diff options
Diffstat (limited to 'numpy')
230 files changed, 12808 insertions, 7725 deletions
diff --git a/numpy/__init__.py b/numpy/__init__.py index e1df236bb..ba88c733f 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -163,6 +163,8 @@ else: from __builtin__ import bool, int, float, complex, object, unicode, str from .core import round, abs, max, min + # now that numpy modules are imported, can initialize limits + core.getlimits._register_known_types() __all__.extend(['__version__', 'show_config']) __all__.extend(core.__all__) diff --git a/numpy/_globals.py b/numpy/_globals.py index 9a7b458f1..f5c0761b5 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -17,7 +17,6 @@ motivated this module. """ from __future__ import division, absolute_import, print_function - __ALL__ = [ 'ModuleDeprecationWarning', 'VisibleDeprecationWarning', '_NoValue' ] @@ -39,7 +38,9 @@ class ModuleDeprecationWarning(DeprecationWarning): nose tester will let pass without making tests fail. """ - pass + + +ModuleDeprecationWarning.__module__ = 'numpy' class VisibleDeprecationWarning(UserWarning): @@ -50,7 +51,10 @@ class VisibleDeprecationWarning(UserWarning): the usage is most likely a user bug. """ - pass + + +VisibleDeprecationWarning.__module__ = 'numpy' + class _NoValueType(object): """Special keyword value. @@ -73,4 +77,5 @@ class _NoValueType(object): def __repr__(self): return "<no value>" + _NoValue = _NoValueType() diff --git a/numpy/_pytesttester.py b/numpy/_pytesttester.py index 30ecc69c7..8d1a3811c 100644 --- a/numpy/_pytesttester.py +++ b/numpy/_pytesttester.py @@ -105,24 +105,18 @@ class PytestTester(object): Notes ----- - Each NumPy module exposes `test` in its namespace to run all tests for it. - For example, to run all tests for numpy.lib: + Each NumPy module exposes `test` in its namespace to run all tests for + it. For example, to run all tests for numpy.lib: >>> np.lib.test() #doctest: +SKIP Examples -------- >>> result = np.lib.test() #doctest: +SKIP - Running unit tests for numpy.lib ... - Ran 976 tests in 3.933s - - OK - - >>> result.errors #doctest: +SKIP - [] - >>> result.knownfail #doctest: +SKIP - [] + 1023 passed, 2 skipped, 6 deselected, 1 xfailed in 10.39 seconds + >>> result + True """ import pytest diff --git a/numpy/compat/_inspect.py b/numpy/compat/_inspect.py index 76bf544a5..439d0d2c2 100644 --- a/numpy/compat/_inspect.py +++ b/numpy/compat/_inspect.py @@ -184,9 +184,8 @@ def formatargvalues(args, varargs, varkw, locals, def convert(name, locals=locals, formatarg=formatarg, formatvalue=formatvalue): return formatarg(name) + formatvalue(locals[name]) - specs = [] - for i in range(len(args)): - specs.append(strseq(args[i], convert, join)) + specs = [strseq(arg, convert, join) for arg in args] + if varargs: specs.append(formatvarargs(varargs) + formatvalue(locals[varargs])) if varkw: diff --git a/numpy/compat/py3k.py b/numpy/compat/py3k.py index ce4543bc3..8e06ead78 100644 --- a/numpy/compat/py3k.py +++ b/numpy/compat/py3k.py @@ -8,13 +8,13 @@ __all__ = ['bytes', 'asbytes', 'isfileobj', 'getexception', 'strchar', 'unicode', 'asunicode', 'asbytes_nested', 'asunicode_nested', 'asstr', 'open_latin1', 'long', 'basestring', 'sixu', 'integer_types', 'is_pathlib_path', 'npy_load_module', 'Path', - 'contextlib_nullcontext'] + 'contextlib_nullcontext', 'os_fspath', 'os_PathLike'] import sys try: - from pathlib import Path + from pathlib import Path, PurePath except ImportError: - Path = None + Path = PurePath = None if sys.version_info[0] >= 3: import io @@ -95,6 +95,8 @@ def asunicode_nested(x): def is_pathlib_path(obj): """ Check whether obj is a pathlib.Path object. + + Prefer using `isinstance(obj, os_PathLike)` instead of this function. """ return Path is not None and isinstance(obj, Path) @@ -177,3 +179,65 @@ else: finally: fo.close() return mod + +# backport abc.ABC +import abc +if sys.version_info[:2] >= (3, 4): + abc_ABC = abc.ABC +else: + abc_ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) + + +# Backport os.fs_path, os.PathLike, and PurePath.__fspath__ +if sys.version_info[:2] >= (3, 6): + import os + os_fspath = os.fspath + os_PathLike = os.PathLike +else: + def _PurePath__fspath__(self): + return str(self) + + class os_PathLike(abc_ABC): + """Abstract base class for implementing the file system path protocol.""" + + @abc.abstractmethod + def __fspath__(self): + """Return the file system path representation of the object.""" + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, subclass): + if PurePath is not None and issubclass(subclass, PurePath): + return True + return hasattr(subclass, '__fspath__') + + + def os_fspath(path): + """Return the path representation of a path-like object. + If str or bytes is passed in, it is returned unchanged. Otherwise the + os.PathLike interface is used to get the path representation. If the + path representation is not str or bytes, TypeError is raised. If the + provided path is not str, bytes, or os.PathLike, TypeError is raised. + """ + if isinstance(path, (str, bytes)): + return path + + # Work from the object's type to match method resolution of other magic + # methods. + path_type = type(path) + try: + path_repr = path_type.__fspath__(path) + except AttributeError: + if hasattr(path_type, '__fspath__'): + raise + elif PurePath is not None and issubclass(path_type, PurePath): + return _PurePath__fspath__(path) + else: + raise TypeError("expected str, bytes or os.PathLike object, " + "not " + path_type.__name__) + if isinstance(path_repr, (str, bytes)): + return path_repr + else: + raise TypeError("expected {}.__fspath__() to return str or bytes, " + "not {}".format(path_type.__name__, + type(path_repr).__name__)) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index ea472f1b3..5f5029aa0 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -49,7 +49,7 @@ add_newdoc('numpy.core', 'flatiter', >>> x = np.arange(6).reshape(2, 3) >>> fl = x.flat >>> type(fl) - <type 'numpy.flatiter'> + <class 'numpy.flatiter'> >>> for item in fl: ... print(item) ... @@ -320,71 +320,68 @@ add_newdoc('numpy.core', 'nditer', Here is how we might write an ``iter_add`` function, using the Python iterator protocol:: - def iter_add_py(x, y, out=None): - addop = np.add - it = np.nditer([x, y, out], [], - [['readonly'], ['readonly'], ['writeonly','allocate']]) - with it: - for (a, b, c) in it: - addop(a, b, out=c) - return it.operands[2] + >>> def iter_add_py(x, y, out=None): + ... addop = np.add + ... it = np.nditer([x, y, out], [], + ... [['readonly'], ['readonly'], ['writeonly','allocate']]) + ... with it: + ... for (a, b, c) in it: + ... addop(a, b, out=c) + ... return it.operands[2] Here is the same function, but following the C-style pattern:: - def iter_add(x, y, out=None): - addop = np.add - - it = np.nditer([x, y, out], [], - [['readonly'], ['readonly'], ['writeonly','allocate']]) - with it: - while not it.finished: - addop(it[0], it[1], out=it[2]) - it.iternext() - - return it.operands[2] + >>> def iter_add(x, y, out=None): + ... addop = np.add + ... it = np.nditer([x, y, out], [], + ... [['readonly'], ['readonly'], ['writeonly','allocate']]) + ... with it: + ... while not it.finished: + ... addop(it[0], it[1], out=it[2]) + ... it.iternext() + ... return it.operands[2] Here is an example outer product function:: - def outer_it(x, y, out=None): - mulop = np.multiply - - it = np.nditer([x, y, out], ['external_loop'], - [['readonly'], ['readonly'], ['writeonly', 'allocate']], - op_axes=[list(range(x.ndim)) + [-1] * y.ndim, - [-1] * x.ndim + list(range(y.ndim)), - None]) - with it: - for (a, b, c) in it: - mulop(a, b, out=c) - return it.operands[2] - - >>> a = np.arange(2)+1 - >>> b = np.arange(3)+1 - >>> outer_it(a,b) - array([[1, 2, 3], - [2, 4, 6]]) + >>> def outer_it(x, y, out=None): + ... mulop = np.multiply + ... it = np.nditer([x, y, out], ['external_loop'], + ... [['readonly'], ['readonly'], ['writeonly', 'allocate']], + ... op_axes=[list(range(x.ndim)) + [-1] * y.ndim, + ... [-1] * x.ndim + list(range(y.ndim)), + ... None]) + ... with it: + ... for (a, b, c) in it: + ... mulop(a, b, out=c) + ... return it.operands[2] + + >>> a = np.arange(2)+1 + >>> b = np.arange(3)+1 + >>> outer_it(a,b) + array([[1, 2, 3], + [2, 4, 6]]) Here is an example function which operates like a "lambda" ufunc:: - def luf(lamdaexpr, *args, **kwargs): - "luf(lambdaexpr, op1, ..., opn, out=None, order='K', casting='safe', buffersize=0)" - nargs = len(args) - op = (kwargs.get('out',None),) + args - it = np.nditer(op, ['buffered','external_loop'], - [['writeonly','allocate','no_broadcast']] + - [['readonly','nbo','aligned']]*nargs, - order=kwargs.get('order','K'), - casting=kwargs.get('casting','safe'), - buffersize=kwargs.get('buffersize',0)) - while not it.finished: - it[0] = lamdaexpr(*it[1:]) - it.iternext() - return it.operands[0] - - >>> a = np.arange(5) - >>> b = np.ones(5) - >>> luf(lambda i,j:i*i + j/2, a, b) - array([ 0.5, 1.5, 4.5, 9.5, 16.5]) + >>> def luf(lamdaexpr, *args, **kwargs): + ... '''luf(lambdaexpr, op1, ..., opn, out=None, order='K', casting='safe', buffersize=0)''' + ... nargs = len(args) + ... op = (kwargs.get('out',None),) + args + ... it = np.nditer(op, ['buffered','external_loop'], + ... [['writeonly','allocate','no_broadcast']] + + ... [['readonly','nbo','aligned']]*nargs, + ... order=kwargs.get('order','K'), + ... casting=kwargs.get('casting','safe'), + ... buffersize=kwargs.get('buffersize',0)) + ... while not it.finished: + ... it[0] = lamdaexpr(*it[1:]) + ... it.iternext() + ... return it.operands[0] + + >>> a = np.arange(5) + >>> b = np.ones(5) + >>> luf(lambda i,j:i*i + j/2, a, b) + array([ 0.5, 1.5, 4.5, 9.5, 16.5]) If operand flags `"writeonly"` or `"readwrite"` are used the operands may be views into the original data with the `WRITEBACKIFCOPY` flag. In this case @@ -393,16 +390,16 @@ add_newdoc('numpy.core', 'nditer', data will be written back to the original data when the `__exit__` function is called but not before: - >>> a = np.arange(6, dtype='i4')[::-2] - >>> with nditer(a, [], - ... [['writeonly', 'updateifcopy']], - ... casting='unsafe', - ... op_dtypes=[np.dtype('f4')]) as i: - ... x = i.operands[0] - ... x[:] = [-1, -2, -3] - ... # a still unchanged here - >>> a, x - array([-1, -2, -3]), array([-1, -2, -3]) + >>> a = np.arange(6, dtype='i4')[::-2] + >>> with np.nditer(a, [], + ... [['writeonly', 'updateifcopy']], + ... casting='unsafe', + ... op_dtypes=[np.dtype('f4')]) as i: + ... x = i.operands[0] + ... x[:] = [-1, -2, -3] + ... # a still unchanged here + >>> a, x + (array([-1, -2, -3], dtype=int32), array([-1., -2., -3.], dtype=float32)) It is important to note that once the iterator is exited, dangling references (like `x` in the example) may or may not share data with @@ -428,10 +425,10 @@ add_newdoc('numpy.core', 'nditer', ('copy', >>> x = np.arange(10) >>> y = x + 1 >>> it = np.nditer([x, y]) - >>> it.next() + >>> next(it) (array(0), array(1)) >>> it2 = it.copy() - >>> it2.next() + >>> next(it2) (array(1), array(2)) """)) @@ -544,7 +541,6 @@ add_newdoc('numpy.core', 'nested_iters', ... print(i.multi_index) ... for y in j: ... print('', j.multi_index, y) - (0,) (0, 0) 0 (0, 1) 1 @@ -617,9 +613,9 @@ add_newdoc('numpy.core', 'broadcast', >>> out = np.empty(b.shape) >>> out.flat = [u+v for (u,v) in b] >>> out - array([[ 5., 6., 7.], - [ 6., 7., 8.], - [ 7., 8., 9.]]) + array([[5., 6., 7.], + [6., 7., 8.], + [7., 8., 9.]]) Compare against built-in broadcasting: @@ -643,7 +639,7 @@ add_newdoc('numpy.core', 'broadcast', ('index', >>> b = np.broadcast(x, y) >>> b.index 0 - >>> b.next(), b.next(), b.next() + >>> next(b), next(b), next(b) ((1, 4), (1, 5), (1, 6)) >>> b.index 3 @@ -762,11 +758,11 @@ add_newdoc('numpy.core', 'broadcast', ('reset', Examples -------- >>> x = np.array([1, 2, 3]) - >>> y = np.array([[4], [5], [6]] + >>> y = np.array([[4], [5], [6]]) >>> b = np.broadcast(x, y) >>> b.index 0 - >>> b.next(), b.next(), b.next() + >>> next(b), next(b), next(b) ((1, 4), (2, 4), (3, 4)) >>> b.index 3 @@ -1072,6 +1068,43 @@ add_newdoc('numpy.core.multiarray', 'fromstring', """) +add_newdoc('numpy.core.multiarray', 'compare_chararrays', + """ + compare_chararrays(a, b, cmp_op, rstrip) + + Performs element-wise comparison of two string arrays using the + comparison operator specified by `cmp_op`. + + Parameters + ---------- + a, b : array_like + Arrays to be compared. + cmp_op : {"<", "<=", "==", ">=", ">", "!="} + Type of comparison. + rstrip : Boolean + If True, the spaces at the end of Strings are removed before the comparison. + + Returns + ------- + out : ndarray + The output array of type Boolean with the same shape as a and b. + + Raises + ------ + ValueError + If `cmp_op` is not valid. + TypeError + If at least one of `a` or `b` is a non-string array + + Examples + -------- + >>> a = np.array(["a", "b", "cde"]) + >>> b = np.array(["a", "a", "dec"]) + >>> np.compare_chararrays(a, b, ">", True) + array([False, True, False]) + + """) + add_newdoc('numpy.core.multiarray', 'fromiter', """ fromiter(iterable, dtype, count=-1) @@ -1152,32 +1185,32 @@ add_newdoc('numpy.core.multiarray', 'fromfile', -------- Construct an ndarray: - >>> dt = np.dtype([('time', [('min', int), ('sec', int)]), + >>> dt = np.dtype([('time', [('min', np.int64), ('sec', np.int64)]), ... ('temp', float)]) >>> x = np.zeros((1,), dtype=dt) >>> x['time']['min'] = 10; x['temp'] = 98.25 >>> x array([((10, 0), 98.25)], - dtype=[('time', [('min', '<i4'), ('sec', '<i4')]), ('temp', '<f8')]) + dtype=[('time', [('min', '<i8'), ('sec', '<i8')]), ('temp', '<f8')]) Save the raw data to disk: - >>> import os - >>> fname = os.tmpnam() + >>> import tempfile + >>> fname = tempfile.mkstemp()[1] >>> x.tofile(fname) Read the raw data from disk: >>> np.fromfile(fname, dtype=dt) array([((10, 0), 98.25)], - dtype=[('time', [('min', '<i4'), ('sec', '<i4')]), ('temp', '<f8')]) + dtype=[('time', [('min', '<i8'), ('sec', '<i8')]), ('temp', '<f8')]) The recommended way to store and load data: >>> np.save(fname, x) >>> np.load(fname + '.npy') array([((10, 0), 98.25)], - dtype=[('time', [('min', '<i4'), ('sec', '<i4')]), ('temp', '<f8')]) + dtype=[('time', [('min', '<i8'), ('sec', '<i8')]), ('temp', '<f8')]) """) @@ -1205,17 +1238,16 @@ add_newdoc('numpy.core.multiarray', 'frombuffer', >>> dt = np.dtype(int) >>> dt = dt.newbyteorder('>') - >>> np.frombuffer(buf, dtype=dt) + >>> np.frombuffer(buf, dtype=dt) # doctest: +SKIP The data of the resulting array will not be byteswapped, but will be interpreted correctly. Examples -------- - >>> s = 'hello world' + >>> s = b'hello world' >>> np.frombuffer(s, dtype='S1', count=5, offset=6) - array(['w', 'o', 'r', 'l', 'd'], - dtype='|S1') + array([b'w', b'o', b'r', b'l', b'd'], dtype='|S1') >>> np.frombuffer(b'\\x01\\x02', dtype=np.uint8) array([1, 2], dtype=uint8) @@ -1320,6 +1352,12 @@ add_newdoc('numpy.core.multiarray', 'set_numeric_ops', Set numerical operators for array objects. + .. deprecated:: 1.16 + + For the general case, use :c:func:`PyUFunc_ReplaceLoopBySignature`. + For ndarray subclasses, define the ``__array_ufunc__`` method and + override the relevant ufunc. + Parameters ---------- op1, op2, ... : callable @@ -1470,134 +1508,11 @@ add_newdoc('numpy.core.multiarray', 'getbuffer', """) -add_newdoc('numpy.core', 'matmul', - """ - matmul(a, b, out=None) - - Matrix product of two arrays. - - The behavior depends on the arguments in the following way. - - - If both arguments are 2-D they are multiplied like conventional - matrices. - - If either argument is N-D, N > 2, it is treated as a stack of - matrices residing in the last two indexes and broadcast accordingly. - - If the first argument is 1-D, it is promoted to a matrix by - prepending a 1 to its dimensions. After matrix multiplication - the prepended 1 is removed. - - If the second argument is 1-D, it is promoted to a matrix by - appending a 1 to its dimensions. After matrix multiplication - the appended 1 is removed. - - Multiplication by a scalar is not allowed, use ``*`` instead. Note that - multiplying a stack of matrices with a vector will result in a stack of - vectors, but matmul will not recognize it as such. - - ``matmul`` differs from ``dot`` in two important ways. - - - Multiplication by scalars is not allowed. - - Stacks of matrices are broadcast together as if the matrices - were elements. - - .. warning:: - This function is preliminary and included in NumPy 1.10.0 for testing - and documentation. Its semantics will not change, but the number and - order of the optional arguments will. - - .. versionadded:: 1.10.0 - - Parameters - ---------- - a : array_like - First argument. - b : array_like - Second argument. - out : ndarray, optional - Output argument. This must have the exact kind that would be returned - if it was not used. In particular, it must have the right type, must be - C-contiguous, and its dtype must be the dtype that would be returned - for `dot(a,b)`. This is a performance feature. Therefore, if these - conditions are not met, an exception is raised, instead of attempting - to be flexible. - - Returns - ------- - output : ndarray - Returns the dot product of `a` and `b`. If `a` and `b` are both - 1-D arrays then a scalar is returned; otherwise an array is - returned. If `out` is given, then it is returned. - - Raises - ------ - ValueError - If the last dimension of `a` is not the same size as - the second-to-last dimension of `b`. - - If scalar value is passed. - - See Also - -------- - vdot : Complex-conjugating dot product. - tensordot : Sum products over arbitrary axes. - einsum : Einstein summation convention. - dot : alternative matrix product with different broadcasting rules. - - Notes - ----- - The matmul function implements the semantics of the `@` operator introduced - in Python 3.5 following PEP465. - - Examples - -------- - For 2-D arrays it is the matrix product: - - >>> a = [[1, 0], [0, 1]] - >>> b = [[4, 1], [2, 2]] - >>> np.matmul(a, b) - array([[4, 1], - [2, 2]]) - - For 2-D mixed with 1-D, the result is the usual. - - >>> a = [[1, 0], [0, 1]] - >>> b = [1, 2] - >>> np.matmul(a, b) - array([1, 2]) - >>> np.matmul(b, a) - array([1, 2]) - - - Broadcasting is conventional for stacks of arrays - - >>> a = np.arange(2*2*4).reshape((2,2,4)) - >>> b = np.arange(2*2*4).reshape((2,4,2)) - >>> np.matmul(a,b).shape - (2, 2, 2) - >>> np.matmul(a,b)[0,1,1] - 98 - >>> sum(a[0,1,:] * b[0,:,1]) - 98 - - Vector, vector returns the scalar inner product, but neither argument - is complex-conjugated: - - >>> np.matmul([2j, 3j], [2j, 3j]) - (-13+0j) - - Scalar multiplication raises an error. - - >>> np.matmul([1,2], 3) - Traceback (most recent call last): - ... - ValueError: Scalar operands are not allowed, use '*' instead - - """) - add_newdoc('numpy.core.multiarray', 'c_einsum', """ c_einsum(subscripts, *operands, out=None, dtype=None, order='K', casting='safe') - + *This documentation shadows that of the native python implementation of the `einsum` function, except all references and examples related to the `optimize` argument (v 0.12.0) have been removed.* @@ -2021,8 +1936,8 @@ add_newdoc('numpy.core.multiarray', 'ndarray', First mode, `buffer` is None: >>> np.ndarray(shape=(2,2), dtype=float, order='F') - array([[ -1.13698227e+002, 4.25087011e-303], - [ 2.88528414e-306, 3.27025015e-309]]) #random + array([[0.0e+000, 0.0e+000], # random + [ nan, 2.5e-323]]) Second mode: @@ -2113,7 +2028,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('ctypes', ----- Below are the public attributes of this object which were documented in "Guide to NumPy" (we have omitted undocumented public attributes, - as well as documented private attributes): + as well as documented private attributes): .. autoattribute:: numpy.core._internal._ctypes.data @@ -2127,14 +2042,6 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('ctypes', .. automethod:: numpy.core._internal._ctypes.strides_as - Be careful using the ctypes attribute - especially on temporary - arrays or arrays constructed on the fly. For example, calling - ``(a+b).ctypes.data_as(ctypes.c_void_p)`` returns a pointer to memory - that is invalid because the array created as (a+b) is deallocated - before the next Python statement. You can avoid this problem using - either ``c=a+b`` or ``ct=(a+b).ctypes``. In the latter case, ct will - hold a reference to the array until ct is deleted or re-assigned. - If the ctypes module is not available, then the ctypes attribute of array objects still returns something useful, but ctypes objects are not returned and errors may be raised instead. In particular, @@ -2336,7 +2243,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('flat', >>> x.T.flat[3] 5 >>> type(x.flat) - <type 'numpy.flatiter'> + <class 'numpy.flatiter'> An assignment example: @@ -2455,7 +2362,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('size', Notes ----- - `a.size` returns a standard arbitrary precision Python integer. This + `a.size` returns a standard arbitrary precision Python integer. This may not be the case with other methods of obtaining the same value (like the suggested ``np.prod(a.shape)``, which returns an instance of ``np.int_``), and may be relevant if the value is used further in @@ -2786,7 +2693,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('astype', -------- >>> x = np.array([1, 2, 2.5]) >>> x - array([ 1. , 2. , 2.5]) + array([1. , 2. , 2.5]) >>> x.astype(int) array([1, 2, 2]) @@ -2817,19 +2724,20 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('byteswap', Examples -------- >>> A = np.array([1, 256, 8755], dtype=np.int16) - >>> map(hex, A) + >>> list(map(hex, A)) ['0x1', '0x100', '0x2233'] >>> A.byteswap(inplace=True) array([ 256, 1, 13090], dtype=int16) - >>> map(hex, A) + >>> list(map(hex, A)) ['0x100', '0x1', '0x3322'] Arrays of strings are not swapped >>> A = np.array(['ceg', 'fac']) >>> A.byteswap() - array(['ceg', 'fac'], - dtype='|S3') + Traceback (most recent call last): + ... + UnicodeDecodeError: ... """)) @@ -3017,14 +2925,14 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('dot', >>> a = np.eye(2) >>> b = np.ones((2, 2)) * 2 >>> a.dot(b) - array([[ 2., 2.], - [ 2., 2.]]) + array([[2., 2.], + [2., 2.]]) This array method can be conveniently chained: >>> a.dot(b).dot(b) - array([[ 8., 8.], - [ 8., 8.]]) + array([[8., 8.], + [8., 8.]]) """)) @@ -3077,7 +2985,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('fill', >>> a = np.empty(2) >>> a.fill(1) >>> a - array([ 1., 1.]) + array([1., 1.]) """)) @@ -3146,18 +3054,18 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('getfield', >>> x = np.diag([1.+1.j]*2) >>> x[1, 1] = 2 + 4.j >>> x - array([[ 1.+1.j, 0.+0.j], - [ 0.+0.j, 2.+4.j]]) + array([[1.+1.j, 0.+0.j], + [0.+0.j, 2.+4.j]]) >>> x.getfield(np.float64) - array([[ 1., 0.], - [ 0., 2.]]) + array([[1., 0.], + [0., 2.]]) By choosing an offset of 8 bytes we can select the complex part of the array for our view: >>> x.getfield(np.float64, offset=8) - array([[ 1., 0.], - [ 0., 4.]]) + array([[1., 0.], + [0., 4.]]) """)) @@ -3203,19 +3111,20 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('item', Examples -------- + >>> np.random.seed(123) >>> x = np.random.randint(9, size=(3, 3)) >>> x - array([[3, 1, 7], - [2, 8, 3], - [8, 5, 3]]) + array([[2, 2, 6], + [1, 3, 6], + [1, 0, 1]]) >>> x.item(3) - 2 + 1 >>> x.item(7) - 5 + 0 >>> x.item((0, 1)) - 1 + 2 >>> x.item((2, 2)) - 3 + 1 """)) @@ -3251,24 +3160,25 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('itemset', Examples -------- + >>> np.random.seed(123) >>> x = np.random.randint(9, size=(3, 3)) >>> x - array([[3, 1, 7], - [2, 8, 3], - [8, 5, 3]]) + array([[2, 2, 6], + [1, 3, 6], + [1, 0, 1]]) >>> x.itemset(4, 0) >>> x.itemset((2, 2), 9) >>> x - array([[3, 1, 7], - [2, 0, 3], - [8, 5, 9]]) + array([[2, 2, 6], + [1, 0, 6], + [1, 0, 9]]) """)) add_newdoc('numpy.core.multiarray', 'ndarray', ('max', """ - a.max(axis=None, out=None, keepdims=False) + a.max(axis=None, out=None, keepdims=False, initial=<no value>, where=True) Return the maximum along a given axis. @@ -3298,7 +3208,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('mean', add_newdoc('numpy.core.multiarray', 'ndarray', ('min', """ - a.min(axis=None, out=None, keepdims=False) + a.min(axis=None, out=None, keepdims=False, initial=<no value>, where=True) Return the minimum along a given axis. @@ -3451,7 +3361,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('nonzero', add_newdoc('numpy.core.multiarray', 'ndarray', ('prod', """ - a.prod(axis=None, dtype=None, out=None, keepdims=False) + a.prod(axis=None, dtype=None, out=None, keepdims=False, initial=1, where=True) Return the product of the array elements over the given axis @@ -3702,7 +3612,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('resize', >>> a.resize((1, 1)) Traceback (most recent call last): ... - ValueError: cannot resize an array that has been referenced ... + ValueError: cannot resize an array that references or is referenced ... Unless `refcheck` is False: @@ -3775,23 +3685,23 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('setfield', -------- >>> x = np.eye(3) >>> x.getfield(np.float64) - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) >>> x.setfield(3, np.int32) >>> x.getfield(np.int32) array([[3, 3, 3], [3, 3, 3], - [3, 3, 3]]) + [3, 3, 3]], dtype=int32) >>> x - array([[ 1.00000000e+000, 1.48219694e-323, 1.48219694e-323], - [ 1.48219694e-323, 1.00000000e+000, 1.48219694e-323], - [ 1.48219694e-323, 1.48219694e-323, 1.00000000e+000]]) + array([[1.0e+000, 1.5e-323, 1.5e-323], + [1.5e-323, 1.0e+000, 1.5e-323], + [1.5e-323, 1.5e-323, 1.0e+000]]) >>> x.setfield(np.eye(3), np.int32) >>> x - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) """)) @@ -3844,6 +3754,9 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('setflags', Examples -------- + >>> y = np.array([[3, 1, 7], + ... [2, 0, 0], + ... [8, 5, 9]]) >>> y array([[3, 1, 7], [2, 0, 0], @@ -3923,8 +3836,8 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('sort', >>> a = np.array([('a', 2), ('c', 1)], dtype=[('x', 'S1'), ('y', int)]) >>> a.sort(order='y') >>> a - array([('c', 1), ('a', 2)], - dtype=[('x', '|S1'), ('y', '<i4')]) + array([(b'c', 1), (b'a', 2)], + dtype=[('x', 'S1'), ('y', '<i8')]) """)) @@ -3980,6 +3893,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('partition', array([2, 1, 3, 4]) >>> a.partition((1, 3)) + >>> a array([1, 2, 3, 4]) """)) @@ -4016,7 +3930,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('std', add_newdoc('numpy.core.multiarray', 'ndarray', ('sum', """ - a.sum(axis=None, dtype=None, out=None, keepdims=False) + a.sum(axis=None, dtype=None, out=None, keepdims=False, initial=0, where=True) Return the sum of the array elements over the given axis. @@ -4090,7 +4004,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('tofile', machines with different endianness. Some of these problems can be overcome by outputting the data as text files, at the expense of speed and file size. - + When fid is a file object, array contents are directly written to the file, bypassing the file object's ``write`` method. As a result, tofile cannot be used with files objects supporting compression (e.g., GzipFile) @@ -4161,13 +4075,13 @@ tobytesdoc = """ Examples -------- - >>> x = np.array([[0, 1], [2, 3]]) + >>> x = np.array([[0, 1], [2, 3]], dtype='<u2') >>> x.tobytes() - b'\\x00\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x03\\x00\\x00\\x00' + b'\\x00\\x00\\x01\\x00\\x02\\x00\\x03\\x00' >>> x.tobytes('C') == x.tobytes() True >>> x.tobytes('F') - b'\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x03\\x00\\x00\\x00' + b'\\x00\\x00\\x02\\x00\\x01\\x00\\x03\\x00' """ @@ -4317,7 +4231,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('view', >>> y matrix([[513]], dtype=int16) >>> print(type(y)) - <class 'numpy.matrixlib.defmatrix.matrix'> + <class 'numpy.matrix'> Creating a view on a structured array so it can be used in calculations @@ -4327,19 +4241,19 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('view', array([[1, 2], [3, 4]], dtype=int8) >>> xv.mean(0) - array([ 2., 3.]) + array([2., 3.]) Making changes to the view changes the underlying array >>> xv[0,1] = 20 - >>> print(x) - [(1, 20) (3, 4)] + >>> x + array([(1, 20), (3, 4)], dtype=[('a', 'i1'), ('b', 'i1')]) Using a view to convert an array to a recarray: >>> z = x.view(np.recarray) >>> z.a - array([1], dtype=int8) + array([1, 3], dtype=int8) Views share data: @@ -4357,8 +4271,8 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('view', [4, 5]], dtype=int16) >>> y.view(dtype=[('width', np.int16), ('length', np.int16)]) Traceback (most recent call last): - File "<stdin>", line 1, in <module> - ValueError: new type not compatible with array. + ... + ValueError: To change to a dtype of a different size, the array must be C-contiguous >>> z = y.copy() >>> z.view(dtype=[('width', np.int16), ('length', np.int16)]) array([[(1, 2)], @@ -4409,10 +4323,9 @@ add_newdoc('numpy.core.umath', 'frompyfunc', >>> oct_array = np.frompyfunc(oct, 1, 1) >>> oct_array(np.array((10, 30, 100))) - array([012, 036, 0144], dtype=object) + array(['0o12', '0o36', '0o144'], dtype=object) >>> np.array((oct(10), oct(30), oct(100))) # for comparison - array(['012', '036', '0144'], - dtype='|S4') + array(['0o12', '0o36', '0o144'], dtype='<U5') """) @@ -4474,7 +4387,7 @@ add_newdoc('numpy.core.umath', 'geterrobj', >>> np.base_repr(np.geterrobj()[1], 8) '0' >>> old_err = np.seterr(divide='warn', over='log', under='call', - invalid='print') + ... invalid='print') >>> np.base_repr(np.geterrobj()[1], 8) '4351' @@ -4546,184 +4459,6 @@ add_newdoc('numpy.core.umath', 'seterrobj', # ############################################################################## -add_newdoc('numpy.core.multiarray', 'bincount', - """ - bincount(x, weights=None, minlength=0) - - Count number of occurrences of each value in array of non-negative ints. - - The number of bins (of size 1) is one larger than the largest value in - `x`. If `minlength` is specified, there will be at least this number - of bins in the output array (though it will be longer if necessary, - depending on the contents of `x`). - Each bin gives the number of occurrences of its index value in `x`. - If `weights` is specified the input array is weighted by it, i.e. if a - value ``n`` is found at position ``i``, ``out[n] += weight[i]`` instead - of ``out[n] += 1``. - - Parameters - ---------- - x : array_like, 1 dimension, nonnegative ints - Input array. - weights : array_like, optional - Weights, array of the same shape as `x`. - minlength : int, optional - A minimum number of bins for the output array. - - .. versionadded:: 1.6.0 - - Returns - ------- - out : ndarray of ints - The result of binning the input array. - The length of `out` is equal to ``np.amax(x)+1``. - - Raises - ------ - ValueError - If the input is not 1-dimensional, or contains elements with negative - values, or if `minlength` is negative. - TypeError - If the type of the input is float or complex. - - See Also - -------- - histogram, digitize, unique - - Examples - -------- - >>> np.bincount(np.arange(5)) - array([1, 1, 1, 1, 1]) - >>> np.bincount(np.array([0, 1, 1, 3, 2, 1, 7])) - array([1, 3, 1, 1, 0, 0, 0, 1]) - - >>> x = np.array([0, 1, 1, 3, 2, 1, 7, 23]) - >>> np.bincount(x).size == np.amax(x)+1 - True - - The input array needs to be of integer dtype, otherwise a - TypeError is raised: - - >>> np.bincount(np.arange(5, dtype=float)) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: array cannot be safely cast to required type - - A possible use of ``bincount`` is to perform sums over - variable-size chunks of an array, using the ``weights`` keyword. - - >>> w = np.array([0.3, 0.5, 0.2, 0.7, 1., -0.6]) # weights - >>> x = np.array([0, 1, 1, 2, 2, 2]) - >>> np.bincount(x, weights=w) - array([ 0.3, 0.7, 1.1]) - - """) - -add_newdoc('numpy.core.multiarray', 'ravel_multi_index', - """ - ravel_multi_index(multi_index, dims, mode='raise', order='C') - - Converts a tuple of index arrays into an array of flat - indices, applying boundary modes to the multi-index. - - Parameters - ---------- - multi_index : tuple of array_like - A tuple of integer arrays, one array for each dimension. - dims : tuple of ints - The shape of array into which the indices from ``multi_index`` apply. - mode : {'raise', 'wrap', 'clip'}, optional - Specifies how out-of-bounds indices are handled. Can specify - either one mode or a tuple of modes, one mode per index. - - * 'raise' -- raise an error (default) - * 'wrap' -- wrap around - * 'clip' -- clip to the range - - In 'clip' mode, a negative index which would normally - wrap will clip to 0 instead. - order : {'C', 'F'}, optional - Determines whether the multi-index should be viewed as - indexing in row-major (C-style) or column-major - (Fortran-style) order. - - Returns - ------- - raveled_indices : ndarray - An array of indices into the flattened version of an array - of dimensions ``dims``. - - See Also - -------- - unravel_index - - Notes - ----- - .. versionadded:: 1.6.0 - - Examples - -------- - >>> arr = np.array([[3,6,6],[4,5,1]]) - >>> np.ravel_multi_index(arr, (7,6)) - array([22, 41, 37]) - >>> np.ravel_multi_index(arr, (7,6), order='F') - array([31, 41, 13]) - >>> np.ravel_multi_index(arr, (4,6), mode='clip') - array([22, 23, 19]) - >>> np.ravel_multi_index(arr, (4,4), mode=('clip','wrap')) - array([12, 13, 13]) - - >>> np.ravel_multi_index((3,1,4,1), (6,7,8,9)) - 1621 - """) - -add_newdoc('numpy.core.multiarray', 'unravel_index', - """ - unravel_index(indices, shape, order='C') - - Converts a flat index or array of flat indices into a tuple - of coordinate arrays. - - Parameters - ---------- - indices : array_like - An integer array whose elements are indices into the flattened - version of an array of dimensions ``shape``. Before version 1.6.0, - this function accepted just one index value. - shape : tuple of ints - The shape of the array to use for unraveling ``indices``. - - .. versionchanged:: 1.16.0 - Renamed from ``dims`` to ``shape``. - - order : {'C', 'F'}, optional - Determines whether the indices should be viewed as indexing in - row-major (C-style) or column-major (Fortran-style) order. - - .. versionadded:: 1.6.0 - - Returns - ------- - unraveled_coords : tuple of ndarray - Each array in the tuple has the same shape as the ``indices`` - array. - - See Also - -------- - ravel_multi_index - - Examples - -------- - >>> np.unravel_index([22, 41, 37], (7,6)) - (array([3, 6, 6]), array([4, 5, 1])) - >>> np.unravel_index([31, 41, 13], (7,6), order='F') - (array([3, 6, 6]), array([4, 5, 1])) - - >>> np.unravel_index(1621, (6,7,8,9)) - (3, 1, 4, 1) - - """) - add_newdoc('numpy.core.multiarray', 'add_docstring', """ add_docstring(obj, docstring) @@ -4798,7 +4533,10 @@ add_newdoc('numpy.core.multiarray', 'packbits', ... [0,0,1]]]) >>> b = np.packbits(a, axis=-1) >>> b - array([[[160],[64]],[[192],[32]]], dtype=uint8) + array([[[160], + [ 64]], + [[192], + [ 32]]], dtype=uint8) Note that in binary 160 = 1010 0000, 64 = 0100 0000, 192 = 1100 0000, and 32 = 0010 0000. @@ -5138,7 +4876,7 @@ add_newdoc('numpy.core', 'ufunc', ('signature', add_newdoc('numpy.core', 'ufunc', ('reduce', """ - reduce(a, axis=0, dtype=None, out=None, keepdims=False, initial) + reduce(a, axis=0, dtype=None, out=None, keepdims=False, initial=<no value>, where=True) Reduces `a`'s dimension by one, by applying ufunc along one axis. @@ -5200,9 +4938,17 @@ add_newdoc('numpy.core', 'ufunc', ('reduce', to None - otherwise it defaults to ufunc.identity. If ``None`` is given, the first element of the reduction is used, and an error is thrown if the reduction is empty. - + .. versionadded:: 1.15.0 + where : array_like of bool, optional + A boolean array which is broadcasted to match the dimensions + of `a`, and selects elements to include in the reduction. Note + that for ufuncs like ``minimum`` that do not have an identity + defined, one has to pass in also ``initial``. + + .. versionadded:: 1.17.0 + Returns ------- r : ndarray @@ -5233,20 +4979,25 @@ add_newdoc('numpy.core', 'ufunc', ('reduce', >>> np.add.reduce(X, 2) array([[ 1, 5], [ 9, 13]]) - - You can use the ``initial`` keyword argument to initialize the reduction with a - different value. - + + You can use the ``initial`` keyword argument to initialize the reduction + with a different value, and ``where`` to select specific elements to include: + >>> np.add.reduce([10], initial=5) 15 - >>> np.add.reduce(np.ones((2, 2, 2)), axis=(0, 2), initializer=10) + >>> np.add.reduce(np.ones((2, 2, 2)), axis=(0, 2), initial=10) array([14., 14.]) - + >>> a = np.array([10., np.nan, 10]) + >>> np.add.reduce(a, where=~np.isnan(a)) + 20.0 + Allows reductions of empty arrays where they would normally fail, i.e. for ufuncs without an identity. - + >>> np.minimum.reduce([], initial=np.inf) inf + >>> np.minimum.reduce([[1., 2.], [3., 4.]], initial=10., where=[True, False]) + array([ 1., 10.]) >>> np.minimum.reduce([]) Traceback (most recent call last): ... @@ -5312,23 +5063,23 @@ add_newdoc('numpy.core', 'ufunc', ('accumulate', >>> I = np.eye(2) >>> I - array([[ 1., 0.], - [ 0., 1.]]) + array([[1., 0.], + [0., 1.]]) Accumulate along axis 0 (rows), down columns: >>> np.add.accumulate(I, 0) - array([[ 1., 0.], - [ 1., 1.]]) + array([[1., 0.], + [1., 1.]]) >>> np.add.accumulate(I) # no axis specified = axis zero - array([[ 1., 0.], - [ 1., 1.]]) + array([[1., 0.], + [1., 1.]]) Accumulate along axis 1 (columns), through rows: >>> np.add.accumulate(I, 1) - array([[ 1., 1.], - [ 0., 1.]]) + array([[1., 1.], + [0., 1.]]) """)) @@ -5405,10 +5156,10 @@ add_newdoc('numpy.core', 'ufunc', ('reduceat', >>> x = np.linspace(0, 15, 16).reshape(4,4) >>> x - array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]]) + array([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) :: @@ -5420,11 +5171,11 @@ add_newdoc('numpy.core', 'ufunc', ('reduceat', # [row1 + row2 + row3 + row4] >>> np.add.reduceat(x, [0, 3, 1, 2, 0]) - array([[ 12., 15., 18., 21.], - [ 12., 13., 14., 15.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.], - [ 24., 28., 32., 36.]]) + array([[12., 15., 18., 21.], + [12., 13., 14., 15.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [24., 28., 32., 36.]]) :: @@ -5432,10 +5183,10 @@ add_newdoc('numpy.core', 'ufunc', ('reduceat', # [col1 * col2 * col3, col4] >>> np.multiply.reduceat(x, [0, 3], 1) - array([[ 0., 3.], - [ 120., 7.], - [ 720., 11.], - [ 2184., 15.]]) + array([[ 0., 3.], + [ 120., 7.], + [ 720., 11.], + [2184., 15.]]) """)) @@ -5534,14 +5285,14 @@ add_newdoc('numpy.core', 'ufunc', ('at', >>> a = np.array([1, 2, 3, 4]) >>> np.negative.at(a, [0, 1]) - >>> print(a) - array([-1, -2, 3, 4]) + >>> a + array([-1, -2, 3, 4]) Increment items 0 and 1, and increment item 2 twice: >>> a = np.array([1, 2, 3, 4]) >>> np.add.at(a, [0, 1, 2, 2], 1) - >>> print(a) + >>> a array([2, 3, 5, 4]) Add items 0 and 1 in first array to second array, @@ -5550,7 +5301,7 @@ add_newdoc('numpy.core', 'ufunc', ('at', >>> a = np.array([1, 2, 3, 4]) >>> b = np.array([1, 2]) >>> np.add.at(a, [0, 1], b) - >>> print(a) + >>> a array([2, 4, 3, 4]) """)) @@ -5615,13 +5366,13 @@ add_newdoc('numpy.core.multiarray', 'dtype', Structured type, two fields: the first field contains an unsigned int, the second an int32: - >>> np.dtype([('f1', np.uint), ('f2', np.int32)]) - dtype([('f1', '<u4'), ('f2', '<i4')]) + >>> np.dtype([('f1', np.uint64), ('f2', np.int32)]) + dtype([('f1', '<u8'), ('f2', '<i4')]) Using array-protocol type strings: >>> np.dtype([('a','f8'),('b','S10')]) - dtype([('a', '<f8'), ('b', '|S10')]) + dtype([('a', '<f8'), ('b', 'S10')]) Using comma-separated field formats. The shape is (2,3): @@ -5631,24 +5382,24 @@ add_newdoc('numpy.core.multiarray', 'dtype', Using tuples. ``int`` is a fixed type, 3 the field's shape. ``void`` is a flexible type, here of size 10: - >>> np.dtype([('hello',(int,3)),('world',np.void,10)]) - dtype([('hello', '<i4', 3), ('world', '|V10')]) + >>> np.dtype([('hello',(np.int64,3)),('world',np.void,10)]) + dtype([('hello', '<i8', (3,)), ('world', 'V10')]) Subdivide ``int16`` into 2 ``int8``'s, called x and y. 0 and 1 are the offsets in bytes: >>> np.dtype((np.int16, {'x':(np.int8,0), 'y':(np.int8,1)})) - dtype(('<i2', [('x', '|i1'), ('y', '|i1')])) + dtype((numpy.int16, [('x', 'i1'), ('y', 'i1')])) Using dictionaries. Two fields named 'gender' and 'age': >>> np.dtype({'names':['gender','age'], 'formats':['S1',np.uint8]}) - dtype([('gender', '|S1'), ('age', '|u1')]) + dtype([('gender', 'S1'), ('age', 'u1')]) Offsets in bytes, here 0 and 25: >>> np.dtype({'surname':('S25',0),'age':(np.uint8,25)}) - dtype([('surname', '|S25'), ('age', '|u1')]) + dtype([('surname', 'S25'), ('age', 'u1')]) """) @@ -5714,13 +5465,13 @@ add_newdoc('numpy.core.multiarray', 'dtype', ('char', add_newdoc('numpy.core.multiarray', 'dtype', ('descr', """ - PEP3118 interface description of the data-type. + `__array_interface__` description of the data-type. The format is that required by the 'descr' key in the - PEP3118 `__array_interface__` attribute. + `__array_interface__` attribute. - Warning: This attribute exists specifically for PEP3118 compliance, and - is not a datatype description compatible with `np.dtype`. + Warning: This attribute exists specifically for `__array_interface__`, + and is not a datatype description compatible with `np.dtype`. """)) add_newdoc('numpy.core.multiarray', 'dtype', ('fields', @@ -6052,7 +5803,7 @@ add_newdoc('numpy.core.multiarray', 'busdaycalendar', ... holidays=['2011-07-01', '2011-07-04', '2011-07-17']) >>> # Default is Monday to Friday weekdays ... bdd.weekmask - array([ True, True, True, True, True, False, False], dtype='bool') + array([ True, True, True, True, True, False, False]) >>> # Any holidays already on the weekend are removed ... bdd.holidays array(['2011-07-01', '2011-07-04'], dtype='datetime64[D]') @@ -6149,7 +5900,7 @@ add_newdoc('numpy.core.multiarray', 'datetime_data', as a timedelta >>> np.datetime64('2010', np.datetime_data(dt_25s)) - numpy.datetime64('2010-01-01T00:00:00', '25s') + numpy.datetime64('2010-01-01T00:00:00','25s') """) @@ -6983,25 +6734,25 @@ add_newdoc('numpy.core.numerictypes', 'generic', ('view', add_newdoc('numpy.core.numerictypes', 'number', """ Abstract base class of all numeric scalar types. - + """) add_newdoc('numpy.core.numerictypes', 'integer', """ Abstract base class of all integer scalar types. - + """) add_newdoc('numpy.core.numerictypes', 'signedinteger', """ Abstract base class of all signed integer scalar types. - + """) add_newdoc('numpy.core.numerictypes', 'unsignedinteger', """ Abstract base class of all unsigned integer scalar types. - + """) add_newdoc('numpy.core.numerictypes', 'inexact', @@ -7009,20 +6760,20 @@ add_newdoc('numpy.core.numerictypes', 'inexact', Abstract base class of all numeric scalar types with a (potentially) inexact representation of the values in its range, such as floating-point numbers. - + """) add_newdoc('numpy.core.numerictypes', 'floating', """ Abstract base class of all floating-point scalar types. - + """) add_newdoc('numpy.core.numerictypes', 'complexfloating', """ Abstract base class of all complex number scalar types that are made up of floating-point numbers. - + """) add_newdoc('numpy.core.numerictypes', 'flexible', @@ -7030,13 +6781,13 @@ add_newdoc('numpy.core.numerictypes', 'flexible', Abstract base class of all scalar types without predefined length. The actual size of these types depends on the specific `np.dtype` instantiation. - + """) add_newdoc('numpy.core.numerictypes', 'character', """ Abstract base class of all character string scalar types. - + """) diff --git a/numpy/core/_dtype.py b/numpy/core/_dtype.py index d115e0fa6..3a12c8fad 100644 --- a/numpy/core/_dtype.py +++ b/numpy/core/_dtype.py @@ -138,7 +138,9 @@ def _scalar_str(dtype, short): else: return "'%sU%d'" % (byteorder, dtype.itemsize / 4) - elif dtype.type == np.void: + # unlike the other types, subclasses of void are preserved - but + # historically the repr does not actually reveal the subclass + elif issubclass(dtype.type, np.void): if _isunsized(dtype): return "'V'" else: diff --git a/numpy/core/_dtype_ctypes.py b/numpy/core/_dtype_ctypes.py new file mode 100644 index 000000000..0852b1ef2 --- /dev/null +++ b/numpy/core/_dtype_ctypes.py @@ -0,0 +1,113 @@ +""" +Conversion from ctypes to dtype. + +In an ideal world, we could acheive this through the PEP3118 buffer protocol, +something like:: + + def dtype_from_ctypes_type(t): + # needed to ensure that the shape of `t` is within memoryview.format + class DummyStruct(ctypes.Structure): + _fields_ = [('a', t)] + + # empty to avoid memory allocation + ctype_0 = (DummyStruct * 0)() + mv = memoryview(ctype_0) + + # convert the struct, and slice back out the field + return _dtype_from_pep3118(mv.format)['a'] + +Unfortunately, this fails because: + +* ctypes cannot handle length-0 arrays with PEP3118 (bpo-32782) +* PEP3118 cannot represent unions, but both numpy and ctypes can +* ctypes cannot handle big-endian structs with PEP3118 (bpo-32780) +""" +import _ctypes +import ctypes + +import numpy as np + + +def _from_ctypes_array(t): + return np.dtype((dtype_from_ctypes_type(t._type_), (t._length_,))) + + +def _from_ctypes_structure(t): + for item in t._fields_: + if len(item) > 2: + raise TypeError( + "ctypes bitfields have no dtype equivalent") + + if hasattr(t, "_pack_"): + formats = [] + offsets = [] + names = [] + current_offset = 0 + for fname, ftyp in t._fields_: + names.append(fname) + formats.append(dtype_from_ctypes_type(ftyp)) + # Each type has a default offset, this is platform dependent for some types. + effective_pack = min(t._pack_, ctypes.alignment(ftyp)) + current_offset = ((current_offset + effective_pack - 1) // effective_pack) * effective_pack + offsets.append(current_offset) + current_offset += ctypes.sizeof(ftyp) + + return np.dtype(dict( + formats=formats, + offsets=offsets, + names=names, + itemsize=ctypes.sizeof(t))) + else: + fields = [] + for fname, ftyp in t._fields_: + fields.append((fname, dtype_from_ctypes_type(ftyp))) + + # by default, ctypes structs are aligned + return np.dtype(fields, align=True) + + +def _from_ctypes_scalar(t): + """ + Return the dtype type with endianness included if it's the case + """ + if getattr(t, '__ctype_be__', None) is t: + return np.dtype('>' + t._type_) + elif getattr(t, '__ctype_le__', None) is t: + return np.dtype('<' + t._type_) + else: + return np.dtype(t._type_) + + +def _from_ctypes_union(t): + formats = [] + offsets = [] + names = [] + for fname, ftyp in t._fields_: + names.append(fname) + formats.append(dtype_from_ctypes_type(ftyp)) + offsets.append(0) # Union fields are offset to 0 + + return np.dtype(dict( + formats=formats, + offsets=offsets, + names=names, + itemsize=ctypes.sizeof(t))) + + +def dtype_from_ctypes_type(t): + """ + Construct a dtype object from a ctypes type + """ + if issubclass(t, _ctypes.Array): + return _from_ctypes_array(t) + elif issubclass(t, _ctypes._Pointer): + raise TypeError("ctypes pointers have no dtype equivalent") + elif issubclass(t, _ctypes.Structure): + return _from_ctypes_structure(t) + elif issubclass(t, _ctypes.Union): + return _from_ctypes_union(t) + elif isinstance(getattr(t, '_type_', None), str): + return _from_ctypes_scalar(t) + else: + raise NotImplementedError( + "Unknown ctypes type {}".format(t.__name__)) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index c4d967dc2..1d3bb5584 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -10,6 +10,7 @@ import re import sys from numpy.compat import unicode +from numpy.core.overrides import set_module from .multiarray import dtype, array, ndarray try: import ctypes @@ -237,19 +238,68 @@ _getintp_ctype.cache = None class _missing_ctypes(object): def cast(self, num, obj): - return num + return num.value + + class c_void_p(object): + def __init__(self, ptr): + self.value = ptr + + +class _unsafe_first_element_pointer(object): + """ + Helper to allow viewing an array as a ctypes pointer to the first element + + This avoids: + * dealing with strides + * `.view` rejecting object-containing arrays + * `memoryview` not supporting overlapping fields + """ + def __init__(self, arr): + self.base = arr + + @property + def __array_interface__(self): + i = dict( + shape=(), + typestr='|V0', + data=(self.base.__array_interface__['data'][0], False), + strides=(), + version=3, + ) + return i + + +def _get_void_ptr(arr): + """ + Get a `ctypes.c_void_p` to arr.data, that keeps a reference to the array + """ + import numpy as np + # convert to a 0d array that has a data pointer referrign to the start + # of arr. This holds a reference to arr. + simple_arr = np.asarray(_unsafe_first_element_pointer(arr)) + + # create a `char[0]` using the same memory. + c_arr = (ctypes.c_char * 0).from_buffer(simple_arr) + + # finally cast to void* + return ctypes.cast(ctypes.pointer(c_arr), ctypes.c_void_p) - def c_void_p(self, num): - return num class _ctypes(object): def __init__(self, array, ptr=None): + self._arr = array + if ctypes: self._ctypes = ctypes + # get a void pointer to the buffer, which keeps the array alive + self._data = _get_void_ptr(array) + assert self._data.value == ptr else: + # fake a pointer-like object that holds onto the reference self._ctypes = _missing_ctypes() - self._arr = array - self._data = ptr + self._data = self._ctypes.c_void_p(ptr) + self._data._objects = array + if self._arr.ndim == 0: self._zerod = True else: @@ -262,6 +312,8 @@ class _ctypes(object): ``self.data_as(ctypes.c_void_p)``. Perhaps you want to use the data as a pointer to a ctypes array of floating-point data: ``self.data_as(ctypes.POINTER(ctypes.c_double))``. + + The returned pointer will keep a reference to the array. """ return self._ctypes.cast(self._data, obj) @@ -283,7 +335,8 @@ class _ctypes(object): return None return (obj*self._arr.ndim)(*self._arr.strides) - def get_data(self): + @property + def data(self): """ A pointer to the memory area of the array as a Python integer. This memory area may contain data that is not aligned, or not in correct @@ -292,10 +345,16 @@ class _ctypes(object): attribute to arbitrary C-code to avoid trouble that can include Python crashing. User Beware! The value of this attribute is exactly the same as ``self._array_interface_['data'][0]``. + + Note that unlike `data_as`, a reference will not be kept to the array: + code like ``ctypes.c_void_p((a + b).ctypes.data)`` will result in a + pointer to a deallocated array, and should be spelt + ``(a + b).ctypes.data_as(ctypes.c_void_p)`` """ - return self._data + return self._data.value - def get_shape(self): + @property + def shape(self): """ (c_intp*self.ndim): A ctypes array of length self.ndim where the basetype is the C-integer corresponding to ``dtype('p')`` on this @@ -306,7 +365,8 @@ class _ctypes(object): """ return self.shape_as(_getintp_ctype()) - def get_strides(self): + @property + def strides(self): """ (c_intp*self.ndim): A ctypes array of length self.ndim where the basetype is the same as for the shape attribute. This ctypes array @@ -316,13 +376,20 @@ class _ctypes(object): """ return self.strides_as(_getintp_ctype()) - def get_as_parameter(self): - return self._ctypes.c_void_p(self._data) + @property + def _as_parameter_(self): + """ + Overrides the ctypes semi-magic method + + Enables `c_func(some_array.ctypes)` + """ + return self._data - data = property(get_data) - shape = property(get_shape) - strides = property(get_strides) - _as_parameter_ = property(get_as_parameter, None, doc="_as parameter_") + # kept for compatibility + get_data = data.fget + get_shape = shape.fget + get_strides = strides.fget + get_as_parameter = _as_parameter_.fget def _newnames(datatype, order): @@ -482,6 +549,12 @@ _pep3118_standard_map = { } _pep3118_standard_typechars = ''.join(_pep3118_standard_map.keys()) +_pep3118_unsupported_map = { + 'u': 'UCS-2 strings', + '&': 'pointers', + 't': 'bitfields', + 'X': 'function pointers', +} class _Stream(object): def __init__(self, s): @@ -593,6 +666,11 @@ def __dtype_from_pep3118(stream, is_subdtype): stream.byteorder, stream.byteorder) value = dtype(numpy_byteorder + dtypechar) align = value.alignment + elif stream.next in _pep3118_unsupported_map: + desc = _pep3118_unsupported_map[stream.next] + raise NotImplementedError( + "Unrepresentable PEP 3118 data type {!r} ({})" + .format(stream.next, desc)) else: raise ValueError("Unknown PEP 3118 data type specifier %r" % stream.s) @@ -718,9 +796,11 @@ def _lcm(a, b): return a // _gcd(a, b) * b # Exception used in shares_memory() +@set_module('numpy') class TooHardError(RuntimeError): pass +@set_module('numpy') class AxisError(ValueError, IndexError): """ Axis supplied was invalid. """ def __init__(self, axis, ndim=None, msg_prefix=None): @@ -750,6 +830,13 @@ def array_ufunc_errmsg_formatter(dummy, ufunc, method, *inputs, **kwargs): .format(ufunc, method, args_string, types_string)) +def array_function_errmsg_formatter(public_api, types): + """ Format the error message for when __array_ufunc__ gives up. """ + func_name = '{}.{}'.format(public_api.__module__, public_api.__name__) + return ("no implementation found for '{}' on types that implement " + '__array_function__: {}'.format(func_name, list(types))) + + def _ufunc_doc_signature_formatter(ufunc): """ Builds a signature string which resembles PEP 457 @@ -796,13 +883,13 @@ def _ufunc_doc_signature_formatter(ufunc): ) -def _is_from_ctypes(obj): - # determine if an object comes from ctypes, in order to work around +def npy_ctypes_check(cls): + # determine if a class comes from ctypes, in order to work around # a bug in the buffer protocol for those objects, bpo-10746 try: # ctypes class are new-style, so have an __mro__. This probably fails # for ctypes classes with multiple inheritance. - ctype_base = type(obj).__mro__[-2] + ctype_base = cls.__mro__[-2] # right now, they're part of the _ctypes module return 'ctypes' in ctype_base.__module__ except Exception: diff --git a/numpy/core/_methods.py b/numpy/core/_methods.py index 8974f0ce1..51362c761 100644 --- a/numpy/core/_methods.py +++ b/numpy/core/_methods.py @@ -24,20 +24,20 @@ umr_all = um.logical_and.reduce # avoid keyword arguments to speed up parsing, saves about 15%-20% for very # small reductions def _amax(a, axis=None, out=None, keepdims=False, - initial=_NoValue): - return umr_maximum(a, axis, None, out, keepdims, initial) + initial=_NoValue, where=True): + return umr_maximum(a, axis, None, out, keepdims, initial, where) def _amin(a, axis=None, out=None, keepdims=False, - initial=_NoValue): - return umr_minimum(a, axis, None, out, keepdims, initial) + initial=_NoValue, where=True): + return umr_minimum(a, axis, None, out, keepdims, initial, where) def _sum(a, axis=None, dtype=None, out=None, keepdims=False, - initial=_NoValue): - return umr_sum(a, axis, dtype, out, keepdims, initial) + initial=_NoValue, where=True): + return umr_sum(a, axis, dtype, out, keepdims, initial, where) def _prod(a, axis=None, dtype=None, out=None, keepdims=False, - initial=_NoValue): - return umr_prod(a, axis, dtype, out, keepdims, initial) + initial=_NoValue, where=True): + return umr_prod(a, axis, dtype, out, keepdims, initial, where) def _any(a, axis=None, dtype=None, out=None, keepdims=False): return umr_any(a, axis, dtype, out, keepdims) @@ -154,18 +154,3 @@ def _ptp(a, axis=None, out=None, keepdims=False): umr_minimum(a, axis, None, None, keepdims), out ) - -_NDARRAY_ARRAY_FUNCTION = mu.ndarray.__array_function__ - -def _array_function(self, func, types, args, kwargs): - # TODO: rewrite this in C - # Cannot handle items that have __array_function__ other than our own. - for t in types: - if t is not mu.ndarray: - method = getattr(t, '__array_function__', _NDARRAY_ARRAY_FUNCTION) - if method is not _NDARRAY_ARRAY_FUNCTION: - return NotImplemented - - # Arguments contain no overrides, so we can safely call the - # overloaded function again. - return func(*args, **kwargs) diff --git a/numpy/core/_type_aliases.py b/numpy/core/_type_aliases.py index cce6c0425..d6e1a1fb7 100644 --- a/numpy/core/_type_aliases.py +++ b/numpy/core/_type_aliases.py @@ -60,7 +60,7 @@ for k, v in typeinfo.items(): else: _concrete_typeinfo[k] = v -_concrete_types = set(v.type for k, v in _concrete_typeinfo.items()) +_concrete_types = {v.type for k, v in _concrete_typeinfo.items()} def _bits_of(obj): diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 1b9fbbfa9..7d8785c32 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -26,6 +26,7 @@ __docformat__ = 'restructuredtext' import sys import functools +import numbers if sys.version_info[0] >= 3: try: from _thread import get_ident @@ -48,7 +49,7 @@ from .fromnumeric import ravel, any from .numeric import concatenate, asarray, errstate from .numerictypes import (longlong, intc, int_, float_, complex_, bool_, flexible) -from .overrides import array_function_dispatch +from .overrides import array_function_dispatch, set_module import warnings import contextlib @@ -86,9 +87,15 @@ def _make_options_dict(precision=None, threshold=None, edgeitems=None, if legacy not in [None, False, '1.13']: warnings.warn("legacy printing option can currently only be '1.13' or " "`False`", stacklevel=3) - + if threshold is not None: + # forbid the bad threshold arg suggested by stack overflow, gh-12351 + if not isinstance(threshold, numbers.Number) or np.isnan(threshold): + raise ValueError("threshold must be numeric and non-NAN, try " + "sys.maxsize for untruncated representation") return options + +@set_module('numpy') def set_printoptions(precision=None, threshold=None, edgeitems=None, linewidth=None, suppress=None, nanstr=None, infstr=None, formatter=None, sign=None, floatmode=None, **kwarg): @@ -194,21 +201,21 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, Floating point precision can be set: >>> np.set_printoptions(precision=4) - >>> print(np.array([1.123456789])) - [ 1.1235] + >>> np.array([1.123456789]) + [1.1235] Long arrays can be summarised: >>> np.set_printoptions(threshold=5) - >>> print(np.arange(10)) - [0 1 2 ..., 7 8 9] + >>> np.arange(10) + array([0, 1, 2, ..., 7, 8, 9]) Small results can be suppressed: >>> eps = np.finfo(float).eps >>> x = np.arange(4.) >>> x**2 - (x + eps)**2 - array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) + array([-4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) >>> np.set_printoptions(suppress=True) >>> x**2 - (x + eps)**2 array([-0., -0., 0., 0.]) @@ -250,6 +257,7 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, set_legacy_print_mode(0) +@set_module('numpy') def get_printoptions(): """ Return the current print options. @@ -279,6 +287,7 @@ def get_printoptions(): return _format_options.copy() +@set_module('numpy') @contextlib.contextmanager def printoptions(*args, **kwargs): """Context manager for setting print options. @@ -290,9 +299,10 @@ def printoptions(*args, **kwargs): Examples -------- + >>> from numpy.testing import assert_equal >>> with np.printoptions(precision=2): - ... print(np.array([2.0])) / 3 - [0.67] + ... np.array([2.0]) / 3 + array([0.67]) The `as`-clause of the `with`-statement gives the current print options: @@ -506,7 +516,7 @@ def _array2string_dispatcher( return (a,) -@array_function_dispatch(_array2string_dispatcher) +@array_function_dispatch(_array2string_dispatcher, module='numpy') def array2string(a, max_line_width=None, precision=None, suppress_small=None, separator=' ', prefix="", style=np._NoValue, formatter=None, threshold=None, @@ -635,9 +645,9 @@ def array2string(a, max_line_width=None, precision=None, Examples -------- >>> x = np.array([1e-16,1,2,3]) - >>> print(np.array2string(x, precision=2, separator=',', - ... suppress_small=True)) - [ 0., 1., 2., 3.] + >>> np.array2string(x, precision=2, separator=',', + ... suppress_small=True) + '[0.,1.,2.,3.]' >>> x = np.arange(3.) >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x}) @@ -645,7 +655,7 @@ def array2string(a, max_line_width=None, precision=None, >>> x = np.arange(3) >>> np.array2string(x, formatter={'int':lambda x: hex(x)}) - '[0x0L 0x1L 0x2L]' + '[0x0 0x1 0x2]' """ legacy = kwarg.pop('legacy', None) @@ -976,6 +986,8 @@ class LongFloatFormat(FloatingFormat): DeprecationWarning, stacklevel=2) super(LongFloatFormat, self).__init__(*args, **kwargs) + +@set_module('numpy') def format_float_scientific(x, precision=None, unique=True, trim='k', sign=False, pad_left=None, exp_digits=None): """ @@ -1043,6 +1055,8 @@ def format_float_scientific(x, precision=None, unique=True, trim='k', trim=trim, sign=sign, pad_left=pad_left, exp_digits=exp_digits) + +@set_module('numpy') def format_float_positional(x, precision=None, unique=True, fractional=True, trim='k', sign=False, pad_left=None, pad_right=None): @@ -1344,7 +1358,7 @@ def dtype_is_implied(dtype): >>> np.core.arrayprint.dtype_is_implied(np.int8) False >>> np.array([1, 2, 3], np.int8) - array([1, 2, 3], dtype=np.int8) + array([1, 2, 3], dtype=int8) """ dtype = np.dtype(dtype) if _format_options['legacy'] == '1.13' and dtype.type == bool_: @@ -1364,6 +1378,7 @@ def dtype_short_repr(dtype): The intent is roughly that the following holds >>> from numpy import * + >>> dt = np.int64([1, 2]).dtype >>> assert eval(dtype_short_repr(dt)) == dt """ if dtype.names is not None: @@ -1381,12 +1396,59 @@ def dtype_short_repr(dtype): return typename +def _array_repr_implementation( + arr, max_line_width=None, precision=None, suppress_small=None, + array2string=array2string): + """Internal version of array_repr() that allows overriding array2string.""" + if max_line_width is None: + max_line_width = _format_options['linewidth'] + + if type(arr) is not ndarray: + class_name = type(arr).__name__ + else: + class_name = "array" + + skipdtype = dtype_is_implied(arr.dtype) and arr.size > 0 + + prefix = class_name + "(" + suffix = ")" if skipdtype else "," + + if (_format_options['legacy'] == '1.13' and + arr.shape == () and not arr.dtype.names): + lst = repr(arr.item()) + elif arr.size > 0 or arr.shape == (0,): + lst = array2string(arr, max_line_width, precision, suppress_small, + ', ', prefix, suffix=suffix) + else: # show zero-length shape unless it is (0,) + lst = "[], shape=%s" % (repr(arr.shape),) + + arr_str = prefix + lst + suffix + + if skipdtype: + return arr_str + + dtype_str = "dtype={})".format(dtype_short_repr(arr.dtype)) + + # compute whether we should put dtype on a new line: Do so if adding the + # dtype would extend the last line past max_line_width. + # Note: This line gives the correct result even when rfind returns -1. + last_line_len = len(arr_str) - (arr_str.rfind('\n') + 1) + spacer = " " + if _format_options['legacy'] == '1.13': + if issubclass(arr.dtype.type, flexible): + spacer = '\n' + ' '*len(class_name + "(") + elif last_line_len + len(dtype_str) + 1 > max_line_width: + spacer = '\n' + ' '*len(class_name + "(") + + return arr_str + spacer + dtype_str + + def _array_repr_dispatcher( arr, max_line_width=None, precision=None, suppress_small=None): return (arr,) -@array_function_dispatch(_array_repr_dispatcher) +@array_function_dispatch(_array_repr_dispatcher, module='numpy') def array_repr(arr, max_line_width=None, precision=None, suppress_small=None): """ Return the string representation of an array. @@ -1420,59 +1482,40 @@ def array_repr(arr, max_line_width=None, precision=None, suppress_small=None): >>> np.array_repr(np.array([1,2])) 'array([1, 2])' >>> np.array_repr(np.ma.array([0.])) - 'MaskedArray([ 0.])' + 'MaskedArray([0.])' >>> np.array_repr(np.array([], np.int32)) 'array([], dtype=int32)' >>> x = np.array([1e-6, 4e-7, 2, 3]) >>> np.array_repr(x, precision=6, suppress_small=True) - 'array([ 0.000001, 0. , 2. , 3. ])' + 'array([0.000001, 0. , 2. , 3. ])' """ - if max_line_width is None: - max_line_width = _format_options['linewidth'] + return _array_repr_implementation( + arr, max_line_width, precision, suppress_small) - if type(arr) is not ndarray: - class_name = type(arr).__name__ - else: - class_name = "array" - skipdtype = dtype_is_implied(arr.dtype) and arr.size > 0 +_guarded_str = _recursive_guard()(str) - prefix = class_name + "(" - suffix = ")" if skipdtype else "," +def _array_str_implementation( + a, max_line_width=None, precision=None, suppress_small=None, + array2string=array2string): + """Internal version of array_str() that allows overriding array2string.""" if (_format_options['legacy'] == '1.13' and - arr.shape == () and not arr.dtype.names): - lst = repr(arr.item()) - elif arr.size > 0 or arr.shape == (0,): - lst = array2string(arr, max_line_width, precision, suppress_small, - ', ', prefix, suffix=suffix) - else: # show zero-length shape unless it is (0,) - lst = "[], shape=%s" % (repr(arr.shape),) - - arr_str = prefix + lst + suffix - - if skipdtype: - return arr_str - - dtype_str = "dtype={})".format(dtype_short_repr(arr.dtype)) - - # compute whether we should put dtype on a new line: Do so if adding the - # dtype would extend the last line past max_line_width. - # Note: This line gives the correct result even when rfind returns -1. - last_line_len = len(arr_str) - (arr_str.rfind('\n') + 1) - spacer = " " - if _format_options['legacy'] == '1.13': - if issubclass(arr.dtype.type, flexible): - spacer = '\n' + ' '*len(class_name + "(") - elif last_line_len + len(dtype_str) + 1 > max_line_width: - spacer = '\n' + ' '*len(class_name + "(") - - return arr_str + spacer + dtype_str + a.shape == () and not a.dtype.names): + return str(a.item()) + # the str of 0d arrays is a special case: It should appear like a scalar, + # so floats are not truncated by `precision`, and strings are not wrapped + # in quotes. So we return the str of the scalar value. + if a.shape == (): + # obtain a scalar and call str on it, avoiding problems for subclasses + # for which indexing with () returns a 0d instead of a scalar by using + # ndarray's getindex. Also guard against recursive 0d object arrays. + return _guarded_str(np.ndarray.__getitem__(a, ())) -_guarded_str = _recursive_guard()(str) + return array2string(a, max_line_width, precision, suppress_small, ' ', "") def _array_str_dispatcher( @@ -1480,7 +1523,7 @@ def _array_str_dispatcher( return (a,) -@array_function_dispatch(_array_str_dispatcher) +@array_function_dispatch(_array_str_dispatcher, module='numpy') def array_str(a, max_line_width=None, precision=None, suppress_small=None): """ Return a string representation of the data in an array. @@ -1515,20 +1558,17 @@ def array_str(a, max_line_width=None, precision=None, suppress_small=None): '[0 1 2]' """ - if (_format_options['legacy'] == '1.13' and - a.shape == () and not a.dtype.names): - return str(a.item()) + return _array_str_implementation( + a, max_line_width, precision, suppress_small) - # the str of 0d arrays is a special case: It should appear like a scalar, - # so floats are not truncated by `precision`, and strings are not wrapped - # in quotes. So we return the str of the scalar value. - if a.shape == (): - # obtain a scalar and call str on it, avoiding problems for subclasses - # for which indexing with () returns a 0d instead of a scalar by using - # ndarray's getindex. Also guard against recursive 0d object arrays. - return _guarded_str(np.ndarray.__getitem__(a, ())) - return array2string(a, max_line_width, precision, suppress_small, ' ', "") +# needed if __array_function__ is disabled +_array2string_impl = getattr(array2string, '__wrapped__', array2string) +_default_array_str = functools.partial(_array_str_implementation, + array2string=_array2string_impl) +_default_array_repr = functools.partial(_array_repr_implementation, + array2string=_array2string_impl) + def set_string_function(f, repr=True): """ @@ -1559,8 +1599,8 @@ def set_string_function(f, repr=True): >>> a = np.arange(10) >>> a HA! - What are you going to do now? - >>> print(a) - [0 1 2 3 4 5 6 7 8 9] + >>> _ = a + >>> # [0 1 2 3 4 5 6 7 8 9] We can reset the function to the default: @@ -1578,16 +1618,16 @@ def set_string_function(f, repr=True): >>> x.__str__() 'random' >>> x.__repr__() - 'array([ 0, 1, 2, 3])' + 'array([0, 1, 2, 3])' """ if f is None: if repr: - return multiarray.set_string_function(array_repr, 1) + return multiarray.set_string_function(_default_array_repr, 1) else: - return multiarray.set_string_function(array_str, 0) + return multiarray.set_string_function(_default_array_str, 0) else: return multiarray.set_string_function(f, repr) -set_string_function(array_str, 0) -set_string_function(array_repr, 1) +set_string_function(_default_array_str, 0) +set_string_function(_default_array_repr, 1) diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index c8b998bfc..00f10df57 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -39,9 +39,12 @@ 0x0000000b = edb1ba83730c650fd9bc5772a919cda7 # Version 12 (NumPy 1.14) Added PyArray_ResolveWritebackIfCopy, -# Version 12 (NumPy 1.15) No change. # PyArray_SetWritebackIfCopyBase and deprecated PyArray_SetUpdateIfCopyBase. +# Version 12 (NumPy 1.15) No change. 0x0000000c = a1bc756c5782853ec2e3616cf66869d8 -# Version 13 (Numpy 1.16) Added fields core_dim_flags and core_dim_sizes to PyUFuncObject -0x0000000d = a1bc756c5782853ec2e3616cf66869d8 +# Version 13 (NumPy 1.16) +# Deprecate PyArray_SetNumericOps and PyArray_GetNumericOps, +# Add fields core_dim_flags and core_dim_sizes to PyUFuncObject. +# Add PyUFunc_FromFuncAndDataAndSignatureAndIdentity to ufunc_funcs_api. +0x0000000d = 5b0e8bbded00b166125974fc71e80a33 diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 42c564a97..4aca2373c 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -19,6 +19,7 @@ __docformat__ = 'restructuredtext' # The files under src/ that are scanned for API functions API_FILES = [join('multiarray', 'alloc.c'), + join('multiarray', 'arrayfunction_override.c'), join('multiarray', 'array_assign_array.c'), join('multiarray', 'array_assign_scalar.c'), join('multiarray', 'arrayobject.c'), @@ -163,9 +164,7 @@ def skip_brackets(s, lbrac, rbrac): def split_arguments(argstr): arguments = [] - bracket_counts = {'(': 0, '[': 0} current_argument = [] - state = 0 i = 0 def finish_arg(): if current_argument: @@ -400,9 +399,7 @@ class FunctionApi(object): return " (void *) %s" % self.name def internal_define(self): - annstr = [] - for a in self.annotations: - annstr.append(str(a)) + annstr = [str(a) for a in self.annotations] annstr = ' '.join(annstr) astr = """\ NPY_NO_EXPORT %s %s %s \\\n (%s);""" % (annstr, self.return_type, @@ -463,10 +460,7 @@ def get_api_functions(tagname, api_dict): functions = [] for f in API_FILES: functions.extend(find_functions(f, tagname)) - dfunctions = [] - for func in functions: - o = api_dict[func.name][0] - dfunctions.append( (o, func) ) + dfunctions = [(api_dict[func.name][0], func) for func in functions] dfunctions.sort() return [a[1] for a in dfunctions] diff --git a/numpy/core/code_generators/generate_numpy_api.py b/numpy/core/code_generators/generate_numpy_api.py index 7f2541667..5e04fb86d 100644 --- a/numpy/core/code_generators/generate_numpy_api.py +++ b/numpy/core/code_generators/generate_numpy_api.py @@ -50,7 +50,6 @@ _import_array(void) PyObject *c_api = NULL; if (numpy == NULL) { - PyErr_SetString(PyExc_ImportError, "numpy.core._multiarray_umath failed to import"); return -1; } c_api = PyObject_GetAttrString(numpy, "_ARRAY_API"); @@ -193,7 +192,9 @@ def do_generate_api(targets, sources): genapi.check_api_dict(multiarray_api_index) numpyapi_list = genapi.get_api_functions('NUMPY_API', - multiarray_funcs) + multiarray_funcs) + + # FIXME: ordered_funcs_api is unused ordered_funcs_api = genapi.order_dict(multiarray_funcs) # Create dict name -> *Api instance diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index 6dc01877b..108fff631 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -10,11 +10,14 @@ sys.path.insert(0, os.path.dirname(__file__)) import ufunc_docstrings as docstrings sys.path.pop(0) -Zero = "PyUFunc_Zero" -One = "PyUFunc_One" -None_ = "PyUFunc_None" -AllOnes = "PyUFunc_MinusOne" -ReorderableNone = "PyUFunc_ReorderableNone" +Zero = "PyInt_FromLong(0)" +One = "PyInt_FromLong(1)" +True_ = "(Py_INCREF(Py_True), Py_True)" +False_ = "(Py_INCREF(Py_False), Py_False)" +None_ = object() +AllOnes = "PyInt_FromLong(-1)" +MinusInfinity = 'PyFloat_FromDouble(-NPY_INFINITY)' +ReorderableNone = "(Py_INCREF(Py_None), Py_None)" # Sentinel value to specify using the full type description in the # function name @@ -74,10 +77,7 @@ class TypeDescription(object): _fdata_map = dict(e='npy_%sf', f='npy_%sf', d='npy_%s', g='npy_%sl', F='nc_%sf', D='nc_%s', G='nc_%sl') def build_func_data(types, f): - func_data = [] - for t in types: - d = _fdata_map.get(t, '%s') % (f,) - func_data.append(d) + func_data = [_fdata_map.get(t, '%s') % (f,) for t in types] return func_data def TD(types, f=None, astype=None, in_=None, out=None, simd=None): @@ -124,7 +124,7 @@ class Ufunc(object): type_descriptions : list of TypeDescription objects """ def __init__(self, nin, nout, identity, docstring, typereso, - *type_descriptions): + *type_descriptions, **kwargs): self.nin = nin self.nout = nout if identity is None: @@ -133,10 +133,13 @@ class Ufunc(object): self.docstring = docstring self.typereso = typereso self.type_descriptions = [] + self.signature = kwargs.pop('signature', None) for td in type_descriptions: self.type_descriptions.extend(td) for td in self.type_descriptions: td.finish_signature(self.nin, self.nout) + if kwargs: + raise ValueError('unknown kwargs %r' % str(kwargs)) # String-handling utilities to avoid locale-dependence. @@ -312,7 +315,7 @@ defdict = { TD(intfltcmplx), [TypeDescription('m', FullTypeDescr, 'mq', 'm'), TypeDescription('m', FullTypeDescr, 'md', 'm'), - #TypeDescription('m', FullTypeDescr, 'mm', 'd'), + TypeDescription('m', FullTypeDescr, 'mm', 'q'), ], TD(O, f='PyNumber_FloorDivide'), ), @@ -458,7 +461,7 @@ defdict = { [TypeDescription('O', FullTypeDescr, 'OO', 'O')], ), 'logical_and': - Ufunc(2, 1, One, + Ufunc(2, 1, True_, docstrings.get('numpy.core.umath.logical_and'), 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(nodatetime_or_obj, out='?', simd=[('avx2', ints)]), @@ -472,14 +475,14 @@ defdict = { TD(O, f='npy_ObjectLogicalNot'), ), 'logical_or': - Ufunc(2, 1, Zero, + Ufunc(2, 1, False_, docstrings.get('numpy.core.umath.logical_or'), 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(nodatetime_or_obj, out='?', simd=[('avx2', ints)]), TD(O, f='npy_ObjectLogicalOr'), ), 'logical_xor': - Ufunc(2, 1, Zero, + Ufunc(2, 1, False_, docstrings.get('numpy.core.umath.logical_xor'), 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(nodatetime_or_obj, out='?'), @@ -514,7 +517,7 @@ defdict = { TD(O, f='npy_ObjectMin') ), 'logaddexp': - Ufunc(2, 1, None, + Ufunc(2, 1, MinusInfinity, docstrings.get('numpy.core.umath.logaddexp'), None, TD(flts, f="logaddexp", astype={'e':'f'}) @@ -791,8 +794,9 @@ defdict = { 'remainder': Ufunc(2, 1, None, docstrings.get('numpy.core.umath.remainder'), - None, + 'PyUFunc_RemainderTypeResolver', TD(intflt), + [TypeDescription('m', FullTypeDescr, 'mm', 'm')], TD(O, f='PyNumber_Remainder'), ), 'divmod': @@ -900,7 +904,14 @@ defdict = { "PyUFunc_SimpleBinaryOperationTypeResolver", TD(ints), TD('O', f='npy_ObjectLCM'), - ) + ), +'matmul' : + Ufunc(2, 1, None, + docstrings.get('numpy.core.umath.matmul'), + "PyUFunc_SimpleBinaryOperationTypeResolver", + TD(notimes_or_obj), + signature='(n?,k),(k,m?)->(n?,m?)', + ), } if sys.version_info[0] >= 3: @@ -1046,19 +1057,44 @@ def make_ufuncs(funcdict): # string literal in C code. We split at endlines because textwrap.wrap # do not play well with \n docstring = '\\n\"\"'.join(docstring.split(r"\n")) + if uf.signature is None: + sig = "NULL" + else: + sig = '"{}"'.format(uf.signature) fmt = textwrap.dedent("""\ - f = PyUFunc_FromFuncAndData( + identity = {identity_expr}; + if ({has_identity} && identity == NULL) {{ + return -1; + }} + f = PyUFunc_FromFuncAndDataAndSignatureAndIdentity( {name}_functions, {name}_data, {name}_signatures, {nloops}, {nin}, {nout}, {identity}, "{name}", - "{doc}", 0 + "{doc}", 0, {sig}, identity ); + if ({has_identity}) {{ + Py_DECREF(identity); + }} if (f == NULL) {{ return -1; - }}""") - mlist.append(fmt.format( + }} + """) + args = dict( name=name, nloops=len(uf.type_descriptions), - nin=uf.nin, nout=uf.nout, identity=uf.identity, doc=docstring - )) + nin=uf.nin, nout=uf.nout, + has_identity='0' if uf.identity is None_ else '1', + identity='PyUFunc_IdentityValue', + identity_expr=uf.identity, + doc=docstring, + sig=sig, + ) + + # Only PyUFunc_None means don't reorder - we pass this using the old + # argument + if uf.identity is None_: + args['identity'] = 'PyUFunc_None' + args['identity_expr'] = 'NULL' + + mlist.append(fmt.format(**args)) if uf.typereso is not None: mlist.append( r"((PyUFuncObject *)f)->type_resolver = &%s;" % uf.typereso) @@ -1082,11 +1118,13 @@ def make_code(funcdict, filename): #include "cpuid.h" #include "ufunc_object.h" #include "ufunc_type_resolution.h" + #include "loops.h" + #include "matmul.h" %s static int InitOperators(PyObject *dictionary) { - PyObject *f; + PyObject *f, *identity; %s %s diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index d8a9ee6b4..a71c236fd 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -402,6 +402,8 @@ ufunc_funcs_api = { # End 1.7 API 'PyUFunc_RegisterLoopForDescr': (41,), # End 1.8 API + 'PyUFunc_FromFuncAndDataAndSignatureAndIdentity': (42,), + # End 1.16 API } # List of all the dicts which define the C API diff --git a/numpy/core/code_generators/ufunc_docstrings.py b/numpy/core/code_generators/ufunc_docstrings.py index 13231de29..267e63b2d 100644 --- a/numpy/core/code_generators/ufunc_docstrings.py +++ b/numpy/core/code_generators/ufunc_docstrings.py @@ -39,7 +39,8 @@ subst = { def add_newdoc(place, name, doc): doc = textwrap.dedent(doc).strip() - if name[0] != '_': + if name[0] != '_' and name != 'matmul': + # matmul is special, it does not use the OUT_SCALAR replacement strings if '\nx :' in doc: assert '$OUT_SCALAR_1' in doc, "in {}".format(name) elif '\nx2 :' in doc or '\nx1, x2 :' in doc: @@ -647,8 +648,8 @@ add_newdoc('numpy.core.umath', 'bitwise_or', array([ 6, 5, 255]) >>> np.array([2, 5, 255]) | np.array([4, 4, 4]) array([ 6, 5, 255]) - >>> np.bitwise_or(np.array([2, 5, 255, 2147483647L], dtype=np.int32), - ... np.array([4, 4, 4, 2147483647L], dtype=np.int32)) + >>> np.bitwise_or(np.array([2, 5, 255, 2147483647], dtype=np.int32), + ... np.array([4, 4, 4, 2147483647], dtype=np.int32)) array([ 6, 5, 255, 2147483647]) >>> np.bitwise_or([True, True], [False, True]) array([ True, True]) @@ -836,6 +837,7 @@ add_newdoc('numpy.core.umath', 'cos', array([ 1.00000000e+00, 6.12303177e-17, -1.00000000e+00]) >>> >>> # Example of providing the optional output parameter + >>> out1 = np.array([0], dtype='d') >>> out2 = np.cos([0.1], out1) >>> out2 is out1 True @@ -844,7 +846,7 @@ add_newdoc('numpy.core.umath', 'cos', >>> np.cos(np.zeros((3,3)),np.zeros((2,2))) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: invalid return array shape + ValueError: operands could not be broadcast together with shapes (3,3) (2,2) """) @@ -911,7 +913,7 @@ add_newdoc('numpy.core.umath', 'degrees', 270., 300., 330.]) >>> out = np.zeros((rad.shape)) - >>> r = degrees(rad, out) + >>> r = np.degrees(rad, out) >>> np.all(r == out) True @@ -1558,33 +1560,31 @@ add_newdoc('numpy.core.umath', 'invert', We've seen that 13 is represented by ``00001101``. The invert or bit-wise NOT of 13 is then: - >>> np.invert(np.array([13], dtype=uint8)) - array([242], dtype=uint8) + >>> x = np.invert(np.array(13, dtype=np.uint8)) + >>> x + 242 >>> np.binary_repr(x, width=8) - '00001101' - >>> np.binary_repr(242, width=8) '11110010' The result depends on the bit-width: - >>> np.invert(np.array([13], dtype=uint16)) - array([65522], dtype=uint16) + >>> x = np.invert(np.array(13, dtype=np.uint16)) + >>> x + 65522 >>> np.binary_repr(x, width=16) - '0000000000001101' - >>> np.binary_repr(65522, width=16) '1111111111110010' When using signed integer types the result is the two's complement of the result for the unsigned type: - >>> np.invert(np.array([13], dtype=int8)) + >>> np.invert(np.array([13], dtype=np.int8)) array([-14], dtype=int8) >>> np.binary_repr(-14, width=8) '11110010' Booleans are accepted as well: - >>> np.invert(array([True, False])) + >>> np.invert(np.array([True, False])) array([False, True]) """) @@ -1968,7 +1968,7 @@ add_newdoc('numpy.core.umath', 'log10', Examples -------- >>> np.log10([1e-15, -3.]) - array([-15., NaN]) + array([-15., nan]) """) @@ -2360,7 +2360,7 @@ add_newdoc('numpy.core.umath', 'maximum', [ 0.5, 2. ]]) >>> np.maximum([np.nan, 0, np.nan], [0, np.nan, np.nan]) - array([ NaN, NaN, NaN]) + array([nan, nan, nan]) >>> np.maximum(np.Inf, 1) inf @@ -2419,7 +2419,7 @@ add_newdoc('numpy.core.umath', 'minimum', [ 0. , 1. ]]) >>> np.minimum([np.nan, 0, np.nan],[0, np.nan, np.nan]) - array([ NaN, NaN, NaN]) + array([nan, nan, nan]) >>> np.minimum(-np.Inf, 1) -inf @@ -2479,7 +2479,7 @@ add_newdoc('numpy.core.umath', 'fmax', [ 0.5, 2. ]]) >>> np.fmax([np.nan, 0, np.nan],[0, np.nan, np.nan]) - array([ 0., 0., NaN]) + array([ 0., 0., nan]) """) @@ -2537,10 +2537,133 @@ add_newdoc('numpy.core.umath', 'fmin', [ 0. , 1. ]]) >>> np.fmin([np.nan, 0, np.nan],[0, np.nan, np.nan]) - array([ 0., 0., NaN]) + array([ 0., 0., nan]) """) +add_newdoc('numpy.core.umath', 'matmul', + """ + Matrix product of two arrays. + + Parameters + ---------- + x1, x2 : array_like + Input arrays, scalars not allowed. + out : ndarray, optional + A location into which the result is stored. If provided, it must have + a shape that matches the signature `(n,k),(k,m)->(n,m)`. If not + provided or `None`, a freshly-allocated array is returned. + **kwargs + For other keyword-only arguments, see the + :ref:`ufunc docs <ufuncs.kwargs>`. + + ..versionadded:: 1.16 + Now handles ufunc kwargs + + Returns + ------- + y : ndarray + The matrix product of the inputs. + This is a scalar only when both x1, x2 are 1-d vectors. + + Raises + ------ + ValueError + If the last dimension of `a` is not the same size as + the second-to-last dimension of `b`. + + If a scalar value is passed in. + + See Also + -------- + vdot : Complex-conjugating dot product. + tensordot : Sum products over arbitrary axes. + einsum : Einstein summation convention. + dot : alternative matrix product with different broadcasting rules. + + Notes + ----- + + The behavior depends on the arguments in the following way. + + - If both arguments are 2-D they are multiplied like conventional + matrices. + - If either argument is N-D, N > 2, it is treated as a stack of + matrices residing in the last two indexes and broadcast accordingly. + - If the first argument is 1-D, it is promoted to a matrix by + prepending a 1 to its dimensions. After matrix multiplication + the prepended 1 is removed. + - If the second argument is 1-D, it is promoted to a matrix by + appending a 1 to its dimensions. After matrix multiplication + the appended 1 is removed. + + ``matmul`` differs from ``dot`` in two important ways: + + - Multiplication by scalars is not allowed, use ``*`` instead. + - Stacks of matrices are broadcast together as if the matrices + were elements, respecting the signature ``(n,k),(k,m)->(n,m)``: + + >>> a = np.ones([9, 5, 7, 4]) + >>> c = np.ones([9, 5, 4, 3]) + >>> np.dot(a, c).shape + (9, 5, 7, 9, 5, 3) + >>> np.matmul(a, c).shape + (9, 5, 7, 3) + >>> # n is 7, k is 4, m is 3 + + The matmul function implements the semantics of the `@` operator introduced + in Python 3.5 following PEP465. + + Examples + -------- + For 2-D arrays it is the matrix product: + + >>> a = np.array([[1, 0], + ... [0, 1]]) + >>> b = np.array([[4, 1], + ... [2, 2]]) + >>> np.matmul(a, b) + array([[4, 1], + [2, 2]]) + + For 2-D mixed with 1-D, the result is the usual. + + >>> a = np.array([[1, 0], + ... [0, 1]]) + >>> b = np.array([1, 2]) + >>> np.matmul(a, b) + array([1, 2]) + >>> np.matmul(b, a) + array([1, 2]) + + + Broadcasting is conventional for stacks of arrays + + >>> a = np.arange(2 * 2 * 4).reshape((2, 2, 4)) + >>> b = np.arange(2 * 2 * 4).reshape((2, 4, 2)) + >>> np.matmul(a,b).shape + (2, 2, 2) + >>> np.matmul(a, b)[0, 1, 1] + 98 + >>> sum(a[0, 1, :] * b[0 , :, 1]) + 98 + + Vector, vector returns the scalar inner product, but neither argument + is complex-conjugated: + + >>> np.matmul([2j, 3j], [2j, 3j]) + (-13+0j) + + Scalar multiplication raises an error. + + >>> np.matmul([1,2], 3) + Traceback (most recent call last): + ... + ValueError: matmul: Input operand 1 does not have enough dimensions ... + + .. versionadded:: 1.10.0 + """) + add_newdoc('numpy.core.umath', 'modf', """ Return the fractional and integral parts of an array, element-wise. @@ -3352,6 +3475,7 @@ add_newdoc('numpy.core.umath', 'sinh', >>> # Discrepancy due to vagaries of floating point arithmetic. >>> # Example of providing the optional output parameter + >>> out1 = np.array([0], dtype='d') >>> out2 = np.sinh([0.1], out1) >>> out2 is out1 True @@ -3360,7 +3484,7 @@ add_newdoc('numpy.core.umath', 'sinh', >>> np.sinh(np.zeros((3,3)),np.zeros((2,2))) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: invalid return array shape + ValueError: operands could not be broadcast together with shapes (3,3) (2,2) """) @@ -3405,8 +3529,8 @@ add_newdoc('numpy.core.umath', 'sqrt', >>> np.sqrt([4, -1, -3+4J]) array([ 2.+0.j, 0.+1.j, 1.+2.j]) - >>> np.sqrt([4, -1, numpy.inf]) - array([ 2., NaN, Inf]) + >>> np.sqrt([4, -1, np.inf]) + array([ 2., nan, inf]) """) @@ -3537,6 +3661,7 @@ add_newdoc('numpy.core.umath', 'tan', >>> >>> # Example of providing the optional output parameter illustrating >>> # that what is returned is a reference to said parameter + >>> out1 = np.array([0], dtype='d') >>> out2 = np.cos([0.1], out1) >>> out2 is out1 True @@ -3545,7 +3670,7 @@ add_newdoc('numpy.core.umath', 'tan', >>> np.cos(np.zeros((3,3)),np.zeros((2,2))) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: invalid return array shape + ValueError: operands could not be broadcast together with shapes (3,3) (2,2) """) @@ -3588,6 +3713,7 @@ add_newdoc('numpy.core.umath', 'tanh', >>> # Example of providing the optional output parameter illustrating >>> # that what is returned is a reference to said parameter + >>> out1 = np.array([0], dtype='d') >>> out2 = np.tanh([0.1], out1) >>> out2 is out1 True @@ -3596,7 +3722,7 @@ add_newdoc('numpy.core.umath', 'tanh', >>> np.tanh(np.zeros((3,3)),np.zeros((2,2))) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: invalid return array shape + ValueError: operands could not be broadcast together with shapes (3,3) (2,2) """) @@ -3638,8 +3764,6 @@ add_newdoc('numpy.core.umath', 'true_divide', >>> np.true_divide(x, 4) array([ 0. , 0.25, 0.5 , 0.75, 1. ]) - >>> x/4 - array([0, 0, 0, 0, 1]) >>> x//4 array([0, 0, 0, 0, 1]) @@ -3735,7 +3859,7 @@ add_newdoc('numpy.core.umath', 'ldexp', Examples -------- >>> np.ldexp(5, np.arange(4)) - array([ 5., 10., 20., 40.], dtype=float32) + array([ 5., 10., 20., 40.], dtype=float16) >>> x = np.arange(6) >>> np.ldexp(*np.frexp(x)) diff --git a/numpy/core/defchararray.py b/numpy/core/defchararray.py index 0a8c7bbec..007fc6186 100644 --- a/numpy/core/defchararray.py +++ b/numpy/core/defchararray.py @@ -17,12 +17,14 @@ The preferred alias for `defchararray` is `numpy.char`. """ from __future__ import division, absolute_import, print_function +import functools import sys from .numerictypes import string_, unicode_, integer, object_, bool_, character from .numeric import ndarray, compare_chararrays from .numeric import array as narray from numpy.core.multiarray import _vec_string -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import set_module +from numpy.core import overrides from numpy.compat import asbytes, long import numpy @@ -48,6 +50,10 @@ else: _bytes = str _len = len +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy.char') + + def _use_unicode(*args): """ Helper function for determining the output type of some string @@ -492,8 +498,7 @@ def count(a, sub, start=0, end=None): -------- >>> c = np.array(['aAaAaA', ' aA ', 'abBABba']) >>> c - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') >>> np.char.count(c, 'A') array([3, 1, 1]) >>> np.char.count(c, 'aA') @@ -546,8 +551,7 @@ def decode(a, encoding=None, errors=None): -------- >>> c = np.array(['aAaAaA', ' aA ', 'abBABba']) >>> c - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') >>> np.char.encode(c, encoding='cp037') array(['\\x81\\xc1\\x81\\xc1\\x81\\xc1', '@@\\x81\\xc1@@', '\\x81\\x82\\xc2\\xc1\\xc2\\x82\\x81'], @@ -631,8 +635,7 @@ def endswith(a, suffix, start=0, end=None): >>> s[0] = 'foo' >>> s[1] = 'bar' >>> s - array(['foo', 'bar'], - dtype='|S3') + array(['foo', 'bar'], dtype='<U3') >>> np.char.endswith(s, 'ar') array([False, True]) >>> np.char.endswith(s, 'a', start=1, end=2) @@ -1030,11 +1033,9 @@ def lower(a): Examples -------- >>> c = np.array(['A1B C', '1BCA', 'BCA1']); c - array(['A1B C', '1BCA', 'BCA1'], - dtype='|S5') + array(['A1B C', '1BCA', 'BCA1'], dtype='<U5') >>> np.char.lower(c) - array(['a1b c', '1bca', 'bca1'], - dtype='|S5') + array(['a1b c', '1bca', 'bca1'], dtype='<U5') """ a_arr = numpy.asarray(a) @@ -1078,23 +1079,20 @@ def lstrip(a, chars=None): -------- >>> c = np.array(['aAaAaA', ' aA ', 'abBABba']) >>> c - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') The 'a' variable is unstripped from c[1] because whitespace leading. >>> np.char.lstrip(c, 'a') - array(['AaAaA', ' aA ', 'bBABba'], - dtype='|S7') + array(['AaAaA', ' aA ', 'bBABba'], dtype='<U7') >>> np.char.lstrip(c, 'A') # leaves c unchanged - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') >>> (np.char.lstrip(c, ' ') == np.char.lstrip(c, '')).all() - ... # XXX: is this a regression? this line now returns False + ... # XXX: is this a regression? This used to return True ... # np.char.lstrip(c,'') does not modify c at all. - True + False >>> (np.char.lstrip(c, ' ') == np.char.lstrip(c, None)).all() True @@ -1394,10 +1392,10 @@ def rstrip(a, chars=None): >>> c = np.array(['aAaAaA', 'abBABba'], dtype='S7'); c array(['aAaAaA', 'abBABba'], dtype='|S7') - >>> np.char.rstrip(c, 'a') + >>> np.char.rstrip(c, b'a') array(['aAaAaA', 'abBABb'], dtype='|S7') - >>> np.char.rstrip(c, 'A') + >>> np.char.rstrip(c, b'A') array(['aAaAa', 'abBABba'], dtype='|S7') @@ -1543,17 +1541,13 @@ def strip(a, chars=None): -------- >>> c = np.array(['aAaAaA', ' aA ', 'abBABba']) >>> c - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') >>> np.char.strip(c) - array(['aAaAaA', 'aA', 'abBABba'], - dtype='|S7') + array(['aAaAaA', 'aA', 'abBABba'], dtype='<U7') >>> np.char.strip(c, 'a') # 'a' unstripped from c[1] because whitespace leads - array(['AaAaA', ' aA ', 'bBABb'], - dtype='|S7') + array(['AaAaA', ' aA ', 'bBABb'], dtype='<U7') >>> np.char.strip(c, 'A') # 'A' unstripped from c[1] because (unprinted) ws trails - array(['aAaAa', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAa', ' aA ', 'abBABba'], dtype='<U7') """ a_arr = numpy.asarray(a) @@ -1705,11 +1699,9 @@ def upper(a): Examples -------- >>> c = np.array(['a1b c', '1bca', 'bca1']); c - array(['a1b c', '1bca', 'bca1'], - dtype='|S5') + array(['a1b c', '1bca', 'bca1'], dtype='<U5') >>> np.char.upper(c) - array(['A1B C', '1BCA', 'BCA1'], - dtype='|S5') + array(['A1B C', '1BCA', 'BCA1'], dtype='<U5') """ a_arr = numpy.asarray(a) @@ -1815,6 +1807,7 @@ def isdecimal(a): return _vec_string(a, bool_, 'isdecimal') +@set_module('numpy') class chararray(ndarray): """ chararray(shape, itemsize=1, unicode=False, buffer=None, offset=0, @@ -1943,18 +1936,16 @@ class chararray(ndarray): >>> charar = np.chararray((3, 3)) >>> charar[:] = 'a' >>> charar - chararray([['a', 'a', 'a'], - ['a', 'a', 'a'], - ['a', 'a', 'a']], - dtype='|S1') + chararray([[b'a', b'a', b'a'], + [b'a', b'a', b'a'], + [b'a', b'a', b'a']], dtype='|S1') >>> charar = np.chararray(charar.shape, itemsize=5) >>> charar[:] = 'abc' >>> charar - chararray([['abc', 'abc', 'abc'], - ['abc', 'abc', 'abc'], - ['abc', 'abc', 'abc']], - dtype='|S5') + chararray([[b'abc', b'abc', b'abc'], + [b'abc', b'abc', b'abc'], + [b'abc', b'abc', b'abc']], dtype='|S5') """ def __new__(subtype, shape, itemsize=1, unicode=False, buffer=None, diff --git a/numpy/core/einsumfunc.py b/numpy/core/einsumfunc.py index 1281b3c98..83b7d8287 100644 --- a/numpy/core/einsumfunc.py +++ b/numpy/core/einsumfunc.py @@ -9,6 +9,7 @@ import itertools from numpy.compat import basestring from numpy.core.multiarray import c_einsum from numpy.core.numeric import asanyarray, tensordot +from numpy.core.overrides import array_function_dispatch __all__ = ['einsum', 'einsum_path'] @@ -40,10 +41,10 @@ def _flop_count(idx_contraction, inner, num_terms, size_dictionary): -------- >>> _flop_count('abc', False, 1, {'a': 2, 'b':3, 'c':5}) - 90 + 30 >>> _flop_count('abc', True, 2, {'a': 2, 'b':3, 'c':5}) - 270 + 60 """ @@ -168,9 +169,9 @@ def _optimal_path(input_sets, output_set, idx_dict, memory_limit): Examples -------- >>> isets = [set('abd'), set('ac'), set('bdc')] - >>> oset = set('') + >>> oset = set() >>> idx_sizes = {'a': 1, 'b':2, 'c':3, 'd':4} - >>> _path__optimal_path(isets, oset, idx_sizes, 5000) + >>> _optimal_path(isets, oset, idx_sizes, 5000) [(0, 2), (0, 1)] """ @@ -339,9 +340,9 @@ def _greedy_path(input_sets, output_set, idx_dict, memory_limit): Examples -------- >>> isets = [set('abd'), set('ac'), set('bdc')] - >>> oset = set('') + >>> oset = set() >>> idx_sizes = {'a': 1, 'b':2, 'c':3, 'd':4} - >>> _path__greedy_path(isets, oset, idx_sizes, 5000) + >>> _greedy_path(isets, oset, idx_sizes, 5000) [(0, 2), (0, 1)] """ @@ -538,13 +539,14 @@ def _parse_einsum_input(operands): -------- The operand list is simplified to reduce printing: + >>> np.random.seed(123) >>> a = np.random.rand(4, 4) >>> b = np.random.rand(4, 4, 4) - >>> __parse_einsum_input(('...a,...a->...', a, b)) - ('za,xza', 'xz', [a, b]) + >>> _parse_einsum_input(('...a,...a->...', a, b)) + ('za,xza', 'xz', [a, b]) # may vary - >>> __parse_einsum_input((a, [Ellipsis, 0], b, [Ellipsis, 0])) - ('za,xza', 'xz', [a, b]) + >>> _parse_einsum_input((a, [Ellipsis, 0], b, [Ellipsis, 0])) + ('za,xza', 'xz', [a, b]) # may vary """ if len(operands) == 0: @@ -689,6 +691,17 @@ def _parse_einsum_input(operands): return (input_subscripts, output_subscript, operands) +def _einsum_path_dispatcher(*operands, **kwargs): + # NOTE: technically, we should only dispatch on array-like arguments, not + # subscripts (given as strings). But separating operands into + # arrays/subscripts is a little tricky/slow (given einsum's two supported + # signatures), so as a practical shortcut we dispatch on everything. + # Strings will be ignored for dispatching since they don't define + # __array_function__. + return operands + + +@array_function_dispatch(_einsum_path_dispatcher, module='numpy') def einsum_path(*operands, **kwargs): """ einsum_path(subscripts, *operands, optimize='greedy') @@ -751,6 +764,7 @@ def einsum_path(*operands, **kwargs): of the contraction and the remaining contraction ``(0, 1)`` is then completed. + >>> np.random.seed(123) >>> a = np.random.rand(2, 2) >>> b = np.random.rand(2, 5) >>> c = np.random.rand(5, 2) @@ -758,7 +772,7 @@ def einsum_path(*operands, **kwargs): >>> print(path_info[0]) ['einsum_path', (1, 2), (0, 1)] >>> print(path_info[1]) - Complete contraction: ij,jk,kl->il + Complete contraction: ij,jk,kl->il # may vary Naive scaling: 4 Optimized scaling: 3 Naive FLOP count: 1.600e+02 @@ -777,12 +791,12 @@ def einsum_path(*operands, **kwargs): >>> I = np.random.rand(10, 10, 10, 10) >>> C = np.random.rand(10, 10) >>> path_info = np.einsum_path('ea,fb,abcd,gc,hd->efgh', C, C, I, C, C, - optimize='greedy') + ... optimize='greedy') >>> print(path_info[0]) ['einsum_path', (0, 2), (0, 3), (0, 2), (0, 1)] - >>> print(path_info[1]) - Complete contraction: ea,fb,abcd,gc,hd->efgh + >>> print(path_info[1]) + Complete contraction: ea,fb,abcd,gc,hd->efgh # may vary Naive scaling: 8 Optimized scaling: 5 Naive FLOP count: 8.000e+08 @@ -837,7 +851,6 @@ def einsum_path(*operands, **kwargs): # Python side parsing input_subscripts, output_subscript, operands = _parse_einsum_input(operands) - subscripts = input_subscripts + '->' + output_subscript # Build a few useful list and sets input_list = input_subscripts.split(',') @@ -876,9 +889,8 @@ def einsum_path(*operands, **kwargs): broadcast_indices = [set(x) for x in broadcast_indices] # Compute size of each input array plus the output array - size_list = [] - for term in input_list + [output_subscript]: - size_list.append(_compute_size_by_dict(term, dimension_dict)) + size_list = [_compute_size_by_dict(term, dimension_dict) + for term in input_list + [output_subscript]] max_size = max(size_list) if memory_limit is None: @@ -980,7 +992,16 @@ def einsum_path(*operands, **kwargs): return (path, path_print) +def _einsum_dispatcher(*operands, **kwargs): + # Arguably we dispatch on more arguments that we really should; see note in + # _einsum_path_dispatcher for why. + for op in operands: + yield op + yield kwargs.get('out') + + # Rewrite einsum to handle different cases +@array_function_dispatch(_einsum_dispatcher, module='numpy') def einsum(*operands, **kwargs): """ einsum(subscripts, *operands, out=None, dtype=None, order='K', @@ -1255,32 +1276,32 @@ def einsum(*operands, **kwargs): >>> a = np.arange(60.).reshape(3,4,5) >>> b = np.arange(24.).reshape(4,3,2) >>> np.einsum('ijk,jil->kl', a, b) - array([[ 4400., 4730.], - [ 4532., 4874.], - [ 4664., 5018.], - [ 4796., 5162.], - [ 4928., 5306.]]) + array([[4400., 4730.], + [4532., 4874.], + [4664., 5018.], + [4796., 5162.], + [4928., 5306.]]) >>> np.einsum(a, [0,1,2], b, [1,0,3], [2,3]) - array([[ 4400., 4730.], - [ 4532., 4874.], - [ 4664., 5018.], - [ 4796., 5162.], - [ 4928., 5306.]]) + array([[4400., 4730.], + [4532., 4874.], + [4664., 5018.], + [4796., 5162.], + [4928., 5306.]]) >>> np.tensordot(a,b, axes=([1,0],[0,1])) - array([[ 4400., 4730.], - [ 4532., 4874.], - [ 4664., 5018.], - [ 4796., 5162.], - [ 4928., 5306.]]) + array([[4400., 4730.], + [4532., 4874.], + [4664., 5018.], + [4796., 5162.], + [4928., 5306.]]) Writeable returned arrays (since version 1.10.0): >>> a = np.zeros((3, 3)) >>> np.einsum('ii->i', a)[:] = 1 >>> a - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) Example of ellipsis use: @@ -1303,19 +1324,27 @@ def einsum(*operands, **kwargs): particularly significant with larger arrays: >>> a = np.ones(64).reshape(2,4,8) - # Basic `einsum`: ~1520ms (benchmarked on 3.1GHz Intel i5.) + + Basic `einsum`: ~1520ms (benchmarked on 3.1GHz Intel i5.) + >>> for iteration in range(500): - ... np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a) - # Sub-optimal `einsum` (due to repeated path calculation time): ~330ms + ... _ = np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a) + + Sub-optimal `einsum` (due to repeated path calculation time): ~330ms + >>> for iteration in range(500): - ... np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='optimal') - # Greedy `einsum` (faster optimal path approximation): ~160ms + ... _ = np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='optimal') + + Greedy `einsum` (faster optimal path approximation): ~160ms + >>> for iteration in range(500): - ... np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='greedy') - # Optimal `einsum` (best usage pattern in some use cases): ~110ms + ... _ = np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='greedy') + + Optimal `einsum` (best usage pattern in some use cases): ~110ms + >>> path = np.einsum_path('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='optimal')[0] >>> for iteration in range(500): - ... np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize=path) + ... _ = np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize=path) """ @@ -1354,9 +1383,7 @@ def einsum(*operands, **kwargs): # Start contraction loop for num, contraction in enumerate(contraction_list): inds, idx_rm, einsum_str, remaining, blas = contraction - tmp_operands = [] - for x in inds: - tmp_operands.append(operands.pop(x)) + tmp_operands = [operands.pop(x) for x in inds] # Do we need to deal with the output? handle_out = specified_out and ((num + 1) == len(contraction_list)) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 2fdbf3e23..d94372986 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -3,16 +3,17 @@ """ from __future__ import division, absolute_import, print_function +import functools import types import warnings import numpy as np from .. import VisibleDeprecationWarning from . import multiarray as mu +from . import overrides from . import umath as um from . import numerictypes as nt from .numeric import asarray, array, asanyarray, concatenate -from .overrides import array_function_dispatch from . import _methods _dt_ = nt.sctype2char @@ -32,6 +33,9 @@ _gentype = types.GeneratorType # save away Python sum _sum_ = sum +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + # functions that are now methods def _wrapit(obj, method, *args, **kwds): @@ -63,10 +67,8 @@ def _wrapfunc(obj, method, *args, **kwds): def _wrapreduction(obj, ufunc, method, axis, dtype, out, **kwargs): - passkwargs = {} - for k, v in kwargs.items(): - if v is not np._NoValue: - passkwargs[k] = v + passkwargs = {k: v for k, v in kwargs.items() + if v is not np._NoValue} if type(obj) is not mu.ndarray: try: @@ -238,12 +240,16 @@ def reshape(a, newshape, order='C'): you should assign the new shape to the shape attribute of the array:: >>> a = np.zeros((10, 2)) + # A transpose makes the array non-contiguous >>> b = a.T + # Taking a view makes it possible to modify the shape without modifying # the initial object. >>> c = b.view() >>> c.shape = (20) + Traceback (most recent call last): + ... AttributeError: incompatible shape for a non-contiguous array The `order` keyword gives the index ordering both for *fetching* the values @@ -1446,7 +1452,7 @@ def diagonal(a, offset=0, axis1=0, axis2=1): same type as `a` is returned unless `a` is a `matrix`, in which case a 1-D array rather than a (2-D) `matrix` is returned in order to maintain backward compatibility. - + If ``a.ndim > 2``, then the dimensions specified by `axis1` and `axis2` are removed, and a new axis inserted at the end corresponding to the diagonal. @@ -1642,21 +1648,21 @@ def ravel(a, order='C'): It is equivalent to ``reshape(-1, order=order)``. >>> x = np.array([[1, 2, 3], [4, 5, 6]]) - >>> print(np.ravel(x)) - [1 2 3 4 5 6] + >>> np.ravel(x) + array([1, 2, 3, 4, 5, 6]) - >>> print(x.reshape(-1)) - [1 2 3 4 5 6] + >>> x.reshape(-1) + array([1, 2, 3, 4, 5, 6]) - >>> print(np.ravel(x, order='F')) - [1 4 2 5 3 6] + >>> np.ravel(x, order='F') + array([1, 4, 2, 5, 3, 6]) When ``order`` is 'A', it will preserve the array's 'C' or 'F' ordering: - >>> print(np.ravel(x.T)) - [1 4 2 5 3 6] - >>> print(np.ravel(x.T, order='A')) - [1 2 3 4 5 6] + >>> np.ravel(x.T) + array([1, 4, 2, 5, 3, 6]) + >>> np.ravel(x.T, order='A') + array([1, 2, 3, 4, 5, 6]) When ``order`` is 'K', it will preserve orderings that are neither 'C' nor 'F', but won't reverse axes: @@ -1745,7 +1751,7 @@ def nonzero(a): array([[0, 0], [1, 1], [2, 0], - [2, 1]) + [2, 1]]) A common use for ``nonzero`` is to find the indices of an array, where a condition is True. Given an array `a`, the condition `a` > 3 is a @@ -1957,12 +1963,13 @@ def clip(a, a_min, a_max, out=None): def _sum_dispatcher(a, axis=None, dtype=None, out=None, keepdims=None, - initial=None): + initial=None, where=None): return (a, out) @array_function_dispatch(_sum_dispatcher) -def sum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._NoValue): +def sum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, + initial=np._NoValue, where=np._NoValue): """ Sum of array elements over a given axis. @@ -2006,6 +2013,11 @@ def sum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._No .. versionadded:: 1.15.0 + where : array_like of bool, optional + Elements to include in the sum. See `~numpy.ufunc.reduce` for details. + + .. versionadded:: 1.17.0 + Returns ------- sum_along_axis : ndarray @@ -2046,6 +2058,8 @@ def sum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._No array([0, 6]) >>> np.sum([[0, 1], [0, 5]], axis=1) array([1, 5]) + >>> np.sum([[0, 1], [np.nan, 5]], where=[False, True], axis=1) + array([1., 5.]) If the accumulator is too small, overflow occurs: @@ -2071,7 +2085,7 @@ def sum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._No return res return _wrapreduction(a, np.add, 'sum', axis, dtype, out, keepdims=keepdims, - initial=initial) + initial=initial, where=where) def _any_dispatcher(a, axis=None, out=None, keepdims=None): @@ -2148,10 +2162,10 @@ def any(a, axis=None, out=None, keepdims=np._NoValue): >>> np.any(np.nan) True - >>> o=np.array([False]) + >>> o=np.array(False) >>> z=np.any([-1, 4, 5], out=o) >>> z, o - (array([ True]), array([ True])) + (array(True), array(True)) >>> # Check now that z is a reference to o >>> z is o True @@ -2234,10 +2248,10 @@ def all(a, axis=None, out=None, keepdims=np._NoValue): >>> np.all([1.0, np.nan]) True - >>> o=np.array([False]) + >>> o=np.array(False) >>> z=np.all([-1, 4, 5], out=o) - >>> id(z), id(o), z # doctest: +SKIP - (28293632, 28293632, array([ True])) + >>> id(z), id(o), z + (28293632, 28293632, array(True)) # may vary """ return _wrapreduction(a, np.logical_and, 'all', axis, None, out, keepdims=keepdims) @@ -2388,12 +2402,14 @@ def ptp(a, axis=None, out=None, keepdims=np._NoValue): return _methods._ptp(a, axis=axis, out=out, **kwargs) -def _amax_dispatcher(a, axis=None, out=None, keepdims=None, initial=None): +def _amax_dispatcher(a, axis=None, out=None, keepdims=None, initial=None, + where=None): return (a, out) @array_function_dispatch(_amax_dispatcher) -def amax(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue): +def amax(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue, + where=np._NoValue): """ Return the maximum of an array or maximum along an axis. @@ -2431,6 +2447,11 @@ def amax(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue): .. versionadded:: 1.15.0 + where : array_like of bool, optional + Elements to compare for the maximum. See `~numpy.ufunc.reduce` + for details. + + .. versionadded:: 1.17.0 Returns ------- @@ -2476,11 +2497,14 @@ def amax(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue): array([2, 3]) >>> np.amax(a, axis=1) # Maxima along the second axis array([1, 3]) - + >>> np.amax(a, where=[False, True], initial=-1, axis=0) + array([-1, 3]) >>> b = np.arange(5, dtype=float) >>> b[2] = np.NaN >>> np.amax(b) nan + >>> np.amax(b, where=~np.isnan(b), initial=-1) + 4.0 >>> np.nanmax(b) 4.0 @@ -2499,16 +2523,18 @@ def amax(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue): >>> max([5], default=6) 5 """ - return _wrapreduction(a, np.maximum, 'max', axis, None, out, keepdims=keepdims, - initial=initial) + return _wrapreduction(a, np.maximum, 'max', axis, None, out, + keepdims=keepdims, initial=initial, where=where) -def _amin_dispatcher(a, axis=None, out=None, keepdims=None, initial=None): +def _amin_dispatcher(a, axis=None, out=None, keepdims=None, initial=None, + where=None): return (a, out) @array_function_dispatch(_amin_dispatcher) -def amin(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue): +def amin(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue, + where=np._NoValue): """ Return the minimum of an array or minimum along an axis. @@ -2546,6 +2572,12 @@ def amin(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue): .. versionadded:: 1.15.0 + where : array_like of bool, optional + Elements to compare for the minimum. See `~numpy.ufunc.reduce` + for details. + + .. versionadded:: 1.17.0 + Returns ------- amin : ndarray or scalar @@ -2590,11 +2622,15 @@ def amin(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue): array([0, 1]) >>> np.amin(a, axis=1) # Minima along the second axis array([0, 2]) + >>> np.amin(a, where=[False, True], initial=10, axis=0) + array([10, 1]) >>> b = np.arange(5, dtype=float) >>> b[2] = np.NaN >>> np.amin(b) nan + >>> np.amin(b, where=~np.isnan(b), initial=10) + 0.0 >>> np.nanmin(b) 0.0 @@ -2612,8 +2648,8 @@ def amin(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue): >>> min([6], default=5) 6 """ - return _wrapreduction(a, np.minimum, 'min', axis, None, out, keepdims=keepdims, - initial=initial) + return _wrapreduction(a, np.minimum, 'min', axis, None, out, + keepdims=keepdims, initial=initial, where=where) def _alen_dispathcer(a): @@ -2654,13 +2690,14 @@ def alen(a): return len(array(a, ndmin=1)) -def _prod_dispatcher( - a, axis=None, dtype=None, out=None, keepdims=None, initial=None): +def _prod_dispatcher(a, axis=None, dtype=None, out=None, keepdims=None, + initial=None, where=None): return (a, out) @array_function_dispatch(_prod_dispatcher) -def prod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._NoValue): +def prod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, + initial=np._NoValue, where=np._NoValue): """ Return the product of array elements over a given axis. @@ -2705,6 +2742,11 @@ def prod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._N .. versionadded:: 1.15.0 + where : array_like of bool, optional + Elements to include in the product. See `~numpy.ufunc.reduce` for details. + + .. versionadded:: 1.17.0 + Returns ------- product_along_axis : ndarray, see `dtype` parameter above. @@ -2722,8 +2764,8 @@ def prod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._N raised on overflow. That means that, on a 32-bit platform: >>> x = np.array([536870910, 536870910, 536870910, 536870910]) - >>> np.prod(x) # random - 16 + >>> np.prod(x) + 16 # may vary The product of an empty array is the neutral element 1: @@ -2747,6 +2789,11 @@ def prod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._N >>> np.prod([[1.,2.],[3.,4.]], axis=1) array([ 2., 12.]) + Or select specific elements to include: + + >>> np.prod([1., np.nan, 3.], where=[True, False, True]) + 3.0 + If the type of `x` is unsigned, then the output type is the unsigned platform integer: @@ -2766,8 +2813,8 @@ def prod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._N >>> np.prod([1, 2], initial=5) 10 """ - return _wrapreduction(a, np.multiply, 'prod', axis, dtype, out, keepdims=keepdims, - initial=initial) + return _wrapreduction(a, np.multiply, 'prod', axis, dtype, out, + keepdims=keepdims, initial=initial, where=where) def _cumprod_dispatcher(a, axis=None, dtype=None, out=None): @@ -2991,11 +3038,11 @@ def around(a, decimals=0, out=None): Examples -------- >>> np.around([0.37, 1.64]) - array([ 0., 2.]) + array([0., 2.]) >>> np.around([0.37, 1.64], decimals=1) - array([ 0.4, 1.6]) + array([0.4, 1.6]) >>> np.around([.5, 1.5, 2.5, 3.5, 4.5]) # rounds to nearest even value - array([ 0., 2., 2., 4., 4.]) + array([0., 2., 2., 4., 4.]) >>> np.around([1,2,3,11], decimals=1) # ndarray of ints is returned array([ 1, 2, 3, 11]) >>> np.around([1,2,3,11], decimals=-1) @@ -3083,9 +3130,9 @@ def mean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): >>> np.mean(a) 2.5 >>> np.mean(a, axis=0) - array([ 2., 3.]) + array([2., 3.]) >>> np.mean(a, axis=1) - array([ 1.5, 3.5]) + array([1.5, 3.5]) In single precision, `mean` can be inaccurate: @@ -3098,7 +3145,7 @@ def mean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): Computing the mean in float64 is more accurate: >>> np.mean(a, dtype=np.float64) - 0.55000000074505806 + 0.55000000074505806 # may vary """ kwargs = {} @@ -3204,11 +3251,11 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): -------- >>> a = np.array([[1, 2], [3, 4]]) >>> np.std(a) - 1.1180339887498949 + 1.1180339887498949 # may vary >>> np.std(a, axis=0) - array([ 1., 1.]) + array([1., 1.]) >>> np.std(a, axis=1) - array([ 0.5, 0.5]) + array([0.5, 0.5]) In single precision, std() can be inaccurate: @@ -3221,7 +3268,7 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): Computing the standard deviation in float64 is more accurate: >>> np.std(a, dtype=np.float64) - 0.44999999925494177 + 0.44999999925494177 # may vary """ kwargs = {} @@ -3328,9 +3375,9 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): >>> np.var(a) 1.25 >>> np.var(a, axis=0) - array([ 1., 1.]) + array([1., 1.]) >>> np.var(a, axis=1) - array([ 0.25, 0.25]) + array([0.25, 0.25]) In single precision, var() can be inaccurate: @@ -3343,7 +3390,7 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): Computing the variance in float64 is more accurate: >>> np.var(a, dtype=np.float64) - 0.20249999932944759 + 0.20249999932944759 # may vary >>> ((1-0.55)**2 + (0.1-0.55)**2)/2 0.2025 diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 799b1418d..f8800b83e 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -1,16 +1,22 @@ from __future__ import division, absolute_import, print_function +import functools import warnings import operator from . import numeric as _nx from .numeric import (result_type, NaN, shares_memory, MAY_SHARE_BOUNDS, - TooHardError,asanyarray) + TooHardError, asanyarray) from numpy.core.multiarray import add_docstring +from numpy.core import overrides __all__ = ['logspace', 'linspace', 'geomspace'] +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + def _index_deprecate(i, stacklevel=2): try: i = operator.index(i) @@ -23,7 +29,14 @@ def _index_deprecate(i, stacklevel=2): return i -def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): +def _linspace_dispatcher(start, stop, num=None, endpoint=None, retstep=None, + dtype=None, axis=None): + return (start, stop) + + +@array_function_dispatch(_linspace_dispatcher) +def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, + axis=0): """ Return evenly spaced numbers over a specified interval. @@ -32,11 +45,14 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): The endpoint of the interval can optionally be excluded. + .. versionchanged:: 1.16.0 + Non-scalar `start` and `stop` are now supported. + Parameters ---------- - start : scalar + start : array_like The starting value of the sequence. - stop : scalar + stop : array_like The end value of the sequence, unless `endpoint` is set to False. In that case, the sequence consists of all but the last of ``num + 1`` evenly spaced samples, so that `stop` is excluded. Note that the step @@ -55,6 +71,13 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): .. versionadded:: 1.9.0 + axis : int, optional + The axis in the result to store the samples. Relevant only if start + or stop are array-like. By default (0), the samples will be along a + new axis inserted at the beginning. Use -1 to get an axis at the end. + + .. versionadded:: 1.16.0 + Returns ------- samples : ndarray @@ -79,11 +102,11 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): Examples -------- >>> np.linspace(2.0, 3.0, num=5) - array([ 2. , 2.25, 2.5 , 2.75, 3. ]) + array([2. , 2.25, 2.5 , 2.75, 3. ]) >>> np.linspace(2.0, 3.0, num=5, endpoint=False) - array([ 2. , 2.2, 2.4, 2.6, 2.8]) + array([2. , 2.2, 2.4, 2.6, 2.8]) >>> np.linspace(2.0, 3.0, num=5, retstep=True) - (array([ 2. , 2.25, 2.5 , 2.75, 3. ]), 0.25) + (array([2. , 2.25, 2.5 , 2.75, 3. ]), 0.25) Graphical illustration: @@ -116,16 +139,15 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): if dtype is None: dtype = dt - y = _nx.arange(0, num, dtype=dt) - delta = stop - start + y = _nx.arange(0, num, dtype=dt).reshape((-1,) + (1,) * delta.ndim) # In-place multiplication y *= delta/div is faster, but prevents the multiplicant # from overriding what class is produced, and thus prevents, e.g. use of Quantities, # see gh-7142. Hence, we multiply in place only for standard scalar types. - _mult_inplace = _nx.isscalar(delta) + _mult_inplace = _nx.isscalar(delta) if num > 1: step = delta / div - if step == 0: + if _nx.any(step == 0): # Special handling for denormal numbers, gh-5437 y /= div if _mult_inplace: @@ -148,13 +170,23 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): if endpoint and num > 1: y[-1] = stop + if axis != 0: + y = _nx.moveaxis(y, 0, axis) + if retstep: return y.astype(dtype, copy=False), step else: return y.astype(dtype, copy=False) -def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): +def _logspace_dispatcher(start, stop, num=None, endpoint=None, base=None, + dtype=None, axis=None): + return (start, stop) + + +@array_function_dispatch(_logspace_dispatcher) +def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, + axis=0): """ Return numbers spaced evenly on a log scale. @@ -162,11 +194,14 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): (`base` to the power of `start`) and ends with ``base ** stop`` (see `endpoint` below). + .. versionchanged:: 1.16.0 + Non-scalar `start` and `stop` are now supported. + Parameters ---------- - start : float + start : array_like ``base ** start`` is the starting value of the sequence. - stop : float + stop : array_like ``base ** stop`` is the final value of the sequence, unless `endpoint` is False. In that case, ``num + 1`` values are spaced over the interval in log-space, of which all but the last (a sequence of @@ -183,6 +218,13 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): dtype : dtype The type of the output array. If `dtype` is not given, infer the data type from the other input arguments. + axis : int, optional + The axis in the result to store the samples. Relevant only if start + or stop are array-like. By default (0), the samples will be along a + new axis inserted at the beginning. Use -1 to get an axis at the end. + + .. versionadded:: 1.16.0 + Returns ------- @@ -210,11 +252,11 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): Examples -------- >>> np.logspace(2.0, 3.0, num=4) - array([ 100. , 215.443469 , 464.15888336, 1000. ]) + array([ 100. , 215.443469 , 464.15888336, 1000. ]) >>> np.logspace(2.0, 3.0, num=4, endpoint=False) - array([ 100. , 177.827941 , 316.22776602, 562.34132519]) + array([100. , 177.827941 , 316.22776602, 562.34132519]) >>> np.logspace(2.0, 3.0, num=4, base=2.0) - array([ 4. , 5.0396842 , 6.34960421, 8. ]) + array([4. , 5.0396842 , 6.34960421, 8. ]) Graphical illustration: @@ -232,24 +274,33 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): >>> plt.show() """ - y = linspace(start, stop, num=num, endpoint=endpoint) + y = linspace(start, stop, num=num, endpoint=endpoint, axis=axis) if dtype is None: return _nx.power(base, y) - return _nx.power(base, y).astype(dtype) + return _nx.power(base, y).astype(dtype, copy=False) + + +def _geomspace_dispatcher(start, stop, num=None, endpoint=None, dtype=None, + axis=None): + return (start, stop) -def geomspace(start, stop, num=50, endpoint=True, dtype=None): +@array_function_dispatch(_geomspace_dispatcher) +def geomspace(start, stop, num=50, endpoint=True, dtype=None, axis=0): """ Return numbers spaced evenly on a log scale (a geometric progression). This is similar to `logspace`, but with endpoints specified directly. Each output sample is a constant multiple of the previous. + .. versionchanged:: 1.16.0 + Non-scalar `start` and `stop` are now supported. + Parameters ---------- - start : scalar + start : array_like The starting value of the sequence. - stop : scalar + stop : array_like The final value of the sequence, unless `endpoint` is False. In that case, ``num + 1`` values are spaced over the interval in log-space, of which all but the last (a sequence of @@ -262,6 +313,12 @@ def geomspace(start, stop, num=50, endpoint=True, dtype=None): dtype : dtype The type of the output array. If `dtype` is not given, infer the data type from the other input arguments. + axis : int, optional + The axis in the result to store the samples. Relevant only if start + or stop are array-like. By default (0), the samples will be along a + new axis inserted at the beginning. Use -1 to get an axis at the end. + + .. versionadded:: 1.16.0 Returns ------- @@ -304,15 +361,15 @@ def geomspace(start, stop, num=50, endpoint=True, dtype=None): Negative, decreasing, and complex inputs are allowed: >>> np.geomspace(1000, 1, num=4) - array([ 1000., 100., 10., 1.]) + array([1000., 100., 10., 1.]) >>> np.geomspace(-1000, -1, num=4) array([-1000., -100., -10., -1.]) >>> np.geomspace(1j, 1000j, num=4) # Straight line - array([ 0. +1.j, 0. +10.j, 0. +100.j, 0.+1000.j]) + array([0. +1.j, 0. +10.j, 0. +100.j, 0.+1000.j]) >>> np.geomspace(-1+0j, 1+0j, num=5) # Circle - array([-1.00000000+0.j , -0.70710678+0.70710678j, - 0.00000000+1.j , 0.70710678+0.70710678j, - 1.00000000+0.j ]) + array([-1.00000000e+00+1.22464680e-16j, -7.07106781e-01+7.07106781e-01j, + 6.12323400e-17+1.00000000e+00j, 7.07106781e-01+7.07106781e-01j, + 1.00000000e+00+0.00000000e+00j]) Graphical illustration of ``endpoint`` parameter: @@ -320,46 +377,57 @@ def geomspace(start, stop, num=50, endpoint=True, dtype=None): >>> N = 10 >>> y = np.zeros(N) >>> plt.semilogx(np.geomspace(1, 1000, N, endpoint=True), y + 1, 'o') + [<matplotlib.lines.Line2D object at 0x...>] >>> plt.semilogx(np.geomspace(1, 1000, N, endpoint=False), y + 2, 'o') + [<matplotlib.lines.Line2D object at 0x...>] >>> plt.axis([0.5, 2000, 0, 3]) + [0.5, 2000, 0, 3] >>> plt.grid(True, color='0.7', linestyle='-', which='both', axis='both') >>> plt.show() """ - if start == 0 or stop == 0: + start = asanyarray(start) + stop = asanyarray(stop) + if _nx.any(start == 0) or _nx.any(stop == 0): raise ValueError('Geometric sequence cannot include zero') - dt = result_type(start, stop, float(num)) + dt = result_type(start, stop, float(num), _nx.zeros((), dtype)) if dtype is None: dtype = dt else: # complex to dtype('complex128'), for instance dtype = _nx.dtype(dtype) + # Promote both arguments to the same dtype in case, for instance, one is + # complex and another is negative and log would produce NaN otherwise. + # Copy since we may change things in-place further down. + start = start.astype(dt, copy=True) + stop = stop.astype(dt, copy=True) + + out_sign = _nx.ones(_nx.broadcast(start, stop).shape, dt) # Avoid negligible real or imaginary parts in output by rotating to # positive real, calculating, then undoing rotation - out_sign = 1 - if start.real == stop.real == 0: - start, stop = start.imag, stop.imag - out_sign = 1j * out_sign - if _nx.sign(start) == _nx.sign(stop) == -1: - start, stop = -start, -stop - out_sign = -out_sign - - # Promote both arguments to the same dtype in case, for instance, one is - # complex and another is negative and log would produce NaN otherwise - start = start + (stop - stop) - stop = stop + (start - start) - if _nx.issubdtype(dtype, _nx.complexfloating): - start = start + 0j - stop = stop + 0j + if _nx.issubdtype(dt, _nx.complexfloating): + all_imag = (start.real == 0.) & (stop.real == 0.) + if _nx.any(all_imag): + start[all_imag] = start[all_imag].imag + stop[all_imag] = stop[all_imag].imag + out_sign[all_imag] = 1j + + both_negative = (_nx.sign(start) == -1) & (_nx.sign(stop) == -1) + if _nx.any(both_negative): + _nx.negative(start, out=start, where=both_negative) + _nx.negative(stop, out=stop, where=both_negative) + _nx.negative(out_sign, out=out_sign, where=both_negative) log_start = _nx.log10(start) log_stop = _nx.log10(stop) result = out_sign * logspace(log_start, log_stop, num=num, endpoint=endpoint, base=10.0, dtype=dtype) + if axis != 0: + result = _nx.moveaxis(result, 0, axis) - return result.astype(dtype) + return result.astype(dtype, copy=False) #always succeed diff --git a/numpy/core/getlimits.py b/numpy/core/getlimits.py index 0e3c58793..544b8b35f 100644 --- a/numpy/core/getlimits.py +++ b/numpy/core/getlimits.py @@ -8,6 +8,7 @@ __all__ = ['finfo', 'iinfo'] import warnings from .machar import MachAr +from .overrides import set_module from . import numeric from . import numerictypes as ntypes from .numeric import array, inf @@ -30,6 +31,32 @@ def _fr1(a): a.shape = () return a +class MachArLike(object): + """ Object to simulate MachAr instance """ + + def __init__(self, + ftype, + **kwargs): + params = _MACHAR_PARAMS[ftype] + float_conv = lambda v: array([v], ftype) + float_to_float = lambda v : _fr1(float_conv(v)) + float_to_str = lambda v: (params['fmt'] % array(_fr0(v)[0], ftype)) + + self.title = params['title'] + # Parameter types same as for discovered MachAr object. + self.epsilon = self.eps = float_to_float(kwargs.pop('eps')) + self.epsneg = float_to_float(kwargs.pop('epsneg')) + self.xmax = self.huge = float_to_float(kwargs.pop('huge')) + self.xmin = self.tiny = float_to_float(kwargs.pop('tiny')) + self.ibeta = params['itype'](kwargs.pop('ibeta')) + self.__dict__.update(kwargs) + self.precision = int(-log10(self.eps)) + self.resolution = float_to_float(float_conv(10) ** (-self.precision)) + self._str_eps = float_to_str(self.eps) + self._str_epsneg = float_to_str(self.epsneg) + self._str_xmin = float_to_str(self.xmin) + self._str_xmax = float_to_str(self.xmax) + self._str_resolution = float_to_str(self.resolution) _convert_to_float = { ntypes.csingle: ntypes.single, @@ -37,7 +64,6 @@ _convert_to_float = { ntypes.clongfloat: ntypes.longfloat } - # Parameters for creating MachAr / MachAr-like objects _title_fmt = 'numpy {} precision floating point number' _MACHAR_PARAMS = { @@ -58,194 +84,156 @@ _MACHAR_PARAMS = { fmt = '%12.5e', title = _title_fmt.format('half'))} - -class MachArLike(object): - """ Object to simulate MachAr instance """ - - def __init__(self, - ftype, - **kwargs): - params = _MACHAR_PARAMS[ftype] - float_conv = lambda v: array([v], ftype) - float_to_float = lambda v : _fr1(float_conv(v)) - self._float_to_str = lambda v: (params['fmt'] % - array(_fr0(v)[0], ftype)) - self.title = params['title'] - # Parameter types same as for discovered MachAr object. - self.epsilon = self.eps = float_to_float(kwargs.pop('eps')) - self.epsneg = float_to_float(kwargs.pop('epsneg')) - self.xmax = self.huge = float_to_float(kwargs.pop('huge')) - self.xmin = self.tiny = float_to_float(kwargs.pop('tiny')) - self.ibeta = params['itype'](kwargs.pop('ibeta')) - self.__dict__.update(kwargs) - self.precision = int(-log10(self.eps)) - self.resolution = float_to_float(float_conv(10) ** (-self.precision)) - - # Properties below to delay need for float_to_str, and thus avoid circular - # imports during early numpy module loading. - # See: https://github.com/numpy/numpy/pull/8983#discussion_r115838683 - - @property - def _str_eps(self): - return self._float_to_str(self.eps) - - @property - def _str_epsneg(self): - return self._float_to_str(self.epsneg) - - @property - def _str_xmin(self): - return self._float_to_str(self.xmin) - - @property - def _str_xmax(self): - return self._float_to_str(self.xmax) - - @property - def _str_resolution(self): - return self._float_to_str(self.resolution) - - -# Known parameters for float16 -# See docstring of MachAr class for description of parameters. -_f16 = ntypes.float16 -_float16_ma = MachArLike(_f16, - machep=-10, - negep=-11, - minexp=-14, - maxexp=16, - it=10, - iexp=5, - ibeta=2, - irnd=5, - ngrd=0, - eps=exp2(_f16(-10)), - epsneg=exp2(_f16(-11)), - huge=_f16(65504), - tiny=_f16(2 ** -14)) - -# Known parameters for float32 -_f32 = ntypes.float32 -_float32_ma = MachArLike(_f32, - machep=-23, - negep=-24, - minexp=-126, - maxexp=128, - it=23, - iexp=8, - ibeta=2, - irnd=5, - ngrd=0, - eps=exp2(_f32(-23)), - epsneg=exp2(_f32(-24)), - huge=_f32((1 - 2 ** -24) * 2**128), - tiny=exp2(_f32(-126))) - -# Known parameters for float64 -_f64 = ntypes.float64 -_epsneg_f64 = 2.0 ** -53.0 -_tiny_f64 = 2.0 ** -1022.0 -_float64_ma = MachArLike(_f64, - machep=-52, - negep=-53, - minexp=-1022, - maxexp=1024, - it=52, - iexp=11, - ibeta=2, - irnd=5, - ngrd=0, - eps=2.0 ** -52.0, - epsneg=_epsneg_f64, - huge=(1.0 - _epsneg_f64) / _tiny_f64 * _f64(4), - tiny=_tiny_f64) - -# Known parameters for IEEE 754 128-bit binary float -_ld = ntypes.longdouble -_epsneg_f128 = exp2(_ld(-113)) -_tiny_f128 = exp2(_ld(-16382)) -# Ignore runtime error when this is not f128 -with numeric.errstate(all='ignore'): - _huge_f128 = (_ld(1) - _epsneg_f128) / _tiny_f128 * _ld(4) -_float128_ma = MachArLike(_ld, - machep=-112, - negep=-113, - minexp=-16382, - maxexp=16384, - it=112, - iexp=15, - ibeta=2, - irnd=5, - ngrd=0, - eps=exp2(_ld(-112)), - epsneg=_epsneg_f128, - huge=_huge_f128, - tiny=_tiny_f128) - -# Known parameters for float80 (Intel 80-bit extended precision) -_epsneg_f80 = exp2(_ld(-64)) -_tiny_f80 = exp2(_ld(-16382)) -# Ignore runtime error when this is not f80 -with numeric.errstate(all='ignore'): - _huge_f80 = (_ld(1) - _epsneg_f80) / _tiny_f80 * _ld(4) -_float80_ma = MachArLike(_ld, - machep=-63, - negep=-64, - minexp=-16382, - maxexp=16384, - it=63, - iexp=15, - ibeta=2, - irnd=5, - ngrd=0, - eps=exp2(_ld(-63)), - epsneg=_epsneg_f80, - huge=_huge_f80, - tiny=_tiny_f80) - -# Guessed / known parameters for double double; see: -# https://en.wikipedia.org/wiki/Quadruple-precision_floating-point_format#Double-double_arithmetic -# These numbers have the same exponent range as float64, but extended number of -# digits in the significand. -_huge_dd = (umath.nextafter(_ld(inf), _ld(0)) - if hasattr(umath, 'nextafter') # Missing on some platforms? - else _float64_ma.huge) -_float_dd_ma = MachArLike(_ld, - machep=-105, - negep=-106, - minexp=-1022, - maxexp=1024, - it=105, - iexp=11, - ibeta=2, - irnd=5, - ngrd=0, - eps=exp2(_ld(-105)), - epsneg= exp2(_ld(-106)), - huge=_huge_dd, - tiny=exp2(_ld(-1022))) - - # Key to identify the floating point type. Key is result of # ftype('-0.1').newbyteorder('<').tobytes() # See: # https://perl5.git.perl.org/perl.git/blob/3118d7d684b56cbeb702af874f4326683c45f045:/Configure -_KNOWN_TYPES = { - b'\x9a\x99\x99\x99\x99\x99\xb9\xbf' : _float64_ma, - b'\xcd\xcc\xcc\xbd' : _float32_ma, - b'f\xae' : _float16_ma, +_KNOWN_TYPES = {} +def _register_type(machar, bytepat): + _KNOWN_TYPES[bytepat] = machar +_float_ma = {} + +def _register_known_types(): + # Known parameters for float16 + # See docstring of MachAr class for description of parameters. + f16 = ntypes.float16 + float16_ma = MachArLike(f16, + machep=-10, + negep=-11, + minexp=-14, + maxexp=16, + it=10, + iexp=5, + ibeta=2, + irnd=5, + ngrd=0, + eps=exp2(f16(-10)), + epsneg=exp2(f16(-11)), + huge=f16(65504), + tiny=f16(2 ** -14)) + _register_type(float16_ma, b'f\xae') + _float_ma[16] = float16_ma + + # Known parameters for float32 + f32 = ntypes.float32 + float32_ma = MachArLike(f32, + machep=-23, + negep=-24, + minexp=-126, + maxexp=128, + it=23, + iexp=8, + ibeta=2, + irnd=5, + ngrd=0, + eps=exp2(f32(-23)), + epsneg=exp2(f32(-24)), + huge=f32((1 - 2 ** -24) * 2**128), + tiny=exp2(f32(-126))) + _register_type(float32_ma, b'\xcd\xcc\xcc\xbd') + _float_ma[32] = float32_ma + + # Known parameters for float64 + f64 = ntypes.float64 + epsneg_f64 = 2.0 ** -53.0 + tiny_f64 = 2.0 ** -1022.0 + float64_ma = MachArLike(f64, + machep=-52, + negep=-53, + minexp=-1022, + maxexp=1024, + it=52, + iexp=11, + ibeta=2, + irnd=5, + ngrd=0, + eps=2.0 ** -52.0, + epsneg=epsneg_f64, + huge=(1.0 - epsneg_f64) / tiny_f64 * f64(4), + tiny=tiny_f64) + _register_type(float64_ma, b'\x9a\x99\x99\x99\x99\x99\xb9\xbf') + _float_ma[64] = float64_ma + + # Known parameters for IEEE 754 128-bit binary float + ld = ntypes.longdouble + epsneg_f128 = exp2(ld(-113)) + tiny_f128 = exp2(ld(-16382)) + # Ignore runtime error when this is not f128 + with numeric.errstate(all='ignore'): + huge_f128 = (ld(1) - epsneg_f128) / tiny_f128 * ld(4) + float128_ma = MachArLike(ld, + machep=-112, + negep=-113, + minexp=-16382, + maxexp=16384, + it=112, + iexp=15, + ibeta=2, + irnd=5, + ngrd=0, + eps=exp2(ld(-112)), + epsneg=epsneg_f128, + huge=huge_f128, + tiny=tiny_f128) + # IEEE 754 128-bit binary float + _register_type(float128_ma, + b'\x9a\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\xfb\xbf') + _register_type(float128_ma, + b'\x9a\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\xfb\xbf') + _float_ma[128] = float128_ma + + # Known parameters for float80 (Intel 80-bit extended precision) + epsneg_f80 = exp2(ld(-64)) + tiny_f80 = exp2(ld(-16382)) + # Ignore runtime error when this is not f80 + with numeric.errstate(all='ignore'): + huge_f80 = (ld(1) - epsneg_f80) / tiny_f80 * ld(4) + float80_ma = MachArLike(ld, + machep=-63, + negep=-64, + minexp=-16382, + maxexp=16384, + it=63, + iexp=15, + ibeta=2, + irnd=5, + ngrd=0, + eps=exp2(ld(-63)), + epsneg=epsneg_f80, + huge=huge_f80, + tiny=tiny_f80) # float80, first 10 bytes containing actual storage - b'\xcd\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xfb\xbf' : _float80_ma, + _register_type(float80_ma, b'\xcd\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xfb\xbf') + _float_ma[80] = float80_ma + + # Guessed / known parameters for double double; see: + # https://en.wikipedia.org/wiki/Quadruple-precision_floating-point_format#Double-double_arithmetic + # These numbers have the same exponent range as float64, but extended number of + # digits in the significand. + huge_dd = (umath.nextafter(ld(inf), ld(0)) + if hasattr(umath, 'nextafter') # Missing on some platforms? + else float64_ma.huge) + float_dd_ma = MachArLike(ld, + machep=-105, + negep=-106, + minexp=-1022, + maxexp=1024, + it=105, + iexp=11, + ibeta=2, + irnd=5, + ngrd=0, + eps=exp2(ld(-105)), + epsneg= exp2(ld(-106)), + huge=huge_dd, + tiny=exp2(ld(-1022))) # double double; low, high order (e.g. PPC 64) - b'\x9a\x99\x99\x99\x99\x99Y<\x9a\x99\x99\x99\x99\x99\xb9\xbf' : - _float_dd_ma, + _register_type(float_dd_ma, + b'\x9a\x99\x99\x99\x99\x99Y<\x9a\x99\x99\x99\x99\x99\xb9\xbf') # double double; high, low order (e.g. PPC 64 le) - b'\x9a\x99\x99\x99\x99\x99\xb9\xbf\x9a\x99\x99\x99\x99\x99Y<' : - _float_dd_ma, - # IEEE 754 128-bit binary float - b'\x9a\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\x99\xfb\xbf' : - _float128_ma, -} + _register_type(float_dd_ma, + b'\x9a\x99\x99\x99\x99\x99\xb9\xbf\x9a\x99\x99\x99\x99\x99Y<') + _float_ma['dd'] = float_dd_ma def _get_machar(ftype): @@ -302,6 +290,7 @@ def _discovered_machar(ftype): params['title']) +@set_module('numpy') class finfo(object): """ finfo(dtype) @@ -452,6 +441,7 @@ class finfo(object): " max=%(_str_max)s, dtype=%(dtype)s)") % d) +@set_module('numpy') class iinfo(object): """ iinfo(type) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index da006909a..b0b749c80 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -505,7 +505,8 @@ typedef struct { PyArray_NonzeroFunc *nonzero; /* - * Used for arange. + * Used for arange. Should return 0 on success + * and -1 on failure. * Can be NULL. */ PyArray_FillFunc *fill; diff --git a/numpy/core/include/numpy/npy_1_7_deprecated_api.h b/numpy/core/include/numpy/npy_1_7_deprecated_api.h index 76b57b748..a6ee21219 100644 --- a/numpy/core/include/numpy/npy_1_7_deprecated_api.h +++ b/numpy/core/include/numpy/npy_1_7_deprecated_api.h @@ -5,6 +5,8 @@ #error "Should never include npy_*_*_deprecated_api directly." #endif +/* Emit a warning if the user did not specifically request the old API */ +#ifndef NPY_NO_DEPRECATED_API #if defined(_WIN32) #define _WARN___STR2__(x) #x #define _WARN___STR1__(x) _WARN___STR2__(x) @@ -16,6 +18,7 @@ "#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION" #endif /* TODO: How to do this warning message for other compilers? */ +#endif /* * This header exists to collect all dangerous/deprecated NumPy API diff --git a/numpy/core/include/numpy/npy_3kcompat.h b/numpy/core/include/numpy/npy_3kcompat.h index a3c69f44e..832bc0599 100644 --- a/numpy/core/include/numpy/npy_3kcompat.h +++ b/numpy/core/include/numpy/npy_3kcompat.h @@ -219,6 +219,7 @@ npy_PyFile_Dup2(PyObject *file, char *mode, npy_off_t *orig_pos) if (handle == NULL) { PyErr_SetString(PyExc_IOError, "Getting a FILE* from a Python file object failed"); + return NULL; } /* Record the original raw file handle position */ @@ -383,6 +384,36 @@ npy_PyFile_CloseFile(PyObject *file) } +/* This is a copy of _PyErr_ChainExceptions + */ +static NPY_INLINE void +npy_PyErr_ChainExceptions(PyObject *exc, PyObject *val, PyObject *tb) +{ + if (exc == NULL) + return; + + if (PyErr_Occurred()) { + /* only py3 supports this anyway */ + #ifdef NPY_PY3K + PyObject *exc2, *val2, *tb2; + PyErr_Fetch(&exc2, &val2, &tb2); + PyErr_NormalizeException(&exc, &val, &tb); + if (tb != NULL) { + PyException_SetTraceback(val, tb); + Py_DECREF(tb); + } + Py_DECREF(exc); + PyErr_NormalizeException(&exc2, &val2, &tb2); + PyException_SetContext(val2, val); + PyErr_Restore(exc2, val2, tb2); + #endif + } + else { + PyErr_Restore(exc, val, tb); + } +} + + /* This is a copy of _PyErr_ChainExceptions, with: * - a minimal implementation for python 2 * - __cause__ used instead of __context__ diff --git a/numpy/core/include/numpy/ufuncobject.h b/numpy/core/include/numpy/ufuncobject.h index 85f8a6c08..90d837a9b 100644 --- a/numpy/core/include/numpy/ufuncobject.h +++ b/numpy/core/include/numpy/ufuncobject.h @@ -223,7 +223,8 @@ typedef struct _tagPyUFuncObject { */ npy_uint32 *core_dim_flags; - + /* Identity for reduction, when identity == PyUFunc_IdentityValue */ + PyObject *identity_value; } PyUFuncObject; @@ -299,6 +300,12 @@ typedef struct _tagPyUFuncObject { * This case allows reduction with multiple axes at once. */ #define PyUFunc_ReorderableNone -2 +/* + * UFunc unit is in identity_value, and the order of operations can be reordered + * This case allows reduction with multiple axes at once. + */ +#define PyUFunc_IdentityValue -3 + #define UFUNC_REDUCE 0 #define UFUNC_ACCUMULATE 1 diff --git a/numpy/core/machar.py b/numpy/core/machar.py index 7578544fe..91fb4eda8 100644 --- a/numpy/core/machar.py +++ b/numpy/core/machar.py @@ -11,9 +11,11 @@ __all__ = ['MachAr'] from numpy.core.fromnumeric import any from numpy.core.numeric import errstate +from numpy.core.overrides import set_module # Need to speed this up...especially for longfloat +@set_module('numpy') class MachAr(object): """ Diagnosing machine parameters. diff --git a/numpy/core/memmap.py b/numpy/core/memmap.py index 8269f537f..9ba4817f4 100644 --- a/numpy/core/memmap.py +++ b/numpy/core/memmap.py @@ -3,8 +3,9 @@ from __future__ import division, absolute_import, print_function import numpy as np from .numeric import uint8, ndarray, dtype from numpy.compat import ( - long, basestring, is_pathlib_path, contextlib_nullcontext + long, basestring, os_fspath, contextlib_nullcontext, is_pathlib_path ) +from numpy.core.overrides import set_module __all__ = ['memmap'] @@ -19,6 +20,8 @@ mode_equivalents = { "write":"w+" } + +@set_module('numpy') class memmap(ndarray): """Create a memory-map to an array stored in a *binary* file on disk. @@ -132,9 +135,9 @@ class memmap(ndarray): >>> fp = np.memmap(filename, dtype='float32', mode='w+', shape=(3,4)) >>> fp - memmap([[ 0., 0., 0., 0.], - [ 0., 0., 0., 0.], - [ 0., 0., 0., 0.]], dtype=float32) + memmap([[0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.]], dtype=float32) Write data to memmap array: @@ -218,10 +221,8 @@ class memmap(ndarray): if hasattr(filename, 'read'): f_ctx = contextlib_nullcontext(filename) - elif is_pathlib_path(filename): - f_ctx = filename.open(('r' if mode == 'c' else mode)+'b') else: - f_ctx = open(filename, ('r' if mode == 'c' else mode)+'b') + f_ctx = open(os_fspath(filename), ('r' if mode == 'c' else mode)+'b') with f_ctx as fid: fid.seek(0, 2) @@ -268,14 +269,13 @@ class memmap(ndarray): self.offset = offset self.mode = mode - if isinstance(filename, basestring): - self.filename = os.path.abspath(filename) - elif is_pathlib_path(filename): + if is_pathlib_path(filename): + # special case - if we were constructed with a pathlib.path, + # then filename is a path object, not a string self.filename = filename.resolve() - # py3 returns int for TemporaryFile().name - elif (hasattr(filename, "name") and - isinstance(filename.name, basestring)): - self.filename = os.path.abspath(filename.name) + elif hasattr(fid, "name") and isinstance(fid.name, basestring): + # py3 returns int for TemporaryFile().name + self.filename = os.path.abspath(fid.name) # same as memmap copies (e.g. memmap + 1) else: self.filename = None diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 4dbd3b0fd..4c2715892 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -6,8 +6,11 @@ by importing from the extension module. """ +import functools +import warnings + +from . import overrides from . import _multiarray_umath -from .overrides import array_function_dispatch import numpy as np from numpy.core._multiarray_umath import * from numpy.core._multiarray_umath import ( @@ -38,13 +41,37 @@ __all__ = [ 'where', 'zeros'] -def _empty_like_dispatcher(prototype, dtype=None, order=None, subok=None): - return (prototype,) - +arange.__module__ = 'numpy' +array.__module__ = 'numpy' +datetime_data.__module__ = 'numpy' +empty.__module__ = 'numpy' +frombuffer.__module__ = 'numpy' +fromfile.__module__ = 'numpy' +fromiter.__module__ = 'numpy' +frompyfunc.__module__ = 'numpy' +fromstring.__module__ = 'numpy' +geterrobj.__module__ = 'numpy' +may_share_memory.__module__ = 'numpy' +nested_iters.__module__ = 'numpy' +promote_types.__module__ = 'numpy' +set_numeric_ops.__module__ = 'numpy' +seterrobj.__module__ = 'numpy' +zeros.__module__ = 'numpy' + + +# We can't verify dispatcher signatures because NumPy's C functions don't +# support introspection. +array_function_from_c_func_and_dispatcher = functools.partial( + overrides.array_function_from_dispatcher, + module='numpy', docs_from_dispatcher=True, verify=False) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.empty_like) +def empty_like(prototype, dtype=None, order=None, subok=None): + """ + empty_like(prototype, dtype=None, order='K', subok=True) -@array_function_dispatch(_empty_like_dispatcher) -def empty_like(prototype, dtype=None, order='K', subok=True): - """Return a new array with the same shape and type as a given array. + Return a new array with the same shape and type as a given array. Parameters ---------- @@ -90,25 +117,19 @@ def empty_like(prototype, dtype=None, order='K', subok=True): -------- >>> a = ([1,2,3], [4,5,6]) # a is array-like >>> np.empty_like(a) - array([[-1073741821, -1073741821, 3], #random + array([[-1073741821, -1073741821, 3], # random [ 0, 0, -1073741821]]) >>> a = np.array([[1., 2., 3.],[4.,5.,6.]]) >>> np.empty_like(a) - array([[ -2.00000715e+000, 1.48219694e-323, -2.00000572e+000],#random + array([[ -2.00000715e+000, 1.48219694e-323, -2.00000572e+000], # random [ 4.38791518e-305, -2.00000715e+000, 4.17269252e-309]]) """ - return _multiarray_umath.empty_like(prototype, dtype, order, subok) - - -def _concatenate_dispatcher(arrays, axis=None, out=None): - for array in arrays: - yield array - yield out + return (prototype,) -@array_function_dispatch(_concatenate_dispatcher) -def concatenate(arrays, axis=0, out=None): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.concatenate) +def concatenate(arrays, axis=None, out=None): """ concatenate((a1, a2, ...), axis=0, out=None) @@ -190,16 +211,18 @@ def concatenate(arrays, axis=0, out=None): fill_value=999999) """ - return _multiarray_umath.concatenate(arrays, axis, out) - - -def _inner_dispatcher(a, b): - return (a, b) + if out is not None: + # optimize for the typical case where only arrays is provided + arrays = list(arrays) + arrays.append(out) + return arrays -@array_function_dispatch(_inner_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.inner) def inner(a, b): """ + inner(a, b) + Inner product of two arrays. Ordinary inner product of vectors for 1-D arrays (without complex @@ -265,19 +288,15 @@ def inner(a, b): An example where `b` is a scalar: >>> np.inner(np.eye(2), 7) - array([[ 7., 0.], - [ 0., 7.]]) + array([[7., 0.], + [0., 7.]]) """ - return _multiarray_umath.inner(a, b) - - -def _where_dispatcher(condition, x=None, y=None): - return (condition, x, y) + return (a, b) -@array_function_dispatch(_where_dispatcher) -def where(condition, x=np._NoValue, y=np._NoValue): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.where) +def where(condition, x=None, y=None): """ where(condition, [x, y]) @@ -348,21 +367,14 @@ def where(condition, x=np._NoValue, y=np._NoValue): [ 0, 2, -1], [ 0, 3, -1]]) """ - # _multiarray_umath.where only accepts positional arguments - args = tuple(a for a in (x, y) if a is not np._NoValue) - return _multiarray_umath.where(condition, *args) - - -def _lexsort_dispatcher(keys, axis=None): - if isinstance(keys, tuple): - return keys - else: - return (keys,) + return (condition, x, y) -@array_function_dispatch(_lexsort_dispatcher) -def lexsort(keys, axis=-1): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.lexsort) +def lexsort(keys, axis=None): """ + lexsort(keys, axis=-1) + Perform an indirect stable sort using a sequence of keys. Given multiple sorting keys, which can be interpreted as columns in a @@ -411,8 +423,8 @@ def lexsort(keys, axis=-1): >>> a = [1,5,1,4,3,4,4] # First column >>> b = [9,4,0,4,0,2,1] # Second column >>> ind = np.lexsort((b,a)) # Sort by a, then by b - >>> print(ind) - [2 0 4 6 5 3 1] + >>> ind + array([2, 0, 4, 6, 5, 3, 1]) >>> [(a[i],b[i]) for i in ind] [(1, 0), (1, 9), (3, 0), (4, 1), (4, 2), (4, 4), (5, 4)] @@ -434,16 +446,17 @@ def lexsort(keys, axis=-1): array([2, 0, 4, 6, 5, 3, 1]) """ - return _multiarray_umath.lexsort(keys, axis) - - -def _can_cast_dispatcher(from_, to, casting=None): - return (from_,) + if isinstance(keys, tuple): + return keys + else: + return (keys,) -@array_function_dispatch(_can_cast_dispatcher) -def can_cast(from_, to, casting='safe'): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.can_cast) +def can_cast(from_, to, casting=None): """ + can_cast(from_, to, casting='safe') + Returns True if cast between data types can occur according to the casting rule. If from is a scalar or array scalar, also returns True if the scalar value can be cast without overflow or truncation @@ -547,16 +560,14 @@ def can_cast(from_, to, casting='safe'): True """ - return _multiarray_umath.can_cast(from_, to, casting) - - -def _min_scalar_type_dispatcher(a): - return (a,) + return (from_,) -@array_function_dispatch(_min_scalar_type_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.min_scalar_type) def min_scalar_type(a): """ + min_scalar_type(a) + For scalar ``a``, returns the data type with the smallest size and smallest scalar kind which can hold its value. For non-scalar array ``a``, returns the vector's dtype unmodified. @@ -600,16 +611,14 @@ def min_scalar_type(a): dtype('float64') """ - return _multiarray_umath.min_scalar_type(a) - - -def _result_type_dispatcher(*arrays_and_dtypes): - return arrays_and_dtypes + return (a,) -@array_function_dispatch(_result_type_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.result_type) def result_type(*arrays_and_dtypes): """ + result_type(*arrays_and_dtypes) + Returns the type that results from applying the NumPy type promotion rules to the arguments. @@ -674,16 +683,14 @@ def result_type(*arrays_and_dtypes): dtype('float64') """ - return _multiarray_umath.result_type(*arrays_and_dtypes) - - -def _dot_dispatcher(a, b, out=None): - return (a, b, out) + return arrays_and_dtypes -@array_function_dispatch(_dot_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.dot) def dot(a, b, out=None): """ + dot(a, b, out=None) + Dot product of two arrays. Specifically, - If both `a` and `b` are 1-D arrays, it is inner product of vectors @@ -764,16 +771,14 @@ def dot(a, b, out=None): 499128 """ - return _multiarray_umath.dot(a, b, out) - - -def _vdot_dispatcher(a, b): - return (a, b) + return (a, b, out) -@array_function_dispatch(_vdot_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.vdot) def vdot(a, b): """ + vdot(a, b) + Return the dot product of two vectors. The vdot(`a`, `b`) function handles complex numbers differently than @@ -824,17 +829,464 @@ def vdot(a, b): 30 """ - return _multiarray_umath.vdot(a, b) + return (a, b) -def _is_busday_dispatcher( - dates, weekmask=None, holidays=None, busdaycal=None, out=None): - return (dates, weekmask, holidays, out) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.bincount) +def bincount(x, weights=None, minlength=None): + """ + bincount(x, weights=None, minlength=0) + + Count number of occurrences of each value in array of non-negative ints. + + The number of bins (of size 1) is one larger than the largest value in + `x`. If `minlength` is specified, there will be at least this number + of bins in the output array (though it will be longer if necessary, + depending on the contents of `x`). + Each bin gives the number of occurrences of its index value in `x`. + If `weights` is specified the input array is weighted by it, i.e. if a + value ``n`` is found at position ``i``, ``out[n] += weight[i]`` instead + of ``out[n] += 1``. + + Parameters + ---------- + x : array_like, 1 dimension, nonnegative ints + Input array. + weights : array_like, optional + Weights, array of the same shape as `x`. + minlength : int, optional + A minimum number of bins for the output array. + + .. versionadded:: 1.6.0 + + Returns + ------- + out : ndarray of ints + The result of binning the input array. + The length of `out` is equal to ``np.amax(x)+1``. + + Raises + ------ + ValueError + If the input is not 1-dimensional, or contains elements with negative + values, or if `minlength` is negative. + TypeError + If the type of the input is float or complex. + + See Also + -------- + histogram, digitize, unique + + Examples + -------- + >>> np.bincount(np.arange(5)) + array([1, 1, 1, 1, 1]) + >>> np.bincount(np.array([0, 1, 1, 3, 2, 1, 7])) + array([1, 3, 1, 1, 0, 0, 0, 1]) + + >>> x = np.array([0, 1, 1, 3, 2, 1, 7, 23]) + >>> np.bincount(x).size == np.amax(x)+1 + True + + The input array needs to be of integer dtype, otherwise a + TypeError is raised: + + >>> np.bincount(np.arange(5, dtype=float)) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + TypeError: array cannot be safely cast to required type + + A possible use of ``bincount`` is to perform sums over + variable-size chunks of an array, using the ``weights`` keyword. + + >>> w = np.array([0.3, 0.5, 0.2, 0.7, 1., -0.6]) # weights + >>> x = np.array([0, 1, 1, 2, 2, 2]) + >>> np.bincount(x, weights=w) + array([ 0.3, 0.7, 1.1]) + + """ + return (x, weights) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.ravel_multi_index) +def ravel_multi_index(multi_index, dims, mode=None, order=None): + """ + ravel_multi_index(multi_index, dims, mode='raise', order='C') + + Converts a tuple of index arrays into an array of flat + indices, applying boundary modes to the multi-index. + + Parameters + ---------- + multi_index : tuple of array_like + A tuple of integer arrays, one array for each dimension. + dims : tuple of ints + The shape of array into which the indices from ``multi_index`` apply. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices are handled. Can specify + either one mode or a tuple of modes, one mode per index. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + In 'clip' mode, a negative index which would normally + wrap will clip to 0 instead. + order : {'C', 'F'}, optional + Determines whether the multi-index should be viewed as + indexing in row-major (C-style) or column-major + (Fortran-style) order. + + Returns + ------- + raveled_indices : ndarray + An array of indices into the flattened version of an array + of dimensions ``dims``. + + See Also + -------- + unravel_index + + Notes + ----- + .. versionadded:: 1.6.0 + + Examples + -------- + >>> arr = np.array([[3,6,6],[4,5,1]]) + >>> np.ravel_multi_index(arr, (7,6)) + array([22, 41, 37]) + >>> np.ravel_multi_index(arr, (7,6), order='F') + array([31, 41, 13]) + >>> np.ravel_multi_index(arr, (4,6), mode='clip') + array([22, 23, 19]) + >>> np.ravel_multi_index(arr, (4,4), mode=('clip','wrap')) + array([12, 13, 13]) + + >>> np.ravel_multi_index((3,1,4,1), (6,7,8,9)) + 1621 + """ + return multi_index + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.unravel_index) +def unravel_index(indices, shape=None, order=None, dims=None): + """ + unravel_index(indices, shape, order='C') + + Converts a flat index or array of flat indices into a tuple + of coordinate arrays. + + Parameters + ---------- + indices : array_like + An integer array whose elements are indices into the flattened + version of an array of dimensions ``shape``. Before version 1.6.0, + this function accepted just one index value. + shape : tuple of ints + The shape of the array to use for unraveling ``indices``. + + .. versionchanged:: 1.16.0 + Renamed from ``dims`` to ``shape``. + + order : {'C', 'F'}, optional + Determines whether the indices should be viewed as indexing in + row-major (C-style) or column-major (Fortran-style) order. + + .. versionadded:: 1.6.0 + + Returns + ------- + unraveled_coords : tuple of ndarray + Each array in the tuple has the same shape as the ``indices`` + array. + + See Also + -------- + ravel_multi_index + + Examples + -------- + >>> np.unravel_index([22, 41, 37], (7,6)) + (array([3, 6, 6]), array([4, 5, 1])) + >>> np.unravel_index([31, 41, 13], (7,6), order='F') + (array([3, 6, 6]), array([4, 5, 1])) + + >>> np.unravel_index(1621, (6,7,8,9)) + (3, 1, 4, 1) + + """ + if dims is not None: + warnings.warn("'shape' argument should be used instead of 'dims'", + DeprecationWarning, stacklevel=3) + return (indices,) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.copyto) +def copyto(dst, src, casting=None, where=None): + """ + copyto(dst, src, casting='same_kind', where=True) + + Copies values from one array to another, broadcasting as necessary. + + Raises a TypeError if the `casting` rule is violated, and if + `where` is provided, it selects which elements to copy. + + .. versionadded:: 1.7.0 + + Parameters + ---------- + dst : ndarray + The array into which values are copied. + src : array_like + The array from which values are copied. + casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional + Controls what kind of data casting may occur when copying. + + * 'no' means the data types should not be cast at all. + * 'equiv' means only byte-order changes are allowed. + * 'safe' means only casts which can preserve values are allowed. + * 'same_kind' means only safe casts or casts within a kind, + like float64 to float32, are allowed. + * 'unsafe' means any data conversions may be done. + where : array_like of bool, optional + A boolean array which is broadcasted to match the dimensions + of `dst`, and selects elements to copy from `src` to `dst` + wherever it contains the value True. + """ + return (dst, src, where) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.putmask) +def putmask(a, mask, values): + """ + putmask(a, mask, values) + + Changes elements of an array based on conditional and input values. + + Sets ``a.flat[n] = values[n]`` for each n where ``mask.flat[n]==True``. + + If `values` is not the same size as `a` and `mask` then it will repeat. + This gives behavior different from ``a[mask] = values``. + + Parameters + ---------- + a : array_like + Target array. + mask : array_like + Boolean mask array. It has to be the same shape as `a`. + values : array_like + Values to put into `a` where `mask` is True. If `values` is smaller + than `a` it will be repeated. + + See Also + -------- + place, put, take, copyto + + Examples + -------- + >>> x = np.arange(6).reshape(2, 3) + >>> np.putmask(x, x>2, x**2) + >>> x + array([[ 0, 1, 2], + [ 9, 16, 25]]) + + If `values` is smaller than `a` it is repeated: + + >>> x = np.arange(5) + >>> np.putmask(x, x>1, [-33, -44]) + >>> x + array([ 0, 1, -33, -44, -33]) + + """ + return (a, mask, values) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.packbits) +def packbits(myarray, axis=None): + """ + packbits(myarray, axis=None) + + Packs the elements of a binary-valued array into bits in a uint8 array. + + The result is padded to full bytes by inserting zero bits at the end. + Parameters + ---------- + myarray : array_like + An array of integers or booleans whose elements should be packed to + bits. + axis : int, optional + The dimension over which bit-packing is done. + ``None`` implies packing the flattened array. + + Returns + ------- + packed : ndarray + Array of type uint8 whose elements represent bits corresponding to the + logical (0 or nonzero) value of the input elements. The shape of + `packed` has the same number of dimensions as the input (unless `axis` + is None, in which case the output is 1-D). + + See Also + -------- + unpackbits: Unpacks elements of a uint8 array into a binary-valued output + array. + + Examples + -------- + >>> a = np.array([[[1,0,1], + ... [0,1,0]], + ... [[1,1,0], + ... [0,0,1]]]) + >>> b = np.packbits(a, axis=-1) + >>> b + array([[[160], + [ 64]], + [[192], + [ 32]]], dtype=uint8) + + Note that in binary 160 = 1010 0000, 64 = 0100 0000, 192 = 1100 0000, + and 32 = 0010 0000. -@array_function_dispatch(_is_busday_dispatcher) -def is_busday(dates, weekmask=None, holidays=None, busdaycal=None, - out=None): + """ + return (myarray,) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.unpackbits) +def unpackbits(myarray, axis=None): + """ + unpackbits(myarray, axis=None) + + Unpacks elements of a uint8 array into a binary-valued output array. + + Each element of `myarray` represents a bit-field that should be unpacked + into a binary-valued output array. The shape of the output array is either + 1-D (if `axis` is None) or the same shape as the input array with unpacking + done along the axis specified. + + Parameters + ---------- + myarray : ndarray, uint8 type + Input array. + axis : int, optional + The dimension over which bit-unpacking is done. + ``None`` implies unpacking the flattened array. + + Returns + ------- + unpacked : ndarray, uint8 type + The elements are binary-valued (0 or 1). + + See Also + -------- + packbits : Packs the elements of a binary-valued array into bits in a uint8 + array. + + Examples + -------- + >>> a = np.array([[2], [7], [23]], dtype=np.uint8) + >>> a + array([[ 2], + [ 7], + [23]], dtype=uint8) + >>> b = np.unpackbits(a, axis=1) + >>> b + array([[0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 0, 1, 1, 1]], dtype=uint8) + + """ + return (myarray,) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.shares_memory) +def shares_memory(a, b, max_work=None): + """ + shares_memory(a, b, max_work=None) + + Determine if two arrays share memory + + Parameters + ---------- + a, b : ndarray + Input arrays + max_work : int, optional + Effort to spend on solving the overlap problem (maximum number + of candidate solutions to consider). The following special + values are recognized: + + max_work=MAY_SHARE_EXACT (default) + The problem is solved exactly. In this case, the function returns + True only if there is an element shared between the arrays. + max_work=MAY_SHARE_BOUNDS + Only the memory bounds of a and b are checked. + + Raises + ------ + numpy.TooHardError + Exceeded max_work. + + Returns + ------- + out : bool + + See Also + -------- + may_share_memory + + Examples + -------- + >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9])) + False + + """ + return (a, b) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.may_share_memory) +def may_share_memory(a, b, max_work=None): + """ + may_share_memory(a, b, max_work=None) + + Determine if two arrays might share memory + + A return of True does not necessarily mean that the two arrays + share any element. It just means that they *might*. + + Only the memory bounds of a and b are checked by default. + + Parameters + ---------- + a, b : ndarray + Input arrays + max_work : int, optional + Effort to spend on solving the overlap problem. See + `shares_memory` for details. Default for ``may_share_memory`` + is to do a bounds check. + + Returns + ------- + out : bool + + See Also + -------- + shares_memory + + Examples + -------- + >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9])) + False + >>> x = np.zeros([3, 4]) + >>> np.may_share_memory(x[:,0], x[:,1]) + True + + """ + return (a, b) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.is_busday) +def is_busday(dates, weekmask=None, holidays=None, busdaycal=None, out=None): """ is_busday(dates, weekmask='1111100', holidays=None, busdaycal=None, out=None) @@ -882,28 +1334,14 @@ def is_busday(dates, weekmask=None, holidays=None, busdaycal=None, >>> # The weekdays are Friday, Saturday, and Monday ... np.is_busday(['2011-07-01', '2011-07-02', '2011-07-18'], ... holidays=['2011-07-01', '2011-07-04', '2011-07-17']) - array([False, False, True], dtype='bool') + array([False, False, True]) """ - kwargs = {} - if weekmask is not None: - kwargs['weekmask'] = weekmask - if holidays is not None: - kwargs['holidays'] = holidays - if busdaycal is not None: - kwargs['busdaycal'] = busdaycal - if out is not None: - kwargs['out'] = out - return _multiarray_umath.is_busday(dates, **kwargs) - - -def _busday_offset_dispatcher(dates, offsets, roll=None, weekmask=None, - holidays=None, busdaycal=None, out=None): - return (dates, offsets, weekmask, holidays, out) + return (dates, weekmask, holidays, out) -@array_function_dispatch(_busday_offset_dispatcher) -def busday_offset(dates, offsets, roll='raise', weekmask=None, - holidays=None, busdaycal=None, out=None): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.busday_offset) +def busday_offset(dates, offsets, roll=None, weekmask=None, holidays=None, + busdaycal=None, out=None): """ busday_offset(dates, offsets, roll='raise', weekmask='1111100', holidays=None, busdaycal=None, out=None) @@ -970,46 +1408,32 @@ def busday_offset(dates, offsets, roll='raise', weekmask=None, -------- >>> # First business day in October 2011 (not accounting for holidays) ... np.busday_offset('2011-10', 0, roll='forward') - numpy.datetime64('2011-10-03','D') + numpy.datetime64('2011-10-03') >>> # Last business day in February 2012 (not accounting for holidays) ... np.busday_offset('2012-03', -1, roll='forward') - numpy.datetime64('2012-02-29','D') + numpy.datetime64('2012-02-29') >>> # Third Wednesday in January 2011 ... np.busday_offset('2011-01', 2, roll='forward', weekmask='Wed') - numpy.datetime64('2011-01-19','D') + numpy.datetime64('2011-01-19') >>> # 2012 Mother's Day in Canada and the U.S. ... np.busday_offset('2012-05', 1, roll='forward', weekmask='Sun') - numpy.datetime64('2012-05-13','D') + numpy.datetime64('2012-05-13') >>> # First business day on or after a date ... np.busday_offset('2011-03-20', 0, roll='forward') - numpy.datetime64('2011-03-21','D') + numpy.datetime64('2011-03-21') >>> np.busday_offset('2011-03-22', 0, roll='forward') - numpy.datetime64('2011-03-22','D') + numpy.datetime64('2011-03-22') >>> # First business day after a date ... np.busday_offset('2011-03-20', 1, roll='backward') - numpy.datetime64('2011-03-21','D') + numpy.datetime64('2011-03-21') >>> np.busday_offset('2011-03-22', 1, roll='backward') - numpy.datetime64('2011-03-23','D') + numpy.datetime64('2011-03-23') """ - kwargs = {} - if weekmask is not None: - kwargs['weekmask'] = weekmask - if holidays is not None: - kwargs['holidays'] = holidays - if busdaycal is not None: - kwargs['busdaycal'] = busdaycal - if out is not None: - kwargs['out'] = out - return _multiarray_umath.busday_offset(dates, offsets, roll, **kwargs) - - -def _busday_count_dispatcher(begindates, enddates, weekmask=None, - holidays=None, busdaycal=None, out=None): - return (begindates, enddates, weekmask, holidays, out) + return (dates, offsets, weekmask, holidays, out) -@array_function_dispatch(_busday_count_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.busday_count) def busday_count(begindates, enddates, weekmask=None, holidays=None, busdaycal=None, out=None): """ @@ -1068,32 +1492,21 @@ def busday_count(begindates, enddates, weekmask=None, holidays=None, ... np.busday_count('2011-01', '2011-02') 21 >>> # Number of weekdays in 2011 - ... np.busday_count('2011', '2012') + >>> np.busday_count('2011', '2012') 260 >>> # Number of Saturdays in 2011 ... np.busday_count('2011', '2012', weekmask='Sat') 53 """ - kwargs = {} - if weekmask is not None: - kwargs['weekmask'] = weekmask - if holidays is not None: - kwargs['holidays'] = holidays - if busdaycal is not None: - kwargs['busdaycal'] = busdaycal - if out is not None: - kwargs['out'] = out - return _multiarray_umath.busday_count(begindates, enddates, **kwargs) - - -def _datetime_as_string_dispatcher( - arr, unit=None, timezone=None, casting=None): - return (arr,) + return (begindates, enddates, weekmask, holidays, out) -@array_function_dispatch(_datetime_as_string_dispatcher) -def datetime_as_string(arr, unit=None, timezone='naive', casting='same_kind'): +@array_function_from_c_func_and_dispatcher( + _multiarray_umath.datetime_as_string) +def datetime_as_string(arr, unit=None, timezone=None, casting=None): """ + datetime_as_string(arr, unit=None, timezone='naive', casting='same_kind') + Convert an array of datetimes into an array of strings. Parameters @@ -1117,6 +1530,7 @@ def datetime_as_string(arr, unit=None, timezone='naive', casting='same_kind'): Examples -------- + >>> import pytz >>> d = np.arange('2002-10-27T04:30', 4*60, 60, dtype='M8[m]') >>> d array(['2002-10-27T04:30', '2002-10-27T05:30', '2002-10-27T06:30', @@ -1147,7 +1561,9 @@ def datetime_as_string(arr, unit=None, timezone='naive', casting='same_kind'): 'casting' can be used to specify whether precision can be changed >>> np.datetime_as_string(d, unit='h', casting='safe') + Traceback (most recent call last): + ... TypeError: Cannot create a datetime string as units 'h' from a NumPy datetime with units 'm' according to the rule 'safe' """ - return _multiarray_umath.datetime_as_string(arr, unit, timezone, casting) + return (arr,) diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 6e4e585c3..8a8efddf3 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -6,6 +6,7 @@ try: import collections.abc as collections_abc except ImportError: import collections as collections_abc +import functools import itertools import operator import sys @@ -27,8 +28,9 @@ from .multiarray import ( if sys.version_info[0] < 3: from .multiarray import newbuffer, getbuffer +from . import overrides from . import umath -from .overrides import array_function_dispatch +from .overrides import set_module from .umath import (multiply, invert, sin, UFUNC_BUFSIZE_DEFAULT, ERR_IGNORE, ERR_WARN, ERR_RAISE, ERR_CALL, ERR_PRINT, ERR_LOG, ERR_DEFAULT, PINF, NAN) @@ -55,6 +57,10 @@ else: import __builtin__ as builtins +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + def loads(*args, **kwargs): # NumPy 1.15.0, 2017-12-10 warnings.warn( @@ -87,6 +93,7 @@ if sys.version_info[0] < 3: __all__.extend(['getbuffer', 'newbuffer']) +@set_module('numpy') class ComplexWarning(RuntimeWarning): """ The warning raised when casting a complex dtype to a real dtype. @@ -153,9 +160,9 @@ def zeros_like(a, dtype=None, order='K', subok=True): >>> y = np.arange(3, dtype=float) >>> y - array([ 0., 1., 2.]) + array([0., 1., 2.]) >>> np.zeros_like(y) - array([ 0., 0., 0.]) + array([0., 0., 0.]) """ res = empty_like(a, dtype=dtype, order=order, subok=subok) @@ -165,6 +172,7 @@ def zeros_like(a, dtype=None, order='K', subok=True): return res +@set_module('numpy') def ones(shape, dtype=None, order='C'): """ Return a new array of given shape and type, filled with ones. @@ -197,19 +205,19 @@ def ones(shape, dtype=None, order='C'): Examples -------- >>> np.ones(5) - array([ 1., 1., 1., 1., 1.]) + array([1., 1., 1., 1., 1.]) >>> np.ones((5,), dtype=int) array([1, 1, 1, 1, 1]) >>> np.ones((2, 1)) - array([[ 1.], - [ 1.]]) + array([[1.], + [1.]]) >>> s = (2,2) >>> np.ones(s) - array([[ 1., 1.], - [ 1., 1.]]) + array([[1., 1.], + [1., 1.]]) """ a = empty(shape, dtype, order) @@ -272,9 +280,9 @@ def ones_like(a, dtype=None, order='K', subok=True): >>> y = np.arange(3, dtype=float) >>> y - array([ 0., 1., 2.]) + array([0., 1., 2.]) >>> np.ones_like(y) - array([ 1., 1., 1.]) + array([1., 1., 1.]) """ res = empty_like(a, dtype=dtype, order=order, subok=subok) @@ -282,6 +290,7 @@ def ones_like(a, dtype=None, order='K', subok=True): return res +@set_module('numpy') def full(shape, fill_value, dtype=None, order='C'): """ Return a new array of given shape and type, filled with `fill_value`. @@ -314,8 +323,8 @@ def full(shape, fill_value, dtype=None, order='C'): Examples -------- >>> np.full((2, 2), np.inf) - array([[ inf, inf], - [ inf, inf]]) + array([[inf, inf], + [inf, inf]]) >>> np.full((2, 2), 10) array([[10, 10], [10, 10]]) @@ -376,13 +385,13 @@ def full_like(a, fill_value, dtype=None, order='K', subok=True): >>> np.full_like(x, 0.1) array([0, 0, 0, 0, 0, 0]) >>> np.full_like(x, 0.1, dtype=np.double) - array([ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) + array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) >>> np.full_like(x, np.nan, dtype=np.double) - array([ nan, nan, nan, nan, nan, nan]) + array([nan, nan, nan, nan, nan, nan]) >>> y = np.arange(6, dtype=np.double) >>> np.full_like(y, 0.1) - array([ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) + array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) """ res = empty_like(a, dtype=dtype, order=order, subok=subok) @@ -457,6 +466,7 @@ def count_nonzero(a, axis=None): return a_bool.sum(axis=axis, dtype=np.intp) +@set_module('numpy') def asarray(a, dtype=None, order=None): """Convert the input to an array. @@ -528,6 +538,7 @@ def asarray(a, dtype=None, order=None): return array(a, dtype, copy=False, order=order) +@set_module('numpy') def asanyarray(a, dtype=None, order=None): """Convert the input to an ndarray, but pass ndarray subclasses through. @@ -580,9 +591,10 @@ def asanyarray(a, dtype=None, order=None): return array(a, dtype, copy=False, order=order, subok=True) +@set_module('numpy') def ascontiguousarray(a, dtype=None): """ - Return a contiguous array in memory (C order). + Return a contiguous array (ndim >= 1) in memory (C order). Parameters ---------- @@ -608,18 +620,22 @@ def ascontiguousarray(a, dtype=None): -------- >>> x = np.arange(6).reshape(2,3) >>> np.ascontiguousarray(x, dtype=np.float32) - array([[ 0., 1., 2.], - [ 3., 4., 5.]], dtype=float32) + array([[0., 1., 2.], + [3., 4., 5.]], dtype=float32) >>> x.flags['C_CONTIGUOUS'] True + Note: This function returns an array with at least one-dimension (1-d) + so it will not preserve 0-d arrays. + """ return array(a, dtype, copy=False, order='C', ndmin=1) +@set_module('numpy') def asfortranarray(a, dtype=None): """ - Return an array laid out in Fortran order in memory. + Return an array (ndim >= 1) laid out in Fortran order in memory. Parameters ---------- @@ -650,10 +666,14 @@ def asfortranarray(a, dtype=None): >>> y.flags['F_CONTIGUOUS'] True + Note: This function returns an array with at least one-dimension (1-d) + so it will not preserve 0-d arrays. + """ return array(a, dtype, copy=False, order='F', ndmin=1) +@set_module('numpy') def require(a, dtype=None, requirements=None): """ Return an ndarray of the provided type that satisfies requirements. @@ -725,7 +745,7 @@ def require(a, dtype=None, requirements=None): if not requirements: return asanyarray(a, dtype=dtype) else: - requirements = set(possible_flags[x.upper()] for x in requirements) + requirements = {possible_flags[x.upper()] for x in requirements} if 'E' in requirements: requirements.remove('E') @@ -734,7 +754,7 @@ def require(a, dtype=None, requirements=None): subok = True order = 'A' - if requirements >= set(['C', 'F']): + if requirements >= {'C', 'F'}: raise ValueError('Cannot specify both "C" and "F" order') elif 'F' in requirements: order = 'F' @@ -752,6 +772,7 @@ def require(a, dtype=None, requirements=None): return arr +@set_module('numpy') def isfortran(a): """ Returns True if the array is Fortran contiguous but *not* C contiguous. @@ -781,7 +802,7 @@ def isfortran(a): >>> np.isfortran(a) False - >>> b = np.array([[1, 2, 3], [4, 5, 6]], order='FORTRAN') + >>> b = np.array([[1, 2, 3], [4, 5, 6]], order='F') >>> b array([[1, 2, 3], [4, 5, 6]]) @@ -966,11 +987,11 @@ def correlate(a, v, mode='valid'): Examples -------- >>> np.correlate([1, 2, 3], [0, 1, 0.5]) - array([ 3.5]) + array([3.5]) >>> np.correlate([1, 2, 3], [0, 1, 0.5], "same") - array([ 2. , 3.5, 3. ]) + array([2. , 3.5, 3. ]) >>> np.correlate([1, 2, 3], [0, 1, 0.5], "full") - array([ 0.5, 2. , 3.5, 3. , 0. ]) + array([0.5, 2. , 3.5, 3. , 0. ]) Using complex sequences: @@ -1066,20 +1087,20 @@ def convolve(a, v, mode='full'): before "sliding" the two across one another: >>> np.convolve([1, 2, 3], [0, 1, 0.5]) - array([ 0. , 1. , 2.5, 4. , 1.5]) + array([0. , 1. , 2.5, 4. , 1.5]) Only return the middle values of the convolution. Contains boundary effects, where zeros are taken into account: >>> np.convolve([1,2,3],[0,1,0.5], 'same') - array([ 1. , 2.5, 4. ]) + array([1. , 2.5, 4. ]) The two arrays are of the same length, so there is only one position where they completely overlap: >>> np.convolve([1,2,3],[0,1,0.5], 'valid') - array([ 2.5]) + array([2.5]) """ a, v = array(a, copy=False, ndmin=1), array(v, copy=False, ndmin=1) @@ -1155,11 +1176,11 @@ def outer(a, b, out=None): [-2., -1., 0., 1., 2.]]) >>> im = np.outer(1j*np.linspace(2, -2, 5), np.ones((5,))) >>> im - array([[ 0.+2.j, 0.+2.j, 0.+2.j, 0.+2.j, 0.+2.j], - [ 0.+1.j, 0.+1.j, 0.+1.j, 0.+1.j, 0.+1.j], - [ 0.+0.j, 0.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.-1.j, 0.-1.j, 0.-1.j, 0.-1.j, 0.-1.j], - [ 0.-2.j, 0.-2.j, 0.-2.j, 0.-2.j, 0.-2.j]]) + array([[0.+2.j, 0.+2.j, 0.+2.j, 0.+2.j, 0.+2.j], + [0.+1.j, 0.+1.j, 0.+1.j, 0.+1.j, 0.+1.j], + [0.+0.j, 0.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], + [0.-1.j, 0.-1.j, 0.-1.j, 0.-1.j, 0.-1.j], + [0.-2.j, 0.-2.j, 0.-2.j, 0.-2.j, 0.-2.j]]) >>> grid = rl + im >>> grid array([[-2.+2.j, -1.+2.j, 0.+2.j, 1.+2.j, 2.+2.j], @@ -1172,9 +1193,9 @@ def outer(a, b, out=None): >>> x = np.array(['a', 'b', 'c'], dtype=object) >>> np.outer(x, [1, 2, 3]) - array([[a, aa, aaa], - [b, bb, bbb], - [c, cc, ccc]], dtype=object) + array([['a', 'aa', 'aaa'], + ['b', 'bb', 'bbb'], + ['c', 'cc', 'ccc']], dtype=object) """ a = asarray(a) @@ -1243,11 +1264,11 @@ def tensordot(a, b, axes=2): >>> c.shape (5, 2) >>> c - array([[ 4400., 4730.], - [ 4532., 4874.], - [ 4664., 5018.], - [ 4796., 5162.], - [ 4928., 5306.]]) + array([[4400., 4730.], + [4532., 4874.], + [4664., 5018.], + [4796., 5162.], + [4928., 5306.]]) >>> # A slower but equivalent way of computing the same... >>> d = np.zeros((5,2)) >>> for i in range(5): @@ -1273,40 +1294,40 @@ def tensordot(a, b, axes=2): [3, 4]], [[5, 6], [7, 8]]]) - array([[a, b], - [c, d]], dtype=object) + array([['a', 'b'], + ['c', 'd']], dtype=object) >>> np.tensordot(a, A) # third argument default is 2 for double-contraction - array([abbcccdddd, aaaaabbbbbbcccccccdddddddd], dtype=object) + array(['abbcccdddd', 'aaaaabbbbbbcccccccdddddddd'], dtype=object) >>> np.tensordot(a, A, 1) - array([[[acc, bdd], - [aaacccc, bbbdddd]], - [[aaaaacccccc, bbbbbdddddd], - [aaaaaaacccccccc, bbbbbbbdddddddd]]], dtype=object) + array([[['acc', 'bdd'], + ['aaacccc', 'bbbdddd']], + [['aaaaacccccc', 'bbbbbdddddd'], + ['aaaaaaacccccccc', 'bbbbbbbdddddddd']]], dtype=object) >>> np.tensordot(a, A, 0) # tensor product (result too long to incl.) - array([[[[[a, b], - [c, d]], + array([[[[['a', 'b'], + ['c', 'd']], ... >>> np.tensordot(a, A, (0, 1)) - array([[[abbbbb, cddddd], - [aabbbbbb, ccdddddd]], - [[aaabbbbbbb, cccddddddd], - [aaaabbbbbbbb, ccccdddddddd]]], dtype=object) + array([[['abbbbb', 'cddddd'], + ['aabbbbbb', 'ccdddddd']], + [['aaabbbbbbb', 'cccddddddd'], + ['aaaabbbbbbbb', 'ccccdddddddd']]], dtype=object) >>> np.tensordot(a, A, (2, 1)) - array([[[abb, cdd], - [aaabbbb, cccdddd]], - [[aaaaabbbbbb, cccccdddddd], - [aaaaaaabbbbbbbb, cccccccdddddddd]]], dtype=object) + array([[['abb', 'cdd'], + ['aaabbbb', 'cccdddd']], + [['aaaaabbbbbb', 'cccccdddddd'], + ['aaaaaaabbbbbbbb', 'cccccccdddddddd']]], dtype=object) >>> np.tensordot(a, A, ((0, 1), (0, 1))) - array([abbbcccccddddddd, aabbbbccccccdddddddd], dtype=object) + array(['abbbcccccddddddd', 'aabbbbccccccdddddddd'], dtype=object) >>> np.tensordot(a, A, ((2, 1), (1, 0))) - array([acccbbdddd, aaaaacccccccbbbbbbdddddddd], dtype=object) + array(['acccbbdddd', 'aaaaacccccccbbbbbbdddddddd'], dtype=object) """ try: @@ -1759,7 +1780,7 @@ def cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None): >>> x = [1,2] >>> y = [4,5] >>> np.cross(x, y) - -3 + array(-3) Multiple vector cross-products. Note that the direction of the cross product vector is defined by the `right-hand rule`. @@ -1878,6 +1899,7 @@ def cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None): little_endian = (sys.byteorder == 'little') +@set_module('numpy') def indices(dimensions, dtype=int): """ Return an array representing the indices of a grid. @@ -1949,6 +1971,7 @@ def indices(dimensions, dtype=int): return res +@set_module('numpy') def fromfunction(function, shape, **kwargs): """ Construct an array by executing a function over each coordinate. @@ -2009,6 +2032,7 @@ def _frombuffer(buf, dtype, shape, order): return frombuffer(buf, dtype=dtype).reshape(shape, order=order) +@set_module('numpy') def isscalar(num): """ Returns True if the type of `num` is a scalar type. @@ -2073,10 +2097,10 @@ def isscalar(num): NumPy supports PEP 3141 numbers: >>> from fractions import Fraction - >>> isscalar(Fraction(5, 17)) + >>> np.isscalar(Fraction(5, 17)) True >>> from numbers import Number - >>> isscalar(Number()) + >>> np.isscalar(Number()) True """ @@ -2085,6 +2109,7 @@ def isscalar(num): or isinstance(num, numbers.Number)) +@set_module('numpy') def binary_repr(num, width=None): """ Return the binary representation of the input number as a string. @@ -2195,6 +2220,7 @@ def binary_repr(num, width=None): return '1' * (outwidth - binwidth) + binary +@set_module('numpy') def base_repr(number, base=2, padding=0): """ Return a string representation of a number in the given base system. @@ -2289,6 +2315,7 @@ def _maketup(descr, val): return tuple(res) +@set_module('numpy') def identity(n, dtype=None): """ Return the identity array. @@ -2312,9 +2339,9 @@ def identity(n, dtype=None): Examples -------- >>> np.identity(3) - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) """ from numpy import eye @@ -2460,23 +2487,23 @@ def isclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False): Examples -------- >>> np.isclose([1e10,1e-7], [1.00001e10,1e-8]) - array([True, False]) + array([ True, False]) >>> np.isclose([1e10,1e-8], [1.00001e10,1e-9]) - array([True, True]) + array([ True, True]) >>> np.isclose([1e10,1e-8], [1.0001e10,1e-9]) - array([False, True]) + array([False, True]) >>> np.isclose([1.0, np.nan], [1.0, np.nan]) - array([True, False]) + array([ True, False]) >>> np.isclose([1.0, np.nan], [1.0, np.nan], equal_nan=True) - array([True, True]) + array([ True, True]) >>> np.isclose([1e-8, 1e-7], [0.0, 0.0]) - array([ True, False], dtype=bool) + array([ True, False]) >>> np.isclose([1e-100, 1e-7], [0.0, 0.0], atol=0.0) - array([False, False], dtype=bool) + array([False, False]) >>> np.isclose([1e-10, 1e-10], [1e-20, 0.0]) - array([ True, True], dtype=bool) + array([ True, True]) >>> np.isclose([1e-10, 1e-10], [1e-20, 0.999999e-10], atol=0.0) - array([False, True], dtype=bool) + array([False, True]) """ def within_tol(x, y, atol, rtol): with errstate(invalid='ignore'): @@ -2623,12 +2650,10 @@ _errdict = {"ignore": ERR_IGNORE, "print": ERR_PRINT, "log": ERR_LOG} -_errdict_rev = {} -for key in _errdict.keys(): - _errdict_rev[_errdict[key]] = key -del key +_errdict_rev = {value: key for key, value in _errdict.items()} +@set_module('numpy') def seterr(all=None, divide=None, over=None, under=None, invalid=None): """ Set how floating-point errors are handled. @@ -2685,11 +2710,9 @@ def seterr(all=None, divide=None, over=None, under=None, invalid=None): -------- >>> old_settings = np.seterr(all='ignore') #seterr to known value >>> np.seterr(over='raise') - {'over': 'ignore', 'divide': 'ignore', 'invalid': 'ignore', - 'under': 'ignore'} + {'divide': 'ignore', 'over': 'ignore', 'under': 'ignore', 'invalid': 'ignore'} >>> np.seterr(**old_settings) # reset to default - {'over': 'raise', 'divide': 'ignore', 'invalid': 'ignore', - 'under': 'ignore'} + {'divide': 'ignore', 'over': 'raise', 'under': 'ignore', 'invalid': 'ignore'} >>> np.int16(32000) * np.int16(3) 30464 @@ -2699,11 +2722,11 @@ def seterr(all=None, divide=None, over=None, under=None, invalid=None): File "<stdin>", line 1, in <module> FloatingPointError: overflow encountered in short_scalars + >>> from collections import OrderedDict >>> old_settings = np.seterr(all='print') - >>> np.geterr() - {'over': 'print', 'divide': 'print', 'invalid': 'print', 'under': 'print'} + >>> OrderedDict(np.geterr()) + OrderedDict([('divide', 'print'), ('over', 'print'), ('under', 'print'), ('invalid', 'print')]) >>> np.int16(32000) * np.int16(3) - Warning: overflow encountered in short_scalars 30464 """ @@ -2730,6 +2753,7 @@ def seterr(all=None, divide=None, over=None, under=None, invalid=None): return old +@set_module('numpy') def geterr(): """ Get the current way of handling floating-point errors. @@ -2753,18 +2777,17 @@ def geterr(): Examples -------- - >>> np.geterr() - {'over': 'warn', 'divide': 'warn', 'invalid': 'warn', - 'under': 'ignore'} + >>> from collections import OrderedDict + >>> sorted(np.geterr().items()) + [('divide', 'warn'), ('invalid', 'warn'), ('over', 'warn'), ('under', 'ignore')] >>> np.arange(3.) / np.arange(3.) - array([ NaN, 1., 1.]) + array([nan, 1., 1.]) >>> oldsettings = np.seterr(all='warn', over='raise') - >>> np.geterr() - {'over': 'raise', 'divide': 'warn', 'invalid': 'warn', 'under': 'warn'} + >>> OrderedDict(sorted(np.geterr().items())) + OrderedDict([('divide', 'warn'), ('invalid', 'warn'), ('over', 'raise'), ('under', 'warn')]) >>> np.arange(3.) / np.arange(3.) - __main__:1: RuntimeWarning: invalid value encountered in divide - array([ NaN, 1., 1.]) + array([nan, 1., 1.]) """ maskvalue = umath.geterrobj()[1] @@ -2781,6 +2804,7 @@ def geterr(): return res +@set_module('numpy') def setbufsize(size): """ Set the size of the buffer used in ufuncs. @@ -2805,6 +2829,7 @@ def setbufsize(size): return old +@set_module('numpy') def getbufsize(): """ Return the size of the buffer used in ufuncs. @@ -2818,6 +2843,7 @@ def getbufsize(): return umath.geterrobj()[0] +@set_module('numpy') def seterrcall(func): """ Set the floating-point error callback function or log object. @@ -2868,15 +2894,16 @@ def seterrcall(func): >>> saved_handler = np.seterrcall(err_handler) >>> save_err = np.seterr(all='call') + >>> from collections import OrderedDict >>> np.array([1, 2, 3]) / 0.0 Floating point error (divide by zero), with flag 1 - array([ Inf, Inf, Inf]) + array([inf, inf, inf]) >>> np.seterrcall(saved_handler) <function err_handler at 0x...> - >>> np.seterr(**save_err) - {'over': 'call', 'divide': 'call', 'invalid': 'call', 'under': 'call'} + >>> OrderedDict(sorted(np.seterr(**save_err).items())) + OrderedDict([('divide', 'call'), ('invalid', 'call'), ('over', 'call'), ('under', 'call')]) Log error message: @@ -2890,14 +2917,13 @@ def seterrcall(func): >>> save_err = np.seterr(all='log') >>> np.array([1, 2, 3]) / 0.0 - LOG: Warning: divide by zero encountered in divide - <BLANKLINE> - array([ Inf, Inf, Inf]) + LOG: Warning: divide by zero encountered in true_divide + array([inf, inf, inf]) >>> np.seterrcall(saved_handler) - <__main__.Log object at 0x...> - >>> np.seterr(**save_err) - {'over': 'log', 'divide': 'log', 'invalid': 'log', 'under': 'log'} + <numpy.core.numeric.Log object at 0x...> + >>> OrderedDict(sorted(np.seterr(**save_err).items())) + OrderedDict([('divide', 'log'), ('invalid', 'log'), ('over', 'log'), ('under', 'log')]) """ if func is not None and not isinstance(func, collections_abc.Callable): @@ -2910,6 +2936,7 @@ def seterrcall(func): return old +@set_module('numpy') def geterrcall(): """ Return the current callback function used on floating-point errors. @@ -2945,7 +2972,7 @@ def geterrcall(): >>> oldhandler = np.seterrcall(err_handler) >>> np.array([1, 2, 3]) / 0.0 Floating point error (divide by zero), with flag 1 - array([ Inf, Inf, Inf]) + array([inf, inf, inf]) >>> cur_handler = np.geterrcall() >>> cur_handler is err_handler @@ -2962,6 +2989,7 @@ class _unspecified(object): _Unspecified = _unspecified() +@set_module('numpy') class errstate(object): """ errstate(**kwargs) @@ -2992,15 +3020,14 @@ class errstate(object): Examples -------- + >>> from collections import OrderedDict >>> olderr = np.seterr(all='ignore') # Set error handling to known state. >>> np.arange(3) / 0. - array([ NaN, Inf, Inf]) + array([nan, inf, inf]) >>> with np.errstate(divide='warn'): ... np.arange(3) / 0. - ... - __main__:2: RuntimeWarning: divide by zero encountered in divide - array([ NaN, Inf, Inf]) + array([nan, inf, inf]) >>> np.sqrt(-1) nan @@ -3012,9 +3039,8 @@ class errstate(object): Outside the context the error handling behavior has not changed: - >>> np.geterr() - {'over': 'warn', 'divide': 'warn', 'invalid': 'warn', - 'under': 'ignore'} + >>> OrderedDict(sorted(np.geterr().items())) + OrderedDict([('divide', 'ignore'), ('invalid', 'ignore'), ('over', 'ignore'), ('under', 'ignore')]) """ # Note that we don't want to run the above doctests because they will fail diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 2fb841f7c..5bc37b73a 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -92,6 +92,7 @@ from numpy.core.multiarray import ( datetime_as_string, busday_offset, busday_count, is_busday, busdaycalendar ) +from numpy.core.overrides import set_module # we add more at the bottom __all__ = ['sctypeDict', 'sctypeNA', 'typeDict', 'typeNA', 'sctypes', @@ -162,19 +163,19 @@ def maximum_sctype(t): Examples -------- >>> np.maximum_sctype(int) - <type 'numpy.int64'> + <class 'numpy.int64'> >>> np.maximum_sctype(np.uint8) - <type 'numpy.uint64'> + <class 'numpy.uint64'> >>> np.maximum_sctype(complex) - <type 'numpy.complex192'> + <class 'numpy.complex256'> # may vary >>> np.maximum_sctype(str) - <type 'numpy.string_'> + <class 'numpy.str_'> >>> np.maximum_sctype('i2') - <type 'numpy.int64'> + <class 'numpy.int64'> >>> np.maximum_sctype('f4') - <type 'numpy.float96'> + <class 'numpy.float128'> # may vary """ g = obj2sctype(t) @@ -187,6 +188,8 @@ def maximum_sctype(t): else: return t + +@set_module('numpy') def issctype(rep): """ Determines whether the given object represents a scalar data-type. @@ -231,6 +234,8 @@ def issctype(rep): except Exception: return False + +@set_module('numpy') def obj2sctype(rep, default=None): """ Return the scalar dtype or NumPy equivalent of Python type of an object. @@ -255,19 +260,18 @@ def obj2sctype(rep, default=None): Examples -------- >>> np.obj2sctype(np.int32) - <type 'numpy.int32'> + <class 'numpy.int32'> >>> np.obj2sctype(np.array([1., 2.])) - <type 'numpy.float64'> + <class 'numpy.float64'> >>> np.obj2sctype(np.array([1.j])) - <type 'numpy.complex128'> + <class 'numpy.complex128'> >>> np.obj2sctype(dict) - <type 'numpy.object_'> + <class 'numpy.object_'> >>> np.obj2sctype('string') - <type 'numpy.string_'> >>> np.obj2sctype(1, default=list) - <type 'list'> + <class 'list'> """ # prevent abtract classes being upcast @@ -285,6 +289,7 @@ def obj2sctype(rep, default=None): return res.type +@set_module('numpy') def issubclass_(arg1, arg2): """ Determine if a class is a subclass of a second class. @@ -313,7 +318,7 @@ def issubclass_(arg1, arg2): Examples -------- >>> np.issubclass_(np.int32, int) - True + False # True on Python 2.7 >>> np.issubclass_(np.int32, float) False @@ -323,6 +328,8 @@ def issubclass_(arg1, arg2): except TypeError: return False + +@set_module('numpy') def issubsctype(arg1, arg2): """ Determine if the first argument is a subclass of the second argument. @@ -344,7 +351,7 @@ def issubsctype(arg1, arg2): Examples -------- >>> np.issubsctype('S8', str) - True + False >>> np.issubsctype(np.array([1]), int) True >>> np.issubsctype(np.array([1]), float) @@ -353,6 +360,8 @@ def issubsctype(arg1, arg2): """ return issubclass(obj2sctype(arg1), obj2sctype(arg2)) + +@set_module('numpy') def issubdtype(arg1, arg2): """ Returns True if first argument is a typecode lower/equal in type hierarchy. @@ -446,6 +455,8 @@ def _construct_lookups(): _construct_lookups() + +@set_module('numpy') def sctype2char(sctype): """ Return the string representation of a scalar dtype. @@ -473,9 +484,9 @@ def sctype2char(sctype): Examples -------- - >>> for sctype in [np.int32, float, complex, np.string_, np.ndarray]: + >>> for sctype in [np.int32, np.double, np.complex, np.string_, np.ndarray]: ... print(np.sctype2char(sctype)) - l + l # may vary d D S @@ -586,6 +597,8 @@ def _register_types(): _register_types() + +@set_module('numpy') def find_common_type(array_types, scalar_types): """ Determine common type following standard coercion rules. diff --git a/numpy/core/overrides.py b/numpy/core/overrides.py index 5be60cd29..c55174ecd 100644 --- a/numpy/core/overrides.py +++ b/numpy/core/overrides.py @@ -1,69 +1,23 @@ -"""Preliminary implementation of NEP-18 - -TODO: rewrite this in C for performance. -""" +"""Implementation of __array_function__ overrides from NEP-18.""" import collections import functools +import os -from numpy.core._multiarray_umath import ndarray +from numpy.core._multiarray_umath import ( + add_docstring, implement_array_function, _get_implementing_args) from numpy.compat._inspect import getargspec -_NDARRAY_ARRAY_FUNCTION = ndarray.__array_function__ - - -def get_overloaded_types_and_args(relevant_args): - """Returns a list of arguments on which to call __array_function__. +ENABLE_ARRAY_FUNCTION = bool( + int(os.environ.get('NUMPY_EXPERIMENTAL_ARRAY_FUNCTION', 0))) - Parameters - ---------- - relevant_args : iterable of array-like - Iterable of array-like arguments to check for __array_function__ - methods. - Returns - ------- - overloaded_types : collection of types - Types of arguments from relevant_args with __array_function__ methods. - overloaded_args : list - Arguments from relevant_args on which to call __array_function__ - methods, in the order in which they should be called. +add_docstring( + implement_array_function, """ - # Runtime is O(num_arguments * num_unique_types) - overloaded_types = [] - overloaded_args = [] - for arg in relevant_args: - arg_type = type(arg) - # We only collect arguments if they have a unique type, which ensures - # reasonable performance even with a long list of possibly overloaded - # arguments. - if (arg_type not in overloaded_types and - hasattr(arg_type, '__array_function__')): - - overloaded_types.append(arg_type) - - # By default, insert this argument at the end, but if it is - # subclass of another argument, insert it before that argument. - # This ensures "subclasses before superclasses". - index = len(overloaded_args) - for i, old_arg in enumerate(overloaded_args): - if issubclass(arg_type, type(old_arg)): - index = i - break - overloaded_args.insert(index, arg) - - # Special handling for ndarray.__array_function__ - overloaded_args = [ - arg for arg in overloaded_args - if type(arg).__array_function__ is not _NDARRAY_ARRAY_FUNCTION - ] - - return overloaded_types, overloaded_args - - -def array_function_implementation_or_override( - implementation, public_api, relevant_args, args, kwargs): - """Implement a function with checks for __array_function__ overrides. + Implement a function with checks for __array_function__ overrides. + + All arguments are required, and can only be passed by position. Arguments --------- @@ -78,36 +32,37 @@ def array_function_implementation_or_override( Iterable of arguments to check for __array_function__ methods. args : tuple Arbitrary positional arguments originally passed into ``public_api``. - kwargs : tuple + kwargs : dict Arbitrary keyword arguments originally passed into ``public_api``. Returns ------- - Result from calling `implementation()` or an `__array_function__` + Result from calling ``implementation()`` or an ``__array_function__`` method, as appropriate. Raises ------ TypeError : if no implementation is found. - """ - # Check for __array_function__ methods. - types, overloaded_args = get_overloaded_types_and_args(relevant_args) - if not overloaded_args: - return implementation(*args, **kwargs) + """) - # Call overrides - for overloaded_arg in overloaded_args: - # Use `public_api` instead of `implemenation` so __array_function__ - # implementations can do equality/identity comparisons. - result = overloaded_arg.__array_function__( - public_api, types, args, kwargs) - if result is not NotImplemented: - return result +# exposed for testing purposes; used internally by implement_array_function +add_docstring( + _get_implementing_args, + """ + Collect arguments on which to call __array_function__. - raise TypeError('no implementation found for {} on types that implement ' - '__array_function__: {}' - .format(public_api, list(map(type, overloaded_args)))) + Parameters + ---------- + relevant_args : iterable of array-like + Iterable of possibly array-like arguments to check for + __array_function__ methods. + + Returns + ------- + Sequence of arguments with __array_function__ methods, in the order in + which they should be called. + """) ArgSpec = collections.namedtuple('ArgSpec', 'args varargs keywords defaults') @@ -135,20 +90,98 @@ def verify_matching_signatures(implementation, dispatcher): 'default argument values') -def array_function_dispatch(dispatcher, verify=True): - """Decorator for adding dispatch with the __array_function__ protocol.""" +def set_module(module): + """Decorator for overriding __module__ on a function or class. + + Example usage:: + + @set_module('numpy') + def example(): + pass + + assert example.__module__ == 'numpy' + """ + def decorator(func): + if module is not None: + func.__module__ = module + return func + return decorator + + +def array_function_dispatch(dispatcher, module=None, verify=True, + docs_from_dispatcher=False): + """Decorator for adding dispatch with the __array_function__ protocol. + + See NEP-18 for example usage. + + Parameters + ---------- + dispatcher : callable + Function that when called like ``dispatcher(*args, **kwargs)`` with + arguments from the NumPy function call returns an iterable of + array-like arguments to check for ``__array_function__``. + module : str, optional + __module__ attribute to set on new function, e.g., ``module='numpy'``. + By default, module is copied from the decorated function. + verify : bool, optional + If True, verify the that the signature of the dispatcher and decorated + function signatures match exactly: all required and optional arguments + should appear in order with the same names, but the default values for + all optional arguments should be ``None``. Only disable verification + if the dispatcher's signature needs to deviate for some particular + reason, e.g., because the function has a signature like + ``func(*args, **kwargs)``. + docs_from_dispatcher : bool, optional + If True, copy docs from the dispatcher function onto the dispatched + function, rather than from the implementation. This is useful for + functions defined in C, which otherwise don't have docstrings. + + Returns + ------- + Function suitable for decorating the implementation of a NumPy function. + """ + + if not ENABLE_ARRAY_FUNCTION: + # __array_function__ requires an explicit opt-in for now + def decorator(implementation): + if module is not None: + implementation.__module__ = module + if docs_from_dispatcher: + add_docstring(implementation, dispatcher.__doc__) + return implementation + return decorator + def decorator(implementation): - # TODO: only do this check when the appropriate flag is enabled or for - # a dev install. We want this check for testing but don't want to - # slow down all numpy imports. if verify: verify_matching_signatures(implementation, dispatcher) + if docs_from_dispatcher: + add_docstring(implementation, dispatcher.__doc__) + @functools.wraps(implementation) def public_api(*args, **kwargs): relevant_args = dispatcher(*args, **kwargs) - return array_function_implementation_or_override( + return implement_array_function( implementation, public_api, relevant_args, args, kwargs) + + if module is not None: + public_api.__module__ = module + + # TODO: remove this when we drop Python 2 support (functools.wraps + # adds __wrapped__ automatically in later versions) + public_api.__wrapped__ = implementation + return public_api return decorator + + +def array_function_from_dispatcher( + implementation, module=None, verify=True, docs_from_dispatcher=True): + """Like array_function_dispatcher, but with function arguments flipped.""" + + def decorator(dispatcher): + return array_function_dispatch( + dispatcher, module, verify=verify, + docs_from_dispatcher=docs_from_dispatcher)(implementation) + return decorator diff --git a/numpy/core/records.py b/numpy/core/records.py index a483871ba..4d18c5712 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -7,10 +7,9 @@ Most commonly, ndarrays contain elements of a single type, e.g. floats, integers, bools etc. However, it is possible for elements to be combinations of these using structured types, such as:: - >>> a = np.array([(1, 2.0), (1, 2.0)], dtype=[('x', int), ('y', float)]) + >>> a = np.array([(1, 2.0), (1, 2.0)], dtype=[('x', np.int64), ('y', np.float64)]) >>> a - array([(1, 2.0), (1, 2.0)], - dtype=[('x', '<i4'), ('y', '<f8')]) + array([(1, 2.), (1, 2.)], dtype=[('x', '<i8'), ('y', '<f8')]) Here, each element consists of two fields: x (and int), and y (a float). This is known as a structured array. The different fields are analogous @@ -21,7 +20,7 @@ one would a dictionary:: array([1, 1]) >>> a['y'] - array([ 2., 2.]) + array([2., 2.]) Record arrays allow us to access fields as properties:: @@ -31,7 +30,7 @@ Record arrays allow us to access fields as properties:: array([1, 1]) >>> ar.y - array([ 2., 2.]) + array([2., 2.]) """ from __future__ import division, absolute_import, print_function @@ -39,10 +38,12 @@ from __future__ import division, absolute_import, print_function import sys import os import warnings +from collections import Counter, OrderedDict from . import numeric as sb from . import numerictypes as nt -from numpy.compat import isfileobj, bytes, long, unicode +from numpy.compat import isfileobj, bytes, long, unicode, os_fspath +from numpy.core.overrides import set_module from .arrayprint import get_printoptions # All of the functions allow formats to be a dtype @@ -73,15 +74,28 @@ _byteorderconv = {'b':'>', numfmt = nt.typeDict +# taken from OrderedDict recipes in the Python documentation +# https://docs.python.org/3.3/library/collections.html#ordereddict-examples-and-recipes +class _OrderedCounter(Counter, OrderedDict): + """Counter that remembers the order elements are first encountered""" + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, OrderedDict(self)) + + def __reduce__(self): + return self.__class__, (OrderedDict(self),) + + def find_duplicate(list): """Find duplication in a list, return a list of duplicated elements""" - dup = [] - for i in range(len(list)): - if (list[i] in list[i + 1:]): - if (list[i] not in dup): - dup.append(list[i]) - return dup + return [ + item + for item, counts in _OrderedCounter(list).items() + if counts > 1 + ] + +@set_module('numpy') class format_parser(object): """ Class to convert formats, names, titles description to a dtype. @@ -125,10 +139,9 @@ class format_parser(object): Examples -------- - >>> np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'], + >>> np.format_parser(['<f8', '<i4', '<a5'], ['col1', 'col2', 'col3'], ... ['T1', 'T2', 'T3']).dtype - dtype([(('T1', 'col1'), '<f8'), (('T2', 'col2'), '<i4'), - (('T3', 'col3'), '|S5')]) + dtype([(('T1', 'col1'), '<f8'), (('T2', 'col2'), '<i4'), (('T3', 'col3'), 'S5')]) `names` and/or `titles` can be empty lists. If `titles` is an empty list, titles will simply not appear. If `names` is empty, default field names @@ -136,9 +149,9 @@ class format_parser(object): >>> np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'], ... []).dtype - dtype([('col1', '<f8'), ('col2', '<i4'), ('col3', '|S5')]) - >>> np.format_parser(['f8', 'i4', 'a5'], [], []).dtype - dtype([('f0', '<f8'), ('f1', '<i4'), ('f2', '|S5')]) + dtype([('col1', '<f8'), ('col2', '<i4'), ('col3', '<S5')]) + >>> np.format_parser(['<f8', '<i4', '<a5'], [], []).dtype + dtype([('f0', '<f8'), ('f1', '<i4'), ('f2', 'S5')]) """ @@ -287,10 +300,8 @@ class record(nt.void): # pretty-print all fields names = self.dtype.names maxlen = max(len(name) for name in names) - rows = [] fmt = '%% %ds: %%s' % maxlen - for name in names: - rows.append(fmt % (name, getattr(self, name))) + rows = [fmt % (name, getattr(self, name)) for name in names] return "\n".join(rows) # The recarray is almost identical to a standard array (which supports @@ -379,20 +390,19 @@ class recarray(ndarray): -------- Create an array with two fields, ``x`` and ``y``: - >>> x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', float), ('y', int)]) + >>> x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', '<f8'), ('y', '<i8')]) >>> x - array([(1.0, 2), (3.0, 4)], - dtype=[('x', '<f8'), ('y', '<i4')]) + array([(1., 2), (3., 4)], dtype=[('x', '<f8'), ('y', '<i8')]) >>> x['x'] - array([ 1., 3.]) + array([1., 3.]) View the array as a record array: >>> x = x.view(np.recarray) >>> x.x - array([ 1., 3.]) + array([1., 3.]) >>> x.y array([2, 4]) @@ -579,7 +589,7 @@ def fromarrays(arrayList, dtype=None, shape=None, formats=None, >>> x3=np.array([1.1,2,3,4]) >>> r = np.core.records.fromarrays([x1,x2,x3],names='a,b,c') >>> print(r[1]) - (2, 'dd', 2.0) + (2, 'dd', 2.0) # may vary >>> x1[1]=34 >>> r.a array([1, 2, 3, 4]) @@ -658,11 +668,11 @@ def fromrecords(recList, dtype=None, shape=None, formats=None, names=None, >>> r.col1 array([456, 2]) >>> r.col2 - array(['dbe', 'de'], - dtype='|S3') + array(['dbe', 'de'], dtype='<U3') >>> import pickle - >>> print(pickle.loads(pickle.dumps(r))) - [(456, 'dbe', 1.2) (2, 'de', 1.3)] + >>> pickle.loads(pickle.dumps(r)) + rec.array([(456, 'dbe', 1.2), ( 2, 'de', 1.3)], + dtype=[('col1', '<i8'), ('col2', '<U3'), ('col3', '<f8')]) """ if formats is None and dtype is None: # slower @@ -737,9 +747,9 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, names=None, titles=None, aligned=False, byteorder=None): """Create an array from binary file data - If file is a string then that file is opened, else it is assumed - to be a file object. The file object must support random access - (i.e. it must have tell and seek methods). + If file is a string or a path-like object then that file is opened, + else it is assumed to be a file object. The file object must + support random access (i.e. it must have tell and seek methods). >>> from tempfile import TemporaryFile >>> a = np.empty(10,dtype='f8,i4,a5') @@ -749,7 +759,7 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, >>> a = a.newbyteorder('<') >>> a.tofile(fd) >>> - >>> fd.seek(0) + >>> _ = fd.seek(0) >>> r=np.core.records.fromfile(fd, formats='f8,i4,a5', shape=10, ... byteorder='<') >>> print(r[5]) @@ -763,10 +773,14 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, elif isinstance(shape, (int, long)): shape = (shape,) - name = 0 - if isinstance(fd, str): + if isfileobj(fd): + # file already opened + name = 0 + else: + # open file + fd = open(os_fspath(fd), 'rb') name = 1 - fd = open(fd, 'rb') + if (offset > 0): fd.seek(offset, 1) size = get_remaining_size(fd) @@ -778,13 +792,13 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, itemsize = descr.itemsize - shapeprod = sb.array(shape).prod() + shapeprod = sb.array(shape).prod(dtype=nt.intp) shapesize = shapeprod * itemsize if shapesize < 0: shape = list(shape) - shape[shape.index(-1)] = size / -shapesize + shape[shape.index(-1)] = size // -shapesize shape = tuple(shape) - shapeprod = sb.array(shape).prod() + shapeprod = sb.array(shape).prod(dtype=nt.intp) nbytes = shapeprod * itemsize diff --git a/numpy/core/setup.py b/numpy/core/setup.py index fc15fe59f..9ccca629e 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -379,8 +379,9 @@ def check_mathlib(config_cmd): def visibility_define(config): """Return the define value to use for NPY_VISIBILITY_HIDDEN (may be empty string).""" - if config.check_compiler_gcc4(): - return '__attribute__((visibility("hidden")))' + hide = '__attribute__((visibility("hidden")))' + if config.check_gcc_function_attribute(hide, 'hideme'): + return hide else: return '' @@ -677,7 +678,7 @@ def configuration(parent_package='',top_path=None): join('src', 'npymath', 'npy_math_complex.c.src'), join('src', 'npymath', 'halffloat.c') ] - + # Must be true for CRT compilers but not MinGW/cygwin. See gh-9977. is_msvc = platform.system() == 'Windows' config.add_installed_library('npymath', @@ -697,7 +698,8 @@ def configuration(parent_package='',top_path=None): ####################################################################### # This library is created for the build but it is not installed - npysort_sources = [join('src', 'npysort', 'quicksort.c.src'), + npysort_sources = [join('src', 'common', 'npy_sort.h.src'), + join('src', 'npysort', 'quicksort.c.src'), join('src', 'npysort', 'mergesort.c.src'), join('src', 'npysort', 'heapsort.c.src'), join('src', 'common', 'npy_partition.h.src'), @@ -730,8 +732,11 @@ def configuration(parent_package='',top_path=None): join('src', 'common', 'cblasfuncs.h'), join('src', 'common', 'lowlevel_strided_loops.h'), join('src', 'common', 'mem_overlap.h'), + join('src', 'common', 'npy_cblas.h'), join('src', 'common', 'npy_config.h'), + join('src', 'common', 'npy_ctypes.h'), join('src', 'common', 'npy_extint128.h'), + join('src', 'common', 'npy_import.h'), join('src', 'common', 'npy_longdouble.h'), join('src', 'common', 'templ_common.h.src'), join('src', 'common', 'ucsnarrow.h'), @@ -770,6 +775,7 @@ def configuration(parent_package='',top_path=None): multiarray_deps = [ join('src', 'multiarray', 'arrayobject.h'), join('src', 'multiarray', 'arraytypes.h'), + join('src', 'multiarray', 'arrayfunction_override.h'), join('src', 'multiarray', 'buffer.h'), join('src', 'multiarray', 'calculation.h'), join('src', 'multiarray', 'common.h'), @@ -822,6 +828,7 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'arraytypes.c.src'), join('src', 'multiarray', 'array_assign_scalar.c'), join('src', 'multiarray', 'array_assign_array.c'), + join('src', 'multiarray', 'arrayfunction_override.c'), join('src', 'multiarray', 'buffer.c'), join('src', 'multiarray', 'calculation.c'), join('src', 'multiarray', 'compiled_base.c'), @@ -888,6 +895,8 @@ def configuration(parent_package='',top_path=None): join('src', 'umath', 'simd.inc.src'), join('src', 'umath', 'loops.h.src'), join('src', 'umath', 'loops.c.src'), + join('src', 'umath', 'matmul.h.src'), + join('src', 'umath', 'matmul.c.src'), join('src', 'umath', 'ufunc_object.c'), join('src', 'umath', 'extobj.c'), join('src', 'umath', 'cpuid.c'), @@ -901,14 +910,15 @@ def configuration(parent_package='',top_path=None): join('include', 'numpy', 'npy_math.h'), join('include', 'numpy', 'halffloat.h'), join('src', 'multiarray', 'common.h'), + join('src', 'multiarray', 'number.h'), join('src', 'common', 'templ_common.h.src'), join('src', 'umath', 'simd.inc.src'), join('src', 'umath', 'override.h'), join(codegen_dir, 'generate_ufunc_api.py'), - ] + ] config.add_extension('_multiarray_umath', - sources=multiarray_src + umath_src + + sources=multiarray_src + umath_src + npymath_sources + common_src + [generate_config_h, generate_numpyconfig_h, @@ -918,7 +928,7 @@ def configuration(parent_package='',top_path=None): generate_umath_c, generate_ufunc_api, ], - depends=deps + multiarray_deps + umath_deps + + depends=deps + multiarray_deps + umath_deps + common_deps, libraries=['npymath', 'npysort'], extra_info=extra_info) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index fde23076b..f8332c362 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -3,11 +3,19 @@ from __future__ import division, absolute_import, print_function __all__ = ['atleast_1d', 'atleast_2d', 'atleast_3d', 'block', 'hstack', 'stack', 'vstack'] +import functools +import operator +import types +import warnings from . import numeric as _nx +from . import overrides from .numeric import array, asanyarray, newaxis from .multiarray import normalize_axis_index -from .overrides import array_function_dispatch + + +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') def _atleast_1d_dispatcher(*arys): @@ -40,13 +48,13 @@ def atleast_1d(*arys): Examples -------- >>> np.atleast_1d(1.0) - array([ 1.]) + array([1.]) >>> x = np.arange(9.0).reshape(3,3) >>> np.atleast_1d(x) - array([[ 0., 1., 2.], - [ 3., 4., 5.], - [ 6., 7., 8.]]) + array([[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]) >>> np.atleast_1d(x) is x True @@ -98,11 +106,11 @@ def atleast_2d(*arys): Examples -------- >>> np.atleast_2d(3.0) - array([[ 3.]]) + array([[3.]]) >>> x = np.arange(3.0) >>> np.atleast_2d(x) - array([[ 0., 1., 2.]]) + array([[0., 1., 2.]]) >>> np.atleast_2d(x).base is x True @@ -158,7 +166,7 @@ def atleast_3d(*arys): Examples -------- >>> np.atleast_3d(3.0) - array([[[ 3.]]]) + array([[[3.]]]) >>> x = np.arange(3.0) >>> np.atleast_3d(x).shape @@ -171,7 +179,7 @@ def atleast_3d(*arys): True >>> for arr in np.atleast_3d([1, 2], [[1, 2]], [[[1, 2]]]): - ... print(arr, arr.shape) + ... print(arr, arr.shape) # doctest: +SKIP ... [[[1] [2]]] (1, 2, 1) @@ -198,11 +206,27 @@ def atleast_3d(*arys): return res -def _vstack_dispatcher(tup): - return tup +def _arrays_for_stack_dispatcher(arrays, stacklevel=4): + if not hasattr(arrays, '__getitem__') and hasattr(arrays, '__iter__'): + warnings.warn('arrays to stack must be passed as a "sequence" type ' + 'such as list or tuple. Support for non-sequence ' + 'iterables such as generators is deprecated as of ' + 'NumPy 1.16 and will raise an error in the future.', + FutureWarning, stacklevel=stacklevel) + return () + return arrays + + +def _warn_for_nonsequence(arrays): + if not overrides.ENABLE_ARRAY_FUNCTION: + _arrays_for_stack_dispatcher(arrays, stacklevel=4) -@array_function_dispatch(_vstack_dispatcher) +def _vhstack_dispatcher(tup): + return _arrays_for_stack_dispatcher(tup) + + +@array_function_dispatch(_vhstack_dispatcher) def vstack(tup): """ Stack arrays in sequence vertically (row wise). @@ -255,14 +279,11 @@ def vstack(tup): [4]]) """ + _warn_for_nonsequence(tup) return _nx.concatenate([atleast_2d(_m) for _m in tup], 0) -def _hstack_dispatcher(tup): - return tup - - -@array_function_dispatch(_hstack_dispatcher) +@array_function_dispatch(_vhstack_dispatcher) def hstack(tup): """ Stack arrays in sequence horizontally (column wise). @@ -310,6 +331,7 @@ def hstack(tup): [3, 4]]) """ + _warn_for_nonsequence(tup) arrs = [atleast_1d(_m) for _m in tup] # As a special case, dimension 0 of 1-dimensional arrays is "horizontal" if arrs and arrs[0].ndim == 1: @@ -319,10 +341,12 @@ def hstack(tup): def _stack_dispatcher(arrays, axis=None, out=None): - for a in arrays: - yield a + arrays = _arrays_for_stack_dispatcher(arrays, stacklevel=6) if out is not None: - yield out + # optimize for the typical case where only arrays is provided + arrays = list(arrays) + arrays.append(out) + return arrays @array_function_dispatch(_stack_dispatcher) @@ -382,11 +406,12 @@ def stack(arrays, axis=0, out=None): [3, 4]]) """ + _warn_for_nonsequence(arrays) arrays = [asanyarray(arr) for arr in arrays] if not arrays: raise ValueError('need at least one array to stack') - shapes = set(arr.shape for arr in arrays) + shapes = {arr.shape for arr in arrays} if len(shapes) != 1: raise ValueError('all input arrays must have the same shape') @@ -432,6 +457,10 @@ def _block_check_depths_match(arrays, parent_index=[]): refer to it, and the last index along the empty axis will be `None`. max_arr_ndim : int The maximum of the ndims of the arrays nested in `arrays`. + final_size: int + The number of elements in the final array. This is used the motivate + the choice of algorithm used using benchmarking wisdom. + """ if type(arrays) is tuple: # not strictly necessary, but saves us from: @@ -450,8 +479,9 @@ def _block_check_depths_match(arrays, parent_index=[]): idxs_ndims = (_block_check_depths_match(arr, parent_index + [i]) for i, arr in enumerate(arrays)) - first_index, max_arr_ndim = next(idxs_ndims) - for index, ndim in idxs_ndims: + first_index, max_arr_ndim, final_size = next(idxs_ndims) + for index, ndim, size in idxs_ndims: + final_size += size if ndim > max_arr_ndim: max_arr_ndim = ndim if len(index) != len(first_index): @@ -466,13 +496,15 @@ def _block_check_depths_match(arrays, parent_index=[]): # propagate our flag that indicates an empty list at the bottom if index[-1] is None: first_index = index - return first_index, max_arr_ndim + + return first_index, max_arr_ndim, final_size elif type(arrays) is list and len(arrays) == 0: # We've 'bottomed out' on an empty list - return parent_index + [None], 0 + return parent_index + [None], 0, 0 else: # We've 'bottomed out' - arrays is either a scalar or an array - return parent_index, _nx.ndim(arrays) + size = _nx.size(arrays) + return parent_index, _nx.ndim(arrays), size def _atleast_nd(a, ndim): @@ -481,9 +513,132 @@ def _atleast_nd(a, ndim): return array(a, ndmin=ndim, copy=False, subok=True) +def _accumulate(values): + # Helper function because Python 2.7 doesn't have + # itertools.accumulate + value = 0 + accumulated = [] + for v in values: + value += v + accumulated.append(value) + return accumulated + + +def _concatenate_shapes(shapes, axis): + """Given array shapes, return the resulting shape and slices prefixes. + + These help in nested concatation. + Returns + ------- + shape: tuple of int + This tuple satisfies: + ``` + shape, _ = _concatenate_shapes([arr.shape for shape in arrs], axis) + shape == concatenate(arrs, axis).shape + ``` + + slice_prefixes: tuple of (slice(start, end), ) + For a list of arrays being concatenated, this returns the slice + in the larger array at axis that needs to be sliced into. + + For example, the following holds: + ``` + ret = concatenate([a, b, c], axis) + _, (sl_a, sl_b, sl_c) = concatenate_slices([a, b, c], axis) + + ret[(slice(None),) * axis + sl_a] == a + ret[(slice(None),) * axis + sl_b] == b + ret[(slice(None),) * axis + sl_c] == c + ``` + + Thses are called slice prefixes since they are used in the recursive + blocking algorithm to compute the left-most slices during the + recursion. Therefore, they must be prepended to rest of the slice + that was computed deeper in the recusion. + + These are returned as tuples to ensure that they can quickly be added + to existing slice tuple without creating a new tuple everytime. + + """ + # Cache a result that will be reused. + shape_at_axis = [shape[axis] for shape in shapes] + + # Take a shape, any shape + first_shape = shapes[0] + first_shape_pre = first_shape[:axis] + first_shape_post = first_shape[axis+1:] + + if any(shape[:axis] != first_shape_pre or + shape[axis+1:] != first_shape_post for shape in shapes): + raise ValueError( + 'Mismatched array shapes in block along axis {}.'.format(axis)) + + shape = (first_shape_pre + (sum(shape_at_axis),) + first_shape[axis+1:]) + + offsets_at_axis = _accumulate(shape_at_axis) + slice_prefixes = [(slice(start, end),) + for start, end in zip([0] + offsets_at_axis, + offsets_at_axis)] + return shape, slice_prefixes + + +def _block_info_recursion(arrays, max_depth, result_ndim, depth=0): + """ + Returns the shape of the final array, along with a list + of slices and a list of arrays that can be used for assignment inside the + new array + + Parameters + ---------- + arrays : nested list of arrays + The arrays to check + max_depth : list of int + The number of nested lists + result_ndim: int + The number of dimensions in thefinal array. + + Returns + ------- + shape : tuple of int + The shape that the final array will take on. + slices: list of tuple of slices + The slices into the full array required for assignment. These are + required to be prepended with ``(Ellipsis, )`` to obtain to correct + final index. + arrays: list of ndarray + The data to assign to each slice of the full array + + """ + if depth < max_depth: + shapes, slices, arrays = zip( + *[_block_info_recursion(arr, max_depth, result_ndim, depth+1) + for arr in arrays]) + + axis = result_ndim - max_depth + depth + shape, slice_prefixes = _concatenate_shapes(shapes, axis) + + # Prepend the slice prefix and flatten the slices + slices = [slice_prefix + the_slice + for slice_prefix, inner_slices in zip(slice_prefixes, slices) + for the_slice in inner_slices] + + # Flatten the array list + arrays = functools.reduce(operator.add, arrays) + + return shape, slices, arrays + else: + # We've 'bottomed out' - arrays is either a scalar or an array + # type(arrays) is not list + # Return the slice and the array inside a list to be consistent with + # the recursive case. + arr = _atleast_nd(arrays, result_ndim) + return arr.shape, [()], [arr] + + def _block(arrays, max_depth, result_ndim, depth=0): """ - Internal implementation of block. `arrays` is the argument passed to + Internal implementation of block based on repeated concatenation. + `arrays` is the argument passed to block. `max_depth` is the depth of nested lists within `arrays` and `result_ndim` is the greatest of the dimensions of the arrays in `arrays` and the depth of the lists in `arrays` (see block docstring @@ -499,7 +654,19 @@ def _block(arrays, max_depth, result_ndim, depth=0): return _atleast_nd(arrays, result_ndim) -# TODO: support array_function_dispatch +def _block_dispatcher(arrays): + # Use type(...) is list to match the behavior of np.block(), which special + # cases list specifically rather than allowing for generic iterables or + # tuple. Also, we know that list.__array_function__ will never exist. + if type(arrays) is list: + for subarrays in arrays: + for subarray in _block_dispatcher(subarrays): + yield subarray + else: + yield arrays + + +@array_function_dispatch(_block_dispatcher) def block(arrays): """ Assemble an nd-array from nested lists of blocks. @@ -594,11 +761,11 @@ def block(arrays): ... [A, np.zeros((2, 3))], ... [np.ones((3, 2)), B ] ... ]) - array([[ 2., 0., 0., 0., 0.], - [ 0., 2., 0., 0., 0.], - [ 1., 1., 3., 0., 0.], - [ 1., 1., 0., 3., 0.], - [ 1., 1., 0., 0., 3.]]) + array([[2., 0., 0., 0., 0.], + [0., 2., 0., 0., 0.], + [1., 1., 3., 0., 0.], + [1., 1., 0., 3., 0.], + [1., 1., 0., 0., 3.]]) With a list of depth 1, `block` can be used as `hstack` @@ -608,7 +775,7 @@ def block(arrays): >>> a = np.array([1, 2, 3]) >>> b = np.array([2, 3, 4]) >>> np.block([a, b, 10]) # hstack([a, b, 10]) - array([1, 2, 3, 2, 3, 4, 10]) + array([ 1, 2, 3, 2, 3, 4, 10]) >>> A = np.ones((2, 2), int) >>> B = 2 * A @@ -648,7 +815,38 @@ def block(arrays): """ - bottom_index, arr_ndim = _block_check_depths_match(arrays) + arrays, list_ndim, result_ndim, final_size = _block_setup(arrays) + + # It was found through benchmarking that making an array of final size + # around 256x256 was faster by straight concatenation on a + # i7-7700HQ processor and dual channel ram 2400MHz. + # It didn't seem to matter heavily on the dtype used. + # + # A 2D array using repeated concatenation requires 2 copies of the array. + # + # The fastest algorithm will depend on the ratio of CPU power to memory + # speed. + # One can monitor the results of the benchmark + # https://pv.github.io/numpy-bench/#bench_shape_base.Block2D.time_block2d + # to tune this parameter until a C version of the `_block_info_recursion` + # algorithm is implemented which would likely be faster than the python + # version. + if list_ndim * final_size > (2 * 512 * 512): + return _block_slicing(arrays, list_ndim, result_ndim) + else: + return _block_concatenate(arrays, list_ndim, result_ndim) + + +# Theses helper functions are mostly used for testing. +# They allow us to write tests that directly call `_block_slicing` +# or `_block_concatenate` wtihout blocking large arrays to forse the wisdom +# to trigger the desired path. +def _block_setup(arrays): + """ + Returns + (`arrays`, list_ndim, result_ndim, final_size) + """ + bottom_index, arr_ndim, final_size = _block_check_depths_match(arrays) list_ndim = len(bottom_index) if bottom_index and bottom_index[-1] is None: raise ValueError( @@ -656,7 +854,31 @@ def block(arrays): _block_format_index(bottom_index) ) ) - result = _block(arrays, list_ndim, max(arr_ndim, list_ndim)) + result_ndim = max(arr_ndim, list_ndim) + return arrays, list_ndim, result_ndim, final_size + + +def _block_slicing(arrays, list_ndim, result_ndim): + shape, slices, arrays = _block_info_recursion( + arrays, list_ndim, result_ndim) + dtype = _nx.result_type(*[arr.dtype for arr in arrays]) + + # Test preferring F only in the case that all input arrays are F + F_order = all(arr.flags['F_CONTIGUOUS'] for arr in arrays) + C_order = all(arr.flags['C_CONTIGUOUS'] for arr in arrays) + order = 'F' if F_order and not C_order else 'C' + result = _nx.empty(shape=shape, dtype=dtype, order=order) + # Note: In a c implementation, the function + # PyArray_CreateMultiSortedStridePerm could be used for more advanced + # guessing of the desired order. + + for the_slice, arr in zip(slices, arrays): + result[(Ellipsis,) + the_slice] = arr + return result + + +def _block_concatenate(arrays, list_ndim, result_ndim): + result = _block(arrays, list_ndim, result_ndim) if list_ndim == 0: # Catch an edge case where _block returns a view because # `arrays` is a single numpy array and not a list of numpy arrays. diff --git a/numpy/core/src/common/array_assign.c b/numpy/core/src/common/array_assign.c index ac3fdbef7..02a423e3a 100644 --- a/numpy/core/src/common/array_assign.c +++ b/numpy/core/src/common/array_assign.c @@ -125,9 +125,13 @@ raw_array_is_aligned(int ndim, npy_intp *shape, return npy_is_aligned((void *)align_check, alignment); } - else { + else if (alignment == 1) { return 1; } + else { + /* always return false for alignment == 0, which means cannot-be-aligned */ + return 0; + } } NPY_NO_EXPORT int diff --git a/numpy/core/src/common/array_assign.h b/numpy/core/src/common/array_assign.h index 07438c5e8..69ef56bb4 100644 --- a/numpy/core/src/common/array_assign.h +++ b/numpy/core/src/common/array_assign.h @@ -87,8 +87,10 @@ broadcast_strides(int ndim, npy_intp *shape, /* * Checks whether a data pointer + set of strides refers to a raw - * array whose elements are all aligned to a given alignment. - * alignment should be a power of two. + * array whose elements are all aligned to a given alignment. Returns + * 1 if data is aligned to alignment or 0 if not. + * alignment should be a power of two, or may be the sentinel value 0 to mean + * cannot-be-aligned, in which case 0 (false) is always returned. */ NPY_NO_EXPORT int raw_array_is_aligned(int ndim, npy_intp *shape, diff --git a/numpy/core/src/common/cblasfuncs.c b/numpy/core/src/common/cblasfuncs.c index 6460c5db1..39572fed4 100644 --- a/numpy/core/src/common/cblasfuncs.c +++ b/numpy/core/src/common/cblasfuncs.c @@ -182,12 +182,13 @@ _select_matrix_shape(PyArrayObject *array) * This also makes sure that the data segment is aligned with * an itemsize address as well by returning one if not true. */ -static int +NPY_NO_EXPORT int _bad_strides(PyArrayObject *ap) { int itemsize = PyArray_ITEMSIZE(ap); int i, N=PyArray_NDIM(ap); npy_intp *strides = PyArray_STRIDES(ap); + npy_intp *dims = PyArray_DIMS(ap); if (((npy_intp)(PyArray_DATA(ap)) % itemsize) != 0) { return 1; @@ -196,12 +197,14 @@ _bad_strides(PyArrayObject *ap) if ((strides[i] < 0) || (strides[i] % itemsize) != 0) { return 1; } + if ((strides[i] == 0 && dims[i] > 1)) { + return 1; + } } return 0; } - /* * dot(a,b) * Returns the dot product of a and b for arrays of floating point types. diff --git a/numpy/core/src/common/get_attr_string.h b/numpy/core/src/common/get_attr_string.h index bec87c5ed..d458d9550 100644 --- a/numpy/core/src/common/get_attr_string.h +++ b/numpy/core/src/common/get_attr_string.h @@ -103,7 +103,6 @@ PyArray_LookupSpecial(PyObject *obj, char *name) if (_is_basic_python_type(tp)) { return NULL; } - return maybe_get_attr((PyObject *)tp, name); } diff --git a/numpy/core/src/common/npy_ctypes.h b/numpy/core/src/common/npy_ctypes.h new file mode 100644 index 000000000..f26db9e05 --- /dev/null +++ b/numpy/core/src/common/npy_ctypes.h @@ -0,0 +1,49 @@ +#ifndef NPY_CTYPES_H +#define NPY_CTYPES_H + +#include <Python.h> + +#include "npy_import.h" + +/* + * Check if a python type is a ctypes class. + * + * Works like the Py<type>_Check functions, returning true if the argument + * looks like a ctypes object. + * + * This entire function is just a wrapper around the Python function of the + * same name. + */ +NPY_INLINE static int +npy_ctypes_check(PyTypeObject *obj) +{ + static PyObject *py_func = NULL; + PyObject *ret_obj; + int ret; + + npy_cache_import("numpy.core._internal", "npy_ctypes_check", &py_func); + if (py_func == NULL) { + goto fail; + } + + ret_obj = PyObject_CallFunctionObjArgs(py_func, (PyObject *)obj, NULL); + if (ret_obj == NULL) { + goto fail; + } + + ret = PyObject_IsTrue(ret_obj); + if (ret == -1) { + goto fail; + } + + return ret; + +fail: + /* If the above fails, then we should just assume that the type is not from + * ctypes + */ + PyErr_Clear(); + return 0; +} + +#endif diff --git a/numpy/core/src/common/npy_sort.h b/numpy/core/src/common/npy_sort.h deleted file mode 100644 index 8c6f05623..000000000 --- a/numpy/core/src/common/npy_sort.h +++ /dev/null @@ -1,204 +0,0 @@ -#ifndef __NPY_SORT_H__ -#define __NPY_SORT_H__ - -/* Python include is for future object sorts */ -#include <Python.h> -#include <numpy/npy_common.h> -#include <numpy/ndarraytypes.h> - -#define NPY_ENOMEM 1 -#define NPY_ECOMP 2 - -static NPY_INLINE int npy_get_msb(npy_uintp unum) -{ - int depth_limit = 0; - while (unum >>= 1) { - depth_limit++; - } - return depth_limit; -} - -int quicksort_bool(void *vec, npy_intp cnt, void *null); -int heapsort_bool(void *vec, npy_intp cnt, void *null); -int mergesort_bool(void *vec, npy_intp cnt, void *null); -int aquicksort_bool(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_bool(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_bool(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_byte(void *vec, npy_intp cnt, void *null); -int heapsort_byte(void *vec, npy_intp cnt, void *null); -int mergesort_byte(void *vec, npy_intp cnt, void *null); -int aquicksort_byte(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_byte(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_byte(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_ubyte(void *vec, npy_intp cnt, void *null); -int heapsort_ubyte(void *vec, npy_intp cnt, void *null); -int mergesort_ubyte(void *vec, npy_intp cnt, void *null); -int aquicksort_ubyte(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_ubyte(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_ubyte(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_short(void *vec, npy_intp cnt, void *null); -int heapsort_short(void *vec, npy_intp cnt, void *null); -int mergesort_short(void *vec, npy_intp cnt, void *null); -int aquicksort_short(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_short(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_short(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_ushort(void *vec, npy_intp cnt, void *null); -int heapsort_ushort(void *vec, npy_intp cnt, void *null); -int mergesort_ushort(void *vec, npy_intp cnt, void *null); -int aquicksort_ushort(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_ushort(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_ushort(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_int(void *vec, npy_intp cnt, void *null); -int heapsort_int(void *vec, npy_intp cnt, void *null); -int mergesort_int(void *vec, npy_intp cnt, void *null); -int aquicksort_int(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_int(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_int(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_uint(void *vec, npy_intp cnt, void *null); -int heapsort_uint(void *vec, npy_intp cnt, void *null); -int mergesort_uint(void *vec, npy_intp cnt, void *null); -int aquicksort_uint(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_uint(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_uint(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_long(void *vec, npy_intp cnt, void *null); -int heapsort_long(void *vec, npy_intp cnt, void *null); -int mergesort_long(void *vec, npy_intp cnt, void *null); -int aquicksort_long(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_long(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_long(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_ulong(void *vec, npy_intp cnt, void *null); -int heapsort_ulong(void *vec, npy_intp cnt, void *null); -int mergesort_ulong(void *vec, npy_intp cnt, void *null); -int aquicksort_ulong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_ulong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_ulong(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_longlong(void *vec, npy_intp cnt, void *null); -int heapsort_longlong(void *vec, npy_intp cnt, void *null); -int mergesort_longlong(void *vec, npy_intp cnt, void *null); -int aquicksort_longlong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_longlong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_longlong(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_ulonglong(void *vec, npy_intp cnt, void *null); -int heapsort_ulonglong(void *vec, npy_intp cnt, void *null); -int mergesort_ulonglong(void *vec, npy_intp cnt, void *null); -int aquicksort_ulonglong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_ulonglong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_ulonglong(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_half(void *vec, npy_intp cnt, void *null); -int heapsort_half(void *vec, npy_intp cnt, void *null); -int mergesort_half(void *vec, npy_intp cnt, void *null); -int aquicksort_half(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_half(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_half(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_float(void *vec, npy_intp cnt, void *null); -int heapsort_float(void *vec, npy_intp cnt, void *null); -int mergesort_float(void *vec, npy_intp cnt, void *null); -int aquicksort_float(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_float(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_float(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_double(void *vec, npy_intp cnt, void *null); -int heapsort_double(void *vec, npy_intp cnt, void *null); -int mergesort_double(void *vec, npy_intp cnt, void *null); -int aquicksort_double(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_double(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_double(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_longdouble(void *vec, npy_intp cnt, void *null); -int heapsort_longdouble(void *vec, npy_intp cnt, void *null); -int mergesort_longdouble(void *vec, npy_intp cnt, void *null); -int aquicksort_longdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_longdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_longdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_cfloat(void *vec, npy_intp cnt, void *null); -int heapsort_cfloat(void *vec, npy_intp cnt, void *null); -int mergesort_cfloat(void *vec, npy_intp cnt, void *null); -int aquicksort_cfloat(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_cfloat(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_cfloat(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_cdouble(void *vec, npy_intp cnt, void *null); -int heapsort_cdouble(void *vec, npy_intp cnt, void *null); -int mergesort_cdouble(void *vec, npy_intp cnt, void *null); -int aquicksort_cdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_cdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_cdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_clongdouble(void *vec, npy_intp cnt, void *null); -int heapsort_clongdouble(void *vec, npy_intp cnt, void *null); -int mergesort_clongdouble(void *vec, npy_intp cnt, void *null); -int aquicksort_clongdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_clongdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_clongdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_string(void *vec, npy_intp cnt, void *arr); -int heapsort_string(void *vec, npy_intp cnt, void *arr); -int mergesort_string(void *vec, npy_intp cnt, void *arr); -int aquicksort_string(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int aheapsort_string(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int amergesort_string(void *vec, npy_intp *ind, npy_intp cnt, void *arr); - - -int quicksort_unicode(void *vec, npy_intp cnt, void *arr); -int heapsort_unicode(void *vec, npy_intp cnt, void *arr); -int mergesort_unicode(void *vec, npy_intp cnt, void *arr); -int aquicksort_unicode(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int aheapsort_unicode(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int amergesort_unicode(void *vec, npy_intp *ind, npy_intp cnt, void *arr); - - -int quicksort_datetime(void *vec, npy_intp cnt, void *null); -int heapsort_datetime(void *vec, npy_intp cnt, void *null); -int mergesort_datetime(void *vec, npy_intp cnt, void *null); -int aquicksort_datetime(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_datetime(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_datetime(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_timedelta(void *vec, npy_intp cnt, void *null); -int heapsort_timedelta(void *vec, npy_intp cnt, void *null); -int mergesort_timedelta(void *vec, npy_intp cnt, void *null); -int aquicksort_timedelta(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_timedelta(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_timedelta(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int npy_quicksort(void *vec, npy_intp cnt, void *arr); -int npy_heapsort(void *vec, npy_intp cnt, void *arr); -int npy_mergesort(void *vec, npy_intp cnt, void *arr); -int npy_aquicksort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int npy_aheapsort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int npy_amergesort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); - -#endif diff --git a/numpy/core/src/common/npy_sort.h.src b/numpy/core/src/common/npy_sort.h.src new file mode 100644 index 000000000..c31a82764 --- /dev/null +++ b/numpy/core/src/common/npy_sort.h.src @@ -0,0 +1,83 @@ +#ifndef __NPY_SORT_H__ +#define __NPY_SORT_H__ + +/* Python include is for future object sorts */ +#include <Python.h> +#include <numpy/npy_common.h> +#include <numpy/ndarraytypes.h> + +#define NPY_ENOMEM 1 +#define NPY_ECOMP 2 + +static NPY_INLINE int npy_get_msb(npy_uintp unum) +{ + int depth_limit = 0; + while (unum >>= 1) { + depth_limit++; + } + return depth_limit; +} + + +/* + ***************************************************************************** + ** NUMERIC SORTS ** + ***************************************************************************** + */ + + +/**begin repeat + * + * #suff = bool, byte, ubyte, short, ushort, int, uint, long, ulong, + * longlong, ulonglong, half, float, double, longdouble, + * cfloat, cdouble, clongdouble, datetime, timedelta# + */ + +int quicksort_@suff@(void *vec, npy_intp cnt, void *null); +int heapsort_@suff@(void *vec, npy_intp cnt, void *null); +int mergesort_@suff@(void *vec, npy_intp cnt, void *null); +int aquicksort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *null); +int aheapsort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *null); +int amergesort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *null); + +/**end repeat**/ + + + +/* + ***************************************************************************** + ** STRING SORTS ** + ***************************************************************************** + */ + + +/**begin repeat + * + * #suff = string, unicode# + */ + +int quicksort_@suff@(void *vec, npy_intp cnt, void *arr); +int heapsort_@suff@(void *vec, npy_intp cnt, void *arr); +int mergesort_@suff@(void *vec, npy_intp cnt, void *arr); +int aquicksort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *arr); +int aheapsort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *arr); +int amergesort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *arr); + +/**end repeat**/ + + +/* + ***************************************************************************** + ** GENERIC SORT ** + ***************************************************************************** + */ + + +int npy_quicksort(void *vec, npy_intp cnt, void *arr); +int npy_heapsort(void *vec, npy_intp cnt, void *arr); +int npy_mergesort(void *vec, npy_intp cnt, void *arr); +int npy_aquicksort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); +int npy_aheapsort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); +int npy_amergesort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); + +#endif diff --git a/numpy/core/src/common/ufunc_override.c b/numpy/core/src/common/ufunc_override.c index 33b54c665..89f08a9cb 100644 --- a/numpy/core/src/common/ufunc_override.c +++ b/numpy/core/src/common/ufunc_override.c @@ -1,10 +1,9 @@ #define NPY_NO_DEPRECATED_API NPY_API_VERSION -#define NO_IMPORT_ARRAY +#define _MULTIARRAYMODULE #include "npy_pycompat.h" #include "get_attr_string.h" #include "npy_import.h" - #include "ufunc_override.h" /* @@ -12,45 +11,39 @@ * is not the default, i.e., the object is not an ndarray, and its * __array_ufunc__ is not the same as that of ndarray. * - * Returns a new reference, the value of type(obj).__array_ufunc__ - * - * If the __array_ufunc__ matches that of ndarray, or does not exist, return - * NULL. - * - * Note that since this module is used with both multiarray and umath, we do - * not have access to PyArray_Type and therewith neither to PyArray_CheckExact - * nor to the default __array_ufunc__ method, so instead we import locally. - * TODO: Can this really not be done more smartly? + * Returns a new reference, the value of type(obj).__array_ufunc__ if it + * exists and is different from that of ndarray, and NULL otherwise. */ NPY_NO_EXPORT PyObject * -get_non_default_array_ufunc(PyObject *obj) +PyUFuncOverride_GetNonDefaultArrayUfunc(PyObject *obj) { - static PyObject *ndarray = NULL; static PyObject *ndarray_array_ufunc = NULL; PyObject *cls_array_ufunc; - /* on first entry, import and cache ndarray and its __array_ufunc__ */ - if (ndarray == NULL) { - npy_cache_import("numpy.core.multiarray", "ndarray", &ndarray); - ndarray_array_ufunc = PyObject_GetAttrString(ndarray, + /* On first entry, cache ndarray's __array_ufunc__ */ + if (ndarray_array_ufunc == NULL) { + ndarray_array_ufunc = PyObject_GetAttrString((PyObject *)&PyArray_Type, "__array_ufunc__"); } /* Fast return for ndarray */ - if ((PyObject *)Py_TYPE(obj) == ndarray) { + if (PyArray_CheckExact(obj)) { return NULL; } - /* does the class define __array_ufunc__? */ + /* + * Does the class define __array_ufunc__? (Note that LookupSpecial has fast + * return for basic python types, so no need to worry about those here) + */ cls_array_ufunc = PyArray_LookupSpecial(obj, "__array_ufunc__"); if (cls_array_ufunc == NULL) { return NULL; } - /* is it different from ndarray.__array_ufunc__? */ - if (cls_array_ufunc != ndarray_array_ufunc) { - return cls_array_ufunc; + /* Ignore if the same as ndarray.__array_ufunc__ */ + if (cls_array_ufunc == ndarray_array_ufunc) { + Py_DECREF(cls_array_ufunc); + return NULL; } - Py_DECREF(cls_array_ufunc); - return NULL; + return cls_array_ufunc; } /* @@ -62,9 +55,9 @@ get_non_default_array_ufunc(PyObject *obj) */ NPY_NO_EXPORT int -has_non_default_array_ufunc(PyObject * obj) +PyUFunc_HasOverride(PyObject * obj) { - PyObject *method = get_non_default_array_ufunc(obj); + PyObject *method = PyUFuncOverride_GetNonDefaultArrayUfunc(obj); if (method) { Py_DECREF(method); return 1; @@ -78,164 +71,51 @@ has_non_default_array_ufunc(PyObject * obj) * Get possible out argument from kwds, and returns the number of outputs * contained within it: if a tuple, the number of elements in it, 1 otherwise. * The out argument itself is returned in out_kwd_obj, and the outputs - * in the out_obj array (all as borrowed references). + * in the out_obj array (as borrowed references). * - * Returns -1 if kwds is not a dict, 0 if no outputs found. + * Returns 0 if no outputs found, -1 if kwds is not a dict (with an error set). */ -static int -get_out_objects(PyObject *kwds, PyObject **out_kwd_obj, PyObject ***out_objs) +NPY_NO_EXPORT int +PyUFuncOverride_GetOutObjects(PyObject *kwds, PyObject **out_kwd_obj, PyObject ***out_objs) { if (kwds == NULL) { + Py_INCREF(Py_None); + *out_kwd_obj = Py_None; return 0; } if (!PyDict_CheckExact(kwds)) { PyErr_SetString(PyExc_TypeError, - "Internal Numpy error: call to PyUFunc_WithOverride " + "Internal Numpy error: call to PyUFuncOverride_GetOutObjects " "with non-dict kwds"); + *out_kwd_obj = NULL; return -1; } /* borrowed reference */ *out_kwd_obj = PyDict_GetItemString(kwds, "out"); if (*out_kwd_obj == NULL) { + Py_INCREF(Py_None); + *out_kwd_obj = Py_None; return 0; } if (PyTuple_CheckExact(*out_kwd_obj)) { - *out_objs = PySequence_Fast_ITEMS(*out_kwd_obj); - return PySequence_Fast_GET_SIZE(*out_kwd_obj); - } - else { - *out_objs = out_kwd_obj; - return 1; - } -} - -/* - * For each positional argument and each argument in a possible "out" - * keyword, look for overrides of the standard ufunc behaviour, i.e., - * non-default __array_ufunc__ methods. - * - * Returns the number of overrides, setting corresponding objects - * in PyObject array ``with_override`` and the corresponding - * __array_ufunc__ methods in ``methods`` (both using new references). - * - * Only the first override for a given class is returned. - * - * returns -1 on failure. - */ -NPY_NO_EXPORT int -PyUFunc_WithOverride(PyObject *args, PyObject *kwds, - PyObject **with_override, PyObject **methods) -{ - int i; - int num_override_args = 0; - int narg, nout = 0; - PyObject *out_kwd_obj; - PyObject **arg_objs, **out_objs; - - narg = PyTuple_Size(args); - if (narg < 0) { - return -1; - } - arg_objs = PySequence_Fast_ITEMS(args); - - nout = get_out_objects(kwds, &out_kwd_obj, &out_objs); - if (nout < 0) { - return -1; - } - - for (i = 0; i < narg + nout; ++i) { - PyObject *obj; - int j; - int new_class = 1; - - if (i < narg) { - obj = arg_objs[i]; - } - else { - obj = out_objs[i - narg]; - } /* - * Have we seen this class before? If so, ignore. + * The C-API recommends calling PySequence_Fast before any of the other + * PySequence_Fast* functions. This is required for PyPy */ - for (j = 0; j < num_override_args; j++) { - new_class = (Py_TYPE(obj) != Py_TYPE(with_override[j])); - if (!new_class) { - break; - } - } - if (new_class) { - /* - * Now see if the object provides an __array_ufunc__. However, we should - * ignore the base ndarray.__ufunc__, so we skip any ndarray as well as - * any ndarray subclass instances that did not override __array_ufunc__. - */ - PyObject *method = get_non_default_array_ufunc(obj); - if (method == NULL) { - continue; - } - if (method == Py_None) { - PyErr_Format(PyExc_TypeError, - "operand '%.200s' does not support ufuncs " - "(__array_ufunc__=None)", - obj->ob_type->tp_name); - Py_DECREF(method); - goto fail; - } - Py_INCREF(obj); - with_override[num_override_args] = obj; - methods[num_override_args] = method; - ++num_override_args; + PyObject *seq; + seq = PySequence_Fast(*out_kwd_obj, + "Could not convert object to sequence"); + if (seq == NULL) { + *out_kwd_obj = NULL; + return -1; } + *out_objs = PySequence_Fast_ITEMS(seq); + *out_kwd_obj = seq; + return PySequence_Fast_GET_SIZE(seq); } - return num_override_args; - -fail: - for (i = 0; i < num_override_args; i++) { - Py_DECREF(with_override[i]); - Py_DECREF(methods[i]); - } - return -1; -} - -/* - * Check whether any of a set of input and output args have a non-default - * __array_ufunc__ method. Return 1 if so, 0 if not. - * - * This function primarily exists to help ndarray.__array_ufunc__ determine - * whether it can support a ufunc (which is the case only if none of the - * operands have an override). Thus, unlike in PyUFunc_CheckOverride, the - * actual overrides are not needed and one can stop looking once one is found. - * - * TODO: move this function and has_non_default_array_ufunc closer to ndarray. - */ -NPY_NO_EXPORT int -PyUFunc_HasOverride(PyObject *args, PyObject *kwds) -{ - int i; - int nin, nout; - PyObject *out_kwd_obj; - PyObject **in_objs, **out_objs; - - /* check inputs */ - nin = PyTuple_Size(args); - if (nin < 0) { - return -1; - } - in_objs = PySequence_Fast_ITEMS(args); - for (i = 0; i < nin; ++i) { - if (has_non_default_array_ufunc(in_objs[i])) { - return 1; - } - } - /* check outputs, if any */ - nout = get_out_objects(kwds, &out_kwd_obj, &out_objs); - if (nout < 0) { - return -1; - } - for (i = 0; i < nout; i++) { - if (has_non_default_array_ufunc(out_objs[i])) { - return 1; - } + else { + Py_INCREF(*out_kwd_obj); + *out_objs = out_kwd_obj; + return 1; } - return 0; } diff --git a/numpy/core/src/common/ufunc_override.h b/numpy/core/src/common/ufunc_override.h index 5b269d270..bf86865c9 100644 --- a/numpy/core/src/common/ufunc_override.h +++ b/numpy/core/src/common/ufunc_override.h @@ -8,18 +8,11 @@ * is not the default, i.e., the object is not an ndarray, and its * __array_ufunc__ is not the same as that of ndarray. * - * Returns a new reference, the value of type(obj).__array_ufunc__ - * - * If the __array_ufunc__ matches that of ndarray, or does not exist, return - * NULL. - * - * Note that since this module is used with both multiarray and umath, we do - * not have access to PyArray_Type and therewith neither to PyArray_CheckExact - * nor to the default __array_ufunc__ method, so instead we import locally. - * TODO: Can this really not be done more smartly? + * Returns a new reference, the value of type(obj).__array_ufunc__ if it + * exists and is different from that of ndarray, and NULL otherwise. */ NPY_NO_EXPORT PyObject * -get_non_default_array_ufunc(PyObject *obj); +PyUFuncOverride_GetNonDefaultArrayUfunc(PyObject *obj); /* * Check whether an object has __array_ufunc__ defined on its class and it @@ -29,18 +22,16 @@ get_non_default_array_ufunc(PyObject *obj); * Returns 1 if this is the case, 0 if not. */ NPY_NO_EXPORT int -has_non_default_array_ufunc(PyObject * obj); +PyUFunc_HasOverride(PyObject *obj); /* - * Check whether a set of input and output args have a non-default - * `__array_ufunc__` method. Returns the number of overrides, setting - * corresponding objects in PyObject array with_override (if not NULL). - * returns -1 on failure. + * Get possible out argument from kwds, and returns the number of outputs + * contained within it: if a tuple, the number of elements in it, 1 otherwise. + * The out argument itself is returned in out_kwd_obj, and the outputs + * in the out_obj array (as borrowed references). + * + * Returns 0 if no outputs found, -1 if kwds is not a dict (with an error set). */ NPY_NO_EXPORT int -PyUFunc_WithOverride(PyObject *args, PyObject *kwds, - PyObject **with_override, PyObject **methods); - -NPY_NO_EXPORT int -PyUFunc_HasOverride(PyObject *args, PyObject *kwds); +PyUFuncOverride_GetOutObjects(PyObject *kwds, PyObject **out_kwd_obj, PyObject ***out_objs); #endif diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index 6c4d49bd1..c26bd16ac 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -11,6 +11,13 @@ #include "npy_extint128.h" #include "common.h" + +#if defined(MS_WIN32) || defined(__CYGWIN__) +#define EXPORT(x) __declspec(dllexport) x +#else +#define EXPORT(x) x +#endif + #define ARRAY_SIZE(a) (sizeof(a)/sizeof(a[0])) /* test PyArray_IsPythonScalar, before including private py3 compat header */ @@ -31,6 +38,12 @@ IsPythonScalar(PyObject * dummy, PyObject *args) #include "npy_pycompat.h" +/** Function to test calling via ctypes */ +EXPORT(void*) forward_pointer(void *x) +{ + return x; +} + /* * TODO: * - Handle mode @@ -1855,6 +1868,19 @@ printf_float_g(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) return PrintFloat_Printf_g(obj, precision); } +static PyObject * +getset_numericops(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) +{ + PyObject *ret; + PyObject *ops = PyArray_GetNumericOps(); + if (ops == NULL) { + return NULL; + } + ret = PyLong_FromLong(PyArray_SetNumericOps(ops)); + Py_DECREF(ops); + return ret; +} + static PyMethodDef Multiarray_TestsMethods[] = { {"IsPythonScalar", IsPythonScalar, @@ -1963,6 +1989,9 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"get_fpu_mode", get_fpu_mode, METH_VARARGS, get_fpu_mode_doc}, + {"getset_numericops", + getset_numericops, + METH_NOARGS, NULL}, /**begin repeat * #name = cabs, carg# */ @@ -2040,3 +2069,9 @@ init_multiarray_tests(void) } return RETVAL; } + +NPY_NO_EXPORT int +test_not_exported(void) +{ + return 1; +} diff --git a/numpy/core/src/multiarray/array_assign_array.c b/numpy/core/src/multiarray/array_assign_array.c index f692e0307..69a527dfa 100644 --- a/numpy/core/src/multiarray/array_assign_array.c +++ b/numpy/core/src/multiarray/array_assign_array.c @@ -48,11 +48,15 @@ raw_array_assign_array(int ndim, npy_intp *shape, NPY_BEGIN_THREADS_DEF; - /* Check alignment */ + /* Check both uint and true alignment */ aligned = raw_array_is_aligned(ndim, shape, dst_data, dst_strides, npy_uint_alignment(dst_dtype->elsize)) && + raw_array_is_aligned(ndim, shape, dst_data, dst_strides, + dst_dtype->alignment) && raw_array_is_aligned(ndim, shape, src_data, src_strides, npy_uint_alignment(src_dtype->elsize)); + raw_array_is_aligned(ndim, shape, src_data, src_strides, + src_dtype->alignment); /* Use raw iteration with no heap allocation */ if (PyArray_PrepareTwoRawArrayIter( @@ -133,11 +137,15 @@ raw_array_wheremasked_assign_array(int ndim, npy_intp *shape, NPY_BEGIN_THREADS_DEF; - /* Check alignment */ + /* Check both uint and true alignment */ aligned = raw_array_is_aligned(ndim, shape, dst_data, dst_strides, npy_uint_alignment(dst_dtype->elsize)) && + raw_array_is_aligned(ndim, shape, dst_data, dst_strides, + dst_dtype->alignment) && raw_array_is_aligned(ndim, shape, src_data, src_strides, npy_uint_alignment(src_dtype->elsize)); + raw_array_is_aligned(ndim, shape, src_data, src_strides, + src_dtype->alignment); /* Use raw iteration with no heap allocation */ if (PyArray_PrepareThreeRawArrayIter( diff --git a/numpy/core/src/multiarray/array_assign_scalar.c b/numpy/core/src/multiarray/array_assign_scalar.c index 841a41850..ecb5be47b 100644 --- a/numpy/core/src/multiarray/array_assign_scalar.c +++ b/numpy/core/src/multiarray/array_assign_scalar.c @@ -45,10 +45,13 @@ raw_array_assign_scalar(int ndim, npy_intp *shape, NPY_BEGIN_THREADS_DEF; - /* Check alignment */ + /* Check both uint and true alignment */ aligned = raw_array_is_aligned(ndim, shape, dst_data, dst_strides, npy_uint_alignment(dst_dtype->elsize)) && - npy_is_aligned(src_data, npy_uint_alignment(src_dtype->elsize)); + raw_array_is_aligned(ndim, shape, dst_data, dst_strides, + dst_dtype->alignment) && + npy_is_aligned(src_data, npy_uint_alignment(src_dtype->elsize) && + npy_is_aligned(src_data, src_dtype->alignment)); /* Use raw iteration with no heap allocation */ if (PyArray_PrepareOneRawArrayIter( @@ -116,10 +119,13 @@ raw_array_wheremasked_assign_scalar(int ndim, npy_intp *shape, NPY_BEGIN_THREADS_DEF; - /* Check alignment */ + /* Check both uint and true alignment */ aligned = raw_array_is_aligned(ndim, shape, dst_data, dst_strides, npy_uint_alignment(dst_dtype->elsize)) && - npy_is_aligned(src_data, npy_uint_alignment(src_dtype->elsize)); + raw_array_is_aligned(ndim, shape, dst_data, dst_strides, + dst_dtype->alignment) && + npy_is_aligned(src_data, npy_uint_alignment(src_dtype->elsize) && + npy_is_aligned(src_data, src_dtype->alignment)); /* Use raw iteration with no heap allocation */ if (PyArray_PrepareTwoRawArrayIter( @@ -220,7 +226,8 @@ PyArray_AssignRawScalar(PyArrayObject *dst, * we also skip this if 'dst' has an object dtype. */ if ((!PyArray_EquivTypes(PyArray_DESCR(dst), src_dtype) || - !npy_is_aligned(src_data, npy_uint_alignment(src_dtype->elsize))) && + !(npy_is_aligned(src_data, npy_uint_alignment(src_dtype->elsize)) && + npy_is_aligned(src_data, src_dtype->alignment))) && PyArray_SIZE(dst) > 1 && !PyDataType_REFCHK(PyArray_DESCR(dst))) { char *tmp_src_data; diff --git a/numpy/core/src/multiarray/arrayfunction_override.c b/numpy/core/src/multiarray/arrayfunction_override.c new file mode 100644 index 000000000..e62b32ab2 --- /dev/null +++ b/numpy/core/src/multiarray/arrayfunction_override.c @@ -0,0 +1,376 @@ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE + +#include "npy_pycompat.h" +#include "get_attr_string.h" +#include "npy_import.h" +#include "multiarraymodule.h" + + +/* Return the ndarray.__array_function__ method. */ +static PyObject * +get_ndarray_array_function(void) +{ + PyObject* method = PyObject_GetAttrString((PyObject *)&PyArray_Type, + "__array_function__"); + assert(method != NULL); + return method; +} + + +/* + * Get an object's __array_function__ method in the fastest way possible. + * Never raises an exception. Returns NULL if the method doesn't exist. + */ +static PyObject * +get_array_function(PyObject *obj) +{ + static PyObject *ndarray_array_function = NULL; + + if (ndarray_array_function == NULL) { + ndarray_array_function = get_ndarray_array_function(); + } + + /* Fast return for ndarray */ + if (PyArray_CheckExact(obj)) { + Py_INCREF(ndarray_array_function); + return ndarray_array_function; + } + + return PyArray_LookupSpecial(obj, "__array_function__"); +} + + +/* + * Like list.insert(), but for C arrays of PyObject*. Skips error checking. + */ +static void +pyobject_array_insert(PyObject **array, int length, int index, PyObject *item) +{ + int j; + + for (j = length; j > index; j--) { + array[j] = array[j - 1]; + } + array[index] = item; +} + + +/* + * Collects arguments with __array_function__ and their corresponding methods + * in the order in which they should be tried (i.e., skipping redundant types). + * `relevant_args` is expected to have been produced by PySequence_Fast. + * Returns the number of arguments, or -1 on failure. + */ +static int +get_implementing_args_and_methods(PyObject *relevant_args, + PyObject **implementing_args, + PyObject **methods) +{ + int num_implementing_args = 0; + Py_ssize_t i; + int j; + + PyObject **items = PySequence_Fast_ITEMS(relevant_args); + Py_ssize_t length = PySequence_Fast_GET_SIZE(relevant_args); + + for (i = 0; i < length; i++) { + int new_class = 1; + PyObject *argument = items[i]; + + /* Have we seen this type before? */ + for (j = 0; j < num_implementing_args; j++) { + if (Py_TYPE(argument) == Py_TYPE(implementing_args[j])) { + new_class = 0; + break; + } + } + if (new_class) { + PyObject *method = get_array_function(argument); + + if (method != NULL) { + int arg_index; + + if (num_implementing_args >= NPY_MAXARGS) { + PyErr_Format( + PyExc_TypeError, + "maximum number (%d) of distinct argument types " \ + "implementing __array_function__ exceeded", + NPY_MAXARGS); + Py_DECREF(method); + goto fail; + } + + /* "subclasses before superclasses, otherwise left to right" */ + arg_index = num_implementing_args; + for (j = 0; j < num_implementing_args; j++) { + PyObject *other_type; + other_type = (PyObject *)Py_TYPE(implementing_args[j]); + if (PyObject_IsInstance(argument, other_type)) { + arg_index = j; + break; + } + } + Py_INCREF(argument); + pyobject_array_insert(implementing_args, num_implementing_args, + arg_index, argument); + pyobject_array_insert(methods, num_implementing_args, + arg_index, method); + ++num_implementing_args; + } + } + } + return num_implementing_args; + +fail: + for (j = 0; j < num_implementing_args; j++) { + Py_DECREF(implementing_args[j]); + Py_DECREF(methods[j]); + } + return -1; +} + + +/* + * Is this object ndarray.__array_function__? + */ +static int +is_default_array_function(PyObject *obj) +{ + static PyObject *ndarray_array_function = NULL; + + if (ndarray_array_function == NULL) { + ndarray_array_function = get_ndarray_array_function(); + } + return obj == ndarray_array_function; +} + + +/* + * Core implementation of ndarray.__array_function__. This is exposed + * separately so we can avoid the overhead of a Python method call from + * within `implement_array_function`. + */ +NPY_NO_EXPORT PyObject * +array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, + PyObject *kwargs) +{ + Py_ssize_t j; + PyObject *implementation, *result; + + PyObject **items = PySequence_Fast_ITEMS(types); + Py_ssize_t length = PySequence_Fast_GET_SIZE(types); + + for (j = 0; j < length; j++) { + int is_subclass = PyObject_IsSubclass( + items[j], (PyObject *)&PyArray_Type); + if (is_subclass == -1) { + return NULL; + } + if (!is_subclass) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + } + + implementation = PyObject_GetAttr(func, npy_ma_str_wrapped); + if (implementation == NULL) { + return NULL; + } + result = PyObject_Call(implementation, args, kwargs); + Py_DECREF(implementation); + return result; +} + + +/* + * Calls __array_function__ on the provided argument, with a fast-path for + * ndarray. + */ +static PyObject * +call_array_function(PyObject* argument, PyObject* method, + PyObject* public_api, PyObject* types, + PyObject* args, PyObject* kwargs) +{ + if (is_default_array_function(method)) { + return array_function_method_impl(public_api, types, args, kwargs); + } + else { + return PyObject_CallFunctionObjArgs( + method, argument, public_api, types, args, kwargs, NULL); + } +} + + +/* + * Implements the __array_function__ protocol for a function, as described in + * in NEP-18. See numpy.core.overrides for a full docstring. + */ +NPY_NO_EXPORT PyObject * +array_implement_array_function( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args) +{ + PyObject *implementation, *public_api, *relevant_args, *args, *kwargs; + + PyObject *types = NULL; + PyObject *implementing_args[NPY_MAXARGS]; + PyObject *array_function_methods[NPY_MAXARGS]; + + int j, any_overrides; + int num_implementing_args = 0; + PyObject *result = NULL; + + static PyObject *errmsg_formatter = NULL; + + if (!PyArg_UnpackTuple( + positional_args, "implement_array_function", 5, 5, + &implementation, &public_api, &relevant_args, &args, &kwargs)) { + return NULL; + } + + relevant_args = PySequence_Fast( + relevant_args, + "dispatcher for __array_function__ did not return an iterable"); + if (relevant_args == NULL) { + return NULL; + } + + /* Collect __array_function__ implementations */ + num_implementing_args = get_implementing_args_and_methods( + relevant_args, implementing_args, array_function_methods); + if (num_implementing_args == -1) { + goto cleanup; + } + + /* + * Handle the typical case of no overrides. This is merely an optimization + * if some arguments are ndarray objects, but is also necessary if no + * arguments implement __array_function__ at all (e.g., if they are all + * built-in types). + */ + any_overrides = 0; + for (j = 0; j < num_implementing_args; j++) { + if (!is_default_array_function(array_function_methods[j])) { + any_overrides = 1; + break; + } + } + if (!any_overrides) { + result = PyObject_Call(implementation, args, kwargs); + goto cleanup; + } + + /* + * Create a Python object for types. + * We use a tuple, because it's the fastest Python collection to create + * and has the bonus of being immutable. + */ + types = PyTuple_New(num_implementing_args); + if (types == NULL) { + goto cleanup; + } + for (j = 0; j < num_implementing_args; j++) { + PyObject *arg_type = (PyObject *)Py_TYPE(implementing_args[j]); + Py_INCREF(arg_type); + PyTuple_SET_ITEM(types, j, arg_type); + } + + /* Call __array_function__ methods */ + for (j = 0; j < num_implementing_args; j++) { + PyObject *argument = implementing_args[j]; + PyObject *method = array_function_methods[j]; + + /* + * We use `public_api` instead of `implementation` here so + * __array_function__ implementations can do equality/identity + * comparisons. + */ + result = call_array_function( + argument, method, public_api, types, args, kwargs); + + if (result == Py_NotImplemented) { + /* Try the next one */ + Py_DECREF(result); + result = NULL; + } + else { + /* Either a good result, or an exception was raised. */ + goto cleanup; + } + } + + /* No acceptable override found, raise TypeError. */ + npy_cache_import("numpy.core._internal", + "array_function_errmsg_formatter", + &errmsg_formatter); + if (errmsg_formatter != NULL) { + PyObject *errmsg = PyObject_CallFunctionObjArgs( + errmsg_formatter, public_api, types, NULL); + if (errmsg != NULL) { + PyErr_SetObject(PyExc_TypeError, errmsg); + Py_DECREF(errmsg); + } + } + +cleanup: + for (j = 0; j < num_implementing_args; j++) { + Py_DECREF(implementing_args[j]); + Py_DECREF(array_function_methods[j]); + } + Py_XDECREF(types); + Py_DECREF(relevant_args); + return result; +} + + +/* + * Python wrapper for get_implementing_args_and_methods, for testing purposes. + */ +NPY_NO_EXPORT PyObject * +array__get_implementing_args( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args) +{ + PyObject *relevant_args; + int j; + int num_implementing_args = 0; + PyObject *implementing_args[NPY_MAXARGS]; + PyObject *array_function_methods[NPY_MAXARGS]; + PyObject *result = NULL; + + if (!PyArg_ParseTuple(positional_args, "O:array__get_implementing_args", + &relevant_args)) { + return NULL; + } + + relevant_args = PySequence_Fast( + relevant_args, + "dispatcher for __array_function__ did not return an iterable"); + if (relevant_args == NULL) { + return NULL; + } + + num_implementing_args = get_implementing_args_and_methods( + relevant_args, implementing_args, array_function_methods); + if (num_implementing_args == -1) { + goto cleanup; + } + + /* create a Python object for implementing_args */ + result = PyList_New(num_implementing_args); + if (result == NULL) { + goto cleanup; + } + for (j = 0; j < num_implementing_args; j++) { + PyObject *argument = implementing_args[j]; + Py_INCREF(argument); + PyList_SET_ITEM(result, j, argument); + } + +cleanup: + for (j = 0; j < num_implementing_args; j++) { + Py_DECREF(implementing_args[j]); + Py_DECREF(array_function_methods[j]); + } + Py_DECREF(relevant_args); + return result; +} diff --git a/numpy/core/src/multiarray/arrayfunction_override.h b/numpy/core/src/multiarray/arrayfunction_override.h new file mode 100644 index 000000000..0d224e2b6 --- /dev/null +++ b/numpy/core/src/multiarray/arrayfunction_override.h @@ -0,0 +1,16 @@ +#ifndef _NPY_PRIVATE__ARRAYFUNCTION_OVERRIDE_H +#define _NPY_PRIVATE__ARRAYFUNCTION_OVERRIDE_H + +NPY_NO_EXPORT PyObject * +array_implement_array_function( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args); + +NPY_NO_EXPORT PyObject * +array__get_implementing_args( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args); + +NPY_NO_EXPORT PyObject * +array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, + PyObject *kwargs); + +#endif diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 341682588..97aaee93d 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -471,7 +471,7 @@ array_dealloc(PyArrayObject *self) { PyArrayObject_fields *fa = (PyArrayObject_fields *)self; - _array_dealloc_buffer_info(self); + _dealloc_cached_buffer_info((PyObject*)self); if (fa->weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *)self); @@ -656,11 +656,9 @@ array_might_be_written(PyArrayObject *obj) { const char *msg = "Numpy has detected that you (may be) writing to an array returned\n" - "by numpy.diagonal or by selecting multiple fields in a structured\n" - "array. This code will likely break in a future numpy release --\n" - "see numpy.diagonal or arrays.indexing reference docs for details.\n" - "The quick fix is to make an explicit copy (e.g., do\n" - "arr.diagonal().copy() or arr[['f0','f1']].copy())."; + "by numpy.diagonal. This code will likely break in a future numpy\n" + "release -- see numpy.diagonal docs for details. The quick fix is\n" + "to make an explicit copy (e.g., do arr.diagonal().copy())."; if (PyArray_FLAGS(obj) & NPY_ARRAY_WARN_ON_WRITE) { /* 2012-07-17, 1.7 */ if (DEPRECATE_FUTUREWARNING(msg) < 0) { diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 46a3ffb3d..823ee7115 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -2,7 +2,8 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" #include "structmember.h" - +#include <limits.h> +#include <assert.h> #define NPY_NO_DEPRECATED_API NPY_API_VERSION #define _MULTIARRAYMODULE @@ -34,8 +35,7 @@ #include "cblasfuncs.h" #include "npy_cblas.h" -#include <limits.h> -#include <assert.h> +#include "buffer.h" /* check for sequences, but ignore the types numpy considers scalars */ static NPY_INLINE npy_bool @@ -921,6 +921,7 @@ VOID_setitem(PyObject *op, void *input, void *vap) memset(ip + view.len, 0, itemsize - view.len); } PyBuffer_Release(&view); + _dealloc_cached_buffer_info(op); } #else { @@ -3593,9 +3594,10 @@ OBJECT_dot(char *ip1, npy_intp is1, char *ip2, npy_intp is2, char *op, npy_intp #define BOOL_fill NULL /* this requires buffer to be filled with objects or NULL */ -static void +static int OBJECT_fill(PyObject **buffer, npy_intp length, void *NPY_UNUSED(ignored)) { + int retval = 0; npy_intp i; PyObject *start = buffer[0]; PyObject *delta = buffer[1]; @@ -3603,27 +3605,31 @@ OBJECT_fill(PyObject **buffer, npy_intp length, void *NPY_UNUSED(ignored)) delta = PyNumber_Subtract(delta, start); if (!delta) { - return; + return -1; } second = start = PyNumber_Add(start, delta); if (!start) { - goto finish; + goto error; } buffer += 2; for (i = 2; i < length; i++, buffer++) { start = PyNumber_Add(start, delta); if (!start) { - goto finish; + goto error; } Py_XDECREF(*buffer); *buffer = start; } + goto finish; + +error: + retval = -1; finish: Py_XDECREF(second); Py_DECREF(delta); - return; + return retval; } /**begin repeat @@ -3637,7 +3643,7 @@ finish: * npy_float, npy_double, npy_longdouble, * npy_datetime, npy_timedelta# */ -static void +static int @NAME@_fill(@type@ *buffer, npy_intp length, void *NPY_UNUSED(ignored)) { npy_intp i; @@ -3648,10 +3654,11 @@ static void for (i = 2; i < length; ++i) { buffer[i] = start + i*delta; } + return 0; } /**end repeat**/ -static void +static int HALF_fill(npy_half *buffer, npy_intp length, void *NPY_UNUSED(ignored)) { npy_intp i; @@ -3662,6 +3669,7 @@ HALF_fill(npy_half *buffer, npy_intp length, void *NPY_UNUSED(ignored)) for (i = 2; i < length; ++i) { buffer[i] = npy_float_to_half(start + i*delta); } + return 0; } /**begin repeat @@ -3669,7 +3677,7 @@ HALF_fill(npy_half *buffer, npy_intp length, void *NPY_UNUSED(ignored)) * #NAME = CFLOAT, CDOUBLE, CLONGDOUBLE# * #type = npy_cfloat, npy_cdouble, npy_clongdouble# */ -static void +static int @NAME@_fill(@type@ *buffer, npy_intp length, void *NPY_UNUSED(ignore)) { npy_intp i; @@ -3687,6 +3695,7 @@ static void buffer->real = start.real + i*delta.real; buffer->imag = start.imag + i*delta.imag; } + return 0; } /**end repeat**/ @@ -4166,6 +4175,53 @@ small_correlate(const char * d_, npy_intp dstride, } /* +*/ + +/* A clone function for the datetime dtype c_metadata */ +static NpyAuxData * +_datetime_dtype_metadata_clone(NpyAuxData *data) +{ + PyArray_DatetimeDTypeMetaData *newdata = + (PyArray_DatetimeDTypeMetaData *)PyArray_malloc( + sizeof(*newdata)); + if (newdata == NULL) { + PyErr_NoMemory(); + return NULL; + } + + memcpy(newdata, data, sizeof(*newdata)); + + return (NpyAuxData *)newdata; +} + +/* + * Allcoate and initialize a PyArray_DatetimeDTypeMetaData object + */ +static NpyAuxData* +_create_datetime_metadata(NPY_DATETIMEUNIT base, int num) +{ + PyArray_DatetimeDTypeMetaData *data; + + /* Allocate memory for the metadata */ + data = PyArray_malloc(sizeof(*data)); + if (data == NULL) { + PyErr_NoMemory(); + return NULL; + } + + /* Initialize the base aux data */ + memset(data, 0, sizeof(PyArray_DatetimeDTypeMetaData)); + data->base.free = (NpyAuxData_FreeFunc *)PyArray_free; + data->base.clone = _datetime_dtype_metadata_clone; + + data->meta.base = base; + data->meta.num = num; + + return (NpyAuxData*)data; +} + + +/* ***************************************************************************** ** SETUP FUNCTION POINTERS ** ***************************************************************************** @@ -4521,66 +4577,6 @@ PyArray_DescrFromType(int type) return ret; } -/* A clone function for the datetime dtype metadata */ -static NpyAuxData * -datetime_dtype_metadata_clone(NpyAuxData *data) -{ - PyArray_DatetimeDTypeMetaData *newdata = - (PyArray_DatetimeDTypeMetaData *)PyArray_malloc( - sizeof(PyArray_DatetimeDTypeMetaData)); - if (newdata == NULL) { - return NULL; - } - - memcpy(newdata, data, sizeof(PyArray_DatetimeDTypeMetaData)); - - return (NpyAuxData *)newdata; -} - -/* - * Initializes the c_metadata field for the _builtin_descrs DATETIME - * and TIMEDELTA. - * - * must not be static, gcc 4.1.2 on redhat 5 then miscompiles this function - * see gh-5163 - * - */ -NPY_NO_EXPORT int -initialize_builtin_datetime_metadata(void) -{ - PyArray_DatetimeDTypeMetaData *data1, *data2; - - /* Allocate memory for the metadata */ - data1 = PyArray_malloc(sizeof(PyArray_DatetimeDTypeMetaData)); - if (data1 == NULL) { - return -1; - } - data2 = PyArray_malloc(sizeof(PyArray_DatetimeDTypeMetaData)); - if (data2 == NULL) { - PyArray_free(data1); - return -1; - } - - /* Initialize the base aux data */ - memset(data1, 0, sizeof(PyArray_DatetimeDTypeMetaData)); - memset(data2, 0, sizeof(PyArray_DatetimeDTypeMetaData)); - data1->base.free = (NpyAuxData_FreeFunc *)PyArray_free; - data2->base.free = (NpyAuxData_FreeFunc *)PyArray_free; - data1->base.clone = datetime_dtype_metadata_clone; - data2->base.clone = datetime_dtype_metadata_clone; - - /* Set to the default metadata */ - data1->meta.base = NPY_DATETIME_DEFAULTUNIT; - data1->meta.num = 1; - data2->meta.base = NPY_DATETIME_DEFAULTUNIT; - data2->meta.num = 1; - - _builtin_descrs[NPY_DATETIME]->c_metadata = (NpyAuxData *)data1; - _builtin_descrs[NPY_TIMEDELTA]->c_metadata = (NpyAuxData *)data2; - - return 0; -} - /* ***************************************************************************** ** SETUP TYPE INFO ** @@ -4649,7 +4645,14 @@ set_typeinfo(PyObject *dict) /**end repeat**/ - if (initialize_builtin_datetime_metadata() < 0) { + _builtin_descrs[NPY_DATETIME]->c_metadata = _create_datetime_metadata( + NPY_DATETIME_DEFAULTUNIT, 1); + if (_builtin_descrs[NPY_DATETIME]->c_metadata == NULL) { + return -1; + } + _builtin_descrs[NPY_TIMEDELTA]->c_metadata = _create_datetime_metadata( + NPY_DATETIME_DEFAULTUNIT, 1); + if (_builtin_descrs[NPY_DATETIME]->c_metadata == NULL) { return -1; } diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 9a2750aea..51676f0b3 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -509,6 +509,10 @@ _buffer_info_new(PyObject *obj) PyArray_Descr *descr = NULL; int err = 0; + /* + * Note that the buffer info is cached as pyints making them appear like + * unreachable lost memory to valgrind. + */ info = malloc(sizeof(_buffer_info_t)); if (info == NULL) { PyErr_NoMemory(); @@ -922,7 +926,7 @@ fail: */ NPY_NO_EXPORT void -_array_dealloc_buffer_info(PyArrayObject *self) +_dealloc_cached_buffer_info(PyObject *self) { int reset_error_state = 0; PyObject *ptype, *pvalue, *ptraceback; @@ -936,7 +940,7 @@ _array_dealloc_buffer_info(PyArrayObject *self) PyErr_Fetch(&ptype, &pvalue, &ptraceback); } - _buffer_clear_info((PyObject*)self); + _buffer_clear_info(self); if (reset_error_state) { PyErr_Restore(ptype, pvalue, ptraceback); diff --git a/numpy/core/src/multiarray/buffer.h b/numpy/core/src/multiarray/buffer.h index d5da8f440..fae413c85 100644 --- a/numpy/core/src/multiarray/buffer.h +++ b/numpy/core/src/multiarray/buffer.h @@ -4,7 +4,7 @@ extern NPY_NO_EXPORT PyBufferProcs array_as_buffer; NPY_NO_EXPORT void -_array_dealloc_buffer_info(PyArrayObject *self); +_dealloc_cached_buffer_info(PyObject *self); NPY_NO_EXPORT PyArray_Descr* _descriptor_from_pep3118_format(char *s); diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 5b4611e8a..addb67732 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -164,7 +164,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, if (string_type == NPY_STRING) { if ((temp = PyObject_Str(obj)) == NULL) { - return -1; + goto fail; } #if defined(NPY_PY3K) #if PY_VERSION_HEX >= 0x03030000 @@ -182,7 +182,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, #else if ((temp = PyObject_Unicode(obj)) == NULL) { #endif - return -1; + goto fail; } itemsize = PyUnicode_GET_DATA_SIZE(temp); #ifndef Py_UNICODE_WIDE @@ -216,7 +216,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, if (string_type == NPY_STRING) { if ((temp = PyObject_Str(obj)) == NULL) { - return -1; + goto fail; } #if defined(NPY_PY3K) #if PY_VERSION_HEX >= 0x03030000 @@ -234,7 +234,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, #else if ((temp = PyObject_Unicode(obj)) == NULL) { #endif - return -1; + goto fail; } itemsize = PyUnicode_GET_DATA_SIZE(temp); #ifndef Py_UNICODE_WIDE @@ -312,6 +312,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, PyErr_Clear(); dtype = _descriptor_from_pep3118_format(buffer_view.format); PyBuffer_Release(&buffer_view); + _dealloc_cached_buffer_info(obj); if (dtype) { goto promote_types; } @@ -323,6 +324,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, dtype = PyArray_DescrNewFromType(NPY_VOID); dtype->elsize = buffer_view.itemsize; PyBuffer_Release(&buffer_view); + _dealloc_cached_buffer_info(obj); goto promote_types; } else { @@ -438,12 +440,18 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, return 0; } - /* Recursive case, first check the sequence contains only one type */ + /* + * The C-API recommends calling PySequence_Fast before any of the other + * PySequence_Fast* functions. This is required for PyPy + */ seq = PySequence_Fast(obj, "Could not convert object to sequence"); if (seq == NULL) { goto fail; } + + /* Recursive case, first check the sequence contains only one type */ size = PySequence_Fast_GET_SIZE(seq); + /* objects is borrowed, do not release seq */ objects = PySequence_Fast_ITEMS(seq); common_type = size > 0 ? Py_TYPE(objects[0]) : NULL; for (i = 1; i < size; ++i) { @@ -503,7 +511,7 @@ promote_types: PyArray_Descr *res_dtype = PyArray_PromoteTypes(dtype, *out_dtype); Py_DECREF(dtype); if (res_dtype == NULL) { - return -1; + goto fail; } if (!string_type && res_dtype->type_num == NPY_UNICODE && @@ -607,12 +615,6 @@ _IsWriteable(PyArrayObject *ap) * If it is a writeable array, then return TRUE * If we can find an array object * or a writeable buffer object as the final base object - * or a string object (for pickling support memory savings). - * - this last could be removed if a proper pickleable - * buffer was added to Python. - * - * MW: I think it would better to disallow switching from READONLY - * to WRITEABLE like this... */ while(PyArray_Check(base)) { @@ -621,21 +623,20 @@ _IsWriteable(PyArrayObject *ap) } base = PyArray_BASE((PyArrayObject *)base); } - - /* - * here so pickle support works seamlessly - * and unpickled array can be set and reset writeable - * -- could be abused -- - */ - if (PyString_Check(base)) { - return NPY_TRUE; - } #if defined(NPY_PY3K) if (PyObject_GetBuffer(base, &view, PyBUF_WRITABLE|PyBUF_SIMPLE) < 0) { PyErr_Clear(); return NPY_FALSE; } PyBuffer_Release(&view); + /* + * The first call to PyObject_GetBuffer stores a reference to a struct + * _buffer_info_t (from buffer.c, with format, ndim, strides and shape) in + * a static dictionary, with id(base) as the key. Usually we release it + * after the call to PyBuffer_Release, via a call to + * _dealloc_cached_buffer_info, but in this case leave it in the cache to + * speed up future calls to _IsWriteable. + */ #else if (PyObject_AsWriteBuffer(base, &dummy, &n) < 0) { PyErr_Clear(); diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 2b8d3d3a4..0e162903d 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -182,6 +182,7 @@ check_and_adjust_axis(int *axis, int ndim) /* used for some alignment checks */ #define _ALIGN(type) offsetof(struct {char c; type v;}, v) +#define _UINT_ALIGN(type) npy_uint_alignment(sizeof(type)) /* * Disable harmless compiler warning "4116: unnamed type definition in * parentheses" which is caused by the _ALIGN macro. @@ -201,6 +202,7 @@ npy_is_aligned(const void * p, const npy_uintp alignment) * Assumes cast from pointer to uintp gives a sensible representation we * can use bitwise & on (not required by C standard, but used by glibc). * This test is faster than a direct modulo. + * Note alignment value of 0 is allowed and returns False. */ return ((npy_uintp)(p) & ((alignment) - 1)) == 0; } diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index e8380e3bc..88924a860 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -1158,13 +1158,18 @@ arr_unravel_index(PyObject *self, PyObject *args, PyObject *kwds) char *kwlist[] = {"indices", "shape", "order", NULL}; - /* Continue to support the older "dims" argument in place + /* + * TODO: remove this in favor of warning raised in the dispatcher when + * __array_function__ is enabled by default. + */ + + /* + * Continue to support the older "dims" argument in place * of the "shape" argument. Issue an appropriate warning * if "dims" is detected in keywords, then replace it with - * the new "shape" argument and continue processing as usual */ - - - if (kwds) { + * the new "shape" argument and continue processing as usual. + */ + if (kwds) { PyObject *dims_item, *shape_item; dims_item = PyDict_GetItemString(kwds, "dims"); shape_item = PyDict_GetItemString(kwds, "shape"); @@ -1570,6 +1575,7 @@ pack_bits(PyObject *input, int axis) if (!PyArray_ISBOOL(inp) && !PyArray_ISINTEGER(inp)) { PyErr_SetString(PyExc_TypeError, "Expected an input array of integer or boolean data type"); + Py_DECREF(inp); goto fail; } @@ -1677,6 +1683,7 @@ unpack_bits(PyObject *input, int axis) if (PyArray_TYPE(inp) != NPY_UBYTE) { PyErr_SetString(PyExc_TypeError, "Expected an input array of unsigned byte data type"); + Py_DECREF(inp); goto fail; } diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 7e92e5991..cef3c27ed 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -16,6 +16,7 @@ #include "conversion_utils.h" #include "alloc.h" +#include "buffer.h" static int PyArray_PyIntAsInt_ErrMsg(PyObject *o, const char * msg) NPY_GCC_NONNULL(2); @@ -185,6 +186,7 @@ PyArray_BufferConverter(PyObject *obj, PyArray_Chunk *buf) * sticks around after the release. */ PyBuffer_Release(&view); + _dealloc_cached_buffer_info(obj); /* Point to the base of the buffer object if present */ if (PyMemoryView_Check(obj)) { diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c index e88582a51..7db467308 100644 --- a/numpy/core/src/multiarray/convert.c +++ b/numpy/core/src/multiarray/convert.c @@ -614,22 +614,6 @@ PyArray_View(PyArrayObject *self, PyArray_Descr *type, PyTypeObject *pytype) } dtype = PyArray_DESCR(self); - - if (type != NULL && !PyArray_EquivTypes(dtype, type) && - (PyArray_FLAGS(self) & NPY_ARRAY_WARN_ON_WRITE)) { - const char *msg = - "Numpy has detected that you may be viewing or writing to an array " - "returned by selecting multiple fields in a structured array. \n\n" - "This code may break in numpy 1.16 because this will return a view " - "instead of a copy -- see release notes for details."; - /* 2016-09-19, 1.12 */ - if (DEPRECATE_FUTUREWARNING(msg) < 0) { - return NULL; - } - /* Only warn once per array */ - PyArray_CLEARFLAGS(self, NPY_ARRAY_WARN_ON_WRITE); - } - flags = PyArray_FLAGS(self); Py_INCREF(dtype); diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index aaaaeee82..bd70eba0c 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -11,7 +11,7 @@ #include "npy_config.h" -#include "npy_import.h" +#include "npy_ctypes.h" #include "npy_pycompat.h" #include "multiarraymodule.h" @@ -743,12 +743,14 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, d[i] = buffer_view.shape[i]; } PyBuffer_Release(&buffer_view); + _dealloc_cached_buffer_info(obj); return 0; } else if (PyObject_GetBuffer(obj, &buffer_view, PyBUF_SIMPLE) == 0) { d[0] = buffer_view.len; *maxndim = 1; PyBuffer_Release(&buffer_view); + _dealloc_cached_buffer_info(obj); return 0; } else { @@ -1381,15 +1383,7 @@ _array_from_buffer_3118(PyObject *memoryview) * Note that even if the above are fixed in master, we have to drop the * early patch versions of python to actually make use of the fixes. */ - - int is_ctypes = _is_from_ctypes(view->obj); - if (is_ctypes < 0) { - /* This error is not useful */ - PyErr_WriteUnraisable(view->obj); - is_ctypes = 0; - } - - if (!is_ctypes) { + if (!npy_ctypes_check(Py_TYPE(view->obj))) { /* This object has no excuse for a broken PEP3118 buffer */ PyErr_Format( PyExc_RuntimeError, @@ -2030,7 +2024,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) newtype = oldtype; Py_INCREF(oldtype); } - if (PyDataType_ISUNSIZED(newtype)) { + else if (PyDataType_ISUNSIZED(newtype)) { PyArray_DESCR_REPLACE(newtype); if (newtype == NULL) { return NULL; @@ -2134,12 +2128,15 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) */ /* 2017-Nov-10 1.14 */ - if (DEPRECATE("NPY_ARRAY_UPDATEIFCOPY, NPY_ARRAY_INOUT_ARRAY, and " - "NPY_ARRAY_INOUT_FARRAY are deprecated, use NPY_WRITEBACKIFCOPY, " - "NPY_ARRAY_INOUT_ARRAY2, or NPY_ARRAY_INOUT_FARRAY2 respectively " - "instead, and call PyArray_ResolveWritebackIfCopy before the " - "array is deallocated, i.e. before the last call to Py_DECREF.") < 0) + if (DEPRECATE( + "NPY_ARRAY_UPDATEIFCOPY, NPY_ARRAY_INOUT_ARRAY, and " + "NPY_ARRAY_INOUT_FARRAY are deprecated, use NPY_WRITEBACKIFCOPY, " + "NPY_ARRAY_INOUT_ARRAY2, or NPY_ARRAY_INOUT_FARRAY2 respectively " + "instead, and call PyArray_ResolveWritebackIfCopy before the " + "array is deallocated, i.e. before the last call to Py_DECREF.") < 0) { + Py_DECREF(ret); return NULL; + } Py_INCREF(arr); if (PyArray_SetWritebackIfCopyBase(ret, arr) < 0) { Py_DECREF(ret); @@ -2166,14 +2163,12 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) Py_DECREF(newtype); if (needview) { - PyArray_Descr *dtype = PyArray_DESCR(arr); PyTypeObject *subtype = NULL; if (flags & NPY_ARRAY_ENSUREARRAY) { subtype = &PyArray_Type; } - Py_INCREF(dtype); ret = (PyArrayObject *)PyArray_View(arr, NULL, subtype); if (ret == NULL) { return NULL; @@ -2471,6 +2466,7 @@ PyArray_FromInterface(PyObject *origin) * sticks around after the release. */ PyBuffer_Release(&view); + _dealloc_cached_buffer_info(base); #else res = PyObject_AsWriteBuffer(base, (void **)&data, &buffer_len); if (res < 0) { @@ -2832,7 +2828,8 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) * contiguous strides, etc. */ if (PyArray_GetDTypeTransferFunction( - IsUintAligned(src) && IsUintAligned(dst), + IsUintAligned(src) && IsAligned(src) && + IsUintAligned(dst) && IsAligned(dst), src_stride, dst_stride, PyArray_DESCR(src), PyArray_DESCR(dst), 0, @@ -3726,6 +3723,7 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type, * sticks around after the release. */ PyBuffer_Release(&view); + _dealloc_cached_buffer_info(buf); #else if (PyObject_AsWriteBuffer(buf, (void *)&data, &ts) == -1) { writeable = 0; diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index a8550d958..54d19d993 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -3822,18 +3822,26 @@ recursive_find_object_timedelta64_type(PyObject *obj, * single object using [()], but not by using * __getitem__(integer) approaches */ - PyObject *item, *meth, *args; + PyObject *item, *args; - meth = PyObject_GetAttrString(obj, "__getitem__"); - args = Py_BuildValue("(())"); - item = PyObject_CallObject(meth, args); + args = PyTuple_New(0); + if (args == NULL) { + return 0; + } + item = PyObject_GetItem(obj, args); + Py_DECREF(args); + if (item == NULL) { + return 0; + } /* * NOTE: may need other type checks here in the future * for expanded 0 D datetime array conversions? */ if (PyDelta_Check(item)) { + Py_DECREF(item); return delta_checker(meta); } + Py_DECREF(item); } } } diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 439980877..0471a2a3e 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -10,7 +10,7 @@ #include "numpy/arrayscalars.h" #include "npy_config.h" - +#include "npy_ctypes.h" #include "npy_pycompat.h" #include "_datetime.h" @@ -19,6 +19,7 @@ #include "descriptor.h" #include "alloc.h" #include "assert.h" +#include "buffer.h" /* * offset: A starting offset. @@ -54,79 +55,46 @@ Borrowed_PyMapping_GetItemString(PyObject *o, char *key) return ret; } -/* - * Creates a dtype object from ctypes inputs. - * - * Returns a new reference to a dtype object, or NULL - * if this is not possible. When it returns NULL, it does - * not set a Python exception. - */ static PyArray_Descr * -_arraydescr_fromctypes(PyObject *obj) +_arraydescr_from_ctypes_type(PyTypeObject *type) { - PyObject *dtypedescr; - PyArray_Descr *newdescr; - int ret; + PyObject *_numpy_dtype_ctypes; + PyObject *res; - /* Understand basic ctypes */ - dtypedescr = PyObject_GetAttrString(obj, "_type_"); - PyErr_Clear(); - if (dtypedescr) { - ret = PyArray_DescrConverter(dtypedescr, &newdescr); - Py_DECREF(dtypedescr); - if (ret == NPY_SUCCEED) { - PyObject *length; - /* Check for ctypes arrays */ - length = PyObject_GetAttrString(obj, "_length_"); - PyErr_Clear(); - if (length) { - /* derived type */ - PyObject *newtup; - PyArray_Descr *derived; - newtup = Py_BuildValue("N(N)", newdescr, length); - ret = PyArray_DescrConverter(newtup, &derived); - Py_DECREF(newtup); - if (ret == NPY_SUCCEED) { - return derived; - } - PyErr_Clear(); - return NULL; - } - return newdescr; - } - PyErr_Clear(); + /* Call the python function of the same name. */ + _numpy_dtype_ctypes = PyImport_ImportModule("numpy.core._dtype_ctypes"); + if (_numpy_dtype_ctypes == NULL) { return NULL; } - /* Understand ctypes structures -- - bit-fields are not supported - automatically aligns */ - dtypedescr = PyObject_GetAttrString(obj, "_fields_"); - PyErr_Clear(); - if (dtypedescr) { - ret = PyArray_DescrAlignConverter(dtypedescr, &newdescr); - Py_DECREF(dtypedescr); - if (ret == NPY_SUCCEED) { - return newdescr; - } - PyErr_Clear(); + res = PyObject_CallMethod(_numpy_dtype_ctypes, "dtype_from_ctypes_type", "O", (PyObject *)type); + Py_DECREF(_numpy_dtype_ctypes); + if (res == NULL) { + return NULL; } - return NULL; + /* + * sanity check that dtype_from_ctypes_type returned the right type, + * since getting it wrong would give segfaults. + */ + if (!PyObject_TypeCheck(res, &PyArrayDescr_Type)) { + Py_DECREF(res); + PyErr_BadInternalCall(); + return NULL; + } + + return (PyArray_Descr *)res; } /* - * This function creates a dtype object when: - * - The object has a "dtype" attribute, and it can be converted - * to a dtype object. - * - The object is a ctypes type object, including array - * and structure types. + * This function creates a dtype object when the object has a "dtype" attribute, + * and it can be converted to a dtype object. * * Returns a new reference to a dtype object, or NULL * if this is not possible. When it returns NULL, it does * not set a Python exception. */ NPY_NO_EXPORT PyArray_Descr * -_arraydescr_fromobj(PyObject *obj) +_arraydescr_from_dtype_attr(PyObject *obj) { PyObject *dtypedescr; PyArray_Descr *newdescr = NULL; @@ -135,15 +103,18 @@ _arraydescr_fromobj(PyObject *obj) /* For arbitrary objects that have a "dtype" attribute */ dtypedescr = PyObject_GetAttrString(obj, "dtype"); PyErr_Clear(); - if (dtypedescr != NULL) { - ret = PyArray_DescrConverter(dtypedescr, &newdescr); - Py_DECREF(dtypedescr); - if (ret == NPY_SUCCEED) { - return newdescr; - } + if (dtypedescr == NULL) { + return NULL; + } + + ret = PyArray_DescrConverter(dtypedescr, &newdescr); + Py_DECREF(dtypedescr); + if (ret != NPY_SUCCEED) { PyErr_Clear(); + return NULL; } - return _arraydescr_fromctypes(obj); + + return newdescr; } /* @@ -286,6 +257,9 @@ _convert_from_tuple(PyObject *obj, int align) return NULL; } PyArray_DESCR_REPLACE(type); + if (type == NULL) { + return NULL; + } if (type->type_num == NPY_UNICODE) { type->elsize = itemsize << 2; } @@ -541,6 +515,7 @@ _convert_from_array_descr(PyObject *obj, int align) #if defined(NPY_PY3K) Py_DECREF(name); #endif + Py_DECREF(conv); goto fail; } dtypeflags |= (conv->flags & NPY_FROM_FIELDS); @@ -863,9 +838,11 @@ _use_inherit(PyArray_Descr *type, PyObject *newobj, int *errflag) else if (new->elsize != conv->elsize) { PyErr_SetString(PyExc_ValueError, "mismatch in size of old and new data-descriptor"); + Py_DECREF(new); goto fail; } else if (invalid_union_object_dtype(new, conv)) { + Py_DECREF(new); goto fail; } @@ -1423,10 +1400,20 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) check_num = NPY_VOID; } else { - *at = _arraydescr_fromobj(obj); + *at = _arraydescr_from_dtype_attr(obj); if (*at) { return NPY_SUCCEED; } + + /* + * Note: this comes after _arraydescr_from_dtype_attr because the ctypes + * type might override the dtype if numpy does not otherwise + * support it. + */ + if (npy_ctypes_check((PyTypeObject *)obj)) { + *at = _arraydescr_from_ctypes_type((PyTypeObject *)obj); + return *at ? NPY_SUCCEED : NPY_FAIL; + } } goto finish; } @@ -1596,13 +1583,23 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) goto fail; } else { - *at = _arraydescr_fromobj(obj); + *at = _arraydescr_from_dtype_attr(obj); if (*at) { return NPY_SUCCEED; } if (PyErr_Occurred()) { return NPY_FAIL; } + + /* + * Note: this comes after _arraydescr_from_dtype_attr because the ctypes + * type might override the dtype if numpy does not otherwise + * support it. + */ + if (npy_ctypes_check(Py_TYPE(obj))) { + *at = _arraydescr_from_ctypes_type(Py_TYPE(obj)); + return *at ? NPY_SUCCEED : NPY_FAIL; + } goto fail; } if (PyErr_Occurred()) { @@ -1660,6 +1657,9 @@ finish: if (PyDataType_ISUNSIZED(*at) && (*at)->elsize != elsize) { PyArray_DESCR_REPLACE(*at); + if (*at == NULL) { + goto error; + } (*at)->elsize = elsize; } if (endian != '=' && PyArray_ISNBO(endian)) { @@ -1668,6 +1668,9 @@ finish: if (endian != '=' && (*at)->byteorder != '|' && (*at)->byteorder != endian) { PyArray_DESCR_REPLACE(*at); + if (*at == NULL) { + goto error; + } (*at)->byteorder = endian; } return NPY_SUCCEED; @@ -1728,6 +1731,7 @@ PyArray_DescrNew(PyArray_Descr *base) newdescr->c_metadata = NPY_AUXDATA_CLONE(base->c_metadata); if (newdescr->c_metadata == NULL) { PyErr_NoMemory(); + /* TODO: This seems wrong, as the old fields get decref'd? */ Py_DECREF(newdescr); return NULL; } @@ -1770,6 +1774,7 @@ arraydescr_dealloc(PyArray_Descr *self) Py_INCREF(self); return; } + _dealloc_cached_buffer_info((PyObject*)self); Py_XDECREF(self->typeobj); Py_XDECREF(self->names); Py_XDECREF(self->fields); @@ -3335,12 +3340,15 @@ static PyObject * _subscript_by_index(PyArray_Descr *self, Py_ssize_t i) { PyObject *name = PySequence_GetItem(self->names, i); + PyObject *ret; if (name == NULL) { PyErr_Format(PyExc_IndexError, "Field index %zd out of range.", i); return NULL; } - return _subscript_by_name(self, name); + ret = _subscript_by_name(self, name); + Py_DECREF(name); + return ret; } static PyObject * diff --git a/numpy/core/src/multiarray/descriptor.h b/numpy/core/src/multiarray/descriptor.h index 5a3e4b15f..a5f3b8cdf 100644 --- a/numpy/core/src/multiarray/descriptor.h +++ b/numpy/core/src/multiarray/descriptor.h @@ -8,7 +8,7 @@ NPY_NO_EXPORT PyObject * array_set_typeDict(PyObject *NPY_UNUSED(ignored), PyObject *args); NPY_NO_EXPORT PyArray_Descr * -_arraydescr_fromobj(PyObject *obj); +_arraydescr_from_dtype_attr(PyObject *obj); NPY_NO_EXPORT int diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 97d899ce0..63b1ead25 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -26,6 +26,7 @@ #include "_datetime.h" #include "datetime_strings.h" #include "descriptor.h" +#include "array_assign.h" #include "shape.h" #include "lowlevel_strided_loops.h" @@ -51,6 +52,20 @@ #endif /**********************************************/ +#if NPY_DT_DBG_TRACING +/* + * Thin wrapper around print that ignores exceptions + */ +static void +_safe_print(PyObject *obj) +{ + if (PyObject_Print(obj, stdout, 0) < 0) { + PyErr_Clear(); + printf("<error during print>"); + } +} +#endif + /* * Returns a transfer function which DECREFs any references in src_type. * @@ -1042,9 +1057,9 @@ get_nbo_cast_datetime_transfer_function(int aligned, #if NPY_DT_DBG_TRACING printf("Dtype transfer from "); - PyObject_Print((PyObject *)src_dtype, stdout, 0); + _safe_print((PyObject *)src_dtype); printf(" to "); - PyObject_Print((PyObject *)dst_dtype, stdout, 0); + _safe_print((PyObject *)dst_dtype); printf("\n"); printf("has conversion fraction %lld/%lld\n", num, denom); #endif @@ -1089,9 +1104,9 @@ get_nbo_datetime_to_string_transfer_function(int aligned, #if NPY_DT_DBG_TRACING printf("Dtype transfer from "); - PyObject_Print((PyObject *)src_dtype, stdout, 0); + _safe_print((PyObject *)src_dtype); printf(" to "); - PyObject_Print((PyObject *)dst_dtype, stdout, 0); + _safe_print((PyObject *)dst_dtype); printf("\n"); #endif @@ -1211,9 +1226,9 @@ get_nbo_string_to_datetime_transfer_function(int aligned, #if NPY_DT_DBG_TRACING printf("Dtype transfer from "); - PyObject_Print((PyObject *)src_dtype, stdout, 0); + _safe_print((PyObject *)src_dtype); printf(" to "); - PyObject_Print((PyObject *)dst_dtype, stdout, 0); + _safe_print((PyObject *)dst_dtype); printf("\n"); #endif @@ -3421,9 +3436,13 @@ PyArray_GetDTypeTransferFunction(int aligned, #if NPY_DT_DBG_TRACING printf("Calculating dtype transfer from "); - PyObject_Print((PyObject *)src_dtype, stdout, 0); + if (PyObject_Print((PyObject *)src_dtype, stdout, 0) < 0) { + return NPY_FAIL; + } printf(" to "); - PyObject_Print((PyObject *)dst_dtype, stdout, 0); + if (PyObject_Print((PyObject *)dst_dtype, stdout, 0) < 0) { + return NPY_FAIL; + } printf("\n"); #endif @@ -3747,11 +3766,15 @@ PyArray_CastRawArrays(npy_intp count, return NPY_SUCCEED; } - /* Check data alignment */ - aligned = (((npy_intp)src | src_stride) & - (src_dtype->alignment - 1)) == 0 && - (((npy_intp)dst | dst_stride) & - (dst_dtype->alignment - 1)) == 0; + /* Check data alignment, both uint and true */ + aligned = raw_array_is_aligned(1, &count, dst, &dst_stride, + npy_uint_alignment(dst_dtype->elsize)) && + raw_array_is_aligned(1, &count, dst, &dst_stride, + dst_dtype->alignment) && + raw_array_is_aligned(1, &count, src, &src_stride, + npy_uint_alignment(src_dtype->elsize)) && + raw_array_is_aligned(1, &count, src, &src_stride, + src_dtype->alignment); /* Get the function to do the casting */ if (PyArray_GetDTypeTransferFunction(aligned, diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index cae4273ff..24962da8a 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -20,6 +20,7 @@ #include "arrayobject.h" #include "mem_overlap.h" #include "alloc.h" +#include "buffer.h" /******************* array attribute get and set routines ******************/ @@ -143,6 +144,7 @@ array_strides_set(PyArrayObject *self, PyObject *obj) offset = PyArray_BYTES(self) - (char *)view.buf; numbytes = view.len + offset; PyBuffer_Release(&view); + _dealloc_cached_buffer_info((PyObject*)new); } #else if (PyArray_BASE(new) && @@ -376,6 +378,7 @@ array_data_set(PyArrayObject *self, PyObject *op) * sticks around after the release. */ PyBuffer_Release(&view); + _dealloc_cached_buffer_info(op); #else if (PyObject_AsWriteBuffer(op, &buf, &buf_len) < 0) { PyErr_Clear(); diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c index de54ca1b3..a7c6b14f4 100644 --- a/numpy/core/src/multiarray/item_selection.c +++ b/numpy/core/src/multiarray/item_selection.c @@ -45,7 +45,7 @@ PyArray_TakeFrom(PyArrayObject *self0, PyObject *indices0, int axis, indices = NULL; self = (PyArrayObject *)PyArray_CheckAxis(self0, &axis, - NPY_ARRAY_CARRAY); + NPY_ARRAY_CARRAY_RO); if (self == NULL) { return NULL; } diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 3e3248f53..a3bc8e742 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -92,114 +92,6 @@ parse_index_entry(PyObject *op, npy_intp *step_size, } -/* - * Parses an index that has no fancy indexing. Populates - * out_dimensions, out_strides, and out_offset. - */ -NPY_NO_EXPORT int -parse_index(PyArrayObject *self, PyObject *op, - npy_intp *out_dimensions, - npy_intp *out_strides, - npy_intp *out_offset, - int check_index) -{ - int i, j, n; - int nd_old, nd_new, n_add, n_ellipsis; - npy_intp n_steps, start, offset, step_size; - PyObject *op1 = NULL; - int is_slice; - - if (PySlice_Check(op) || op == Py_Ellipsis || op == Py_None) { - n = 1; - op1 = op; - Py_INCREF(op); - /* this relies on the fact that n==1 for loop below */ - is_slice = 1; - } - else { - if (!PySequence_Check(op)) { - PyErr_SetString(PyExc_IndexError, - "index must be either an int " - "or a sequence"); - return -1; - } - n = PySequence_Length(op); - is_slice = 0; - } - - nd_old = nd_new = 0; - - offset = 0; - for (i = 0; i < n; i++) { - if (!is_slice) { - op1 = PySequence_GetItem(op, i); - if (op1 == NULL) { - return -1; - } - } - start = parse_index_entry(op1, &step_size, &n_steps, - nd_old < PyArray_NDIM(self) ? - PyArray_DIMS(self)[nd_old] : 0, - nd_old, check_index ? - nd_old < PyArray_NDIM(self) : 0); - Py_DECREF(op1); - if (start == -1) { - break; - } - if (n_steps == NEWAXIS_INDEX) { - out_dimensions[nd_new] = 1; - out_strides[nd_new] = 0; - nd_new++; - } - else if (n_steps == ELLIPSIS_INDEX) { - for (j = i + 1, n_ellipsis = 0; j < n; j++) { - op1 = PySequence_GetItem(op, j); - if (op1 == Py_None) { - n_ellipsis++; - } - Py_DECREF(op1); - } - n_add = PyArray_NDIM(self)-(n-i-n_ellipsis-1+nd_old); - if (n_add < 0) { - PyErr_SetString(PyExc_IndexError, "too many indices"); - return -1; - } - for (j = 0; j < n_add; j++) { - out_dimensions[nd_new] = PyArray_DIMS(self)[nd_old]; - out_strides[nd_new] = PyArray_STRIDES(self)[nd_old]; - nd_new++; nd_old++; - } - } - else { - if (nd_old >= PyArray_NDIM(self)) { - PyErr_SetString(PyExc_IndexError, "too many indices"); - return -1; - } - offset += PyArray_STRIDES(self)[nd_old]*start; - nd_old++; - if (n_steps != SINGLE_INDEX) { - out_dimensions[nd_new] = n_steps; - out_strides[nd_new] = step_size * - PyArray_STRIDES(self)[nd_old-1]; - nd_new++; - } - } - } - if (i < n) { - return -1; - } - n_add = PyArray_NDIM(self)-nd_old; - for (j = 0; j < n_add; j++) { - out_dimensions[nd_new] = PyArray_DIMS(self)[nd_old]; - out_strides[nd_new] = PyArray_STRIDES(self)[nd_old]; - nd_new++; - nd_old++; - } - *out_offset = offset; - return nd_new; -} - - /*********************** Element-wise Array Iterator ***********************/ /* Aided by Peter J. Verveer's nd_image package and numpy's arraymap ****/ /* and Python's array iterator ***/ diff --git a/numpy/core/src/multiarray/iterators.h b/numpy/core/src/multiarray/iterators.h index 04f57c885..376dc154a 100644 --- a/numpy/core/src/multiarray/iterators.h +++ b/numpy/core/src/multiarray/iterators.h @@ -1,17 +1,6 @@ #ifndef _NPY_ARRAYITERATORS_H_ #define _NPY_ARRAYITERATORS_H_ -/* - * Parses an index that has no fancy indexing. Populates - * out_dimensions, out_strides, and out_offset. - */ -NPY_NO_EXPORT int -parse_index(PyArrayObject *self, PyObject *op, - npy_intp *out_dimensions, - npy_intp *out_strides, - npy_intp *out_offset, - int check_index); - NPY_NO_EXPORT PyObject *iter_subscript(PyArrayIterObject *, PyObject *); diff --git a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src index 159bb4103..16bacf1ab 100644 --- a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src +++ b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src @@ -82,7 +82,7 @@ /**begin repeat * #elsize = 1, 2, 4, 8, 16# * #elsize_half = 0, 1, 2, 4, 8# - * #type = npy_uint8, npy_uint16, npy_uint32, npy_uint64, npy_uint128# + * #type = npy_uint8, npy_uint16, npy_uint32, npy_uint64, npy_uint64# */ /**begin repeat1 * #oper = strided_to_strided, strided_to_contig, @@ -119,10 +119,10 @@ static void npy_intp N, npy_intp NPY_UNUSED(src_itemsize), NpyAuxData *NPY_UNUSED(data)) { -#if @is_aligned@ && @elsize@ != 16 +#if @is_aligned@ /* sanity check */ - assert(npy_is_aligned(dst, _ALIGN(@type@))); - assert(npy_is_aligned(src, _ALIGN(@type@))); + assert(N == 0 || npy_is_aligned(dst, _UINT_ALIGN(@type@))); + assert(N == 0 || npy_is_aligned(src, _UINT_ALIGN(@type@))); #endif /*printf("fn @prefix@_@oper@_size@elsize@\n");*/ while (N > 0) { @@ -201,8 +201,8 @@ static NPY_GCC_OPT_3 void } #if @is_aligned@ && @elsize@ != 16 /* sanity check */ - assert(npy_is_aligned(dst, _ALIGN(@type@))); - assert(npy_is_aligned(src, _ALIGN(@type@))); + assert(N == 0 || npy_is_aligned(dst, _UINT_ALIGN(@type@))); + assert(N == 0 || npy_is_aligned(src, _UINT_ALIGN(@type@))); #endif #if @elsize@ == 1 && @dst_contig@ memset(dst, *src, N); @@ -808,12 +808,8 @@ static NPY_GCC_OPT_3 void #if @aligned@ /* sanity check */ -# if !@is_complex1@ - assert(npy_is_aligned(src, _ALIGN(_TYPE1))); -# endif -# if !@is_complex2@ - assert(npy_is_aligned(dst, _ALIGN(_TYPE2))); -# endif + assert(N == 0 || npy_is_aligned(src, _ALIGN(_TYPE1))); + assert(N == 0 || npy_is_aligned(dst, _ALIGN(_TYPE2))); #endif /*printf("@prefix@_cast_@name1@_to_@name2@\n");*/ @@ -1425,7 +1421,7 @@ mapiter_trivial_@name@(PyArrayObject *self, PyArrayObject *ind, while (itersize--) { char * self_ptr; npy_intp indval = *((npy_intp*)ind_ptr); - assert(npy_is_aligned(ind_ptr, _ALIGN(npy_intp))); + assert(npy_is_aligned(ind_ptr, _UINT_ALIGN(npy_intp))); #if @isget@ if (check_and_adjust_index(&indval, fancy_dim, 0, _save) < 0 ) { return -1; @@ -1439,8 +1435,8 @@ mapiter_trivial_@name@(PyArrayObject *self, PyArrayObject *ind, #if @isget@ #if @elsize@ - assert(npy_is_aligned(result_ptr, _ALIGN(@copytype@))); - assert(npy_is_aligned(self_ptr, _ALIGN(@copytype@))); + assert(npy_is_aligned(result_ptr, _UINT_ALIGN(@copytype@))); + assert(npy_is_aligned(self_ptr, _UINT_ALIGN(@copytype@))); *(@copytype@ *)result_ptr = *(@copytype@ *)self_ptr; #else copyswap(result_ptr, self_ptr, 0, self); @@ -1448,8 +1444,8 @@ mapiter_trivial_@name@(PyArrayObject *self, PyArrayObject *ind, #else /* !@isget@ */ #if @elsize@ - assert(npy_is_aligned(result_ptr, _ALIGN(@copytype@))); - assert(npy_is_aligned(self_ptr, _ALIGN(@copytype@))); + assert(npy_is_aligned(result_ptr, _UINT_ALIGN(@copytype@))); + assert(npy_is_aligned(self_ptr, _UINT_ALIGN(@copytype@))); *(@copytype@ *)self_ptr = *(@copytype@ *)result_ptr; #else copyswap(self_ptr, result_ptr, 0, self); @@ -1571,7 +1567,7 @@ mapiter_@name@(PyArrayMapIterObject *mit) for (i=0; i < @numiter@; i++) { npy_intp indval = *((npy_intp*)outer_ptrs[i]); assert(npy_is_aligned(outer_ptrs[i], - _ALIGN(npy_intp))); + _UINT_ALIGN(npy_intp))); #if @isget@ && @one_iter@ if (check_and_adjust_index(&indval, fancy_dims[i], @@ -1591,16 +1587,20 @@ mapiter_@name@(PyArrayMapIterObject *mit) #if @isget@ #if @elsize@ - assert(npy_is_aligned(outer_ptrs[i], _ALIGN(@copytype@))); - assert(npy_is_aligned(self_ptr, _ALIGN(@copytype@))); + assert(npy_is_aligned(outer_ptrs[i], + _UINT_ALIGN(@copytype@))); + assert(npy_is_aligned(self_ptr, + _UINT_ALIGN(@copytype@))); *(@copytype@ *)(outer_ptrs[i]) = *(@copytype@ *)self_ptr; #else copyswap(outer_ptrs[i], self_ptr, 0, array); #endif #else /* !@isget@ */ #if @elsize@ - assert(npy_is_aligned(outer_ptrs[i], _ALIGN(@copytype@))); - assert(npy_is_aligned(self_ptr, _ALIGN(@copytype@))); + assert(npy_is_aligned(outer_ptrs[i], + _UINT_ALIGN(@copytype@))); + assert(npy_is_aligned(self_ptr, + _UINT_ALIGN(@copytype@))); *(@copytype@ *)self_ptr = *(@copytype@ *)(outer_ptrs[i]); #else copyswap(self_ptr, outer_ptrs[i], 0, array); diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index d371ae762..17edd2bbf 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -1064,7 +1064,8 @@ array_boolean_subscript(PyArrayObject *self, /* Get a dtype transfer function */ NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); - if (PyArray_GetDTypeTransferFunction(IsUintAligned(self), + if (PyArray_GetDTypeTransferFunction( + IsUintAligned(self) && IsAligned(self), fixed_strides[0], itemsize, dtype, dtype, 0, @@ -1253,7 +1254,8 @@ array_assign_boolean_subscript(PyArrayObject *self, /* Get a dtype transfer function */ NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); if (PyArray_GetDTypeTransferFunction( - IsUintAligned(self) && IsUintAligned(v), + IsUintAligned(self) && IsAligned(self) && + IsUintAligned(v) && IsAligned(v), v_stride, fixed_strides[0], PyArray_DESCR(v), PyArray_DESCR(self), 0, @@ -1389,54 +1391,14 @@ array_subscript_asarray(PyArrayObject *self, PyObject *op) } /* - * Helper function for _get_field_view which turns a multifield - * view into a "packed" copy, as done in numpy 1.15 and before. - * In numpy 1.16 this function should be removed. - */ -NPY_NO_EXPORT int -_multifield_view_to_copy(PyArrayObject **view) { - static PyObject *copyfunc = NULL; - PyObject *viewcopy; - - /* return a repacked copy of the view */ - npy_cache_import("numpy.lib.recfunctions", "repack_fields", ©func); - if (copyfunc == NULL) { - goto view_fail; - } - - PyArray_CLEARFLAGS(*view, NPY_ARRAY_WARN_ON_WRITE); - viewcopy = PyObject_CallFunction(copyfunc, "O", *view); - if (viewcopy == NULL) { - goto view_fail; - } - Py_DECREF(*view); - *view = (PyArrayObject*)viewcopy; - - /* warn when writing to the copy */ - PyArray_ENABLEFLAGS(*view, NPY_ARRAY_WARN_ON_WRITE); - return 0; - -view_fail: - Py_DECREF(*view); - *view = NULL; - return 0; -} - -/* * Attempts to subscript an array using a field name or list of field names. * * If an error occurred, return 0 and set view to NULL. If the subscript is not * a string or list of strings, return -1 and set view to NULL. Otherwise * return 0 and set view to point to a new view into arr for the given fields. - * - * In numpy 1.15 and before, in the case of a list of field names the returned - * view will actually be a copy by default, with fields packed together. - * The `force_view` argument causes a view to be returned. This argument can be - * removed in 1.16 when we plan to return a view always. */ NPY_NO_EXPORT int -_get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view, - int force_view) +_get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view) { *view = NULL; @@ -1597,11 +1559,7 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view, return 0; } - /* the code below can be replaced by "return 0" in 1.16 */ - if (force_view) { - return 0; - } - return _multifield_view_to_copy(view); + return 0; } return -1; } @@ -1629,7 +1587,7 @@ array_subscript(PyArrayObject *self, PyObject *op) /* return fields if op is a string index */ if (PyDataType_HASFIELDS(PyArray_DESCR(self))) { PyArrayObject *view; - int ret = _get_field_view(self, op, &view, 0); + int ret = _get_field_view(self, op, &view); if (ret == 0){ if (view == NULL) { return NULL; @@ -1911,7 +1869,7 @@ array_assign_subscript(PyArrayObject *self, PyObject *ind, PyObject *op) /* field access */ if (PyDataType_HASFIELDS(PyArray_DESCR(self))){ PyArrayObject *view; - int ret = _get_field_view(self, ind, &view, 1); + int ret = _get_field_view(self, ind, &view); if (ret == 0){ if (view == NULL) { return -1; diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 23b0bfd24..252378828 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -8,6 +8,7 @@ #include "numpy/arrayobject.h" #include "numpy/arrayscalars.h" +#include "arrayfunction_override.h" #include "npy_config.h" #include "npy_pycompat.h" #include "npy_import.h" @@ -187,7 +188,7 @@ array_reshape(PyArrayObject *self, PyObject *args, PyObject *kwds) } if (n <= 1) { - if (PyTuple_GET_ITEM(args, 0) == Py_None) { + if (n != 0 && PyTuple_GET_ITEM(args, 0) == Py_None) { return PyArray_View(self, NULL, NULL); } if (!PyArg_ParseTuple(args, "O&:reshape", PyArray_IntpConverter, @@ -988,8 +989,58 @@ array_getarray(PyArrayObject *self, PyObject *args) } } +/* + * Check whether any of a set of input and output args have a non-default + * __array_ufunc__ method. Return 1 if so, 0 if not, and -1 on error. + * + * This function primarily exists to help ndarray.__array_ufunc__ determine + * whether it can support a ufunc (which is the case only if none of the + * operands have an override). Thus, unlike in umath/override.c, the + * actual overrides are not needed and one can stop looking once one is found. + */ +static int +any_array_ufunc_overrides(PyObject *args, PyObject *kwds) +{ + int i; + int nin, nout; + PyObject *out_kwd_obj; + PyObject *fast; + PyObject **in_objs, **out_objs; -static PyObject * + /* check inputs */ + nin = PyTuple_Size(args); + if (nin < 0) { + return -1; + } + fast = PySequence_Fast(args, "Could not convert object to sequence"); + if (fast == NULL) { + return -1; + } + in_objs = PySequence_Fast_ITEMS(fast); + for (i = 0; i < nin; ++i) { + if (PyUFunc_HasOverride(in_objs[i])) { + Py_DECREF(fast); + return 1; + } + } + Py_DECREF(fast); + /* check outputs, if any */ + nout = PyUFuncOverride_GetOutObjects(kwds, &out_kwd_obj, &out_objs); + if (nout < 0) { + return -1; + } + for (i = 0; i < nout; i++) { + if (PyUFunc_HasOverride(out_objs[i])) { + Py_DECREF(out_kwd_obj); + return 1; + } + } + Py_DECREF(out_kwd_obj); + return 0; +} + + +NPY_NO_EXPORT PyObject * array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) { PyObject *ufunc, *method_name, *normal_args, *ufunc_method; @@ -1009,7 +1060,7 @@ array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } /* ndarray cannot handle overrides itself */ - has_override = PyUFunc_HasOverride(normal_args, kwds); + has_override = any_array_ufunc_overrides(normal_args, kwds); if (has_override < 0) { goto cleanup; } @@ -1038,13 +1089,29 @@ cleanup: return result; } - static PyObject * -array_function(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_function(PyArrayObject *self, PyObject *c_args, PyObject *c_kwds) { - NPY_FORWARD_NDARRAY_METHOD("_array_function"); -} + PyObject *func, *types, *args, *kwargs, *result; + static char *kwlist[] = {"func", "types", "args", "kwargs", NULL}; + + if (!PyArg_ParseTupleAndKeywords( + c_args, c_kwds, "OOOO:__array_function__", kwlist, + &func, &types, &args, &kwargs)) { + return NULL; + } + + types = PySequence_Fast( + types, + "types argument to ndarray.__array_function__ must be iterable"); + if (types == NULL) { + return NULL; + } + result = array_function_method_impl(func, types, args, kwargs); + Py_DECREF(types); + return result; +} static PyObject * array_copy(PyArrayObject *self, PyObject *args, PyObject *kwds) @@ -1314,6 +1381,7 @@ array_argsort(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } newd = PyArray_DescrNew(saved); + Py_DECREF(newd->names); newd->names = new_name; ((PyArrayObject_fields *)self)->descr = newd; } @@ -1368,6 +1436,7 @@ array_argpartition(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } newd = PyArray_DescrNew(saved); + Py_DECREF(newd->names); newd->names = new_name; ((PyArrayObject_fields *)self)->descr = newd; } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 8f782cff6..166533b3f 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -34,6 +34,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; /* Internal APIs */ +#include "arrayfunction_override.h" #include "arraytypes.h" #include "arrayobject.h" #include "hashdescr.h" @@ -72,10 +73,10 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; ***************************************************************************** */ #include "funcs.inc" -#include "loops.h" #include "umathmodule.h" NPY_NO_EXPORT int initscalarmath(PyObject *); +NPY_NO_EXPORT int set_matmul_flags(PyObject *d); /* in ufunc_object.c */ /* * global variable to determine if legacy printing is enabled, accessible from @@ -833,7 +834,10 @@ PyArray_InnerProduct(PyObject *op1, PyObject *op2) typenum = PyArray_ObjectType(op2, typenum); typec = PyArray_DescrFromType(typenum); if (typec == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot find a common data type."); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "Cannot find a common data type."); + } goto fail; } @@ -919,7 +923,10 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) typenum = PyArray_ObjectType(op2, typenum); typec = PyArray_DescrFromType(typenum); if (typec == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot find a common data type."); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "Cannot find a common data type."); + } return NULL; } @@ -976,7 +983,7 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) for (i = 0; i < PyArray_NDIM(ap2) - 2; i++) { dimensions[j++] = PyArray_DIMS(ap2)[i]; } - if(PyArray_NDIM(ap2) > 1) { + if (PyArray_NDIM(ap2) > 1) { dimensions[j++] = PyArray_DIMS(ap2)[PyArray_NDIM(ap2)-1]; } @@ -1312,7 +1319,7 @@ PyArray_Correlate2(PyObject *op1, PyObject *op2, int mode) */ if (inverted) { st = _pyarray_revert(ret); - if(st) { + if (st) { goto clean_ret; } } @@ -1359,7 +1366,7 @@ PyArray_Correlate(PyObject *op1, PyObject *op2, int mode) } ret = _pyarray_correlate(ap1, ap2, typenum, mode, &unused); - if(ret == NULL) { + if (ret == NULL) { goto fail; } Py_DECREF(ap1); @@ -1648,7 +1655,7 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) } full_path: - if(!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&i:array", kwd, + if (!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&i:array", kwd, &op, PyArray_DescrConverter2, &type, PyArray_BoolConverter, ©, @@ -2044,6 +2051,7 @@ static PyObject * array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) { PyObject *file = NULL, *ret; + PyObject *err_type = NULL, *err_value = NULL, *err_traceback = NULL; char *sep = ""; Py_ssize_t nin = -1; static char *kwlist[] = {"file", "dtype", "count", "sep", NULL}; @@ -2079,18 +2087,26 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) } ret = PyArray_FromFile(fp, type, (npy_intp) nin, sep); + /* If an exception is thrown in the call to PyArray_FromFile + * we need to clear it, and restore it later to ensure that + * we can cleanup the duplicated file descriptor properly. + */ + PyErr_Fetch(&err_type, &err_value, &err_traceback); if (npy_PyFile_DupClose2(file, fp, orig_pos) < 0) { + npy_PyErr_ChainExceptions(err_type, err_value, err_traceback); goto fail; } if (own && npy_PyFile_CloseFile(file) < 0) { + npy_PyErr_ChainExceptions(err_type, err_value, err_traceback); goto fail; } + PyErr_Restore(err_type, err_value, err_traceback); Py_DECREF(file); return ret; fail: Py_DECREF(file); - Py_DECREF(ret); + Py_XDECREF(ret); return NULL; } @@ -2303,154 +2319,6 @@ fail: return NULL; } - - -/* - * matmul - * - * Implements the protocol used by the '@' operator defined in PEP 364. - * Not in the NUMPY API at this time, maybe later. - * - * - * in1: Left hand side operand - * in2: Right hand side operand - * out: Either NULL, or an array into which the output should be placed. - * - * Returns NULL on error. - */ -static PyObject * -array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) -{ - PyObject *in1, *in2, *out = NULL; - char* kwlist[] = {"a", "b", "out", NULL }; - PyArrayObject *ap1, *ap2, *ret = NULL; - NPY_ORDER order = NPY_KEEPORDER; - NPY_CASTING casting = NPY_SAFE_CASTING; - PyArray_Descr *dtype; - int nd1, nd2, typenum; - char *subscripts; - PyArrayObject *ops[2]; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matmul", kwlist, - &in1, &in2, &out)) { - return NULL; - } - - if (out != NULL) { - if (out == Py_None) { - out = NULL; - } - else if (!PyArray_Check(out)) { - PyErr_SetString(PyExc_TypeError, "'out' must be an array"); - return NULL; - } - } - - dtype = PyArray_DescrFromObject(in1, NULL); - dtype = PyArray_DescrFromObject(in2, dtype); - if (dtype == NULL) { - PyErr_SetString(PyExc_ValueError, "Cannot find a common data type."); - return NULL; - } - typenum = dtype->type_num; - - if (typenum == NPY_OBJECT) { - /* matmul is not currently implemented for object arrays */ - PyErr_SetString(PyExc_TypeError, - "Object arrays are not currently supported"); - Py_DECREF(dtype); - return NULL; - } - - ap1 = (PyArrayObject *)PyArray_FromAny(in1, dtype, 0, 0, - NPY_ARRAY_ALIGNED, NULL); - if (ap1 == NULL) { - return NULL; - } - - Py_INCREF(dtype); - ap2 = (PyArrayObject *)PyArray_FromAny(in2, dtype, 0, 0, - NPY_ARRAY_ALIGNED, NULL); - if (ap2 == NULL) { - Py_DECREF(ap1); - return NULL; - } - - if (PyArray_NDIM(ap1) == 0 || PyArray_NDIM(ap2) == 0) { - /* Scalars are rejected */ - PyErr_SetString(PyExc_ValueError, - "Scalar operands are not allowed, use '*' instead"); - return NULL; - } - - nd1 = PyArray_NDIM(ap1); - nd2 = PyArray_NDIM(ap2); - -#if defined(HAVE_CBLAS) - if (nd1 <= 2 && nd2 <= 2 && - (NPY_DOUBLE == typenum || NPY_CDOUBLE == typenum || - NPY_FLOAT == typenum || NPY_CFLOAT == typenum)) { - return cblas_matrixproduct(typenum, ap1, ap2, (PyArrayObject *)out); - } -#endif - - /* - * Use einsum for the stacked cases. This is a quick implementation - * to avoid setting up the proper iterators. Einsum broadcasts, so - * we need to check dimensions before the call. - */ - if (nd1 == 1 && nd2 == 1) { - /* vector vector */ - if (PyArray_DIM(ap1, 0) != PyArray_DIM(ap2, 0)) { - dot_alignment_error(ap1, 0, ap2, 0); - goto fail; - } - subscripts = "i, i"; - } - else if (nd1 == 1) { - /* vector matrix */ - if (PyArray_DIM(ap1, 0) != PyArray_DIM(ap2, nd2 - 2)) { - dot_alignment_error(ap1, 0, ap2, nd2 - 2); - goto fail; - } - subscripts = "i, ...ij"; - } - else if (nd2 == 1) { - /* matrix vector */ - if (PyArray_DIM(ap1, nd1 - 1) != PyArray_DIM(ap2, 0)) { - dot_alignment_error(ap1, nd1 - 1, ap2, 0); - goto fail; - } - subscripts = "...i, i"; - } - else { - /* matrix * matrix */ - if (PyArray_DIM(ap1, nd1 - 1) != PyArray_DIM(ap2, nd2 - 2)) { - dot_alignment_error(ap1, nd1 - 1, ap2, nd2 - 2); - goto fail; - } - subscripts = "...ij, ...jk"; - } - ops[0] = ap1; - ops[1] = ap2; - ret = PyArray_EinsteinSum(subscripts, 2, ops, NULL, order, casting, - (PyArrayObject *)out); - Py_DECREF(ap1); - Py_DECREF(ap2); - - /* If no output was supplied, possibly convert to a scalar */ - if (ret != NULL && out == NULL) { - return PyArray_Return((PyArrayObject *)ret); - } - return (PyObject *)ret; - -fail: - Py_XDECREF(ap1); - Py_XDECREF(ap2); - return NULL; -} - - static int einsum_sub_op_from_str(PyObject *args, PyObject **str_obj, char **subscripts, PyArrayObject **op) @@ -2622,7 +2490,7 @@ einsum_sub_op_from_lists(PyObject *args, "operand and a subscripts list to einsum"); return -1; } - else if(nop >= NPY_MAXARGS) { + else if (nop >= NPY_MAXARGS) { PyErr_SetString(PyExc_ValueError, "too many operands"); return -1; } @@ -2857,7 +2725,7 @@ array_arange(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) { static char *kwd[]= {"start", "stop", "step", "dtype", NULL}; PyArray_Descr *typecode = NULL; - if(!PyArg_ParseTupleAndKeywords(args, kws, "O|OOO&:arange", kwd, + if (!PyArg_ParseTupleAndKeywords(args, kws, "O|OOO&:arange", kwd, &o_start, &o_stop, &o_step, @@ -2895,7 +2763,7 @@ array__get_ndarray_c_version(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObje { static char *kwlist[] = {NULL}; - if(!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist )) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist )) { return NULL; } return PyInt_FromLong( (long) PyArray_GetNDArrayCVersion() ); @@ -2968,7 +2836,7 @@ array_set_string_function(PyObject *NPY_UNUSED(self), PyObject *args, int repr = 1; static char *kwlist[] = {"f", "repr", NULL}; - if(!PyArg_ParseTupleAndKeywords(args, kwds, "|Oi:set_string_function", kwlist, &op, &repr)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oi:set_string_function", kwlist, &op, &repr)) { return NULL; } /* reset the array_repr function to built-in */ @@ -2990,7 +2858,7 @@ array_set_ops_function(PyObject *NPY_UNUSED(self), PyObject *NPY_UNUSED(args), { PyObject *oldops = NULL; - if ((oldops = PyArray_GetNumericOps()) == NULL) { + if ((oldops = _PyArray_GetNumericOps()) == NULL) { return NULL; } /* @@ -3000,8 +2868,10 @@ array_set_ops_function(PyObject *NPY_UNUSED(self), PyObject *NPY_UNUSED(args), */ if (kwds && PyArray_SetNumericOps(kwds) == -1) { Py_DECREF(oldops); - PyErr_SetString(PyExc_ValueError, + if (PyErr_Occurred() == NULL) { + PyErr_SetString(PyExc_ValueError, "one or more objects not callable"); + } return NULL; } return oldops; @@ -3276,7 +3146,7 @@ array_promote_types(PyObject *NPY_UNUSED(dummy), PyObject *args) PyArray_Descr *d1 = NULL; PyArray_Descr *d2 = NULL; PyObject *ret = NULL; - if(!PyArg_ParseTuple(args, "O&O&:promote_types", + if (!PyArg_ParseTuple(args, "O&O&:promote_types", PyArray_DescrConverter2, &d1, PyArray_DescrConverter2, &d2)) { goto finish; } @@ -3302,7 +3172,7 @@ array_min_scalar_type(PyObject *NPY_UNUSED(dummy), PyObject *args) PyArrayObject *array; PyObject *ret = NULL; - if(!PyArg_ParseTuple(args, "O:min_scalar_type", &array_in)) { + if (!PyArg_ParseTuple(args, "O:min_scalar_type", &array_in)) { return NULL; } @@ -3379,12 +3249,13 @@ array_datetime_data(PyObject *NPY_UNUSED(dummy), PyObject *args) PyArray_Descr *dtype; PyArray_DatetimeMetaData *meta; - if(!PyArg_ParseTuple(args, "O&:datetime_data", + if (!PyArg_ParseTuple(args, "O&:datetime_data", PyArray_DescrConverter, &dtype)) { return NULL; } meta = get_datetime_metadata_from_dtype(dtype); + Py_DECREF(dtype); if (meta == NULL) { return NULL; } @@ -3398,7 +3269,7 @@ new_buffer(PyObject *NPY_UNUSED(dummy), PyObject *args) { int size; - if(!PyArg_ParseTuple(args, "i:buffer", &size)) { + if (!PyArg_ParseTuple(args, "i:buffer", &size)) { return NULL; } return PyBuffer_New(size); @@ -3749,6 +3620,7 @@ _vec_string_with_args(PyArrayObject* char_array, PyArray_Descr* type, if (nargs == -1 || nargs > NPY_MAXARGS) { PyErr_Format(PyExc_ValueError, "len(args) must be < %d", NPY_MAXARGS - 1); + Py_DECREF(type); goto err; } @@ -3756,6 +3628,7 @@ _vec_string_with_args(PyArrayObject* char_array, PyArray_Descr* type, for (i = 1; i < nargs; i++) { PyObject* item = PySequence_GetItem(args, i-1); if (item == NULL) { + Py_DECREF(type); goto err; } broadcast_args[i] = item; @@ -3764,6 +3637,7 @@ _vec_string_with_args(PyArrayObject* char_array, PyArray_Descr* type, in_iter = (PyArrayMultiIterObject*)PyArray_MultiIterFromObjects (broadcast_args, nargs, 0); if (in_iter == NULL) { + Py_DECREF(type); goto err; } n = in_iter->numiter; @@ -3844,6 +3718,7 @@ _vec_string_no_args(PyArrayObject* char_array, in_iter = (PyArrayIterObject*)PyArray_IterNew((PyObject*)char_array); if (in_iter == NULL) { + Py_DECREF(type); goto err; } @@ -3900,7 +3775,7 @@ static PyObject * _vec_string(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) { PyArrayObject* char_array = NULL; - PyArray_Descr *type = NULL; + PyArray_Descr *type; PyObject* method_name; PyObject* args_seq = NULL; @@ -3937,6 +3812,7 @@ _vec_string(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) result = _vec_string_with_args(char_array, type, method, args_seq); } else { + Py_DECREF(type); PyErr_SetString(PyExc_TypeError, "'args' must be a sequence of arguments"); goto err; @@ -4193,6 +4069,9 @@ normalize_axis_index(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) } static struct PyMethodDef array_module_methods[] = { + {"_get_implementing_args", + (PyCFunction)array__get_implementing_args, + METH_VARARGS, NULL}, {"_get_ndarray_c_version", (PyCFunction)array__get_ndarray_c_version, METH_VARARGS|METH_KEYWORDS, NULL}, @@ -4265,9 +4144,6 @@ static struct PyMethodDef array_module_methods[] = { {"vdot", (PyCFunction)array_vdot, METH_VARARGS | METH_KEYWORDS, NULL}, - {"matmul", - (PyCFunction)array_matmul, - METH_VARARGS | METH_KEYWORDS, NULL}, {"c_einsum", (PyCFunction)array_einsum, METH_VARARGS|METH_KEYWORDS, NULL}, @@ -4358,6 +4234,9 @@ static struct PyMethodDef array_module_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"_monotonicity", (PyCFunction)arr__monotonicity, METH_VARARGS | METH_KEYWORDS, NULL}, + {"implement_array_function", + (PyCFunction)array_implement_array_function, + METH_VARARGS, NULL}, {"interp", (PyCFunction)arr_interp, METH_VARARGS | METH_KEYWORDS, NULL}, {"interp_complex", (PyCFunction)arr_interp_complex, @@ -4610,6 +4489,7 @@ NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_array_wrap = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_array_finalize = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_buffer = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_ufunc = NULL; +NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_wrapped = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_order = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_copy = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_dtype = NULL; @@ -4626,6 +4506,7 @@ intern_strings(void) npy_ma_str_array_finalize = PyUString_InternFromString("__array_finalize__"); npy_ma_str_buffer = PyUString_InternFromString("__buffer__"); npy_ma_str_ufunc = PyUString_InternFromString("__array_ufunc__"); + npy_ma_str_wrapped = PyUString_InternFromString("__wrapped__"); npy_ma_str_order = PyUString_InternFromString("order"); npy_ma_str_copy = PyUString_InternFromString("copy"); npy_ma_str_dtype = PyUString_InternFromString("dtype"); @@ -4635,12 +4516,11 @@ intern_strings(void) return npy_ma_str_array && npy_ma_str_array_prepare && npy_ma_str_array_wrap && npy_ma_str_array_finalize && - npy_ma_str_buffer && npy_ma_str_ufunc && + npy_ma_str_buffer && npy_ma_str_ufunc && npy_ma_str_wrapped && npy_ma_str_order && npy_ma_str_copy && npy_ma_str_dtype && npy_ma_str_ndmin && npy_ma_str_axis1 && npy_ma_str_axis2; } - #if defined(NPY_PY3K) static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, @@ -4705,15 +4585,23 @@ PyMODINIT_FUNC init_multiarray_umath(void) { */ PyArray_Type.tp_hash = PyObject_HashNotImplemented; + if (PyType_Ready(&PyUFunc_Type) < 0) { + goto err; + } + /* Load the ufunc operators into the array module's namespace */ if (InitOperators(d) < 0) { goto err; } + if (set_matmul_flags(d) < 0) { + goto err; + } initialize_casting_tables(); initialize_numeric_types(); - if(initscalarmath(m) < 0) + if (initscalarmath(m) < 0) { goto err; + } if (PyType_Ready(&PyArray_Type) < 0) { goto err; diff --git a/numpy/core/src/multiarray/multiarraymodule.h b/numpy/core/src/multiarray/multiarraymodule.h index 3de68c549..60a3965c9 100644 --- a/numpy/core/src/multiarray/multiarraymodule.h +++ b/numpy/core/src/multiarray/multiarraymodule.h @@ -7,6 +7,7 @@ NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_array_wrap; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_array_finalize; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_buffer; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_ufunc; +NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_wrapped; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_order; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_copy; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_dtype; diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index dbb24f26b..ba7041f32 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -1132,7 +1132,7 @@ npyiter_prepare_one_operand(PyArrayObject **op, /* Check if the operand is aligned */ if (op_flags & NPY_ITER_ALIGNED) { /* Check alignment */ - if (!IsUintAligned(*op)) { + if (!(IsUintAligned(*op) && IsAligned(*op))) { NPY_IT_DBG_PRINT("Iterator: Setting NPY_OP_ITFLAG_CAST " "because of NPY_ITER_ALIGNED\n"); *op_itflags |= NPY_OP_ITFLAG_CAST; @@ -1248,9 +1248,9 @@ npyiter_prepare_operands(int nop, PyArrayObject **op_in, return 1; fail_nop: - iop = nop; + iop = nop - 1; fail_iop: - for (i = 0; i < iop; ++i) { + for (i = 0; i < iop+1; ++i) { Py_XDECREF(op[i]); Py_XDECREF(op_dtype[i]); } @@ -2851,8 +2851,14 @@ npyiter_allocate_arrays(NpyIter *iter, npyiter_replace_axisdata(iter, iop, op[iop], ondim, PyArray_DATA(op[iop]), op_axes ? op_axes[iop] : NULL); - /* New arrays are aligned and need no cast */ - op_itflags[iop] |= NPY_OP_ITFLAG_ALIGNED; + /* + * New arrays are guaranteed true-aligned, but copy/cast code + * needs uint-alignment in addition. + */ + if (IsUintAligned(out)) { + op_itflags[iop] |= NPY_OP_ITFLAG_ALIGNED; + } + /* New arrays need no cast */ op_itflags[iop] &= ~NPY_OP_ITFLAG_CAST; } /* @@ -2888,11 +2894,17 @@ npyiter_allocate_arrays(NpyIter *iter, PyArray_DATA(op[iop]), NULL); /* - * New arrays are aligned need no cast, and in the case + * New arrays are guaranteed true-aligned, but copy/cast code + * needs uint-alignment in addition. + */ + if (IsUintAligned(temp)) { + op_itflags[iop] |= NPY_OP_ITFLAG_ALIGNED; + } + /* + * New arrays need no cast, and in the case * of scalars, always have stride 0 so never need buffering */ - op_itflags[iop] |= (NPY_OP_ITFLAG_ALIGNED | - NPY_OP_ITFLAG_BUFNEVER); + op_itflags[iop] |= NPY_OP_ITFLAG_BUFNEVER; op_itflags[iop] &= ~NPY_OP_ITFLAG_CAST; if (itflags & NPY_ITFLAG_BUFFER) { NBF_STRIDES(bufferdata)[iop] = 0; @@ -2953,8 +2965,14 @@ npyiter_allocate_arrays(NpyIter *iter, npyiter_replace_axisdata(iter, iop, op[iop], ondim, PyArray_DATA(op[iop]), op_axes ? op_axes[iop] : NULL); - /* The temporary copy is aligned and needs no cast */ - op_itflags[iop] |= NPY_OP_ITFLAG_ALIGNED; + /* + * New arrays are guaranteed true-aligned, but copy/cast code + * additionally needs uint-alignment in addition. + */ + if (IsUintAligned(temp)) { + op_itflags[iop] |= NPY_OP_ITFLAG_ALIGNED; + } + /* The temporary copy needs no cast */ op_itflags[iop] &= ~NPY_OP_ITFLAG_CAST; } else { @@ -3157,6 +3175,7 @@ npyiter_allocate_transfer_functions(NpyIter *iter) &stransfer, &transferdata, &needs_api) != NPY_SUCCEED) { + iop -= 1; /* This one cannot be cleaned up yet. */ goto fail; } readtransferfn[iop] = stransfer; @@ -3250,7 +3269,7 @@ npyiter_allocate_transfer_functions(NpyIter *iter) return 1; fail: - for (i = 0; i < iop; ++i) { + for (i = 0; i < iop+1; ++i) { if (readtransferdata[iop] != NULL) { NPY_AUXDATA_FREE(readtransferdata[iop]); readtransferdata[iop] = NULL; diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 5a9f3c5fa..30a81e0ca 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -2355,6 +2355,8 @@ npyiter_close(NewNpyArrayIterObject *self) } ret = NpyIter_Deallocate(iter); self->iter = NULL; + Py_XDECREF(self->nested_child); + self->nested_child = NULL; if (ret < 0) { return NULL; } diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index dabbae064..420501ce2 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -71,12 +71,8 @@ array_inplace_power(PyArrayObject *a1, PyObject *o2, PyObject *NPY_UNUSED(modulo n_ops.op = temp; \ } - -/*NUMPY_API - *Set internal structure with number functions that all arrays will use - */ NPY_NO_EXPORT int -PyArray_SetNumericOps(PyObject *dict) +_PyArray_SetNumericOps(PyObject *dict) { PyObject *temp = NULL; SET(add); @@ -116,19 +112,32 @@ PyArray_SetNumericOps(PyObject *dict) SET(minimum); SET(rint); SET(conjugate); + SET(matmul); return 0; } +/*NUMPY_API + *Set internal structure with number functions that all arrays will use + */ +NPY_NO_EXPORT int +PyArray_SetNumericOps(PyObject *dict) +{ + /* 2018-09-09, 1.16 */ + if (DEPRECATE("PyArray_SetNumericOps is deprecated. Use " + "PyUFunc_ReplaceLoopBySignature to replace ufunc inner loop functions " + "instead.") < 0) { + return -1; + } + return _PyArray_SetNumericOps(dict); +} + /* Note - macro contains goto */ #define GET(op) if (n_ops.op && \ (PyDict_SetItemString(dict, #op, n_ops.op)==-1)) \ goto fail; -/*NUMPY_API - Get dictionary showing number functions that all arrays will use -*/ NPY_NO_EXPORT PyObject * -PyArray_GetNumericOps(void) +_PyArray_GetNumericOps(void) { PyObject *dict; if ((dict = PyDict_New())==NULL) @@ -169,6 +178,7 @@ PyArray_GetNumericOps(void) GET(minimum); GET(rint); GET(conjugate); + GET(matmul); return dict; fail: @@ -176,6 +186,19 @@ PyArray_GetNumericOps(void) return NULL; } +/*NUMPY_API + Get dictionary showing number functions that all arrays will use +*/ +NPY_NO_EXPORT PyObject * +PyArray_GetNumericOps(void) +{ + /* 2018-09-09, 1.16 */ + if (DEPRECATE("PyArray_GetNumericOps is deprecated.") < 0) { + return NULL; + } + return _PyArray_GetNumericOps(); +} + static PyObject * _get_keywords(int rtype, PyArrayObject *out) { @@ -361,14 +384,8 @@ array_divmod(PyArrayObject *m1, PyObject *m2) static PyObject * array_matrix_multiply(PyArrayObject *m1, PyObject *m2) { - static PyObject *matmul = NULL; - - npy_cache_import("numpy.core.multiarray", "matmul", &matmul); - if (matmul == NULL) { - return NULL; - } BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_matrix_multiply, array_matrix_multiply); - return PyArray_GenericBinaryFunction(m1, m2, matmul); + return PyArray_GenericBinaryFunction(m1, m2, n_ops.matmul); } static PyObject * @@ -578,19 +595,20 @@ array_positive(PyArrayObject *m1) */ PyObject *exc, *val, *tb; PyErr_Fetch(&exc, &val, &tb); - if (has_non_default_array_ufunc((PyObject *)m1)) { + if (PyUFunc_HasOverride((PyObject *)m1)) { PyErr_Restore(exc, val, tb); return NULL; } + Py_XDECREF(exc); + Py_XDECREF(val); + Py_XDECREF(tb); + /* 2018-06-28, 1.16.0 */ if (DEPRECATE("Applying '+' to a non-numerical array is " "ill-defined. Returning a copy, but in the future " "this will error.") < 0) { return NULL; } - Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); value = PyArray_Return((PyArrayObject *)PyArray_Copy(m1)); } return value; diff --git a/numpy/core/src/multiarray/number.h b/numpy/core/src/multiarray/number.h index 99a2a722b..33a7cf872 100644 --- a/numpy/core/src/multiarray/number.h +++ b/numpy/core/src/multiarray/number.h @@ -39,6 +39,7 @@ typedef struct { PyObject *minimum; PyObject *rint; PyObject *conjugate; + PyObject *matmul; } NumericOps; extern NPY_NO_EXPORT NumericOps n_ops; @@ -48,10 +49,10 @@ NPY_NO_EXPORT PyObject * array_int(PyArrayObject *v); NPY_NO_EXPORT int -PyArray_SetNumericOps(PyObject *dict); +_PyArray_SetNumericOps(PyObject *dict); NPY_NO_EXPORT PyObject * -PyArray_GetNumericOps(void); +_PyArray_GetNumericOps(void); NPY_NO_EXPORT PyObject * PyArray_GenericBinaryFunction(PyArrayObject *m1, PyObject *m2, PyObject *op); diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index 5ef6c0bbf..bc435d1ca 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -471,7 +471,7 @@ PyArray_DescrFromTypeObject(PyObject *type) /* Do special thing for VOID sub-types */ if (PyType_IsSubtype((PyTypeObject *)type, &PyVoidArrType_Type)) { new = PyArray_DescrNewFromType(NPY_VOID); - conv = _arraydescr_fromobj(type); + conv = _arraydescr_from_dtype_attr(type); if (conv) { new->fields = conv->fields; Py_INCREF(new->fields); diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 6dd8b1a29..2f71c8ae9 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -139,6 +139,7 @@ gentype_alloc(PyTypeObject *type, Py_ssize_t nitems) static void gentype_dealloc(PyObject *v) { + _dealloc_cached_buffer_info(v); Py_TYPE(v)->tp_free(v); } @@ -1103,8 +1104,7 @@ static PyNumberMethods gentype_as_number = { (binaryfunc)gentype_add, /*nb_add*/ (binaryfunc)gentype_subtract, /*nb_subtract*/ (binaryfunc)gentype_multiply, /*nb_multiply*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) (binaryfunc)gentype_divide, /*nb_divide*/ #endif (binaryfunc)gentype_remainder, /*nb_remainder*/ @@ -1120,8 +1120,7 @@ static PyNumberMethods gentype_as_number = { (binaryfunc)gentype_and, /*nb_and*/ (binaryfunc)gentype_xor, /*nb_xor*/ (binaryfunc)gentype_or, /*nb_or*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) 0, /*nb_coerce*/ #endif (unaryfunc)gentype_int, /*nb_int*/ @@ -1131,16 +1130,14 @@ static PyNumberMethods gentype_as_number = { (unaryfunc)gentype_long, /*nb_long*/ #endif (unaryfunc)gentype_float, /*nb_float*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) (unaryfunc)gentype_oct, /*nb_oct*/ (unaryfunc)gentype_hex, /*nb_hex*/ #endif 0, /*inplace_add*/ 0, /*inplace_subtract*/ 0, /*inplace_multiply*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) 0, /*inplace_divide*/ #endif 0, /*inplace_remainder*/ @@ -1155,6 +1152,10 @@ static PyNumberMethods gentype_as_number = { 0, /*nb_inplace_floor_divide*/ 0, /*nb_inplace_true_divide*/ (unaryfunc)NULL, /*nb_index*/ +#if PY_VERSION_HEX >= 0x03050000 + 0, /*np_matmul*/ + 0, /*np_inplace_matmul*/ +#endif }; @@ -1858,6 +1859,7 @@ gentype_reduce(PyObject *self, PyObject *NPY_UNUSED(args)) * sticks around after the release. */ PyBuffer_Release(&view); + _dealloc_cached_buffer_info(self); } else { Py_DECREF(ret); diff --git a/numpy/core/src/multiarray/temp_elide.c b/numpy/core/src/multiarray/temp_elide.c index 3d2f976f2..09b948218 100644 --- a/numpy/core/src/multiarray/temp_elide.c +++ b/numpy/core/src/multiarray/temp_elide.c @@ -166,7 +166,7 @@ check_callers(int * cannot) return 0; } /* get multiarray base address */ - if (dladdr(&PyArray_SetNumericOps, &info)) { + if (dladdr(&PyArray_INCREF, &info)) { pos_ma_start = info.dli_fbase; pos_ma_end = info.dli_fbase; } diff --git a/numpy/core/src/umath/_struct_ufunc_tests.c.src b/numpy/core/src/umath/_struct_ufunc_tests.c.src index b831d5c2a..5c6e235e0 100644 --- a/numpy/core/src/umath/_struct_ufunc_tests.c.src +++ b/numpy/core/src/umath/_struct_ufunc_tests.c.src @@ -114,6 +114,7 @@ PyMODINIT_FUNC init_struct_ufunc_tests(void) dtypes, NULL); + Py_DECREF(dtype); d = PyModule_GetDict(m); PyDict_SetItemString(d, "add_triplet", add_triplet); diff --git a/numpy/core/src/umath/_umath_tests.c.src b/numpy/core/src/umath/_umath_tests.c.src index 8cb74f177..6c3bcce71 100644 --- a/numpy/core/src/umath/_umath_tests.c.src +++ b/numpy/core/src/umath/_umath_tests.c.src @@ -564,7 +564,7 @@ UMath_Tests_test_signature(PyObject *NPY_UNUSED(dummy), PyObject *args) core_dim_sizes = Py_None; } Py_DECREF(f); - return Py_BuildValue("iOOOO", core_enabled, core_num_dims, + return Py_BuildValue("iNNNN", core_enabled, core_num_dims, core_dim_ixs, core_dim_flags, core_dim_sizes); fail: diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index e62942efd..53afa817b 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -1591,6 +1591,59 @@ TIMEDELTA_mm_d_divide(char **args, npy_intp *dimensions, npy_intp *steps, void * } } +NPY_NO_EXPORT void +TIMEDELTA_mm_m_remainder(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)) +{ + BINARY_LOOP { + const npy_timedelta in1 = *(npy_timedelta *)ip1; + const npy_timedelta in2 = *(npy_timedelta *)ip2; + if (in1 == NPY_DATETIME_NAT || in2 == NPY_DATETIME_NAT) { + *((npy_timedelta *)op1) = NPY_DATETIME_NAT; + } + else { + if (in2 == 0) { + npy_set_floatstatus_divbyzero(); + *((npy_timedelta *)op1) = 0; + } + else { + /* handle mixed case the way Python does */ + const npy_timedelta rem = in1 % in2; + if ((in1 > 0) == (in2 > 0) || rem == 0) { + *((npy_timedelta *)op1) = rem; + } + else { + *((npy_timedelta *)op1) = rem + in2; + } + } + } + } +} + +NPY_NO_EXPORT void +TIMEDELTA_mm_q_floor_divide(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)) +{ + BINARY_LOOP { + const npy_timedelta in1 = *(npy_timedelta *)ip1; + const npy_timedelta in2 = *(npy_timedelta *)ip2; + if (in1 == NPY_DATETIME_NAT || in2 == NPY_DATETIME_NAT) { + npy_set_floatstatus_invalid(); + *((npy_timedelta *)op1) = 0; + } + else if (in2 == 0) { + npy_set_floatstatus_divbyzero(); + *((npy_timedelta *)op1) = 0; + } + else { + if (((in1 > 0) != (in2 > 0)) && (in1 % in2 != 0)) { + *((npy_timedelta *)op1) = in1/in2 - 1; + } + else { + *((npy_timedelta *)op1) = in1/in2; + } + } + } +} + /* ***************************************************************************** ** FLOAT LOOPS ** @@ -1833,11 +1886,9 @@ NPY_NO_EXPORT void if (!run_unary_reduce_simd_@kind@_@TYPE@(args, dimensions, steps)) { BINARY_REDUCE_LOOP(@type@) { const @type@ in2 = *(@type@ *)ip2; + /* Order of operations important for MSVC 2015 */ io1 = (io1 @OP@ in2 || npy_isnan(io1)) ? io1 : in2; } - if (npy_isnan(io1)) { - npy_set_floatstatus_invalid(); - } *((@type@ *)iop1) = io1; } } @@ -1845,13 +1896,12 @@ NPY_NO_EXPORT void BINARY_LOOP { @type@ in1 = *(@type@ *)ip1; const @type@ in2 = *(@type@ *)ip2; + /* Order of operations important for MSVC 2015 */ in1 = (in1 @OP@ in2 || npy_isnan(in1)) ? in1 : in2; - if (npy_isnan(in1)) { - npy_set_floatstatus_invalid(); - } *((@type@ *)op1) = in1; } } + npy_clear_floatstatus_barrier((char*)dimensions); } /**end repeat1**/ @@ -1866,6 +1916,7 @@ NPY_NO_EXPORT void if (IS_BINARY_REDUCE) { BINARY_REDUCE_LOOP(@type@) { const @type@ in2 = *(@type@ *)ip2; + /* Order of operations important for MSVC 2015 */ io1 = (io1 @OP@ in2 || npy_isnan(in2)) ? io1 : in2; } *((@type@ *)iop1) = io1; @@ -1874,6 +1925,7 @@ NPY_NO_EXPORT void BINARY_LOOP { const @type@ in1 = *(@type@ *)ip1; const @type@ in2 = *(@type@ *)ip2; + /* Order of operations important for MSVC 2015 */ *((@type@ *)op1) = (in1 @OP@ in2 || npy_isnan(in2)) ? in1 : in2; } } @@ -2195,14 +2247,11 @@ HALF_@kind@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED { /* */ BINARY_LOOP { - npy_half in1 = *(npy_half *)ip1; + const npy_half in1 = *(npy_half *)ip1; const npy_half in2 = *(npy_half *)ip2; - in1 = (@OP@(in1, in2) || npy_half_isnan(in1)) ? in1 : in2; - if (npy_half_isnan(in1)) { - npy_set_floatstatus_invalid(); - } - *((npy_half *)op1) = in1; + *((npy_half *)op1) = (@OP@(in1, in2) || npy_half_isnan(in1)) ? in1 : in2; } + /* npy_half_isnan will never set floatstatus_invalid, so do not clear */ } /**end repeat**/ @@ -2219,7 +2268,7 @@ HALF_@kind@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED const npy_half in2 = *(npy_half *)ip2; *((npy_half *)op1) = (@OP@(in1, in2) || npy_half_isnan(in2)) ? in1 : in2; } - npy_clear_floatstatus_barrier((char*)dimensions); + /* npy_half_isnan will never set floatstatus_invalid, so do not clear */ } /**end repeat**/ @@ -2761,16 +2810,14 @@ NPY_NO_EXPORT void @ftype@ in1i = ((@ftype@ *)ip1)[1]; const @ftype@ in2r = ((@ftype@ *)ip2)[0]; const @ftype@ in2i = ((@ftype@ *)ip2)[1]; - if ( !(@OP@(in1r, in1i, in2r, in2i) || npy_isnan(in1r) || npy_isnan(in1i))) { + if ( !(npy_isnan(in1r) || npy_isnan(in1i) || @OP@(in1r, in1i, in2r, in2i))) { in1r = in2r; in1i = in2i; } - if (npy_isnan(in1r) || npy_isnan(in1i)) { - npy_set_floatstatus_invalid(); - } ((@ftype@ *)op1)[0] = in1r; ((@ftype@ *)op1)[1] = in1i; } + npy_clear_floatstatus_barrier((char*)dimensions); } /**end repeat1**/ @@ -2786,7 +2833,7 @@ NPY_NO_EXPORT void const @ftype@ in1i = ((@ftype@ *)ip1)[1]; const @ftype@ in2r = ((@ftype@ *)ip2)[0]; const @ftype@ in2i = ((@ftype@ *)ip2)[1]; - if (@OP@(in1r, in1i, in2r, in2i) || npy_isnan(in2r) || npy_isnan(in2i)) { + if (npy_isnan(in2r) || npy_isnan(in2i) || @OP@(in1r, in1i, in2r, in2i)) { ((@ftype@ *)op1)[0] = in1r; ((@ftype@ *)op1)[1] = in1i; } @@ -2795,6 +2842,7 @@ NPY_NO_EXPORT void ((@ftype@ *)op1)[1] = in2i; } } + npy_clear_floatstatus_barrier((char*)dimensions); } /**end repeat1**/ diff --git a/numpy/core/src/umath/loops.h.src b/numpy/core/src/umath/loops.h.src index 5c2b2c22c..3c908121e 100644 --- a/numpy/core/src/umath/loops.h.src +++ b/numpy/core/src/umath/loops.h.src @@ -473,6 +473,12 @@ TIMEDELTA_md_m_divide(char **args, npy_intp *dimensions, npy_intp *steps, void * NPY_NO_EXPORT void TIMEDELTA_mm_d_divide(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); +NPY_NO_EXPORT void +TIMEDELTA_mm_q_floor_divide(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); + +NPY_NO_EXPORT void +TIMEDELTA_mm_m_remainder(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); + /* Special case equivalents to above functions */ #define TIMEDELTA_mq_m_true_divide TIMEDELTA_mq_m_divide diff --git a/numpy/core/src/umath/matmul.c.src b/numpy/core/src/umath/matmul.c.src new file mode 100644 index 000000000..0cb3c82ad --- /dev/null +++ b/numpy/core/src/umath/matmul.c.src @@ -0,0 +1,402 @@ +/* -*- c -*- */ + +#define _UMATHMODULE +#define _MULTIARRAYMODULE +#define NPY_NO_DEPRECATED_API NPY_API_VERSION + +#include "Python.h" + +#include "npy_config.h" +#include "numpy/npy_common.h" +#include "numpy/arrayobject.h" +#include "numpy/ufuncobject.h" +#include "numpy/npy_math.h" +#include "numpy/halffloat.h" +#include "lowlevel_strided_loops.h" + +#include "npy_pycompat.h" + +#include "npy_cblas.h" +#include "arraytypes.h" /* For TYPE_dot functions */ + +#include <assert.h> + +/* + ***************************************************************************** + ** BASICS ** + ***************************************************************************** + */ + +/* + * -1 to be conservative, in case blas internally uses a for loop with an + * inclusive upper bound + */ +#define BLAS_MAXSIZE (NPY_MAX_INT - 1) + +/* + * Determine if a 2d matrix can be used by BLAS + * 1. Strides must not alias or overlap + * 2. The faster (second) axis must be contiguous + * 3. The slower (first) axis stride, in unit steps, must be larger than + * the faster axis dimension + */ +static NPY_INLINE npy_bool +is_blasable2d(npy_intp byte_stride1, npy_intp byte_stride2, + npy_intp d1, npy_intp d2, npy_intp itemsize) +{ + npy_intp unit_stride1 = byte_stride1 / itemsize; + if (byte_stride2 != itemsize) { + return NPY_FALSE; + } + if ((byte_stride1 % itemsize ==0) && + (unit_stride1 >= d2) && + (unit_stride1 <= BLAS_MAXSIZE)) + { + return NPY_TRUE; + } + return NPY_FALSE; +} + +#if defined(HAVE_CBLAS) +static const npy_cdouble oneD = {1.0, 0.0}, zeroD = {0.0, 0.0}; +static const npy_cfloat oneF = {1.0, 0.0}, zeroF = {0.0, 0.0}; + +/**begin repeat + * + * #name = FLOAT, DOUBLE, CFLOAT, CDOUBLE# + * #ctype = npy_float, npy_double, npy_cfloat, npy_cdouble# + * #typ = npy_float, npy_double, npy_cfloat, npy_cdouble# + * #prefix = s, d, c, z# + * #step1 = 1.F, 1., &oneF, &oneD# + * #step0 = 0.F, 0., &zeroF, &zeroD# + */ +NPY_NO_EXPORT void +@name@_gemv(void *ip1, npy_intp is1_m, npy_intp is1_n, + void *ip2, npy_intp is2_n, npy_intp NPY_UNUSED(is2_p), + void *op, npy_intp op_m, npy_intp NPY_UNUSED(op_p), + npy_intp m, npy_intp n, npy_intp NPY_UNUSED(p)) +{ + /* + * Vector matrix multiplication -- Level 2 BLAS + * arguments + * ip1: contiguous data, m*n shape + * ip2: data in c order, n*1 shape + * op: data in c order, m shape + */ + enum CBLAS_ORDER order; + int M, N, lda; + + assert(m <= BLAS_MAXSIZE && n <= BLAS_MAXSIZE); + assert (is_blasable2d(is2_n, sizeof(@typ@), n, 1, sizeof(@typ@))); + M = (int)m; + N = (int)n; + + if (is_blasable2d(is1_m, is1_n, m, n, sizeof(@typ@))) { + order = CblasColMajor; + lda = (int)(is1_m / sizeof(@typ@)); + } + else { + /* If not ColMajor, caller should have ensured we are RowMajor */ + /* will not assert in release mode */ + order = CblasRowMajor; + assert(is_blasable2d(is1_n, is1_m, n, m, sizeof(@typ@))); + lda = (int)(is1_n / sizeof(@typ@)); + } + cblas_@prefix@gemv(order, CblasTrans, N, M, @step1@, ip1, lda, ip2, + is2_n / sizeof(@typ@), @step0@, op, op_m / sizeof(@typ@)); +} + +NPY_NO_EXPORT void +@name@_matmul_matrixmatrix(void *ip1, npy_intp is1_m, npy_intp is1_n, + void *ip2, npy_intp is2_n, npy_intp is2_p, + void *op, npy_intp os_m, npy_intp os_p, + npy_intp m, npy_intp n, npy_intp p) +{ + /* + * matrix matrix multiplication -- Level 3 BLAS + */ + enum CBLAS_ORDER order = CblasRowMajor; + enum CBLAS_TRANSPOSE trans1, trans2; + int M, N, P, lda, ldb, ldc; + assert(m <= BLAS_MAXSIZE && n <= BLAS_MAXSIZE && p <= BLAS_MAXSIZE); + M = (int)m; + N = (int)n; + P = (int)p; + + assert(is_blasable2d(os_m, os_p, m, p, sizeof(@typ@))); + ldc = (int)(os_m / sizeof(@typ@)); + + if (is_blasable2d(is1_m, is1_n, m, n, sizeof(@typ@))) { + trans1 = CblasNoTrans; + lda = (int)(is1_m / sizeof(@typ@)); + } + else { + /* If not ColMajor, caller should have ensured we are RowMajor */ + /* will not assert in release mode */ + assert(is_blasable2d(is1_n, is1_m, n, m, sizeof(@typ@))); + trans1 = CblasTrans; + lda = (int)(is1_n / sizeof(@typ@)); + } + + if (is_blasable2d(is2_n, is2_p, n, p, sizeof(@typ@))) { + trans2 = CblasNoTrans; + ldb = (int)(is2_n / sizeof(@typ@)); + } + else { + /* If not ColMajor, caller should have ensured we are RowMajor */ + /* will not assert in release mode */ + assert(is_blasable2d(is2_p, is2_n, p, n, sizeof(@typ@))); + trans2 = CblasTrans; + ldb = (int)(is2_p / sizeof(@typ@)); + } + /* + * Use syrk if we have a case of a matrix times its transpose. + * Otherwise, use gemm for all other cases. + */ + if ( + (ip1 == ip2) && + (m == p) && + (is1_m == is2_p) && + (is1_n == is2_n) && + (trans1 != trans2) + ) { + npy_intp i,j; + if (trans1 == CblasNoTrans) { + cblas_@prefix@syrk(order, CblasUpper, trans1, P, N, @step1@, + ip1, lda, @step0@, op, ldc); + } + else { + cblas_@prefix@syrk(order, CblasUpper, trans1, P, N, @step1@, + ip1, ldb, @step0@, op, ldc); + } + /* Copy the triangle */ + for (i = 0; i < P; i++) { + for (j = i + 1; j < P; j++) { + ((@typ@*)op)[j * ldc + i] = ((@typ@*)op)[i * ldc + j]; + } + } + + } + else { + cblas_@prefix@gemm(order, trans1, trans2, M, P, N, @step1@, ip1, lda, + ip2, ldb, @step0@, op, ldc); + } +} + +/**end repeat**/ +#endif + +/* + * matmul loops + * signature is (m?,n),(n,p?)->(m?,p?) + */ + +/**begin repeat + * #TYPE = LONGDOUBLE, + * FLOAT, DOUBLE, HALF, + * CFLOAT, CDOUBLE, CLONGDOUBLE, + * UBYTE, USHORT, UINT, ULONG, ULONGLONG, + * BYTE, SHORT, INT, LONG, LONGLONG, + * BOOL# + * #typ = npy_longdouble, + * npy_float,npy_double,npy_half, + * npy_cfloat, npy_cdouble, npy_clongdouble, + * npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong, + * npy_byte, npy_short, npy_int, npy_long, npy_longlong, + * npy_bool# + * #IS_COMPLEX = 0, 0, 0, 0, 1, 1, 1, 0*11# + * #IS_HALF = 0, 0, 0, 1, 0*14# + */ + +NPY_NO_EXPORT void +@TYPE@_matmul_inner_noblas(void *_ip1, npy_intp is1_m, npy_intp is1_n, + void *_ip2, npy_intp is2_n, npy_intp is2_p, + void *_op, npy_intp os_m, npy_intp os_p, + npy_intp dm, npy_intp dn, npy_intp dp) + +{ + npy_intp m, n, p; + npy_intp ib1_n, ib2_n, ib2_p, ob_p; + char *ip1 = (char *)_ip1, *ip2 = (char *)_ip2, *op = (char *)_op; + + ib1_n = is1_n * dn; + ib2_n = is2_n * dn; + ib2_p = is2_p * dp; + ob_p = os_p * dp; + + for (m = 0; m < dm; m++) { + for (p = 0; p < dp; p++) { +#if @IS_COMPLEX@ == 1 + (*(@typ@ *)op).real = 0; + (*(@typ@ *)op).imag = 0; +#elif @IS_HALF@ + float sum = 0; +#else + *(@typ@ *)op = 0; +#endif + for (n = 0; n < dn; n++) { + @typ@ val1 = (*(@typ@ *)ip1); + @typ@ val2 = (*(@typ@ *)ip2); +#if @IS_HALF@ + sum += npy_half_to_float(val1) * npy_half_to_float(val2); +#elif @IS_COMPLEX@ == 1 + (*(@typ@ *)op).real += (val1.real * val2.real) - + (val1.imag * val2.imag); + (*(@typ@ *)op).imag += (val1.real * val2.imag) + + (val1.imag * val2.real); +#else + *(@typ@ *)op += val1 * val2; +#endif + ip2 += is2_n; + ip1 += is1_n; + } +#if @IS_HALF@ + *(@typ@ *)op = npy_float_to_half(sum); +#endif + ip1 -= ib1_n; + ip2 -= ib2_n; + op += os_p; + ip2 += is2_p; + } + op -= ob_p; + ip2 -= ib2_p; + ip1 += is1_m; + op += os_m; + } +} + +/**end repeat**/ + +/**begin repeat + * #TYPE = FLOAT, DOUBLE, LONGDOUBLE, HALF, + * CFLOAT, CDOUBLE, CLONGDOUBLE, + * UBYTE, USHORT, UINT, ULONG, ULONGLONG, + * BYTE, SHORT, INT, LONG, LONGLONG, + * BOOL# + * #typ = npy_float,npy_double,npy_longdouble, npy_half, + * npy_cfloat, npy_cdouble, npy_clongdouble, + * npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong, + * npy_byte, npy_short, npy_int, npy_long, npy_longlong, + * npy_bool# + * #IS_COMPLEX = 0, 0, 0, 0, 1, 1, 1, 0*11# + * #USEBLAS = 1, 1, 0, 0, 1, 1, 0*12# + */ + + +NPY_NO_EXPORT void +@TYPE@_matmul(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)) +{ + npy_intp dOuter = *dimensions++; + npy_intp iOuter; + npy_intp s0 = *steps++; + npy_intp s1 = *steps++; + npy_intp s2 = *steps++; + npy_intp dm = dimensions[0]; + npy_intp dn = dimensions[1]; + npy_intp dp = dimensions[2]; + npy_intp is1_m=steps[0], is1_n=steps[1], is2_n=steps[2], is2_p=steps[3], + os_m=steps[4], os_p=steps[5]; +#if @USEBLAS@ && defined(HAVE_CBLAS) + npy_intp sz = sizeof(@typ@); + npy_bool special_case = (dm == 1 || dn == 1 || dp == 1); + npy_bool any_zero_dim = (dm == 0 || dn == 0 || dp == 0); + npy_bool scalar_out = (dm == 1 && dp == 1); + npy_bool scalar_vec = (dn == 1 && (dp == 1 || dm == 1)); + npy_bool too_big_for_blas = (dm > BLAS_MAXSIZE || dn > BLAS_MAXSIZE || + dp > BLAS_MAXSIZE); + npy_bool i1_c_blasable = is_blasable2d(is1_m, is1_n, dm, dn, sz); + npy_bool i2_c_blasable = is_blasable2d(is2_n, is2_p, dn, dp, sz); + npy_bool i1_f_blasable = is_blasable2d(is1_n, is1_m, dn, dm, sz); + npy_bool i2_f_blasable = is_blasable2d(is2_p, is2_n, dp, dn, sz); + npy_bool i1blasable = i1_c_blasable || i1_f_blasable; + npy_bool i2blasable = i2_c_blasable || i2_f_blasable; + npy_bool o_c_blasable = is_blasable2d(os_m, os_p, dm, dp, sz); + npy_bool o_f_blasable = is_blasable2d(os_p, os_m, dp, dm, sz); + npy_bool vector_matrix = ((dm == 1) && i2blasable && + is_blasable2d(is1_n, sz, dn, 1, sz)); + npy_bool matrix_vector = ((dp == 1) && i1blasable && + is_blasable2d(is2_n, sz, dn, 1, sz)); +#endif + + for (iOuter = 0; iOuter < dOuter; iOuter++, + args[0] += s0, args[1] += s1, args[2] += s2) { + void *ip1=args[0], *ip2=args[1], *op=args[2]; +#if @USEBLAS@ && defined(HAVE_CBLAS) + /* + * TODO: refactor this out to a inner_loop_selector, in + * PyUFunc_MatmulLoopSelector. But that call does not have access to + * n, m, p and strides. + */ + if (too_big_for_blas || any_zero_dim) { + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + } + else if (special_case) { + /* Special case variants that have a 1 in the core dimensions */ + if (scalar_out) { + /* row @ column, 1,1 output */ + @TYPE@_dot(ip1, is1_n, ip2, is2_n, op, dn, NULL); + } else if (scalar_vec){ + /* + * 1,1d @ vector or vector @ 1,1d + * could use cblas_Xaxy, but that requires 0ing output + * and would not be faster (XXX prove it) + */ + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + } else if (vector_matrix) { + /* vector @ matrix, switch ip1, ip2, p and m */ + @TYPE@_gemv(ip2, is2_p, is2_n, ip1, is1_n, is1_m, + op, os_p, os_m, dp, dn, dm); + } else if (matrix_vector) { + /* matrix @ vector */ + @TYPE@_gemv(ip1, is1_m, is1_n, ip2, is2_n, is2_p, + + op, os_m, os_p, dm, dn, dp); + } else { + /* column @ row, 2d output, no blas needed or non-blas-able input */ + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + } + } else { + /* matrix @ matrix */ + if (i1blasable && i2blasable && o_c_blasable) { + @TYPE@_matmul_matrixmatrix(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, + dm, dn, dp); + } else if (i1blasable && i2blasable && o_f_blasable) { + /* + * Use transpose equivalence: + * matmul(a, b, o) == matmul(b.T, a.T, o.T) + */ + @TYPE@_matmul_matrixmatrix(ip2, is2_p, is2_n, + ip1, is1_n, is1_m, + op, os_p, os_m, + dp, dn, dm); + } else { + /* + * If parameters are castable to int and we copy the + * non-blasable (or non-ccontiguous output) + * we could still use BLAS, see gh-12365. + */ + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + } + } +#else + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + +#endif + } +} + +/**end repeat**/ + + diff --git a/numpy/core/src/umath/matmul.h.src b/numpy/core/src/umath/matmul.h.src new file mode 100644 index 000000000..16be7675b --- /dev/null +++ b/numpy/core/src/umath/matmul.h.src @@ -0,0 +1,12 @@ +/**begin repeat + * #TYPE = FLOAT, DOUBLE, LONGDOUBLE, HALF, + * CFLOAT, CDOUBLE, CLONGDOUBLE, + * UBYTE, USHORT, UINT, ULONG, ULONGLONG, + * BYTE, SHORT, INT, LONG, LONGLONG, + * BOOL# + **/ +NPY_NO_EXPORT void +@TYPE@_matmul(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); +/**end repeat**/ + + diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c index 4a381ba12..8d67f96ac 100644 --- a/numpy/core/src/umath/override.c +++ b/numpy/core/src/umath/override.c @@ -5,8 +5,98 @@ #include "numpy/ufuncobject.h" #include "npy_import.h" -#include "ufunc_override.h" #include "override.h" +#include "ufunc_override.h" + +/* + * For each positional argument and each argument in a possible "out" + * keyword, look for overrides of the standard ufunc behaviour, i.e., + * non-default __array_ufunc__ methods. + * + * Returns the number of overrides, setting corresponding objects + * in PyObject array ``with_override`` and the corresponding + * __array_ufunc__ methods in ``methods`` (both using new references). + * + * Only the first override for a given class is returned. + * + * Returns -1 on failure. + */ +static int +get_array_ufunc_overrides(PyObject *args, PyObject *kwds, + PyObject **with_override, PyObject **methods) +{ + int i; + int num_override_args = 0; + int narg, nout = 0; + PyObject *out_kwd_obj; + PyObject **arg_objs, **out_objs; + + narg = PyTuple_Size(args); + if (narg < 0) { + return -1; + } + arg_objs = PySequence_Fast_ITEMS(args); + + nout = PyUFuncOverride_GetOutObjects(kwds, &out_kwd_obj, &out_objs); + if (nout < 0) { + return -1; + } + + for (i = 0; i < narg + nout; ++i) { + PyObject *obj; + int j; + int new_class = 1; + + if (i < narg) { + obj = arg_objs[i]; + } + else { + obj = out_objs[i - narg]; + } + /* + * Have we seen this class before? If so, ignore. + */ + for (j = 0; j < num_override_args; j++) { + new_class = (Py_TYPE(obj) != Py_TYPE(with_override[j])); + if (!new_class) { + break; + } + } + if (new_class) { + /* + * Now see if the object provides an __array_ufunc__. However, we should + * ignore the base ndarray.__ufunc__, so we skip any ndarray as well as + * any ndarray subclass instances that did not override __array_ufunc__. + */ + PyObject *method = PyUFuncOverride_GetNonDefaultArrayUfunc(obj); + if (method == NULL) { + continue; + } + if (method == Py_None) { + PyErr_Format(PyExc_TypeError, + "operand '%.200s' does not support ufuncs " + "(__array_ufunc__=None)", + obj->ob_type->tp_name); + Py_DECREF(method); + goto fail; + } + Py_INCREF(obj); + with_override[num_override_args] = obj; + methods[num_override_args] = method; + ++num_override_args; + } + } + Py_DECREF(out_kwd_obj); + return num_override_args; + +fail: + for (i = 0; i < num_override_args; i++) { + Py_DECREF(with_override[i]); + Py_DECREF(methods[i]); + } + Py_DECREF(out_kwd_obj); + return -1; +} /* * The following functions normalize ufunc arguments. The work done is similar @@ -136,14 +226,14 @@ normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, PyObject *obj; static PyObject *NoValue = NULL; static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims", - "initial"}; + "initial", "where"}; npy_cache_import("numpy", "_NoValue", &NoValue); if (NoValue == NULL) return -1; - if (nargs < 1 || nargs > 6) { + if (nargs < 1 || nargs > 7) { PyErr_Format(PyExc_TypeError, - "ufunc.reduce() takes from 1 to 6 positional " + "ufunc.reduce() takes from 1 to 7 positional " "arguments but %"NPY_INTP_FMT" were given", nargs); return -1; } @@ -359,7 +449,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* * Check inputs for overrides */ - num_override_args = PyUFunc_WithOverride( + num_override_args = get_array_ufunc_overrides( args, kwds, with_override, array_ufunc_methods); if (num_override_args == -1) { goto fail; diff --git a/numpy/core/src/umath/reduction.c b/numpy/core/src/umath/reduction.c index 6d04ce372..4174e69a8 100644 --- a/numpy/core/src/umath/reduction.c +++ b/numpy/core/src/umath/reduction.c @@ -186,7 +186,6 @@ conform_reduce_result(int ndim, npy_bool *axis_flags, return NULL; } - Py_INCREF(ret); if (PyArray_SetWritebackIfCopyBase(ret_copy, (PyArrayObject *)ret) < 0) { Py_DECREF(ret); Py_DECREF(ret_copy); @@ -444,9 +443,9 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, /* Iterator parameters */ NpyIter *iter = NULL; - PyArrayObject *op[2]; - PyArray_Descr *op_dtypes[2]; - npy_uint32 flags, op_flags[2]; + PyArrayObject *op[3]; + PyArray_Descr *op_dtypes[3]; + npy_uint32 flags, op_flags[3]; /* More than one axis means multiple orders are possible */ if (!reorderable && count_axes(PyArray_NDIM(operand), axis_flags) > 1) { @@ -456,13 +455,12 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, funcname); return NULL; } - - - /* Validate that the parameters for future expansion are NULL */ - if (wheremask != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "Reduce operations in NumPy do not yet support " - "a where mask"); + /* Can only use where with an initial ( from identity or argument) */ + if (wheremask != NULL && identity == Py_None) { + PyErr_Format(PyExc_ValueError, + "reduction operation '%s' does not have an identity, " + "so to use a where mask one has to specify 'initial'", + funcname); return NULL; } @@ -524,8 +522,16 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, NPY_ITER_NO_SUBTYPE; op_flags[1] = NPY_ITER_READONLY | NPY_ITER_ALIGNED; + if (wheremask != NULL) { + op[2] = wheremask; + op_dtypes[2] = PyArray_DescrFromType(NPY_BOOL); + if (op_dtypes[2] == NULL) { + goto fail; + } + op_flags[2] = NPY_ITER_READONLY; + } - iter = NpyIter_AdvancedNew(2, op, flags, + iter = NpyIter_AdvancedNew(wheremask == NULL ? 2 : 3, op, flags, NPY_KEEPORDER, casting, op_flags, op_dtypes, @@ -568,7 +574,7 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, goto fail; } } - + /* Check whether any errors occurred during the loop */ if (PyErr_Occurred() || _check_ufunc_fperr(errormask, NULL, "reduce") < 0) { diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index e98d9f865..a7987acda 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -1564,7 +1564,6 @@ static PyObject* } /**end repeat**/ - /**begin repeat * #name = byte, ubyte, short, ushort, int, uint, * long, ulong, longlong, ulonglong, @@ -1575,8 +1574,7 @@ static PyNumberMethods @name@_as_number = { (binaryfunc)@name@_add, /*nb_add*/ (binaryfunc)@name@_subtract, /*nb_subtract*/ (binaryfunc)@name@_multiply, /*nb_multiply*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) (binaryfunc)@name@_divide, /*nb_divide*/ #endif (binaryfunc)@name@_remainder, /*nb_remainder*/ @@ -1596,8 +1594,7 @@ static PyNumberMethods @name@_as_number = { (binaryfunc)@name@_and, /*nb_and*/ (binaryfunc)@name@_xor, /*nb_xor*/ (binaryfunc)@name@_or, /*nb_or*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) 0, /*nb_coerce*/ #endif (unaryfunc)@name@_int, /*nb_int*/ @@ -1607,16 +1604,14 @@ static PyNumberMethods @name@_as_number = { (unaryfunc)@name@_long, /*nb_long*/ #endif (unaryfunc)@name@_float, /*nb_float*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) (unaryfunc)@name@_oct, /*nb_oct*/ (unaryfunc)@name@_hex, /*nb_hex*/ #endif 0, /*inplace_add*/ 0, /*inplace_subtract*/ 0, /*inplace_multiply*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) 0, /*inplace_divide*/ #endif 0, /*inplace_remainder*/ @@ -1631,6 +1626,10 @@ static PyNumberMethods @name@_as_number = { 0, /*nb_inplace_floor_divide*/ 0, /*nb_inplace_true_divide*/ (unaryfunc)NULL, /*nb_index*/ +#if PY_VERSION_HEX >= 0x03050000 + 0, /*nb_matrix_multiply*/ + 0, /*nb_inplace_matrix_multiply*/ +#endif }; /**end repeat**/ diff --git a/numpy/core/src/umath/simd.inc.src b/numpy/core/src/umath/simd.inc.src index 47f9168e5..4bb8569be 100644 --- a/numpy/core/src/umath/simd.inc.src +++ b/numpy/core/src/umath/simd.inc.src @@ -32,6 +32,8 @@ #include <float.h> #include <string.h> /* for memcpy */ +#define VECTOR_SIZE_BYTES 16 + static NPY_INLINE npy_uintp abs_ptrdiff(char *a, char *b) { @@ -144,7 +146,7 @@ static NPY_INLINE int run_@name@_simd_@func@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps) { #if @vector@ && defined NPY_HAVE_SSE2_INTRINSICS - if (@check@(sizeof(@type@), 16)) { + if (@check@(sizeof(@type@), VECTOR_SIZE_BYTES)) { sse2_@func@_@TYPE@((@type@*)args[1], (@type@*)args[0], dimensions[0]); return 1; } @@ -182,17 +184,24 @@ run_binary_simd_@kind@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps @type@ * ip2 = (@type@ *)args[1]; @type@ * op = (@type@ *)args[2]; npy_intp n = dimensions[0]; +#if defined __AVX512F__ + const npy_intp vector_size_bytes = 64; +#elif defined __AVX2__ + const npy_intp vector_size_bytes = 32; +#else + const npy_intp vector_size_bytes = 32; +#endif /* argument one scalar */ - if (IS_BLOCKABLE_BINARY_SCALAR1(sizeof(@type@), 16)) { + if (IS_BLOCKABLE_BINARY_SCALAR1(sizeof(@type@), vector_size_bytes)) { sse2_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } /* argument two scalar */ - else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), 16)) { + else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), vector_size_bytes)) { sse2_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } - else if (IS_BLOCKABLE_BINARY(sizeof(@type@), 16)) { + else if (IS_BLOCKABLE_BINARY(sizeof(@type@), vector_size_bytes)) { sse2_binary_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } @@ -232,16 +241,16 @@ run_binary_simd_@kind@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps npy_bool * op = (npy_bool *)args[2]; npy_intp n = dimensions[0]; /* argument one scalar */ - if (IS_BLOCKABLE_BINARY_SCALAR1_BOOL(sizeof(@type@), 16)) { + if (IS_BLOCKABLE_BINARY_SCALAR1_BOOL(sizeof(@type@), VECTOR_SIZE_BYTES)) { sse2_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } /* argument two scalar */ - else if (IS_BLOCKABLE_BINARY_SCALAR2_BOOL(sizeof(@type@), 16)) { + else if (IS_BLOCKABLE_BINARY_SCALAR2_BOOL(sizeof(@type@), VECTOR_SIZE_BYTES)) { sse2_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } - else if (IS_BLOCKABLE_BINARY_BOOL(sizeof(@type@), 16)) { + else if (IS_BLOCKABLE_BINARY_BOOL(sizeof(@type@), VECTOR_SIZE_BYTES)) { sse2_binary_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } @@ -302,7 +311,8 @@ static NPY_INLINE int run_binary_simd_@kind@_BOOL(char **args, npy_intp *dimensions, npy_intp *steps) { #if defined NPY_HAVE_SSE2_INTRINSICS - if (sizeof(npy_bool) == 1 && IS_BLOCKABLE_BINARY(sizeof(npy_bool), 16)) { + if (sizeof(npy_bool) == 1 && + IS_BLOCKABLE_BINARY(sizeof(npy_bool), VECTOR_SIZE_BYTES)) { sse2_binary_@kind@_BOOL((npy_bool*)args[2], (npy_bool*)args[0], (npy_bool*)args[1], dimensions[0]); return 1; @@ -316,7 +326,8 @@ static NPY_INLINE int run_reduce_simd_@kind@_BOOL(char **args, npy_intp *dimensions, npy_intp *steps) { #if defined NPY_HAVE_SSE2_INTRINSICS - if (sizeof(npy_bool) == 1 && IS_BLOCKABLE_REDUCE(sizeof(npy_bool), 16)) { + if (sizeof(npy_bool) == 1 && + IS_BLOCKABLE_REDUCE(sizeof(npy_bool), VECTOR_SIZE_BYTES)) { sse2_reduce_@kind@_BOOL((npy_bool*)args[0], (npy_bool*)args[1], dimensions[0]); return 1; @@ -340,7 +351,8 @@ static NPY_INLINE int run_unary_simd_@kind@_BOOL(char **args, npy_intp *dimensions, npy_intp *steps) { #if defined NPY_HAVE_SSE2_INTRINSICS - if (sizeof(npy_bool) == 1 && IS_BLOCKABLE_UNARY(sizeof(npy_bool), 16)) { + if (sizeof(npy_bool) == 1 && + IS_BLOCKABLE_UNARY(sizeof(npy_bool), VECTOR_SIZE_BYTES)) { sse2_@kind@_BOOL((npy_bool*)args[1], (npy_bool*)args[0], dimensions[0]); return 1; } @@ -416,19 +428,20 @@ static void sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) { #ifdef __AVX512F__ - LOOP_BLOCK_ALIGN_VAR(op, @type@, 64) + const npy_intp vector_size_bytes = 64; + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[i] @OP@ ip2[i]; /* lots of specializations, to squeeze out max performance */ - if (npy_is_aligned(&ip1[i], 64) && npy_is_aligned(&ip2[i], 64)) { + if (npy_is_aligned(&ip1[i], vector_size_bytes) && npy_is_aligned(&ip2[i], vector_size_bytes)) { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, a); @vpre512@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @@ -436,16 +449,16 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } } - else if (npy_is_aligned(&ip1[i], 64)) { - LOOP_BLOCKED(@type@, 64) { + else if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); } } - else if (npy_is_aligned(&ip2[i], 64)) { - LOOP_BLOCKED(@type@, 64) { + else if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @@ -454,14 +467,14 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } else { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, a); @vpre512@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @@ -470,19 +483,21 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } #elif __AVX2__ - LOOP_BLOCK_ALIGN_VAR(op, @type@, 32) + const npy_intp vector_size_bytes = 32; + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[i] @OP@ ip2[i]; /* lots of specializations, to squeeze out max performance */ - if (npy_is_aligned(&ip1[i], 32) && npy_is_aligned(&ip2[i], 32)) { + if (npy_is_aligned(&ip1[i], vector_size_bytes) && + npy_is_aligned(&ip2[i], vector_size_bytes)) { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, a); @vpre256@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @@ -490,16 +505,16 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } } - else if (npy_is_aligned(&ip1[i], 32)) { - LOOP_BLOCKED(@type@, 32) { + else if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); } } - else if (npy_is_aligned(&ip2[i], 32)) { - LOOP_BLOCKED(@type@, 32) { + else if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @@ -508,14 +523,14 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } else { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, a); @vpre256@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @@ -524,19 +539,20 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } #else - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = ip1[i] @OP@ ip2[i]; /* lots of specializations, to squeeze out max performance */ - if (npy_is_aligned(&ip1[i], 16) && npy_is_aligned(&ip2[i], 16)) { + if (npy_is_aligned(&ip1[i], VECTOR_SIZE_BYTES) && + npy_is_aligned(&ip2[i], VECTOR_SIZE_BYTES)) { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a); @vpre@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @@ -544,16 +560,16 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } } - else if (npy_is_aligned(&ip1[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + else if (npy_is_aligned(&ip1[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); } } - else if (npy_is_aligned(&ip2[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + else if (npy_is_aligned(&ip2[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @@ -562,14 +578,14 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } else { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a); @vpre@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @@ -588,18 +604,19 @@ static void sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) { #ifdef __AVX512F__ + const npy_intp vector_size_bytes = 64; const @vtype512@ a = @vpre512@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 64) + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[0] @OP@ ip2[i]; - if (npy_is_aligned(&ip2[i], 64)) { - LOOP_BLOCKED(@type@, 64) { + if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); @@ -608,18 +625,19 @@ sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_i #elif __AVX2__ + const npy_intp vector_size_bytes = 32; const @vtype256@ a = @vpre256@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 32) + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[0] @OP@ ip2[i]; - if (npy_is_aligned(&ip2[i], 32)) { - LOOP_BLOCKED(@type@, 32) { + if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); @@ -627,17 +645,17 @@ sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_i } #else const @vtype@ a = @vpre@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = ip1[0] @OP@ ip2[i]; - if (npy_is_aligned(&ip2[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + if (npy_is_aligned(&ip2[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); @@ -654,18 +672,19 @@ static void sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) { #ifdef __AVX512F__ + const npy_intp vector_size_bytes = 64; const @vtype512@ b = @vpre512@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 64) + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[i] @OP@ ip2[0]; - if (npy_is_aligned(&ip1[i], 64)) { - LOOP_BLOCKED(@type@, 64) { + if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); @@ -673,18 +692,19 @@ sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_i } #elif __AVX2__ + const npy_intp vector_size_bytes = 32; const @vtype256@ b = @vpre256@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 32) + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[i] @OP@ ip2[0]; - if (npy_is_aligned(&ip1[i], 32)) { - LOOP_BLOCKED(@type@, 32) { + if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); @@ -692,17 +712,17 @@ sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_i } #else const @vtype@ b = @vpre@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = ip1[i] @OP@ ip2[0]; - if (npy_is_aligned(&ip1[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + if (npy_is_aligned(&ip1[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); @@ -742,10 +762,10 @@ sse2_compress4_to_byte_@TYPE@(@vtype@ r1, @vtype@ r2, @vtype@ r3, @vtype@ * r4, static void sse2_signbit_@TYPE@(npy_bool * op, @type@ * ip1, npy_intp n) { - LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { op[i] = npy_signbit(ip1[i]) != 0; } - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); int r = @vpre@_movemask_@vsuf@(a); if (sizeof(@type@) == 8) { @@ -783,14 +803,14 @@ sse2_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, npy_intp n) const @vtype@ fltmax = @vpre@_set1_@vsuf@(FLT_MAX); #endif #endif - LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { op[i] = npy_@kind@(ip1[i]) != 0; } - LOOP_BLOCKED(@type@, 64) { - @vtype@ a = @vpre@_load_@vsuf@(&ip1[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b = @vpre@_load_@vsuf@(&ip1[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c = @vpre@_load_@vsuf@(&ip1[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d = @vpre@_load_@vsuf@(&ip1[i + 3 * 16 / sizeof(@type@)]); + LOOP_BLOCKED(@type@, 4 * VECTOR_SIZE_BYTES) { + @vtype@ a = @vpre@_load_@vsuf@(&ip1[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b = @vpre@_load_@vsuf@(&ip1[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c = @vpre@_load_@vsuf@(&ip1[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d = @vpre@_load_@vsuf@(&ip1[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); @vtype@ r1, r2, r3, r4; #if @var@ != 0 /* isinf/isfinite */ /* fabs via masking of sign bit */ @@ -853,18 +873,18 @@ sse2_ordered_cmp_@kind@_@TYPE@(const @type@ a, const @type@ b) static void sse2_binary_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy_intp n) { - LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[i], ip2[i]); } - LOOP_BLOCKED(@type@, 64) { - @vtype@ a1 = @vpre@_load_@vsuf@(&ip1[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b1 = @vpre@_load_@vsuf@(&ip1[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c1 = @vpre@_load_@vsuf@(&ip1[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d1 = @vpre@_load_@vsuf@(&ip1[i + 3 * 16 / sizeof(@type@)]); - @vtype@ a2 = @vpre@_loadu_@vsuf@(&ip2[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b2 = @vpre@_loadu_@vsuf@(&ip2[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c2 = @vpre@_loadu_@vsuf@(&ip2[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d2 = @vpre@_loadu_@vsuf@(&ip2[i + 3 * 16 / sizeof(@type@)]); + LOOP_BLOCKED(@type@, 4 * VECTOR_SIZE_BYTES) { + @vtype@ a1 = @vpre@_load_@vsuf@(&ip1[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b1 = @vpre@_load_@vsuf@(&ip1[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c1 = @vpre@_load_@vsuf@(&ip1[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d1 = @vpre@_load_@vsuf@(&ip1[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ a2 = @vpre@_loadu_@vsuf@(&ip2[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b2 = @vpre@_loadu_@vsuf@(&ip2[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c2 = @vpre@_loadu_@vsuf@(&ip2[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d2 = @vpre@_loadu_@vsuf@(&ip2[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); @vtype@ r1 = @vpre@_@VOP@_@vsuf@(a1, a2); @vtype@ r2 = @vpre@_@VOP@_@vsuf@(b1, b2); @vtype@ r3 = @vpre@_@VOP@_@vsuf@(c1, c2); @@ -881,14 +901,14 @@ static void sse2_binary_scalar1_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy_intp n) { @vtype@ s = @vpre@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(ip2, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip2, @type@, VECTOR_SIZE_BYTES) { op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[0], ip2[i]); } - LOOP_BLOCKED(@type@, 64) { - @vtype@ a = @vpre@_load_@vsuf@(&ip2[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b = @vpre@_load_@vsuf@(&ip2[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c = @vpre@_load_@vsuf@(&ip2[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d = @vpre@_load_@vsuf@(&ip2[i + 3 * 16 / sizeof(@type@)]); + LOOP_BLOCKED(@type@, 4 * VECTOR_SIZE_BYTES) { + @vtype@ a = @vpre@_load_@vsuf@(&ip2[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b = @vpre@_load_@vsuf@(&ip2[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c = @vpre@_load_@vsuf@(&ip2[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d = @vpre@_load_@vsuf@(&ip2[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); @vtype@ r1 = @vpre@_@VOP@_@vsuf@(s, a); @vtype@ r2 = @vpre@_@VOP@_@vsuf@(s, b); @vtype@ r3 = @vpre@_@VOP@_@vsuf@(s, c); @@ -905,14 +925,14 @@ static void sse2_binary_scalar2_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy_intp n) { @vtype@ s = @vpre@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[i], ip2[0]); } - LOOP_BLOCKED(@type@, 64) { - @vtype@ a = @vpre@_load_@vsuf@(&ip1[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b = @vpre@_load_@vsuf@(&ip1[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c = @vpre@_load_@vsuf@(&ip1[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d = @vpre@_load_@vsuf@(&ip1[i + 3 * 16 / sizeof(@type@)]); + LOOP_BLOCKED(@type@, 4 * VECTOR_SIZE_BYTES) { + @vtype@ a = @vpre@_load_@vsuf@(&ip1[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b = @vpre@_load_@vsuf@(&ip1[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c = @vpre@_load_@vsuf@(&ip1[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d = @vpre@_load_@vsuf@(&ip1[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); @vtype@ r1 = @vpre@_@VOP@_@vsuf@(a, s); @vtype@ r2 = @vpre@_@VOP@_@vsuf@(b, s); @vtype@ r3 = @vpre@_@VOP@_@vsuf@(c, s); @@ -928,19 +948,20 @@ sse2_binary_scalar2_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy static void sse2_sqrt_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n) { - /* align output to 16 bytes */ - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) { + /* align output to VECTOR_SIZE_BYTES bytes */ + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) { op[i] = @scalarf@(ip[i]); } - assert(n < (16 / sizeof(@type@)) || npy_is_aligned(&op[i], 16)); - if (npy_is_aligned(&ip[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + assert(n < (VECTOR_SIZE_BYTES / sizeof(@type@)) || + npy_is_aligned(&op[i], VECTOR_SIZE_BYTES)); + if (npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ d = @vpre@_load_@vsuf@(&ip[i]); @vpre@_store_@vsuf@(&op[i], @vpre@_sqrt_@vsuf@(d)); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ d = @vpre@_loadu_@vsuf@(&ip[i]); @vpre@_store_@vsuf@(&op[i], @vpre@_sqrt_@vsuf@(d)); } @@ -979,19 +1000,20 @@ sse2_@kind@_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n) */ const @vtype@ mask = @vpre@_set1_@vsuf@(-0.@c@); - /* align output to 16 bytes */ - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) { + /* align output to VECTOR_SIZE_BYTES bytes */ + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) { op[i] = @scalar@_@type@(ip[i]); } - assert(n < (16 / sizeof(@type@)) || npy_is_aligned(&op[i], 16)); - if (npy_is_aligned(&ip[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + assert(n < (VECTOR_SIZE_BYTES / sizeof(@type@)) || + npy_is_aligned(&op[i], VECTOR_SIZE_BYTES)); + if (npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip[i]); @vpre@_store_@vsuf@(&op[i], @vpre@_@VOP@_@vsuf@(mask, a)); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip[i]); @vpre@_store_@vsuf@(&op[i], @vpre@_@VOP@_@vsuf@(mask, a)); } @@ -1012,11 +1034,12 @@ sse2_@kind@_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n) static void sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n) { - const npy_intp stride = 16 / (npy_intp)sizeof(@type@); - LOOP_BLOCK_ALIGN_VAR(ip, @type@, 16) { + const npy_intp stride = VECTOR_SIZE_BYTES / (npy_intp)sizeof(@type@); + LOOP_BLOCK_ALIGN_VAR(ip, @type@, VECTOR_SIZE_BYTES) { + /* Order of operations important for MSVC 2015 */ *op = (*op @OP@ ip[i] || npy_isnan(*op)) ? *op : ip[i]; } - assert(n < (stride) || npy_is_aligned(&ip[i], 16)); + assert(n < (stride) || npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)); if (i + 3 * stride <= n) { /* load the first elements */ @vtype@ c1 = @vpre@_load_@vsuf@((@type@*)&ip[i]); @@ -1025,7 +1048,7 @@ sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n) /* minps/minpd will set invalid flag if nan is encountered */ npy_clear_floatstatus_barrier((char*)&c1); - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, 2 * VECTOR_SIZE_BYTES) { @vtype@ v1 = @vpre@_load_@vsuf@((@type@*)&ip[i]); @vtype@ v2 = @vpre@_load_@vsuf@((@type@*)&ip[i + stride]); c1 = @vpre@_@VOP@_@vsuf@(c1, v1); @@ -1038,15 +1061,15 @@ sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n) } else { @type@ tmp = sse2_horizontal_@VOP@_@vtype@(c1); + /* Order of operations important for MSVC 2015 */ *op = (*op @OP@ tmp || npy_isnan(*op)) ? *op : tmp; } } LOOP_BLOCKED_END { + /* Order of operations important for MSVC 2015 */ *op = (*op @OP@ ip[i] || npy_isnan(*op)) ? *op : ip[i]; } - if (npy_isnan(*op)) { - npy_set_floatstatus_invalid(); - } + npy_clear_floatstatus_barrier((char*)op); } /**end repeat1**/ @@ -1092,9 +1115,9 @@ static NPY_INLINE @vtype@ byte_to_true(@vtype@ v) static void sse2_binary_@kind@_BOOL(npy_bool * op, npy_bool * ip1, npy_bool * ip2, npy_intp n) { - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = ip1[i] @op@ ip2[i]; - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vloadu@((@vtype@*)&ip1[i]); @vtype@ b = @vloadu@((@vtype@*)&ip2[i]); #if @and@ @@ -1119,16 +1142,16 @@ static void sse2_reduce_@kind@_BOOL(npy_bool * op, npy_bool * ip, const npy_intp n) { const @vtype@ zero = @vpre@_setzero_@vsuf@(); - LOOP_BLOCK_ALIGN_VAR(ip, npy_bool, 16) { + LOOP_BLOCK_ALIGN_VAR(ip, npy_bool, VECTOR_SIZE_BYTES) { *op = *op @op@ ip[i]; if (*op @sc@ 0) { return; } } /* unrolled once to replace a slow movmsk with a fast pmaxb */ - LOOP_BLOCKED(npy_bool, 32) { + LOOP_BLOCKED(npy_bool, 2 * VECTOR_SIZE_BYTES) { @vtype@ v = @vload@((@vtype@*)&ip[i]); - @vtype@ v2 = @vload@((@vtype@*)&ip[i + 16]); + @vtype@ v2 = @vload@((@vtype@*)&ip[i + VECTOR_SIZE_BYTES]); v = @vpre@_cmpeq_epi8(v, zero); v2 = @vpre@_cmpeq_epi8(v2, zero); #if @and@ @@ -1166,9 +1189,9 @@ sse2_reduce_@kind@_BOOL(npy_bool * op, npy_bool * ip, const npy_intp n) static void sse2_@kind@_BOOL(@type@ * op, @type@ * ip, const npy_intp n) { - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = (ip[i] @op@ 0); - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vloadu@((@vtype@*)&ip[i]); #if @not@ const @vtype@ zero = @vpre@_setzero_@vsuf@(); @@ -1189,6 +1212,8 @@ sse2_@kind@_BOOL(@type@ * op, @type@ * ip, const npy_intp n) /**end repeat**/ +#undef VECTOR_SIZE_BYTES + #endif /* NPY_HAVE_SSE2_INTRINSICS */ #endif diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index b82c74109..f950915f5 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -308,6 +308,78 @@ _find_array_prepare(ufunc_full_args args, return; } +#define NPY_UFUNC_DEFAULT_INPUT_FLAGS \ + NPY_ITER_READONLY | \ + NPY_ITER_ALIGNED | \ + NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE + +#define NPY_UFUNC_DEFAULT_OUTPUT_FLAGS \ + NPY_ITER_ALIGNED | \ + NPY_ITER_ALLOCATE | \ + NPY_ITER_NO_BROADCAST | \ + NPY_ITER_NO_SUBTYPE | \ + NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE + +/* Called at module initialization to set the matmul ufunc output flags */ +NPY_NO_EXPORT int +set_matmul_flags(PyObject *d) +{ + PyObject *matmul = PyDict_GetItemString(d, "matmul"); + if (matmul == NULL) { + return -1; + } + /* + * The default output flag NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE allows + * perfectly overlapping input and output (in-place operations). While + * correct for the common mathematical operations, this assumption is + * incorrect in the general case and specifically in the case of matmul. + * + * NPY_ITER_UPDATEIFCOPY is added by default in + * PyUFunc_GeneralizedFunction, which is the variant called for gufuncs + * with a signature + * + * Enabling NPY_ITER_WRITEONLY can prevent a copy in some cases. + */ + ((PyUFuncObject *)matmul)->op_flags[2] = (NPY_ITER_WRITEONLY | + NPY_ITER_UPDATEIFCOPY | + NPY_UFUNC_DEFAULT_OUTPUT_FLAGS) & + ~NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; + return 0; +} + + +/* + * Set per-operand flags according to desired input or output flags. + * op_flags[i] for i in input (as determined by ufunc->nin) will be + * merged with op_in_flags, perhaps overriding per-operand flags set + * in previous stages. + * op_flags[i] for i in output will be set to op_out_flags only if previously + * unset. + * The input flag behavior preserves backward compatibility, while the + * output flag behaviour is the "correct" one for maximum flexibility. + */ +NPY_NO_EXPORT void +_ufunc_setup_flags(PyUFuncObject *ufunc, npy_uint32 op_in_flags, + npy_uint32 op_out_flags, npy_uint32 *op_flags) +{ + int nin = ufunc->nin; + int nout = ufunc->nout; + int nop = nin + nout, i; + /* Set up the flags */ + for (i = 0; i < nin; ++i) { + op_flags[i] = ufunc->op_flags[i] | op_in_flags; + /* + * If READWRITE flag has been set for this operand, + * then clear default READONLY flag + */ + if (op_flags[i] & (NPY_ITER_READWRITE | NPY_ITER_WRITEONLY)) { + op_flags[i] &= ~NPY_ITER_READONLY; + } + } + for (i = nin; i < nop; ++i) { + op_flags[i] = ufunc->op_flags[i] ? ufunc->op_flags[i] : op_out_flags; + } +} /* * This function analyzes the input arguments @@ -747,7 +819,7 @@ _set_out_array(PyObject *obj, PyArrayObject **store) /* Translate None to NULL */ return 0; } - if PyArray_Check(obj) { + if (PyArray_Check(obj)) { /* If it's an array, store it */ if (PyArray_FailUnlessWriteable((PyArrayObject *)obj, "output array") < 0) { @@ -1394,11 +1466,11 @@ iterator_loop(PyUFuncObject *ufunc, PyObject **arr_prep, ufunc_full_args full_args, PyUFuncGenericFunction innerloop, - void *innerloopdata) + void *innerloopdata, + npy_uint32 *op_flags) { npy_intp i, nin = ufunc->nin, nout = ufunc->nout; npy_intp nop = nin + nout; - npy_uint32 op_flags[NPY_MAXARGS]; NpyIter *iter; char *baseptrs[NPY_MAXARGS]; @@ -1412,29 +1484,6 @@ iterator_loop(PyUFuncObject *ufunc, NPY_BEGIN_THREADS_DEF; - /* Set up the flags */ - for (i = 0; i < nin; ++i) { - op_flags[i] = NPY_ITER_READONLY | - NPY_ITER_ALIGNED | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - /* - * If READWRITE flag has been set for this operand, - * then clear default READONLY flag - */ - op_flags[i] |= ufunc->op_flags[i]; - if (op_flags[i] & (NPY_ITER_READWRITE | NPY_ITER_WRITEONLY)) { - op_flags[i] &= ~NPY_ITER_READONLY; - } - } - for (i = nin; i < nop; ++i) { - op_flags[i] = NPY_ITER_WRITEONLY | - NPY_ITER_ALIGNED | - NPY_ITER_ALLOCATE | - NPY_ITER_NO_BROADCAST | - NPY_ITER_NO_SUBTYPE | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - } - iter_flags = ufunc->iter_flags | NPY_ITER_EXTERNAL_LOOP | NPY_ITER_REFS_OK | @@ -1538,15 +1587,15 @@ iterator_loop(PyUFuncObject *ufunc, } /* + * ufunc - the ufunc to call * trivial_loop_ok - 1 if no alignment, data conversion, etc required - * nin - number of inputs - * nout - number of outputs - * op - the operands (nin + nout of them) + * op - the operands (ufunc->nin + ufunc->nout of them) + * dtypes - the dtype of each operand * order - the loop execution order/output memory order * buffersize - how big of a buffer to use * arr_prep - the __array_prepare__ functions for the outputs - * innerloop - the inner loop function - * innerloopdata - data to pass to the inner loop + * full_args - the original input, output PyObject * + * op_flags - per-operand flags, a combination of NPY_ITER_* constants */ static int execute_legacy_ufunc_loop(PyUFuncObject *ufunc, @@ -1556,7 +1605,8 @@ execute_legacy_ufunc_loop(PyUFuncObject *ufunc, NPY_ORDER order, npy_intp buffersize, PyObject **arr_prep, - ufunc_full_args full_args) + ufunc_full_args full_args, + npy_uint32 *op_flags) { npy_intp nin = ufunc->nin, nout = ufunc->nout; PyUFuncGenericFunction innerloop; @@ -1691,7 +1741,7 @@ execute_legacy_ufunc_loop(PyUFuncObject *ufunc, NPY_UF_DBG_PRINT("iterator loop\n"); if (iterator_loop(ufunc, op, dtypes, order, buffersize, arr_prep, full_args, - innerloop, innerloopdata) < 0) { + innerloop, innerloopdata, op_flags) < 0) { return -1; } @@ -1717,14 +1767,13 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, NPY_ORDER order, npy_intp buffersize, PyObject **arr_prep, - ufunc_full_args full_args) + ufunc_full_args full_args, + npy_uint32 *op_flags) { int i, nin = ufunc->nin, nout = ufunc->nout; int nop = nin + nout; - npy_uint32 op_flags[NPY_MAXARGS]; NpyIter *iter; int needs_api; - npy_intp default_op_in_flags = 0, default_op_out_flags = 0; NpyIter_IterNextFunc *iternext; char **dataptr; @@ -1734,48 +1783,10 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, PyArrayObject **op_it; npy_uint32 iter_flags; - if (wheremask != NULL) { - if (nop + 1 > NPY_MAXARGS) { - PyErr_SetString(PyExc_ValueError, - "Too many operands when including where= parameter"); - return -1; - } - op[nop] = wheremask; - dtypes[nop] = NULL; - default_op_out_flags |= NPY_ITER_WRITEMASKED; - } - - /* Set up the flags */ - for (i = 0; i < nin; ++i) { - op_flags[i] = default_op_in_flags | - NPY_ITER_READONLY | - NPY_ITER_ALIGNED | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - /* - * If READWRITE flag has been set for this operand, - * then clear default READONLY flag - */ - op_flags[i] |= ufunc->op_flags[i]; - if (op_flags[i] & (NPY_ITER_READWRITE | NPY_ITER_WRITEONLY)) { - op_flags[i] &= ~NPY_ITER_READONLY; - } - } for (i = nin; i < nop; ++i) { - /* - * We don't write to all elements, and the iterator may make - * UPDATEIFCOPY temporary copies. The output arrays (unless they are - * allocated by the iterator itself) must be considered READWRITE by the - * iterator, so that the elements we don't write to are copied to the - * possible temporary array. - */ - op_flags[i] = default_op_out_flags | - (op[i] != NULL ? NPY_ITER_READWRITE : NPY_ITER_WRITEONLY) | - NPY_ITER_ALIGNED | - NPY_ITER_ALLOCATE | - NPY_ITER_NO_BROADCAST | - NPY_ITER_NO_SUBTYPE | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; + op_flags[i] |= (op[i] != NULL ? NPY_ITER_READWRITE : NPY_ITER_WRITEONLY); } + if (wheremask != NULL) { op_flags[nop] = NPY_ITER_READONLY | NPY_ITER_ARRAYMASK; } @@ -2471,6 +2482,11 @@ _get_identity(PyUFuncObject *ufunc, npy_bool *reorderable) { *reorderable = 0; Py_RETURN_NONE; + case PyUFunc_IdentityValue: + *reorderable = 1; + Py_INCREF(ufunc->identity_value); + return ufunc->identity_value; + default: PyErr_Format(PyExc_ValueError, "ufunc %s has an invalid identity", ufunc_get_name_cstr(ufunc)); @@ -2785,6 +2801,18 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, if (retval < 0) { goto fail; } + /* + * We don't write to all elements, and the iterator may make + * UPDATEIFCOPY temporary copies. The output arrays (unless they are + * allocated by the iterator itself) must be considered READWRITE by the + * iterator, so that the elements we don't write to are copied to the + * possible temporary array. + */ + _ufunc_setup_flags(ufunc, NPY_ITER_COPY | NPY_UFUNC_DEFAULT_INPUT_FLAGS, + NPY_ITER_UPDATEIFCOPY | + NPY_ITER_READWRITE | + NPY_UFUNC_DEFAULT_OUTPUT_FLAGS, + op_flags); /* For the generalized ufunc, we get the loop right away too */ retval = ufunc->legacy_inner_loop_selector(ufunc, dtypes, &innerloop, &innerloopdata, &needs_api); @@ -2827,28 +2855,6 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, * Set up the iterator per-op flags. For generalized ufuncs, we * can't do buffering, so must COPY or UPDATEIFCOPY. */ - for (i = 0; i < nin; ++i) { - op_flags[i] = NPY_ITER_READONLY | - NPY_ITER_COPY | - NPY_ITER_ALIGNED | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - /* - * If READWRITE flag has been set for this operand, - * then clear default READONLY flag - */ - op_flags[i] |= ufunc->op_flags[i]; - if (op_flags[i] & (NPY_ITER_READWRITE | NPY_ITER_WRITEONLY)) { - op_flags[i] &= ~NPY_ITER_READONLY; - } - } - for (i = nin; i < nop; ++i) { - op_flags[i] = NPY_ITER_READWRITE| - NPY_ITER_UPDATEIFCOPY| - NPY_ITER_ALIGNED| - NPY_ITER_ALLOCATE| - NPY_ITER_NO_BROADCAST| - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - } iter_flags = ufunc->iter_flags | NPY_ITER_MULTI_INDEX | @@ -2868,13 +2874,15 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, } /* Fill in any allocated outputs */ - for (i = nin; i < nop; ++i) { - if (op[i] == NULL) { - op[i] = NpyIter_GetOperandArray(iter)[i]; - Py_INCREF(op[i]); + { + PyArrayObject **operands = NpyIter_GetOperandArray(iter); + for (i = 0; i < nop; ++i) { + if (op[i] == NULL) { + op[i] = operands[i]; + Py_INCREF(op[i]); + } } } - /* * Set up the inner strides array. Because we're not doing * buffering, the strides are fixed throughout the looping. @@ -3055,6 +3063,8 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, Py_XDECREF(axis); Py_XDECREF(full_args.in); Py_XDECREF(full_args.out); + PyArray_free(remap_axis_memory); + PyArray_free(remap_axis); NPY_UF_DBG_PRINT1("Returning code %d\n", reval); @@ -3097,7 +3107,8 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, int i, nop; const char *ufunc_name; int retval = -1, subok = 1; - int need_fancy = 0; + npy_uint32 op_flags[NPY_MAXARGS]; + npy_intp default_op_out_flags; PyArray_Descr *dtypes[NPY_MAXARGS]; @@ -3156,13 +3167,6 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, return retval; } - /* - * Use the masked loop if a wheremask was specified. - */ - if (wheremask != NULL) { - need_fancy = 1; - } - /* Get the buffersize and errormask */ if (_get_bufsize_errmask(extobj, ufunc_name, &buffersize, &errormask) < 0) { retval = -1; @@ -3177,16 +3181,20 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, goto fail; } - /* Only do the trivial loop check for the unmasked version. */ - if (!need_fancy) { - /* - * This checks whether a trivial loop is ok, making copies of - * scalar and one dimensional operands if that will help. - */ - trivial_loop_ok = check_for_trivial_loop(ufunc, op, dtypes, buffersize); - if (trivial_loop_ok < 0) { - goto fail; - } + if (wheremask != NULL) { + /* Set up the flags. */ + default_op_out_flags = NPY_ITER_NO_SUBTYPE | + NPY_ITER_WRITEMASKED | + NPY_UFUNC_DEFAULT_OUTPUT_FLAGS; + _ufunc_setup_flags(ufunc, NPY_UFUNC_DEFAULT_INPUT_FLAGS, + default_op_out_flags, op_flags); + } + else { + /* Set up the flags. */ + default_op_out_flags = NPY_ITER_WRITEONLY | + NPY_UFUNC_DEFAULT_OUTPUT_FLAGS; + _ufunc_setup_flags(ufunc, NPY_UFUNC_DEFAULT_INPUT_FLAGS, + default_op_out_flags, op_flags); } #if NPY_UF_DBG_TRACING @@ -3214,23 +3222,46 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, _find_array_prepare(full_args, arr_prep, nin, nout); } - /* Start with the floating-point exception flags cleared */ - npy_clear_floatstatus_barrier((char*)&ufunc); /* Do the ufunc loop */ - if (need_fancy) { + if (wheremask != NULL) { NPY_UF_DBG_PRINT("Executing fancy inner loop\n"); + if (nop + 1 > NPY_MAXARGS) { + PyErr_SetString(PyExc_ValueError, + "Too many operands when including where= parameter"); + return -1; + } + op[nop] = wheremask; + dtypes[nop] = NULL; + + /* Set up the flags */ + + npy_clear_floatstatus_barrier((char*)&ufunc); retval = execute_fancy_ufunc_loop(ufunc, wheremask, op, dtypes, order, - buffersize, arr_prep, full_args); + buffersize, arr_prep, full_args, op_flags); } else { NPY_UF_DBG_PRINT("Executing legacy inner loop\n"); + /* + * This checks whether a trivial loop is ok, making copies of + * scalar and one dimensional operands if that will help. + * Since it requires dtypes, it can only be called after + * ufunc->type_resolver + */ + trivial_loop_ok = check_for_trivial_loop(ufunc, op, dtypes, buffersize); + if (trivial_loop_ok < 0) { + goto fail; + } + + /* check_for_trivial_loop on half-floats can overflow */ + npy_clear_floatstatus_barrier((char*)&ufunc); + retval = execute_legacy_ufunc_loop(ufunc, trivial_loop_ok, op, dtypes, order, - buffersize, arr_prep, full_args); + buffersize, arr_prep, full_args, op_flags); } if (retval < 0) { goto fail; @@ -3437,12 +3468,15 @@ reduce_loop(NpyIter *iter, char **dataptrs, npy_intp *strides, PyUFuncObject *ufunc = (PyUFuncObject *)data; char *dataptrs_copy[3]; npy_intp strides_copy[3]; + npy_bool masked; /* The normal selected inner loop */ PyUFuncGenericFunction innerloop = NULL; void *innerloopdata = NULL; NPY_BEGIN_THREADS_DEF; + /* Get the number of operands, to determine whether "where" is used */ + masked = (NpyIter_GetNOp(iter) == 3); /* Get the inner loop */ iter_dtypes = NpyIter_GetDescrArray(iter); @@ -3502,8 +3536,36 @@ reduce_loop(NpyIter *iter, char **dataptrs, npy_intp *strides, strides_copy[0] = strides[0]; strides_copy[1] = strides[1]; strides_copy[2] = strides[0]; - innerloop(dataptrs_copy, countptr, - strides_copy, innerloopdata); + + if (!masked) { + innerloop(dataptrs_copy, countptr, + strides_copy, innerloopdata); + } + else { + npy_intp count = *countptr; + char *maskptr = dataptrs[2]; + npy_intp mask_stride = strides[2]; + /* Optimization for when the mask is broadcast */ + npy_intp n = mask_stride == 0 ? count : 1; + while (count) { + char mask = *maskptr; + maskptr += mask_stride; + while (n < count && mask == *maskptr) { + n++; + maskptr += mask_stride; + } + /* If mask set, apply inner loop on this contiguous region */ + if (mask) { + innerloop(dataptrs_copy, &n, + strides_copy, innerloopdata); + } + dataptrs_copy[0] += n * strides[0]; + dataptrs_copy[1] += n * strides[1]; + dataptrs_copy[2] = dataptrs_copy[0]; + count -= n; + n = 1; + } + } } while (iternext(iter)); finish_loop: @@ -3532,7 +3594,7 @@ finish_loop: static PyArrayObject * PyUFunc_Reduce(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, int naxes, int *axes, PyArray_Descr *odtype, int keepdims, - PyObject *initial) + PyObject *initial, PyArrayObject *wheremask) { int iaxes, ndim; npy_bool reorderable; @@ -3598,7 +3660,7 @@ PyUFunc_Reduce(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, return NULL; } - result = PyUFunc_ReduceWrapper(arr, out, NULL, dtype, dtype, + result = PyUFunc_ReduceWrapper(arr, out, wheremask, dtype, dtype, NPY_UNSAFE_CASTING, axis_flags, reorderable, keepdims, 0, @@ -4355,7 +4417,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, int i, naxes=0, ndim; int axes[NPY_MAXDIMS]; PyObject *axes_in = NULL; - PyArrayObject *mp = NULL, *ret = NULL; + PyArrayObject *mp = NULL, *wheremask = NULL, *ret = NULL; PyObject *op; PyObject *obj_ind, *context; PyArrayObject *indices = NULL; @@ -4364,7 +4426,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, int keepdims = 0; PyObject *initial = NULL; static char *reduce_kwlist[] = { - "array", "axis", "dtype", "out", "keepdims", "initial", NULL}; + "array", "axis", "dtype", "out", "keepdims", "initial", "where", NULL}; static char *accumulate_kwlist[] = { "array", "axis", "dtype", "out", NULL}; static char *reduceat_kwlist[] = { @@ -4427,22 +4489,23 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, } else if (operation == UFUNC_ACCUMULATE) { if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO&O&:accumulate", - accumulate_kwlist, - &op, - &axes_in, - PyArray_DescrConverter2, &otype, - PyArray_OutputConverter, &out)) { + accumulate_kwlist, + &op, + &axes_in, + PyArray_DescrConverter2, &otype, + PyArray_OutputConverter, &out)) { goto fail; } } else { - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO&O&iO:reduce", - reduce_kwlist, - &op, - &axes_in, - PyArray_DescrConverter2, &otype, - PyArray_OutputConverter, &out, - &keepdims, &initial)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO&O&iOO&:reduce", + reduce_kwlist, + &op, + &axes_in, + PyArray_DescrConverter2, &otype, + PyArray_OutputConverter, &out, + &keepdims, &initial, + _wheremask_converter, &wheremask)) { goto fail; } } @@ -4573,7 +4636,8 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, switch(operation) { case UFUNC_REDUCE: ret = PyUFunc_Reduce(ufunc, mp, out, naxes, axes, - otype, keepdims, initial); + otype, keepdims, initial, wheremask); + Py_XDECREF(wheremask); break; case UFUNC_ACCUMULATE: if (naxes != 1) { @@ -4631,6 +4695,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, fail: Py_XDECREF(otype); Py_XDECREF(mp); + Py_XDECREF(wheremask); return NULL; } @@ -4840,6 +4905,20 @@ PyUFunc_FromFuncAndDataAndSignature(PyUFuncGenericFunction *func, void **data, const char *name, const char *doc, int unused, const char *signature) { + return PyUFunc_FromFuncAndDataAndSignatureAndIdentity( + func, data, types, ntypes, nin, nout, identity, name, doc, + unused, signature, NULL); +} + +/*UFUNC_API*/ +NPY_NO_EXPORT PyObject * +PyUFunc_FromFuncAndDataAndSignatureAndIdentity(PyUFuncGenericFunction *func, void **data, + char *types, int ntypes, + int nin, int nout, int identity, + const char *name, const char *doc, + int unused, const char *signature, + PyObject *identity_value) +{ PyUFuncObject *ufunc; if (nin + nout > NPY_MAXARGS) { PyErr_Format(PyExc_ValueError, @@ -4860,6 +4939,10 @@ PyUFunc_FromFuncAndDataAndSignature(PyUFuncGenericFunction *func, void **data, ufunc->nout = nout; ufunc->nargs = nin+nout; ufunc->identity = identity; + if (ufunc->identity == PyUFunc_IdentityValue) { + Py_INCREF(identity_value); + } + ufunc->identity_value = identity_value; ufunc->functions = func; ufunc->data = data; @@ -4881,6 +4964,7 @@ PyUFunc_FromFuncAndDataAndSignature(PyUFuncGenericFunction *func, void **data, ufunc->op_flags = PyArray_malloc(sizeof(npy_uint32)*ufunc->nargs); if (ufunc->op_flags == NULL) { + Py_DECREF(ufunc); return PyErr_NoMemory(); } memset(ufunc->op_flags, 0, sizeof(npy_uint32)*ufunc->nargs); @@ -5231,12 +5315,17 @@ ufunc_dealloc(PyUFuncObject *ufunc) { PyArray_free(ufunc->core_num_dims); PyArray_free(ufunc->core_dim_ixs); + PyArray_free(ufunc->core_dim_sizes); + PyArray_free(ufunc->core_dim_flags); PyArray_free(ufunc->core_offsets); PyArray_free(ufunc->core_signature); PyArray_free(ufunc->ptr); PyArray_free(ufunc->op_flags); Py_XDECREF(ufunc->userloops); Py_XDECREF(ufunc->obj); + if (ufunc->identity == PyUFunc_IdentityValue) { + Py_DECREF(ufunc->identity_value); + } PyArray_free(ufunc); } diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index 5ddfe29ef..3cf507258 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -22,6 +22,11 @@ #include "ufunc_object.h" #include "common.h" +#include "mem_overlap.h" +#if defined(HAVE_CBLAS) +#include "cblasfuncs.h" +#endif + static const char * npy_casting_to_string(NPY_CASTING casting) { @@ -1109,7 +1114,16 @@ PyUFunc_DivisionTypeResolver(PyUFuncObject *ufunc, } out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); + + /* + * TODO: split function into truediv and floordiv resolvers + */ + if (strcmp(ufunc->name, "floor_divide") == 0) { + out_dtypes[2] = PyArray_DescrFromType(NPY_LONGLONG); + } + else { out_dtypes[2] = PyArray_DescrFromType(NPY_DOUBLE); + } if (out_dtypes[2] == NULL) { Py_DECREF(out_dtypes[0]); out_dtypes[0] = NULL; @@ -1173,6 +1187,57 @@ PyUFunc_DivisionTypeResolver(PyUFuncObject *ufunc, } +NPY_NO_EXPORT int +PyUFunc_RemainderTypeResolver(PyUFuncObject *ufunc, + NPY_CASTING casting, + PyArrayObject **operands, + PyObject *type_tup, + PyArray_Descr **out_dtypes) +{ + int type_num1, type_num2; + int i; + + type_num1 = PyArray_DESCR(operands[0])->type_num; + type_num2 = PyArray_DESCR(operands[1])->type_num; + + /* Use the default when datetime and timedelta are not involved */ + if (!PyTypeNum_ISDATETIME(type_num1) && !PyTypeNum_ISDATETIME(type_num2)) { + return PyUFunc_DefaultTypeResolver(ufunc, casting, operands, + type_tup, out_dtypes); + } + if (type_num1 == NPY_TIMEDELTA) { + if (type_num2 == NPY_TIMEDELTA) { + out_dtypes[0] = PyArray_PromoteTypes(PyArray_DESCR(operands[0]), + PyArray_DESCR(operands[1])); + if (out_dtypes[0] == NULL) { + return -1; + } + out_dtypes[1] = out_dtypes[0]; + Py_INCREF(out_dtypes[1]); + out_dtypes[2] = out_dtypes[0]; + Py_INCREF(out_dtypes[2]); + } + else { + return raise_binary_type_reso_error(ufunc, operands); + } + } + else { + return raise_binary_type_reso_error(ufunc, operands); + } + + /* Check against the casting rules */ + if (PyUFunc_ValidateCasting(ufunc, casting, operands, out_dtypes) < 0) { + for (i = 0; i < 3; ++i) { + Py_DECREF(out_dtypes[i]); + out_dtypes[i] = NULL; + } + return -1; + } + + return 0; +} + + /* * True division should return float64 results when both inputs are integer * types. The PyUFunc_DefaultTypeResolver promotes 8 bit integers to float16 @@ -1248,7 +1313,6 @@ PyUFunc_MixedDivisionTypeResolver(PyUFuncObject *ufunc, type_tup, out_dtypes); } - static int find_userloop(PyUFuncObject *ufunc, PyArray_Descr **dtypes, diff --git a/numpy/core/src/umath/ufunc_type_resolution.h b/numpy/core/src/umath/ufunc_type_resolution.h index fa9f1dbfa..2f37af753 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.h +++ b/numpy/core/src/umath/ufunc_type_resolution.h @@ -92,6 +92,13 @@ PyUFunc_DivisionTypeResolver(PyUFuncObject *ufunc, PyObject *type_tup, PyArray_Descr **out_dtypes); +NPY_NO_EXPORT int +PyUFunc_RemainderTypeResolver(PyUFuncObject *ufunc, + NPY_CASTING casting, + PyArrayObject **operands, + PyObject *type_tup, + PyArray_Descr **out_dtypes); + /* * Does a linear search for the best inner loop of the ufunc. * @@ -138,5 +145,4 @@ PyUFunc_DefaultMaskedInnerLoopSelector(PyUFuncObject *ufunc, NpyAuxData **out_innerloopdata, int *out_needs_api); - #endif diff --git a/numpy/core/src/umath/umathmodule.c b/numpy/core/src/umath/umathmodule.c index 20bd2b0a8..4d3151328 100644 --- a/numpy/core/src/umath/umathmodule.c +++ b/numpy/core/src/umath/umathmodule.c @@ -29,6 +29,7 @@ #include "abstract.h" #include "numpy/npy_math.h" +#include "number.h" static PyUFuncGenericFunction pyfunc_functions[] = {PyUFunc_On_Om}; @@ -169,7 +170,7 @@ PyObject * add_newdoc_ufunc(PyObject *NPY_UNUSED(dummy), PyObject *args) { PyUFuncObject *ufunc; - PyObject *str; + PyObject *str, *tmp; char *docstr, *newdocstr; #if defined(NPY_PY3K) @@ -177,7 +178,11 @@ add_newdoc_ufunc(PyObject *NPY_UNUSED(dummy), PyObject *args) &PyUnicode_Type, &str)) { return NULL; } - docstr = PyBytes_AS_STRING(PyUnicode_AsUTF8String(str)); + tmp = PyUnicode_AsUTF8String(str); + if (tmp == NULL) { + return NULL; + } + docstr = PyBytes_AS_STRING(tmp); #else if (!PyArg_ParseTuple(args, "O!O!:_add_newdoc_ufunc", &PyUFunc_Type, &ufunc, &PyString_Type, &str)) { @@ -189,6 +194,9 @@ add_newdoc_ufunc(PyObject *NPY_UNUSED(dummy), PyObject *args) if (NULL != ufunc->doc) { PyErr_SetString(PyExc_ValueError, "Cannot change docstring of ufunc with non-NULL docstring"); +#if defined(NPY_PY3K) + Py_DECREF(tmp); +#endif return NULL; } @@ -202,6 +210,9 @@ add_newdoc_ufunc(PyObject *NPY_UNUSED(dummy), PyObject *args) strcpy(newdocstr, docstr); ufunc->doc = newdocstr; +#if defined(NPY_PY3K) + Py_DECREF(tmp); +#endif Py_RETURN_NONE; } @@ -267,10 +278,6 @@ int initumath(PyObject *m) UFUNC_FLOATING_POINT_SUPPORT = 0; #endif - /* Initialize the types */ - if (PyType_Ready(&PyUFunc_Type) < 0) - return -1; - /* Add some symbolic constants to the module */ d = PyModule_GetDict(m); @@ -325,7 +332,7 @@ int initumath(PyObject *m) s2 = PyDict_GetItemString(d, "remainder"); /* Setup the array object's numerical structures with appropriate ufuncs in d*/ - PyArray_SetNumericOps(d); + _PyArray_SetNumericOps(d); PyDict_SetItemString(d, "conj", s); PyDict_SetItemString(d, "mod", s2); diff --git a/numpy/core/tests/test_arrayprint.py b/numpy/core/tests/test_arrayprint.py index 6522c6e8a..f2b8fdca7 100644 --- a/numpy/core/tests/test_arrayprint.py +++ b/numpy/core/tests/test_arrayprint.py @@ -90,6 +90,7 @@ class TestArrayRepr(object): assert_equal(repr(x), 'sub(sub(sub(..., dtype=object), dtype=object), dtype=object)') assert_equal(str(x), '...') + x[()] = 0 # resolve circular references for garbage collector # nested 0d-subclass-object x = sub(None) @@ -124,11 +125,13 @@ class TestArrayRepr(object): arr0d[()] = arr0d assert_equal(repr(arr0d), 'array(array(..., dtype=object), dtype=object)') + arr0d[()] = 0 # resolve recursion for garbage collector arr1d = np.array([None, None]) arr1d[1] = arr1d assert_equal(repr(arr1d), 'array([None, array(..., dtype=object)], dtype=object)') + arr1d[1] = 0 # resolve recursion for garbage collector first = np.array(None) second = np.array(None) @@ -136,6 +139,7 @@ class TestArrayRepr(object): second[()] = first assert_equal(repr(first), 'array(array(array(..., dtype=object), dtype=object), dtype=object)') + first[()] = 0 # resolve circular references for garbage collector def test_containing_list(self): # printing square brackets directly would be ambiguuous @@ -842,6 +846,10 @@ class TestPrintOptions(object): [[ 0.]]]])""") ) + def test_bad_args(self): + assert_raises(ValueError, np.set_printoptions, threshold='nan') + assert_raises(ValueError, np.set_printoptions, threshold=u'1') + assert_raises(ValueError, np.set_printoptions, threshold=b'1') def test_unicode_object_array(): import sys diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index e4446e07f..cb7555a34 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -7,6 +7,7 @@ import datetime import pytest from numpy.testing import ( assert_, assert_equal, assert_raises, assert_warns, suppress_warnings, + assert_raises_regex, ) from numpy.core.numeric import pickle @@ -1080,6 +1081,86 @@ class TestDateTime(object): check(np.timedelta64(0), f, nat) check(nat, f, nat) + @pytest.mark.parametrize("op1, op2, exp", [ + # m8 same units round down + (np.timedelta64(7, 's'), + np.timedelta64(4, 's'), + 1), + # m8 same units round down with negative + (np.timedelta64(7, 's'), + np.timedelta64(-4, 's'), + -2), + # m8 same units negative no round down + (np.timedelta64(8, 's'), + np.timedelta64(-4, 's'), + -2), + # m8 different units + (np.timedelta64(1, 'm'), + np.timedelta64(31, 's'), + 1), + # m8 generic units + (np.timedelta64(1890), + np.timedelta64(31), + 60), + # Y // M works + (np.timedelta64(2, 'Y'), + np.timedelta64('13', 'M'), + 1), + # handle 1D arrays + (np.array([1, 2, 3], dtype='m8'), + np.array([2], dtype='m8'), + np.array([0, 1, 1], dtype=np.int64)), + ]) + def test_timedelta_floor_divide(self, op1, op2, exp): + assert_equal(op1 // op2, exp) + + @pytest.mark.parametrize("op1, op2", [ + # div by 0 + (np.timedelta64(10, 'us'), + np.timedelta64(0, 'us')), + # div with NaT + (np.timedelta64('NaT'), + np.timedelta64(50, 'us')), + # special case for int64 min + # in integer floor division + (np.timedelta64(np.iinfo(np.int64).min), + np.timedelta64(-1)), + ]) + def test_timedelta_floor_div_warnings(self, op1, op2): + with assert_warns(RuntimeWarning): + actual = op1 // op2 + assert_equal(actual, 0) + assert_equal(actual.dtype, np.int64) + + @pytest.mark.parametrize("val1, val2", [ + # the smallest integer that can't be represented + # exactly in a double should be preserved if we avoid + # casting to double in floordiv operation + (9007199254740993, 1), + # stress the alternate floordiv code path where + # operand signs don't match and remainder isn't 0 + (9007199254740999, -2), + ]) + def test_timedelta_floor_div_precision(self, val1, val2): + op1 = np.timedelta64(val1) + op2 = np.timedelta64(val2) + actual = op1 // op2 + # Python reference integer floor + expected = val1 // val2 + assert_equal(actual, expected) + + @pytest.mark.parametrize("val1, val2", [ + # years and months sometimes can't be unambiguously + # divided for floor division operation + (np.timedelta64(7, 'Y'), + np.timedelta64(3, 's')), + (np.timedelta64(7, 'M'), + np.timedelta64(1, 'D')), + ]) + def test_timedelta_floor_div_error(self, val1, val2): + with assert_raises_regex(TypeError, "common metadata divisor"): + val1 // val2 + def test_datetime_divide(self): for dta, tda, tdb, tdc, tdd in \ [ @@ -1110,8 +1191,6 @@ class TestDateTime(object): assert_equal(tda / tdd, 60.0) assert_equal(tdd / tda, 1.0 / 60.0) - # m8 // m8 - assert_raises(TypeError, np.floor_divide, tda, tdb) # int / m8 assert_raises(TypeError, np.divide, 2, tdb) # float / m8 @@ -1626,6 +1705,76 @@ class TestDateTime(object): assert_raises(TypeError, np.arange, np.timedelta64(0, 'Y'), np.timedelta64(5, 'D')) + @pytest.mark.parametrize("val1, val2, expected", [ + # case from gh-12092 + (np.timedelta64(7, 's'), + np.timedelta64(3, 's'), + np.timedelta64(1, 's')), + # negative value cases + (np.timedelta64(3, 's'), + np.timedelta64(-2, 's'), + np.timedelta64(-1, 's')), + (np.timedelta64(-3, 's'), + np.timedelta64(2, 's'), + np.timedelta64(1, 's')), + # larger value cases + (np.timedelta64(17, 's'), + np.timedelta64(22, 's'), + np.timedelta64(17, 's')), + (np.timedelta64(22, 's'), + np.timedelta64(17, 's'), + np.timedelta64(5, 's')), + # different units + (np.timedelta64(1, 'm'), + np.timedelta64(57, 's'), + np.timedelta64(3, 's')), + (np.timedelta64(1, 'us'), + np.timedelta64(727, 'ns'), + np.timedelta64(273, 'ns')), + # NaT is propagated + (np.timedelta64('NaT'), + np.timedelta64(50, 'ns'), + np.timedelta64('NaT')), + # Y % M works + (np.timedelta64(2, 'Y'), + np.timedelta64(22, 'M'), + np.timedelta64(2, 'M')), + ]) + def test_timedelta_modulus(self, val1, val2, expected): + assert_equal(val1 % val2, expected) + + @pytest.mark.parametrize("val1, val2", [ + # years and months sometimes can't be unambiguously + # divided for modulus operation + (np.timedelta64(7, 'Y'), + np.timedelta64(3, 's')), + (np.timedelta64(7, 'M'), + np.timedelta64(1, 'D')), + ]) + def test_timedelta_modulus_error(self, val1, val2): + with assert_raises_regex(TypeError, "common metadata divisor"): + val1 % val2 + + def test_timedelta_modulus_div_by_zero(self): + with assert_warns(RuntimeWarning): + actual = np.timedelta64(10, 's') % np.timedelta64(0, 's') + assert_equal(actual, np.timedelta64(0, 's')) + + @pytest.mark.parametrize("val1, val2", [ + # cases where one operand is not + # timedelta64 + (np.timedelta64(7, 'Y'), + 15,), + (7.5, + np.timedelta64(1, 'D')), + ]) + def test_timedelta_modulus_type_resolution(self, val1, val2): + # NOTE: some of the operations may be supported + # in the future + with assert_raises_regex(TypeError, + "remainder cannot use operands with types"): + val1 % val2 + def test_timedelta_arange_no_dtype(self): d = np.array(5, dtype="m8[D]") assert_equal(np.arange(d, d + 1), d) diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 10ef16800..edb5d5e46 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -13,8 +13,7 @@ import pytest import numpy as np from numpy.testing import ( - assert_raises, assert_warns, assert_no_warnings, assert_array_equal, - assert_ + assert_raises, assert_warns, assert_ ) try: @@ -523,3 +522,14 @@ class TestFromstring(_DeprecationTestCase): # 2017-10-19, 1.14 def test_fromstring(self): self.assert_deprecated(np.fromstring, args=('\x00'*80,)) + +class Test_GetSet_NumericOps(_DeprecationTestCase): + # 2018-09-20, 1.16.0 + def test_get_numeric_ops(self): + from numpy.core._multiarray_tests import getset_numericops + self.assert_deprecated(getset_numericops, num=2) + + # empty kwargs prevents any state actually changing which would break + # other tests. + self.assert_deprecated(np.set_numeric_ops, kwargs={}) + assert_raises(ValueError, np.set_numeric_ops, add='abc') diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 1bce86a5a..c55751e3c 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -156,9 +156,9 @@ class TestRecord(object): the dtype constructor. """ assert_raises(TypeError, np.dtype, - dict(names=set(['A', 'B']), formats=['f8', 'i4'])) + dict(names={'A', 'B'}, formats=['f8', 'i4'])) assert_raises(TypeError, np.dtype, - dict(names=['A', 'B'], formats=set(['f8', 'i4']))) + dict(names=['A', 'B'], formats={'f8', 'i4'})) def test_aligned_size(self): # Check that structured dtypes get padded to an aligned size @@ -620,6 +620,25 @@ class TestString(object): # Pull request #4722 np.array(["", ""]).astype(object) + def test_void_subclass_unsized(self): + dt = np.dtype(np.record) + assert_equal(repr(dt), "dtype('V')") + assert_equal(str(dt), '|V0') + assert_equal(dt.name, 'record') + + def test_void_subclass_sized(self): + dt = np.dtype((np.record, 2)) + assert_equal(repr(dt), "dtype('V2')") + assert_equal(str(dt), '|V2') + assert_equal(dt.name, 'record16') + + def test_void_subclass_fields(self): + dt = np.dtype((np.record, [('a', '<u2')])) + assert_equal(repr(dt), "dtype((numpy.record, [('a', '<u2')]))") + assert_equal(str(dt), "(numpy.record, [('a', '<u2')])") + assert_equal(dt.name, 'record16') + + class TestDtypeAttributeDeletion(object): def test_dtype_non_writable_attributes_deletion(self): @@ -775,7 +794,82 @@ class TestFromCTypes(object): ], align=True) self.check(PaddedStruct, expected) - @pytest.mark.xfail(reason="_pack_ is ignored - see gh-11651") + def test_bit_fields(self): + class BitfieldStruct(ctypes.Structure): + _fields_ = [ + ('a', ctypes.c_uint8, 7), + ('b', ctypes.c_uint8, 1) + ] + assert_raises(TypeError, np.dtype, BitfieldStruct) + assert_raises(TypeError, np.dtype, BitfieldStruct()) + + def test_pointer(self): + p_uint8 = ctypes.POINTER(ctypes.c_uint8) + assert_raises(TypeError, np.dtype, p_uint8) + + def test_void_pointer(self): + self.check(ctypes.c_void_p, np.uintp) + + def test_union(self): + class Union(ctypes.Union): + _fields_ = [ + ('a', ctypes.c_uint8), + ('b', ctypes.c_uint16), + ] + expected = np.dtype(dict( + names=['a', 'b'], + formats=[np.uint8, np.uint16], + offsets=[0, 0], + itemsize=2 + )) + self.check(Union, expected) + + def test_union_with_struct_packed(self): + class Struct(ctypes.Structure): + _pack_ = 1 + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + + class Union(ctypes.Union): + _fields_ = [ + ('a', ctypes.c_uint8), + ('b', ctypes.c_uint16), + ('c', ctypes.c_uint32), + ('d', Struct), + ] + expected = np.dtype(dict( + names=['a', 'b', 'c', 'd'], + formats=['u1', np.uint16, np.uint32, [('one', 'u1'), ('two', np.uint32)]], + offsets=[0, 0, 0, 0], + itemsize=ctypes.sizeof(Union) + )) + self.check(Union, expected) + + def test_union_packed(self): + class Struct(ctypes.Structure): + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + _pack_ = 1 + class Union(ctypes.Union): + _pack_ = 1 + _fields_ = [ + ('a', ctypes.c_uint8), + ('b', ctypes.c_uint16), + ('c', ctypes.c_uint32), + ('d', Struct), + ] + expected = np.dtype(dict( + names=['a', 'b', 'c', 'd'], + formats=['u1', np.uint16, np.uint32, [('one', 'u1'), ('two', np.uint32)]], + offsets=[0, 0, 0, 0], + itemsize=ctypes.sizeof(Union) + )) + self.check(Union, expected) + def test_packed_structure(self): class PackedStructure(ctypes.Structure): _pack_ = 1 @@ -789,8 +883,45 @@ class TestFromCTypes(object): ]) self.check(PackedStructure, expected) - @pytest.mark.xfail(sys.byteorder != 'little', - reason="non-native endianness does not work - see gh-10533") + def test_large_packed_structure(self): + class PackedStructure(ctypes.Structure): + _pack_ = 2 + _fields_ = [ + ('a', ctypes.c_uint8), + ('b', ctypes.c_uint16), + ('c', ctypes.c_uint8), + ('d', ctypes.c_uint16), + ('e', ctypes.c_uint32), + ('f', ctypes.c_uint32), + ('g', ctypes.c_uint8) + ] + expected = np.dtype(dict( + formats=[np.uint8, np.uint16, np.uint8, np.uint16, np.uint32, np.uint32, np.uint8 ], + offsets=[0, 2, 4, 6, 8, 12, 16], + names=['a', 'b', 'c', 'd', 'e', 'f', 'g'], + itemsize=18)) + self.check(PackedStructure, expected) + + def test_big_endian_structure_packed(self): + class BigEndStruct(ctypes.BigEndianStructure): + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + _pack_ = 1 + expected = np.dtype([('one', 'u1'), ('two', '>u4')]) + self.check(BigEndStruct, expected) + + def test_little_endian_structure_packed(self): + class LittleEndStruct(ctypes.LittleEndianStructure): + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + _pack_ = 1 + expected = np.dtype([('one', 'u1'), ('two', '<u4')]) + self.check(LittleEndStruct, expected) + def test_little_endian_structure(self): class PaddedStruct(ctypes.LittleEndianStructure): _fields_ = [ @@ -803,8 +934,6 @@ class TestFromCTypes(object): ], align=True) self.check(PaddedStruct, expected) - @pytest.mark.xfail(sys.byteorder != 'big', - reason="non-native endianness does not work - see gh-10533") def test_big_endian_structure(self): class PaddedStruct(ctypes.BigEndianStructure): _fields_ = [ @@ -816,3 +945,9 @@ class TestFromCTypes(object): ('b', '>H') ], align=True) self.check(PaddedStruct, expected) + + def test_simple_endian_types(self): + self.check(ctypes.c_uint16.__ctype_le__, np.dtype('<u2')) + self.check(ctypes.c_uint16.__ctype_be__, np.dtype('>u2')) + self.check(ctypes.c_uint8.__ctype_le__, np.dtype('u1')) + self.check(ctypes.c_uint8.__ctype_be__, np.dtype('u1')) diff --git a/numpy/core/tests/test_einsum.py b/numpy/core/tests/test_einsum.py index 6b5b9c06e..3be4a8a26 100644 --- a/numpy/core/tests/test_einsum.py +++ b/numpy/core/tests/test_einsum.py @@ -11,9 +11,7 @@ from numpy.testing import ( # Setup for optimize einsum chars = 'abcdefghij' sizes = np.array([2, 3, 4, 5, 4, 3, 2, 6, 5, 4, 3]) -global_size_dict = {} -for size, char in zip(sizes, chars): - global_size_dict[char] = size +global_size_dict = dict(zip(chars, sizes)) class TestEinsum(object): diff --git a/numpy/core/tests/test_extint128.py b/numpy/core/tests/test_extint128.py index 0e9c07fd5..7c454a603 100644 --- a/numpy/core/tests/test_extint128.py +++ b/numpy/core/tests/test_extint128.py @@ -1,6 +1,5 @@ from __future__ import division, absolute_import, print_function -import sys import itertools import contextlib import operator @@ -8,7 +7,6 @@ import pytest import numpy as np import numpy.core._multiarray_tests as mt -from numpy.compat import long from numpy.testing import assert_raises, assert_equal diff --git a/numpy/core/tests/test_function_base.py b/numpy/core/tests/test_function_base.py index d0ff1c15f..459bacab0 100644 --- a/numpy/core/tests/test_function_base.py +++ b/numpy/core/tests/test_function_base.py @@ -2,7 +2,7 @@ from __future__ import division, absolute_import, print_function from numpy import ( logspace, linspace, geomspace, dtype, array, sctypes, arange, isnan, - ndarray, sqrt, nextafter + ndarray, sqrt, nextafter, stack ) from numpy.testing import ( assert_, assert_equal, assert_raises, assert_array_equal, assert_allclose, @@ -54,6 +54,20 @@ class TestLogspace(object): y = logspace(0, 6, num=7) assert_array_equal(y, [1, 10, 100, 1e3, 1e4, 1e5, 1e6]) + def test_start_stop_array(self): + start = array([0., 1.]) + stop = array([6., 7.]) + t1 = logspace(start, stop, 6) + t2 = stack([logspace(_start, _stop, 6) + for _start, _stop in zip(start, stop)], axis=1) + assert_equal(t1, t2) + t3 = logspace(start, stop[0], 6) + t4 = stack([logspace(_start, stop[0], 6) + for _start in start], axis=1) + assert_equal(t3, t4) + t5 = logspace(start, stop, 6, axis=-1) + assert_equal(t5, t2.T) + def test_dtype(self): y = logspace(0, 6, dtype='float32') assert_equal(y.dtype, dtype('float32')) @@ -156,7 +170,7 @@ class TestGeomspace(object): y = geomspace(1, 1e6, dtype=complex) assert_equal(y.dtype, dtype('complex')) - def test_array_scalar(self): + def test_start_stop_array_scalar(self): lim1 = array([120, 100], dtype="int8") lim2 = array([-120, -100], dtype="int8") lim3 = array([1200, 1000], dtype="uint16") @@ -172,6 +186,21 @@ class TestGeomspace(object): assert_allclose(t2, t5, rtol=1e-2) assert_allclose(t3, t6, rtol=1e-5) + def test_start_stop_array(self): + # Try to use all special cases. + start = array([1.e0, 32., 1j, -4j, 1+1j, -1]) + stop = array([1.e4, 2., 16j, -324j, 10000+10000j, 1]) + t1 = geomspace(start, stop, 5) + t2 = stack([geomspace(_start, _stop, 5) + for _start, _stop in zip(start, stop)], axis=1) + assert_equal(t1, t2) + t3 = geomspace(start, stop[0], 5) + t4 = stack([geomspace(_start, stop[0], 5) + for _start in start], axis=1) + assert_equal(t3, t4) + t5 = geomspace(start, stop, 5, axis=-1) + assert_equal(t5, t2.T) + def test_physical_quantities(self): a = PhysicalQuantity(1.0) b = PhysicalQuantity(5.0) @@ -227,7 +256,7 @@ class TestLinspace(object): y = linspace(0, 6, dtype='int32') assert_equal(y.dtype, dtype('int32')) - def test_array_scalar(self): + def test_start_stop_array_scalar(self): lim1 = array([-120, 100], dtype="int8") lim2 = array([120, -100], dtype="int8") lim3 = array([1200, 1000], dtype="uint16") @@ -241,6 +270,20 @@ class TestLinspace(object): assert_equal(t2, t5) assert_equal(t3, t6) + def test_start_stop_array(self): + start = array([-120, 120], dtype="int8") + stop = array([100, -100], dtype="int8") + t1 = linspace(start, stop, 5) + t2 = stack([linspace(_start, _stop, 5) + for _start, _stop in zip(start, stop)], axis=1) + assert_equal(t1, t2) + t3 = linspace(start, stop[0], 5) + t4 = stack([linspace(_start, stop[0], 5) + for _start in start], axis=1) + assert_equal(t3, t4) + t5 = linspace(start, stop, 5, axis=-1) + assert_equal(t5, t2.T) + def test_complex(self): lim1 = linspace(1 + 2j, 3 + 4j, 5) t1 = array([1.0+2.j, 1.5+2.5j, 2.0+3j, 2.5+3.5j, 3.0+4j]) @@ -285,9 +328,7 @@ class TestLinspace(object): @property def __array_interface__(self): - # Ideally should be `'shape': ()` but the current interface - # does not allow that - return {'shape': (1,), 'typestr': '<i4', 'data': self._data, + return {'shape': (), 'typestr': '<i4', 'data': self._data, 'version': 3} def __mul__(self, other): diff --git a/numpy/core/tests/test_getlimits.py b/numpy/core/tests/test_getlimits.py index ca8093c62..2f6648183 100644 --- a/numpy/core/tests/test_getlimits.py +++ b/numpy/core/tests/test_getlimits.py @@ -7,10 +7,7 @@ import numpy as np from numpy.core import finfo, iinfo from numpy import half, single, double, longdouble from numpy.testing import assert_equal, assert_, assert_raises -from numpy.core.getlimits import ( - _discovered_machar, _float16_ma, _float32_ma, _float64_ma, _float128_ma, - _float80_ma - ) +from numpy.core.getlimits import _discovered_machar, _float_ma ################################################## @@ -101,9 +98,9 @@ def assert_ma_equal(discovered, ma_like): def test_known_types(): # Test we are correctly compiling parameters for known types - for ftype, ma_like in ((np.float16, _float16_ma), - (np.float32, _float32_ma), - (np.float64, _float64_ma)): + for ftype, ma_like in ((np.float16, _float_ma[16]), + (np.float32, _float_ma[32]), + (np.float64, _float_ma[64])): assert_ma_equal(_discovered_machar(ftype), ma_like) # Suppress warning for broken discovery of double double on PPC with np.errstate(all='ignore'): @@ -111,10 +108,10 @@ def test_known_types(): bytes = np.dtype(np.longdouble).itemsize if (ld_ma.it, ld_ma.maxexp) == (63, 16384) and bytes in (12, 16): # 80-bit extended precision - assert_ma_equal(ld_ma, _float80_ma) + assert_ma_equal(ld_ma, _float_ma[80]) elif (ld_ma.it, ld_ma.maxexp) == (112, 16384) and bytes == 16: # IEE 754 128-bit - assert_ma_equal(ld_ma, _float128_ma) + assert_ma_equal(ld_ma, _float_ma[128]) def test_plausible_finfo(): diff --git a/numpy/core/tests/test_half.py b/numpy/core/tests/test_half.py index d715569f8..b28c933db 100644 --- a/numpy/core/tests/test_half.py +++ b/numpy/core/tests/test_half.py @@ -5,7 +5,7 @@ import pytest import numpy as np from numpy import uint16, float16, float32, float64 -from numpy.testing import assert_, assert_equal, suppress_warnings +from numpy.testing import assert_, assert_equal def assert_raises_fpe(strmatch, callable, *args, **kwargs): @@ -301,21 +301,19 @@ class TestHalf(object): assert_equal(np.copysign(b, a), [2, 5, 1, 4, 3]) assert_equal(np.maximum(a, b), [0, 5, 2, 4, 3]) - with suppress_warnings() as sup: - sup.record(RuntimeWarning) - x = np.maximum(b, c) - assert_(np.isnan(x[3])) - assert_equal(len(sup.log), 1) + + x = np.maximum(b, c) + assert_(np.isnan(x[3])) x[3] = 0 assert_equal(x, [0, 5, 1, 0, 6]) + assert_equal(np.minimum(a, b), [-2, 1, 1, 4, 2]) - with suppress_warnings() as sup: - sup.record(RuntimeWarning) - x = np.minimum(b, c) - assert_(np.isnan(x[3])) - assert_equal(len(sup.log), 1) + + x = np.minimum(b, c) + assert_(np.isnan(x[3])) x[3] = 0 assert_equal(x, [-2, -1, -np.inf, 0, 3]) + assert_equal(np.fmax(a, b), [0, 5, 2, 4, 3]) assert_equal(np.fmax(b, c), [0, 5, 1, 4, 6]) assert_equal(np.fmin(a, b), [-2, 1, 1, 4, 2]) diff --git a/numpy/core/tests/test_mem_overlap.py b/numpy/core/tests/test_mem_overlap.py index f4ce6a84a..3c8e0e722 100644 --- a/numpy/core/tests/test_mem_overlap.py +++ b/numpy/core/tests/test_mem_overlap.py @@ -10,7 +10,7 @@ from numpy.core import _umath_tests from numpy.lib.stride_tricks import as_strided from numpy.compat import long from numpy.testing import ( - assert_, assert_raises, assert_equal, assert_array_equal, assert_allclose + assert_, assert_raises, assert_equal, assert_array_equal ) if sys.version_info[0] >= 3: diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 4b2a38990..70cf74c4a 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -54,7 +54,12 @@ else: def _aligned_zeros(shape, dtype=float, order="C", align=None): - """Allocate a new ndarray with aligned memory.""" + """ + Allocate a new ndarray with aligned memory. + + The ndarray is guaranteed *not* aligned to twice the requested alignment. + Eg, if align=4, guarantees it is not aligned to 8. If align=None uses + dtype.alignment.""" dtype = np.dtype(dtype) if dtype == np.dtype(object): # Can't do this, fall back to standard allocation (which @@ -67,10 +72,15 @@ def _aligned_zeros(shape, dtype=float, order="C", align=None): if not hasattr(shape, '__len__'): shape = (shape,) size = functools.reduce(operator.mul, shape) * dtype.itemsize - buf = np.empty(size + align + 1, np.uint8) - offset = buf.__array_interface__['data'][0] % align + buf = np.empty(size + 2*align + 1, np.uint8) + + ptr = buf.__array_interface__['data'][0] + offset = ptr % align if offset != 0: offset = align - offset + if (ptr % (2*align)) == 0: + offset += align + # Note: slices producing 0-size arrays do not necessarily change # data pointer --- so we use and allocate size+1 buf = buf[offset:offset+size+1][:-1] @@ -92,6 +102,46 @@ class TestFlags(object): self.a[0] = 5 self.a[0] = 0 + def test_writeable_from_readonly(self): + # gh-9440 - make sure fromstring, from buffer on readonly buffers + # set writeable False + data = b'\x00' * 100 + vals = np.frombuffer(data, 'B') + assert_raises(ValueError, vals.setflags, write=True) + types = np.dtype( [('vals', 'u1'), ('res3', 'S4')] ) + values = np.core.records.fromstring(data, types) + vals = values['vals'] + assert_raises(ValueError, vals.setflags, write=True) + + def test_writeable_from_buffer(self): + data = bytearray(b'\x00' * 100) + vals = np.frombuffer(data, 'B') + assert_(vals.flags.writeable) + vals.setflags(write=False) + assert_(vals.flags.writeable is False) + vals.setflags(write=True) + assert_(vals.flags.writeable) + types = np.dtype( [('vals', 'u1'), ('res3', 'S4')] ) + values = np.core.records.fromstring(data, types) + vals = values['vals'] + assert_(vals.flags.writeable) + vals.setflags(write=False) + assert_(vals.flags.writeable is False) + vals.setflags(write=True) + assert_(vals.flags.writeable) + + @pytest.mark.skipif(sys.version_info[0] < 3, reason="Python 2 always copies") + def test_writeable_pickle(self): + import pickle + # Small arrays will be copied without setting base. + # See condition for using PyArray_SetBaseObject in + # array_setstate. + a = np.arange(1000) + for v in range(pickle.HIGHEST_PROTOCOL): + vals = pickle.loads(pickle.dumps(a, v)) + assert_(vals.flags.writeable) + assert_(isinstance(vals.base, bytes)) + def test_otherflags(self): assert_equal(self.a.flags.carray, True) assert_equal(self.a.flags['C'], True) @@ -2595,7 +2645,8 @@ class TestMethods(object): assert_equal(x1.flatten('F'), y1f) assert_equal(x1.flatten('F'), x1.T.flatten()) - def test_dot(self): + @pytest.mark.parametrize('func', (np.dot, np.matmul)) + def test_arr_mult(self, func): a = np.array([[1, 0], [0, 1]]) b = np.array([[0, 1], [1, 0]]) c = np.array([[9, 1], [1, -9]]) @@ -2619,49 +2670,49 @@ class TestMethods(object): # gemm vs syrk optimizations for et in [np.float32, np.float64, np.complex64, np.complex128]: eaf = a.astype(et) - assert_equal(np.dot(eaf, eaf), eaf) - assert_equal(np.dot(eaf.T, eaf), eaf) - assert_equal(np.dot(eaf, eaf.T), eaf) - assert_equal(np.dot(eaf.T, eaf.T), eaf) - assert_equal(np.dot(eaf.T.copy(), eaf), eaf) - assert_equal(np.dot(eaf, eaf.T.copy()), eaf) - assert_equal(np.dot(eaf.T.copy(), eaf.T.copy()), eaf) + assert_equal(func(eaf, eaf), eaf) + assert_equal(func(eaf.T, eaf), eaf) + assert_equal(func(eaf, eaf.T), eaf) + assert_equal(func(eaf.T, eaf.T), eaf) + assert_equal(func(eaf.T.copy(), eaf), eaf) + assert_equal(func(eaf, eaf.T.copy()), eaf) + assert_equal(func(eaf.T.copy(), eaf.T.copy()), eaf) # syrk validations for et in [np.float32, np.float64, np.complex64, np.complex128]: eaf = a.astype(et) ebf = b.astype(et) - assert_equal(np.dot(ebf, ebf), eaf) - assert_equal(np.dot(ebf.T, ebf), eaf) - assert_equal(np.dot(ebf, ebf.T), eaf) - assert_equal(np.dot(ebf.T, ebf.T), eaf) + assert_equal(func(ebf, ebf), eaf) + assert_equal(func(ebf.T, ebf), eaf) + assert_equal(func(ebf, ebf.T), eaf) + assert_equal(func(ebf.T, ebf.T), eaf) # syrk - different shape, stride, and view validations for et in [np.float32, np.float64, np.complex64, np.complex128]: edf = d.astype(et) assert_equal( - np.dot(edf[::-1, :], edf.T), - np.dot(edf[::-1, :].copy(), edf.T.copy()) + func(edf[::-1, :], edf.T), + func(edf[::-1, :].copy(), edf.T.copy()) ) assert_equal( - np.dot(edf[:, ::-1], edf.T), - np.dot(edf[:, ::-1].copy(), edf.T.copy()) + func(edf[:, ::-1], edf.T), + func(edf[:, ::-1].copy(), edf.T.copy()) ) assert_equal( - np.dot(edf, edf[::-1, :].T), - np.dot(edf, edf[::-1, :].T.copy()) + func(edf, edf[::-1, :].T), + func(edf, edf[::-1, :].T.copy()) ) assert_equal( - np.dot(edf, edf[:, ::-1].T), - np.dot(edf, edf[:, ::-1].T.copy()) + func(edf, edf[:, ::-1].T), + func(edf, edf[:, ::-1].T.copy()) ) assert_equal( - np.dot(edf[:edf.shape[0] // 2, :], edf[::2, :].T), - np.dot(edf[:edf.shape[0] // 2, :].copy(), edf[::2, :].T.copy()) + func(edf[:edf.shape[0] // 2, :], edf[::2, :].T), + func(edf[:edf.shape[0] // 2, :].copy(), edf[::2, :].T.copy()) ) assert_equal( - np.dot(edf[::2, :], edf[:edf.shape[0] // 2, :].T), - np.dot(edf[::2, :].copy(), edf[:edf.shape[0] // 2, :].T.copy()) + func(edf[::2, :], edf[:edf.shape[0] // 2, :].T), + func(edf[::2, :].copy(), edf[:edf.shape[0] // 2, :].T.copy()) ) # syrk - different shape @@ -2669,9 +2720,43 @@ class TestMethods(object): edf = d.astype(et) eddtf = ddt.astype(et) edtdf = dtd.astype(et) - assert_equal(np.dot(edf, edf.T), eddtf) - assert_equal(np.dot(edf.T, edf), edtdf) + assert_equal(func(edf, edf.T), eddtf) + assert_equal(func(edf.T, edf), edtdf) + + @pytest.mark.parametrize('func', (np.dot, np.matmul)) + @pytest.mark.parametrize('dtype', 'ifdFD') + def test_no_dgemv(self, func, dtype): + # check vector arg for contiguous before gemv + # gh-12156 + a = np.arange(8.0, dtype=dtype).reshape(2, 4) + b = np.broadcast_to(1., (4, 1)) + ret1 = func(a, b) + ret2 = func(a, b.copy()) + assert_equal(ret1, ret2) + + ret1 = func(b.T, a.T) + ret2 = func(b.T.copy(), a.T) + assert_equal(ret1, ret2) + + # check for unaligned data + dt = np.dtype(dtype) + a = np.zeros(8 * dt.itemsize // 2 + 1, dtype='int16')[1:].view(dtype) + a = a.reshape(2, 4) + b = a[0] + # make sure it is not aligned + assert_(a.__array_interface__['data'][0] % dt.itemsize != 0) + ret1 = func(a, b) + ret2 = func(a.copy(), b.copy()) + assert_equal(ret1, ret2) + + ret1 = func(b.T, a.T) + ret2 = func(b.T.copy(), a.T.copy()) + assert_equal(ret1, ret2) + def test_dot(self): + a = np.array([[1, 0], [0, 1]]) + b = np.array([[0, 1], [1, 0]]) + c = np.array([[9, 1], [1, -9]]) # function versus methods assert_equal(np.dot(a, b), a.dot(b)) assert_equal(np.dot(np.dot(a, b), c), a.dot(b).dot(c)) @@ -2727,6 +2812,29 @@ class TestMethods(object): np.dot(a, b, out=out) np.matmul(a, b, out=out) + def test_dot_matmul_inner_array_casting_fails(self): + + class A(object): + def __array__(self, *args, **kwargs): + raise NotImplementedError + + # Don't override the error from calling __array__() + assert_raises(NotImplementedError, np.dot, A(), A()) + assert_raises(NotImplementedError, np.matmul, A(), A()) + assert_raises(NotImplementedError, np.inner, A(), A()) + + def test_matmul_out(self): + # overlapping memory + a = np.arange(18).reshape(2, 3, 3) + b = np.matmul(a, a) + c = np.matmul(a, a, out=a) + assert_(c is a) + assert_equal(c, b) + a = np.arange(18).reshape(2, 3, 3) + c = np.matmul(a, a, out=a[::-1, ...]) + assert_(c.base is a.base) + assert_equal(c, b) + def test_diagonal(self): a = np.arange(12).reshape((3, 4)) assert_equal(a.diagonal(), [0, 5, 10]) @@ -3136,6 +3244,8 @@ class TestBinop(object): # 'eq': (np.equal, False), # 'ne': (np.not_equal, False), } + if sys.version_info >= (3, 5): + ops['matmul'] = (np.matmul, False, float) class Coerced(Exception): pass @@ -3178,7 +3288,7 @@ class TestBinop(object): if issubclass(MyType, np.ndarray): # Use this range to avoid special case weirdnesses around # divide-by-0, pow(x, 2), overflow due to pow(big, big), etc. - return np.arange(3, 5).view(MyType) + return np.arange(3, 7).reshape(2, 2).view(MyType) else: return MyType() @@ -3187,7 +3297,7 @@ class TestBinop(object): for op, (ufunc, has_inplace, dtype) in ops.items(): err_msg = ('op: %s, ufunc: %s, has_inplace: %s, dtype: %s' % (op, ufunc, has_inplace, dtype)) - check_objs = [np.arange(3, 5, dtype=dtype)] + check_objs = [np.arange(3, 7, dtype=dtype).reshape(2, 2)] if check_scalar: check_objs.append(check_objs[0][0]) for arr in check_objs: @@ -3582,7 +3692,6 @@ class TestPickling(object): 'protocol 5 although it is not available')) def test_correct_protocol5_error_message(self): array = np.arange(10) - f = io.BytesIO() if sys.version_info[:2] in ((3, 6), (3, 7)): # For the specific case of python3.6 and 3.7, raise a clear import @@ -4541,6 +4650,19 @@ class TestIO(object): f.close() assert_equal(pos, 10, err_msg=err_msg) + def test_load_object_array_fromfile(self): + # gh-12300 + with open(self.filename, 'w') as f: + # Ensure we have a file with consistent contents + pass + + with open(self.filename, 'rb') as f: + assert_raises_regex(ValueError, "Cannot read into object array", + np.fromfile, f, dtype=object) + + assert_raises_regex(ValueError, "Cannot read into object array", + np.fromfile, self.filename, dtype=object) + def _check_from(self, s, value, **kw): if 'sep' not in kw: y = np.frombuffer(s, **kw) @@ -4843,11 +4965,11 @@ class TestRecord(object): assert_equal(dt.names, ['p', 'q']) def test_multiple_field_name_occurrence(self): - def test_assign(): - dtype = np.dtype([("A", "f8"), ("B", "f8"), ("A", "f8")]) + def test_dtype_init(): + np.dtype([("A", "f8"), ("B", "f8"), ("A", "f8")]) # Error raised when multiple fields have the same name - assert_raises(ValueError, test_assign) + assert_raises(ValueError, test_dtype_init) @pytest.mark.skipif(sys.version_info[0] < 3, reason="Not Python 3") def test_bytes_fields(self): @@ -4867,13 +4989,11 @@ class TestRecord(object): @pytest.mark.skipif(sys.version_info[0] < 3, reason="Not Python 3") def test_multiple_field_name_unicode(self): - def test_assign_unicode(): - dt = np.dtype([("\u20B9", "f8"), - ("B", "f8"), - ("\u20B9", "f8")]) + def test_dtype_unicode(): + np.dtype([("\u20B9", "f8"), ("B", "f8"), ("\u20B9", "f8")]) # Error raised when multiple fields have the same name(unicode included) - assert_raises(ValueError, test_assign_unicode) + assert_raises(ValueError, test_dtype_unicode) @pytest.mark.skipif(sys.version_info[0] >= 3, reason="Not Python 2") def test_unicode_field_titles(self): @@ -4955,25 +5075,9 @@ class TestRecord(object): fn2 = func('f2') b[fn2] = 3 - # In 1.16 code below can be replaced by: - # assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3)) - # assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2)) - # assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,))) - with suppress_warnings() as sup: - sup.filter(FutureWarning, - ".* selecting multiple fields .*") - - assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3)) - assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2)) - assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,))) - # view of subfield view/copy - assert_equal(b[['f1', 'f2']][0].view(('i4', 2)).tolist(), - (2, 3)) - assert_equal(b[['f2', 'f1']][0].view(('i4', 2)).tolist(), - (3, 2)) - view_dtype = [('f1', 'i4'), ('f3', [('', 'i4')])] - assert_equal(b[['f1', 'f3']][0].view(view_dtype).tolist(), - (2, (1,))) + assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3)) + assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2)) + assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,))) # non-ascii unicode field indexing is well behaved if not is_py3: @@ -4983,50 +5087,6 @@ class TestRecord(object): assert_raises(ValueError, a.__setitem__, u'\u03e0', 1) assert_raises(ValueError, a.__getitem__, u'\u03e0') - # can be removed in 1.16 - def test_field_names_deprecation(self): - - def collect_warnings(f, *args, **kwargs): - with warnings.catch_warnings(record=True) as log: - warnings.simplefilter("always") - f(*args, **kwargs) - return [w.category for w in log] - - a = np.zeros((1,), dtype=[('f1', 'i4'), - ('f2', 'i4'), - ('f3', [('sf1', 'i4')])]) - a['f1'][0] = 1 - a['f2'][0] = 2 - a['f3'][0] = (3,) - b = np.zeros((1,), dtype=[('f1', 'i4'), - ('f2', 'i4'), - ('f3', [('sf1', 'i4')])]) - b['f1'][0] = 1 - b['f2'][0] = 2 - b['f3'][0] = (3,) - - # All the different functions raise a warning, but not an error - assert_equal(collect_warnings(a[['f1', 'f2']].__setitem__, 0, (10, 20)), - [FutureWarning]) - # For <=1.12 a is not modified, but it will be in 1.13 - assert_equal(a, b) - - # Views also warn - subset = a[['f1', 'f2']] - subset_view = subset.view() - assert_equal(collect_warnings(subset_view['f1'].__setitem__, 0, 10), - [FutureWarning]) - # But the write goes through: - assert_equal(subset['f1'][0], 10) - # Only one warning per multiple field indexing, though (even if there - # are multiple views involved): - assert_equal(collect_warnings(subset['f1'].__setitem__, 0, 10), []) - - # make sure views of a multi-field index warn too - c = np.zeros(3, dtype='i8,i8,i8') - assert_equal(collect_warnings(c[['f0', 'f2']].view, 'i8,i8'), - [FutureWarning]) - def test_record_hash(self): a = np.array([(1, 2), (1, 2)], dtype='i1,i2') a.flags.writeable = False @@ -5050,6 +5110,16 @@ class TestRecord(object): np.array([(), (), (), (), ()], dtype={'names': [], 'formats': [], 'offsets': [], 'itemsize': 12}) + def test_multifield_indexing_view(self): + a = np.ones(3, dtype=[('a', 'i4'), ('b', 'f4'), ('c', 'u4')]) + v = a[['a', 'c']] + assert_(v.base is a) + assert_(v.dtype == np.dtype({'names': ['a', 'c'], + 'formats': ['i4', 'u4'], + 'offsets': [0, 8]})) + v[:] = (4,5) + assert_equal(a[0].item(), (4, 1, 5)) + class TestView(object): def test_basic(self): x = np.array([(1, 2, 3, 4), (5, 6, 7, 8)], @@ -5673,15 +5743,38 @@ class MatmulCommon(object): res = self.matmul(v, v) assert_(type(res) is np.dtype(dt).type) - def test_vector_vector_values(self): - vec = np.array([1, 2]) - tgt = 5 + def test_scalar_output(self): + vec1 = np.array([2]) + vec2 = np.array([3, 4]).reshape(1, -1) + tgt = np.array([6, 8]) for dt in self.types[1:]: - v1 = vec.astype(dt) - res = self.matmul(v1, v1) + v1 = vec1.astype(dt) + v2 = vec2.astype(dt) + res = self.matmul(v1, v2) + assert_equal(res, tgt) + res = self.matmul(v2.T, v1) assert_equal(res, tgt) # boolean type + vec = np.array([True, True], dtype='?').reshape(1, -1) + res = self.matmul(vec[:, 0], vec) + assert_equal(res, True) + + def test_vector_vector_values(self): + vec1 = np.array([1, 2]) + vec2 = np.array([3, 4]).reshape(-1, 1) + tgt1 = np.array([11]) + tgt2 = np.array([[3, 6], [4, 8]]) + for dt in self.types[1:]: + v1 = vec1.astype(dt) + v2 = vec2.astype(dt) + res = self.matmul(v1, v2) + assert_equal(res, tgt1) + # no broadcast, we must make v1 into a 2d ndarray + res = self.matmul(v2, v1.reshape(1, -1)) + assert_equal(res, tgt2) + + # boolean type vec = np.array([True, True], dtype='?') res = self.matmul(vec, vec) assert_equal(res, True) @@ -5810,44 +5903,96 @@ class TestMatmul(MatmulCommon): matmul = np.matmul def test_out_arg(self): - a = np.ones((2, 2), dtype=float) - b = np.ones((2, 2), dtype=float) - tgt = np.full((2,2), 2, dtype=float) + a = np.ones((5, 2), dtype=float) + b = np.array([[1, 3], [5, 7]], dtype=float) + tgt = np.dot(a, b) # test as positional argument msg = "out positional argument" - out = np.zeros((2, 2), dtype=float) + out = np.zeros((5, 2), dtype=float) self.matmul(a, b, out) assert_array_equal(out, tgt, err_msg=msg) # test as keyword argument msg = "out keyword argument" - out = np.zeros((2, 2), dtype=float) + out = np.zeros((5, 2), dtype=float) self.matmul(a, b, out=out) assert_array_equal(out, tgt, err_msg=msg) # test out with not allowed type cast (safe casting) - # einsum and cblas raise different error types, so - # use Exception. - msg = "out argument with illegal cast" - out = np.zeros((2, 2), dtype=np.int32) - assert_raises(Exception, self.matmul, a, b, out=out) - - # skip following tests for now, cblas does not allow non-contiguous - # outputs and consistency with dot would require same type, - # dimensions, subtype, and c_contiguous. - - # test out with allowed type cast - # msg = "out argument with allowed cast" - # out = np.zeros((2, 2), dtype=np.complex128) - # self.matmul(a, b, out=out) - # assert_array_equal(out, tgt, err_msg=msg) + msg = "Cannot cast ufunc matmul output" + out = np.zeros((5, 2), dtype=np.int32) + assert_raises_regex(TypeError, msg, self.matmul, a, b, out=out) + + # test out with type upcast to complex + out = np.zeros((5, 2), dtype=np.complex128) + c = self.matmul(a, b, out=out) + assert_(c is out) + with suppress_warnings() as sup: + sup.filter(np.ComplexWarning, '') + c = c.astype(tgt.dtype) + assert_array_equal(c, tgt) + + def test_out_contiguous(self): + a = np.ones((5, 2), dtype=float) + b = np.array([[1, 3], [5, 7]], dtype=float) + v = np.array([1, 3], dtype=float) + tgt = np.dot(a, b) + tgt_mv = np.dot(a, v) # test out non-contiguous - # msg = "out argument with non-contiguous layout" - # c = np.zeros((2, 2, 2), dtype=float) - # self.matmul(a, b, out=c[..., 0]) - # assert_array_equal(c, tgt, err_msg=msg) + out = np.ones((5, 2, 2), dtype=float) + c = self.matmul(a, b, out=out[..., 0]) + assert c.base is out + assert_array_equal(c, tgt) + c = self.matmul(a, v, out=out[:, 0, 0]) + assert_array_equal(c, tgt_mv) + c = self.matmul(v, a.T, out=out[:, 0, 0]) + assert_array_equal(c, tgt_mv) + + # test out contiguous in only last dim + out = np.ones((10, 2), dtype=float) + c = self.matmul(a, b, out=out[::2, :]) + assert_array_equal(c, tgt) + + # test transposes of out, args + out = np.ones((5, 2), dtype=float) + c = self.matmul(b.T, a.T, out=out.T) + assert_array_equal(out, tgt) + + m1 = np.arange(15.).reshape(5, 3) + m2 = np.arange(21.).reshape(3, 7) + m3 = np.arange(30.).reshape(5, 6)[:, ::2] # non-contiguous + vc = np.arange(10.) + vr = np.arange(6.) + m0 = np.zeros((3, 0)) + @pytest.mark.parametrize('args', ( + # matrix-matrix + (m1, m2), (m2.T, m1.T), (m2.T.copy(), m1.T), (m2.T, m1.T.copy()), + # matrix-matrix-transpose, contiguous and non + (m1, m1.T), (m1.T, m1), (m1, m3.T), (m3, m1.T), + (m3, m3.T), (m3.T, m3), + # matrix-matrix non-contiguous + (m3, m2), (m2.T, m3.T), (m2.T.copy(), m3.T), + # vector-matrix, matrix-vector, contiguous + (m1, vr[:3]), (vc[:5], m1), (m1.T, vc[:5]), (vr[:3], m1.T), + # vector-matrix, matrix-vector, vector non-contiguous + (m1, vr[::2]), (vc[::2], m1), (m1.T, vc[::2]), (vr[::2], m1.T), + # vector-matrix, matrix-vector, matrix non-contiguous + (m3, vr[:3]), (vc[:5], m3), (m3.T, vc[:5]), (vr[:3], m3.T), + # vector-matrix, matrix-vector, both non-contiguous + (m3, vr[::2]), (vc[::2], m3), (m3.T, vc[::2]), (vr[::2], m3.T), + # size == 0 + (m0, m0.T), (m0.T, m0), (m1, m0), (m0.T, m1.T), + )) + def test_dot_equivalent(self, args): + r1 = np.matmul(*args) + r2 = np.dot(*args) + assert_equal(r1, r2) + + r3 = np.matmul(args[0].copy(), args[1].copy()) + assert_equal(r1, r3) + if sys.version_info[:2] >= (3, 5): @@ -5871,6 +6016,11 @@ if sys.version_info[:2] >= (3, 5): assert_equal(self.matmul(a, b), "A") assert_equal(self.matmul(b, a), "A") + def test_matmul_raises(self): + assert_raises(TypeError, self.matmul, np.int8(5), np.int8(5)) + assert_raises(TypeError, self.matmul, np.void(b'abc'), np.void(b'abc')) + assert_raises(ValueError, self.matmul, np.arange(10), np.void(b'abc')) + def test_matmul_inplace(): # It would be nice to support in-place matmul eventually, but for now # we don't have a working implementation, so better just to error out @@ -5885,6 +6035,17 @@ if sys.version_info[:2] >= (3, 5): exec_ = getattr(builtins, "exec") assert_raises(TypeError, exec_, "a @= b", globals(), locals()) + def test_matmul_axes(): + a = np.arange(3*4*5).reshape(3, 4, 5) + c = np.matmul(a, a, axes=[(-2, -1), (-1, -2), (1, 2)]) + assert c.shape == (3, 4, 4) + d = np.matmul(a, a, axes=[(-2, -1), (-1, -2), (0, 1)]) + assert d.shape == (4, 4, 3) + e = np.swapaxes(d, 0, 2) + assert_array_equal(e, c) + f = np.matmul(a, np.arange(3), axes=[(1, 0), (0), (0)]) + assert f.shape == (4, 5) + class TestInner(object): @@ -6775,7 +6936,7 @@ class TestNewBufferProtocol(object): ValueError, "format string", np.array, m) - def test_error_message(self): + def test_error_message_unsupported(self): # wchar has no corresponding numpy type - if this changes in future, we # need a better way to construct an invalid memoryview format. t = ctypes.c_wchar * 4 @@ -6784,14 +6945,17 @@ class TestNewBufferProtocol(object): exc = cm.exception if sys.version_info.major > 2: - with assert_raises_regex(ValueError, "Unknown .* specifier 'u'"): + with assert_raises_regex( + NotImplementedError, + r"Unrepresentable .* 'u' \(UCS-2 strings\)" + ): raise exc.__cause__ def test_ctypes_integer_via_memoryview(self): # gh-11150, due to bpo-10746 for c_integer in {ctypes.c_int, ctypes.c_long, ctypes.c_longlong}: value = c_integer(42) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(record=True): warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning) np.asarray(value) @@ -6801,7 +6965,7 @@ class TestNewBufferProtocol(object): _fields_ = [('a', ctypes.c_uint8), ('b', ctypes.c_uint32)] f = foo(a=1, b=2) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(record=True): warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning) arr = np.asarray(f) @@ -7472,6 +7636,55 @@ class TestCTypes(object): finally: _internal.ctypes = ctypes + def _make_readonly(x): + x.flags.writeable = False + return x + + @pytest.mark.parametrize('arr', [ + np.array([1, 2, 3]), + np.array([['one', 'two'], ['three', 'four']]), + np.array((1, 2), dtype='i4,i4'), + np.zeros((2,), dtype= + np.dtype(dict( + formats=['<i4', '<i4'], + names=['a', 'b'], + offsets=[0, 2], + itemsize=6 + )) + ), + np.array([None], dtype=object), + np.array([]), + np.empty((0, 0)), + _make_readonly(np.array([1, 2, 3])), + ], ids=[ + '1d', + '2d', + 'structured', + 'overlapping', + 'object', + 'empty', + 'empty-2d', + 'readonly' + ]) + def test_ctypes_data_as_holds_reference(self, arr): + # gh-9647 + # create a copy to ensure that pytest does not mess with the refcounts + arr = arr.copy() + + arr_ref = weakref.ref(arr) + + ctypes_ptr = arr.ctypes.data_as(ctypes.c_void_p) + + # `ctypes_ptr` should hold onto `arr` + del arr + gc.collect() + assert_(arr_ref() is not None, "ctypes pointer did not hold onto a reference") + + # but when the `ctypes_ptr` object dies, so should `arr` + del ctypes_ptr + gc.collect() + assert_(arr_ref() is None, "unknowable whether ctypes pointer holds a reference") + class TestWritebackIfCopy(object): # all these tests use the WRITEBACKIFCOPY mechanism @@ -7491,6 +7704,7 @@ class TestWritebackIfCopy(object): mat = np.eye(5) out = np.eye(5, dtype='i2') res = np.clip(mat, a_min=-10, a_max=0, out=out) + assert_(res is out) assert_equal(np.sum(out), 0) def test_insert_noncontiguous(self): @@ -7768,6 +7982,44 @@ def test_uintalignment_and_alignment(): dst = np.zeros((2,2), dtype='c8') dst[:,1] = src[:,1] # assert in lowlevel_strided_loops fails? +class TestAlignment(object): + # adapted from scipy._lib.tests.test__util.test__aligned_zeros + # Checks that unusual memory alignments don't trip up numpy. + # In particular, check RELAXED_STRIDES don't trip alignment assertions in + # NDEBUG mode for size-0 arrays (gh-12503) + + def check(self, shape, dtype, order, align): + err_msg = repr((shape, dtype, order, align)) + x = _aligned_zeros(shape, dtype, order, align=align) + if align is None: + align = np.dtype(dtype).alignment + assert_equal(x.__array_interface__['data'][0] % align, 0) + if hasattr(shape, '__len__'): + assert_equal(x.shape, shape, err_msg) + else: + assert_equal(x.shape, (shape,), err_msg) + assert_equal(x.dtype, dtype) + if order == "C": + assert_(x.flags.c_contiguous, err_msg) + elif order == "F": + if x.size > 0: + assert_(x.flags.f_contiguous, err_msg) + elif order is None: + assert_(x.flags.c_contiguous, err_msg) + else: + raise ValueError() + + def test_various_alignments(self): + for align in [1, 2, 3, 4, 8, 12, 16, 32, 64, None]: + for n in [0, 1, 3, 11]: + for order in ["C", "F", None]: + for dtype in list(np.typecodes["All"]) + ['i4,i4,i4']: + if dtype == 'O': + # object dtype can't be misaligned + continue + for shape in [n, (1, 2, 3, n)]: + self.check(shape, np.dtype(dtype), order, align) + def test_getfield(): a = np.arange(32, dtype='uint16') if sys.byteorder == 'little': diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index 5e8165bc5..26fd9c346 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -1,14 +1,13 @@ from __future__ import division, absolute_import, print_function import sys -import warnings import pytest import numpy as np import numpy.core._multiarray_tests as _multiarray_tests from numpy import array, arange, nditer, all from numpy.testing import ( - assert_, assert_equal, assert_array_equal, assert_raises, assert_warns, + assert_, assert_equal, assert_array_equal, assert_raises, HAS_REFCOUNT, suppress_warnings ) @@ -2196,21 +2195,15 @@ class TestIterNested(object): a = arange(12).reshape(2, 3, 2) i, j = np.nested_iters(a, [[0], [1, 2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9, 10, 11]]) i, j = np.nested_iters(a, [[0, 1], [2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11]]) i, j = np.nested_iters(a, [[0, 2], [1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4], [1, 3, 5], [6, 8, 10], [7, 9, 11]]) def test_reorder(self): @@ -2219,40 +2212,28 @@ class TestIterNested(object): # In 'K' order (default), it gets reordered i, j = np.nested_iters(a, [[0], [2, 1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9, 10, 11]]) i, j = np.nested_iters(a, [[1, 0], [2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11]]) i, j = np.nested_iters(a, [[2, 0], [1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4], [1, 3, 5], [6, 8, 10], [7, 9, 11]]) # In 'C' order, it doesn't i, j = np.nested_iters(a, [[0], [2, 1]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4, 1, 3, 5], [6, 8, 10, 7, 9, 11]]) i, j = np.nested_iters(a, [[1, 0], [2]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1], [6, 7], [2, 3], [8, 9], [4, 5], [10, 11]]) i, j = np.nested_iters(a, [[2, 0], [1]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4], [6, 8, 10], [1, 3, 5], [7, 9, 11]]) def test_flip_axes(self): @@ -2261,40 +2242,28 @@ class TestIterNested(object): # In 'K' order (default), the axes all get flipped i, j = np.nested_iters(a, [[0], [1, 2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9, 10, 11]]) i, j = np.nested_iters(a, [[0, 1], [2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11]]) i, j = np.nested_iters(a, [[0, 2], [1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4], [1, 3, 5], [6, 8, 10], [7, 9, 11]]) # In 'C' order, flipping axes is disabled i, j = np.nested_iters(a, [[0], [1, 2]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[11, 10, 9, 8, 7, 6], [5, 4, 3, 2, 1, 0]]) i, j = np.nested_iters(a, [[0, 1], [2]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[11, 10], [9, 8], [7, 6], [5, 4], [3, 2], [1, 0]]) i, j = np.nested_iters(a, [[0, 2], [1]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[11, 9, 7], [10, 8, 6], [5, 3, 1], [4, 2, 0]]) def test_broadcast(self): @@ -2303,15 +2272,11 @@ class TestIterNested(object): b = arange(3).reshape(1, 3) i, j = np.nested_iters([a, b], [[0], [1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[[0, 0], [0, 1], [0, 2]], [[1, 0], [1, 1], [1, 2]]]) i, j = np.nested_iters([a, b], [[1], [0]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[[0, 0], [1, 0]], [[0, 1], [1, 1]], [[0, 2], [1, 2]]]) def test_dtype_copy(self): @@ -2323,9 +2288,7 @@ class TestIterNested(object): op_flags=['readonly', 'copy'], op_dtypes='f8') assert_equal(j[0].dtype, np.dtype('f8')) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2], [3, 4, 5]]) vals = None @@ -2376,15 +2339,11 @@ class TestIterNested(object): def test_0d(self): a = np.arange(12).reshape(2, 3, 2) i, j = np.nested_iters(a, [[], [1, 0, 2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]]) i, j = np.nested_iters(a, [[1, 0, 2], []]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11]]) i, j, k = np.nested_iters(a, [[2, 0], [], [1]]) @@ -2556,10 +2515,8 @@ def test_iter_buffering_reduction_reuse_reduce_loops(): op_flags=[['readonly'], ['readwrite']], buffersize=5) - bufsizes = [] with it: - for x, y in it: - bufsizes.append(x.shape[0]) + bufsizes = [x.shape[0] for x, y in it] assert_equal(bufsizes, [5, 2, 5, 2]) assert_equal(sum(bufsizes), a.size) diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index f264c4ab0..37534720a 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -13,7 +13,7 @@ from numpy.random import rand, randint, randn from numpy.testing import ( assert_, assert_equal, assert_raises, assert_raises_regex, assert_array_equal, assert_almost_equal, assert_array_almost_equal, - suppress_warnings, HAS_REFCOUNT + HAS_REFCOUNT ) diff --git a/numpy/core/tests/test_numerictypes.py b/numpy/core/tests/test_numerictypes.py index 27e4fdeec..71f7b7150 100644 --- a/numpy/core/tests/test_numerictypes.py +++ b/numpy/core/tests/test_numerictypes.py @@ -87,10 +87,8 @@ def normalize_descr(descr): else: nitem = (item[0], dtype) out.append(nitem) - elif isinstance(item[1], list): - l = [] - for j in normalize_descr(item[1]): - l.append(j) + elif isinstance(dtype, list): + l = normalize_descr(dtype) out.append((item[0], l)) else: raise ValueError("Expected a str or list and got %s" % diff --git a/numpy/core/tests/test_overrides.py b/numpy/core/tests/test_overrides.py index 3f87a6afe..8f1c16539 100644 --- a/numpy/core/tests/test_overrides.py +++ b/numpy/core/tests/test_overrides.py @@ -1,51 +1,62 @@ from __future__ import division, absolute_import, print_function +import inspect import sys import numpy as np from numpy.testing import ( assert_, assert_equal, assert_raises, assert_raises_regex) from numpy.core.overrides import ( - get_overloaded_types_and_args, array_function_dispatch, - verify_matching_signatures) + _get_implementing_args, array_function_dispatch, + verify_matching_signatures, ENABLE_ARRAY_FUNCTION) from numpy.core.numeric import pickle +import pytest -def _get_overloaded_args(relevant_args): - types, args = get_overloaded_types_and_args(relevant_args) - return args +requires_array_function = pytest.mark.skipif( + not ENABLE_ARRAY_FUNCTION, + reason="__array_function__ dispatch not enabled.") -def _return_self(self, *args, **kwargs): - return self +def _return_not_implemented(self, *args, **kwargs): + return NotImplemented -class TestGetOverloadedTypesAndArgs(object): +# need to define this at the top level to test pickling +@array_function_dispatch(lambda array: (array,)) +def dispatched_one_arg(array): + """Docstring.""" + return 'original' + + +@array_function_dispatch(lambda array1, array2: (array1, array2)) +def dispatched_two_arg(array1, array2): + """Docstring.""" + return 'original' + + +@requires_array_function +class TestGetImplementingArgs(object): def test_ndarray(self): array = np.array(1) - types, args = get_overloaded_types_and_args([array]) - assert_equal(set(types), {np.ndarray}) - assert_equal(list(args), []) + args = _get_implementing_args([array]) + assert_equal(list(args), [array]) - types, args = get_overloaded_types_and_args([array, array]) - assert_equal(len(types), 1) - assert_equal(set(types), {np.ndarray}) - assert_equal(list(args), []) + args = _get_implementing_args([array, array]) + assert_equal(list(args), [array]) - types, args = get_overloaded_types_and_args([array, 1]) - assert_equal(set(types), {np.ndarray}) - assert_equal(list(args), []) + args = _get_implementing_args([array, 1]) + assert_equal(list(args), [array]) - types, args = get_overloaded_types_and_args([1, array]) - assert_equal(set(types), {np.ndarray}) - assert_equal(list(args), []) + args = _get_implementing_args([1, array]) + assert_equal(list(args), [array]) def test_ndarray_subclasses(self): class OverrideSub(np.ndarray): - __array_function__ = _return_self + __array_function__ = _return_not_implemented class NoOverrideSub(np.ndarray): pass @@ -54,117 +65,148 @@ class TestGetOverloadedTypesAndArgs(object): override_sub = np.array(1).view(OverrideSub) no_override_sub = np.array(1).view(NoOverrideSub) - types, args = get_overloaded_types_and_args([array, override_sub]) - assert_equal(set(types), {np.ndarray, OverrideSub}) - assert_equal(list(args), [override_sub]) + args = _get_implementing_args([array, override_sub]) + assert_equal(list(args), [override_sub, array]) - types, args = get_overloaded_types_and_args([array, no_override_sub]) - assert_equal(set(types), {np.ndarray, NoOverrideSub}) - assert_equal(list(args), []) + args = _get_implementing_args([array, no_override_sub]) + assert_equal(list(args), [no_override_sub, array]) - types, args = get_overloaded_types_and_args( + args = _get_implementing_args( [override_sub, no_override_sub]) - assert_equal(set(types), {OverrideSub, NoOverrideSub}) - assert_equal(list(args), [override_sub]) + assert_equal(list(args), [override_sub, no_override_sub]) def test_ndarray_and_duck_array(self): class Other(object): - __array_function__ = _return_self + __array_function__ = _return_not_implemented array = np.array(1) other = Other() - types, args = get_overloaded_types_and_args([other, array]) - assert_equal(set(types), {np.ndarray, Other}) - assert_equal(list(args), [other]) + args = _get_implementing_args([other, array]) + assert_equal(list(args), [other, array]) - types, args = get_overloaded_types_and_args([array, other]) - assert_equal(set(types), {np.ndarray, Other}) - assert_equal(list(args), [other]) + args = _get_implementing_args([array, other]) + assert_equal(list(args), [array, other]) def test_ndarray_subclass_and_duck_array(self): class OverrideSub(np.ndarray): - __array_function__ = _return_self + __array_function__ = _return_not_implemented class Other(object): - __array_function__ = _return_self + __array_function__ = _return_not_implemented array = np.array(1) subarray = np.array(1).view(OverrideSub) other = Other() - assert_equal(_get_overloaded_args([array, subarray, other]), - [subarray, other]) - assert_equal(_get_overloaded_args([array, other, subarray]), - [subarray, other]) + assert_equal(_get_implementing_args([array, subarray, other]), + [subarray, array, other]) + assert_equal(_get_implementing_args([array, other, subarray]), + [subarray, array, other]) def test_many_duck_arrays(self): class A(object): - __array_function__ = _return_self + __array_function__ = _return_not_implemented class B(A): - __array_function__ = _return_self + __array_function__ = _return_not_implemented class C(A): - __array_function__ = _return_self + __array_function__ = _return_not_implemented class D(object): - __array_function__ = _return_self + __array_function__ = _return_not_implemented a = A() b = B() c = C() d = D() - assert_equal(_get_overloaded_args([1]), []) - assert_equal(_get_overloaded_args([a]), [a]) - assert_equal(_get_overloaded_args([a, 1]), [a]) - assert_equal(_get_overloaded_args([a, a, a]), [a]) - assert_equal(_get_overloaded_args([a, d, a]), [a, d]) - assert_equal(_get_overloaded_args([a, b]), [b, a]) - assert_equal(_get_overloaded_args([b, a]), [b, a]) - assert_equal(_get_overloaded_args([a, b, c]), [b, c, a]) - assert_equal(_get_overloaded_args([a, c, b]), [c, b, a]) + assert_equal(_get_implementing_args([1]), []) + assert_equal(_get_implementing_args([a]), [a]) + assert_equal(_get_implementing_args([a, 1]), [a]) + assert_equal(_get_implementing_args([a, a, a]), [a]) + assert_equal(_get_implementing_args([a, d, a]), [a, d]) + assert_equal(_get_implementing_args([a, b]), [b, a]) + assert_equal(_get_implementing_args([b, a]), [b, a]) + assert_equal(_get_implementing_args([a, b, c]), [b, c, a]) + assert_equal(_get_implementing_args([a, c, b]), [c, b, a]) + def test_too_many_duck_arrays(self): + namespace = dict(__array_function__=_return_not_implemented) + types = [type('A' + str(i), (object,), namespace) for i in range(33)] + relevant_args = [t() for t in types] + actual = _get_implementing_args(relevant_args[:32]) + assert_equal(actual, relevant_args[:32]) + + with assert_raises_regex(TypeError, 'distinct argument types'): + _get_implementing_args(relevant_args) + + +@requires_array_function class TestNDArrayArrayFunction(object): def test_method(self): - class SubOverride(np.ndarray): - __array_function__ = _return_self + class Other(object): + __array_function__ = _return_not_implemented class NoOverrideSub(np.ndarray): pass - array = np.array(1) + class OverrideSub(np.ndarray): + __array_function__ = _return_not_implemented - def func(): - return 'original' + array = np.array([1]) + other = Other() + no_override_sub = array.view(NoOverrideSub) + override_sub = array.view(OverrideSub) - result = array.__array_function__( - func=func, types=(np.ndarray,), args=(), kwargs={}) + result = array.__array_function__(func=dispatched_two_arg, + types=(np.ndarray,), + args=(array, 1.), kwargs={}) assert_equal(result, 'original') - result = array.__array_function__( - func=func, types=(np.ndarray, SubOverride), args=(), kwargs={}) + result = array.__array_function__(func=dispatched_two_arg, + types=(np.ndarray, Other), + args=(array, other), kwargs={}) assert_(result is NotImplemented) - result = array.__array_function__( - func=func, types=(np.ndarray, NoOverrideSub), args=(), kwargs={}) + result = array.__array_function__(func=dispatched_two_arg, + types=(np.ndarray, NoOverrideSub), + args=(array, no_override_sub), + kwargs={}) assert_equal(result, 'original') + result = array.__array_function__(func=dispatched_two_arg, + types=(np.ndarray, OverrideSub), + args=(array, override_sub), + kwargs={}) + assert_equal(result, 'original') -# need to define this at the top level to test pickling -@array_function_dispatch(lambda array: (array,)) -def dispatched_one_arg(array): - """Docstring.""" - return 'original' + with assert_raises_regex(TypeError, 'no implementation found'): + np.concatenate((array, other)) + + expected = np.concatenate((array, array)) + result = np.concatenate((array, no_override_sub)) + assert_equal(result, expected.view(NoOverrideSub)) + result = np.concatenate((array, override_sub)) + assert_equal(result, expected.view(OverrideSub)) + + def test_no_wrapper(self): + array = np.array(1) + func = dispatched_one_arg.__wrapped__ + with assert_raises_regex(AttributeError, '__wrapped__'): + array.__array_function__(func=func, + types=(np.ndarray,), + args=(array,), kwargs={}) +@requires_array_function class TestArrayFunctionDispatch(object): def test_pickle(self): @@ -189,7 +231,8 @@ class TestArrayFunctionDispatch(object): assert_(obj is original) assert_(func is dispatched_one_arg) assert_equal(set(types), {MyArray}) - assert_equal(args, (original,)) + # assert_equal uses the overloaded np.iscomplexobj() internally + assert_(args == (original,)) assert_equal(kwargs, {}) def test_not_implemented(self): @@ -203,6 +246,7 @@ class TestArrayFunctionDispatch(object): dispatched_one_arg(array) +@requires_array_function class TestVerifyMatchingSignatures(object): def test_verify_matching_signatures(self): @@ -255,6 +299,7 @@ def _new_duck_type_and_implements(): return (MyArray, implements) +@requires_array_function class TestArrayFunctionImplementation(object): def test_one_arg(self): @@ -295,12 +340,53 @@ class TestArrayFunctionImplementation(object): def test_not_implemented(self): MyArray, implements = _new_duck_type_and_implements() - @array_function_dispatch(lambda array: (array,)) + @array_function_dispatch(lambda array: (array,), module='my') def func(array): return array array = np.array(1) assert_(func(array) is array) + assert_equal(func.__module__, 'my') - with assert_raises_regex(TypeError, 'no implementation found'): + with assert_raises_regex( + TypeError, "no implementation found for 'my.func'"): func(MyArray()) + + +class TestNDArrayMethods(object): + + def test_repr(self): + # gh-12162: should still be defined even if __array_function__ doesn't + # implement np.array_repr() + + class MyArray(np.ndarray): + def __array_function__(*args, **kwargs): + return NotImplemented + + array = np.array(1).view(MyArray) + assert_equal(repr(array), 'MyArray(1)') + assert_equal(str(array), '1') + + +class TestNumPyFunctions(object): + + def test_set_module(self): + assert_equal(np.sum.__module__, 'numpy') + assert_equal(np.char.equal.__module__, 'numpy.char') + assert_equal(np.fft.fft.__module__, 'numpy.fft') + assert_equal(np.linalg.solve.__module__, 'numpy.linalg') + + @pytest.mark.skipif(sys.version_info[0] < 3, reason="Python 3 only") + def test_inspect_sum(self): + signature = inspect.signature(np.sum) + assert_('axis' in signature.parameters) + + @requires_array_function + def test_override_sum(self): + MyArray, implements = _new_duck_type_and_implements() + + @implements(np.sum) + def _(array): + return 'yes' + + assert_equal(np.sum(MyArray()), 'yes') diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py index a77eef404..c059ef510 100644 --- a/numpy/core/tests/test_records.py +++ b/numpy/core/tests/test_records.py @@ -7,15 +7,15 @@ try: import collections.abc as collections_abc except ImportError: import collections as collections_abc -import warnings import textwrap from os import path import pytest import numpy as np +from numpy.compat import Path from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_array_almost_equal, - assert_raises, assert_warns + assert_raises, temppath ) from numpy.core.numeric import pickle @@ -325,6 +325,23 @@ class TestFromrecords(object): assert_equal(rec['f1'], [b'', b'', b'']) +@pytest.mark.skipif(Path is None, reason="No pathlib.Path") +class TestPathUsage(object): + # Test that pathlib.Path can be used + def test_tofile_fromfile(self): + with temppath(suffix='.bin') as path: + path = Path(path) + np.random.seed(123) + a = np.random.rand(10).astype('f8,i4,a5') + a[5] = (0.5,10,'abcde') + with path.open("wb") as fd: + a.tofile(fd) + x = np.core.records.fromfile(path, + formats='f8,i4,a5', + shape=10) + assert_array_equal(x, a) + + class TestRecord(object): def setup(self): self.data = np.rec.fromrecords([(1, 2, 3), (4, 5, 6)], @@ -361,7 +378,6 @@ class TestRecord(object): with assert_raises(ValueError): r.setfield([2,3], *r.dtype.fields['f']) - @pytest.mark.xfail(reason="See gh-10411, becomes real error in 1.16") def test_out_of_order_fields(self): # names in the same order, padding added to descr x = self.data[['col1', 'col2']] diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index d53f6da84..2421a1161 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -2,9 +2,7 @@ from __future__ import division, absolute_import, print_function import copy import sys -import platform import gc -import warnings import tempfile import pytest from os import path @@ -1557,10 +1555,7 @@ class TestRegression(object): def test_complex_nan_maximum(self): cnan = complex(0, np.nan) - with suppress_warnings() as sup: - sup.record(RuntimeWarning) - assert_equal(np.maximum(1, cnan), cnan) - assert_equal(len(sup.log), 1) + assert_equal(np.maximum(1, cnan), cnan) def test_subclass_int_tuple_assignment(self): # ticket #1563 @@ -2412,7 +2407,7 @@ class TestRegression(object): t = np.dtype([((s, 'f1'), np.float64)]) data = np.zeros(10, t) for i in range(10): - v = str(data[['f1']]) + str(data[['f1']]) if HAS_REFCOUNT: assert_(base <= sys.getrefcount(s)) diff --git a/numpy/core/tests/test_scalarinherit.py b/numpy/core/tests/test_scalarinherit.py index 28436f6c7..9e32cf624 100644 --- a/numpy/core/tests/test_scalarinherit.py +++ b/numpy/core/tests/test_scalarinherit.py @@ -69,6 +69,7 @@ class TestCharacter(object): np_s = np.string_('abc') np_u = np.unicode_('abc') np_i = np.int(5) - res_np = np_s * np_i res_s = b'abc' * 5 - assert_(res_np == res_s) + res_u = u'abc' * 5 + assert_(np_s * np_i == res_s) + assert_(np_u * np_i == res_u) diff --git a/numpy/core/tests/test_scalarmath.py b/numpy/core/tests/test_scalarmath.py index a55f06b69..51bcf2b8d 100644 --- a/numpy/core/tests/test_scalarmath.py +++ b/numpy/core/tests/test_scalarmath.py @@ -9,7 +9,7 @@ import pytest import numpy as np from numpy.testing import ( - assert_, assert_equal, assert_raises, assert_almost_equal, assert_allclose, + assert_, assert_equal, assert_raises, assert_almost_equal, assert_array_equal, IS_PYPY, suppress_warnings, _gen_alignment_data, assert_warns ) @@ -184,7 +184,7 @@ class TestPower(object): a = 5 b = 4 c = 10 - expected = pow(a, b, c) + expected = pow(a, b, c) # noqa: F841 for t in (np.int32, np.float32, np.complex64): # note that 3-operand power only dispatches on the first argument assert_raises(TypeError, operator.pow, t(a), b, c) @@ -565,10 +565,10 @@ class TestMultiply(object): # Some of this behaviour may be controversial and could be open for # change. accepted_types = set(np.typecodes["AllInteger"]) - deprecated_types = set('?') + deprecated_types = {'?'} forbidden_types = ( set(np.typecodes["All"]) - accepted_types - deprecated_types) - forbidden_types -= set('V') # can't default-construct void scalars + forbidden_types -= {'V'} # can't default-construct void scalars for seq_type in (list, tuple): seq = seq_type([1, 2, 3]) diff --git a/numpy/core/tests/test_scalarprint.py b/numpy/core/tests/test_scalarprint.py index 472ff691d..cde1355aa 100644 --- a/numpy/core/tests/test_scalarprint.py +++ b/numpy/core/tests/test_scalarprint.py @@ -10,7 +10,7 @@ import pytest from tempfile import TemporaryFile import numpy as np -from numpy.testing import assert_, assert_equal, suppress_warnings, dec +from numpy.testing import assert_, assert_equal, suppress_warnings class TestRealScalars(object): def test_str(self): diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py index df819b73f..b996321c2 100644 --- a/numpy/core/tests/test_shape_base.py +++ b/numpy/core/tests/test_shape_base.py @@ -1,14 +1,17 @@ from __future__ import division, absolute_import, print_function -import warnings +import pytest +import sys import numpy as np from numpy.core import ( array, arange, atleast_1d, atleast_2d, atleast_3d, block, vstack, hstack, newaxis, concatenate, stack ) +from numpy.core.shape_base import (_block_dispatcher, _block_setup, + _block_concatenate, _block_slicing) from numpy.testing import ( assert_, assert_raises, assert_array_equal, assert_equal, - assert_raises_regex, assert_almost_equal + assert_raises_regex, assert_warns ) from numpy.compat import long @@ -153,6 +156,14 @@ class TestHstack(object): desired = array([[1, 1], [2, 2]]) assert_array_equal(res, desired) + def test_generator(self): + with assert_warns(FutureWarning): + hstack((np.arange(3) for _ in range(2))) + if sys.version_info.major > 2: + # map returns a list on Python 2 + with assert_warns(FutureWarning): + hstack(map(lambda x: x, np.ones((3, 2)))) + class TestVstack(object): def test_non_iterable(self): @@ -189,6 +200,10 @@ class TestVstack(object): desired = array([[1, 2], [1, 2]]) assert_array_equal(res, desired) + def test_generator(self): + with assert_warns(FutureWarning): + vstack((np.arange(3) for _ in range(2))) + class TestConcatenate(object): def test_returns_copy(self): @@ -352,12 +367,16 @@ def test_stack(): arrays = [np.random.randn(3, 4) for _ in range(10)] axes = [0, 1, 2, -1, -2, -3] expected_shapes = [(10, 3, 4), (3, 10, 4), (3, 4, 10), - (3, 4, 10), (3, 10, 4), (10, 3, 4)] + (3, 4, 10), (3, 10, 4), (10, 3, 4)] for axis, expected_shape in zip(axes, expected_shapes): assert_equal(np.stack(arrays, axis).shape, expected_shape) # empty arrays assert_(stack([[], [], []]).shape == (3, 0)) assert_(stack([[], [], []], axis=1).shape == (0, 3)) + # out + out = np.zeros_like(r1) + np.stack((a, b), out=out) + assert_array_equal(out, r1) # edge cases assert_raises_regex(ValueError, 'need at least one array', stack, []) assert_raises_regex(ValueError, 'must have the same shape', @@ -370,16 +389,62 @@ def test_stack(): stack, [np.zeros((3, 3)), np.zeros(3)], axis=1) assert_raises_regex(ValueError, 'must have the same shape', stack, [np.arange(2), np.arange(3)]) + # generator is deprecated + with assert_warns(FutureWarning): + result = stack((x for x in range(3))) + assert_array_equal(result, np.array([0, 1, 2])) class TestBlock(object): - def test_returns_copy(self): + @pytest.fixture(params=['block', 'force_concatenate', 'force_slicing']) + def block(self, request): + # blocking small arrays and large arrays go through different paths. + # the algorithm is triggered depending on the number of element + # copies required. + # We define a test fixture that forces most tests to go through + # both code paths. + # Ultimately, this should be removed if a single algorithm is found + # to be faster for both small and large arrays. + def _block_force_concatenate(arrays): + arrays, list_ndim, result_ndim, _ = _block_setup(arrays) + return _block_concatenate(arrays, list_ndim, result_ndim) + + def _block_force_slicing(arrays): + arrays, list_ndim, result_ndim, _ = _block_setup(arrays) + return _block_slicing(arrays, list_ndim, result_ndim) + + if request.param == 'force_concatenate': + return _block_force_concatenate + elif request.param == 'force_slicing': + return _block_force_slicing + elif request.param == 'block': + return block + else: + raise ValueError('Unknown blocking request. There is a typo in the tests.') + + def test_returns_copy(self, block): a = np.eye(3) - b = np.block(a) + b = block(a) b[0, 0] = 2 assert b[0, 0] != a[0, 0] - def test_block_simple_row_wise(self): + def test_block_total_size_estimate(self, block): + _, _, _, total_size = _block_setup([1]) + assert total_size == 1 + + _, _, _, total_size = _block_setup([[1]]) + assert total_size == 1 + + _, _, _, total_size = _block_setup([[1, 1]]) + assert total_size == 2 + + _, _, _, total_size = _block_setup([[1], [1]]) + assert total_size == 2 + + _, _, _, total_size = _block_setup([[1, 2], [3, 4]]) + assert total_size == 4 + + def test_block_simple_row_wise(self, block): a_2d = np.ones((2, 2)) b_2d = 2 * a_2d desired = np.array([[1, 1, 2, 2], @@ -387,7 +452,7 @@ class TestBlock(object): result = block([a_2d, b_2d]) assert_equal(desired, result) - def test_block_simple_column_wise(self): + def test_block_simple_column_wise(self, block): a_2d = np.ones((2, 2)) b_2d = 2 * a_2d expected = np.array([[1, 1], @@ -397,7 +462,7 @@ class TestBlock(object): result = block([[a_2d], [b_2d]]) assert_equal(expected, result) - def test_block_with_1d_arrays_row_wise(self): + def test_block_with_1d_arrays_row_wise(self, block): # # # 1-D vectors are treated as row arrays a = np.array([1, 2, 3]) b = np.array([2, 3, 4]) @@ -405,7 +470,7 @@ class TestBlock(object): result = block([a, b]) assert_equal(expected, result) - def test_block_with_1d_arrays_multiple_rows(self): + def test_block_with_1d_arrays_multiple_rows(self, block): a = np.array([1, 2, 3]) b = np.array([2, 3, 4]) expected = np.array([[1, 2, 3, 2, 3, 4], @@ -413,7 +478,7 @@ class TestBlock(object): result = block([[a, b], [a, b]]) assert_equal(expected, result) - def test_block_with_1d_arrays_column_wise(self): + def test_block_with_1d_arrays_column_wise(self, block): # # # 1-D vectors are treated as row arrays a_1d = np.array([1, 2, 3]) b_1d = np.array([2, 3, 4]) @@ -422,7 +487,7 @@ class TestBlock(object): result = block([[a_1d], [b_1d]]) assert_equal(expected, result) - def test_block_mixed_1d_and_2d(self): + def test_block_mixed_1d_and_2d(self, block): a_2d = np.ones((2, 2)) b_1d = np.array([2, 2]) result = block([[a_2d], [b_1d]]) @@ -431,7 +496,7 @@ class TestBlock(object): [2, 2]]) assert_equal(expected, result) - def test_block_complicated(self): + def test_block_complicated(self, block): # a bit more complicated one_2d = np.array([[1, 1, 1]]) two_2d = np.array([[2, 2, 2]]) @@ -455,7 +520,7 @@ class TestBlock(object): [zero_2d]]) assert_equal(result, expected) - def test_nested(self): + def test_nested(self, block): one = np.array([1, 1, 1]) two = np.array([[2, 2, 2], [2, 2, 2], [2, 2, 2]]) three = np.array([3, 3, 3]) @@ -464,9 +529,9 @@ class TestBlock(object): six = np.array([6, 6, 6, 6, 6]) zero = np.zeros((2, 6)) - result = np.block([ + result = block([ [ - np.block([ + block([ [one], [three], [four] @@ -485,7 +550,7 @@ class TestBlock(object): assert_equal(result, expected) - def test_3d(self): + def test_3d(self, block): a000 = np.ones((2, 2, 2), int) * 1 a100 = np.ones((3, 2, 2), int) * 2 @@ -498,7 +563,7 @@ class TestBlock(object): a111 = np.ones((3, 3, 3), int) * 8 - result = np.block([ + result = block([ [ [a000, a001], [a010, a011], @@ -540,55 +605,102 @@ class TestBlock(object): assert_array_equal(result, expected) - def test_block_with_mismatched_shape(self): + def test_block_with_mismatched_shape(self, block): a = np.array([0, 0]) b = np.eye(2) - assert_raises(ValueError, np.block, [a, b]) - assert_raises(ValueError, np.block, [b, a]) + assert_raises(ValueError, block, [a, b]) + assert_raises(ValueError, block, [b, a]) - def test_no_lists(self): - assert_equal(np.block(1), np.array(1)) - assert_equal(np.block(np.eye(3)), np.eye(3)) + to_block = [[np.ones((2,3)), np.ones((2,2))], + [np.ones((2,2)), np.ones((2,2))]] + assert_raises(ValueError, block, to_block) + def test_no_lists(self, block): + assert_equal(block(1), np.array(1)) + assert_equal(block(np.eye(3)), np.eye(3)) - def test_invalid_nesting(self): + def test_invalid_nesting(self, block): msg = 'depths are mismatched' - assert_raises_regex(ValueError, msg, np.block, [1, [2]]) - assert_raises_regex(ValueError, msg, np.block, [1, []]) - assert_raises_regex(ValueError, msg, np.block, [[1], 2]) - assert_raises_regex(ValueError, msg, np.block, [[], 2]) - assert_raises_regex(ValueError, msg, np.block, [ + assert_raises_regex(ValueError, msg, block, [1, [2]]) + assert_raises_regex(ValueError, msg, block, [1, []]) + assert_raises_regex(ValueError, msg, block, [[1], 2]) + assert_raises_regex(ValueError, msg, block, [[], 2]) + assert_raises_regex(ValueError, msg, block, [ [[1], [2]], [[3, 4]], [5] # missing brackets ]) - def test_empty_lists(self): - assert_raises_regex(ValueError, 'empty', np.block, []) - assert_raises_regex(ValueError, 'empty', np.block, [[]]) - assert_raises_regex(ValueError, 'empty', np.block, [[1], []]) + def test_empty_lists(self, block): + assert_raises_regex(ValueError, 'empty', block, []) + assert_raises_regex(ValueError, 'empty', block, [[]]) + assert_raises_regex(ValueError, 'empty', block, [[1], []]) - def test_tuple(self): - assert_raises_regex(TypeError, 'tuple', np.block, ([1, 2], [3, 4])) - assert_raises_regex(TypeError, 'tuple', np.block, [(1, 2), (3, 4)]) + def test_tuple(self, block): + assert_raises_regex(TypeError, 'tuple', block, ([1, 2], [3, 4])) + assert_raises_regex(TypeError, 'tuple', block, [(1, 2), (3, 4)]) - def test_different_ndims(self): + def test_different_ndims(self, block): a = 1. b = 2 * np.ones((1, 2)) c = 3 * np.ones((1, 1, 3)) - result = np.block([a, b, c]) + result = block([a, b, c]) expected = np.array([[[1., 2., 2., 3., 3., 3.]]]) assert_equal(result, expected) - def test_different_ndims_depths(self): + def test_different_ndims_depths(self, block): a = 1. b = 2 * np.ones((1, 2)) c = 3 * np.ones((1, 2, 3)) - result = np.block([[a, b], [c]]) + result = block([[a, b], [c]]) expected = np.array([[[1., 2., 2.], [3., 3., 3.], [3., 3., 3.]]]) assert_equal(result, expected) + + def test_block_memory_order(self, block): + # 3D + arr_c = np.zeros((3,)*3, order='C') + arr_f = np.zeros((3,)*3, order='F') + + b_c = [[[arr_c, arr_c], + [arr_c, arr_c]], + [[arr_c, arr_c], + [arr_c, arr_c]]] + + b_f = [[[arr_f, arr_f], + [arr_f, arr_f]], + [[arr_f, arr_f], + [arr_f, arr_f]]] + + assert block(b_c).flags['C_CONTIGUOUS'] + assert block(b_f).flags['F_CONTIGUOUS'] + + arr_c = np.zeros((3, 3), order='C') + arr_f = np.zeros((3, 3), order='F') + # 2D + b_c = [[arr_c, arr_c], + [arr_c, arr_c]] + + b_f = [[arr_f, arr_f], + [arr_f, arr_f]] + + assert block(b_c).flags['C_CONTIGUOUS'] + assert block(b_f).flags['F_CONTIGUOUS'] + + +def test_block_dispatcher(): + class ArrayLike(object): + pass + a = ArrayLike() + b = ArrayLike() + c = ArrayLike() + assert_equal(list(_block_dispatcher(a)), [a]) + assert_equal(list(_block_dispatcher([a])), [a]) + assert_equal(list(_block_dispatcher([a, b])), [a, b]) + assert_equal(list(_block_dispatcher([[a], [b, [c]]])), [a, b, c]) + # don't recurse into non-lists + assert_equal(list(_block_dispatcher((a, b))), [(a, b)]) diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index b83b8ccff..b52152243 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -3,6 +3,8 @@ from __future__ import division, absolute_import, print_function import warnings import itertools +import pytest + import numpy as np import numpy.core._umath_tests as umt import numpy.linalg._umath_linalg as uml @@ -596,6 +598,12 @@ class TestUfunc(object): assert_equal(np.sum(np.ones((2, 3, 5), dtype=np.int64), axis=(0, 2), initial=2), [12, 12, 12]) + def test_sum_where(self): + # More extensive tests done in test_reduction_with_where. + assert_equal(np.sum([[1., 2.], [3., 4.]], where=[True, False]), 4.) + assert_equal(np.sum([[1., 2.], [3., 4.]], axis=0, initial=5., + where=[True, False]), [9., 5.]) + def test_inner1d(self): a = np.arange(6).reshape((2, 3)) assert_array_equal(umt.inner1d(a, a), np.sum(a*a, axis=-1)) @@ -1162,6 +1170,8 @@ class TestUfunc(object): assert_equal(np.array([[1]], dtype=object).sum(), 1) assert_equal(np.array([[[1, 2]]], dtype=object).sum((0, 1)), [1, 2]) assert_equal(np.array([1], dtype=object).sum(initial=1), 2) + assert_equal(np.array([[1], [2, 3]], dtype=object) + .sum(initial=[0], where=[False, True]), [0, 2, 3]) def test_object_array_accumulate_inplace(self): # Checks that in-place accumulates work, see also gh-7402 @@ -1396,6 +1406,44 @@ class TestUfunc(object): res = np.add.reduce(a, initial=5) assert_equal(res, 15) + @pytest.mark.parametrize('axis', (0, 1, None)) + @pytest.mark.parametrize('where', (np.array([False, True, True]), + np.array([[True], [False], [True]]), + np.array([[True, False, False], + [False, True, False], + [False, True, True]]))) + def test_reduction_with_where(self, axis, where): + a = np.arange(9.).reshape(3, 3) + a_copy = a.copy() + a_check = np.zeros_like(a) + np.positive(a, out=a_check, where=where) + + res = np.add.reduce(a, axis=axis, where=where) + check = a_check.sum(axis) + assert_equal(res, check) + # Check we do not overwrite elements of a internally. + assert_array_equal(a, a_copy) + + @pytest.mark.parametrize(('axis', 'where'), + ((0, np.array([True, False, True])), + (1, [True, True, False]), + (None, True))) + @pytest.mark.parametrize('initial', (-np.inf, 5.)) + def test_reduction_with_where_and_initial(self, axis, where, initial): + a = np.arange(9.).reshape(3, 3) + a_copy = a.copy() + a_check = np.full(a.shape, -np.inf) + np.positive(a, out=a_check, where=where) + + res = np.maximum.reduce(a, axis=axis, where=where, initial=initial) + check = a_check.max(axis, initial=initial) + assert_equal(res, check) + + def test_reduction_where_initial_needed(self): + a = np.arange(9.).reshape(3, 3) + m = [False, True, False] + assert_raises(ValueError, np.maximum.reduce, a, where=m) + def test_identityless_reduction_nonreorderable(self): a = np.array([[8.0, 2.0, 2.0], [1.0, 0.5, 0.25]]) @@ -1749,16 +1797,19 @@ class TestUfunc(object): assert_equal(f(d, 0, None, None, True), r.reshape((1,) + r.shape)) assert_equal(f(d, 0, None, None, False, 0), r) assert_equal(f(d, 0, None, None, False, initial=0), r) + assert_equal(f(d, 0, None, None, False, 0, True), r) + assert_equal(f(d, 0, None, None, False, 0, where=True), r) # multiple keywords assert_equal(f(d, axis=0, dtype=None, out=None, keepdims=False), r) assert_equal(f(d, 0, dtype=None, out=None, keepdims=False), r) assert_equal(f(d, 0, None, out=None, keepdims=False), r) - assert_equal(f(d, 0, None, out=None, keepdims=False, initial=0), r) + assert_equal(f(d, 0, None, out=None, keepdims=False, initial=0, + where=True), r) # too little assert_raises(TypeError, f) # too much - assert_raises(TypeError, f, d, 0, None, None, False, 0, 1) + assert_raises(TypeError, f, d, 0, None, None, False, 0, True, 1) # invalid axis assert_raises(TypeError, f, d, "invalid") assert_raises(TypeError, f, d, axis="invalid") diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index c15ce83f6..21097244f 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1,6 +1,5 @@ from __future__ import division, absolute_import, print_function -import sys import platform import warnings import fnmatch @@ -14,7 +13,7 @@ from numpy.testing import ( assert_, assert_equal, assert_raises, assert_raises_regex, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_allclose, assert_no_warnings, suppress_warnings, - _gen_alignment_data, assert_warns + _gen_alignment_data ) @@ -685,6 +684,10 @@ class TestLogAddExp(_FilterInvalids): assert_(np.isnan(np.logaddexp(0, np.nan))) assert_(np.isnan(np.logaddexp(np.nan, np.nan))) + def test_reduce(self): + assert_equal(np.logaddexp.identity, -np.inf) + assert_equal(np.logaddexp.reduce([]), -np.inf) + class TestLog1p(object): def test_log1p(self): @@ -1294,6 +1297,7 @@ class TestSign(object): # In reference to github issue #6229 def test_nan(): foo = np.array([np.nan]) + # FIXME: a not used a = np.sign(foo.astype(object)) assert_raises(TypeError, test_nan) @@ -1327,21 +1331,18 @@ class TestMinMax(object): assert_equal(d.max(), d[0]) assert_equal(d.min(), d[0]) - def test_reduce_warns(self): + def test_reduce_reorder(self): # gh 10370, 11029 Some compilers reorder the call to npy_getfloatstatus # and put it before the call to an intrisic function that causes - # invalid status to be set. Also make sure warnings are emitted + # invalid status to be set. Also make sure warnings are not emitted for n in (2, 4, 8, 16, 32): for dt in (np.float32, np.float16, np.complex64): - with suppress_warnings() as sup: - sup.record(RuntimeWarning) - for r in np.diagflat(np.array([np.nan] * n, dtype=dt)): - assert_equal(np.min(r), np.nan) - assert_equal(len(sup.log), n) + for r in np.diagflat(np.array([np.nan] * n, dtype=dt)): + assert_equal(np.min(r), np.nan) - def test_minimize_warns(self): - # gh 11589 - assert_warns(RuntimeWarning, np.minimum, np.nan, 1) + def test_minimize_no_warns(self): + a = np.minimum(np.nan, 1) + assert_equal(a, np.nan) class TestAbsoluteNegative(object): @@ -1893,7 +1894,8 @@ class TestSpecialMethods(object): # reduce, kwargs res = np.multiply.reduce(a, axis='axis0', dtype='dtype0', out='out0', - keepdims='keep0', initial='init0') + keepdims='keep0', initial='init0', + where='where0') assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], 'reduce') @@ -1902,7 +1904,8 @@ class TestSpecialMethods(object): 'out': ('out0',), 'keepdims': 'keep0', 'axis': 'axis0', - 'initial': 'init0'}) + 'initial': 'init0', + 'where': 'where0'}) # reduce, output equal to None removed, but not other explicit ones, # even if they are at their default value. @@ -1912,14 +1915,18 @@ class TestSpecialMethods(object): assert_equal(res[4], {'axis': 0, 'keepdims': True}) res = np.multiply.reduce(a, None, out=(None,), dtype=None) assert_equal(res[4], {'axis': None, 'dtype': None}) - res = np.multiply.reduce(a, 0, None, None, False, 2) - assert_equal(res[4], {'axis': 0, 'dtype': None, 'keepdims': False, 'initial': 2}) - # np._NoValue ignored for initial. - res = np.multiply.reduce(a, 0, None, None, False, np._NoValue) - assert_equal(res[4], {'axis': 0, 'dtype': None, 'keepdims': False}) - # None kept for initial. - res = np.multiply.reduce(a, 0, None, None, False, None) - assert_equal(res[4], {'axis': 0, 'dtype': None, 'keepdims': False, 'initial': None}) + res = np.multiply.reduce(a, 0, None, None, False, 2, True) + assert_equal(res[4], {'axis': 0, 'dtype': None, 'keepdims': False, + 'initial': 2, 'where': True}) + # np._NoValue ignored for initial + res = np.multiply.reduce(a, 0, None, None, False, + np._NoValue, True) + assert_equal(res[4], {'axis': 0, 'dtype': None, 'keepdims': False, + 'where': True}) + # None kept for initial, True for where. + res = np.multiply.reduce(a, 0, None, None, False, None, True) + assert_equal(res[4], {'axis': 0, 'dtype': None, 'keepdims': False, + 'initial': None, 'where': True}) # reduce, wrong args assert_raises(ValueError, np.multiply.reduce, a, out=()) @@ -2447,11 +2454,6 @@ class TestRationalFunctions(object): assert_equal(np.gcd(2**100, 3**100), 1) -def is_longdouble_finfo_bogus(): - info = np.finfo(np.longcomplex) - return not np.isfinite(np.log10(info.tiny/info.eps)) - - class TestComplexFunctions(object): funcs = [np.arcsin, np.arccos, np.arctan, np.arcsinh, np.arccosh, np.arctanh, np.sin, np.cos, np.tan, np.exp, @@ -2547,7 +2549,8 @@ class TestComplexFunctions(object): b = cfunc(p) assert_(abs(a - b) < atol, "%s %s: %s; cmath: %s" % (fname, p, a, b)) - def check_loss_of_precision(self, dtype): + @pytest.mark.parametrize('dtype', [np.complex64, np.complex_, np.longcomplex]) + def test_loss_of_precision(self, dtype): """Check loss of precision in complex arc* functions""" # Check against known-good functions @@ -2589,10 +2592,11 @@ class TestComplexFunctions(object): # It's not guaranteed that the system-provided arc functions # are accurate down to a few epsilons. (Eg. on Linux 64-bit) # So, give more leeway for long complex tests here: - check(x_series, 50*eps) + # Can use 2.1 for > Ubuntu LTS Trusty (2014), glibc = 2.19. + check(x_series, 50.0*eps) else: check(x_series, 2.1*eps) - check(x_basic, 2*eps/1e-3) + check(x_basic, 2.0*eps/1e-3) # Check a few points @@ -2632,15 +2636,6 @@ class TestComplexFunctions(object): check(func, pts, 1j) check(func, pts, 1+1j) - def test_loss_of_precision(self): - for dtype in [np.complex64, np.complex_]: - self.check_loss_of_precision(dtype) - - @pytest.mark.skipif(is_longdouble_finfo_bogus(), - reason="Bogus long double finfo") - def test_loss_of_precision_longcomplex(self): - self.check_loss_of_precision(np.longcomplex) - class TestAttributes(object): def test_attributes(self): diff --git a/numpy/ctypeslib.py b/numpy/ctypeslib.py index 24cfc6762..2e9781286 100644 --- a/numpy/ctypeslib.py +++ b/numpy/ctypeslib.py @@ -55,7 +55,9 @@ __all__ = ['load_library', 'ndpointer', 'test', 'ctypes_load_library', 'c_intp', 'as_ctypes', 'as_array'] import os -from numpy import integer, ndarray, dtype as _dtype, deprecate, array +from numpy import ( + integer, ndarray, dtype as _dtype, deprecate, array, frombuffer +) from numpy.core.multiarray import _flagdict, flagsobj try: @@ -91,7 +93,7 @@ else: def load_library(libname, loader_path): """ It is possible to load a library using - >>> lib = ctypes.cdll[<full_path_name>] + >>> lib = ctypes.cdll[<full_path_name>] # doctest: +SKIP But there are cross-platform considerations, such as library file extensions, plus the fact Windows will just load the first library it finds with that name. @@ -175,24 +177,6 @@ def _flags_fromnum(num): class _ndptr(_ndptr_base): - - def _check_retval_(self): - """This method is called when this class is used as the .restype - attribute for a shared-library function. It constructs a numpy - array from a void pointer.""" - return array(self) - - @property - def __array_interface__(self): - return {'descr': self._dtype_.descr, - '__ref': self, - 'strides': None, - 'shape': self._shape_, - 'version': 3, - 'typestr': self._dtype_.descr[0][1], - 'data': (self.value, False), - } - @classmethod def from_param(cls, obj): if not isinstance(obj, ndarray): @@ -213,6 +197,34 @@ class _ndptr(_ndptr_base): return obj.ctypes +class _concrete_ndptr(_ndptr): + """ + Like _ndptr, but with `_shape_` and `_dtype_` specified. + + Notably, this means the pointer has enough information to reconstruct + the array, which is not generally true. + """ + def _check_retval_(self): + """ + This method is called when this class is used as the .restype + attribute for a shared-library function, to automatically wrap the + pointer into an array. + """ + return self.contents + + @property + def contents(self): + """ + Get an ndarray viewing the data pointed to by this pointer. + + This mirrors the `contents` attribute of a normal ctypes pointer + """ + full_dtype = _dtype((self._dtype_, self._shape_)) + full_ctype = ctypes.c_char * full_dtype.itemsize + buffer = ctypes.cast(self, ctypes.POINTER(full_ctype)).contents + return frombuffer(buffer, dtype=full_dtype).squeeze(axis=0) + + # Factory for an array-checking class with from_param defined for # use with ctypes argtypes mechanism _pointer_type_cache = {} @@ -269,8 +281,11 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): """ + # normalize dtype to an Optional[dtype] if dtype is not None: dtype = _dtype(dtype) + + # normalize flags to an Optional[int] num = None if flags is not None: if isinstance(flags, str): @@ -287,10 +302,23 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): except Exception: raise TypeError("invalid flags specification") num = _num_fromflags(flags) + + # normalize shape to an Optional[tuple] + if shape is not None: + try: + shape = tuple(shape) + except TypeError: + # single integer -> 1-tuple + shape = (shape,) + + cache_key = (dtype, ndim, shape, num) + try: - return _pointer_type_cache[(dtype, ndim, shape, num)] + return _pointer_type_cache[cache_key] except KeyError: pass + + # produce a name for the new type if dtype is None: name = 'any' elif dtype.names: @@ -300,23 +328,21 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): if ndim is not None: name += "_%dd" % ndim if shape is not None: - try: - strshape = [str(x) for x in shape] - except TypeError: - strshape = [str(shape)] - shape = (shape,) - shape = tuple(shape) - name += "_"+"x".join(strshape) + name += "_"+"x".join(str(x) for x in shape) if flags is not None: name += "_"+"_".join(flags) + + if dtype is not None and shape is not None: + base = _concrete_ndptr else: - flags = [] - klass = type("ndpointer_%s"%name, (_ndptr,), + base = _ndptr + + klass = type("ndpointer_%s"%name, (base,), {"_dtype_": dtype, "_shape_" : shape, "_ndim_" : ndim, "_flags_" : num}) - _pointer_type_cache[(dtype, shape, ndim, num)] = klass + _pointer_type_cache[cache_key] = klass return klass @@ -375,5 +401,5 @@ if ctypes is not None: raise TypeError("readonly arrays unsupported") tp = _ctype_ndarray(_typecodes[ai["typestr"]], ai["shape"]) result = tp.from_address(addr) - result.__keep = ai + result.__keep = obj return result diff --git a/numpy/distutils/__init__.py b/numpy/distutils/__init__.py index 8dd326920..55514750e 100644 --- a/numpy/distutils/__init__.py +++ b/numpy/distutils/__init__.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import sys - from .__version__ import version as __version__ # Must import local ccompiler ASAP in order to get # customized CCompiler.spawn effective. diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py index b03fb96b2..100d0d069 100644 --- a/numpy/distutils/ccompiler.py +++ b/numpy/distutils/ccompiler.py @@ -6,6 +6,7 @@ import sys import types import shlex import time +import subprocess from copy import copy from distutils import ccompiler from distutils.ccompiler import * @@ -16,9 +17,11 @@ from distutils.version import LooseVersion from numpy.distutils import log from numpy.distutils.compat import get_exception -from numpy.distutils.exec_command import exec_command +from numpy.distutils.exec_command import ( + filepath_from_subprocess_output, forward_bytes_to_stdout +) from numpy.distutils.misc_util import cyg2win32, is_sequence, mingw32, \ - quote_args, get_num_build_jobs, \ + get_num_build_jobs, \ _commandline_dep_string # globals for parallel build management @@ -136,20 +139,37 @@ def CCompiler_spawn(self, cmd, display=None): if is_sequence(display): display = ' '.join(list(display)) log.info(display) - s, o = exec_command(cmd) - if s: - if is_sequence(cmd): - cmd = ' '.join(list(cmd)) - try: - print(o) - except UnicodeError: - # When installing through pip, `o` can contain non-ascii chars - pass - if re.search('Too many open files', o): - msg = '\nTry rerunning setup command until build succeeds.' - else: - msg = '' - raise DistutilsExecError('Command "%s" failed with exit status %d%s' % (cmd, s, msg)) + try: + subprocess.check_output(cmd) + except subprocess.CalledProcessError as exc: + o = exc.output + s = exc.returncode + except OSError: + # OSError doesn't have the same hooks for the exception + # output, but exec_command() historically would use an + # empty string for EnvironmentError (base class for + # OSError) + o = b'' + # status previously used by exec_command() for parent + # of OSError + s = 127 + else: + # use a convenience return here so that any kind of + # caught exception will execute the default code after the + # try / except block, which handles various exceptions + return None + + if is_sequence(cmd): + cmd = ' '.join(list(cmd)) + + forward_bytes_to_stdout(o) + + if re.search(b'Too many open files', o): + msg = '\nTry rerunning setup command until build succeeds.' + else: + msg = '' + raise DistutilsExecError('Command "%s" failed with exit status %d%s' % + (cmd, s, msg)) replace_method(CCompiler, 'spawn', CCompiler_spawn) @@ -404,10 +424,8 @@ def _compiler_to_string(compiler): v = getattr(compiler, key) mx = max(mx, len(key)) props.append((key, repr(v))) - lines = [] - format = '%-' + repr(mx+1) + 's = %s' - for prop in props: - lines.append(format % prop) + fmt = '%-' + repr(mx+1) + 's = %s' + lines = [fmt % prop for prop in props] return '\n'.join(lines) def CCompiler_show_customization(self): @@ -620,7 +638,21 @@ def CCompiler_get_version(self, force=False, ok_status=[0]): version = m.group('version') return version - status, output = exec_command(version_cmd, use_tee=0) + try: + output = subprocess.check_output(version_cmd) + except subprocess.CalledProcessError as exc: + output = exc.output + status = exc.returncode + except OSError: + # match the historical returns for a parent + # exception class caught by exec_command() + status = 127 + output = b'' + else: + # output isn't actually a filepath but we do this + # for now to match previous distutils behavior + output = filepath_from_subprocess_output(output) + status = 0 version = None if status in ok_status: @@ -738,8 +770,13 @@ ccompiler.new_compiler = new_compiler _distutils_gen_lib_options = gen_lib_options def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries): - library_dirs = quote_args(library_dirs) - runtime_library_dirs = quote_args(runtime_library_dirs) + # the version of this function provided by CPython allows the following + # to return lists, which are unpacked automatically: + # - compiler.runtime_library_dir_option + # our version extends the behavior to: + # - compiler.library_dir_option + # - compiler.library_option + # - compiler.find_library_file r = _distutils_gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) lib_opts = [] @@ -759,11 +796,6 @@ for _cc in ['msvc9', 'msvc', '_msvc', 'bcpp', 'cygwinc', 'emxc', 'unixc']: if _m is not None: setattr(_m, 'gen_lib_options', gen_lib_options) -_distutils_gen_preprocess_options = gen_preprocess_options -def gen_preprocess_options (macros, include_dirs): - include_dirs = quote_args(include_dirs) - return _distutils_gen_preprocess_options(macros, include_dirs) -ccompiler.gen_preprocess_options = gen_preprocess_options ##Fix distutils.util.split_quoted: # NOTE: I removed this fix in revision 4481 (see ticket #619), but it appears diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index 18d36480a..ab9d585a5 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -265,10 +265,10 @@ class build_ext (old_build_ext): # we blindly assume that both packages need all of the libraries, # resulting in a larger wheel than is required. This should be fixed, # but it's so rare that I won't bother to handle it. - pkg_roots = set( + pkg_roots = { self.get_ext_fullname(ext.name).split('.')[0] for ext in self.extensions - ) + } for pkg_root in pkg_roots: shared_lib_dir = os.path.join(pkg_root, '.libs') if not self.inplace: diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index 9def37822..668bc23fe 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -204,7 +204,6 @@ class build_src(build_ext.build_ext): def _build_npy_pkg_config(self, info, gd): - import shutil template, install_dir, subst_dict = info template_dir = os.path.dirname(template) for k, v in gd.items(): @@ -239,7 +238,6 @@ class build_src(build_ext.build_ext): if not install_cmd.finalized == 1: install_cmd.finalize_options() build_npkg = False - gd = {} if self.inplace == 1: top_prefix = '.' build_npkg = True diff --git a/numpy/distutils/command/config.py b/numpy/distutils/command/config.py index 6b904d6ef..d9b1e8488 100644 --- a/numpy/distutils/command/config.py +++ b/numpy/distutils/command/config.py @@ -95,7 +95,7 @@ Original exception was: %s, and the Compiler class was %s try: ret = mth(*((self,)+args)) except (DistutilsExecError, CompileError): - msg = str(get_exception()) + str(get_exception()) self.compiler = save_compiler raise CompileError self.compiler = save_compiler diff --git a/numpy/distutils/conv_template.py b/numpy/distutils/conv_template.py index 4a8746236..b33e315b4 100644 --- a/numpy/distutils/conv_template.py +++ b/numpy/distutils/conv_template.py @@ -206,10 +206,8 @@ def parse_loop_header(loophead) : dlist = [] if nsub is None : raise ValueError("No substitution variables found") - for i in range(nsub) : - tmp = {} - for name, vals in names : - tmp[name] = vals[i] + for i in range(nsub): + tmp = {name: vals[i] for name, vals in names} dlist.append(tmp) return dlist diff --git a/numpy/distutils/exec_command.py b/numpy/distutils/exec_command.py index af7810d75..ede347b03 100644 --- a/numpy/distutils/exec_command.py +++ b/numpy/distutils/exec_command.py @@ -67,8 +67,10 @@ def filepath_from_subprocess_output(output): Inherited from `exec_command`, and possibly incorrect. """ - output = output.decode(locale.getpreferredencoding(False), - errors='replace') + mylocale = locale.getpreferredencoding(False) + if mylocale is None: + mylocale = 'ascii' + output = output.decode(mylocale, errors='replace') output = output.replace('\r\n', '\n') # Another historical oddity if output[-1:] == '\n': @@ -79,6 +81,29 @@ def filepath_from_subprocess_output(output): output = output.encode('ascii', errors='replace') return output + +def forward_bytes_to_stdout(val): + """ + Forward bytes from a subprocess call to the console, without attempting to + decode them. + + The assumption is that the subprocess call already returned bytes in + a suitable encoding. + """ + if sys.version_info.major < 3: + # python 2 has binary output anyway + sys.stdout.write(val) + elif hasattr(sys.stdout, 'buffer'): + # use the underlying binary output if there is one + sys.stdout.buffer.write(val) + elif hasattr(sys.stdout, 'encoding'): + # round-trip the encoding if necessary + sys.stdout.write(val.decode(sys.stdout.encoding)) + else: + # make a best-guess at the encoding + sys.stdout.write(val.decode('utf8', errors='replace')) + + def temp_file_name(): fo, name = make_temp_file() fo.close() @@ -146,9 +171,7 @@ def find_executable(exe, path=None, _cache={}): def _preserve_environment( names ): log.debug('_preserve_environment(%r)' % (names)) - env = {} - for name in names: - env[name] = os.environ.get(name) + env = {name: os.environ.get(name) for name in names} return env def _update_environment( **env ): @@ -278,9 +301,10 @@ def _exec_command(command, use_shell=None, use_tee = None, **env): return 127, '' text, err = proc.communicate() - text = text.decode(locale.getpreferredencoding(False), - errors='replace') - + mylocale = locale.getpreferredencoding(False) + if mylocale is None: + mylocale = 'ascii' + text = text.decode(mylocale, errors='replace') text = text.replace('\r\n', '\n') # Another historical oddity if text[-1:] == '\n': diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py index 3bd8057b4..12b32832e 100644 --- a/numpy/distutils/fcompiler/__init__.py +++ b/numpy/distutils/fcompiler/__init__.py @@ -22,6 +22,7 @@ import os import sys import re import types +import shlex from numpy.compat import open_latin1 @@ -465,8 +466,10 @@ class FCompiler(CCompiler): noarch = self.distutils_vars.get('noarch', noopt) debug = self.distutils_vars.get('debug', False) - f77 = self.command_vars.compiler_f77 - f90 = self.command_vars.compiler_f90 + f77 = shlex.split(self.command_vars.compiler_f77, + posix=(os.name == 'posix')) + f90 = shlex.split(self.command_vars.compiler_f90, + posix=(os.name == 'posix')) f77flags = [] f90flags = [] @@ -480,6 +483,14 @@ class FCompiler(CCompiler): freeflags = self.flag_vars.free # XXX Assuming that free format is default for f90 compiler. fix = self.command_vars.compiler_fix + # NOTE: this and similar examples are probably just + # exluding --coverage flag when F90 = gfortran --coverage + # instead of putting that flag somewhere more appropriate + # this and similar examples where a Fortran compiler + # environment variable has been customized by CI or a user + # should perhaps eventually be more throughly tested and more + # robustly handled + fix = shlex.split(fix, posix=(os.name == 'posix')) if fix: fixflags = self.flag_vars.fix + f90flags @@ -506,11 +517,11 @@ class FCompiler(CCompiler): fflags = self.flag_vars.flags + dflags + oflags + aflags if f77: - self.set_commands(compiler_f77=[f77]+f77flags+fflags) + self.set_commands(compiler_f77=f77+f77flags+fflags) if f90: - self.set_commands(compiler_f90=[f90]+freeflags+f90flags+fflags) + self.set_commands(compiler_f90=f90+freeflags+f90flags+fflags) if fix: - self.set_commands(compiler_fix=[fix]+fixflags+fflags) + self.set_commands(compiler_fix=fix+fixflags+fflags) #XXX: Do we need LDSHARED->SOSHARED, LDFLAGS->SOFLAGS diff --git a/numpy/distutils/fcompiler/environment.py b/numpy/distutils/fcompiler/environment.py index 489784580..4238f35cb 100644 --- a/numpy/distutils/fcompiler/environment.py +++ b/numpy/distutils/fcompiler/environment.py @@ -1,6 +1,7 @@ from __future__ import division, absolute_import, print_function import os +import warnings from distutils.dist import Distribution __metaclass__ = type @@ -54,8 +55,18 @@ class EnvironmentConfig(object): if envvar is not None: envvar_contents = os.environ.get(envvar) if envvar_contents is not None: - if var and append and os.environ.get('NPY_DISTUTILS_APPEND_FLAGS', '0') == '1': - var = var + [envvar_contents] + if var and append: + if os.environ.get('NPY_DISTUTILS_APPEND_FLAGS', '0') == '1': + var = var + [envvar_contents] + else: + var = envvar_contents + if 'NPY_DISTUTILS_APPEND_FLAGS' not in os.environ.keys(): + msg = "{} is used as is, not appended ".format(envvar) + \ + "to flags already defined " + \ + "by numpy.distutils! Use NPY_DISTUTILS_APPEND_FLAGS=1 " + \ + "to obtain appending behavior instead (this " + \ + "behavior will become default in a future release)." + warnings.warn(msg, UserWarning, stacklevel=3) else: var = envvar_contents if confvar is not None and self._conf: diff --git a/numpy/distutils/fcompiler/gnu.py b/numpy/distutils/fcompiler/gnu.py index f151809c7..81769e562 100644 --- a/numpy/distutils/fcompiler/gnu.py +++ b/numpy/distutils/fcompiler/gnu.py @@ -265,6 +265,10 @@ class GnuFCompiler(FCompiler): return [] def runtime_library_dir_option(self, dir): + if sys.platform[:3] == 'aix' or sys.platform == 'win32': + # Linux/Solaris/Unix support RPATH, Windows and AIX do not + raise NotImplementedError + sep = ',' if sys.platform == 'darwin' else '=' return '-Wl,-rpath%s"%s"' % (sep, dir) @@ -311,6 +315,12 @@ class Gnu95FCompiler(GnuFCompiler): module_dir_switch = '-J' module_include_switch = '-I' + if sys.platform[:3] == 'aix': + executables['linker_so'].append('-lpthread') + if platform.architecture()[0][:2] == '64': + for key in ['compiler_f77', 'compiler_f90','compiler_fix','linker_so', 'linker_exe']: + executables[key].append('-maix64') + g2c = 'gfortran' def _universal_flags(self, cmd): diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index eba0d9ba1..67a5f7234 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -1691,7 +1691,6 @@ class Configuration(object): """ if subst_dict is None: subst_dict = {} - basename = os.path.splitext(template)[0] template = os.path.join(self.package_path, template) if self.name in self.installed_pkg_config: @@ -2225,7 +2224,6 @@ def is_bootstrapping(): return True except AttributeError: return False - __NUMPY_SETUP__ = False ######################### diff --git a/numpy/distutils/npy_pkg_config.py b/numpy/distutils/npy_pkg_config.py index ea16e772d..bfe8b9f77 100644 --- a/numpy/distutils/npy_pkg_config.py +++ b/numpy/distutils/npy_pkg_config.py @@ -222,9 +222,7 @@ def parse_meta(config): if not config.has_section('meta'): raise FormatError("No meta section found !") - d = {} - for name, value in config.items('meta'): - d[name] = value + d = dict(config.items('meta')) for k in ['name', 'description', 'version']: if not k in d: diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 79adcc334..cd63cc849 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -1124,8 +1124,9 @@ class atlas_info(system_info): lapack = None atlas_1 = None for d in lib_dirs: - atlas = self.check_libs2(d, atlas_libs, []) + # FIXME: lapack_atlas is unused lapack_atlas = self.check_libs2(d, ['lapack_atlas'], []) + atlas = self.check_libs2(d, atlas_libs, []) if atlas is not None: lib_dirs2 = [d] + self.combine_paths(d, ['atlas*', 'ATLAS*']) lapack = self.check_libs2(lib_dirs2, lapack_libs, []) @@ -2107,17 +2108,17 @@ class numerix_info(system_info): if which[0] is None: which = "numpy", "defaulted" try: - import numpy + import numpy # noqa: F401 which = "numpy", "defaulted" except ImportError: msg1 = str(get_exception()) try: - import Numeric + import Numeric # noqa: F401 which = "numeric", "defaulted" except ImportError: msg2 = str(get_exception()) try: - import numarray + import numarray # noqa: F401 which = "numarray", "defaulted" except ImportError: msg3 = str(get_exception()) @@ -2437,7 +2438,6 @@ class umfpack_info(system_info): define_macros=[('SCIPY_UMFPACK_H', None)], swig_opts=['-I' + inc_dir]) - amd = get_info('amd') dict_append(info, **get_info('amd')) self.set_info(**info) @@ -2533,6 +2533,7 @@ def show_all(argv=None): del show_only[show_only.index(name)] conf = c() conf.verbosity = 2 + # FIXME: r not used r = conf.get_info() if show_only: log.info('Info classes not defined: %s', ','.join(show_only)) diff --git a/numpy/distutils/tests/test_fcompiler.py b/numpy/distutils/tests/test_fcompiler.py index 95e44b051..ba19a97ea 100644 --- a/numpy/distutils/tests/test_fcompiler.py +++ b/numpy/distutils/tests/test_fcompiler.py @@ -1,6 +1,8 @@ from __future__ import division, absolute_import, print_function -from numpy.testing import assert_ +import pytest + +from numpy.testing import assert_, suppress_warnings import numpy.distutils.fcompiler customizable_flags = [ @@ -25,6 +27,7 @@ def test_fcompiler_flags(monkeypatch): monkeypatch.setenv(envvar, new_flag) new_flags = getattr(flag_vars, opt) + monkeypatch.delenv(envvar) assert_(new_flags == [new_flag]) @@ -33,12 +36,46 @@ def test_fcompiler_flags(monkeypatch): for opt, envvar in customizable_flags: new_flag = '-dummy-{}-flag'.format(opt) prev_flags = getattr(flag_vars, opt) - monkeypatch.setenv(envvar, new_flag) new_flags = getattr(flag_vars, opt) + monkeypatch.delenv(envvar) if prev_flags is None: assert_(new_flags == [new_flag]) else: assert_(new_flags == prev_flags + [new_flag]) + +def test_fcompiler_flags_append_warning(monkeypatch): + # Test to check that the warning for append behavior changing in future + # is triggered. Need to use a real compiler instance so that we have + # non-empty flags to start with (otherwise the "if var and append" check + # will always be false). + try: + with suppress_warnings() as sup: + sup.record() + fc = numpy.distutils.fcompiler.new_fcompiler(compiler='gnu95') + fc.customize() + except numpy.distutils.fcompiler.CompilerNotFound: + pytest.skip("gfortran not found, so can't execute this test") + + # Ensure NPY_DISTUTILS_APPEND_FLAGS not defined + monkeypatch.delenv('NPY_DISTUTILS_APPEND_FLAGS', raising=False) + + for opt, envvar in customizable_flags: + new_flag = '-dummy-{}-flag'.format(opt) + with suppress_warnings() as sup: + sup.record() + prev_flags = getattr(fc.flag_vars, opt) + + monkeypatch.setenv(envvar, new_flag) + with suppress_warnings() as sup: + sup.record() + new_flags = getattr(fc.flag_vars, opt) + if prev_flags: + # Check that warning was issued + assert len(sup.log) == 1 + + monkeypatch.delenv(envvar) + assert_(new_flags == [new_flag]) + diff --git a/numpy/distutils/tests/test_misc_util.py b/numpy/distutils/tests/test_misc_util.py index 2a3294ddf..3e239cf48 100644 --- a/numpy/distutils/tests/test_misc_util.py +++ b/numpy/distutils/tests/test_misc_util.py @@ -79,3 +79,6 @@ def test_installed_npymath_ini(): # Regression test for gh-7707. If npymath.ini wasn't installed, then this # will give an error. info = get_info('npymath') + + assert isinstance(info, dict) + assert "define_macros" in info diff --git a/numpy/doc/broadcasting.py b/numpy/doc/broadcasting.py index 6c3a4bc75..0bdb6ae7d 100644 --- a/numpy/doc/broadcasting.py +++ b/numpy/doc/broadcasting.py @@ -3,6 +3,12 @@ Broadcasting over arrays ======================== +.. note:: + See `this article + <https://numpy.org/devdocs/user/theory.broadcasting.html>`_ + for illustrations of broadcasting concepts. + + The term broadcasting describes how numpy treats arrays with different shapes during arithmetic operations. Subject to certain constraints, the smaller array is "broadcast" across the larger array so that they @@ -172,8 +178,5 @@ Here the ``newaxis`` index operator inserts a new axis into ``a``, making it a two-dimensional ``4x1`` array. Combining the ``4x1`` array with ``b``, which has shape ``(3,)``, yields a ``4x3`` array. -See `this article <https://scipy.github.io/old-wiki/pages/EricsBroadcastingDoc>`_ -for illustrations of broadcasting concepts. - """ from __future__ import division, absolute_import, print_function diff --git a/numpy/doc/glossary.py b/numpy/doc/glossary.py index a3b9423a8..a3707340d 100644 --- a/numpy/doc/glossary.py +++ b/numpy/doc/glossary.py @@ -270,13 +270,11 @@ Glossary masked_array(data = [-- 2.0 --], mask = [ True False True], fill_value = 1e+20) - <BLANKLINE> >>> x + [1, 2, 3] masked_array(data = [-- 4.0 --], mask = [ True False True], fill_value = 1e+20) - <BLANKLINE> Masked arrays are often used when operating on arrays containing diff --git a/numpy/doc/structured_arrays.py b/numpy/doc/structured_arrays.py index ab97c5df6..e92a06124 100644 --- a/numpy/doc/structured_arrays.py +++ b/numpy/doc/structured_arrays.py @@ -35,26 +35,24 @@ with the field name:: array([('Rex', 5, 81.0), ('Fido', 5, 27.0)], dtype=[('name', 'S10'), ('age', '<i4'), ('weight', '<f4')]) -Structured arrays are designed for low-level manipulation of structured data, -for example, for interpreting binary blobs. Structured datatypes are -designed to mimic 'structs' in the C language, making them also useful for -interfacing with C code. For these purposes, numpy supports specialized -features such as subarrays and nested datatypes, and allows manual control over -the memory layout of the structure. - -For simple manipulation of tabular data other pydata projects, such as pandas, -xarray, or DataArray, provide higher-level interfaces that may be more -suitable. These projects may also give better performance for tabular data -analysis because the C-struct-like memory layout of structured arrays can lead -to poor cache behavior. +Structured datatypes are designed to be able to mimic 'structs' in the C +language, and share a similar memory layout. They are meant for interfacing with +C code and for low-level manipulation of structured buffers, for example for +interpreting binary blobs. For these purposes they support specialized features +such as subarrays, nested datatypes, and unions, and allow control over the +memory layout of the structure. + +Users looking to manipulate tabular data, such as stored in csv files, may find +other pydata projects more suitable, such as xarray, pandas, or DataArray. +These provide a high-level interface for tabular data analysis and are better +optimized for that use. For instance, the C-struct-like memory layout of +structured arrays in numpy can lead to poor cache behavior in comparison. .. _defining-structured-types: Structured Datatypes ==================== -To use structured arrays one first needs to define a structured datatype. - A structured datatype can be thought of as a sequence of bytes of a certain length (the structure's :term:`itemsize`) which is interpreted as a collection of fields. Each field has a name, a datatype, and a byte offset within the @@ -180,7 +178,9 @@ values are tuples containing the dtype and byte offset of each field. :: mappingproxy({'x': (dtype('int64'), 0), 'y': (dtype('float32'), 8)}) Both the ``names`` and ``fields`` attributes will equal ``None`` for -unstructured arrays. +unstructured arrays. The recommended way to test if a dtype is structured is +with `if dt.names is not None` rather than `if dt.names`, to account for dtypes +with 0 fields. The string representation of a structured datatype is shown in the "list of tuples" form if possible, otherwise numpy falls back to using the more general @@ -397,6 +397,15 @@ typically a non-structured array, except in the case of nested structures. >>> y.dtype, y.shape, y.strides (dtype('float32'), (2,), (12,)) +If the accessed field is a subarray, the dimensions of the subarray +are appended to the shape of the result:: + + >>> x = np.zeros((2,2), dtype=[('a', np.int32), ('b', np.float64, (3,3))]) + >>> x['a'].shape + (2, 2) + >>> x['b'].shape + (2, 2, 3, 3) + Accessing Multiple Fields ``````````````````````````` @@ -404,11 +413,10 @@ One can index and assign to a structured array with a multi-field index, where the index is a list of field names. .. warning:: - The behavior of multi-field indexes will change from Numpy 1.15 to Numpy - 1.16. + The behavior of multi-field indexes changed from Numpy 1.15 to Numpy 1.16. -In Numpy 1.16, the result of indexing with a multi-field index will be a view -into the original array, as follows:: +The result of indexing with a multi-field index is a view into the original +array, as follows:: >>> a = np.zeros(3, dtype=[('a', 'i4'), ('b', 'i4'), ('c', 'f4')]) >>> a[['a', 'c']] @@ -420,32 +428,58 @@ in the order they were indexed. Note that unlike for single-field indexing, the view's dtype has the same itemsize as the original array, and has fields at the same offsets as in the original array, and unindexed fields are merely missing. -In Numpy 1.15, indexing an array with a multi-field index returns a copy of -the result above for 1.16, but with fields packed together in memory as if -passed through :func:`numpy.lib.recfunctions.repack_fields`. This is the -behavior since Numpy 1.7. - .. warning:: - The new behavior in Numpy 1.16 leads to extra "padding" bytes at the - location of unindexed fields. You will need to update any code which depends - on the data having a "packed" layout. For instance code such as:: + In Numpy 1.15, indexing an array with a multi-field index returned a copy of + the result above, but with fields packed together in memory as if + passed through :func:`numpy.lib.recfunctions.repack_fields`. + + The new behavior as of Numpy 1.16 leads to extra "padding" bytes at the + location of unindexed fields compared to 1.15. You will need to update any + code which depends on the data having a "packed" layout. For instance code + such as:: + + >>> a = np.zeros(3, dtype=[('a', 'i4'), ('b', 'i4'), ('c', 'f4')]) + >>> a[['a','c']].view('i8') # Fails in Numpy 1.16 + ValueError: When changing to a smaller dtype, its size must be a divisor of the size of original dtype + + will need to be changed. This code has raised a ``FutureWarning`` since + Numpy 1.12, and similar code has raised ``FutureWarning`` since 1.7. + + In 1.16 a number of functions have been introduced in the + :module:`numpy.lib.recfunctions` module to help users account for this + change. These are + :func:`numpy.lib.recfunctions.repack_fields`. + :func:`numpy.lib.recfunctions.structured_to_unstructured`, + :func:`numpy.lib.recfunctions.unstructured_to_structured`, + :func:`numpy.lib.recfunctions.apply_along_fields`, + :func:`numpy.lib.recfunctions.assign_fields_by_name`, and + :func:`numpy.lib.recfunctions.require_fields`. + + The function :func:`numpy.lib.recfunctions.repack_fields` can always be + used to reproduce the old behavior, as it will return a packed copy of the + structured array. The code above, for example, can be replaced with: + + >>> repack_fields(a[['a','c']]).view('i8') # supported in 1.16 + array([0, 0, 0]) + + Furthermore, numpy now provides a new function + :func:`numpy.lib.recfunctions.structured_to_unstructured` which is a safer + and more efficient alternative for users who wish to convert structured + arrays to unstructured arrays, as the view above is often indeded to do. + This function allows safe conversion to an unstructured type taking into + account padding, often avoids a copy, and also casts the datatypes + as needed, unlike the view. Code such as: - >>> a[['a','c']].view('i8') # will fail in Numpy 1.16 - ValueError: When changing to a smaller dtype, its size must be a divisor of the size of original dtype + >>> a = np.zeros(3, dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) + >>> a[['x', 'z']].view('f4') - will need to be changed. This code has raised a ``FutureWarning`` since - Numpy 1.12. + can be made safer by replacing with: - The following is a recommended fix, which will behave identically in Numpy - 1.15 and Numpy 1.16:: + >>> structured_to_unstructured(a[['x', 'z']]) + array([0, 0, 0]) - >>> from numpy.lib.recfunctions import repack_fields - >>> repack_fields(a[['a','c']]).view('i8') # supported 1.15 and 1.16 - array([0, 0, 0]) -Assigning to an array with a multi-field index will behave the same in Numpy -1.15 and Numpy 1.16. In both versions the assignment will modify the original -array:: +Assignment to an array with a multi-field index modifies the original array:: >>> a[['a', 'c']] = (2, 3) >>> a diff --git a/numpy/dual.py b/numpy/dual.py index 3a16a8ec5..651e845bb 100644 --- a/numpy/dual.py +++ b/numpy/dual.py @@ -51,14 +51,14 @@ _restore_dict = {} def register_func(name, func): if name not in __all__: - raise ValueError("%s not a dual function." % name) + raise ValueError("{} not a dual function.".format(name)) f = sys._getframe(0).f_globals _restore_dict[name] = f[name] f[name] = func def restore_func(name): if name not in __all__: - raise ValueError("%s not a dual function." % name) + raise ValueError("{} not a dual function.".format(name)) try: val = _restore_dict[name] except KeyError: diff --git a/numpy/f2py/__main__.py b/numpy/f2py/__main__.py index 6eff41099..708f7f362 100644 --- a/numpy/f2py/__main__.py +++ b/numpy/f2py/__main__.py @@ -1,6 +1,6 @@ # See http://cens.ioc.ee/projects/f2py2e/ from __future__ import division, print_function -from f2py2e import main +from numpy.f2py.f2py2e import main main() diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index 8e63d3cff..c41dd77c6 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -718,10 +718,7 @@ def modsign2map(m): def cb_sign2map(a, var, index=None): ret = {'varname': a} - if index is None or 1: # disable 7712 patch - ret['varname_i'] = ret['varname'] - else: - ret['varname_i'] = ret['varname'] + '_' + str(index) + ret['varname_i'] = ret['varname'] ret['ctype'] = getctype(var) if ret['ctype'] in c2capi_map: ret['atype'] = c2capi_map[ret['ctype']] diff --git a/numpy/f2py/common_rules.py b/numpy/f2py/common_rules.py index 1940d4211..62c1ba207 100644 --- a/numpy/f2py/common_rules.py +++ b/numpy/f2py/common_rules.py @@ -31,11 +31,9 @@ from .crackfortran import rmbadname def findcommonblocks(block, top=1): ret = [] if hascommon(block): - for n in block['common'].keys(): - vars = {} - for v in block['common'][n]: - vars[v] = block['vars'][v] - ret.append((n, block['common'][n], vars)) + for key, value in block['common'].items(): + vars_ = {v: block['vars'][v] for v in value} + ret.append((key, value, vars_)) elif hasbody(block): for b in block['body']: ret = ret + findcommonblocks(b, 0) diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index 361203a57..2620fc9b7 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -1849,10 +1849,8 @@ def postcrack2(block, tab='', param_map=None): if not f90modulevars: return block if isinstance(block, list): - ret = [] - for g in block: - g = postcrack2(g, tab=tab + '\t', param_map=param_map) - ret.append(g) + ret = [postcrack2(g, tab=tab + '\t', param_map=param_map) + for g in block] return ret setmesstext(block) outmess('%sBlock: %s\n' % (tab, block['name']), 0) @@ -1870,10 +1868,8 @@ def postcrack2(block, tab='', param_map=None): val = kind['kind'] if val in param_map: kind['kind'] = param_map[val] - new_body = [] - for b in block['body']: - b = postcrack2(b, tab=tab + '\t', param_map=param_map) - new_body.append(b) + new_body = [postcrack2(b, tab=tab + '\t', param_map=param_map) + for b in block['body']] block['body'] = new_body return block @@ -3211,10 +3207,8 @@ def vars2fortran(block, vars, args, tab='', as_interface=False): vardef = '%s(kind=%s)' % (vardef, selector['kind']) c = ' ' if 'attrspec' in vars[a]: - attr = [] - for l in vars[a]['attrspec']: - if l not in ['external']: - attr.append(l) + attr = [l for l in vars[a]['attrspec'] + if l not in ['external']] if attr: vardef = '%s, %s' % (vardef, ','.join(attr)) c = ',' diff --git a/numpy/f2py/src/test/foomodule.c b/numpy/f2py/src/test/foomodule.c index d7ecc2519..733fab0be 100644 --- a/numpy/f2py/src/test/foomodule.c +++ b/numpy/f2py/src/test/foomodule.c @@ -116,8 +116,6 @@ static PyMethodDef foo_module_methods[] = { void initfoo() { int i; PyObject *m, *d, *s; - PyTypeObject *t; - PyObject *f; import_array(); m = Py_InitModule("foo", foo_module_methods); diff --git a/numpy/f2py/tests/test_block_docstring.py b/numpy/f2py/tests/test_block_docstring.py index b2981aa82..8fc072a5e 100644 --- a/numpy/f2py/tests/test_block_docstring.py +++ b/numpy/f2py/tests/test_block_docstring.py @@ -1,6 +1,5 @@ from __future__ import division, absolute_import, print_function -import textwrap import sys import pytest from . import util diff --git a/numpy/f2py/tests/test_parameter.py b/numpy/f2py/tests/test_parameter.py index a0bbd9460..6a378687a 100644 --- a/numpy/f2py/tests/test_parameter.py +++ b/numpy/f2py/tests/test_parameter.py @@ -1,7 +1,6 @@ from __future__ import division, absolute_import, print_function import os -import math import pytest import numpy as np diff --git a/numpy/f2py/tests/test_quoted_character.py b/numpy/f2py/tests/test_quoted_character.py index 38e380802..c9a1c36f5 100644 --- a/numpy/f2py/tests/test_quoted_character.py +++ b/numpy/f2py/tests/test_quoted_character.py @@ -4,13 +4,9 @@ from __future__ import division, absolute_import, print_function import sys -import os -import uuid from importlib import import_module import pytest -import numpy.f2py - from numpy.testing import assert_equal from . import util diff --git a/numpy/f2py/tests/test_regression.py b/numpy/f2py/tests/test_regression.py index d695de61b..3adae635d 100644 --- a/numpy/f2py/tests/test_regression.py +++ b/numpy/f2py/tests/test_regression.py @@ -1,7 +1,6 @@ from __future__ import division, absolute_import, print_function import os -import math import pytest import numpy as np diff --git a/numpy/f2py/tests/util.py b/numpy/f2py/tests/util.py index 73fc27b96..5fa5dadd2 100644 --- a/numpy/f2py/tests/util.py +++ b/numpy/f2py/tests/util.py @@ -15,9 +15,7 @@ import shutil import atexit import textwrap import re -import random import pytest -import numpy.f2py from numpy.compat import asbytes, asstr from numpy.testing import temppath @@ -26,7 +24,7 @@ from importlib import import_module try: from hashlib import md5 except ImportError: - from md5 import new as md5 + from md5 import new as md5 # noqa: F401 # # Maintaining a temporary module directory diff --git a/numpy/fft/README.md b/numpy/fft/README.md new file mode 100644 index 000000000..7040a2e9b --- /dev/null +++ b/numpy/fft/README.md @@ -0,0 +1,53 @@ +PocketFFT +--------- + +This is a heavily modified implementation of FFTPack [1,2], with the following +advantages: + +- strictly C99 compliant +- more accurate twiddle factor computation +- very fast plan generation +- worst case complexity for transform sizes with large prime factors is + `N*log(N)`, because Bluestein's algorithm [3] is used for these cases. + +License +------- + +3-clause BSD (see LICENSE.md) + + +Some code details +----------------- + +Twiddle factor computation: + +- making use of symmetries to reduce number of sin/cos evaluations +- all angles are reduced to the range `[0; pi/4]` for higher accuracy +- an adapted implementation of `sincospi()` is used, which actually computes + `sin(x)` and `(cos(x)-1)`. +- if `n` sin/cos pairs are required, the adjusted `sincospi()` is only called + `2*sqrt(n)` times; the remaining values are obtained by evaluating the + angle addition theorems in a numerically accurate way. + +Parallel invocation: + +- Plans only contain read-only data; all temporary arrays are allocated and + deallocated during an individual FFT execution. This means that a single plan + can be used in several threads at the same time. + +Efficient codelets are available for the factors: + +- 2, 3, 4, 5, 7, 11 for complex-valued FFTs +- 2, 3, 4, 5 for real-valued FFTs + +Larger prime factors are handled by somewhat less efficient, generic routines. + +For lengths with very large prime factors, Bluestein's algorithm is used, and +instead of an FFT of length `n`, a convolution of length `n2 >= 2*n-1` +is performed, where `n2` is chosen to be highly composite. + + +[1] Swarztrauber, P. 1982, Vectorizing the Fast Fourier Transforms + (New York: Academic Press), 51 +[2] https://www.netlib.org/fftpack/ +[3] https://en.wikipedia.org/wiki/Chirp_Z-transform diff --git a/numpy/fft/__init__.py b/numpy/fft/__init__.py index 44243b483..64b35bc19 100644 --- a/numpy/fft/__init__.py +++ b/numpy/fft/__init__.py @@ -3,7 +3,7 @@ from __future__ import division, absolute_import, print_function # To get sub-modules from .info import __doc__ -from .fftpack import * +from .pocketfft import * from .helper import * from numpy._pytesttester import PytestTester diff --git a/numpy/fft/fftpack.c b/numpy/fft/fftpack.c deleted file mode 100644 index 07fa2bf4c..000000000 --- a/numpy/fft/fftpack.c +++ /dev/null @@ -1,1536 +0,0 @@ -/* - * fftpack.c : A set of FFT routines in C. - * Algorithmically based on Fortran-77 FFTPACK by Paul N. Swarztrauber (Version 4, 1985). -*/ -#define NPY_NO_DEPRECATED_API NPY_API_VERSION - -#include <Python.h> -#include <math.h> -#include <stdio.h> -#include <numpy/ndarraytypes.h> - -#define DOUBLE -#ifdef DOUBLE -#define Treal double -#else -#define Treal float -#endif - -#define ref(u,a) u[a] - -/* Macros for accurate calculation of the twiddle factors. */ -#define TWOPI 6.283185307179586476925286766559005768391 -#define cos2pi(m, n) cos((TWOPI * (m)) / (n)) -#define sin2pi(m, n) sin((TWOPI * (m)) / (n)) - -#define MAXFAC 13 /* maximum number of factors in factorization of n */ -#define NSPECIAL 4 /* number of factors for which we have special-case routines */ - -#ifdef __cplusplus -extern "C" { -#endif - -static void sincos2pi(int m, int n, Treal* si, Treal* co) -/* Calculates sin(2pi * m/n) and cos(2pi * m/n). It is more accurate - * than the naive calculation as the fraction m/n is reduced to [0, 1/8) first. - * Due to the symmetry of sin(x) and cos(x) the values for all x can be - * determined from the function values of the reduced argument in the first - * octant. - */ - { - int n8, m8, octant; - n8 = 8 * n; - m8 = (8 * m) % n8; - octant = m8 / n; - m8 = m8 % n; - switch(octant) { - case 0: - *co = cos2pi(m8, n8); - *si = sin2pi(m8, n8); - break; - case 1: - *co = sin2pi(n-m8, n8); - *si = cos2pi(n-m8, n8); - break; - case 2: - *co = -sin2pi(m8, n8); - *si = cos2pi(m8, n8); - break; - case 3: - *co = -cos2pi(n-m8, n8); - *si = sin2pi(n-m8, n8); - break; - case 4: - *co = -cos2pi(m8, n8); - *si = -sin2pi(m8, n8); - break; - case 5: - *co = -sin2pi(n-m8, n8); - *si = -cos2pi(n-m8, n8); - break; - case 6: - *co = sin2pi(m8, n8); - *si = -cos2pi(m8, n8); - break; - case 7: - *co = cos2pi(n-m8, n8); - *si = -sin2pi(n-m8, n8); - break; - } - } - -/* ---------------------------------------------------------------------- - passf2, passf3, passf4, passf5, passf. Complex FFT passes fwd and bwd. ------------------------------------------------------------------------ */ - -static void passf2(int ido, int l1, const Treal cc[], Treal ch[], const Treal wa1[], int isign) - /* isign==+1 for backward transform */ - { - int i, k, ah, ac; - Treal ti2, tr2; - if (ido <= 2) { - for (k=0; k<l1; k++) { - ah = k*ido; - ac = 2*k*ido; - ch[ah] = ref(cc,ac) + ref(cc,ac + ido); - ch[ah + ido*l1] = ref(cc,ac) - ref(cc,ac + ido); - ch[ah+1] = ref(cc,ac+1) + ref(cc,ac + ido + 1); - ch[ah + ido*l1 + 1] = ref(cc,ac+1) - ref(cc,ac + ido + 1); - } - } else { - for (k=0; k<l1; k++) { - for (i=0; i<ido-1; i+=2) { - ah = i + k*ido; - ac = i + 2*k*ido; - ch[ah] = ref(cc,ac) + ref(cc,ac + ido); - tr2 = ref(cc,ac) - ref(cc,ac + ido); - ch[ah+1] = ref(cc,ac+1) + ref(cc,ac + 1 + ido); - ti2 = ref(cc,ac+1) - ref(cc,ac + 1 + ido); - ch[ah+l1*ido+1] = wa1[i]*ti2 + isign*wa1[i+1]*tr2; - ch[ah+l1*ido] = wa1[i]*tr2 - isign*wa1[i+1]*ti2; - } - } - } - } /* passf2 */ - - -static void passf3(int ido, int l1, const Treal cc[], Treal ch[], - const Treal wa1[], const Treal wa2[], int isign) - /* isign==+1 for backward transform */ - { - static const Treal taur = -0.5; - static const Treal taui = 0.86602540378443864676; - int i, k, ac, ah; - Treal ci2, ci3, di2, di3, cr2, cr3, dr2, dr3, ti2, tr2; - if (ido == 2) { - for (k=1; k<=l1; k++) { - ac = (3*k - 2)*ido; - tr2 = ref(cc,ac) + ref(cc,ac + ido); - cr2 = ref(cc,ac - ido) + taur*tr2; - ah = (k - 1)*ido; - ch[ah] = ref(cc,ac - ido) + tr2; - - ti2 = ref(cc,ac + 1) + ref(cc,ac + ido + 1); - ci2 = ref(cc,ac - ido + 1) + taur*ti2; - ch[ah + 1] = ref(cc,ac - ido + 1) + ti2; - - cr3 = isign*taui*(ref(cc,ac) - ref(cc,ac + ido)); - ci3 = isign*taui*(ref(cc,ac + 1) - ref(cc,ac + ido + 1)); - ch[ah + l1*ido] = cr2 - ci3; - ch[ah + 2*l1*ido] = cr2 + ci3; - ch[ah + l1*ido + 1] = ci2 + cr3; - ch[ah + 2*l1*ido + 1] = ci2 - cr3; - } - } else { - for (k=1; k<=l1; k++) { - for (i=0; i<ido-1; i+=2) { - ac = i + (3*k - 2)*ido; - tr2 = ref(cc,ac) + ref(cc,ac + ido); - cr2 = ref(cc,ac - ido) + taur*tr2; - ah = i + (k-1)*ido; - ch[ah] = ref(cc,ac - ido) + tr2; - ti2 = ref(cc,ac + 1) + ref(cc,ac + ido + 1); - ci2 = ref(cc,ac - ido + 1) + taur*ti2; - ch[ah + 1] = ref(cc,ac - ido + 1) + ti2; - cr3 = isign*taui*(ref(cc,ac) - ref(cc,ac + ido)); - ci3 = isign*taui*(ref(cc,ac + 1) - ref(cc,ac + ido + 1)); - dr2 = cr2 - ci3; - dr3 = cr2 + ci3; - di2 = ci2 + cr3; - di3 = ci2 - cr3; - ch[ah + l1*ido + 1] = wa1[i]*di2 + isign*wa1[i+1]*dr2; - ch[ah + l1*ido] = wa1[i]*dr2 - isign*wa1[i+1]*di2; - ch[ah + 2*l1*ido + 1] = wa2[i]*di3 + isign*wa2[i+1]*dr3; - ch[ah + 2*l1*ido] = wa2[i]*dr3 - isign*wa2[i+1]*di3; - } - } - } - } /* passf3 */ - - -static void passf4(int ido, int l1, const Treal cc[], Treal ch[], - const Treal wa1[], const Treal wa2[], const Treal wa3[], int isign) - /* isign == -1 for forward transform and +1 for backward transform */ - { - int i, k, ac, ah; - Treal ci2, ci3, ci4, cr2, cr3, cr4, ti1, ti2, ti3, ti4, tr1, tr2, tr3, tr4; - if (ido == 2) { - for (k=0; k<l1; k++) { - ac = 4*k*ido + 1; - ti1 = ref(cc,ac) - ref(cc,ac + 2*ido); - ti2 = ref(cc,ac) + ref(cc,ac + 2*ido); - tr4 = ref(cc,ac + 3*ido) - ref(cc,ac + ido); - ti3 = ref(cc,ac + ido) + ref(cc,ac + 3*ido); - tr1 = ref(cc,ac - 1) - ref(cc,ac + 2*ido - 1); - tr2 = ref(cc,ac - 1) + ref(cc,ac + 2*ido - 1); - ti4 = ref(cc,ac + ido - 1) - ref(cc,ac + 3*ido - 1); - tr3 = ref(cc,ac + ido - 1) + ref(cc,ac + 3*ido - 1); - ah = k*ido; - ch[ah] = tr2 + tr3; - ch[ah + 2*l1*ido] = tr2 - tr3; - ch[ah + 1] = ti2 + ti3; - ch[ah + 2*l1*ido + 1] = ti2 - ti3; - ch[ah + l1*ido] = tr1 + isign*tr4; - ch[ah + 3*l1*ido] = tr1 - isign*tr4; - ch[ah + l1*ido + 1] = ti1 + isign*ti4; - ch[ah + 3*l1*ido + 1] = ti1 - isign*ti4; - } - } else { - for (k=0; k<l1; k++) { - for (i=0; i<ido-1; i+=2) { - ac = i + 1 + 4*k*ido; - ti1 = ref(cc,ac) - ref(cc,ac + 2*ido); - ti2 = ref(cc,ac) + ref(cc,ac + 2*ido); - ti3 = ref(cc,ac + ido) + ref(cc,ac + 3*ido); - tr4 = ref(cc,ac + 3*ido) - ref(cc,ac + ido); - tr1 = ref(cc,ac - 1) - ref(cc,ac + 2*ido - 1); - tr2 = ref(cc,ac - 1) + ref(cc,ac + 2*ido - 1); - ti4 = ref(cc,ac + ido - 1) - ref(cc,ac + 3*ido - 1); - tr3 = ref(cc,ac + ido - 1) + ref(cc,ac + 3*ido - 1); - ah = i + k*ido; - ch[ah] = tr2 + tr3; - cr3 = tr2 - tr3; - ch[ah + 1] = ti2 + ti3; - ci3 = ti2 - ti3; - cr2 = tr1 + isign*tr4; - cr4 = tr1 - isign*tr4; - ci2 = ti1 + isign*ti4; - ci4 = ti1 - isign*ti4; - ch[ah + l1*ido] = wa1[i]*cr2 - isign*wa1[i + 1]*ci2; - ch[ah + l1*ido + 1] = wa1[i]*ci2 + isign*wa1[i + 1]*cr2; - ch[ah + 2*l1*ido] = wa2[i]*cr3 - isign*wa2[i + 1]*ci3; - ch[ah + 2*l1*ido + 1] = wa2[i]*ci3 + isign*wa2[i + 1]*cr3; - ch[ah + 3*l1*ido] = wa3[i]*cr4 -isign*wa3[i + 1]*ci4; - ch[ah + 3*l1*ido + 1] = wa3[i]*ci4 + isign*wa3[i + 1]*cr4; - } - } - } - } /* passf4 */ - - -static void passf5(int ido, int l1, const Treal cc[], Treal ch[], - const Treal wa1[], const Treal wa2[], const Treal wa3[], const Treal wa4[], int isign) - /* isign == -1 for forward transform and +1 for backward transform */ - { - static const Treal tr11 = 0.3090169943749474241; - static const Treal ti11 = 0.95105651629515357212; - static const Treal tr12 = -0.8090169943749474241; - static const Treal ti12 = 0.58778525229247312917; - int i, k, ac, ah; - Treal ci2, ci3, ci4, ci5, di3, di4, di5, di2, cr2, cr3, cr5, cr4, ti2, ti3, - ti4, ti5, dr3, dr4, dr5, dr2, tr2, tr3, tr4, tr5; - if (ido == 2) { - for (k = 1; k <= l1; ++k) { - ac = (5*k - 4)*ido + 1; - ti5 = ref(cc,ac) - ref(cc,ac + 3*ido); - ti2 = ref(cc,ac) + ref(cc,ac + 3*ido); - ti4 = ref(cc,ac + ido) - ref(cc,ac + 2*ido); - ti3 = ref(cc,ac + ido) + ref(cc,ac + 2*ido); - tr5 = ref(cc,ac - 1) - ref(cc,ac + 3*ido - 1); - tr2 = ref(cc,ac - 1) + ref(cc,ac + 3*ido - 1); - tr4 = ref(cc,ac + ido - 1) - ref(cc,ac + 2*ido - 1); - tr3 = ref(cc,ac + ido - 1) + ref(cc,ac + 2*ido - 1); - ah = (k - 1)*ido; - ch[ah] = ref(cc,ac - ido - 1) + tr2 + tr3; - ch[ah + 1] = ref(cc,ac - ido) + ti2 + ti3; - cr2 = ref(cc,ac - ido - 1) + tr11*tr2 + tr12*tr3; - ci2 = ref(cc,ac - ido) + tr11*ti2 + tr12*ti3; - cr3 = ref(cc,ac - ido - 1) + tr12*tr2 + tr11*tr3; - ci3 = ref(cc,ac - ido) + tr12*ti2 + tr11*ti3; - cr5 = isign*(ti11*tr5 + ti12*tr4); - ci5 = isign*(ti11*ti5 + ti12*ti4); - cr4 = isign*(ti12*tr5 - ti11*tr4); - ci4 = isign*(ti12*ti5 - ti11*ti4); - ch[ah + l1*ido] = cr2 - ci5; - ch[ah + 4*l1*ido] = cr2 + ci5; - ch[ah + l1*ido + 1] = ci2 + cr5; - ch[ah + 2*l1*ido + 1] = ci3 + cr4; - ch[ah + 2*l1*ido] = cr3 - ci4; - ch[ah + 3*l1*ido] = cr3 + ci4; - ch[ah + 3*l1*ido + 1] = ci3 - cr4; - ch[ah + 4*l1*ido + 1] = ci2 - cr5; - } - } else { - for (k=1; k<=l1; k++) { - for (i=0; i<ido-1; i+=2) { - ac = i + 1 + (k*5 - 4)*ido; - ti5 = ref(cc,ac) - ref(cc,ac + 3*ido); - ti2 = ref(cc,ac) + ref(cc,ac + 3*ido); - ti4 = ref(cc,ac + ido) - ref(cc,ac + 2*ido); - ti3 = ref(cc,ac + ido) + ref(cc,ac + 2*ido); - tr5 = ref(cc,ac - 1) - ref(cc,ac + 3*ido - 1); - tr2 = ref(cc,ac - 1) + ref(cc,ac + 3*ido - 1); - tr4 = ref(cc,ac + ido - 1) - ref(cc,ac + 2*ido - 1); - tr3 = ref(cc,ac + ido - 1) + ref(cc,ac + 2*ido - 1); - ah = i + (k - 1)*ido; - ch[ah] = ref(cc,ac - ido - 1) + tr2 + tr3; - ch[ah + 1] = ref(cc,ac - ido) + ti2 + ti3; - cr2 = ref(cc,ac - ido - 1) + tr11*tr2 + tr12*tr3; - - ci2 = ref(cc,ac - ido) + tr11*ti2 + tr12*ti3; - cr3 = ref(cc,ac - ido - 1) + tr12*tr2 + tr11*tr3; - - ci3 = ref(cc,ac - ido) + tr12*ti2 + tr11*ti3; - cr5 = isign*(ti11*tr5 + ti12*tr4); - ci5 = isign*(ti11*ti5 + ti12*ti4); - cr4 = isign*(ti12*tr5 - ti11*tr4); - ci4 = isign*(ti12*ti5 - ti11*ti4); - dr3 = cr3 - ci4; - dr4 = cr3 + ci4; - di3 = ci3 + cr4; - di4 = ci3 - cr4; - dr5 = cr2 + ci5; - dr2 = cr2 - ci5; - di5 = ci2 - cr5; - di2 = ci2 + cr5; - ch[ah + l1*ido] = wa1[i]*dr2 - isign*wa1[i+1]*di2; - ch[ah + l1*ido + 1] = wa1[i]*di2 + isign*wa1[i+1]*dr2; - ch[ah + 2*l1*ido] = wa2[i]*dr3 - isign*wa2[i+1]*di3; - ch[ah + 2*l1*ido + 1] = wa2[i]*di3 + isign*wa2[i+1]*dr3; - ch[ah + 3*l1*ido] = wa3[i]*dr4 - isign*wa3[i+1]*di4; - ch[ah + 3*l1*ido + 1] = wa3[i]*di4 + isign*wa3[i+1]*dr4; - ch[ah + 4*l1*ido] = wa4[i]*dr5 - isign*wa4[i+1]*di5; - ch[ah + 4*l1*ido + 1] = wa4[i]*di5 + isign*wa4[i+1]*dr5; - } - } - } - } /* passf5 */ - - -static void passf(int *nac, int ido, int ip, int l1, int idl1, - Treal cc[], Treal ch[], - const Treal wa[], int isign) - /* isign is -1 for forward transform and +1 for backward transform */ - { - int idij, idlj, idot, ipph, i, j, k, l, jc, lc, ik, idj, idl, inc,idp; - Treal wai, war; - - idot = ido / 2; - /* nt = ip*idl1;*/ - ipph = (ip + 1) / 2; - idp = ip*ido; - if (ido >= l1) { - for (j=1; j<ipph; j++) { - jc = ip - j; - for (k=0; k<l1; k++) { - for (i=0; i<ido; i++) { - ch[i + (k + j*l1)*ido] = - ref(cc,i + (j + k*ip)*ido) + ref(cc,i + (jc + k*ip)*ido); - ch[i + (k + jc*l1)*ido] = - ref(cc,i + (j + k*ip)*ido) - ref(cc,i + (jc + k*ip)*ido); - } - } - } - for (k=0; k<l1; k++) - for (i=0; i<ido; i++) - ch[i + k*ido] = ref(cc,i + k*ip*ido); - } else { - for (j=1; j<ipph; j++) { - jc = ip - j; - for (i=0; i<ido; i++) { - for (k=0; k<l1; k++) { - ch[i + (k + j*l1)*ido] = ref(cc,i + (j + k*ip)*ido) + ref(cc,i + (jc + k* - ip)*ido); - ch[i + (k + jc*l1)*ido] = ref(cc,i + (j + k*ip)*ido) - ref(cc,i + (jc + k* - ip)*ido); - } - } - } - for (i=0; i<ido; i++) - for (k=0; k<l1; k++) - ch[i + k*ido] = ref(cc,i + k*ip*ido); - } - - idl = 2 - ido; - inc = 0; - for (l=1; l<ipph; l++) { - lc = ip - l; - idl += ido; - for (ik=0; ik<idl1; ik++) { - cc[ik + l*idl1] = ch[ik] + wa[idl - 2]*ch[ik + idl1]; - cc[ik + lc*idl1] = isign*wa[idl-1]*ch[ik + (ip-1)*idl1]; - } - idlj = idl; - inc += ido; - for (j=2; j<ipph; j++) { - jc = ip - j; - idlj += inc; - if (idlj > idp) idlj -= idp; - war = wa[idlj - 2]; - wai = wa[idlj-1]; - for (ik=0; ik<idl1; ik++) { - cc[ik + l*idl1] += war*ch[ik + j*idl1]; - cc[ik + lc*idl1] += isign*wai*ch[ik + jc*idl1]; - } - } - } - for (j=1; j<ipph; j++) - for (ik=0; ik<idl1; ik++) - ch[ik] += ch[ik + j*idl1]; - for (j=1; j<ipph; j++) { - jc = ip - j; - for (ik=1; ik<idl1; ik+=2) { - ch[ik - 1 + j*idl1] = cc[ik - 1 + j*idl1] - cc[ik + jc*idl1]; - ch[ik - 1 + jc*idl1] = cc[ik - 1 + j*idl1] + cc[ik + jc*idl1]; - ch[ik + j*idl1] = cc[ik + j*idl1] + cc[ik - 1 + jc*idl1]; - ch[ik + jc*idl1] = cc[ik + j*idl1] - cc[ik - 1 + jc*idl1]; - } - } - *nac = 1; - if (ido == 2) return; - *nac = 0; - for (ik=0; ik<idl1; ik++) - cc[ik] = ch[ik]; - for (j=1; j<ip; j++) { - for (k=0; k<l1; k++) { - cc[(k + j*l1)*ido + 0] = ch[(k + j*l1)*ido + 0]; - cc[(k + j*l1)*ido + 1] = ch[(k + j*l1)*ido + 1]; - } - } - if (idot <= l1) { - idij = 0; - for (j=1; j<ip; j++) { - idij += 2; - for (i=3; i<ido; i+=2) { - idij += 2; - for (k=0; k<l1; k++) { - cc[i - 1 + (k + j*l1)*ido] = - wa[idij - 2]*ch[i - 1 + (k + j*l1)*ido] - - isign*wa[idij-1]*ch[i + (k + j*l1)*ido]; - cc[i + (k + j*l1)*ido] = - wa[idij - 2]*ch[i + (k + j*l1)*ido] + - isign*wa[idij-1]*ch[i - 1 + (k + j*l1)*ido]; - } - } - } - } else { - idj = 2 - ido; - for (j=1; j<ip; j++) { - idj += ido; - for (k = 0; k < l1; k++) { - idij = idj; - for (i=3; i<ido; i+=2) { - idij += 2; - cc[i - 1 + (k + j*l1)*ido] = - wa[idij - 2]*ch[i - 1 + (k + j*l1)*ido] - - isign*wa[idij-1]*ch[i + (k + j*l1)*ido]; - cc[i + (k + j*l1)*ido] = - wa[idij - 2]*ch[i + (k + j*l1)*ido] + - isign*wa[idij-1]*ch[i - 1 + (k + j*l1)*ido]; - } - } - } - } - } /* passf */ - - - /* ---------------------------------------------------------------------- -radf2,radb2, radf3,radb3, radf4,radb4, radf5,radb5, radfg,radbg. -Treal FFT passes fwd and bwd. ----------------------------------------------------------------------- */ - -static void radf2(int ido, int l1, const Treal cc[], Treal ch[], const Treal wa1[]) - { - int i, k, ic; - Treal ti2, tr2; - for (k=0; k<l1; k++) { - ch[2*k*ido] = - ref(cc,k*ido) + ref(cc,(k + l1)*ido); - ch[(2*k+1)*ido + ido-1] = - ref(cc,k*ido) - ref(cc,(k + l1)*ido); - } - if (ido < 2) return; - if (ido != 2) { - for (k=0; k<l1; k++) { - for (i=2; i<ido; i+=2) { - ic = ido - i; - tr2 = wa1[i - 2]*ref(cc, i-1 + (k + l1)*ido) + wa1[i - 1]*ref(cc, i + (k + l1)*ido); - ti2 = wa1[i - 2]*ref(cc, i + (k + l1)*ido) - wa1[i - 1]*ref(cc, i-1 + (k + l1)*ido); - ch[i + 2*k*ido] = ref(cc,i + k*ido) + ti2; - ch[ic + (2*k+1)*ido] = ti2 - ref(cc,i + k*ido); - ch[i - 1 + 2*k*ido] = ref(cc,i - 1 + k*ido) + tr2; - ch[ic - 1 + (2*k+1)*ido] = ref(cc,i - 1 + k*ido) - tr2; - } - } - if (ido % 2 == 1) return; - } - for (k=0; k<l1; k++) { - ch[(2*k+1)*ido] = -ref(cc,ido-1 + (k + l1)*ido); - ch[ido-1 + 2*k*ido] = ref(cc,ido-1 + k*ido); - } - } /* radf2 */ - - -static void radb2(int ido, int l1, const Treal cc[], Treal ch[], const Treal wa1[]) - { - int i, k, ic; - Treal ti2, tr2; - for (k=0; k<l1; k++) { - ch[k*ido] = - ref(cc,2*k*ido) + ref(cc,ido-1 + (2*k+1)*ido); - ch[(k + l1)*ido] = - ref(cc,2*k*ido) - ref(cc,ido-1 + (2*k+1)*ido); - } - if (ido < 2) return; - if (ido != 2) { - for (k = 0; k < l1; ++k) { - for (i = 2; i < ido; i += 2) { - ic = ido - i; - ch[i-1 + k*ido] = - ref(cc,i-1 + 2*k*ido) + ref(cc,ic-1 + (2*k+1)*ido); - tr2 = ref(cc,i-1 + 2*k*ido) - ref(cc,ic-1 + (2*k+1)*ido); - ch[i + k*ido] = - ref(cc,i + 2*k*ido) - ref(cc,ic + (2*k+1)*ido); - ti2 = ref(cc,i + (2*k)*ido) + ref(cc,ic + (2*k+1)*ido); - ch[i-1 + (k + l1)*ido] = - wa1[i - 2]*tr2 - wa1[i - 1]*ti2; - ch[i + (k + l1)*ido] = - wa1[i - 2]*ti2 + wa1[i - 1]*tr2; - } - } - if (ido % 2 == 1) return; - } - for (k = 0; k < l1; k++) { - ch[ido-1 + k*ido] = 2*ref(cc,ido-1 + 2*k*ido); - ch[ido-1 + (k + l1)*ido] = -2*ref(cc,(2*k+1)*ido); - } - } /* radb2 */ - - -static void radf3(int ido, int l1, const Treal cc[], Treal ch[], - const Treal wa1[], const Treal wa2[]) - { - static const Treal taur = -0.5; - static const Treal taui = 0.86602540378443864676; - int i, k, ic; - Treal ci2, di2, di3, cr2, dr2, dr3, ti2, ti3, tr2, tr3; - for (k=0; k<l1; k++) { - cr2 = ref(cc,(k + l1)*ido) + ref(cc,(k + 2*l1)*ido); - ch[3*k*ido] = ref(cc,k*ido) + cr2; - ch[(3*k+2)*ido] = taui*(ref(cc,(k + l1*2)*ido) - ref(cc,(k + l1)*ido)); - ch[ido-1 + (3*k + 1)*ido] = ref(cc,k*ido) + taur*cr2; - } - if (ido == 1) return; - for (k=0; k<l1; k++) { - for (i=2; i<ido; i+=2) { - ic = ido - i; - dr2 = wa1[i - 2]*ref(cc,i - 1 + (k + l1)*ido) + - wa1[i - 1]*ref(cc,i + (k + l1)*ido); - di2 = wa1[i - 2]*ref(cc,i + (k + l1)*ido) - wa1[i - 1]*ref(cc,i - 1 + (k + l1)*ido); - dr3 = wa2[i - 2]*ref(cc,i - 1 + (k + l1*2)*ido) + wa2[i - 1]*ref(cc,i + (k + l1*2)*ido); - di3 = wa2[i - 2]*ref(cc,i + (k + l1*2)*ido) - wa2[i - 1]*ref(cc,i - 1 + (k + l1*2)*ido); - cr2 = dr2 + dr3; - ci2 = di2 + di3; - ch[i - 1 + 3*k*ido] = ref(cc,i - 1 + k*ido) + cr2; - ch[i + 3*k*ido] = ref(cc,i + k*ido) + ci2; - tr2 = ref(cc,i - 1 + k*ido) + taur*cr2; - ti2 = ref(cc,i + k*ido) + taur*ci2; - tr3 = taui*(di2 - di3); - ti3 = taui*(dr3 - dr2); - ch[i - 1 + (3*k + 2)*ido] = tr2 + tr3; - ch[ic - 1 + (3*k + 1)*ido] = tr2 - tr3; - ch[i + (3*k + 2)*ido] = ti2 + ti3; - ch[ic + (3*k + 1)*ido] = ti3 - ti2; - } - } - } /* radf3 */ - - -static void radb3(int ido, int l1, const Treal cc[], Treal ch[], - const Treal wa1[], const Treal wa2[]) - { - static const Treal taur = -0.5; - static const Treal taui = 0.86602540378443864676; - int i, k, ic; - Treal ci2, ci3, di2, di3, cr2, cr3, dr2, dr3, ti2, tr2; - for (k=0; k<l1; k++) { - tr2 = 2*ref(cc,ido-1 + (3*k + 1)*ido); - cr2 = ref(cc,3*k*ido) + taur*tr2; - ch[k*ido] = ref(cc,3*k*ido) + tr2; - ci3 = 2*taui*ref(cc,(3*k + 2)*ido); - ch[(k + l1)*ido] = cr2 - ci3; - ch[(k + 2*l1)*ido] = cr2 + ci3; - } - if (ido == 1) return; - for (k=0; k<l1; k++) { - for (i=2; i<ido; i+=2) { - ic = ido - i; - tr2 = ref(cc,i - 1 + (3*k + 2)*ido) + ref(cc,ic - 1 + (3*k + 1)*ido); - cr2 = ref(cc,i - 1 + 3*k*ido) + taur*tr2; - ch[i - 1 + k*ido] = ref(cc,i - 1 + 3*k*ido) + tr2; - ti2 = ref(cc,i + (3*k + 2)*ido) - ref(cc,ic + (3*k + 1)*ido); - ci2 = ref(cc,i + 3*k*ido) + taur*ti2; - ch[i + k*ido] = ref(cc,i + 3*k*ido) + ti2; - cr3 = taui*(ref(cc,i - 1 + (3*k + 2)*ido) - ref(cc,ic - 1 + (3*k + 1)*ido)); - ci3 = taui*(ref(cc,i + (3*k + 2)*ido) + ref(cc,ic + (3*k + 1)*ido)); - dr2 = cr2 - ci3; - dr3 = cr2 + ci3; - di2 = ci2 + cr3; - di3 = ci2 - cr3; - ch[i - 1 + (k + l1)*ido] = wa1[i - 2]*dr2 - wa1[i - 1]*di2; - ch[i + (k + l1)*ido] = wa1[i - 2]*di2 + wa1[i - 1]*dr2; - ch[i - 1 + (k + 2*l1)*ido] = wa2[i - 2]*dr3 - wa2[i - 1]*di3; - ch[i + (k + 2*l1)*ido] = wa2[i - 2]*di3 + wa2[i - 1]*dr3; - } - } - } /* radb3 */ - - -static void radf4(int ido, int l1, const Treal cc[], Treal ch[], - const Treal wa1[], const Treal wa2[], const Treal wa3[]) - { - static const Treal hsqt2 = 0.70710678118654752440; - int i, k, ic; - Treal ci2, ci3, ci4, cr2, cr3, cr4, ti1, ti2, ti3, ti4, tr1, tr2, tr3, tr4; - for (k=0; k<l1; k++) { - tr1 = ref(cc,(k + l1)*ido) + ref(cc,(k + 3*l1)*ido); - tr2 = ref(cc,k*ido) + ref(cc,(k + 2*l1)*ido); - ch[4*k*ido] = tr1 + tr2; - ch[ido-1 + (4*k + 3)*ido] = tr2 - tr1; - ch[ido-1 + (4*k + 1)*ido] = ref(cc,k*ido) - ref(cc,(k + 2*l1)*ido); - ch[(4*k + 2)*ido] = ref(cc,(k + 3*l1)*ido) - ref(cc,(k + l1)*ido); - } - if (ido < 2) return; - if (ido != 2) { - for (k=0; k<l1; k++) { - for (i=2; i<ido; i += 2) { - ic = ido - i; - cr2 = wa1[i - 2]*ref(cc,i - 1 + (k + l1)*ido) + wa1[i - 1]*ref(cc,i + (k + l1)*ido); - ci2 = wa1[i - 2]*ref(cc,i + (k + l1)*ido) - wa1[i - 1]*ref(cc,i - 1 + (k + l1)*ido); - cr3 = wa2[i - 2]*ref(cc,i - 1 + (k + 2*l1)*ido) + wa2[i - 1]*ref(cc,i + (k + 2*l1)* - ido); - ci3 = wa2[i - 2]*ref(cc,i + (k + 2*l1)*ido) - wa2[i - 1]*ref(cc,i - 1 + (k + 2*l1)* - ido); - cr4 = wa3[i - 2]*ref(cc,i - 1 + (k + 3*l1)*ido) + wa3[i - 1]*ref(cc,i + (k + 3*l1)* - ido); - ci4 = wa3[i - 2]*ref(cc,i + (k + 3*l1)*ido) - wa3[i - 1]*ref(cc,i - 1 + (k + 3*l1)* - ido); - tr1 = cr2 + cr4; - tr4 = cr4 - cr2; - ti1 = ci2 + ci4; - ti4 = ci2 - ci4; - ti2 = ref(cc,i + k*ido) + ci3; - ti3 = ref(cc,i + k*ido) - ci3; - tr2 = ref(cc,i - 1 + k*ido) + cr3; - tr3 = ref(cc,i - 1 + k*ido) - cr3; - ch[i - 1 + 4*k*ido] = tr1 + tr2; - ch[ic - 1 + (4*k + 3)*ido] = tr2 - tr1; - ch[i + 4*k*ido] = ti1 + ti2; - ch[ic + (4*k + 3)*ido] = ti1 - ti2; - ch[i - 1 + (4*k + 2)*ido] = ti4 + tr3; - ch[ic - 1 + (4*k + 1)*ido] = tr3 - ti4; - ch[i + (4*k + 2)*ido] = tr4 + ti3; - ch[ic + (4*k + 1)*ido] = tr4 - ti3; - } - } - if (ido % 2 == 1) return; - } - for (k=0; k<l1; k++) { - ti1 = -hsqt2*(ref(cc,ido-1 + (k + l1)*ido) + ref(cc,ido-1 + (k + 3*l1)*ido)); - tr1 = hsqt2*(ref(cc,ido-1 + (k + l1)*ido) - ref(cc,ido-1 + (k + 3*l1)*ido)); - ch[ido-1 + 4*k*ido] = tr1 + ref(cc,ido-1 + k*ido); - ch[ido-1 + (4*k + 2)*ido] = ref(cc,ido-1 + k*ido) - tr1; - ch[(4*k + 1)*ido] = ti1 - ref(cc,ido-1 + (k + 2*l1)*ido); - ch[(4*k + 3)*ido] = ti1 + ref(cc,ido-1 + (k + 2*l1)*ido); - } - } /* radf4 */ - - -static void radb4(int ido, int l1, const Treal cc[], Treal ch[], - const Treal wa1[], const Treal wa2[], const Treal wa3[]) - { - static const Treal sqrt2 = 1.41421356237309504880; - int i, k, ic; - Treal ci2, ci3, ci4, cr2, cr3, cr4, ti1, ti2, ti3, ti4, tr1, tr2, tr3, tr4; - for (k = 0; k < l1; k++) { - tr1 = ref(cc,4*k*ido) - ref(cc,ido-1 + (4*k + 3)*ido); - tr2 = ref(cc,4*k*ido) + ref(cc,ido-1 + (4*k + 3)*ido); - tr3 = ref(cc,ido-1 + (4*k + 1)*ido) + ref(cc,ido-1 + (4*k + 1)*ido); - tr4 = ref(cc,(4*k + 2)*ido) + ref(cc,(4*k + 2)*ido); - ch[k*ido] = tr2 + tr3; - ch[(k + l1)*ido] = tr1 - tr4; - ch[(k + 2*l1)*ido] = tr2 - tr3; - ch[(k + 3*l1)*ido] = tr1 + tr4; - } - if (ido < 2) return; - if (ido != 2) { - for (k = 0; k < l1; ++k) { - for (i = 2; i < ido; i += 2) { - ic = ido - i; - ti1 = ref(cc,i + 4*k*ido) + ref(cc,ic + (4*k + 3)*ido); - ti2 = ref(cc,i + 4*k*ido) - ref(cc,ic + (4*k + 3)*ido); - ti3 = ref(cc,i + (4*k + 2)*ido) - ref(cc,ic + (4*k + 1)*ido); - tr4 = ref(cc,i + (4*k + 2)*ido) + ref(cc,ic + (4*k + 1)*ido); - tr1 = ref(cc,i - 1 + 4*k*ido) - ref(cc,ic - 1 + (4*k + 3)*ido); - tr2 = ref(cc,i - 1 + 4*k*ido) + ref(cc,ic - 1 + (4*k + 3)*ido); - ti4 = ref(cc,i - 1 + (4*k + 2)*ido) - ref(cc,ic - 1 + (4*k + 1)*ido); - tr3 = ref(cc,i - 1 + (4*k + 2)*ido) + ref(cc,ic - 1 + (4*k + 1)*ido); - ch[i - 1 + k*ido] = tr2 + tr3; - cr3 = tr2 - tr3; - ch[i + k*ido] = ti2 + ti3; - ci3 = ti2 - ti3; - cr2 = tr1 - tr4; - cr4 = tr1 + tr4; - ci2 = ti1 + ti4; - ci4 = ti1 - ti4; - ch[i - 1 + (k + l1)*ido] = wa1[i - 2]*cr2 - wa1[i - 1]*ci2; - ch[i + (k + l1)*ido] = wa1[i - 2]*ci2 + wa1[i - 1]*cr2; - ch[i - 1 + (k + 2*l1)*ido] = wa2[i - 2]*cr3 - wa2[i - 1]*ci3; - ch[i + (k + 2*l1)*ido] = wa2[i - 2]*ci3 + wa2[i - 1]*cr3; - ch[i - 1 + (k + 3*l1)*ido] = wa3[i - 2]*cr4 - wa3[i - 1]*ci4; - ch[i + (k + 3*l1)*ido] = wa3[i - 2]*ci4 + wa3[i - 1]*cr4; - } - } - if (ido % 2 == 1) return; - } - for (k = 0; k < l1; k++) { - ti1 = ref(cc,(4*k + 1)*ido) + ref(cc,(4*k + 3)*ido); - ti2 = ref(cc,(4*k + 3)*ido) - ref(cc,(4*k + 1)*ido); - tr1 = ref(cc,ido-1 + 4*k*ido) - ref(cc,ido-1 + (4*k + 2)*ido); - tr2 = ref(cc,ido-1 + 4*k*ido) + ref(cc,ido-1 + (4*k + 2)*ido); - ch[ido-1 + k*ido] = tr2 + tr2; - ch[ido-1 + (k + l1)*ido] = sqrt2*(tr1 - ti1); - ch[ido-1 + (k + 2*l1)*ido] = ti2 + ti2; - ch[ido-1 + (k + 3*l1)*ido] = -sqrt2*(tr1 + ti1); - } - } /* radb4 */ - - -static void radf5(int ido, int l1, const Treal cc[], Treal ch[], - const Treal wa1[], const Treal wa2[], const Treal wa3[], const Treal wa4[]) - { - static const Treal tr11 = 0.3090169943749474241; - static const Treal ti11 = 0.95105651629515357212; - static const Treal tr12 = -0.8090169943749474241; - static const Treal ti12 = 0.58778525229247312917; - int i, k, ic; - Treal ci2, di2, ci4, ci5, di3, di4, di5, ci3, cr2, cr3, dr2, dr3, dr4, dr5, - cr5, cr4, ti2, ti3, ti5, ti4, tr2, tr3, tr4, tr5; - for (k = 0; k < l1; k++) { - cr2 = ref(cc,(k + 4*l1)*ido) + ref(cc,(k + l1)*ido); - ci5 = ref(cc,(k + 4*l1)*ido) - ref(cc,(k + l1)*ido); - cr3 = ref(cc,(k + 3*l1)*ido) + ref(cc,(k + 2*l1)*ido); - ci4 = ref(cc,(k + 3*l1)*ido) - ref(cc,(k + 2*l1)*ido); - ch[5*k*ido] = ref(cc,k*ido) + cr2 + cr3; - ch[ido-1 + (5*k + 1)*ido] = ref(cc,k*ido) + tr11*cr2 + tr12*cr3; - ch[(5*k + 2)*ido] = ti11*ci5 + ti12*ci4; - ch[ido-1 + (5*k + 3)*ido] = ref(cc,k*ido) + tr12*cr2 + tr11*cr3; - ch[(5*k + 4)*ido] = ti12*ci5 - ti11*ci4; - } - if (ido == 1) return; - for (k = 0; k < l1; ++k) { - for (i = 2; i < ido; i += 2) { - ic = ido - i; - dr2 = wa1[i - 2]*ref(cc,i - 1 + (k + l1)*ido) + wa1[i - 1]*ref(cc,i + (k + l1)*ido); - di2 = wa1[i - 2]*ref(cc,i + (k + l1)*ido) - wa1[i - 1]*ref(cc,i - 1 + (k + l1)*ido); - dr3 = wa2[i - 2]*ref(cc,i - 1 + (k + 2*l1)*ido) + wa2[i - 1]*ref(cc,i + (k + 2*l1)*ido); - di3 = wa2[i - 2]*ref(cc,i + (k + 2*l1)*ido) - wa2[i - 1]*ref(cc,i - 1 + (k + 2*l1)*ido); - dr4 = wa3[i - 2]*ref(cc,i - 1 + (k + 3*l1)*ido) + wa3[i - 1]*ref(cc,i + (k + 3*l1)*ido); - di4 = wa3[i - 2]*ref(cc,i + (k + 3*l1)*ido) - wa3[i - 1]*ref(cc,i - 1 + (k + 3*l1)*ido); - dr5 = wa4[i - 2]*ref(cc,i - 1 + (k + 4*l1)*ido) + wa4[i - 1]*ref(cc,i + (k + 4*l1)*ido); - di5 = wa4[i - 2]*ref(cc,i + (k + 4*l1)*ido) - wa4[i - 1]*ref(cc,i - 1 + (k + 4*l1)*ido); - cr2 = dr2 + dr5; - ci5 = dr5 - dr2; - cr5 = di2 - di5; - ci2 = di2 + di5; - cr3 = dr3 + dr4; - ci4 = dr4 - dr3; - cr4 = di3 - di4; - ci3 = di3 + di4; - ch[i - 1 + 5*k*ido] = ref(cc,i - 1 + k*ido) + cr2 + cr3; - ch[i + 5*k*ido] = ref(cc,i + k*ido) + ci2 + ci3; - tr2 = ref(cc,i - 1 + k*ido) + tr11*cr2 + tr12*cr3; - ti2 = ref(cc,i + k*ido) + tr11*ci2 + tr12*ci3; - tr3 = ref(cc,i - 1 + k*ido) + tr12*cr2 + tr11*cr3; - ti3 = ref(cc,i + k*ido) + tr12*ci2 + tr11*ci3; - tr5 = ti11*cr5 + ti12*cr4; - ti5 = ti11*ci5 + ti12*ci4; - tr4 = ti12*cr5 - ti11*cr4; - ti4 = ti12*ci5 - ti11*ci4; - ch[i - 1 + (5*k + 2)*ido] = tr2 + tr5; - ch[ic - 1 + (5*k + 1)*ido] = tr2 - tr5; - ch[i + (5*k + 2)*ido] = ti2 + ti5; - ch[ic + (5*k + 1)*ido] = ti5 - ti2; - ch[i - 1 + (5*k + 4)*ido] = tr3 + tr4; - ch[ic - 1 + (5*k + 3)*ido] = tr3 - tr4; - ch[i + (5*k + 4)*ido] = ti3 + ti4; - ch[ic + (5*k + 3)*ido] = ti4 - ti3; - } - } - } /* radf5 */ - - -static void radb5(int ido, int l1, const Treal cc[], Treal ch[], - const Treal wa1[], const Treal wa2[], const Treal wa3[], const Treal wa4[]) - { - static const Treal tr11 = 0.3090169943749474241; - static const Treal ti11 = 0.95105651629515357212; - static const Treal tr12 = -0.8090169943749474241; - static const Treal ti12 = 0.58778525229247312917; - int i, k, ic; - Treal ci2, ci3, ci4, ci5, di3, di4, di5, di2, cr2, cr3, cr5, cr4, ti2, ti3, - ti4, ti5, dr3, dr4, dr5, dr2, tr2, tr3, tr4, tr5; - for (k = 0; k < l1; k++) { - ti5 = 2*ref(cc,(5*k + 2)*ido); - ti4 = 2*ref(cc,(5*k + 4)*ido); - tr2 = 2*ref(cc,ido-1 + (5*k + 1)*ido); - tr3 = 2*ref(cc,ido-1 + (5*k + 3)*ido); - ch[k*ido] = ref(cc,5*k*ido) + tr2 + tr3; - cr2 = ref(cc,5*k*ido) + tr11*tr2 + tr12*tr3; - cr3 = ref(cc,5*k*ido) + tr12*tr2 + tr11*tr3; - ci5 = ti11*ti5 + ti12*ti4; - ci4 = ti12*ti5 - ti11*ti4; - ch[(k + l1)*ido] = cr2 - ci5; - ch[(k + 2*l1)*ido] = cr3 - ci4; - ch[(k + 3*l1)*ido] = cr3 + ci4; - ch[(k + 4*l1)*ido] = cr2 + ci5; - } - if (ido == 1) return; - for (k = 0; k < l1; ++k) { - for (i = 2; i < ido; i += 2) { - ic = ido - i; - ti5 = ref(cc,i + (5*k + 2)*ido) + ref(cc,ic + (5*k + 1)*ido); - ti2 = ref(cc,i + (5*k + 2)*ido) - ref(cc,ic + (5*k + 1)*ido); - ti4 = ref(cc,i + (5*k + 4)*ido) + ref(cc,ic + (5*k + 3)*ido); - ti3 = ref(cc,i + (5*k + 4)*ido) - ref(cc,ic + (5*k + 3)*ido); - tr5 = ref(cc,i - 1 + (5*k + 2)*ido) - ref(cc,ic - 1 + (5*k + 1)*ido); - tr2 = ref(cc,i - 1 + (5*k + 2)*ido) + ref(cc,ic - 1 + (5*k + 1)*ido); - tr4 = ref(cc,i - 1 + (5*k + 4)*ido) - ref(cc,ic - 1 + (5*k + 3)*ido); - tr3 = ref(cc,i - 1 + (5*k + 4)*ido) + ref(cc,ic - 1 + (5*k + 3)*ido); - ch[i - 1 + k*ido] = ref(cc,i - 1 + 5*k*ido) + tr2 + tr3; - ch[i + k*ido] = ref(cc,i + 5*k*ido) + ti2 + ti3; - cr2 = ref(cc,i - 1 + 5*k*ido) + tr11*tr2 + tr12*tr3; - - ci2 = ref(cc,i + 5*k*ido) + tr11*ti2 + tr12*ti3; - cr3 = ref(cc,i - 1 + 5*k*ido) + tr12*tr2 + tr11*tr3; - - ci3 = ref(cc,i + 5*k*ido) + tr12*ti2 + tr11*ti3; - cr5 = ti11*tr5 + ti12*tr4; - ci5 = ti11*ti5 + ti12*ti4; - cr4 = ti12*tr5 - ti11*tr4; - ci4 = ti12*ti5 - ti11*ti4; - dr3 = cr3 - ci4; - dr4 = cr3 + ci4; - di3 = ci3 + cr4; - di4 = ci3 - cr4; - dr5 = cr2 + ci5; - dr2 = cr2 - ci5; - di5 = ci2 - cr5; - di2 = ci2 + cr5; - ch[i - 1 + (k + l1)*ido] = wa1[i - 2]*dr2 - wa1[i - 1]*di2; - ch[i + (k + l1)*ido] = wa1[i - 2]*di2 + wa1[i - 1]*dr2; - ch[i - 1 + (k + 2*l1)*ido] = wa2[i - 2]*dr3 - wa2[i - 1]*di3; - ch[i + (k + 2*l1)*ido] = wa2[i - 2]*di3 + wa2[i - 1]*dr3; - ch[i - 1 + (k + 3*l1)*ido] = wa3[i - 2]*dr4 - wa3[i - 1]*di4; - ch[i + (k + 3*l1)*ido] = wa3[i - 2]*di4 + wa3[i - 1]*dr4; - ch[i - 1 + (k + 4*l1)*ido] = wa4[i - 2]*dr5 - wa4[i - 1]*di5; - ch[i + (k + 4*l1)*ido] = wa4[i - 2]*di5 + wa4[i - 1]*dr5; - } - } - } /* radb5 */ - - -static void radfg(int ido, int ip, int l1, int idl1, - Treal cc[], Treal ch[], const Treal wa[]) - { - int idij, ipph, i, j, k, l, j2, ic, jc, lc, ik, is, nbd; - Treal dc2, ai1, ai2, ar1, ar2, ds2, dcp, dsp, ar1h, ar2h; - sincos2pi(1, ip, &dsp, &dcp); - ipph = (ip + 1) / 2; - nbd = (ido - 1) / 2; - if (ido != 1) { - for (ik=0; ik<idl1; ik++) ch[ik] = cc[ik]; - for (j=1; j<ip; j++) - for (k=0; k<l1; k++) - ch[(k + j*l1)*ido] = cc[(k + j*l1)*ido]; - if (nbd <= l1) { - is = -ido; - for (j=1; j<ip; j++) { - is += ido; - idij = is-1; - for (i=2; i<ido; i+=2) { - idij += 2; - for (k=0; k<l1; k++) { - ch[i - 1 + (k + j*l1)*ido] = - wa[idij - 1]*cc[i - 1 + (k + j*l1)*ido] + wa[idij]*cc[i + (k + j*l1)*ido]; - ch[i + (k + j*l1)*ido] = - wa[idij - 1]*cc[i + (k + j*l1)*ido] - wa[idij]*cc[i - 1 + (k + j*l1)*ido]; - } - } - } - } else { - is = -ido; - for (j=1; j<ip; j++) { - is += ido; - for (k=0; k<l1; k++) { - idij = is-1; - for (i=2; i<ido; i+=2) { - idij += 2; - ch[i - 1 + (k + j*l1)*ido] = - wa[idij - 1]*cc[i - 1 + (k + j*l1)*ido] + wa[idij]*cc[i + (k + j*l1)*ido]; - ch[i + (k + j*l1)*ido] = - wa[idij - 1]*cc[i + (k + j*l1)*ido] - wa[idij]*cc[i - 1 + (k + j*l1)*ido]; - } - } - } - } - if (nbd >= l1) { - for (j=1; j<ipph; j++) { - jc = ip - j; - for (k=0; k<l1; k++) { - for (i=2; i<ido; i+=2) { - cc[i - 1 + (k + j*l1)*ido] = ch[i - 1 + (k + j*l1)*ido] + ch[i - 1 + (k + jc*l1)*ido]; - cc[i - 1 + (k + jc*l1)*ido] = ch[i + (k + j*l1)*ido] - ch[i + (k + jc*l1)*ido]; - cc[i + (k + j*l1)*ido] = ch[i + (k + j*l1)*ido] + ch[i + (k + jc*l1)*ido]; - cc[i + (k + jc*l1)*ido] = ch[i - 1 + (k + jc*l1)*ido] - ch[i - 1 + (k + j*l1)*ido]; - } - } - } - } else { - for (j=1; j<ipph; j++) { - jc = ip - j; - for (i=2; i<ido; i+=2) { - for (k=0; k<l1; k++) { - cc[i - 1 + (k + j*l1)*ido] = - ch[i - 1 + (k + j*l1)*ido] + ch[i - 1 + (k + jc*l1)*ido]; - cc[i - 1 + (k + jc*l1)*ido] = ch[i + (k + j*l1)*ido] - ch[i + (k + jc*l1)*ido]; - cc[i + (k + j*l1)*ido] = ch[i + (k + j*l1)*ido] + ch[i + (k + jc*l1)*ido]; - cc[i + (k + jc*l1)*ido] = ch[i - 1 + (k + jc*l1)*ido] - ch[i - 1 + (k + j*l1)*ido]; - } - } - } - } - } else { /* now ido == 1 */ - for (ik=0; ik<idl1; ik++) cc[ik] = ch[ik]; - } - for (j=1; j<ipph; j++) { - jc = ip - j; - for (k=0; k<l1; k++) { - cc[(k + j*l1)*ido] = ch[(k + j*l1)*ido] + ch[(k + jc*l1)*ido]; - cc[(k + jc*l1)*ido] = ch[(k + jc*l1)*ido] - ch[(k + j*l1)*ido]; - } - } - - ar1 = 1; - ai1 = 0; - for (l=1; l<ipph; l++) { - lc = ip - l; - ar1h = dcp*ar1 - dsp*ai1; - ai1 = dcp*ai1 + dsp*ar1; - ar1 = ar1h; - for (ik=0; ik<idl1; ik++) { - ch[ik + l*idl1] = cc[ik] + ar1*cc[ik + idl1]; - ch[ik + lc*idl1] = ai1*cc[ik + (ip-1)*idl1]; - } - dc2 = ar1; - ds2 = ai1; - ar2 = ar1; - ai2 = ai1; - for (j=2; j<ipph; j++) { - jc = ip - j; - ar2h = dc2*ar2 - ds2*ai2; - ai2 = dc2*ai2 + ds2*ar2; - ar2 = ar2h; - for (ik=0; ik<idl1; ik++) { - ch[ik + l*idl1] += ar2*cc[ik + j*idl1]; - ch[ik + lc*idl1] += ai2*cc[ik + jc*idl1]; - } - } - } - - for (j=1; j<ipph; j++) - for (ik=0; ik<idl1; ik++) - ch[ik] += cc[ik + j*idl1]; - - if (ido >= l1) { - for (k=0; k<l1; k++) { - for (i=0; i<ido; i++) { - ref(cc,i + k*ip*ido) = ch[i + k*ido]; - } - } - } else { - for (i=0; i<ido; i++) { - for (k=0; k<l1; k++) { - ref(cc,i + k*ip*ido) = ch[i + k*ido]; - } - } - } - for (j=1; j<ipph; j++) { - jc = ip - j; - j2 = 2*j; - for (k=0; k<l1; k++) { - ref(cc,ido-1 + (j2 - 1 + k*ip)*ido) = - ch[(k + j*l1)*ido]; - ref(cc,(j2 + k*ip)*ido) = - ch[(k + jc*l1)*ido]; - } - } - if (ido == 1) return; - if (nbd >= l1) { - for (j=1; j<ipph; j++) { - jc = ip - j; - j2 = 2*j; - for (k=0; k<l1; k++) { - for (i=2; i<ido; i+=2) { - ic = ido - i; - ref(cc,i - 1 + (j2 + k*ip)*ido) = ch[i - 1 + (k + j*l1)*ido] + ch[i - 1 + (k + jc*l1)*ido]; - ref(cc,ic - 1 + (j2 - 1 + k*ip)*ido) = ch[i - 1 + (k + j*l1)*ido] - ch[i - 1 + (k + jc*l1)*ido]; - ref(cc,i + (j2 + k*ip)*ido) = ch[i + (k + j*l1)*ido] + ch[i + (k + jc*l1)*ido]; - ref(cc,ic + (j2 - 1 + k*ip)*ido) = ch[i + (k + jc*l1)*ido] - ch[i + (k + j*l1)*ido]; - } - } - } - } else { - for (j=1; j<ipph; j++) { - jc = ip - j; - j2 = 2*j; - for (i=2; i<ido; i+=2) { - ic = ido - i; - for (k=0; k<l1; k++) { - ref(cc,i - 1 + (j2 + k*ip)*ido) = ch[i - 1 + (k + j*l1)*ido] + ch[i - 1 + (k + jc*l1)*ido]; - ref(cc,ic - 1 + (j2 - 1 + k*ip)*ido) = ch[i - 1 + (k + j*l1)*ido] - ch[i - 1 + (k + jc*l1)*ido]; - ref(cc,i + (j2 + k*ip)*ido) = ch[i + (k + j*l1)*ido] + ch[i + (k + jc*l1)*ido]; - ref(cc,ic + (j2 - 1 + k*ip)*ido) = ch[i + (k + jc*l1)*ido] - ch[i + (k + j*l1)*ido]; - } - } - } - } - } /* radfg */ - - -static void radbg(int ido, int ip, int l1, int idl1, - Treal cc[], Treal ch[], const Treal wa[]) - { - int idij, ipph, i, j, k, l, j2, ic, jc, lc, ik, is; - Treal dc2, ai1, ai2, ar1, ar2, ds2; - int nbd; - Treal dcp, dsp, ar1h, ar2h; - sincos2pi(1, ip, &dsp, &dcp); - nbd = (ido - 1) / 2; - ipph = (ip + 1) / 2; - if (ido >= l1) { - for (k=0; k<l1; k++) { - for (i=0; i<ido; i++) { - ch[i + k*ido] = ref(cc,i + k*ip*ido); - } - } - } else { - for (i=0; i<ido; i++) { - for (k=0; k<l1; k++) { - ch[i + k*ido] = ref(cc,i + k*ip*ido); - } - } - } - for (j=1; j<ipph; j++) { - jc = ip - j; - j2 = 2*j; - for (k=0; k<l1; k++) { - ch[(k + j*l1)*ido] = ref(cc,ido-1 + (j2 - 1 + k*ip)*ido) + ref(cc,ido-1 + (j2 - 1 + k*ip)* - ido); - ch[(k + jc*l1)*ido] = ref(cc,(j2 + k*ip)*ido) + ref(cc,(j2 + k*ip)*ido); - } - } - - if (ido != 1) { - if (nbd >= l1) { - for (j=1; j<ipph; j++) { - jc = ip - j; - for (k=0; k<l1; k++) { - for (i=2; i<ido; i+=2) { - ic = ido - i; - ch[i - 1 + (k + j*l1)*ido] = ref(cc,i - 1 + (2*j + k*ip)*ido) + ref(cc, - ic - 1 + (2*j - 1 + k*ip)*ido); - ch[i - 1 + (k + jc*l1)*ido] = ref(cc,i - 1 + (2*j + k*ip)*ido) - - ref(cc,ic - 1 + (2*j - 1 + k*ip)*ido); - ch[i + (k + j*l1)*ido] = ref(cc,i + (2*j + k*ip)*ido) - ref(cc,ic - + (2*j - 1 + k*ip)*ido); - ch[i + (k + jc*l1)*ido] = ref(cc,i + (2*j + k*ip)*ido) + ref(cc,ic - + (2*j - 1 + k*ip)*ido); - } - } - } - } else { - for (j=1; j<ipph; j++) { - jc = ip - j; - for (i=2; i<ido; i+=2) { - ic = ido - i; - for (k=0; k<l1; k++) { - ch[i - 1 + (k + j*l1)*ido] = ref(cc,i - 1 + (2*j + k*ip)*ido) + ref(cc, - ic - 1 + (2*j - 1 + k*ip)*ido); - ch[i - 1 + (k + jc*l1)*ido] = ref(cc,i - 1 + (2*j + k*ip)*ido) - - ref(cc,ic - 1 + (2*j - 1 + k*ip)*ido); - ch[i + (k + j*l1)*ido] = ref(cc,i + (2*j + k*ip)*ido) - ref(cc,ic - + (2*j - 1 + k*ip)*ido); - ch[i + (k + jc*l1)*ido] = ref(cc,i + (2*j + k*ip)*ido) + ref(cc,ic - + (2*j - 1 + k*ip)*ido); - } - } - } - } - } - - ar1 = 1; - ai1 = 0; - for (l=1; l<ipph; l++) { - lc = ip - l; - ar1h = dcp*ar1 - dsp*ai1; - ai1 = dcp*ai1 + dsp*ar1; - ar1 = ar1h; - for (ik=0; ik<idl1; ik++) { - cc[ik + l*idl1] = ch[ik] + ar1*ch[ik + idl1]; - cc[ik + lc*idl1] = ai1*ch[ik + (ip-1)*idl1]; - } - dc2 = ar1; - ds2 = ai1; - ar2 = ar1; - ai2 = ai1; - for (j=2; j<ipph; j++) { - jc = ip - j; - ar2h = dc2*ar2 - ds2*ai2; - ai2 = dc2*ai2 + ds2*ar2; - ar2 = ar2h; - for (ik=0; ik<idl1; ik++) { - cc[ik + l*idl1] += ar2*ch[ik + j*idl1]; - cc[ik + lc*idl1] += ai2*ch[ik + jc*idl1]; - } - } - } - for (j=1; j<ipph; j++) { - for (ik=0; ik<idl1; ik++) { - ch[ik] += ch[ik + j*idl1]; - } - } - for (j=1; j<ipph; j++) { - jc = ip - j; - for (k=0; k<l1; k++) { - ch[(k + j*l1)*ido] = cc[(k + j*l1)*ido] - cc[(k + jc*l1)*ido]; - ch[(k + jc*l1)*ido] = cc[(k + j*l1)*ido] + cc[(k + jc*l1)*ido]; - } - } - - if (ido == 1) return; - if (nbd >= l1) { - for (j=1; j<ipph; j++) { - jc = ip - j; - for (k=0; k<l1; k++) { - for (i=2; i<ido; i+=2) { - ch[i - 1 + (k + j*l1)*ido] = cc[i - 1 + (k + j*l1)*ido] - cc[i + (k + jc*l1)*ido]; - ch[i - 1 + (k + jc*l1)*ido] = cc[i - 1 + (k + j*l1)*ido] + cc[i + (k + jc*l1)*ido]; - ch[i + (k + j*l1)*ido] = cc[i + (k + j*l1)*ido] + cc[i - 1 + (k + jc*l1)*ido]; - ch[i + (k + jc*l1)*ido] = cc[i + (k + j*l1)*ido] - cc[i - 1 + (k + jc*l1)*ido]; - } - } - } - } else { - for (j=1; j<ipph; j++) { - jc = ip - j; - for (i=2; i<ido; i+=2) { - for (k=0; k<l1; k++) { - ch[i - 1 + (k + j*l1)*ido] = cc[i - 1 + (k + j*l1)*ido] - cc[i + (k + jc*l1)*ido]; - ch[i - 1 + (k + jc*l1)*ido] = cc[i - 1 + (k + j *l1)*ido] + cc[i + (k + jc*l1)*ido]; - ch[i + (k + j*l1)*ido] = cc[i + (k + j*l1)*ido] + cc[i - 1 + (k + jc*l1)*ido]; - ch[i + (k + jc*l1)*ido] = cc[i + (k + j*l1)*ido] - cc[i - 1 + (k + jc*l1)*ido]; - } - } - } - } - for (ik=0; ik<idl1; ik++) cc[ik] = ch[ik]; - for (j=1; j<ip; j++) - for (k=0; k<l1; k++) - cc[(k + j*l1)*ido] = ch[(k + j*l1)*ido]; - if (nbd <= l1) { - is = -ido; - for (j=1; j<ip; j++) { - is += ido; - idij = is-1; - for (i=2; i<ido; i+=2) { - idij += 2; - for (k=0; k<l1; k++) { - cc[i - 1 + (k + j*l1)*ido] = wa[idij - 1]*ch[i - 1 + (k + j*l1)*ido] - wa[idij]* - ch[i + (k + j*l1)*ido]; - cc[i + (k + j*l1)*ido] = wa[idij - 1]*ch[i + (k + j*l1)*ido] + wa[idij]*ch[i - 1 + (k + j*l1)*ido]; - } - } - } - } else { - is = -ido; - for (j=1; j<ip; j++) { - is += ido; - for (k=0; k<l1; k++) { - idij = is - 1; - for (i=2; i<ido; i+=2) { - idij += 2; - cc[i - 1 + (k + j*l1)*ido] = wa[idij-1]*ch[i - 1 + (k + j*l1)*ido] - wa[idij]* - ch[i + (k + j*l1)*ido]; - cc[i + (k + j*l1)*ido] = wa[idij-1]*ch[i + (k + j*l1)*ido] + wa[idij]*ch[i - 1 + (k + j*l1)*ido]; - } - } - } - } - } /* radbg */ - - /* ------------------------------------------------------------ -cfftf1, npy_cfftf, npy_cfftb, cffti1, npy_cffti. Complex FFTs. ---------------------------------------------------------------- */ - -static void cfftf1(int n, Treal c[], Treal ch[], const Treal wa[], const int ifac[MAXFAC+2], int isign) - { - int idot, i; - int k1, l1, l2; - int na, nf, ip, iw, ix2, ix3, ix4, nac, ido, idl1; - Treal *cinput, *coutput; - nf = ifac[1]; - na = 0; - l1 = 1; - iw = 0; - for (k1=2; k1<=nf+1; k1++) { - ip = ifac[k1]; - l2 = ip*l1; - ido = n / l2; - idot = ido + ido; - idl1 = idot*l1; - if (na) { - cinput = ch; - coutput = c; - } else { - cinput = c; - coutput = ch; - } - switch (ip) { - case 4: - ix2 = iw + idot; - ix3 = ix2 + idot; - passf4(idot, l1, cinput, coutput, &wa[iw], &wa[ix2], &wa[ix3], isign); - na = !na; - break; - case 2: - passf2(idot, l1, cinput, coutput, &wa[iw], isign); - na = !na; - break; - case 3: - ix2 = iw + idot; - passf3(idot, l1, cinput, coutput, &wa[iw], &wa[ix2], isign); - na = !na; - break; - case 5: - ix2 = iw + idot; - ix3 = ix2 + idot; - ix4 = ix3 + idot; - passf5(idot, l1, cinput, coutput, &wa[iw], &wa[ix2], &wa[ix3], &wa[ix4], isign); - na = !na; - break; - default: - passf(&nac, idot, ip, l1, idl1, cinput, coutput, &wa[iw], isign); - if (nac != 0) na = !na; - } - l1 = l2; - iw += (ip - 1)*idot; - } - if (na == 0) return; - for (i=0; i<2*n; i++) c[i] = ch[i]; - } /* cfftf1 */ - - -NPY_VISIBILITY_HIDDEN void npy_cfftf(int n, Treal c[], Treal wsave[]) - { - int iw1, iw2; - if (n == 1) return; - iw1 = 2*n; - iw2 = iw1 + 2*n; - cfftf1(n, c, wsave, wsave+iw1, (int*)(wsave+iw2), -1); - } /* npy_cfftf */ - - -NPY_VISIBILITY_HIDDEN void npy_cfftb(int n, Treal c[], Treal wsave[]) - { - int iw1, iw2; - if (n == 1) return; - iw1 = 2*n; - iw2 = iw1 + 2*n; - cfftf1(n, c, wsave, wsave+iw1, (int*)(wsave+iw2), +1); - } /* npy_cfftb */ - - -static void factorize(int n, int ifac[MAXFAC+2], const int ntryh[NSPECIAL]) - /* Factorize n in factors in ntryh and rest. On exit, -ifac[0] contains n and ifac[1] contains number of factors, -the factors start from ifac[2]. */ - { - int ntry=3, i, j=0, ib, nf=0, nl=n, nq, nr; -startloop: - if (j < NSPECIAL) - ntry = ntryh[j]; - else - ntry+= 2; - j++; - do { - nq = nl / ntry; - nr = nl - ntry*nq; - if (nr != 0) goto startloop; - nf++; - ifac[nf + 1] = ntry; - nl = nq; - if (ntry == 2 && nf != 1) { - for (i=2; i<=nf; i++) { - ib = nf - i + 2; - ifac[ib + 1] = ifac[ib]; - } - ifac[2] = 2; - } - } while (nl != 1); - ifac[0] = n; - ifac[1] = nf; - } - - -static void cffti1(int n, Treal wa[], int ifac[MAXFAC+2]) - { - int fi, idot, i, j; - int i1, k1, l1, l2; - int ld, ii, nf, ip; - int ido, ipm; - - static const int ntryh[NSPECIAL] = { - 3,4,2,5 }; /* Do not change the order of these. */ - - factorize(n,ifac,ntryh); - nf = ifac[1]; - i = 1; - l1 = 1; - for (k1=1; k1<=nf; k1++) { - ip = ifac[k1+1]; - ld = 0; - l2 = l1*ip; - ido = n / l2; - idot = ido + ido + 2; - ipm = ip - 1; - for (j=1; j<=ipm; j++) { - i1 = i; - wa[i-1] = 1; - wa[i] = 0; - ld += l1; - fi = 0; - for (ii=4; ii<=idot; ii+=2) { - i+= 2; - fi+= 1; - sincos2pi(fi*ld, n, wa+i, wa+i-1); - } - if (ip > 5) { - wa[i1-1] = wa[i-1]; - wa[i1] = wa[i]; - } - } - l1 = l2; - } - } /* cffti1 */ - - -NPY_VISIBILITY_HIDDEN void npy_cffti(int n, Treal wsave[]) - { - int iw1, iw2; - if (n == 1) return; - iw1 = 2*n; - iw2 = iw1 + 2*n; - cffti1(n, wsave+iw1, (int*)(wsave+iw2)); - } /* npy_cffti */ - - /* ------------------------------------------------------------------- -rfftf1, rfftb1, npy_rfftf, npy_rfftb, rffti1, npy_rffti. Treal FFTs. ----------------------------------------------------------------------- */ - -static void rfftf1(int n, Treal c[], Treal ch[], const Treal wa[], const int ifac[MAXFAC+2]) - { - int i; - int k1, l1, l2, na, kh, nf, ip, iw, ix2, ix3, ix4, ido, idl1; - Treal *cinput, *coutput; - nf = ifac[1]; - na = 1; - l2 = n; - iw = n-1; - for (k1 = 1; k1 <= nf; ++k1) { - kh = nf - k1; - ip = ifac[kh + 2]; - l1 = l2 / ip; - ido = n / l2; - idl1 = ido*l1; - iw -= (ip - 1)*ido; - na = !na; - if (na) { - cinput = ch; - coutput = c; - } else { - cinput = c; - coutput = ch; - } - switch (ip) { - case 4: - ix2 = iw + ido; - ix3 = ix2 + ido; - radf4(ido, l1, cinput, coutput, &wa[iw], &wa[ix2], &wa[ix3]); - break; - case 2: - radf2(ido, l1, cinput, coutput, &wa[iw]); - break; - case 3: - ix2 = iw + ido; - radf3(ido, l1, cinput, coutput, &wa[iw], &wa[ix2]); - break; - case 5: - ix2 = iw + ido; - ix3 = ix2 + ido; - ix4 = ix3 + ido; - radf5(ido, l1, cinput, coutput, &wa[iw], &wa[ix2], &wa[ix3], &wa[ix4]); - break; - default: - if (ido == 1) - na = !na; - if (na == 0) { - radfg(ido, ip, l1, idl1, c, ch, &wa[iw]); - na = 1; - } else { - radfg(ido, ip, l1, idl1, ch, c, &wa[iw]); - na = 0; - } - } - l2 = l1; - } - if (na == 1) return; - for (i = 0; i < n; i++) c[i] = ch[i]; - } /* rfftf1 */ - - -static void rfftb1(int n, Treal c[], Treal ch[], const Treal wa[], const int ifac[MAXFAC+2]) - { - int i; - int k1, l1, l2, na, nf, ip, iw, ix2, ix3, ix4, ido, idl1; - Treal *cinput, *coutput; - nf = ifac[1]; - na = 0; - l1 = 1; - iw = 0; - for (k1=1; k1<=nf; k1++) { - ip = ifac[k1 + 1]; - l2 = ip*l1; - ido = n / l2; - idl1 = ido*l1; - if (na) { - cinput = ch; - coutput = c; - } else { - cinput = c; - coutput = ch; - } - switch (ip) { - case 4: - ix2 = iw + ido; - ix3 = ix2 + ido; - radb4(ido, l1, cinput, coutput, &wa[iw], &wa[ix2], &wa[ix3]); - na = !na; - break; - case 2: - radb2(ido, l1, cinput, coutput, &wa[iw]); - na = !na; - break; - case 3: - ix2 = iw + ido; - radb3(ido, l1, cinput, coutput, &wa[iw], &wa[ix2]); - na = !na; - break; - case 5: - ix2 = iw + ido; - ix3 = ix2 + ido; - ix4 = ix3 + ido; - radb5(ido, l1, cinput, coutput, &wa[iw], &wa[ix2], &wa[ix3], &wa[ix4]); - na = !na; - break; - default: - radbg(ido, ip, l1, idl1, cinput, coutput, &wa[iw]); - if (ido == 1) na = !na; - } - l1 = l2; - iw += (ip - 1)*ido; - } - if (na == 0) return; - for (i=0; i<n; i++) c[i] = ch[i]; - } /* rfftb1 */ - - -NPY_VISIBILITY_HIDDEN void npy_rfftf(int n, Treal r[], Treal wsave[]) - { - if (n == 1) return; - rfftf1(n, r, wsave, wsave+n, (int*)(wsave+2*n)); - } /* npy_rfftf */ - - -NPY_VISIBILITY_HIDDEN void npy_rfftb(int n, Treal r[], Treal wsave[]) - { - if (n == 1) return; - rfftb1(n, r, wsave, wsave+n, (int*)(wsave+2*n)); - } /* npy_rfftb */ - - -static void rffti1(int n, Treal wa[], int ifac[MAXFAC+2]) - { - int fi, i, j; - int k1, l1, l2; - int ld, ii, nf, ip, is; - int ido, ipm, nfm1; - static const int ntryh[NSPECIAL] = { - 4,2,3,5 }; /* Do not change the order of these. */ - factorize(n,ifac,ntryh); - nf = ifac[1]; - is = 0; - nfm1 = nf - 1; - l1 = 1; - if (nfm1 == 0) return; - for (k1 = 1; k1 <= nfm1; k1++) { - ip = ifac[k1 + 1]; - ld = 0; - l2 = l1*ip; - ido = n / l2; - ipm = ip - 1; - for (j = 1; j <= ipm; ++j) { - ld += l1; - i = is; - fi = 0; - for (ii = 3; ii <= ido; ii += 2) { - i += 2; - fi += 1; - sincos2pi(fi*ld, n, wa+i-1, wa+i-2); - } - is += ido; - } - l1 = l2; - } - } /* rffti1 */ - - -NPY_VISIBILITY_HIDDEN void npy_rffti(int n, Treal wsave[]) - { - if (n == 1) return; - rffti1(n, wsave+n, (int*)(wsave+2*n)); - } /* npy_rffti */ - -#ifdef __cplusplus -} -#endif diff --git a/numpy/fft/fftpack.h b/numpy/fft/fftpack.h deleted file mode 100644 index 5e8f4631c..000000000 --- a/numpy/fft/fftpack.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * This file is part of tela the Tensor Language. - * Copyright (c) 1994-1995 Pekka Janhunen - */ - -#ifdef __cplusplus -extern "C" { -#endif - -#define DOUBLE - -#ifdef DOUBLE -#define Treal double -#else -#define Treal float -#endif - -extern NPY_VISIBILITY_HIDDEN void npy_cfftf(int N, Treal data[], const Treal wrk[]); -extern NPY_VISIBILITY_HIDDEN void npy_cfftb(int N, Treal data[], const Treal wrk[]); -extern NPY_VISIBILITY_HIDDEN void npy_cffti(int N, Treal wrk[]); - -extern NPY_VISIBILITY_HIDDEN void npy_rfftf(int N, Treal data[], const Treal wrk[]); -extern NPY_VISIBILITY_HIDDEN void npy_rfftb(int N, Treal data[], const Treal wrk[]); -extern NPY_VISIBILITY_HIDDEN void npy_rffti(int N, Treal wrk[]); - -#ifdef __cplusplus -} -#endif diff --git a/numpy/fft/fftpack_litemodule.c b/numpy/fft/fftpack_litemodule.c deleted file mode 100644 index bd6cfc120..000000000 --- a/numpy/fft/fftpack_litemodule.c +++ /dev/null @@ -1,366 +0,0 @@ -#define NPY_NO_DEPRECATED_API NPY_API_VERSION - -#include "Python.h" -#include "numpy/arrayobject.h" -#include "fftpack.h" - -static PyObject *ErrorObject; - -static const char fftpack_cfftf__doc__[] = ""; - -static PyObject * -fftpack_cfftf(PyObject *NPY_UNUSED(self), PyObject *args) -{ - PyObject *op1, *op2; - PyArrayObject *data; - PyArray_Descr *descr; - double *wsave, *dptr; - npy_intp nsave; - int npts, nrepeats, i; - - if(!PyArg_ParseTuple(args, "OO:cfftf", &op1, &op2)) { - return NULL; - } - data = (PyArrayObject *)PyArray_CopyFromObject(op1, - NPY_CDOUBLE, 1, 0); - if (data == NULL) { - return NULL; - } - descr = PyArray_DescrFromType(NPY_DOUBLE); - if (PyArray_AsCArray(&op2, (void *)&wsave, &nsave, 1, descr) == -1) { - goto fail; - } - if (data == NULL) { - goto fail; - } - - npts = PyArray_DIM(data, PyArray_NDIM(data) - 1); - if (nsave != npts*4 + 15) { - PyErr_SetString(ErrorObject, "invalid work array for fft size"); - goto fail; - } - - nrepeats = PyArray_SIZE(data)/npts; - dptr = (double *)PyArray_DATA(data); - Py_BEGIN_ALLOW_THREADS; - NPY_SIGINT_ON; - for (i = 0; i < nrepeats; i++) { - npy_cfftf(npts, dptr, wsave); - dptr += npts*2; - } - NPY_SIGINT_OFF; - Py_END_ALLOW_THREADS; - PyArray_Free(op2, (char *)wsave); - return (PyObject *)data; - -fail: - PyArray_Free(op2, (char *)wsave); - Py_DECREF(data); - return NULL; -} - -static const char fftpack_cfftb__doc__[] = ""; - -static PyObject * -fftpack_cfftb(PyObject *NPY_UNUSED(self), PyObject *args) -{ - PyObject *op1, *op2; - PyArrayObject *data; - PyArray_Descr *descr; - double *wsave, *dptr; - npy_intp nsave; - int npts, nrepeats, i; - - if(!PyArg_ParseTuple(args, "OO:cfftb", &op1, &op2)) { - return NULL; - } - data = (PyArrayObject *)PyArray_CopyFromObject(op1, - NPY_CDOUBLE, 1, 0); - if (data == NULL) { - return NULL; - } - descr = PyArray_DescrFromType(NPY_DOUBLE); - if (PyArray_AsCArray(&op2, (void *)&wsave, &nsave, 1, descr) == -1) { - goto fail; - } - if (data == NULL) { - goto fail; - } - - npts = PyArray_DIM(data, PyArray_NDIM(data) - 1); - if (nsave != npts*4 + 15) { - PyErr_SetString(ErrorObject, "invalid work array for fft size"); - goto fail; - } - - nrepeats = PyArray_SIZE(data)/npts; - dptr = (double *)PyArray_DATA(data); - Py_BEGIN_ALLOW_THREADS; - NPY_SIGINT_ON; - for (i = 0; i < nrepeats; i++) { - npy_cfftb(npts, dptr, wsave); - dptr += npts*2; - } - NPY_SIGINT_OFF; - Py_END_ALLOW_THREADS; - PyArray_Free(op2, (char *)wsave); - return (PyObject *)data; - -fail: - PyArray_Free(op2, (char *)wsave); - Py_DECREF(data); - return NULL; -} - -static const char fftpack_cffti__doc__[] = ""; - -static PyObject * -fftpack_cffti(PyObject *NPY_UNUSED(self), PyObject *args) -{ - PyArrayObject *op; - npy_intp dim; - long n; - - if (!PyArg_ParseTuple(args, "l:cffti", &n)) { - return NULL; - } - /*Magic size needed by npy_cffti*/ - dim = 4*n + 15; - /*Create a 1 dimensional array of dimensions of type double*/ - op = (PyArrayObject *)PyArray_SimpleNew(1, &dim, NPY_DOUBLE); - if (op == NULL) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS; - NPY_SIGINT_ON; - npy_cffti(n, (double *)PyArray_DATA((PyArrayObject*)op)); - NPY_SIGINT_OFF; - Py_END_ALLOW_THREADS; - - return (PyObject *)op; -} - -static const char fftpack_rfftf__doc__[] = ""; - -static PyObject * -fftpack_rfftf(PyObject *NPY_UNUSED(self), PyObject *args) -{ - PyObject *op1, *op2; - PyArrayObject *data, *ret; - PyArray_Descr *descr; - double *wsave = NULL, *dptr, *rptr; - npy_intp nsave; - int npts, nrepeats, i, rstep; - - if(!PyArg_ParseTuple(args, "OO:rfftf", &op1, &op2)) { - return NULL; - } - data = (PyArrayObject *)PyArray_ContiguousFromObject(op1, - NPY_DOUBLE, 1, 0); - if (data == NULL) { - return NULL; - } - /* FIXME, direct access changing contents of data->dimensions */ - npts = PyArray_DIM(data, PyArray_NDIM(data) - 1); - PyArray_DIMS(data)[PyArray_NDIM(data) - 1] = npts/2 + 1; - ret = (PyArrayObject *)PyArray_Zeros(PyArray_NDIM(data), - PyArray_DIMS(data), PyArray_DescrFromType(NPY_CDOUBLE), 0); - if (ret == NULL) { - goto fail; - } - PyArray_DIMS(data)[PyArray_NDIM(data) - 1] = npts; - rstep = PyArray_DIM(ret, PyArray_NDIM(ret) - 1)*2; - - descr = PyArray_DescrFromType(NPY_DOUBLE); - if (PyArray_AsCArray(&op2, (void *)&wsave, &nsave, 1, descr) == -1) { - goto fail; - } - if (data == NULL || ret == NULL) { - goto fail; - } - if (nsave != npts*2+15) { - PyErr_SetString(ErrorObject, "invalid work array for fft size"); - goto fail; - } - - nrepeats = PyArray_SIZE(data)/npts; - rptr = (double *)PyArray_DATA(ret); - dptr = (double *)PyArray_DATA(data); - - Py_BEGIN_ALLOW_THREADS; - NPY_SIGINT_ON; - for (i = 0; i < nrepeats; i++) { - memcpy((char *)(rptr+1), dptr, npts*sizeof(double)); - npy_rfftf(npts, rptr+1, wsave); - rptr[0] = rptr[1]; - rptr[1] = 0.0; - rptr += rstep; - dptr += npts; - } - NPY_SIGINT_OFF; - Py_END_ALLOW_THREADS; - PyArray_Free(op2, (char *)wsave); - Py_DECREF(data); - return (PyObject *)ret; - -fail: - PyArray_Free(op2, (char *)wsave); - Py_XDECREF(data); - Py_XDECREF(ret); - return NULL; -} - -static const char fftpack_rfftb__doc__[] = ""; - -static PyObject * -fftpack_rfftb(PyObject *NPY_UNUSED(self), PyObject *args) -{ - PyObject *op1, *op2; - PyArrayObject *data, *ret; - PyArray_Descr *descr; - double *wsave, *dptr, *rptr; - npy_intp nsave; - int npts, nrepeats, i; - - if(!PyArg_ParseTuple(args, "OO:rfftb", &op1, &op2)) { - return NULL; - } - data = (PyArrayObject *)PyArray_ContiguousFromObject(op1, - NPY_CDOUBLE, 1, 0); - if (data == NULL) { - return NULL; - } - npts = PyArray_DIM(data, PyArray_NDIM(data) - 1); - ret = (PyArrayObject *)PyArray_Zeros(PyArray_NDIM(data), PyArray_DIMS(data), - PyArray_DescrFromType(NPY_DOUBLE), 0); - - descr = PyArray_DescrFromType(NPY_DOUBLE); - if (PyArray_AsCArray(&op2, (void *)&wsave, &nsave, 1, descr) == -1) { - goto fail; - } - if (data == NULL || ret == NULL) { - goto fail; - } - if (nsave != npts*2 + 15) { - PyErr_SetString(ErrorObject, "invalid work array for fft size"); - goto fail; - } - - nrepeats = PyArray_SIZE(ret)/npts; - rptr = (double *)PyArray_DATA(ret); - dptr = (double *)PyArray_DATA(data); - - Py_BEGIN_ALLOW_THREADS; - NPY_SIGINT_ON; - for (i = 0; i < nrepeats; i++) { - memcpy((char *)(rptr + 1), (dptr + 2), (npts - 1)*sizeof(double)); - rptr[0] = dptr[0]; - npy_rfftb(npts, rptr, wsave); - rptr += npts; - dptr += npts*2; - } - NPY_SIGINT_OFF; - Py_END_ALLOW_THREADS; - PyArray_Free(op2, (char *)wsave); - Py_DECREF(data); - return (PyObject *)ret; - -fail: - PyArray_Free(op2, (char *)wsave); - Py_XDECREF(data); - Py_XDECREF(ret); - return NULL; -} - -static const char fftpack_rffti__doc__[] = ""; - -static PyObject * -fftpack_rffti(PyObject *NPY_UNUSED(self), PyObject *args) -{ - PyArrayObject *op; - npy_intp dim; - long n; - - if (!PyArg_ParseTuple(args, "l:rffti", &n)) { - return NULL; - } - /*Magic size needed by npy_rffti*/ - dim = 2*n + 15; - /*Create a 1 dimensional array of dimensions of type double*/ - op = (PyArrayObject *)PyArray_SimpleNew(1, &dim, NPY_DOUBLE); - if (op == NULL) { - return NULL; - } - Py_BEGIN_ALLOW_THREADS; - NPY_SIGINT_ON; - npy_rffti(n, (double *)PyArray_DATA((PyArrayObject*)op)); - NPY_SIGINT_OFF; - Py_END_ALLOW_THREADS; - - return (PyObject *)op; -} - - -/* List of methods defined in the module */ - -static struct PyMethodDef fftpack_methods[] = { - {"cfftf", fftpack_cfftf, 1, fftpack_cfftf__doc__}, - {"cfftb", fftpack_cfftb, 1, fftpack_cfftb__doc__}, - {"cffti", fftpack_cffti, 1, fftpack_cffti__doc__}, - {"rfftf", fftpack_rfftf, 1, fftpack_rfftf__doc__}, - {"rfftb", fftpack_rfftb, 1, fftpack_rfftb__doc__}, - {"rffti", fftpack_rffti, 1, fftpack_rffti__doc__}, - {NULL, NULL, 0, NULL} /* sentinel */ -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "fftpack_lite", - NULL, - -1, - fftpack_methods, - NULL, - NULL, - NULL, - NULL -}; -#endif - -/* Initialization function for the module */ -#if PY_MAJOR_VERSION >= 3 -#define RETVAL(x) x -PyMODINIT_FUNC PyInit_fftpack_lite(void) -#else -#define RETVAL(x) -PyMODINIT_FUNC -initfftpack_lite(void) -#endif -{ - PyObject *m,*d; -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&moduledef); -#else - static const char fftpack_module_documentation[] = ""; - - m = Py_InitModule4("fftpack_lite", fftpack_methods, - fftpack_module_documentation, - (PyObject*)NULL,PYTHON_API_VERSION); -#endif - if (m == NULL) { - return RETVAL(NULL); - } - - /* Import the array object */ - import_array(); - - /* Add some symbolic constants to the module */ - d = PyModule_GetDict(m); - ErrorObject = PyErr_NewException("fftpack.error", NULL, NULL); - PyDict_SetItemString(d, "error", ErrorObject); - - /* XXXX Add constants here */ - - return RETVAL(m); -} diff --git a/numpy/fft/helper.py b/numpy/fft/helper.py index 4b698bb4d..a920a4ac0 100644 --- a/numpy/fft/helper.py +++ b/numpy/fft/helper.py @@ -4,14 +4,9 @@ Discrete Fourier Transforms - helper.py """ from __future__ import division, absolute_import, print_function -import collections -try: - import threading -except ImportError: - import dummy_threading as threading from numpy.compat import integer_types from numpy.core import integer, empty, arange, asarray, roll -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import array_function_dispatch, set_module # Created by Pearu Peterson, September 2002 @@ -24,7 +19,7 @@ def _fftshift_dispatcher(x, axes=None): return (x,) -@array_function_dispatch(_fftshift_dispatcher) +@array_function_dispatch(_fftshift_dispatcher, module='numpy.fft') def fftshift(x, axes=None): """ Shift the zero-frequency component to the center of the spectrum. @@ -52,7 +47,7 @@ def fftshift(x, axes=None): -------- >>> freqs = np.fft.fftfreq(10, 0.1) >>> freqs - array([ 0., 1., 2., 3., 4., -5., -4., -3., -2., -1.]) + array([ 0., 1., 2., ..., -3., -2., -1.]) >>> np.fft.fftshift(freqs) array([-5., -4., -3., -2., -1., 0., 1., 2., 3., 4.]) @@ -81,7 +76,7 @@ def fftshift(x, axes=None): return roll(x, shift, axes) -@array_function_dispatch(_fftshift_dispatcher) +@array_function_dispatch(_fftshift_dispatcher, module='numpy.fft') def ifftshift(x, axes=None): """ The inverse of `fftshift`. Although identical for even-length `x`, the @@ -128,6 +123,7 @@ def ifftshift(x, axes=None): return roll(x, shift, axes) +@set_module('numpy.fft') def fftfreq(n, d=1.0): """ Return the Discrete Fourier Transform sample frequencies. @@ -161,7 +157,7 @@ def fftfreq(n, d=1.0): >>> timestep = 0.1 >>> freq = np.fft.fftfreq(n, d=timestep) >>> freq - array([ 0. , 1.25, 2.5 , 3.75, -5. , -3.75, -2.5 , -1.25]) + array([ 0. , 1.25, 2.5 , ..., -3.75, -2.5 , -1.25]) """ if not isinstance(n, integer_types): @@ -174,9 +170,9 @@ def fftfreq(n, d=1.0): p2 = arange(-(n//2), 0, dtype=int) results[N:] = p2 return results * val - #return hstack((arange(0,(n-1)/2 + 1), arange(-(n/2),0))) / (n*d) +@set_module('numpy.fft') def rfftfreq(n, d=1.0): """ Return the Discrete Fourier Transform sample frequencies @@ -214,7 +210,7 @@ def rfftfreq(n, d=1.0): >>> sample_rate = 100 >>> freq = np.fft.fftfreq(n, d=1./sample_rate) >>> freq - array([ 0., 10., 20., 30., 40., -50., -40., -30., -20., -10.]) + array([ 0., 10., 20., ..., -30., -20., -10.]) >>> freq = np.fft.rfftfreq(n, d=1./sample_rate) >>> freq array([ 0., 10., 20., 30., 40., 50.]) @@ -226,99 +222,3 @@ def rfftfreq(n, d=1.0): N = n//2 + 1 results = arange(0, N, dtype=int) return results * val - - -class _FFTCache(object): - """ - Cache for the FFT twiddle factors as an LRU (least recently used) cache. - - Parameters - ---------- - max_size_in_mb : int - Maximum memory usage of the cache before items are being evicted. - max_item_count : int - Maximum item count of the cache before items are being evicted. - - Notes - ----- - Items will be evicted if either limit has been reached upon getting and - setting. The maximum memory usages is not strictly the given - ``max_size_in_mb`` but rather - ``max(max_size_in_mb, 1.5 * size_of_largest_item)``. Thus the cache will - never be completely cleared - at least one item will remain and a single - large item can cause the cache to retain several smaller items even if the - given maximum cache size has been exceeded. - """ - def __init__(self, max_size_in_mb, max_item_count): - self._max_size_in_bytes = max_size_in_mb * 1024 ** 2 - self._max_item_count = max_item_count - self._dict = collections.OrderedDict() - self._lock = threading.Lock() - - def put_twiddle_factors(self, n, factors): - """ - Store twiddle factors for an FFT of length n in the cache. - - Putting multiple twiddle factors for a certain n will store it multiple - times. - - Parameters - ---------- - n : int - Data length for the FFT. - factors : ndarray - The actual twiddle values. - """ - with self._lock: - # Pop + later add to move it to the end for LRU behavior. - # Internally everything is stored in a dictionary whose values are - # lists. - try: - value = self._dict.pop(n) - except KeyError: - value = [] - value.append(factors) - self._dict[n] = value - self._prune_cache() - - def pop_twiddle_factors(self, n): - """ - Pop twiddle factors for an FFT of length n from the cache. - - Will return None if the requested twiddle factors are not available in - the cache. - - Parameters - ---------- - n : int - Data length for the FFT. - - Returns - ------- - out : ndarray or None - The retrieved twiddle factors if available, else None. - """ - with self._lock: - if n not in self._dict or not self._dict[n]: - return None - # Pop + later add to move it to the end for LRU behavior. - all_values = self._dict.pop(n) - value = all_values.pop() - # Only put pack if there are still some arrays left in the list. - if all_values: - self._dict[n] = all_values - return value - - def _prune_cache(self): - # Always keep at least one item. - while len(self._dict) > 1 and ( - len(self._dict) > self._max_item_count or self._check_size()): - self._dict.popitem(last=False) - - def _check_size(self): - item_sizes = [sum(_j.nbytes for _j in _i) - for _i in self._dict.values() if _i] - if not item_sizes: - return False - max_size = max(self._max_size_in_bytes, 1.5 * max(item_sizes)) - return sum(item_sizes) > max_size diff --git a/numpy/fft/pocketfft.c b/numpy/fft/pocketfft.c new file mode 100644 index 000000000..10a741b6f --- /dev/null +++ b/numpy/fft/pocketfft.c @@ -0,0 +1,2398 @@ +/* + * This file is part of pocketfft. + * Licensed under a 3-clause BSD style license - see LICENSE.md + */ + +/* + * Main implementation file. + * + * Copyright (C) 2004-2018 Max-Planck-Society + * \author Martin Reinecke + */ + +#include <math.h> +#include <string.h> +#include <stdlib.h> + +#include "npy_config.h" +#define restrict NPY_RESTRICT + +#define RALLOC(type,num) \ + ((type *)malloc((num)*sizeof(type))) +#define DEALLOC(ptr) \ + do { free(ptr); (ptr)=NULL; } while(0) + +#define SWAP(a,b,type) \ + do { type tmp_=(a); (a)=(b); (b)=tmp_; } while(0) + +#ifdef __GNUC__ +#define NOINLINE __attribute__((noinline)) +#define WARN_UNUSED_RESULT __attribute__ ((warn_unused_result)) +#else +#define NOINLINE +#define WARN_UNUSED_RESULT +#endif + +struct cfft_plan_i; +typedef struct cfft_plan_i * cfft_plan; +struct rfft_plan_i; +typedef struct rfft_plan_i * rfft_plan; + +// adapted from https://stackoverflow.com/questions/42792939/ +// CAUTION: this function only works for arguments in the range [-0.25; 0.25]! +static void my_sincosm1pi (double a, double *restrict res) + { + double s = a * a; + /* Approximate cos(pi*x)-1 for x in [-0.25,0.25] */ + double r = -1.0369917389758117e-4; + r = fma (r, s, 1.9294935641298806e-3); + r = fma (r, s, -2.5806887942825395e-2); + r = fma (r, s, 2.3533063028328211e-1); + r = fma (r, s, -1.3352627688538006e+0); + r = fma (r, s, 4.0587121264167623e+0); + r = fma (r, s, -4.9348022005446790e+0); + double c = r*s; + /* Approximate sin(pi*x) for x in [-0.25,0.25] */ + r = 4.6151442520157035e-4; + r = fma (r, s, -7.3700183130883555e-3); + r = fma (r, s, 8.2145868949323936e-2); + r = fma (r, s, -5.9926452893214921e-1); + r = fma (r, s, 2.5501640398732688e+0); + r = fma (r, s, -5.1677127800499516e+0); + s = s * a; + r = r * s; + s = fma (a, 3.1415926535897931e+0, r); + res[0] = c; + res[1] = s; + } + +NOINLINE static void calc_first_octant(size_t den, double * restrict res) + { + size_t n = (den+4)>>3; + if (n==0) return; + res[0]=1.; res[1]=0.; + if (n==1) return; + size_t l1=(size_t)sqrt(n); + for (size_t i=1; i<l1; ++i) + my_sincosm1pi((2.*i)/den,&res[2*i]); + size_t start=l1; + while(start<n) + { + double cs[2]; + my_sincosm1pi((2.*start)/den,cs); + res[2*start] = cs[0]+1.; + res[2*start+1] = cs[1]; + size_t end = l1; + if (start+end>n) end = n-start; + for (size_t i=1; i<end; ++i) + { + double csx[2]={res[2*i], res[2*i+1]}; + res[2*(start+i)] = ((cs[0]*csx[0] - cs[1]*csx[1] + cs[0]) + csx[0]) + 1.; + res[2*(start+i)+1] = (cs[0]*csx[1] + cs[1]*csx[0]) + cs[1] + csx[1]; + } + start += l1; + } + for (size_t i=1; i<l1; ++i) + res[2*i] += 1.; + } + +NOINLINE static void calc_first_quadrant(size_t n, double * restrict res) + { + double * restrict p = res+n; + calc_first_octant(n<<1, p); + size_t ndone=(n+2)>>2; + size_t i=0, idx1=0, idx2=2*ndone-2; + for (; i+1<ndone; i+=2, idx1+=2, idx2-=2) + { + res[idx1] = p[2*i]; + res[idx1+1] = p[2*i+1]; + res[idx2] = p[2*i+3]; + res[idx2+1] = p[2*i+2]; + } + if (i!=ndone) + { + res[idx1 ] = p[2*i]; + res[idx1+1] = p[2*i+1]; + } + } + +NOINLINE static void calc_first_half(size_t n, double * restrict res) + { + int ndone=(n+1)>>1; + double * p = res+n-1; + calc_first_octant(n<<2, p); + int i4=0, in=n, i=0; + for (; i4<=in-i4; ++i, i4+=4) // octant 0 + { + res[2*i] = p[2*i4]; res[2*i+1] = p[2*i4+1]; + } + for (; i4-in <= 0; ++i, i4+=4) // octant 1 + { + int xm = in-i4; + res[2*i] = p[2*xm+1]; res[2*i+1] = p[2*xm]; + } + for (; i4<=3*in-i4; ++i, i4+=4) // octant 2 + { + int xm = i4-in; + res[2*i] = -p[2*xm+1]; res[2*i+1] = p[2*xm]; + } + for (; i<ndone; ++i, i4+=4) // octant 3 + { + int xm = 2*in-i4; + res[2*i] = -p[2*xm]; res[2*i+1] = p[2*xm+1]; + } + } + +NOINLINE static void fill_first_quadrant(size_t n, double * restrict res) + { + const double hsqt2 = 0.707106781186547524400844362104849; + size_t quart = n>>2; + if ((n&7)==0) + res[quart] = res[quart+1] = hsqt2; + for (size_t i=2, j=2*quart-2; i<quart; i+=2, j-=2) + { + res[j ] = res[i+1]; + res[j+1] = res[i ]; + } + } + +NOINLINE static void fill_first_half(size_t n, double * restrict res) + { + size_t half = n>>1; + if ((n&3)==0) + for (size_t i=0; i<half; i+=2) + { + res[i+half] = -res[i+1]; + res[i+half+1] = res[i ]; + } + else + for (size_t i=2, j=2*half-2; i<half; i+=2, j-=2) + { + res[j ] = -res[i ]; + res[j+1] = res[i+1]; + } + } + +NOINLINE static void fill_second_half(size_t n, double * restrict res) + { + if ((n&1)==0) + for (size_t i=0; i<n; ++i) + res[i+n] = -res[i]; + else + for (size_t i=2, j=2*n-2; i<n; i+=2, j-=2) + { + res[j ] = res[i ]; + res[j+1] = -res[i+1]; + } + } + +NOINLINE static void sincos_2pibyn_half(size_t n, double * restrict res) + { + if ((n&3)==0) + { + calc_first_octant(n, res); + fill_first_quadrant(n, res); + fill_first_half(n, res); + } + else if ((n&1)==0) + { + calc_first_quadrant(n, res); + fill_first_half(n, res); + } + else + calc_first_half(n, res); + } + +NOINLINE static void sincos_2pibyn(size_t n, double * restrict res) + { + sincos_2pibyn_half(n, res); + fill_second_half(n, res); + } + +NOINLINE static size_t largest_prime_factor (size_t n) + { + size_t res=1; + size_t tmp; + while (((tmp=(n>>1))<<1)==n) + { res=2; n=tmp; } + + size_t limit=(size_t)sqrt(n+0.01); + for (size_t x=3; x<=limit; x+=2) + while (((tmp=(n/x))*x)==n) + { + res=x; + n=tmp; + limit=(size_t)sqrt(n+0.01); + } + if (n>1) res=n; + + return res; + } + +NOINLINE static double cost_guess (size_t n) + { + const double lfp=1.1; // penalty for non-hardcoded larger factors + size_t ni=n; + double result=0.; + size_t tmp; + while (((tmp=(n>>1))<<1)==n) + { result+=2; n=tmp; } + + size_t limit=(size_t)sqrt(n+0.01); + for (size_t x=3; x<=limit; x+=2) + while ((tmp=(n/x))*x==n) + { + result+= (x<=5) ? x : lfp*x; // penalize larger prime factors + n=tmp; + limit=(size_t)sqrt(n+0.01); + } + if (n>1) result+=(n<=5) ? n : lfp*n; + + return result*ni; + } + +/* returns the smallest composite of 2, 3, 5, 7 and 11 which is >= n */ +NOINLINE static size_t good_size(size_t n) + { + if (n<=6) return n; + + size_t bestfac=2*n; + for (size_t f2=1; f2<bestfac; f2*=2) + for (size_t f23=f2; f23<bestfac; f23*=3) + for (size_t f235=f23; f235<bestfac; f235*=5) + for (size_t f2357=f235; f2357<bestfac; f2357*=7) + for (size_t f235711=f2357; f235711<bestfac; f235711*=11) + if (f235711>=n) bestfac=f235711; + return bestfac; + } + +typedef struct cmplx { + double r,i; +} cmplx; + +#define NFCT 25 +typedef struct cfftp_fctdata + { + size_t fct; + cmplx *tw, *tws; + } cfftp_fctdata; + +typedef struct cfftp_plan_i + { + size_t length, nfct; + cmplx *mem; + cfftp_fctdata fct[NFCT]; + } cfftp_plan_i; +typedef struct cfftp_plan_i * cfftp_plan; + +#define PMC(a,b,c,d) { a.r=c.r+d.r; a.i=c.i+d.i; b.r=c.r-d.r; b.i=c.i-d.i; } +#define ADDC(a,b,c) { a.r=b.r+c.r; a.i=b.i+c.i; } +#define SCALEC(a,b) { a.r*=b; a.i*=b; } +#define ROT90(a) { double tmp_=a.r; a.r=-a.i; a.i=tmp_; } +#define ROTM90(a) { double tmp_=-a.r; a.r=a.i; a.i=tmp_; } +#define CH(a,b,c) ch[(a)+ido*((b)+l1*(c))] +#define CC(a,b,c) cc[(a)+ido*((b)+cdim*(c))] +#define WA(x,i) wa[(i)-1+(x)*(ido-1)] +/* a = b*c */ +#define A_EQ_B_MUL_C(a,b,c) { a.r=b.r*c.r-b.i*c.i; a.i=b.r*c.i+b.i*c.r; } +/* a = conj(b)*c*/ +#define A_EQ_CB_MUL_C(a,b,c) { a.r=b.r*c.r+b.i*c.i; a.i=b.r*c.i-b.i*c.r; } + +#define PMSIGNC(a,b,c,d) { a.r=c.r+sign*d.r; a.i=c.i+sign*d.i; b.r=c.r-sign*d.r; b.i=c.i-sign*d.i; } +/* a = b*c */ +#define MULPMSIGNC(a,b,c) { a.r=b.r*c.r-sign*b.i*c.i; a.i=b.r*c.i+sign*b.i*c.r; } +/* a *= b */ +#define MULPMSIGNCEQ(a,b) { double xtmp=a.r; a.r=b.r*a.r-sign*b.i*a.i; a.i=b.r*a.i+sign*b.i*xtmp; } + +NOINLINE static void pass2b (size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa) + { + const size_t cdim=2; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + PMC (CH(0,k,0),CH(0,k,1),CC(0,0,k),CC(0,1,k)) + else + for (size_t k=0; k<l1; ++k) + { + PMC (CH(0,k,0),CH(0,k,1),CC(0,0,k),CC(0,1,k)) + for (size_t i=1; i<ido; ++i) + { + cmplx t; + PMC (CH(i,k,0),t,CC(i,0,k),CC(i,1,k)) + A_EQ_B_MUL_C (CH(i,k,1),WA(0,i),t) + } + } + } + +NOINLINE static void pass2f (size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa) + { + const size_t cdim=2; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + PMC (CH(0,k,0),CH(0,k,1),CC(0,0,k),CC(0,1,k)) + else + for (size_t k=0; k<l1; ++k) + { + PMC (CH(0,k,0),CH(0,k,1),CC(0,0,k),CC(0,1,k)) + for (size_t i=1; i<ido; ++i) + { + cmplx t; + PMC (CH(i,k,0),t,CC(i,0,k),CC(i,1,k)) + A_EQ_CB_MUL_C (CH(i,k,1),WA(0,i),t) + } + } + } + +#define PREP3(idx) \ + cmplx t0 = CC(idx,0,k), t1, t2; \ + PMC (t1,t2,CC(idx,1,k),CC(idx,2,k)) \ + CH(idx,k,0).r=t0.r+t1.r; \ + CH(idx,k,0).i=t0.i+t1.i; +#define PARTSTEP3a(u1,u2,twr,twi) \ + { \ + cmplx ca,cb; \ + ca.r=t0.r+twr*t1.r; \ + ca.i=t0.i+twr*t1.i; \ + cb.i=twi*t2.r; \ + cb.r=-(twi*t2.i); \ + PMC(CH(0,k,u1),CH(0,k,u2),ca,cb) \ + } + +#define PARTSTEP3b(u1,u2,twr,twi) \ + { \ + cmplx ca,cb,da,db; \ + ca.r=t0.r+twr*t1.r; \ + ca.i=t0.i+twr*t1.i; \ + cb.i=twi*t2.r; \ + cb.r=-(twi*t2.i); \ + PMC(da,db,ca,cb) \ + A_EQ_B_MUL_C (CH(i,k,u1),WA(u1-1,i),da) \ + A_EQ_B_MUL_C (CH(i,k,u2),WA(u2-1,i),db) \ + } +NOINLINE static void pass3b (size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa) + { + const size_t cdim=3; + const double tw1r=-0.5, tw1i= 0.86602540378443864676; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + { + PREP3(0) + PARTSTEP3a(1,2,tw1r,tw1i) + } + else + for (size_t k=0; k<l1; ++k) + { + { + PREP3(0) + PARTSTEP3a(1,2,tw1r,tw1i) + } + for (size_t i=1; i<ido; ++i) + { + PREP3(i) + PARTSTEP3b(1,2,tw1r,tw1i) + } + } + } +#define PARTSTEP3f(u1,u2,twr,twi) \ + { \ + cmplx ca,cb,da,db; \ + ca.r=t0.r+twr*t1.r; \ + ca.i=t0.i+twr*t1.i; \ + cb.i=twi*t2.r; \ + cb.r=-(twi*t2.i); \ + PMC(da,db,ca,cb) \ + A_EQ_CB_MUL_C (CH(i,k,u1),WA(u1-1,i),da) \ + A_EQ_CB_MUL_C (CH(i,k,u2),WA(u2-1,i),db) \ + } +NOINLINE static void pass3f (size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa) + { + const size_t cdim=3; + const double tw1r=-0.5, tw1i= -0.86602540378443864676; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + { + PREP3(0) + PARTSTEP3a(1,2,tw1r,tw1i) + } + else + for (size_t k=0; k<l1; ++k) + { + { + PREP3(0) + PARTSTEP3a(1,2,tw1r,tw1i) + } + for (size_t i=1; i<ido; ++i) + { + PREP3(i) + PARTSTEP3f(1,2,tw1r,tw1i) + } + } + } + +NOINLINE static void pass4b (size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa) + { + const size_t cdim=4; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + { + cmplx t1, t2, t3, t4; + PMC(t2,t1,CC(0,0,k),CC(0,2,k)) + PMC(t3,t4,CC(0,1,k),CC(0,3,k)) + ROT90(t4) + PMC(CH(0,k,0),CH(0,k,2),t2,t3) + PMC(CH(0,k,1),CH(0,k,3),t1,t4) + } + else + for (size_t k=0; k<l1; ++k) + { + { + cmplx t1, t2, t3, t4; + PMC(t2,t1,CC(0,0,k),CC(0,2,k)) + PMC(t3,t4,CC(0,1,k),CC(0,3,k)) + ROT90(t4) + PMC(CH(0,k,0),CH(0,k,2),t2,t3) + PMC(CH(0,k,1),CH(0,k,3),t1,t4) + } + for (size_t i=1; i<ido; ++i) + { + cmplx c2, c3, c4, t1, t2, t3, t4; + cmplx cc0=CC(i,0,k), cc1=CC(i,1,k),cc2=CC(i,2,k),cc3=CC(i,3,k); + PMC(t2,t1,cc0,cc2) + PMC(t3,t4,cc1,cc3) + ROT90(t4) + cmplx wa0=WA(0,i), wa1=WA(1,i),wa2=WA(2,i); + PMC(CH(i,k,0),c3,t2,t3) + PMC(c2,c4,t1,t4) + A_EQ_B_MUL_C (CH(i,k,1),wa0,c2) + A_EQ_B_MUL_C (CH(i,k,2),wa1,c3) + A_EQ_B_MUL_C (CH(i,k,3),wa2,c4) + } + } + } +NOINLINE static void pass4f (size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa) + { + const size_t cdim=4; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + { + cmplx t1, t2, t3, t4; + PMC(t2,t1,CC(0,0,k),CC(0,2,k)) + PMC(t3,t4,CC(0,1,k),CC(0,3,k)) + ROTM90(t4) + PMC(CH(0,k,0),CH(0,k,2),t2,t3) + PMC(CH(0,k,1),CH(0,k,3),t1,t4) + } + else + for (size_t k=0; k<l1; ++k) + { + { + cmplx t1, t2, t3, t4; + PMC(t2,t1,CC(0,0,k),CC(0,2,k)) + PMC(t3,t4,CC(0,1,k),CC(0,3,k)) + ROTM90(t4) + PMC(CH(0,k,0),CH(0,k,2),t2,t3) + PMC (CH(0,k,1),CH(0,k,3),t1,t4) + } + for (size_t i=1; i<ido; ++i) + { + cmplx c2, c3, c4, t1, t2, t3, t4; + cmplx cc0=CC(i,0,k), cc1=CC(i,1,k),cc2=CC(i,2,k),cc3=CC(i,3,k); + PMC(t2,t1,cc0,cc2) + PMC(t3,t4,cc1,cc3) + ROTM90(t4) + cmplx wa0=WA(0,i), wa1=WA(1,i),wa2=WA(2,i); + PMC(CH(i,k,0),c3,t2,t3) + PMC(c2,c4,t1,t4) + A_EQ_CB_MUL_C (CH(i,k,1),wa0,c2) + A_EQ_CB_MUL_C (CH(i,k,2),wa1,c3) + A_EQ_CB_MUL_C (CH(i,k,3),wa2,c4) + } + } + } + +#define PREP5(idx) \ + cmplx t0 = CC(idx,0,k), t1, t2, t3, t4; \ + PMC (t1,t4,CC(idx,1,k),CC(idx,4,k)) \ + PMC (t2,t3,CC(idx,2,k),CC(idx,3,k)) \ + CH(idx,k,0).r=t0.r+t1.r+t2.r; \ + CH(idx,k,0).i=t0.i+t1.i+t2.i; + +#define PARTSTEP5a(u1,u2,twar,twbr,twai,twbi) \ + { \ + cmplx ca,cb; \ + ca.r=t0.r+twar*t1.r+twbr*t2.r; \ + ca.i=t0.i+twar*t1.i+twbr*t2.i; \ + cb.i=twai*t4.r twbi*t3.r; \ + cb.r=-(twai*t4.i twbi*t3.i); \ + PMC(CH(0,k,u1),CH(0,k,u2),ca,cb) \ + } + +#define PARTSTEP5b(u1,u2,twar,twbr,twai,twbi) \ + { \ + cmplx ca,cb,da,db; \ + ca.r=t0.r+twar*t1.r+twbr*t2.r; \ + ca.i=t0.i+twar*t1.i+twbr*t2.i; \ + cb.i=twai*t4.r twbi*t3.r; \ + cb.r=-(twai*t4.i twbi*t3.i); \ + PMC(da,db,ca,cb) \ + A_EQ_B_MUL_C (CH(i,k,u1),WA(u1-1,i),da) \ + A_EQ_B_MUL_C (CH(i,k,u2),WA(u2-1,i),db) \ + } +NOINLINE static void pass5b (size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa) + { + const size_t cdim=5; + const double tw1r= 0.3090169943749474241, + tw1i= 0.95105651629515357212, + tw2r= -0.8090169943749474241, + tw2i= 0.58778525229247312917; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + { + PREP5(0) + PARTSTEP5a(1,4,tw1r,tw2r,+tw1i,+tw2i) + PARTSTEP5a(2,3,tw2r,tw1r,+tw2i,-tw1i) + } + else + for (size_t k=0; k<l1; ++k) + { + { + PREP5(0) + PARTSTEP5a(1,4,tw1r,tw2r,+tw1i,+tw2i) + PARTSTEP5a(2,3,tw2r,tw1r,+tw2i,-tw1i) + } + for (size_t i=1; i<ido; ++i) + { + PREP5(i) + PARTSTEP5b(1,4,tw1r,tw2r,+tw1i,+tw2i) + PARTSTEP5b(2,3,tw2r,tw1r,+tw2i,-tw1i) + } + } + } +#define PARTSTEP5f(u1,u2,twar,twbr,twai,twbi) \ + { \ + cmplx ca,cb,da,db; \ + ca.r=t0.r+twar*t1.r+twbr*t2.r; \ + ca.i=t0.i+twar*t1.i+twbr*t2.i; \ + cb.i=twai*t4.r twbi*t3.r; \ + cb.r=-(twai*t4.i twbi*t3.i); \ + PMC(da,db,ca,cb) \ + A_EQ_CB_MUL_C (CH(i,k,u1),WA(u1-1,i),da) \ + A_EQ_CB_MUL_C (CH(i,k,u2),WA(u2-1,i),db) \ + } +NOINLINE static void pass5f (size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa) + { + const size_t cdim=5; + const double tw1r= 0.3090169943749474241, + tw1i= -0.95105651629515357212, + tw2r= -0.8090169943749474241, + tw2i= -0.58778525229247312917; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + { + PREP5(0) + PARTSTEP5a(1,4,tw1r,tw2r,+tw1i,+tw2i) + PARTSTEP5a(2,3,tw2r,tw1r,+tw2i,-tw1i) + } + else + for (size_t k=0; k<l1; ++k) + { + { + PREP5(0) + PARTSTEP5a(1,4,tw1r,tw2r,+tw1i,+tw2i) + PARTSTEP5a(2,3,tw2r,tw1r,+tw2i,-tw1i) + } + for (size_t i=1; i<ido; ++i) + { + PREP5(i) + PARTSTEP5f(1,4,tw1r,tw2r,+tw1i,+tw2i) + PARTSTEP5f(2,3,tw2r,tw1r,+tw2i,-tw1i) + } + } + } + +#define PREP7(idx) \ + cmplx t1 = CC(idx,0,k), t2, t3, t4, t5, t6, t7; \ + PMC (t2,t7,CC(idx,1,k),CC(idx,6,k)) \ + PMC (t3,t6,CC(idx,2,k),CC(idx,5,k)) \ + PMC (t4,t5,CC(idx,3,k),CC(idx,4,k)) \ + CH(idx,k,0).r=t1.r+t2.r+t3.r+t4.r; \ + CH(idx,k,0).i=t1.i+t2.i+t3.i+t4.i; + +#define PARTSTEP7a0(u1,u2,x1,x2,x3,y1,y2,y3,out1,out2) \ + { \ + cmplx ca,cb; \ + ca.r=t1.r+x1*t2.r+x2*t3.r+x3*t4.r; \ + ca.i=t1.i+x1*t2.i+x2*t3.i+x3*t4.i; \ + cb.i=y1*t7.r y2*t6.r y3*t5.r; \ + cb.r=-(y1*t7.i y2*t6.i y3*t5.i); \ + PMC(out1,out2,ca,cb) \ + } +#define PARTSTEP7a(u1,u2,x1,x2,x3,y1,y2,y3) \ + PARTSTEP7a0(u1,u2,x1,x2,x3,y1,y2,y3,CH(0,k,u1),CH(0,k,u2)) +#define PARTSTEP7(u1,u2,x1,x2,x3,y1,y2,y3) \ + { \ + cmplx da,db; \ + PARTSTEP7a0(u1,u2,x1,x2,x3,y1,y2,y3,da,db) \ + MULPMSIGNC (CH(i,k,u1),WA(u1-1,i),da) \ + MULPMSIGNC (CH(i,k,u2),WA(u2-1,i),db) \ + } + +NOINLINE static void pass7(size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa, const int sign) + { + const size_t cdim=7; + const double tw1r= 0.623489801858733530525, + tw1i= sign * 0.7818314824680298087084, + tw2r= -0.222520933956314404289, + tw2i= sign * 0.9749279121818236070181, + tw3r= -0.9009688679024191262361, + tw3i= sign * 0.4338837391175581204758; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + { + PREP7(0) + PARTSTEP7a(1,6,tw1r,tw2r,tw3r,+tw1i,+tw2i,+tw3i) + PARTSTEP7a(2,5,tw2r,tw3r,tw1r,+tw2i,-tw3i,-tw1i) + PARTSTEP7a(3,4,tw3r,tw1r,tw2r,+tw3i,-tw1i,+tw2i) + } + else + for (size_t k=0; k<l1; ++k) + { + { + PREP7(0) + PARTSTEP7a(1,6,tw1r,tw2r,tw3r,+tw1i,+tw2i,+tw3i) + PARTSTEP7a(2,5,tw2r,tw3r,tw1r,+tw2i,-tw3i,-tw1i) + PARTSTEP7a(3,4,tw3r,tw1r,tw2r,+tw3i,-tw1i,+tw2i) + } + for (size_t i=1; i<ido; ++i) + { + PREP7(i) + PARTSTEP7(1,6,tw1r,tw2r,tw3r,+tw1i,+tw2i,+tw3i) + PARTSTEP7(2,5,tw2r,tw3r,tw1r,+tw2i,-tw3i,-tw1i) + PARTSTEP7(3,4,tw3r,tw1r,tw2r,+tw3i,-tw1i,+tw2i) + } + } + } + +#define PREP11(idx) \ + cmplx t1 = CC(idx,0,k), t2, t3, t4, t5, t6, t7, t8, t9, t10, t11; \ + PMC (t2,t11,CC(idx,1,k),CC(idx,10,k)) \ + PMC (t3,t10,CC(idx,2,k),CC(idx, 9,k)) \ + PMC (t4,t9 ,CC(idx,3,k),CC(idx, 8,k)) \ + PMC (t5,t8 ,CC(idx,4,k),CC(idx, 7,k)) \ + PMC (t6,t7 ,CC(idx,5,k),CC(idx, 6,k)) \ + CH(idx,k,0).r=t1.r+t2.r+t3.r+t4.r+t5.r+t6.r; \ + CH(idx,k,0).i=t1.i+t2.i+t3.i+t4.i+t5.i+t6.i; + +#define PARTSTEP11a0(u1,u2,x1,x2,x3,x4,x5,y1,y2,y3,y4,y5,out1,out2) \ + { \ + cmplx ca,cb; \ + ca.r=t1.r+x1*t2.r+x2*t3.r+x3*t4.r+x4*t5.r+x5*t6.r; \ + ca.i=t1.i+x1*t2.i+x2*t3.i+x3*t4.i+x4*t5.i+x5*t6.i; \ + cb.i=y1*t11.r y2*t10.r y3*t9.r y4*t8.r y5*t7.r; \ + cb.r=-(y1*t11.i y2*t10.i y3*t9.i y4*t8.i y5*t7.i ); \ + PMC(out1,out2,ca,cb) \ + } +#define PARTSTEP11a(u1,u2,x1,x2,x3,x4,x5,y1,y2,y3,y4,y5) \ + PARTSTEP11a0(u1,u2,x1,x2,x3,x4,x5,y1,y2,y3,y4,y5,CH(0,k,u1),CH(0,k,u2)) +#define PARTSTEP11(u1,u2,x1,x2,x3,x4,x5,y1,y2,y3,y4,y5) \ + { \ + cmplx da,db; \ + PARTSTEP11a0(u1,u2,x1,x2,x3,x4,x5,y1,y2,y3,y4,y5,da,db) \ + MULPMSIGNC (CH(i,k,u1),WA(u1-1,i),da) \ + MULPMSIGNC (CH(i,k,u2),WA(u2-1,i),db) \ + } + +NOINLINE static void pass11 (size_t ido, size_t l1, const cmplx * restrict cc, + cmplx * restrict ch, const cmplx * restrict wa, const int sign) + { + const size_t cdim=11; + const double tw1r = 0.8412535328311811688618, + tw1i = sign * 0.5406408174555975821076, + tw2r = 0.4154150130018864255293, + tw2i = sign * 0.9096319953545183714117, + tw3r = -0.1423148382732851404438, + tw3i = sign * 0.9898214418809327323761, + tw4r = -0.6548607339452850640569, + tw4i = sign * 0.755749574354258283774, + tw5r = -0.9594929736144973898904, + tw5i = sign * 0.2817325568414296977114; + + if (ido==1) + for (size_t k=0; k<l1; ++k) + { + PREP11(0) + PARTSTEP11a(1,10,tw1r,tw2r,tw3r,tw4r,tw5r,+tw1i,+tw2i,+tw3i,+tw4i,+tw5i) + PARTSTEP11a(2, 9,tw2r,tw4r,tw5r,tw3r,tw1r,+tw2i,+tw4i,-tw5i,-tw3i,-tw1i) + PARTSTEP11a(3, 8,tw3r,tw5r,tw2r,tw1r,tw4r,+tw3i,-tw5i,-tw2i,+tw1i,+tw4i) + PARTSTEP11a(4, 7,tw4r,tw3r,tw1r,tw5r,tw2r,+tw4i,-tw3i,+tw1i,+tw5i,-tw2i) + PARTSTEP11a(5, 6,tw5r,tw1r,tw4r,tw2r,tw3r,+tw5i,-tw1i,+tw4i,-tw2i,+tw3i) + } + else + for (size_t k=0; k<l1; ++k) + { + { + PREP11(0) + PARTSTEP11a(1,10,tw1r,tw2r,tw3r,tw4r,tw5r,+tw1i,+tw2i,+tw3i,+tw4i,+tw5i) + PARTSTEP11a(2, 9,tw2r,tw4r,tw5r,tw3r,tw1r,+tw2i,+tw4i,-tw5i,-tw3i,-tw1i) + PARTSTEP11a(3, 8,tw3r,tw5r,tw2r,tw1r,tw4r,+tw3i,-tw5i,-tw2i,+tw1i,+tw4i) + PARTSTEP11a(4, 7,tw4r,tw3r,tw1r,tw5r,tw2r,+tw4i,-tw3i,+tw1i,+tw5i,-tw2i) + PARTSTEP11a(5, 6,tw5r,tw1r,tw4r,tw2r,tw3r,+tw5i,-tw1i,+tw4i,-tw2i,+tw3i) + } + for (size_t i=1; i<ido; ++i) + { + PREP11(i) + PARTSTEP11(1,10,tw1r,tw2r,tw3r,tw4r,tw5r,+tw1i,+tw2i,+tw3i,+tw4i,+tw5i) + PARTSTEP11(2, 9,tw2r,tw4r,tw5r,tw3r,tw1r,+tw2i,+tw4i,-tw5i,-tw3i,-tw1i) + PARTSTEP11(3, 8,tw3r,tw5r,tw2r,tw1r,tw4r,+tw3i,-tw5i,-tw2i,+tw1i,+tw4i) + PARTSTEP11(4, 7,tw4r,tw3r,tw1r,tw5r,tw2r,+tw4i,-tw3i,+tw1i,+tw5i,-tw2i) + PARTSTEP11(5, 6,tw5r,tw1r,tw4r,tw2r,tw3r,+tw5i,-tw1i,+tw4i,-tw2i,+tw3i) + } + } + } + +#define CX(a,b,c) cc[(a)+ido*((b)+l1*(c))] +#define CX2(a,b) cc[(a)+idl1*(b)] +#define CH2(a,b) ch[(a)+idl1*(b)] + +NOINLINE static int passg (size_t ido, size_t ip, size_t l1, + cmplx * restrict cc, cmplx * restrict ch, const cmplx * restrict wa, + const cmplx * restrict csarr, const int sign) + { + const size_t cdim=ip; + size_t ipph = (ip+1)/2; + size_t idl1 = ido*l1; + + cmplx * restrict wal=RALLOC(cmplx,ip); + if (!wal) return -1; + wal[0]=(cmplx){1.,0.}; + for (size_t i=1; i<ip; ++i) + wal[i]=(cmplx){csarr[i].r,sign*csarr[i].i}; + + for (size_t k=0; k<l1; ++k) + for (size_t i=0; i<ido; ++i) + CH(i,k,0) = CC(i,0,k); + for (size_t j=1, jc=ip-1; j<ipph; ++j, --jc) + for (size_t k=0; k<l1; ++k) + for (size_t i=0; i<ido; ++i) + PMC(CH(i,k,j),CH(i,k,jc),CC(i,j,k),CC(i,jc,k)) + for (size_t k=0; k<l1; ++k) + for (size_t i=0; i<ido; ++i) + { + cmplx tmp = CH(i,k,0); + for (size_t j=1; j<ipph; ++j) + ADDC(tmp,tmp,CH(i,k,j)) + CX(i,k,0) = tmp; + } + for (size_t l=1, lc=ip-1; l<ipph; ++l, --lc) + { + // j=0 + for (size_t ik=0; ik<idl1; ++ik) + { + CX2(ik,l).r = CH2(ik,0).r+wal[l].r*CH2(ik,1).r+wal[2*l].r*CH2(ik,2).r; + CX2(ik,l).i = CH2(ik,0).i+wal[l].r*CH2(ik,1).i+wal[2*l].r*CH2(ik,2).i; + CX2(ik,lc).r=-wal[l].i*CH2(ik,ip-1).i-wal[2*l].i*CH2(ik,ip-2).i; + CX2(ik,lc).i=wal[l].i*CH2(ik,ip-1).r+wal[2*l].i*CH2(ik,ip-2).r; + } + + size_t iwal=2*l; + size_t j=3, jc=ip-3; + for (; j<ipph-1; j+=2, jc-=2) + { + iwal+=l; if (iwal>ip) iwal-=ip; + cmplx xwal=wal[iwal]; + iwal+=l; if (iwal>ip) iwal-=ip; + cmplx xwal2=wal[iwal]; + for (size_t ik=0; ik<idl1; ++ik) + { + CX2(ik,l).r += CH2(ik,j).r*xwal.r+CH2(ik,j+1).r*xwal2.r; + CX2(ik,l).i += CH2(ik,j).i*xwal.r+CH2(ik,j+1).i*xwal2.r; + CX2(ik,lc).r -= CH2(ik,jc).i*xwal.i+CH2(ik,jc-1).i*xwal2.i; + CX2(ik,lc).i += CH2(ik,jc).r*xwal.i+CH2(ik,jc-1).r*xwal2.i; + } + } + for (; j<ipph; ++j, --jc) + { + iwal+=l; if (iwal>ip) iwal-=ip; + cmplx xwal=wal[iwal]; + for (size_t ik=0; ik<idl1; ++ik) + { + CX2(ik,l).r += CH2(ik,j).r*xwal.r; + CX2(ik,l).i += CH2(ik,j).i*xwal.r; + CX2(ik,lc).r -= CH2(ik,jc).i*xwal.i; + CX2(ik,lc).i += CH2(ik,jc).r*xwal.i; + } + } + } + DEALLOC(wal); + + // shuffling and twiddling + if (ido==1) + for (size_t j=1, jc=ip-1; j<ipph; ++j, --jc) + for (size_t ik=0; ik<idl1; ++ik) + { + cmplx t1=CX2(ik,j), t2=CX2(ik,jc); + PMC(CX2(ik,j),CX2(ik,jc),t1,t2) + } + else + { + for (size_t j=1, jc=ip-1; j<ipph; ++j,--jc) + for (size_t k=0; k<l1; ++k) + { + cmplx t1=CX(0,k,j), t2=CX(0,k,jc); + PMC(CX(0,k,j),CX(0,k,jc),t1,t2) + for (size_t i=1; i<ido; ++i) + { + cmplx x1, x2; + PMC(x1,x2,CX(i,k,j),CX(i,k,jc)) + size_t idij=(j-1)*(ido-1)+i-1; + MULPMSIGNC (CX(i,k,j),wa[idij],x1) + idij=(jc-1)*(ido-1)+i-1; + MULPMSIGNC (CX(i,k,jc),wa[idij],x2) + } + } + } + return 0; + } + +#undef CH2 +#undef CX2 +#undef CX + +NOINLINE WARN_UNUSED_RESULT static int pass_all(cfftp_plan plan, cmplx c[], double fct, + const int sign) + { + if (plan->length==1) return 0; + size_t len=plan->length; + size_t l1=1, nf=plan->nfct; + cmplx *ch = RALLOC(cmplx, len); + if (!ch) return -1; + cmplx *p1=c, *p2=ch; + + for(size_t k1=0; k1<nf; k1++) + { + size_t ip=plan->fct[k1].fct; + size_t l2=ip*l1; + size_t ido = len/l2; + if (ip==4) + sign>0 ? pass4b (ido, l1, p1, p2, plan->fct[k1].tw) + : pass4f (ido, l1, p1, p2, plan->fct[k1].tw); + else if(ip==2) + sign>0 ? pass2b (ido, l1, p1, p2, plan->fct[k1].tw) + : pass2f (ido, l1, p1, p2, plan->fct[k1].tw); + else if(ip==3) + sign>0 ? pass3b (ido, l1, p1, p2, plan->fct[k1].tw) + : pass3f (ido, l1, p1, p2, plan->fct[k1].tw); + else if(ip==5) + sign>0 ? pass5b (ido, l1, p1, p2, plan->fct[k1].tw) + : pass5f (ido, l1, p1, p2, plan->fct[k1].tw); + else if(ip==7) pass7 (ido, l1, p1, p2, plan->fct[k1].tw, sign); + else if(ip==11) pass11(ido, l1, p1, p2, plan->fct[k1].tw, sign); + else + { + if (passg(ido, ip, l1, p1, p2, plan->fct[k1].tw, plan->fct[k1].tws, sign)) + { DEALLOC(ch); return -1; } + SWAP(p1,p2,cmplx *); + } + SWAP(p1,p2,cmplx *); + l1=l2; + } + if (p1!=c) + { + if (fct!=1.) + for (size_t i=0; i<len; ++i) + { + c[i].r = ch[i].r*fct; + c[i].i = ch[i].i*fct; + } + else + memcpy (c,p1,len*sizeof(cmplx)); + } + else + if (fct!=1.) + for (size_t i=0; i<len; ++i) + { + c[i].r *= fct; + c[i].i *= fct; + } + DEALLOC(ch); + return 0; + } + +#undef PMSIGNC +#undef A_EQ_B_MUL_C +#undef A_EQ_CB_MUL_C +#undef MULPMSIGNC +#undef MULPMSIGNCEQ + +#undef WA +#undef CC +#undef CH +#undef ROT90 +#undef SCALEC +#undef ADDC +#undef PMC + +NOINLINE WARN_UNUSED_RESULT +static int cfftp_forward(cfftp_plan plan, double c[], double fct) + { return pass_all(plan,(cmplx *)c, fct, -1); } + +NOINLINE WARN_UNUSED_RESULT +static int cfftp_backward(cfftp_plan plan, double c[], double fct) + { return pass_all(plan,(cmplx *)c, fct, 1); } + +NOINLINE WARN_UNUSED_RESULT +static int cfftp_factorize (cfftp_plan plan) + { + size_t length=plan->length; + size_t nfct=0; + while ((length%4)==0) + { if (nfct>=NFCT) return -1; plan->fct[nfct++].fct=4; length>>=2; } + if ((length%2)==0) + { + length>>=1; + // factor 2 should be at the front of the factor list + if (nfct>=NFCT) return -1; + plan->fct[nfct++].fct=2; + SWAP(plan->fct[0].fct, plan->fct[nfct-1].fct,size_t); + } + size_t maxl=(size_t)(sqrt((double)length))+1; + for (size_t divisor=3; (length>1)&&(divisor<maxl); divisor+=2) + if ((length%divisor)==0) + { + while ((length%divisor)==0) + { + if (nfct>=NFCT) return -1; + plan->fct[nfct++].fct=divisor; + length/=divisor; + } + maxl=(size_t)(sqrt((double)length))+1; + } + if (length>1) plan->fct[nfct++].fct=length; + plan->nfct=nfct; + return 0; + } + +NOINLINE static size_t cfftp_twsize (cfftp_plan plan) + { + size_t twsize=0, l1=1; + for (size_t k=0; k<plan->nfct; ++k) + { + size_t ip=plan->fct[k].fct, ido= plan->length/(l1*ip); + twsize+=(ip-1)*(ido-1); + if (ip>11) + twsize+=ip; + l1*=ip; + } + return twsize; + } + +NOINLINE WARN_UNUSED_RESULT static int cfftp_comp_twiddle (cfftp_plan plan) + { + size_t length=plan->length; + double *twid = RALLOC(double, 2*length); + if (!twid) return -1; + sincos_2pibyn(length, twid); + size_t l1=1; + size_t memofs=0; + for (size_t k=0; k<plan->nfct; ++k) + { + size_t ip=plan->fct[k].fct, ido= length/(l1*ip); + plan->fct[k].tw=plan->mem+memofs; + memofs+=(ip-1)*(ido-1); + for (size_t j=1; j<ip; ++j) + for (size_t i=1; i<ido; ++i) + { + plan->fct[k].tw[(j-1)*(ido-1)+i-1].r = twid[2*j*l1*i]; + plan->fct[k].tw[(j-1)*(ido-1)+i-1].i = twid[2*j*l1*i+1]; + } + if (ip>11) + { + plan->fct[k].tws=plan->mem+memofs; + memofs+=ip; + for (size_t j=0; j<ip; ++j) + { + plan->fct[k].tws[j].r = twid[2*j*l1*ido]; + plan->fct[k].tws[j].i = twid[2*j*l1*ido+1]; + } + } + l1*=ip; + } + DEALLOC(twid); + return 0; + } + +static cfftp_plan make_cfftp_plan (size_t length) + { + if (length==0) return NULL; + cfftp_plan plan = RALLOC(cfftp_plan_i,1); + if (!plan) return NULL; + plan->length=length; + plan->nfct=0; + for (size_t i=0; i<NFCT; ++i) + plan->fct[i]=(cfftp_fctdata){0,0,0}; + plan->mem=0; + if (length==1) return plan; + if (cfftp_factorize(plan)!=0) { DEALLOC(plan); return NULL; } + size_t tws=cfftp_twsize(plan); + plan->mem=RALLOC(cmplx,tws); + if (!plan->mem) { DEALLOC(plan); return NULL; } + if (cfftp_comp_twiddle(plan)!=0) + { DEALLOC(plan->mem); DEALLOC(plan); return NULL; } + return plan; + } + +static void destroy_cfftp_plan (cfftp_plan plan) + { + DEALLOC(plan->mem); + DEALLOC(plan); + } + +typedef struct rfftp_fctdata + { + size_t fct; + double *tw, *tws; + } rfftp_fctdata; + +typedef struct rfftp_plan_i + { + size_t length, nfct; + double *mem; + rfftp_fctdata fct[NFCT]; + } rfftp_plan_i; +typedef struct rfftp_plan_i * rfftp_plan; + +#define WA(x,i) wa[(i)+(x)*(ido-1)] +#define PM(a,b,c,d) { a=c+d; b=c-d; } +/* (a+ib) = conj(c+id) * (e+if) */ +#define MULPM(a,b,c,d,e,f) { a=c*e+d*f; b=c*f-d*e; } + +#define CC(a,b,c) cc[(a)+ido*((b)+l1*(c))] +#define CH(a,b,c) ch[(a)+ido*((b)+cdim*(c))] + +NOINLINE static void radf2 (size_t ido, size_t l1, const double * restrict cc, + double * restrict ch, const double * restrict wa) + { + const size_t cdim=2; + + for (size_t k=0; k<l1; k++) + PM (CH(0,0,k),CH(ido-1,1,k),CC(0,k,0),CC(0,k,1)) + if ((ido&1)==0) + for (size_t k=0; k<l1; k++) + { + CH( 0,1,k) = -CC(ido-1,k,1); + CH(ido-1,0,k) = CC(ido-1,k,0); + } + if (ido<=2) return; + for (size_t k=0; k<l1; k++) + for (size_t i=2; i<ido; i+=2) + { + size_t ic=ido-i; + double tr2, ti2; + MULPM (tr2,ti2,WA(0,i-2),WA(0,i-1),CC(i-1,k,1),CC(i,k,1)) + PM (CH(i-1,0,k),CH(ic-1,1,k),CC(i-1,k,0),tr2) + PM (CH(i ,0,k),CH(ic ,1,k),ti2,CC(i ,k,0)) + } + } + +NOINLINE static void radf3(size_t ido, size_t l1, const double * restrict cc, + double * restrict ch, const double * restrict wa) + { + const size_t cdim=3; + static const double taur=-0.5, taui=0.86602540378443864676; + + for (size_t k=0; k<l1; k++) + { + double cr2=CC(0,k,1)+CC(0,k,2); + CH(0,0,k) = CC(0,k,0)+cr2; + CH(0,2,k) = taui*(CC(0,k,2)-CC(0,k,1)); + CH(ido-1,1,k) = CC(0,k,0)+taur*cr2; + } + if (ido==1) return; + for (size_t k=0; k<l1; k++) + for (size_t i=2; i<ido; i+=2) + { + size_t ic=ido-i; + double di2, di3, dr2, dr3; + MULPM (dr2,di2,WA(0,i-2),WA(0,i-1),CC(i-1,k,1),CC(i,k,1)) // d2=conj(WA0)*CC1 + MULPM (dr3,di3,WA(1,i-2),WA(1,i-1),CC(i-1,k,2),CC(i,k,2)) // d3=conj(WA1)*CC2 + double cr2=dr2+dr3; // c add + double ci2=di2+di3; + CH(i-1,0,k) = CC(i-1,k,0)+cr2; // c add + CH(i ,0,k) = CC(i ,k,0)+ci2; + double tr2 = CC(i-1,k,0)+taur*cr2; // c add + double ti2 = CC(i ,k,0)+taur*ci2; + double tr3 = taui*(di2-di3); // t3 = taui*i*(d3-d2)? + double ti3 = taui*(dr3-dr2); + PM(CH(i-1,2,k),CH(ic-1,1,k),tr2,tr3) // PM(i) = t2+t3 + PM(CH(i ,2,k),CH(ic ,1,k),ti3,ti2) // PM(ic) = conj(t2-t3) + } + } + +NOINLINE static void radf4(size_t ido, size_t l1, const double * restrict cc, + double * restrict ch, const double * restrict wa) + { + const size_t cdim=4; + static const double hsqt2=0.70710678118654752440; + + for (size_t k=0; k<l1; k++) + { + double tr1,tr2; + PM (tr1,CH(0,2,k),CC(0,k,3),CC(0,k,1)) + PM (tr2,CH(ido-1,1,k),CC(0,k,0),CC(0,k,2)) + PM (CH(0,0,k),CH(ido-1,3,k),tr2,tr1) + } + if ((ido&1)==0) + for (size_t k=0; k<l1; k++) + { + double ti1=-hsqt2*(CC(ido-1,k,1)+CC(ido-1,k,3)); + double tr1= hsqt2*(CC(ido-1,k,1)-CC(ido-1,k,3)); + PM (CH(ido-1,0,k),CH(ido-1,2,k),CC(ido-1,k,0),tr1) + PM (CH( 0,3,k),CH( 0,1,k),ti1,CC(ido-1,k,2)) + } + if (ido<=2) return; + for (size_t k=0; k<l1; k++) + for (size_t i=2; i<ido; i+=2) + { + size_t ic=ido-i; + double ci2, ci3, ci4, cr2, cr3, cr4, ti1, ti2, ti3, ti4, tr1, tr2, tr3, tr4; + MULPM(cr2,ci2,WA(0,i-2),WA(0,i-1),CC(i-1,k,1),CC(i,k,1)) + MULPM(cr3,ci3,WA(1,i-2),WA(1,i-1),CC(i-1,k,2),CC(i,k,2)) + MULPM(cr4,ci4,WA(2,i-2),WA(2,i-1),CC(i-1,k,3),CC(i,k,3)) + PM(tr1,tr4,cr4,cr2) + PM(ti1,ti4,ci2,ci4) + PM(tr2,tr3,CC(i-1,k,0),cr3) + PM(ti2,ti3,CC(i ,k,0),ci3) + PM(CH(i-1,0,k),CH(ic-1,3,k),tr2,tr1) + PM(CH(i ,0,k),CH(ic ,3,k),ti1,ti2) + PM(CH(i-1,2,k),CH(ic-1,1,k),tr3,ti4) + PM(CH(i ,2,k),CH(ic ,1,k),tr4,ti3) + } + } + +NOINLINE static void radf5(size_t ido, size_t l1, const double * restrict cc, + double * restrict ch, const double * restrict wa) + { + const size_t cdim=5; + static const double tr11= 0.3090169943749474241, ti11=0.95105651629515357212, + tr12=-0.8090169943749474241, ti12=0.58778525229247312917; + + for (size_t k=0; k<l1; k++) + { + double cr2, cr3, ci4, ci5; + PM (cr2,ci5,CC(0,k,4),CC(0,k,1)) + PM (cr3,ci4,CC(0,k,3),CC(0,k,2)) + CH(0,0,k)=CC(0,k,0)+cr2+cr3; + CH(ido-1,1,k)=CC(0,k,0)+tr11*cr2+tr12*cr3; + CH(0,2,k)=ti11*ci5+ti12*ci4; + CH(ido-1,3,k)=CC(0,k,0)+tr12*cr2+tr11*cr3; + CH(0,4,k)=ti12*ci5-ti11*ci4; + } + if (ido==1) return; + for (size_t k=0; k<l1;++k) + for (size_t i=2; i<ido; i+=2) + { + double ci2, di2, ci4, ci5, di3, di4, di5, ci3, cr2, cr3, dr2, dr3, + dr4, dr5, cr5, cr4, ti2, ti3, ti5, ti4, tr2, tr3, tr4, tr5; + size_t ic=ido-i; + MULPM (dr2,di2,WA(0,i-2),WA(0,i-1),CC(i-1,k,1),CC(i,k,1)) + MULPM (dr3,di3,WA(1,i-2),WA(1,i-1),CC(i-1,k,2),CC(i,k,2)) + MULPM (dr4,di4,WA(2,i-2),WA(2,i-1),CC(i-1,k,3),CC(i,k,3)) + MULPM (dr5,di5,WA(3,i-2),WA(3,i-1),CC(i-1,k,4),CC(i,k,4)) + PM(cr2,ci5,dr5,dr2) + PM(ci2,cr5,di2,di5) + PM(cr3,ci4,dr4,dr3) + PM(ci3,cr4,di3,di4) + CH(i-1,0,k)=CC(i-1,k,0)+cr2+cr3; + CH(i ,0,k)=CC(i ,k,0)+ci2+ci3; + tr2=CC(i-1,k,0)+tr11*cr2+tr12*cr3; + ti2=CC(i ,k,0)+tr11*ci2+tr12*ci3; + tr3=CC(i-1,k,0)+tr12*cr2+tr11*cr3; + ti3=CC(i ,k,0)+tr12*ci2+tr11*ci3; + MULPM(tr5,tr4,cr5,cr4,ti11,ti12) + MULPM(ti5,ti4,ci5,ci4,ti11,ti12) + PM(CH(i-1,2,k),CH(ic-1,1,k),tr2,tr5) + PM(CH(i ,2,k),CH(ic ,1,k),ti5,ti2) + PM(CH(i-1,4,k),CH(ic-1,3,k),tr3,tr4) + PM(CH(i ,4,k),CH(ic ,3,k),ti4,ti3) + } + } + +#undef CC +#undef CH +#define C1(a,b,c) cc[(a)+ido*((b)+l1*(c))] +#define C2(a,b) cc[(a)+idl1*(b)] +#define CH2(a,b) ch[(a)+idl1*(b)] +#define CC(a,b,c) cc[(a)+ido*((b)+cdim*(c))] +#define CH(a,b,c) ch[(a)+ido*((b)+l1*(c))] +NOINLINE static void radfg(size_t ido, size_t ip, size_t l1, + double * restrict cc, double * restrict ch, const double * restrict wa, + const double * restrict csarr) + { + const size_t cdim=ip; + size_t ipph=(ip+1)/2; + size_t idl1 = ido*l1; + + if (ido>1) + { + for (size_t j=1, jc=ip-1; j<ipph; ++j,--jc) // 114 + { + size_t is=(j-1)*(ido-1), + is2=(jc-1)*(ido-1); + for (size_t k=0; k<l1; ++k) // 113 + { + size_t idij=is; + size_t idij2=is2; + for (size_t i=1; i<=ido-2; i+=2) // 112 + { + double t1=C1(i,k,j ), t2=C1(i+1,k,j ), + t3=C1(i,k,jc), t4=C1(i+1,k,jc); + double x1=wa[idij]*t1 + wa[idij+1]*t2, + x2=wa[idij]*t2 - wa[idij+1]*t1, + x3=wa[idij2]*t3 + wa[idij2+1]*t4, + x4=wa[idij2]*t4 - wa[idij2+1]*t3; + C1(i ,k,j ) = x1+x3; + C1(i ,k,jc) = x2-x4; + C1(i+1,k,j ) = x2+x4; + C1(i+1,k,jc) = x3-x1; + idij+=2; + idij2+=2; + } + } + } + } + + for (size_t j=1, jc=ip-1; j<ipph; ++j,--jc) // 123 + for (size_t k=0; k<l1; ++k) // 122 + { + double t1=C1(0,k,j), t2=C1(0,k,jc); + C1(0,k,j ) = t1+t2; + C1(0,k,jc) = t2-t1; + } + +//everything in C +//memset(ch,0,ip*l1*ido*sizeof(double)); + + for (size_t l=1,lc=ip-1; l<ipph; ++l,--lc) // 127 + { + for (size_t ik=0; ik<idl1; ++ik) // 124 + { + CH2(ik,l ) = C2(ik,0)+csarr[2*l]*C2(ik,1)+csarr[4*l]*C2(ik,2); + CH2(ik,lc) = csarr[2*l+1]*C2(ik,ip-1)+csarr[4*l+1]*C2(ik,ip-2); + } + size_t iang = 2*l; + size_t j=3, jc=ip-3; + for (; j<ipph-3; j+=4,jc-=4) // 126 + { + iang+=l; if (iang>=ip) iang-=ip; + double ar1=csarr[2*iang], ai1=csarr[2*iang+1]; + iang+=l; if (iang>=ip) iang-=ip; + double ar2=csarr[2*iang], ai2=csarr[2*iang+1]; + iang+=l; if (iang>=ip) iang-=ip; + double ar3=csarr[2*iang], ai3=csarr[2*iang+1]; + iang+=l; if (iang>=ip) iang-=ip; + double ar4=csarr[2*iang], ai4=csarr[2*iang+1]; + for (size_t ik=0; ik<idl1; ++ik) // 125 + { + CH2(ik,l ) += ar1*C2(ik,j )+ar2*C2(ik,j +1) + +ar3*C2(ik,j +2)+ar4*C2(ik,j +3); + CH2(ik,lc) += ai1*C2(ik,jc)+ai2*C2(ik,jc-1) + +ai3*C2(ik,jc-2)+ai4*C2(ik,jc-3); + } + } + for (; j<ipph-1; j+=2,jc-=2) // 126 + { + iang+=l; if (iang>=ip) iang-=ip; + double ar1=csarr[2*iang], ai1=csarr[2*iang+1]; + iang+=l; if (iang>=ip) iang-=ip; + double ar2=csarr[2*iang], ai2=csarr[2*iang+1]; + for (size_t ik=0; ik<idl1; ++ik) // 125 + { + CH2(ik,l ) += ar1*C2(ik,j )+ar2*C2(ik,j +1); + CH2(ik,lc) += ai1*C2(ik,jc)+ai2*C2(ik,jc-1); + } + } + for (; j<ipph; ++j,--jc) // 126 + { + iang+=l; if (iang>=ip) iang-=ip; + double ar=csarr[2*iang], ai=csarr[2*iang+1]; + for (size_t ik=0; ik<idl1; ++ik) // 125 + { + CH2(ik,l ) += ar*C2(ik,j ); + CH2(ik,lc) += ai*C2(ik,jc); + } + } + } + for (size_t ik=0; ik<idl1; ++ik) // 101 + CH2(ik,0) = C2(ik,0); + for (size_t j=1; j<ipph; ++j) // 129 + for (size_t ik=0; ik<idl1; ++ik) // 128 + CH2(ik,0) += C2(ik,j); + +// everything in CH at this point! +//memset(cc,0,ip*l1*ido*sizeof(double)); + + for (size_t k=0; k<l1; ++k) // 131 + for (size_t i=0; i<ido; ++i) // 130 + CC(i,0,k) = CH(i,k,0); + + for (size_t j=1, jc=ip-1; j<ipph; ++j,--jc) // 137 + { + size_t j2=2*j-1; + for (size_t k=0; k<l1; ++k) // 136 + { + CC(ido-1,j2,k) = CH(0,k,j); + CC(0,j2+1,k) = CH(0,k,jc); + } + } + + if (ido==1) return; + + for (size_t j=1, jc=ip-1; j<ipph; ++j,--jc) // 140 + { + size_t j2=2*j-1; + for(size_t k=0; k<l1; ++k) // 139 + for(size_t i=1, ic=ido-i-2; i<=ido-2; i+=2, ic-=2) // 138 + { + CC(i ,j2+1,k) = CH(i ,k,j )+CH(i ,k,jc); + CC(ic ,j2 ,k) = CH(i ,k,j )-CH(i ,k,jc); + CC(i+1 ,j2+1,k) = CH(i+1,k,j )+CH(i+1,k,jc); + CC(ic+1,j2 ,k) = CH(i+1,k,jc)-CH(i+1,k,j ); + } + } + } +#undef C1 +#undef C2 +#undef CH2 + +#undef CH +#undef CC +#define CH(a,b,c) ch[(a)+ido*((b)+l1*(c))] +#define CC(a,b,c) cc[(a)+ido*((b)+cdim*(c))] + +NOINLINE static void radb2(size_t ido, size_t l1, const double * restrict cc, + double * restrict ch, const double * restrict wa) + { + const size_t cdim=2; + + for (size_t k=0; k<l1; k++) + PM (CH(0,k,0),CH(0,k,1),CC(0,0,k),CC(ido-1,1,k)) + if ((ido&1)==0) + for (size_t k=0; k<l1; k++) + { + CH(ido-1,k,0) = 2.*CC(ido-1,0,k); + CH(ido-1,k,1) =-2.*CC(0 ,1,k); + } + if (ido<=2) return; + for (size_t k=0; k<l1;++k) + for (size_t i=2; i<ido; i+=2) + { + size_t ic=ido-i; + double ti2, tr2; + PM (CH(i-1,k,0),tr2,CC(i-1,0,k),CC(ic-1,1,k)) + PM (ti2,CH(i ,k,0),CC(i ,0,k),CC(ic ,1,k)) + MULPM (CH(i,k,1),CH(i-1,k,1),WA(0,i-2),WA(0,i-1),ti2,tr2) + } + } + +NOINLINE static void radb3(size_t ido, size_t l1, const double * restrict cc, + double * restrict ch, const double * restrict wa) + { + const size_t cdim=3; + static const double taur=-0.5, taui=0.86602540378443864676; + + for (size_t k=0; k<l1; k++) + { + double tr2=2.*CC(ido-1,1,k); + double cr2=CC(0,0,k)+taur*tr2; + CH(0,k,0)=CC(0,0,k)+tr2; + double ci3=2.*taui*CC(0,2,k); + PM (CH(0,k,2),CH(0,k,1),cr2,ci3); + } + if (ido==1) return; + for (size_t k=0; k<l1; k++) + for (size_t i=2; i<ido; i+=2) + { + size_t ic=ido-i; + double tr2=CC(i-1,2,k)+CC(ic-1,1,k); // t2=CC(I) + conj(CC(ic)) + double ti2=CC(i ,2,k)-CC(ic ,1,k); + double cr2=CC(i-1,0,k)+taur*tr2; // c2=CC +taur*t2 + double ci2=CC(i ,0,k)+taur*ti2; + CH(i-1,k,0)=CC(i-1,0,k)+tr2; // CH=CC+t2 + CH(i ,k,0)=CC(i ,0,k)+ti2; + double cr3=taui*(CC(i-1,2,k)-CC(ic-1,1,k));// c3=taui*(CC(i)-conj(CC(ic))) + double ci3=taui*(CC(i ,2,k)+CC(ic ,1,k)); + double di2, di3, dr2, dr3; + PM(dr3,dr2,cr2,ci3) // d2= (cr2-ci3, ci2+cr3) = c2+i*c3 + PM(di2,di3,ci2,cr3) // d3= (cr2+ci3, ci2-cr3) = c2-i*c3 + MULPM(CH(i,k,1),CH(i-1,k,1),WA(0,i-2),WA(0,i-1),di2,dr2) // ch = WA*d2 + MULPM(CH(i,k,2),CH(i-1,k,2),WA(1,i-2),WA(1,i-1),di3,dr3) + } + } + +NOINLINE static void radb4(size_t ido, size_t l1, const double * restrict cc, + double * restrict ch, const double * restrict wa) + { + const size_t cdim=4; + static const double sqrt2=1.41421356237309504880; + + for (size_t k=0; k<l1; k++) + { + double tr1, tr2; + PM (tr2,tr1,CC(0,0,k),CC(ido-1,3,k)) + double tr3=2.*CC(ido-1,1,k); + double tr4=2.*CC(0,2,k); + PM (CH(0,k,0),CH(0,k,2),tr2,tr3) + PM (CH(0,k,3),CH(0,k,1),tr1,tr4) + } + if ((ido&1)==0) + for (size_t k=0; k<l1; k++) + { + double tr1,tr2,ti1,ti2; + PM (ti1,ti2,CC(0 ,3,k),CC(0 ,1,k)) + PM (tr2,tr1,CC(ido-1,0,k),CC(ido-1,2,k)) + CH(ido-1,k,0)=tr2+tr2; + CH(ido-1,k,1)=sqrt2*(tr1-ti1); + CH(ido-1,k,2)=ti2+ti2; + CH(ido-1,k,3)=-sqrt2*(tr1+ti1); + } + if (ido<=2) return; + for (size_t k=0; k<l1;++k) + for (size_t i=2; i<ido; i+=2) + { + double ci2, ci3, ci4, cr2, cr3, cr4, ti1, ti2, ti3, ti4, tr1, tr2, tr3, tr4; + size_t ic=ido-i; + PM (tr2,tr1,CC(i-1,0,k),CC(ic-1,3,k)) + PM (ti1,ti2,CC(i ,0,k),CC(ic ,3,k)) + PM (tr4,ti3,CC(i ,2,k),CC(ic ,1,k)) + PM (tr3,ti4,CC(i-1,2,k),CC(ic-1,1,k)) + PM (CH(i-1,k,0),cr3,tr2,tr3) + PM (CH(i ,k,0),ci3,ti2,ti3) + PM (cr4,cr2,tr1,tr4) + PM (ci2,ci4,ti1,ti4) + MULPM (CH(i,k,1),CH(i-1,k,1),WA(0,i-2),WA(0,i-1),ci2,cr2) + MULPM (CH(i,k,2),CH(i-1,k,2),WA(1,i-2),WA(1,i-1),ci3,cr3) + MULPM (CH(i,k,3),CH(i-1,k,3),WA(2,i-2),WA(2,i-1),ci4,cr4) + } + } + +NOINLINE static void radb5(size_t ido, size_t l1, const double * restrict cc, + double * restrict ch, const double * restrict wa) + { + const size_t cdim=5; + static const double tr11= 0.3090169943749474241, ti11=0.95105651629515357212, + tr12=-0.8090169943749474241, ti12=0.58778525229247312917; + + for (size_t k=0; k<l1; k++) + { + double ti5=CC(0,2,k)+CC(0,2,k); + double ti4=CC(0,4,k)+CC(0,4,k); + double tr2=CC(ido-1,1,k)+CC(ido-1,1,k); + double tr3=CC(ido-1,3,k)+CC(ido-1,3,k); + CH(0,k,0)=CC(0,0,k)+tr2+tr3; + double cr2=CC(0,0,k)+tr11*tr2+tr12*tr3; + double cr3=CC(0,0,k)+tr12*tr2+tr11*tr3; + double ci4, ci5; + MULPM(ci5,ci4,ti5,ti4,ti11,ti12) + PM(CH(0,k,4),CH(0,k,1),cr2,ci5) + PM(CH(0,k,3),CH(0,k,2),cr3,ci4) + } + if (ido==1) return; + for (size_t k=0; k<l1;++k) + for (size_t i=2; i<ido; i+=2) + { + size_t ic=ido-i; + double tr2, tr3, tr4, tr5, ti2, ti3, ti4, ti5; + PM(tr2,tr5,CC(i-1,2,k),CC(ic-1,1,k)) + PM(ti5,ti2,CC(i ,2,k),CC(ic ,1,k)) + PM(tr3,tr4,CC(i-1,4,k),CC(ic-1,3,k)) + PM(ti4,ti3,CC(i ,4,k),CC(ic ,3,k)) + CH(i-1,k,0)=CC(i-1,0,k)+tr2+tr3; + CH(i ,k,0)=CC(i ,0,k)+ti2+ti3; + double cr2=CC(i-1,0,k)+tr11*tr2+tr12*tr3; + double ci2=CC(i ,0,k)+tr11*ti2+tr12*ti3; + double cr3=CC(i-1,0,k)+tr12*tr2+tr11*tr3; + double ci3=CC(i ,0,k)+tr12*ti2+tr11*ti3; + double ci4, ci5, cr5, cr4; + MULPM(cr5,cr4,tr5,tr4,ti11,ti12) + MULPM(ci5,ci4,ti5,ti4,ti11,ti12) + double dr2, dr3, dr4, dr5, di2, di3, di4, di5; + PM(dr4,dr3,cr3,ci4) + PM(di3,di4,ci3,cr4) + PM(dr5,dr2,cr2,ci5) + PM(di2,di5,ci2,cr5) + MULPM(CH(i,k,1),CH(i-1,k,1),WA(0,i-2),WA(0,i-1),di2,dr2) + MULPM(CH(i,k,2),CH(i-1,k,2),WA(1,i-2),WA(1,i-1),di3,dr3) + MULPM(CH(i,k,3),CH(i-1,k,3),WA(2,i-2),WA(2,i-1),di4,dr4) + MULPM(CH(i,k,4),CH(i-1,k,4),WA(3,i-2),WA(3,i-1),di5,dr5) + } + } + +#undef CC +#undef CH +#define CC(a,b,c) cc[(a)+ido*((b)+cdim*(c))] +#define CH(a,b,c) ch[(a)+ido*((b)+l1*(c))] +#define C1(a,b,c) cc[(a)+ido*((b)+l1*(c))] +#define C2(a,b) cc[(a)+idl1*(b)] +#define CH2(a,b) ch[(a)+idl1*(b)] + +NOINLINE static void radbg(size_t ido, size_t ip, size_t l1, + double * restrict cc, double * restrict ch, const double * restrict wa, + const double * restrict csarr) + { + const size_t cdim=ip; + size_t ipph=(ip+1)/ 2; + size_t idl1 = ido*l1; + + for (size_t k=0; k<l1; ++k) // 102 + for (size_t i=0; i<ido; ++i) // 101 + CH(i,k,0) = CC(i,0,k); + for (size_t j=1, jc=ip-1; j<ipph; ++j, --jc) // 108 + { + size_t j2=2*j-1; + for (size_t k=0; k<l1; ++k) + { + CH(0,k,j ) = 2*CC(ido-1,j2,k); + CH(0,k,jc) = 2*CC(0,j2+1,k); + } + } + + if (ido!=1) + { + for (size_t j=1, jc=ip-1; j<ipph; ++j,--jc) // 111 + { + size_t j2=2*j-1; + for (size_t k=0; k<l1; ++k) + for (size_t i=1, ic=ido-i-2; i<=ido-2; i+=2, ic-=2) // 109 + { + CH(i ,k,j ) = CC(i ,j2+1,k)+CC(ic ,j2,k); + CH(i ,k,jc) = CC(i ,j2+1,k)-CC(ic ,j2,k); + CH(i+1,k,j ) = CC(i+1,j2+1,k)-CC(ic+1,j2,k); + CH(i+1,k,jc) = CC(i+1,j2+1,k)+CC(ic+1,j2,k); + } + } + } + for (size_t l=1,lc=ip-1; l<ipph; ++l,--lc) + { + for (size_t ik=0; ik<idl1; ++ik) + { + C2(ik,l ) = CH2(ik,0)+csarr[2*l]*CH2(ik,1)+csarr[4*l]*CH2(ik,2); + C2(ik,lc) = csarr[2*l+1]*CH2(ik,ip-1)+csarr[4*l+1]*CH2(ik,ip-2); + } + size_t iang=2*l; + size_t j=3,jc=ip-3; + for(; j<ipph-3; j+=4,jc-=4) + { + iang+=l; if(iang>ip) iang-=ip; + double ar1=csarr[2*iang], ai1=csarr[2*iang+1]; + iang+=l; if(iang>ip) iang-=ip; + double ar2=csarr[2*iang], ai2=csarr[2*iang+1]; + iang+=l; if(iang>ip) iang-=ip; + double ar3=csarr[2*iang], ai3=csarr[2*iang+1]; + iang+=l; if(iang>ip) iang-=ip; + double ar4=csarr[2*iang], ai4=csarr[2*iang+1]; + for (size_t ik=0; ik<idl1; ++ik) + { + C2(ik,l ) += ar1*CH2(ik,j )+ar2*CH2(ik,j +1) + +ar3*CH2(ik,j +2)+ar4*CH2(ik,j +3); + C2(ik,lc) += ai1*CH2(ik,jc)+ai2*CH2(ik,jc-1) + +ai3*CH2(ik,jc-2)+ai4*CH2(ik,jc-3); + } + } + for(; j<ipph-1; j+=2,jc-=2) + { + iang+=l; if(iang>ip) iang-=ip; + double ar1=csarr[2*iang], ai1=csarr[2*iang+1]; + iang+=l; if(iang>ip) iang-=ip; + double ar2=csarr[2*iang], ai2=csarr[2*iang+1]; + for (size_t ik=0; ik<idl1; ++ik) + { + C2(ik,l ) += ar1*CH2(ik,j )+ar2*CH2(ik,j +1); + C2(ik,lc) += ai1*CH2(ik,jc)+ai2*CH2(ik,jc-1); + } + } + for(; j<ipph; ++j,--jc) + { + iang+=l; if(iang>ip) iang-=ip; + double war=csarr[2*iang], wai=csarr[2*iang+1]; + for (size_t ik=0; ik<idl1; ++ik) + { + C2(ik,l ) += war*CH2(ik,j ); + C2(ik,lc) += wai*CH2(ik,jc); + } + } + } + for (size_t j=1; j<ipph; ++j) + for (size_t ik=0; ik<idl1; ++ik) + CH2(ik,0) += CH2(ik,j); + for (size_t j=1, jc=ip-1; j<ipph; ++j,--jc) // 124 + for (size_t k=0; k<l1; ++k) + { + CH(0,k,j ) = C1(0,k,j)-C1(0,k,jc); + CH(0,k,jc) = C1(0,k,j)+C1(0,k,jc); + } + + if (ido==1) return; + + for (size_t j=1, jc=ip-1; j<ipph; ++j, --jc) // 127 + for (size_t k=0; k<l1; ++k) + for (size_t i=1; i<=ido-2; i+=2) + { + CH(i ,k,j ) = C1(i ,k,j)-C1(i+1,k,jc); + CH(i ,k,jc) = C1(i ,k,j)+C1(i+1,k,jc); + CH(i+1,k,j ) = C1(i+1,k,j)+C1(i ,k,jc); + CH(i+1,k,jc) = C1(i+1,k,j)-C1(i ,k,jc); + } + +// All in CH + + for (size_t j=1; j<ip; ++j) + { + size_t is = (j-1)*(ido-1); + for (size_t k=0; k<l1; ++k) + { + size_t idij = is; + for (size_t i=1; i<=ido-2; i+=2) + { + double t1=CH(i,k,j), t2=CH(i+1,k,j); + CH(i ,k,j) = wa[idij]*t1-wa[idij+1]*t2; + CH(i+1,k,j) = wa[idij]*t2+wa[idij+1]*t1; + idij+=2; + } + } + } + } +#undef C1 +#undef C2 +#undef CH2 + +#undef CC +#undef CH +#undef PM +#undef MULPM +#undef WA + +static void copy_and_norm(double *c, double *p1, size_t n, double fct) + { + if (p1!=c) + { + if (fct!=1.) + for (size_t i=0; i<n; ++i) + c[i] = fct*p1[i]; + else + memcpy (c,p1,n*sizeof(double)); + } + else + if (fct!=1.) + for (size_t i=0; i<n; ++i) + c[i] *= fct; + } + +WARN_UNUSED_RESULT +static int rfftp_forward(rfftp_plan plan, double c[], double fct) + { + if (plan->length==1) return 0; + size_t n=plan->length; + size_t l1=n, nf=plan->nfct; + double *ch = RALLOC(double, n); + if (!ch) return -1; + double *p1=c, *p2=ch; + + for(size_t k1=0; k1<nf;++k1) + { + size_t k=nf-k1-1; + size_t ip=plan->fct[k].fct; + size_t ido=n / l1; + l1 /= ip; + if(ip==4) + radf4(ido, l1, p1, p2, plan->fct[k].tw); + else if(ip==2) + radf2(ido, l1, p1, p2, plan->fct[k].tw); + else if(ip==3) + radf3(ido, l1, p1, p2, plan->fct[k].tw); + else if(ip==5) + radf5(ido, l1, p1, p2, plan->fct[k].tw); + else + { + radfg(ido, ip, l1, p1, p2, plan->fct[k].tw, plan->fct[k].tws); + SWAP (p1,p2,double *); + } + SWAP (p1,p2,double *); + } + copy_and_norm(c,p1,n,fct); + DEALLOC(ch); + return 0; + } + +WARN_UNUSED_RESULT +static int rfftp_backward(rfftp_plan plan, double c[], double fct) + { + if (plan->length==1) return 0; + size_t n=plan->length; + size_t l1=1, nf=plan->nfct; + double *ch = RALLOC(double, n); + if (!ch) return -1; + double *p1=c, *p2=ch; + + for(size_t k=0; k<nf; k++) + { + size_t ip = plan->fct[k].fct, + ido= n/(ip*l1); + if(ip==4) + radb4(ido, l1, p1, p2, plan->fct[k].tw); + else if(ip==2) + radb2(ido, l1, p1, p2, plan->fct[k].tw); + else if(ip==3) + radb3(ido, l1, p1, p2, plan->fct[k].tw); + else if(ip==5) + radb5(ido, l1, p1, p2, plan->fct[k].tw); + else + radbg(ido, ip, l1, p1, p2, plan->fct[k].tw, plan->fct[k].tws); + SWAP (p1,p2,double *); + l1*=ip; + } + copy_and_norm(c,p1,n,fct); + DEALLOC(ch); + return 0; + } + +WARN_UNUSED_RESULT +static int rfftp_factorize (rfftp_plan plan) + { + size_t length=plan->length; + size_t nfct=0; + while ((length%4)==0) + { if (nfct>=NFCT) return -1; plan->fct[nfct++].fct=4; length>>=2; } + if ((length%2)==0) + { + length>>=1; + // factor 2 should be at the front of the factor list + if (nfct>=NFCT) return -1; + plan->fct[nfct++].fct=2; + SWAP(plan->fct[0].fct, plan->fct[nfct-1].fct,size_t); + } + size_t maxl=(size_t)(sqrt((double)length))+1; + for (size_t divisor=3; (length>1)&&(divisor<maxl); divisor+=2) + if ((length%divisor)==0) + { + while ((length%divisor)==0) + { + if (nfct>=NFCT) return -1; + plan->fct[nfct++].fct=divisor; + length/=divisor; + } + maxl=(size_t)(sqrt((double)length))+1; + } + if (length>1) plan->fct[nfct++].fct=length; + plan->nfct=nfct; + return 0; + } + +static size_t rfftp_twsize(rfftp_plan plan) + { + size_t twsize=0, l1=1; + for (size_t k=0; k<plan->nfct; ++k) + { + size_t ip=plan->fct[k].fct, ido= plan->length/(l1*ip); + twsize+=(ip-1)*(ido-1); + if (ip>5) twsize+=2*ip; + l1*=ip; + } + return twsize; + return 0; + } + +WARN_UNUSED_RESULT NOINLINE static int rfftp_comp_twiddle (rfftp_plan plan) + { + size_t length=plan->length; + double *twid = RALLOC(double, 2*length); + if (!twid) return -1; + sincos_2pibyn_half(length, twid); + size_t l1=1; + double *ptr=plan->mem; + for (size_t k=0; k<plan->nfct; ++k) + { + size_t ip=plan->fct[k].fct, ido=length/(l1*ip); + if (k<plan->nfct-1) // last factor doesn't need twiddles + { + plan->fct[k].tw=ptr; ptr+=(ip-1)*(ido-1); + for (size_t j=1; j<ip; ++j) + for (size_t i=1; i<=(ido-1)/2; ++i) + { + plan->fct[k].tw[(j-1)*(ido-1)+2*i-2] = twid[2*j*l1*i]; + plan->fct[k].tw[(j-1)*(ido-1)+2*i-1] = twid[2*j*l1*i+1]; + } + } + if (ip>5) // special factors required by *g functions + { + plan->fct[k].tws=ptr; ptr+=2*ip; + plan->fct[k].tws[0] = 1.; + plan->fct[k].tws[1] = 0.; + for (size_t i=1; i<=(ip>>1); ++i) + { + plan->fct[k].tws[2*i ] = twid[2*i*(length/ip)]; + plan->fct[k].tws[2*i+1] = twid[2*i*(length/ip)+1]; + plan->fct[k].tws[2*(ip-i) ] = twid[2*i*(length/ip)]; + plan->fct[k].tws[2*(ip-i)+1] = -twid[2*i*(length/ip)+1]; + } + } + l1*=ip; + } + DEALLOC(twid); + return 0; + } + +NOINLINE static rfftp_plan make_rfftp_plan (size_t length) + { + if (length==0) return NULL; + rfftp_plan plan = RALLOC(rfftp_plan_i,1); + if (!plan) return NULL; + plan->length=length; + plan->nfct=0; + plan->mem=NULL; + for (size_t i=0; i<NFCT; ++i) + plan->fct[i]=(rfftp_fctdata){0,0,0}; + if (length==1) return plan; + if (rfftp_factorize(plan)!=0) { DEALLOC(plan); return NULL; } + size_t tws=rfftp_twsize(plan); + plan->mem=RALLOC(double,tws); + if (!plan->mem) { DEALLOC(plan); return NULL; } + if (rfftp_comp_twiddle(plan)!=0) + { DEALLOC(plan->mem); DEALLOC(plan); return NULL; } + return plan; + } + +NOINLINE static void destroy_rfftp_plan (rfftp_plan plan) + { + DEALLOC(plan->mem); + DEALLOC(plan); + } + +typedef struct fftblue_plan_i + { + size_t n, n2; + cfftp_plan plan; + double *mem; + double *bk, *bkf; + } fftblue_plan_i; +typedef struct fftblue_plan_i * fftblue_plan; + +NOINLINE static fftblue_plan make_fftblue_plan (size_t length) + { + fftblue_plan plan = RALLOC(fftblue_plan_i,1); + if (!plan) return NULL; + plan->n = length; + plan->n2 = good_size(plan->n*2-1); + plan->mem = RALLOC(double, 2*plan->n+2*plan->n2); + if (!plan->mem) { DEALLOC(plan); return NULL; } + plan->bk = plan->mem; + plan->bkf = plan->bk+2*plan->n; + +/* initialize b_k */ + double *tmp = RALLOC(double,4*plan->n); + if (!tmp) { DEALLOC(plan->mem); DEALLOC(plan); return NULL; } + sincos_2pibyn(2*plan->n,tmp); + plan->bk[0] = 1; + plan->bk[1] = 0; + + size_t coeff=0; + for (size_t m=1; m<plan->n; ++m) + { + coeff+=2*m-1; + if (coeff>=2*plan->n) coeff-=2*plan->n; + plan->bk[2*m ] = tmp[2*coeff ]; + plan->bk[2*m+1] = tmp[2*coeff+1]; + } + + /* initialize the zero-padded, Fourier transformed b_k. Add normalisation. */ + double xn2 = 1./plan->n2; + plan->bkf[0] = plan->bk[0]*xn2; + plan->bkf[1] = plan->bk[1]*xn2; + for (size_t m=2; m<2*plan->n; m+=2) + { + plan->bkf[m] = plan->bkf[2*plan->n2-m] = plan->bk[m] *xn2; + plan->bkf[m+1] = plan->bkf[2*plan->n2-m+1] = plan->bk[m+1] *xn2; + } + for (size_t m=2*plan->n;m<=(2*plan->n2-2*plan->n+1);++m) + plan->bkf[m]=0.; + plan->plan=make_cfftp_plan(plan->n2); + if (!plan->plan) + { DEALLOC(tmp); DEALLOC(plan->mem); DEALLOC(plan); return NULL; } + if (cfftp_forward(plan->plan,plan->bkf,1.)!=0) + { DEALLOC(tmp); DEALLOC(plan->mem); DEALLOC(plan); return NULL; } + DEALLOC(tmp); + + return plan; + } + +NOINLINE static void destroy_fftblue_plan (fftblue_plan plan) + { + DEALLOC(plan->mem); + destroy_cfftp_plan(plan->plan); + DEALLOC(plan); + } + +NOINLINE WARN_UNUSED_RESULT +static int fftblue_fft(fftblue_plan plan, double c[], int isign, double fct) + { + size_t n=plan->n; + size_t n2=plan->n2; + double *bk = plan->bk; + double *bkf = plan->bkf; + double *akf = RALLOC(double, 2*n2); + if (!akf) return -1; + +/* initialize a_k and FFT it */ + if (isign>0) + for (size_t m=0; m<2*n; m+=2) + { + akf[m] = c[m]*bk[m] - c[m+1]*bk[m+1]; + akf[m+1] = c[m]*bk[m+1] + c[m+1]*bk[m]; + } + else + for (size_t m=0; m<2*n; m+=2) + { + akf[m] = c[m]*bk[m] + c[m+1]*bk[m+1]; + akf[m+1] =-c[m]*bk[m+1] + c[m+1]*bk[m]; + } + for (size_t m=2*n; m<2*n2; ++m) + akf[m]=0; + + if (cfftp_forward (plan->plan,akf,fct)!=0) + { DEALLOC(akf); return -1; } + +/* do the convolution */ + if (isign>0) + for (size_t m=0; m<2*n2; m+=2) + { + double im = -akf[m]*bkf[m+1] + akf[m+1]*bkf[m]; + akf[m ] = akf[m]*bkf[m] + akf[m+1]*bkf[m+1]; + akf[m+1] = im; + } + else + for (size_t m=0; m<2*n2; m+=2) + { + double im = akf[m]*bkf[m+1] + akf[m+1]*bkf[m]; + akf[m ] = akf[m]*bkf[m] - akf[m+1]*bkf[m+1]; + akf[m+1] = im; + } + +/* inverse FFT */ + if (cfftp_backward (plan->plan,akf,1.)!=0) + { DEALLOC(akf); return -1; } + +/* multiply by b_k */ + if (isign>0) + for (size_t m=0; m<2*n; m+=2) + { + c[m] = bk[m] *akf[m] - bk[m+1]*akf[m+1]; + c[m+1] = bk[m+1]*akf[m] + bk[m] *akf[m+1]; + } + else + for (size_t m=0; m<2*n; m+=2) + { + c[m] = bk[m] *akf[m] + bk[m+1]*akf[m+1]; + c[m+1] =-bk[m+1]*akf[m] + bk[m] *akf[m+1]; + } + DEALLOC(akf); + return 0; + } + +WARN_UNUSED_RESULT +static int cfftblue_backward(fftblue_plan plan, double c[], double fct) + { return fftblue_fft(plan,c,1,fct); } + +WARN_UNUSED_RESULT +static int cfftblue_forward(fftblue_plan plan, double c[], double fct) + { return fftblue_fft(plan,c,-1,fct); } + +WARN_UNUSED_RESULT +static int rfftblue_backward(fftblue_plan plan, double c[], double fct) + { + size_t n=plan->n; + double *tmp = RALLOC(double,2*n); + if (!tmp) return -1; + tmp[0]=c[0]; + tmp[1]=0.; + memcpy (tmp+2,c+1, (n-1)*sizeof(double)); + if ((n&1)==0) tmp[n+1]=0.; + for (size_t m=2; m<n; m+=2) + { + tmp[2*n-m]=tmp[m]; + tmp[2*n-m+1]=-tmp[m+1]; + } + if (fftblue_fft(plan,tmp,1,fct)!=0) + { DEALLOC(tmp); return -1; } + for (size_t m=0; m<n; ++m) + c[m] = tmp[2*m]; + DEALLOC(tmp); + return 0; + } + +WARN_UNUSED_RESULT +static int rfftblue_forward(fftblue_plan plan, double c[], double fct) + { + size_t n=plan->n; + double *tmp = RALLOC(double,2*n); + if (!tmp) return -1; + for (size_t m=0; m<n; ++m) + { + tmp[2*m] = c[m]; + tmp[2*m+1] = 0.; + } + if (fftblue_fft(plan,tmp,-1,fct)!=0) + { DEALLOC(tmp); return -1; } + c[0] = tmp[0]; + memcpy (c+1, tmp+2, (n-1)*sizeof(double)); + DEALLOC(tmp); + return 0; + } + +typedef struct cfft_plan_i + { + cfftp_plan packplan; + fftblue_plan blueplan; + } cfft_plan_i; + +static cfft_plan make_cfft_plan (size_t length) + { + if (length==0) return NULL; + cfft_plan plan = RALLOC(cfft_plan_i,1); + if (!plan) return NULL; + plan->blueplan=0; + plan->packplan=0; + if ((length<50) || (largest_prime_factor(length)<=sqrt(length))) + { + plan->packplan=make_cfftp_plan(length); + if (!plan->packplan) { DEALLOC(plan); return NULL; } + return plan; + } + double comp1 = cost_guess(length); + double comp2 = 2*cost_guess(good_size(2*length-1)); + comp2*=1.5; /* fudge factor that appears to give good overall performance */ + if (comp2<comp1) // use Bluestein + { + plan->blueplan=make_fftblue_plan(length); + if (!plan->blueplan) { DEALLOC(plan); return NULL; } + } + else + { + plan->packplan=make_cfftp_plan(length); + if (!plan->packplan) { DEALLOC(plan); return NULL; } + } + return plan; + } + +static void destroy_cfft_plan (cfft_plan plan) + { + if (plan->blueplan) + destroy_fftblue_plan(plan->blueplan); + if (plan->packplan) + destroy_cfftp_plan(plan->packplan); + DEALLOC(plan); + } + +WARN_UNUSED_RESULT static int cfft_backward(cfft_plan plan, double c[], double fct) + { + if (plan->packplan) + return cfftp_backward(plan->packplan,c,fct); + // if (plan->blueplan) + return cfftblue_backward(plan->blueplan,c,fct); + } + +WARN_UNUSED_RESULT static int cfft_forward(cfft_plan plan, double c[], double fct) + { + if (plan->packplan) + return cfftp_forward(plan->packplan,c,fct); + // if (plan->blueplan) + return cfftblue_forward(plan->blueplan,c,fct); + } + +typedef struct rfft_plan_i + { + rfftp_plan packplan; + fftblue_plan blueplan; + } rfft_plan_i; + +static rfft_plan make_rfft_plan (size_t length) + { + if (length==0) return NULL; + rfft_plan plan = RALLOC(rfft_plan_i,1); + if (!plan) return NULL; + plan->blueplan=0; + plan->packplan=0; + if ((length<50) || (largest_prime_factor(length)<=sqrt(length))) + { + plan->packplan=make_rfftp_plan(length); + if (!plan->packplan) { DEALLOC(plan); return NULL; } + return plan; + } + double comp1 = 0.5*cost_guess(length); + double comp2 = 2*cost_guess(good_size(2*length-1)); + comp2*=1.5; /* fudge factor that appears to give good overall performance */ + if (comp2<comp1) // use Bluestein + { + plan->blueplan=make_fftblue_plan(length); + if (!plan->blueplan) { DEALLOC(plan); return NULL; } + } + else + { + plan->packplan=make_rfftp_plan(length); + if (!plan->packplan) { DEALLOC(plan); return NULL; } + } + return plan; + } + +static void destroy_rfft_plan (rfft_plan plan) + { + if (plan->blueplan) + destroy_fftblue_plan(plan->blueplan); + if (plan->packplan) + destroy_rfftp_plan(plan->packplan); + DEALLOC(plan); + } + +WARN_UNUSED_RESULT static int rfft_backward(rfft_plan plan, double c[], double fct) + { + if (plan->packplan) + return rfftp_backward(plan->packplan,c,fct); + else // if (plan->blueplan) + return rfftblue_backward(plan->blueplan,c,fct); + } + +WARN_UNUSED_RESULT static int rfft_forward(rfft_plan plan, double c[], double fct) + { + if (plan->packplan) + return rfftp_forward(plan->packplan,c,fct); + else // if (plan->blueplan) + return rfftblue_forward(plan->blueplan,c,fct); + } + +#define NPY_NO_DEPRECATED_API NPY_API_VERSION + +#include "Python.h" +#include "numpy/arrayobject.h" + +static PyObject * +execute_complex(PyObject *a1, int is_forward, double fct) +{ + PyArrayObject *data = (PyArrayObject *)PyArray_CopyFromObject(a1, NPY_CDOUBLE, 1, 0); + if (!data) return NULL; + + int npts = PyArray_DIM(data, PyArray_NDIM(data) - 1); + cfft_plan plan=NULL; + + int nrepeats = PyArray_SIZE(data)/npts; + double *dptr = (double *)PyArray_DATA(data); + int fail=0; + Py_BEGIN_ALLOW_THREADS; + NPY_SIGINT_ON; + plan = make_cfft_plan(npts); + if (!plan) fail=1; + if (!fail) + for (int i = 0; i < nrepeats; i++) { + int res = is_forward ? + cfft_forward(plan, dptr, fct) : cfft_backward(plan, dptr, fct); + if (res!=0) { fail=1; break; } + dptr += npts*2; + } + if (plan) destroy_cfft_plan(plan); + NPY_SIGINT_OFF; + Py_END_ALLOW_THREADS; + if (fail) { + Py_XDECREF(data); + return PyErr_NoMemory(); + } + return (PyObject *)data; +} + +static PyObject * +execute_real_forward(PyObject *a1, double fct) +{ + rfft_plan plan=NULL; + int fail = 0; + PyArrayObject *data = (PyArrayObject *)PyArray_ContiguousFromObject(a1, + NPY_DOUBLE, 1, 0); + if (!data) return NULL; + + int ndim = PyArray_NDIM(data); + const npy_intp *odim = PyArray_DIMS(data); + int npts = odim[ndim - 1]; + npy_intp *tdim=(npy_intp *)malloc(ndim*sizeof(npy_intp)); + if (!tdim) + { Py_XDECREF(data); return NULL; } + for (int d=0; d<ndim-1; ++d) + tdim[d] = odim[d]; + tdim[ndim-1] = npts/2 + 1; + PyArrayObject *ret = (PyArrayObject *)PyArray_Empty(ndim, + tdim, PyArray_DescrFromType(NPY_CDOUBLE), 0); + free(tdim); + if (!ret) fail=1; + if (!fail) { + int rstep = PyArray_DIM(ret, PyArray_NDIM(ret) - 1)*2; + + int nrepeats = PyArray_SIZE(data)/npts; + double *rptr = (double *)PyArray_DATA(ret), + *dptr = (double *)PyArray_DATA(data); + + Py_BEGIN_ALLOW_THREADS; + NPY_SIGINT_ON; + plan = make_rfft_plan(npts); + if (!plan) fail=1; + if (!fail) + for (int i = 0; i < nrepeats; i++) { + rptr[rstep-1] = 0.0; + memcpy((char *)(rptr+1), dptr, npts*sizeof(double)); + if (rfft_forward(plan, rptr+1, fct)!=0) {fail=1; break;} + rptr[0] = rptr[1]; + rptr[1] = 0.0; + rptr += rstep; + dptr += npts; + } + if (plan) destroy_rfft_plan(plan); + NPY_SIGINT_OFF; + Py_END_ALLOW_THREADS; + } + if (fail) { + Py_XDECREF(data); + Py_XDECREF(ret); + return PyErr_NoMemory(); + } + Py_DECREF(data); + return (PyObject *)ret; +} +static PyObject * +execute_real_backward(PyObject *a1, double fct) +{ + rfft_plan plan=NULL; + PyArrayObject *data = (PyArrayObject *)PyArray_ContiguousFromObject(a1, + NPY_CDOUBLE, 1, 0); + if (!data) return NULL; + int npts = PyArray_DIM(data, PyArray_NDIM(data) - 1); + PyArrayObject *ret = (PyArrayObject *)PyArray_Empty(PyArray_NDIM(data), + PyArray_DIMS(data), PyArray_DescrFromType(NPY_DOUBLE), 0); + int fail = 0; + if (!ret) fail=1; + if (!fail) { + int nrepeats = PyArray_SIZE(ret)/npts; + double *rptr = (double *)PyArray_DATA(ret), + *dptr = (double *)PyArray_DATA(data); + + Py_BEGIN_ALLOW_THREADS; + NPY_SIGINT_ON; + plan = make_rfft_plan(npts); + if (!plan) fail=1; + if (!fail) { + for (int i = 0; i < nrepeats; i++) { + memcpy((char *)(rptr + 1), (dptr + 2), (npts - 1)*sizeof(double)); + rptr[0] = dptr[0]; + if (rfft_backward(plan, rptr, fct)!=0) {fail=1; break;} + rptr += npts; + dptr += npts*2; + } + } + if (plan) destroy_rfft_plan(plan); + NPY_SIGINT_OFF; + Py_END_ALLOW_THREADS; + } + if (fail) { + Py_XDECREF(data); + Py_XDECREF(ret); + return PyErr_NoMemory(); + } + Py_DECREF(data); + return (PyObject *)ret; +} + +static PyObject * +execute_real(PyObject *a1, int is_forward, double fct) +{ + return is_forward ? execute_real_forward(a1, fct) + : execute_real_backward(a1, fct); +} + +static const char execute__doc__[] = ""; + +static PyObject * +execute(PyObject *NPY_UNUSED(self), PyObject *args) +{ + PyObject *a1; + int is_real, is_forward; + double fct; + + if(!PyArg_ParseTuple(args, "Oiid:execute", &a1, &is_real, &is_forward, &fct)) { + return NULL; + } + + return is_real ? execute_real(a1, is_forward, fct) + : execute_complex(a1, is_forward, fct); +} + +/* List of methods defined in the module */ + +static struct PyMethodDef methods[] = { + {"execute", execute, 1, execute__doc__}, + {NULL, NULL, 0, NULL} /* sentinel */ +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "pocketfft_internal", + NULL, + -1, + methods, + NULL, + NULL, + NULL, + NULL +}; +#endif + +/* Initialization function for the module */ +#if PY_MAJOR_VERSION >= 3 +#define RETVAL(x) x +PyMODINIT_FUNC PyInit_pocketfft_internal(void) +#else +#define RETVAL(x) +PyMODINIT_FUNC +initpocketfft_internal(void) +#endif +{ + PyObject *m; +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&moduledef); +#else + static const char module_documentation[] = ""; + + m = Py_InitModule4("pocketfft_internal", methods, + module_documentation, + (PyObject*)NULL,PYTHON_API_VERSION); +#endif + if (m == NULL) { + return RETVAL(NULL); + } + + /* Import the array object */ + import_array(); + + /* XXXX Add constants here */ + + return RETVAL(m); +} diff --git a/numpy/fft/fftpack.py b/numpy/fft/pocketfft.py index d88990373..45dc162f6 100644 --- a/numpy/fft/fftpack.py +++ b/numpy/fft/pocketfft.py @@ -26,31 +26,26 @@ n = n-dimensional transform (Note: 2D routines are just nD routines with different default behavior.) -The underlying code for these functions is an f2c-translated and modified -version of the FFTPACK routines. - """ from __future__ import division, absolute_import, print_function __all__ = ['fft', 'ifft', 'rfft', 'irfft', 'hfft', 'ihfft', 'rfftn', 'irfftn', 'rfft2', 'irfft2', 'fft2', 'ifft2', 'fftn', 'ifftn'] -from numpy.core import (array, asarray, zeros, swapaxes, shape, conjugate, - take, sqrt) +import functools + +from numpy.core import asarray, zeros, swapaxes, conjugate, take, sqrt +from . import pocketfft_internal as pfi from numpy.core.multiarray import normalize_axis_index -from numpy.core.overrides import array_function_dispatch -from . import fftpack_lite as fftpack -from .helper import _FFTCache +from numpy.core import overrides -_fft_cache = _FFTCache(max_size_in_mb=100, max_item_count=32) -_real_fft_cache = _FFTCache(max_size_in_mb=100, max_item_count=32) +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy.fft') -def _raw_fft(a, n=None, axis=-1, init_function=fftpack.cffti, - work_function=fftpack.cfftf, fft_cache=_fft_cache): - a = asarray(a) - axis = normalize_axis_index(axis, a.ndim) +def _raw_fft(a, n, axis, is_real, is_forward, fct): + axis = normalize_axis_index(axis, a.ndim) if n is None: n = a.shape[axis] @@ -58,15 +53,6 @@ def _raw_fft(a, n=None, axis=-1, init_function=fftpack.cffti, raise ValueError("Invalid number of FFT data points (%d) specified." % n) - # We have to ensure that only a single thread can access a wsave array - # at any given time. Thus we remove it from the cache and insert it - # again after it has been used. Multiple threads might create multiple - # copies of the wsave array. This is intentional and a limitation of - # the current C code. - wsave = fft_cache.pop_twiddle_factors(n) - if wsave is None: - wsave = init_function(n) - if a.shape[axis] != n: s = list(a.shape) if s[axis] > n: @@ -81,25 +67,22 @@ def _raw_fft(a, n=None, axis=-1, init_function=fftpack.cffti, z[tuple(index)] = a a = z - if axis != a.ndim - 1: + if axis == a.ndim-1: + r = pfi.execute(a, is_real, is_forward, fct) + else: a = swapaxes(a, axis, -1) - r = work_function(a, wsave) - if axis != a.ndim - 1: + r = pfi.execute(a, is_real, is_forward, fct) r = swapaxes(r, axis, -1) - - # As soon as we put wsave back into the cache, another thread could pick it - # up and start using it, so we must not do this until after we're - # completely done using it ourselves. - fft_cache.put_twiddle_factors(n, wsave) - return r def _unitary(norm): - if norm not in (None, "ortho"): - raise ValueError("Invalid norm value %s, should be None or \"ortho\"." - % norm) - return norm is not None + if norm is None: + return False + if norm=="ortho": + return True + raise ValueError("Invalid norm value %s, should be None or \"ortho\"." + % norm) def _fft_dispatcher(a, n=None, axis=None, norm=None): @@ -171,14 +154,10 @@ def fft(a, n=None, axis=-1, norm=None): Examples -------- >>> np.fft.fft(np.exp(2j * np.pi * np.arange(8) / 8)) - array([ -3.44505240e-16 +1.14383329e-17j, - 8.00000000e+00 -5.71092652e-15j, - 2.33482938e-16 +1.22460635e-16j, - 1.64863782e-15 +1.77635684e-15j, - 9.95839695e-17 +2.33482938e-16j, - 0.00000000e+00 +1.66837030e-15j, - 1.14383329e-17 +1.22460635e-16j, - -1.64863782e-15 +1.77635684e-15j]) + array([-2.33486982e-16+1.14423775e-17j, 8.00000000e+00-1.25557246e-15j, + 2.33486982e-16+2.33486982e-16j, 0.00000000e+00+1.22464680e-16j, + -1.14423775e-17+2.33486982e-16j, 0.00000000e+00+5.20784380e-16j, + 1.14423775e-17+1.14423775e-17j, 0.00000000e+00+1.22464680e-16j]) In this example, real input has an FFT which is Hermitian, i.e., symmetric in the real part and anti-symmetric in the imaginary part, as described in @@ -194,12 +173,13 @@ def fft(a, n=None, axis=-1, norm=None): """ - a = asarray(a).astype(complex, copy=False) + a = asarray(a) if n is None: n = a.shape[axis] - output = _raw_fft(a, n, axis, fftpack.cffti, fftpack.cfftf, _fft_cache) - if _unitary(norm): - output *= 1 / sqrt(n) + fct = 1 + if norm is not None and _unitary(norm): + fct = 1 / sqrt(n) + output = _raw_fft(a, n, axis, False, True, fct) return output @@ -272,7 +252,7 @@ def ifft(a, n=None, axis=-1, norm=None): Examples -------- >>> np.fft.ifft([0, 4, 0, 0]) - array([ 1.+0.j, 0.+1.j, -1.+0.j, 0.-1.j]) + array([ 1.+0.j, 0.+1.j, -1.+0.j, 0.-1.j]) # may vary Create and plot a band-limited signal with random phases: @@ -282,19 +262,20 @@ def ifft(a, n=None, axis=-1, norm=None): >>> n[40:60] = np.exp(1j*np.random.uniform(0, 2*np.pi, (20,))) >>> s = np.fft.ifft(n) >>> plt.plot(t, s.real, 'b-', t, s.imag, 'r--') - ... + [<matplotlib.lines.Line2D object at ...>, <matplotlib.lines.Line2D object at ...>] >>> plt.legend(('real', 'imaginary')) - ... + <matplotlib.legend.Legend object at ...> >>> plt.show() """ - # The copy may be required for multithreading. - a = array(a, copy=True, dtype=complex) + a = asarray(a) if n is None: n = a.shape[axis] - unitary = _unitary(norm) - output = _raw_fft(a, n, axis, fftpack.cffti, fftpack.cfftb, _fft_cache) - return output * (1 / (sqrt(n) if unitary else n)) + fct = 1/n + if norm is not None and _unitary(norm): + fct = 1/sqrt(n) + output = _raw_fft(a, n, axis, False, False, fct) + return output @@ -368,23 +349,22 @@ def rfft(a, n=None, axis=-1, norm=None): Examples -------- >>> np.fft.fft([0, 1, 0, 0]) - array([ 1.+0.j, 0.-1.j, -1.+0.j, 0.+1.j]) + array([ 1.+0.j, 0.-1.j, -1.+0.j, 0.+1.j]) # may vary >>> np.fft.rfft([0, 1, 0, 0]) - array([ 1.+0.j, 0.-1.j, -1.+0.j]) + array([ 1.+0.j, 0.-1.j, -1.+0.j]) # may vary Notice how the final element of the `fft` output is the complex conjugate of the second element, for real input. For `rfft`, this symmetry is exploited to compute only the non-negative frequency terms. """ - # The copy may be required for multithreading. - a = array(a, copy=True, dtype=float) - output = _raw_fft(a, n, axis, fftpack.rffti, fftpack.rfftf, - _real_fft_cache) - if _unitary(norm): + a = asarray(a) + fct = 1 + if norm is not None and _unitary(norm): if n is None: n = a.shape[axis] - output *= 1 / sqrt(n) + fct = 1/sqrt(n) + output = _raw_fft(a, n, axis, True, True, fct) return output @@ -459,9 +439,9 @@ def irfft(a, n=None, axis=-1, norm=None): Examples -------- >>> np.fft.ifft([1, -1j, -1, 1j]) - array([ 0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j]) + array([0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j]) # may vary >>> np.fft.irfft([1, -1j, -1]) - array([ 0., 1., 0., 0.]) + array([0., 1., 0., 0.]) Notice how the last term in the input to the ordinary `ifft` is the complex conjugate of the second term, and the output has zero imaginary @@ -469,14 +449,14 @@ def irfft(a, n=None, axis=-1, norm=None): specified, and the output array is purely real. """ - # The copy may be required for multithreading. - a = array(a, copy=True, dtype=complex) + a = asarray(a) if n is None: n = (a.shape[axis] - 1) * 2 - unitary = _unitary(norm) - output = _raw_fft(a, n, axis, fftpack.rffti, fftpack.rfftb, - _real_fft_cache) - return output * (1 / (sqrt(n) if unitary else n)) + fct = 1/n + if norm is not None and _unitary(norm): + fct = 1/sqrt(n) + output = _raw_fft(a, n, axis, True, False, fct) + return output @array_function_dispatch(_fft_dispatcher) @@ -537,16 +517,16 @@ def hfft(a, n=None, axis=-1, norm=None): -------- >>> signal = np.array([1, 2, 3, 4, 3, 2]) >>> np.fft.fft(signal) - array([ 15.+0.j, -4.+0.j, 0.+0.j, -1.-0.j, 0.+0.j, -4.+0.j]) + array([15.+0.j, -4.+0.j, 0.+0.j, -1.-0.j, 0.+0.j, -4.+0.j]) # may vary >>> np.fft.hfft(signal[:4]) # Input first half of signal - array([ 15., -4., 0., -1., 0., -4.]) + array([15., -4., 0., -1., 0., -4.]) >>> np.fft.hfft(signal, 6) # Input entire signal and truncate - array([ 15., -4., 0., -1., 0., -4.]) + array([15., -4., 0., -1., 0., -4.]) >>> signal = np.array([[1, 1.j], [-1.j, 2]]) >>> np.conj(signal.T) - signal # check Hermitian symmetry - array([[ 0.-0.j, 0.+0.j], + array([[ 0.-0.j, -0.+0.j], # may vary [ 0.+0.j, 0.-0.j]]) >>> freq_spectrum = np.fft.hfft(signal) >>> freq_spectrum @@ -554,8 +534,7 @@ def hfft(a, n=None, axis=-1, norm=None): [ 2., -2.]]) """ - # The copy may be required for multithreading. - a = array(a, copy=True, dtype=complex) + a = asarray(a) if n is None: n = (a.shape[axis] - 1) * 2 unitary = _unitary(norm) @@ -610,13 +589,12 @@ def ihfft(a, n=None, axis=-1, norm=None): -------- >>> spectrum = np.array([ 15, -4, 0, -1, 0, -4]) >>> np.fft.ifft(spectrum) - array([ 1.+0.j, 2.-0.j, 3.+0.j, 4.+0.j, 3.+0.j, 2.-0.j]) + array([1.+0.j, 2.+0.j, 3.+0.j, 4.+0.j, 3.+0.j, 2.+0.j]) # may vary >>> np.fft.ihfft(spectrum) - array([ 1.-0.j, 2.-0.j, 3.-0.j, 4.-0.j]) + array([ 1.-0.j, 2.-0.j, 3.-0.j, 4.-0.j]) # may vary """ - # The copy may be required for multithreading. - a = array(a, copy=True, dtype=float) + a = asarray(a) if n is None: n = a.shape[axis] unitary = _unitary(norm) @@ -726,17 +704,17 @@ def fftn(a, s=None, axes=None, norm=None): -------- >>> a = np.mgrid[:3, :3, :3][0] >>> np.fft.fftn(a, axes=(1, 2)) - array([[[ 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j]], - [[ 9.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j]], - [[ 18.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j]]]) + array([[[ 0.+0.j, 0.+0.j, 0.+0.j], # may vary + [ 0.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j]], + [[ 9.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j]], + [[18.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j]]]) >>> np.fft.fftn(a, (2, 2), axes=(0, 1)) - array([[[ 2.+0.j, 2.+0.j, 2.+0.j], + array([[[ 2.+0.j, 2.+0.j, 2.+0.j], # may vary [ 0.+0.j, 0.+0.j, 0.+0.j]], [[-2.+0.j, -2.+0.j, -2.+0.j], [ 0.+0.j, 0.+0.j, 0.+0.j]]]) @@ -832,10 +810,10 @@ def ifftn(a, s=None, axes=None, norm=None): -------- >>> a = np.eye(4) >>> np.fft.ifftn(np.fft.fftn(a, axes=(0,)), axes=(1,)) - array([[ 1.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 1.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j, 1.+0.j]]) + array([[1.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], # may vary + [0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j], + [0.+0.j, 0.+0.j, 1.+0.j, 0.+0.j], + [0.+0.j, 0.+0.j, 0.+0.j, 1.+0.j]]) Create and plot an image with band-limited frequency content: @@ -928,16 +906,16 @@ def fft2(a, s=None, axes=(-2, -1), norm=None): -------- >>> a = np.mgrid[:5, :5][0] >>> np.fft.fft2(a) - array([[ 50.0 +0.j , 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ], - [-12.5+17.20477401j, 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ], - [-12.5 +4.0614962j , 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ], - [-12.5 -4.0614962j , 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ], - [-12.5-17.20477401j, 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ]]) + array([[ 50. +0.j , 0. +0.j , 0. +0.j , # may vary + 0. +0.j , 0. +0.j ], + [-12.5+17.20477401j, 0. +0.j , 0. +0.j , + 0. +0.j , 0. +0.j ], + [-12.5 +4.0614962j , 0. +0.j , 0. +0.j , + 0. +0.j , 0. +0.j ], + [-12.5 -4.0614962j , 0. +0.j , 0. +0.j , + 0. +0.j , 0. +0.j ], + [-12.5-17.20477401j, 0. +0.j , 0. +0.j , + 0. +0.j , 0. +0.j ]]) """ @@ -1022,10 +1000,10 @@ def ifft2(a, s=None, axes=(-2, -1), norm=None): -------- >>> a = 4 * np.eye(4) >>> np.fft.ifft2(a) - array([[ 1.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j, 1.+0.j], - [ 0.+0.j, 0.+0.j, 1.+0.j, 0.+0.j], - [ 0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j]]) + array([[1.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], # may vary + [0.+0.j, 0.+0.j, 0.+0.j, 1.+0.j], + [0.+0.j, 0.+0.j, 1.+0.j, 0.+0.j], + [0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j]]) """ @@ -1104,20 +1082,19 @@ def rfftn(a, s=None, axes=None, norm=None): -------- >>> a = np.ones((2, 2, 2)) >>> np.fft.rfftn(a) - array([[[ 8.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j]], - [[ 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j]]]) + array([[[8.+0.j, 0.+0.j], # may vary + [0.+0.j, 0.+0.j]], + [[0.+0.j, 0.+0.j], + [0.+0.j, 0.+0.j]]]) >>> np.fft.rfftn(a, axes=(2, 0)) - array([[[ 4.+0.j, 0.+0.j], - [ 4.+0.j, 0.+0.j]], - [[ 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j]]]) + array([[[4.+0.j, 0.+0.j], # may vary + [4.+0.j, 0.+0.j]], + [[0.+0.j, 0.+0.j], + [0.+0.j, 0.+0.j]]]) """ - # The copy may be required for multithreading. - a = array(a, copy=True, dtype=float) + a = asarray(a) s, axes = _cook_nd_args(a, s, axes) a = rfft(a, s[-1], axes[-1], norm) for ii in range(len(axes)-1): @@ -1241,16 +1218,15 @@ def irfftn(a, s=None, axes=None, norm=None): >>> a = np.zeros((3, 2, 2)) >>> a[0, 0, 0] = 3 * 2 * 2 >>> np.fft.irfftn(a) - array([[[ 1., 1.], - [ 1., 1.]], - [[ 1., 1.], - [ 1., 1.]], - [[ 1., 1.], - [ 1., 1.]]]) + array([[[1., 1.], + [1., 1.]], + [[1., 1.], + [1., 1.]], + [[1., 1.], + [1., 1.]]]) """ - # The copy may be required for multithreading. - a = array(a, copy=True, dtype=complex) + a = asarray(a) s, axes = _cook_nd_args(a, s, axes, invreal=1) for ii in range(len(axes)-1): a = ifft(a, s[ii], axes[ii], norm) diff --git a/numpy/fft/setup.py b/numpy/fft/setup.py index cd99a82d7..6c3548b65 100644 --- a/numpy/fft/setup.py +++ b/numpy/fft/setup.py @@ -7,9 +7,9 @@ def configuration(parent_package='',top_path=None): config.add_data_dir('tests') - # Configure fftpack_lite - config.add_extension('fftpack_lite', - sources=['fftpack_litemodule.c', 'fftpack.c'] + # Configure pocketfft_internal + config.add_extension('pocketfft_internal', + sources=['pocketfft.c'] ) return config diff --git a/numpy/fft/tests/test_helper.py b/numpy/fft/tests/test_helper.py index 8d315fa02..6613c8002 100644 --- a/numpy/fft/tests/test_helper.py +++ b/numpy/fft/tests/test_helper.py @@ -7,7 +7,6 @@ from __future__ import division, absolute_import, print_function import numpy as np from numpy.testing import assert_array_almost_equal, assert_equal from numpy import fft, pi -from numpy.fft.helper import _FFTCache class TestFFTShift(object): @@ -168,81 +167,3 @@ class TestIRFFTN(object): # Should not raise error fft.irfftn(a, axes=axes) - - -class TestFFTCache(object): - - def test_basic_behaviour(self): - c = _FFTCache(max_size_in_mb=1, max_item_count=4) - - # Put - c.put_twiddle_factors(1, np.ones(2, dtype=np.float32)) - c.put_twiddle_factors(2, np.zeros(2, dtype=np.float32)) - - # Get - assert_array_almost_equal(c.pop_twiddle_factors(1), - np.ones(2, dtype=np.float32)) - assert_array_almost_equal(c.pop_twiddle_factors(2), - np.zeros(2, dtype=np.float32)) - - # Nothing should be left. - assert_equal(len(c._dict), 0) - - # Now put everything in twice so it can be retrieved once and each will - # still have one item left. - for _ in range(2): - c.put_twiddle_factors(1, np.ones(2, dtype=np.float32)) - c.put_twiddle_factors(2, np.zeros(2, dtype=np.float32)) - assert_array_almost_equal(c.pop_twiddle_factors(1), - np.ones(2, dtype=np.float32)) - assert_array_almost_equal(c.pop_twiddle_factors(2), - np.zeros(2, dtype=np.float32)) - assert_equal(len(c._dict), 2) - - def test_automatic_pruning(self): - # That's around 2600 single precision samples. - c = _FFTCache(max_size_in_mb=0.01, max_item_count=4) - - c.put_twiddle_factors(1, np.ones(200, dtype=np.float32)) - c.put_twiddle_factors(2, np.ones(200, dtype=np.float32)) - assert_equal(list(c._dict.keys()), [1, 2]) - - # This is larger than the limit but should still be kept. - c.put_twiddle_factors(3, np.ones(3000, dtype=np.float32)) - assert_equal(list(c._dict.keys()), [1, 2, 3]) - # Add one more. - c.put_twiddle_factors(4, np.ones(3000, dtype=np.float32)) - # The other three should no longer exist. - assert_equal(list(c._dict.keys()), [4]) - - # Now test the max item count pruning. - c = _FFTCache(max_size_in_mb=0.01, max_item_count=2) - c.put_twiddle_factors(2, np.empty(2)) - c.put_twiddle_factors(1, np.empty(2)) - # Can still be accessed. - assert_equal(list(c._dict.keys()), [2, 1]) - - c.put_twiddle_factors(3, np.empty(2)) - # 1 and 3 can still be accessed - c[2] has been touched least recently - # and is thus evicted. - assert_equal(list(c._dict.keys()), [1, 3]) - - # One last test. We will add a single large item that is slightly - # bigger then the cache size. Some small items can still be added. - c = _FFTCache(max_size_in_mb=0.01, max_item_count=5) - c.put_twiddle_factors(1, np.ones(3000, dtype=np.float32)) - c.put_twiddle_factors(2, np.ones(2, dtype=np.float32)) - c.put_twiddle_factors(3, np.ones(2, dtype=np.float32)) - c.put_twiddle_factors(4, np.ones(2, dtype=np.float32)) - assert_equal(list(c._dict.keys()), [1, 2, 3, 4]) - - # One more big item. This time it is 6 smaller ones but they are - # counted as one big item. - for _ in range(6): - c.put_twiddle_factors(5, np.ones(500, dtype=np.float32)) - # '1' no longer in the cache. Rest still in the cache. - assert_equal(list(c._dict.keys()), [2, 3, 4, 5]) - - # Another big item - should now be the only item in the cache. - c.put_twiddle_factors(6, np.ones(4000, dtype=np.float32)) - assert_equal(list(c._dict.keys()), [6]) diff --git a/numpy/fft/tests/test_fftpack.py b/numpy/fft/tests/test_pocketfft.py index 8d6cd8407..0552f6afd 100644 --- a/numpy/fft/tests/test_fftpack.py +++ b/numpy/fft/tests/test_pocketfft.py @@ -28,6 +28,16 @@ class TestFFTShift(object): class TestFFT1D(object): + def test_identity(self): + maxlen = 512 + x = random(maxlen) + 1j*random(maxlen) + xr = random(maxlen) + for i in range(1,maxlen): + assert_array_almost_equal(np.fft.ifft(np.fft.fft(x[0:i])), x[0:i], + decimal=12) + assert_array_almost_equal(np.fft.irfft(np.fft.rfft(xr[0:i]),i), + xr[0:i], decimal=12) + def test_fft(self): x = random(30) + 1j*random(30) assert_array_almost_equal(fft1(x), np.fft.fft(x)) diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index ab00b1444..3a0e67f60 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -20,17 +20,18 @@ gzip, bz2 and xz are supported. Example:: >>> # Create a DataSource, use os.curdir (default) for local storage. - >>> ds = datasource.DataSource() + >>> from numpy import DataSource + >>> ds = DataSource() >>> >>> # Open a remote file. >>> # DataSource downloads the file, stores it locally in: >>> # './www.google.com/index.html' >>> # opens the file and returns a file object. - >>> fp = ds.open('http://www.google.com/index.html') + >>> fp = ds.open('http://www.google.com/') # doctest: +SKIP >>> >>> # Use the file as you normally would - >>> fp.read() - >>> fp.close() + >>> fp.read() # doctest: +SKIP + >>> fp.close() # doctest: +SKIP """ from __future__ import division, absolute_import, print_function @@ -41,8 +42,12 @@ import warnings import shutil import io +from numpy.core.overrides import set_module + + _open = open + def _check_mode(mode, encoding, newline): """Check mode and that encoding and newline are compatible. @@ -152,6 +157,7 @@ class _FileOpeners(object): Examples -------- + >>> import gzip >>> np.lib._datasource._file_openers.keys() [None, '.bz2', '.gz', '.xz', '.lzma'] >>> np.lib._datasource._file_openers['.gz'] is gzip.open @@ -262,7 +268,8 @@ def open(path, mode='r', destpath=os.curdir, encoding=None, newline=None): return ds.open(path, mode, encoding=encoding, newline=newline) -class DataSource (object): +@set_module('numpy') +class DataSource(object): """ DataSource(destpath='.') @@ -285,7 +292,7 @@ class DataSource (object): URLs require a scheme string (``http://``) to be used, without it they will fail:: - >>> repos = DataSource() + >>> repos = np.DataSource() >>> repos.exists('www.google.com/index.html') False >>> repos.exists('http://www.google.com/index.html') @@ -297,17 +304,17 @@ class DataSource (object): -------- :: - >>> ds = DataSource('/home/guido') - >>> urlname = 'http://www.google.com/index.html' - >>> gfile = ds.open('http://www.google.com/index.html') # remote file + >>> ds = np.DataSource('/home/guido') + >>> urlname = 'http://www.google.com/' + >>> gfile = ds.open('http://www.google.com/') >>> ds.abspath(urlname) - '/home/guido/www.google.com/site/index.html' + '/home/guido/www.google.com/index.html' - >>> ds = DataSource(None) # use with temporary file + >>> ds = np.DataSource(None) # use with temporary file >>> ds.open('/home/guido/foobar.txt') <open file '/home/guido.foobar.txt', mode 'r' at 0x91d4430> >>> ds.abspath('/home/guido/foobar.txt') - '/tmp/tmpy4pgsP/home/guido/foobar.txt' + '/tmp/.../home/guido/foobar.txt' """ @@ -323,7 +330,7 @@ class DataSource (object): def __del__(self): # Remove temp directories - if self._istmpdest: + if hasattr(self, '_istmpdest') and self._istmpdest: shutil.rmtree(self._destpath) def _iszip(self, filename): diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index b604b8c52..0ebd39b8c 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -8,7 +8,7 @@ __docformat__ = "restructuredtext en" import sys import numpy as np import numpy.core.numeric as nx -from numpy.compat import asbytes, asunicode, bytes, asbytes_nested, basestring +from numpy.compat import asbytes, asunicode, bytes, basestring if sys.version_info[0] >= 3: from builtins import bool, int, float, complex, object, str @@ -146,11 +146,17 @@ def flatten_dtype(ndtype, flatten_base=False): >>> dt = np.dtype([('name', 'S4'), ('x', float), ('y', float), ... ('block', int, (2, 3))]) >>> np.lib._iotools.flatten_dtype(dt) - [dtype('|S4'), dtype('float64'), dtype('float64'), dtype('int32')] + [dtype('S4'), dtype('float64'), dtype('float64'), dtype('int64')] >>> np.lib._iotools.flatten_dtype(dt, flatten_base=True) - [dtype('|S4'), dtype('float64'), dtype('float64'), dtype('int32'), - dtype('int32'), dtype('int32'), dtype('int32'), dtype('int32'), - dtype('int32')] + [dtype('S4'), + dtype('float64'), + dtype('float64'), + dtype('int64'), + dtype('int64'), + dtype('int64'), + dtype('int64'), + dtype('int64'), + dtype('int64')] """ names = ndtype.names @@ -309,13 +315,13 @@ class NameValidator(object): -------- >>> validator = np.lib._iotools.NameValidator() >>> validator(['file', 'field2', 'with space', 'CaSe']) - ['file_', 'field2', 'with_space', 'CaSe'] + ('file_', 'field2', 'with_space', 'CaSe') >>> validator = np.lib._iotools.NameValidator(excludelist=['excl'], - deletechars='q', - case_sensitive='False') + ... deletechars='q', + ... case_sensitive=False) >>> validator(['excl', 'field2', 'no_q', 'with space', 'CaSe']) - ['excl_', 'field2', 'no_', 'with_space', 'case'] + ('EXCL', 'FIELD2', 'NO_Q', 'WITH_SPACE', 'CASE') """ # @@ -599,7 +605,7 @@ class StringConverter(object): -------- >>> import dateutil.parser >>> import datetime - >>> dateparser = datetustil.parser.parse + >>> dateparser = dateutil.parser.parse >>> defaultdate = datetime.date(2000, 1, 1) >>> StringConverter.upgrade_mapper(dateparser, default=defaultdate) """ @@ -693,7 +699,7 @@ class StringConverter(object): self.func = lambda x: int(float(x)) # Store the list of strings corresponding to missing values. if missing_values is None: - self.missing_values = set(['']) + self.missing_values = {''} else: if isinstance(missing_values, basestring): missing_values = missing_values.split(",") diff --git a/numpy/lib/_version.py b/numpy/lib/_version.py index c3563a7fa..8aa999fc9 100644 --- a/numpy/lib/_version.py +++ b/numpy/lib/_version.py @@ -47,9 +47,12 @@ class NumpyVersion(): >>> from numpy.lib import NumpyVersion >>> if NumpyVersion(np.__version__) < '1.7.0': ... print('skip') - skip + >>> # skip >>> NumpyVersion('1.7') # raises ValueError, add ".0" + Traceback (most recent call last): + ... + ValueError: Not a valid numpy version string """ diff --git a/numpy/lib/arraypad.py b/numpy/lib/arraypad.py index f76ad456f..b236cc449 100644 --- a/numpy/lib/arraypad.py +++ b/numpy/lib/arraypad.py @@ -886,105 +886,71 @@ def _pad_wrap(arr, pad_amt, axis=-1): return np.concatenate((wrap_chunk1, arr, wrap_chunk2), axis=axis) -def _normalize_shape(ndarray, shape, cast_to_int=True): +def _as_pairs(x, ndim, as_index=False): """ - Private function which does some checks and normalizes the possibly - much simpler representations of 'pad_width', 'stat_length', - 'constant_values', 'end_values'. + Broadcast `x` to an array with the shape (`ndim`, 2). - Parameters - ---------- - narray : ndarray - Input ndarray - shape : {sequence, array_like, float, int}, optional - The width of padding (pad_width), the number of elements on the - edge of the narray used for statistics (stat_length), the constant - value(s) to use when filling padded regions (constant_values), or the - endpoint target(s) for linear ramps (end_values). - ((before_1, after_1), ... (before_N, after_N)) unique number of - elements for each axis where `N` is rank of `narray`. - ((before, after),) yields same before and after constants for each - axis. - (constant,) or val is a shortcut for before = after = constant for - all axes. - cast_to_int : bool, optional - Controls if values in ``shape`` will be rounded and cast to int - before being returned. - - Returns - ------- - normalized_shape : tuple of tuples - val => ((val, val), (val, val), ...) - [[val1, val2], [val3, val4], ...] => ((val1, val2), (val3, val4), ...) - ((val1, val2), (val3, val4), ...) => no change - [[val1, val2], ] => ((val1, val2), (val1, val2), ...) - ((val1, val2), ) => ((val1, val2), (val1, val2), ...) - [[val , ], ] => ((val, val), (val, val), ...) - ((val , ), ) => ((val, val), (val, val), ...) - - """ - ndims = ndarray.ndim - - # Shortcut shape=None - if shape is None: - return ((None, None), ) * ndims - - # Convert any input `info` to a NumPy array - shape_arr = np.asarray(shape) - - try: - shape_arr = np.broadcast_to(shape_arr, (ndims, 2)) - except ValueError: - fmt = "Unable to create correctly shaped tuple from %s" - raise ValueError(fmt % (shape,)) - - # Cast if necessary - if cast_to_int is True: - shape_arr = np.round(shape_arr).astype(int) - - # Convert list of lists to tuple of tuples - return tuple(tuple(axis) for axis in shape_arr.tolist()) - - -def _validate_lengths(narray, number_elements): - """ - Private function which does some checks and reformats pad_width and - stat_length using _normalize_shape. + A helper function for `pad` that prepares and validates arguments like + `pad_width` for iteration in pairs. Parameters ---------- - narray : ndarray - Input ndarray - number_elements : {sequence, int}, optional - The width of padding (pad_width) or the number of elements on the edge - of the narray used for statistics (stat_length). - ((before_1, after_1), ... (before_N, after_N)) unique number of - elements for each axis. - ((before, after),) yields same before and after constants for each - axis. - (constant,) or int is a shortcut for before = after = constant for all - axes. + x : {None, scalar, array-like} + The object to broadcast to the shape (`ndim`, 2). + ndim : int + Number of pairs the broadcasted `x` will have. + as_index : bool, optional + If `x` is not None, try to round each element of `x` to an integer + (dtype `np.intp`) and ensure every element is positive. Returns ------- - _validate_lengths : tuple of tuples - int => ((int, int), (int, int), ...) - [[int1, int2], [int3, int4], ...] => ((int1, int2), (int3, int4), ...) - ((int1, int2), (int3, int4), ...) => no change - [[int1, int2], ] => ((int1, int2), (int1, int2), ...) - ((int1, int2), ) => ((int1, int2), (int1, int2), ...) - [[int , ], ] => ((int, int), (int, int), ...) - ((int , ), ) => ((int, int), (int, int), ...) - + pairs : nested iterables, shape (`ndim`, 2) + The broadcasted version of `x`. + + Raises + ------ + ValueError + If `as_index` is True and `x` contains negative elements. + Or if `x` is not broadcastable to the shape (`ndim`, 2). """ - normshp = _normalize_shape(narray, number_elements) - for i in normshp: - chk = [1 if x is None else x for x in i] - chk = [1 if x >= 0 else -1 for x in chk] - if (chk[0] < 0) or (chk[1] < 0): - fmt = "%s cannot contain negative values." - raise ValueError(fmt % (number_elements,)) - return normshp + if x is None: + # Pass through None as a special case, otherwise np.round(x) fails + # with an AttributeError + return ((None, None),) * ndim + + x = np.array(x) + if as_index: + x = np.round(x).astype(np.intp, copy=False) + + if x.ndim < 3: + # Optimization: Possibly use faster paths for cases where `x` has + # only 1 or 2 elements. `np.broadcast_to` could handle these as well + # but is currently slower + + if x.size == 1: + # x was supplied as a single value + x = x.ravel() # Ensure x[0] works for x.ndim == 0, 1, 2 + if as_index and x < 0: + raise ValueError("index can't contain negative values") + return ((x[0], x[0]),) * ndim + + if x.size == 2 and x.shape != (2, 1): + # x was supplied with a single value for each side + # but except case when each dimension has a single value + # which should be broadcasted to a pair, + # e.g. [[1], [2]] -> [[1, 1], [2, 2]] not [[1, 2], [1, 2]] + x = x.ravel() # Ensure x[0], x[1] works + if as_index and (x[0] < 0 or x[1] < 0): + raise ValueError("index can't contain negative values") + return ((x[0], x[1]),) * ndim + + if as_index and x.min() < 0: + raise ValueError("index can't contain negative values") + + # Converting the array with `tolist` seems to improve performance + # when iterating and indexing the result (see usage in `pad`) + return np.broadcast_to(x, (ndim, 2)).tolist() ############################################################################### @@ -995,7 +961,7 @@ def _pad_dispatcher(array, pad_width, mode, **kwargs): return (array,) -@array_function_dispatch(_pad_dispatcher) +@array_function_dispatch(_pad_dispatcher, module='numpy') def pad(array, pad_width, mode, **kwargs): """ Pads an array. @@ -1134,10 +1100,10 @@ def pad(array, pad_width, mode, **kwargs): -------- >>> a = [1, 2, 3, 4, 5] >>> np.pad(a, (2,3), 'constant', constant_values=(4, 6)) - array([4, 4, 1, 2, 3, 4, 5, 6, 6, 6]) + array([4, 4, 1, ..., 6, 6, 6]) >>> np.pad(a, (2, 3), 'edge') - array([1, 1, 1, 2, 3, 4, 5, 5, 5, 5]) + array([1, 1, 1, ..., 5, 5, 5]) >>> np.pad(a, (2, 3), 'linear_ramp', end_values=(5, -4)) array([ 5, 3, 1, 2, 3, 4, 5, 2, -1, -4]) @@ -1203,7 +1169,7 @@ def pad(array, pad_width, mode, **kwargs): raise TypeError('`pad_width` must be of integral type.') narray = np.array(array) - pad_width = _validate_lengths(narray, pad_width) + pad_width = _as_pairs(pad_width, narray.ndim, as_index=True) allowedkwargs = { 'constant': ['constant_values'], @@ -1239,10 +1205,9 @@ def pad(array, pad_width, mode, **kwargs): # Need to only normalize particular keywords. for i in kwargs: if i == 'stat_length': - kwargs[i] = _validate_lengths(narray, kwargs[i]) + kwargs[i] = _as_pairs(kwargs[i], narray.ndim, as_index=True) if i in ['end_values', 'constant_values']: - kwargs[i] = _normalize_shape(narray, kwargs[i], - cast_to_int=False) + kwargs[i] = _as_pairs(kwargs[i], narray.ndim) else: # Drop back to old, slower np.apply_along_axis mode for user-supplied # vector function diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index ec62cd7a6..558150e48 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -27,8 +27,14 @@ To do: Optionally return indices analogously to unique for all functions. """ from __future__ import division, absolute_import, print_function +import functools + import numpy as np -from numpy.core.overrides import array_function_dispatch +from numpy.core import overrides + + +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') __all__ = [ @@ -76,7 +82,7 @@ def ediff1d(ary, to_end=None, to_begin=None): array([ 1, 2, 3, -7]) >>> np.ediff1d(x, to_begin=-99, to_end=np.array([88, 99])) - array([-99, 1, 2, 3, -7, 88, 99]) + array([-99, 1, 2, ..., -7, 88, 99]) The returned array is always 1D. @@ -235,13 +241,11 @@ def unique(ar, return_index=False, return_inverse=False, >>> a = np.array(['a', 'b', 'b', 'c', 'a']) >>> u, indices = np.unique(a, return_index=True) >>> u - array(['a', 'b', 'c'], - dtype='|S1') + array(['a', 'b', 'c'], dtype='<U1') >>> indices array([0, 1, 3]) >>> a[indices] - array(['a', 'b', 'c'], - dtype='|S1') + array(['a', 'b', 'c'], dtype='<U1') Reconstruct the input array from the unique values: @@ -250,9 +254,9 @@ def unique(ar, return_index=False, return_inverse=False, >>> u array([1, 2, 3, 4, 6]) >>> indices - array([0, 1, 4, 3, 1, 2, 1]) + array([0, 1, 4, ..., 1, 2, 1]) >>> u[indices] - array([1, 2, 6, 4, 2, 3, 2]) + array([1, 2, 6, ..., 2, 3, 2]) """ ar = np.asanyarray(ar) @@ -473,6 +477,11 @@ def setxor1d(ar1, ar2, assume_unique=False): return aux[flag[1:] & flag[:-1]] +def _in1d_dispatcher(ar1, ar2, assume_unique=None, invert=None): + return (ar1, ar2) + + +@array_function_dispatch(_in1d_dispatcher) def in1d(ar1, ar2, assume_unique=False, invert=False): """ Test whether each element of a 1-D array is also present in a second array. @@ -650,8 +659,8 @@ def isin(element, test_elements, assume_unique=False, invert=False): >>> test_elements = [1, 2, 4, 8] >>> mask = np.isin(element, test_elements) >>> mask - array([[ False, True], - [ True, False]]) + array([[False, True], + [ True, False]]) >>> element[mask] array([2, 4]) @@ -665,7 +674,7 @@ def isin(element, test_elements, assume_unique=False, invert=False): >>> mask = np.isin(element, test_elements, invert=True) >>> mask array([[ True, False], - [ False, True]]) + [False, True]]) >>> element[mask] array([0, 6]) @@ -674,14 +683,14 @@ def isin(element, test_elements, assume_unique=False, invert=False): >>> test_set = {1, 2, 4, 8} >>> np.isin(element, test_set) - array([[ False, False], - [ False, False]]) + array([[False, False], + [False, False]]) Casting the set to a list gives the expected result: >>> np.isin(element, list(test_set)) - array([[ False, True], - [ True, False]]) + array([[False, True], + [ True, False]]) """ element = np.asarray(element) return in1d(element, test_elements, assume_unique=assume_unique, diff --git a/numpy/lib/arrayterator.py b/numpy/lib/arrayterator.py index f2d4fe9fd..c16668582 100644 --- a/numpy/lib/arrayterator.py +++ b/numpy/lib/arrayterator.py @@ -80,9 +80,8 @@ class Arrayterator(object): >>> for subarr in a_itor: ... if not subarr.all(): - ... print(subarr, subarr.shape) - ... - [[[[0 1]]]] (1, 1, 1, 2) + ... print(subarr, subarr.shape) # doctest: +SKIP + >>> # [[[[0 1]]]] (1, 1, 1, 2) """ @@ -160,7 +159,7 @@ class Arrayterator(object): ... if not subarr: ... print(subarr, type(subarr)) ... - 0 <type 'numpy.int32'> + 0 <class 'numpy.int64'> """ for block in self: diff --git a/numpy/lib/financial.py b/numpy/lib/financial.py index d1a0cd9c0..216687475 100644 --- a/numpy/lib/financial.py +++ b/numpy/lib/financial.py @@ -13,9 +13,14 @@ otherwise stated. from __future__ import division, absolute_import, print_function from decimal import Decimal +import functools import numpy as np -from numpy.core.overrides import array_function_dispatch +from numpy.core import overrides + + +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') __all__ = ['fv', 'pmt', 'nper', 'ipmt', 'ppmt', 'pv', 'rate', @@ -122,7 +127,7 @@ def fv(rate, nper, pmt, pv, when='end'): >>> a = np.array((0.05, 0.06, 0.07))/12 >>> np.fv(a, 10*12, -100, -100) - array([ 15692.92889434, 16569.87435405, 17509.44688102]) + array([ 15692.92889434, 16569.87435405, 17509.44688102]) # may vary """ when = _convert_when(when) @@ -270,7 +275,7 @@ def nper(rate, pmt, pv, fv=0, when='end'): If you only had $150/month to pay towards the loan, how long would it take to pay-off a loan of $8,000 at 7% annual interest? - >>> print(round(np.nper(0.07/12, -150, 8000), 5)) + >>> print(np.round(np.nper(0.07/12, -150, 8000), 5)) 64.07335 So, over 64 months would be required to pay off the loan. @@ -281,10 +286,10 @@ def nper(rate, pmt, pv, fv=0, when='end'): >>> np.nper(*(np.ogrid[0.07/12: 0.08/12: 0.01/12, ... -150 : -99 : 50 , ... 8000 : 9001 : 1000])) - array([[[ 64.07334877, 74.06368256], - [ 108.07548412, 127.99022654]], - [[ 66.12443902, 76.87897353], - [ 114.70165583, 137.90124779]]]) + array([[[ 64.07334877, 74.06368256], + [108.07548412, 127.99022654]], + [[ 66.12443902, 76.87897353], + [114.70165583, 137.90124779]]]) """ when = _convert_when(when) @@ -534,7 +539,7 @@ def pv(rate, nper, pmt, fv=0, when='end'): >>> a = np.array((0.05, 0.04, 0.03))/12 >>> np.pv(a, 10*12, -100, 15692.93) - array([ -100.00067132, -649.26771385, -1273.78633713]) + array([ -100.00067132, -649.26771385, -1273.78633713]) # may vary So, to end up with the same $15692.93 under the same $100 per month "savings plan," for annual interest rates of 4% and 3%, one would @@ -699,15 +704,15 @@ def irr(values): Examples -------- - >>> round(irr([-100, 39, 59, 55, 20]), 5) + >>> round(np.irr([-100, 39, 59, 55, 20]), 5) 0.28095 - >>> round(irr([-100, 0, 0, 74]), 5) + >>> round(np.irr([-100, 0, 0, 74]), 5) -0.0955 - >>> round(irr([-100, 100, 0, -7]), 5) + >>> round(np.irr([-100, 100, 0, -7]), 5) -0.0833 - >>> round(irr([-100, 100, 0, 7]), 5) + >>> round(np.irr([-100, 100, 0, 7]), 5) 0.06206 - >>> round(irr([-5, 10.5, 1, -8, 1]), 5) + >>> round(np.irr([-5, 10.5, 1, -8, 1]), 5) 0.0886 (Compare with the Example given for numpy.lib.financial.npv) @@ -772,7 +777,7 @@ def npv(rate, values): Examples -------- >>> np.npv(0.281,[-100, 39, 59, 55, 20]) - -0.0084785916384548798 + -0.0084785916384548798 # may vary (Compare with the Example given for numpy.lib.financial.irr) diff --git a/numpy/lib/format.py b/numpy/lib/format.py index e25868236..10945e5e8 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -161,7 +161,9 @@ import sys import io import warnings from numpy.lib.utils import safe_eval -from numpy.compat import asbytes, asstr, isfileobj, long, basestring +from numpy.compat import ( + asbytes, asstr, isfileobj, long, os_fspath + ) from numpy.core.numeric import pickle @@ -257,6 +259,43 @@ def dtype_to_descr(dtype): else: return dtype.str +def descr_to_dtype(descr): + ''' + descr may be stored as dtype.descr, which is a list of + (name, format, [shape]) tuples. Offsets are not explicitly saved, rather + empty fields with name,format == '', '|Vn' are added as padding. + + This function reverses the process, eliminating the empty padding fields. + ''' + if isinstance(descr, (str, dict)): + # No padding removal needed + return numpy.dtype(descr) + + fields = [] + offset = 0 + for field in descr: + if len(field) == 2: + name, descr_str = field + dt = descr_to_dtype(descr_str) + else: + name, descr_str, shape = field + dt = numpy.dtype((descr_to_dtype(descr_str), shape)) + + # Ignore padding bytes, which will be void bytes with '' as name + # Once support for blank names is removed, only "if name == ''" needed) + is_pad = (name == '' and dt.type is numpy.void and dt.names is None) + if not is_pad: + fields.append((name, dt, offset)) + + offset += dt.itemsize + + names, formats, offsets = zip(*fields) + # names may be (title, names) tuples + nametups = (n if isinstance(n, tuple) else (None, n) for n in names) + titles, names = zip(*nametups) + return numpy.dtype({'names': names, 'formats': formats, 'titles': titles, + 'offsets': offsets, 'itemsize': offset}) + def header_data_from_array_1_0(array): """ Get the dictionary of header metadata from a numpy.ndarray. @@ -521,7 +560,7 @@ def _read_array_header(fp, version): msg = "fortran_order is not a valid bool: %r" raise ValueError(msg % (d['fortran_order'],)) try: - dtype = numpy.dtype(d['descr']) + dtype = descr_to_dtype(d['descr']) except TypeError as e: msg = "descr is not a valid dtype descriptor: %r" raise ValueError(msg % (d['descr'],)) @@ -706,7 +745,7 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, Parameters ---------- - filename : str + filename : str or path-like The name of the file on disk. This may *not* be a file-like object. mode : str, optional @@ -747,9 +786,9 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, memmap """ - if not isinstance(filename, basestring): - raise ValueError("Filename must be a string. Memmap cannot use" - " existing file handles.") + if isfileobj(filename): + raise ValueError("Filename must be a string or a path-like object." + " Memmap cannot use existing file handles.") if 'w' in mode: # We are creating the file, not reading it. @@ -767,7 +806,7 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, shape=shape, ) # If we got here, then it should be safe to create the file. - fp = open(filename, mode+'b') + fp = open(os_fspath(filename), mode+'b') try: used_ver = _write_array_header(fp, d, version) # this warning can be removed when 1.9 has aged enough @@ -779,7 +818,7 @@ def open_memmap(filename, mode='r+', dtype=None, shape=None, fp.close() else: # Read the header of the file first. - fp = open(filename, 'rb') + fp = open(os_fspath(filename), 'rb') try: version = read_magic(fp) _check_version(version) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index c52ecdbd8..cee7b3a62 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -6,6 +6,7 @@ try: import collections.abc as collections_abc except ImportError: import collections as collections_abc +import functools import re import sys import warnings @@ -26,7 +27,8 @@ from numpy.core.fromnumeric import ( ravel, nonzero, partition, mean, any, sum ) from numpy.core.numerictypes import typecodes -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import set_module +from numpy.core import overrides from numpy.core.function_base import add_newdoc from numpy.lib.twodim_base import diag from .utils import deprecate @@ -44,6 +46,11 @@ if sys.version_info[0] < 3: else: import builtins + +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + # needed in this module for compatibility from numpy.lib.histograms import histogram, histogramdd @@ -211,12 +218,12 @@ def flip(m, axis=None): [2, 3]], [[4, 5], [6, 7]]]) - >>> flip(A, 0) + >>> np.flip(A, 0) array([[[4, 5], [6, 7]], [[0, 1], [2, 3]]]) - >>> flip(A, 1) + >>> np.flip(A, 1) array([[[2, 3], [0, 1]], [[6, 7], @@ -232,7 +239,7 @@ def flip(m, axis=None): [[1, 0], [3, 2]]]) >>> A = np.random.randn(3,4,5) - >>> np.all(flip(A,2) == A[:,:,::-1,...]) + >>> np.all(np.flip(A,2) == A[:,:,::-1,...]) True """ if not hasattr(m, 'ndim'): @@ -248,6 +255,7 @@ def flip(m, axis=None): return m[indexer] +@set_module('numpy') def iterable(y): """ Check whether or not an object can be iterated over. @@ -351,7 +359,7 @@ def average(a, axis=None, weights=None, returned=False): Examples -------- - >>> data = range(1,5) + >>> data = list(range(1,5)) >>> data [1, 2, 3, 4] >>> np.average(data) @@ -365,11 +373,10 @@ def average(a, axis=None, weights=None, returned=False): [2, 3], [4, 5]]) >>> np.average(data, axis=1, weights=[1./4, 3./4]) - array([ 0.75, 2.75, 4.75]) + array([0.75, 2.75, 4.75]) >>> np.average(data, weights=[1./4, 3./4]) - Traceback (most recent call last): - ... + ... TypeError: Axis must be specified when shapes of a and weights differ. >>> a = np.ones(5, dtype=np.float128) @@ -423,6 +430,7 @@ def average(a, axis=None, weights=None, returned=False): return avg +@set_module('numpy') def asarray_chkfinite(a, dtype=None, order=None): """Convert the input to an array, checking for NaNs or Infs. @@ -577,7 +585,7 @@ def piecewise(x, condlist, funclist, *args, **kw): ``x >= 0``. >>> np.piecewise(x, [x < 0, x >= 0], [lambda x: -x, lambda x: x]) - array([ 2.5, 1.5, 0.5, 0.5, 1.5, 2.5]) + array([2.5, 1.5, 0.5, 0.5, 1.5, 2.5]) Apply the same function to a scalar value. @@ -662,7 +670,7 @@ def select(condlist, choicelist, default=0): >>> condlist = [x<3, x>5] >>> choicelist = [x, x**2] >>> np.select(condlist, choicelist) - array([ 0, 1, 2, 0, 0, 0, 36, 49, 64, 81]) + array([ 0, 1, 2, ..., 49, 64, 81]) """ # Check the size of condlist and choicelist are the same, or abort. @@ -845,9 +853,9 @@ def gradient(f, *varargs, **kwargs): -------- >>> f = np.array([1, 2, 4, 7, 11, 16], dtype=float) >>> np.gradient(f) - array([ 1. , 1.5, 2.5, 3.5, 4.5, 5. ]) + array([1. , 1.5, 2.5, 3.5, 4.5, 5. ]) >>> np.gradient(f, 2) - array([ 0.5 , 0.75, 1.25, 1.75, 2.25, 2.5 ]) + array([0.5 , 0.75, 1.25, 1.75, 2.25, 2.5 ]) Spacing can be also specified with an array that represents the coordinates of the values F along the dimensions. @@ -855,13 +863,13 @@ def gradient(f, *varargs, **kwargs): >>> x = np.arange(f.size) >>> np.gradient(f, x) - array([ 1. , 1.5, 2.5, 3.5, 4.5, 5. ]) + array([1. , 1.5, 2.5, 3.5, 4.5, 5. ]) Or a non uniform one: >>> x = np.array([0., 1., 1.5, 3.5, 4., 6.], dtype=float) >>> np.gradient(f, x) - array([ 1. , 3. , 3.5, 6.7, 6.9, 2.5]) + array([1. , 3. , 3.5, 6.7, 6.9, 2.5]) For two dimensional arrays, the return will be two arrays ordered by axis. In this example the first array stands for the gradient in @@ -869,8 +877,8 @@ def gradient(f, *varargs, **kwargs): >>> np.gradient(np.array([[1, 2, 6], [3, 4, 5]], dtype=float)) [array([[ 2., 2., -1.], - [ 2., 2., -1.]]), array([[ 1. , 2.5, 4. ], - [ 1. , 1. , 1. ]])] + [ 2., 2., -1.]]), array([[1. , 2.5, 4. ], + [1. , 1. , 1. ]])] In this example the spacing is also specified: uniform for axis=0 and non uniform for axis=1 @@ -879,17 +887,17 @@ def gradient(f, *varargs, **kwargs): >>> y = [1., 1.5, 3.5] >>> np.gradient(np.array([[1, 2, 6], [3, 4, 5]], dtype=float), dx, y) [array([[ 1. , 1. , -0.5], - [ 1. , 1. , -0.5]]), array([[ 2. , 2. , 2. ], - [ 2. , 1.7, 0.5]])] + [ 1. , 1. , -0.5]]), array([[2. , 2. , 2. ], + [2. , 1.7, 0.5]])] It is possible to specify how boundaries are treated using `edge_order` >>> x = np.array([0, 1, 2, 3, 4]) >>> f = x**2 >>> np.gradient(f, edge_order=1) - array([ 1., 2., 4., 6., 7.]) + array([1., 2., 4., 6., 7.]) >>> np.gradient(f, edge_order=2) - array([-0., 2., 4., 6., 8.]) + array([0., 2., 4., 6., 8.]) The `axis` keyword can be used to specify a subset of axes of which the gradient is calculated @@ -1191,7 +1199,7 @@ def diff(a, n=1, axis=-1, prepend=np._NoValue, append=np._NoValue): >>> np.diff(u8_arr) array([255], dtype=uint8) >>> u8_arr[1,...] - u8_arr[0,...] - array(255, np.uint8) + 255 If this is not desirable, then the array should be cast to a larger integer type first: @@ -1331,7 +1339,7 @@ def interp(x, xp, fp, left=None, right=None, period=None): >>> np.interp(2.5, xp, fp) 1.0 >>> np.interp([0, 1, 1.5, 2.72, 3.14], xp, fp) - array([ 3. , 3. , 2.5 , 0.56, 0. ]) + array([3. , 3. , 2.5 , 0.56, 0. ]) >>> UNDEF = -99.0 >>> np.interp(3.14, xp, fp, right=UNDEF) -99.0 @@ -1355,7 +1363,7 @@ def interp(x, xp, fp, left=None, right=None, period=None): >>> xp = [190, -190, 350, -350] >>> fp = [5, 10, 3, 4] >>> np.interp(x, xp, fp, period=360) - array([7.5, 5., 8.75, 6.25, 3., 3.25, 3.5, 3.75]) + array([7.5 , 5. , 8.75, 6.25, 3. , 3.25, 3.5 , 3.75]) Complex interpolation: @@ -1363,7 +1371,7 @@ def interp(x, xp, fp, left=None, right=None, period=None): >>> xp = [2,3,5] >>> fp = [1.0j, 0, 2+3j] >>> np.interp(x, xp, fp) - array([ 0.+1.j , 1.+1.5j]) + array([0.+1.j , 1.+1.5j]) """ @@ -1436,7 +1444,7 @@ def angle(z, deg=False): Examples -------- >>> np.angle([1.0, 1.0j, 1+1j]) # in radians - array([ 0. , 1.57079633, 0.78539816]) + array([ 0. , 1.57079633, 0.78539816]) # may vary >>> np.angle(1+1j, deg=True) # in degrees 45.0 @@ -1496,9 +1504,9 @@ def unwrap(p, discont=pi, axis=-1): >>> phase = np.linspace(0, np.pi, num=5) >>> phase[3:] += np.pi >>> phase - array([ 0. , 0.78539816, 1.57079633, 5.49778714, 6.28318531]) + array([ 0. , 0.78539816, 1.57079633, 5.49778714, 6.28318531]) # may vary >>> np.unwrap(phase) - array([ 0. , 0.78539816, 1.57079633, -0.78539816, 0. ]) + array([ 0. , 0.78539816, 1.57079633, -0.78539816, 0. ]) # may vary """ p = asarray(p) @@ -1538,10 +1546,10 @@ def sort_complex(a): Examples -------- >>> np.sort_complex([5, 3, 6, 2, 1]) - array([ 1.+0.j, 2.+0.j, 3.+0.j, 5.+0.j, 6.+0.j]) + array([1.+0.j, 2.+0.j, 3.+0.j, 5.+0.j, 6.+0.j]) >>> np.sort_complex([1 + 2j, 2 - 1j, 3 - 2j, 3 - 3j, 3 + 5j]) - array([ 1.+2.j, 2.-1.j, 3.-3.j, 3.-2.j, 3.+5.j]) + array([1.+2.j, 2.-1.j, 3.-3.j, 3.-2.j, 3.+5.j]) """ b = array(a, copy=True) @@ -1587,7 +1595,7 @@ def trim_zeros(filt, trim='fb'): array([1, 2, 3, 0, 2, 1]) >>> np.trim_zeros(a, 'b') - array([0, 0, 0, 1, 2, 3, 0, 2, 1]) + array([0, 0, 0, ..., 0, 2, 1]) The input data type is preserved, list/tuple in means list/tuple out. @@ -1612,25 +1620,6 @@ def trim_zeros(filt, trim='fb'): last = last - 1 return filt[first:last] - -@deprecate -def unique(x): - """ - This function is deprecated. Use numpy.lib.arraysetops.unique() - instead. - """ - try: - tmp = x.flatten() - if tmp.size == 0: - return tmp - tmp.sort() - idx = concatenate(([True], tmp[1:] != tmp[:-1])) - return tmp[idx] - except AttributeError: - items = sorted(set(x)) - return asarray(items) - - def _extract_dispatcher(condition, arr): return (condition, arr) @@ -1885,6 +1874,7 @@ def _create_arrays(broadcast_shape, dim_sizes, list_of_core_dims, dtypes): return arrays +@set_module('numpy') class vectorize(object): """ vectorize(pyfunc, otypes=None, doc=None, excluded=None, cache=False, @@ -1967,11 +1957,11 @@ class vectorize(object): >>> out = vfunc([1, 2, 3, 4], 2) >>> type(out[0]) - <type 'numpy.int32'> + <class 'numpy.int64'> >>> vfunc = np.vectorize(myfunc, otypes=[float]) >>> out = vfunc([1, 2, 3, 4], 2) >>> type(out[0]) - <type 'numpy.float64'> + <class 'numpy.float64'> The `excluded` argument can be used to prevent vectorizing over certain arguments. This can be useful for array-like arguments of a fixed length @@ -1999,18 +1989,18 @@ class vectorize(object): >>> import scipy.stats >>> pearsonr = np.vectorize(scipy.stats.pearsonr, - ... signature='(n),(n)->(),()') - >>> pearsonr([[0, 1, 2, 3]], [[1, 2, 3, 4], [4, 3, 2, 1]]) + ... signature='(n),(n)->(),()') + >>> pearsonr([[0, 1, 2, 3]], [[1, 2, 3, 4], [4, 3, 2, 1]]) (array([ 1., -1.]), array([ 0., 0.])) Or for a vectorized convolution: >>> convolve = np.vectorize(np.convolve, signature='(n),(m)->(k)') >>> convolve(np.eye(4), [1, 2, 1]) - array([[ 1., 2., 1., 0., 0., 0.], - [ 0., 1., 2., 1., 0., 0.], - [ 0., 0., 1., 2., 1., 0.], - [ 0., 0., 0., 1., 2., 1.]]) + array([[1., 2., 1., 0., 0., 0.], + [0., 1., 2., 1., 0., 0.], + [0., 0., 1., 2., 1., 0.], + [0., 0., 0., 1., 2., 1.]]) See Also -------- @@ -2320,10 +2310,14 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, array `m` and let ``f = fweights`` and ``a = aweights`` for brevity. The steps to compute the weighted covariance are as follows:: + >>> m = np.arange(10, dtype=np.float64) + >>> f = np.arange(10) * 2 + >>> a = np.arange(10) ** 2. + >>> ddof = 9 # N - 1 >>> w = f * a >>> v1 = np.sum(w) >>> v2 = np.sum(w * a) - >>> m -= np.sum(m * w, axis=1, keepdims=True) / v1 + >>> m -= np.sum(m * w, axis=None, keepdims=True) / v1 >>> cov = np.dot(m * w, m.T) * v1 / (v1**2 - ddof * v2) Note that when ``a == 1``, the normalization factor @@ -2355,14 +2349,14 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, >>> x = [-2.1, -1, 4.3] >>> y = [3, 1.1, 0.12] >>> X = np.stack((x, y), axis=0) - >>> print(np.cov(X)) - [[ 11.71 -4.286 ] - [ -4.286 2.14413333]] - >>> print(np.cov(x, y)) - [[ 11.71 -4.286 ] - [ -4.286 2.14413333]] - >>> print(np.cov(x)) - 11.71 + >>> np.cov(X) + array([[11.71 , -4.286 ], # may vary + [-4.286 , 2.144133]]) + >>> np.cov(x, y) + array([[11.71 , -4.286 ], # may vary + [-4.286 , 2.144133]]) + >>> np.cov(x) + array(11.71) """ # Check inputs @@ -2549,6 +2543,7 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): return c +@set_module('numpy') def blackman(M): """ Return the Blackman window. @@ -2598,12 +2593,12 @@ def blackman(M): Examples -------- + >>> import matplotlib.pyplot as plt >>> np.blackman(12) - array([ -1.38777878e-17, 3.26064346e-02, 1.59903635e-01, - 4.14397981e-01, 7.36045180e-01, 9.67046769e-01, - 9.67046769e-01, 7.36045180e-01, 4.14397981e-01, - 1.59903635e-01, 3.26064346e-02, -1.38777878e-17]) - + array([-1.38777878e-17, 3.26064346e-02, 1.59903635e-01, # may vary + 4.14397981e-01, 7.36045180e-01, 9.67046769e-01, + 9.67046769e-01, 7.36045180e-01, 4.14397981e-01, + 1.59903635e-01, 3.26064346e-02, -1.38777878e-17]) Plot the window and the frequency response: @@ -2612,15 +2607,15 @@ def blackman(M): >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Blackman window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Blackman window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -2629,13 +2624,12 @@ def blackman(M): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of Blackman window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of Blackman window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> - >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + Text(0.5, 0, 'Normalized frequency [cycles per sample]') + >>> _ = plt.axis('tight') >>> plt.show() """ @@ -2647,6 +2641,7 @@ def blackman(M): return 0.42 - 0.5*cos(2.0*pi*n/(M-1)) + 0.08*cos(4.0*pi*n/(M-1)) +@set_module('numpy') def bartlett(M): """ Return the Bartlett window. @@ -2706,8 +2701,9 @@ def bartlett(M): Examples -------- + >>> import matplotlib.pyplot as plt >>> np.bartlett(12) - array([ 0. , 0.18181818, 0.36363636, 0.54545455, 0.72727273, + array([ 0. , 0.18181818, 0.36363636, 0.54545455, 0.72727273, # may vary 0.90909091, 0.90909091, 0.72727273, 0.54545455, 0.36363636, 0.18181818, 0. ]) @@ -2718,15 +2714,15 @@ def bartlett(M): >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Bartlett window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Bartlett window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -2735,13 +2731,12 @@ def bartlett(M): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of Bartlett window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of Bartlett window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> - >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + Text(0.5, 0, 'Normalized frequency [cycles per sample]') + >>> _ = plt.axis('tight') >>> plt.show() """ @@ -2753,6 +2748,7 @@ def bartlett(M): return where(less_equal(n, (M-1)/2.0), 2.0*n/(M-1), 2.0 - 2.0*n/(M-1)) +@set_module('numpy') def hanning(M): """ Return the Hanning window. @@ -2807,26 +2803,27 @@ def hanning(M): Examples -------- >>> np.hanning(12) - array([ 0. , 0.07937323, 0.29229249, 0.57115742, 0.82743037, - 0.97974649, 0.97974649, 0.82743037, 0.57115742, 0.29229249, - 0.07937323, 0. ]) + array([0. , 0.07937323, 0.29229249, 0.57115742, 0.82743037, + 0.97974649, 0.97974649, 0.82743037, 0.57115742, 0.29229249, + 0.07937323, 0. ]) Plot the window and its frequency response: + >>> import matplotlib.pyplot as plt >>> from numpy.fft import fft, fftshift >>> window = np.hanning(51) >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Hann window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Hann window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -2835,13 +2832,13 @@ def hanning(M): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of the Hann window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of the Hann window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Normalized frequency [cycles per sample]') >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + ... >>> plt.show() """ @@ -2853,6 +2850,7 @@ def hanning(M): return 0.5 - 0.5*cos(2.0*pi*n/(M-1)) +@set_module('numpy') def hamming(M): """ Return the Hamming window. @@ -2905,26 +2903,27 @@ def hamming(M): Examples -------- >>> np.hamming(12) - array([ 0.08 , 0.15302337, 0.34890909, 0.60546483, 0.84123594, + array([ 0.08 , 0.15302337, 0.34890909, 0.60546483, 0.84123594, # may vary 0.98136677, 0.98136677, 0.84123594, 0.60546483, 0.34890909, 0.15302337, 0.08 ]) Plot the window and the frequency response: + >>> import matplotlib.pyplot as plt >>> from numpy.fft import fft, fftshift >>> window = np.hamming(51) >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Hamming window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Hamming window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -2933,13 +2932,13 @@ def hamming(M): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of Hamming window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of Hamming window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Normalized frequency [cycles per sample]') >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + ... >>> plt.show() """ @@ -3088,9 +3087,9 @@ def i0(x): Examples -------- >>> np.i0([0.]) - array(1.0) + array(1.0) # may vary >>> np.i0([0., 1. + 2j]) - array([ 1.00000000+0.j , 0.18785373+0.64616944j]) + array([ 1.00000000+0.j , 0.18785373+0.64616944j]) # may vary """ x = atleast_1d(x).copy() @@ -3106,6 +3105,7 @@ def i0(x): ## End of cephes code for i0 +@set_module('numpy') def kaiser(M, beta): """ Return the Kaiser window. @@ -3184,11 +3184,12 @@ def kaiser(M, beta): Examples -------- + >>> import matplotlib.pyplot as plt >>> np.kaiser(12, 14) - array([ 7.72686684e-06, 3.46009194e-03, 4.65200189e-02, - 2.29737120e-01, 5.99885316e-01, 9.45674898e-01, - 9.45674898e-01, 5.99885316e-01, 2.29737120e-01, - 4.65200189e-02, 3.46009194e-03, 7.72686684e-06]) + array([7.72686684e-06, 3.46009194e-03, 4.65200189e-02, # may vary + 2.29737120e-01, 5.99885316e-01, 9.45674898e-01, + 9.45674898e-01, 5.99885316e-01, 2.29737120e-01, + 4.65200189e-02, 3.46009194e-03, 7.72686684e-06]) Plot the window and the frequency response: @@ -3198,15 +3199,15 @@ def kaiser(M, beta): >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Kaiser window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Kaiser window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -3215,13 +3216,13 @@ def kaiser(M, beta): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of Kaiser window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of Kaiser window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Normalized frequency [cycles per sample]') >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + (-0.5, 0.5, -100.0, ...) # may vary >>> plt.show() """ @@ -3277,31 +3278,32 @@ def sinc(x): Examples -------- + >>> import matplotlib.pyplot as plt >>> x = np.linspace(-4, 4, 41) >>> np.sinc(x) - array([ -3.89804309e-17, -4.92362781e-02, -8.40918587e-02, + array([-3.89804309e-17, -4.92362781e-02, -8.40918587e-02, # may vary -8.90384387e-02, -5.84680802e-02, 3.89804309e-17, - 6.68206631e-02, 1.16434881e-01, 1.26137788e-01, - 8.50444803e-02, -3.89804309e-17, -1.03943254e-01, + 6.68206631e-02, 1.16434881e-01, 1.26137788e-01, + 8.50444803e-02, -3.89804309e-17, -1.03943254e-01, -1.89206682e-01, -2.16236208e-01, -1.55914881e-01, - 3.89804309e-17, 2.33872321e-01, 5.04551152e-01, - 7.56826729e-01, 9.35489284e-01, 1.00000000e+00, - 9.35489284e-01, 7.56826729e-01, 5.04551152e-01, - 2.33872321e-01, 3.89804309e-17, -1.55914881e-01, - -2.16236208e-01, -1.89206682e-01, -1.03943254e-01, - -3.89804309e-17, 8.50444803e-02, 1.26137788e-01, - 1.16434881e-01, 6.68206631e-02, 3.89804309e-17, + 3.89804309e-17, 2.33872321e-01, 5.04551152e-01, + 7.56826729e-01, 9.35489284e-01, 1.00000000e+00, + 9.35489284e-01, 7.56826729e-01, 5.04551152e-01, + 2.33872321e-01, 3.89804309e-17, -1.55914881e-01, + -2.16236208e-01, -1.89206682e-01, -1.03943254e-01, + -3.89804309e-17, 8.50444803e-02, 1.26137788e-01, + 1.16434881e-01, 6.68206631e-02, 3.89804309e-17, -5.84680802e-02, -8.90384387e-02, -8.40918587e-02, -4.92362781e-02, -3.89804309e-17]) >>> plt.plot(x, np.sinc(x)) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Sinc Function") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Sinc Function') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("X") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'X') >>> plt.show() It works in 2-D as well: @@ -3473,18 +3475,18 @@ def median(a, axis=None, out=None, overwrite_input=False, keepdims=False): >>> np.median(a) 3.5 >>> np.median(a, axis=0) - array([ 6.5, 4.5, 2.5]) + array([6.5, 4.5, 2.5]) >>> np.median(a, axis=1) - array([ 7., 2.]) + array([7., 2.]) >>> m = np.median(a, axis=0) >>> out = np.zeros_like(m) >>> np.median(a, axis=0, out=m) - array([ 6.5, 4.5, 2.5]) + array([6.5, 4.5, 2.5]) >>> m - array([ 6.5, 4.5, 2.5]) + array([6.5, 4.5, 2.5]) >>> b = a.copy() >>> np.median(b, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a==b) >>> b = a.copy() >>> np.median(b, axis=None, overwrite_input=True) @@ -3651,23 +3653,23 @@ def percentile(a, q, axis=None, out=None, >>> np.percentile(a, 50) 3.5 >>> np.percentile(a, 50, axis=0) - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> np.percentile(a, 50, axis=1) - array([ 7., 2.]) + array([7., 2.]) >>> np.percentile(a, 50, axis=1, keepdims=True) - array([[ 7.], - [ 2.]]) + array([[7.], + [2.]]) >>> m = np.percentile(a, 50, axis=0) >>> out = np.zeros_like(m) >>> np.percentile(a, 50, axis=0, out=out) - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> m - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> b = a.copy() >>> np.percentile(b, 50, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a == b) The different types of interpolation can be visualized graphically: @@ -3793,21 +3795,21 @@ def quantile(a, q, axis=None, out=None, >>> np.quantile(a, 0.5) 3.5 >>> np.quantile(a, 0.5, axis=0) - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> np.quantile(a, 0.5, axis=1) - array([ 7., 2.]) + array([7., 2.]) >>> np.quantile(a, 0.5, axis=1, keepdims=True) - array([[ 7.], - [ 2.]]) + array([[7.], + [2.]]) >>> m = np.quantile(a, 0.5, axis=0) >>> out = np.zeros_like(m) >>> np.quantile(a, 0.5, axis=0, out=out) - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> m - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> b = a.copy() >>> np.quantile(b, 0.5, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a == b) """ q = np.asanyarray(q) @@ -4036,9 +4038,9 @@ def trapz(y, x=None, dx=1.0, axis=-1): array([[0, 1, 2], [3, 4, 5]]) >>> np.trapz(a, axis=0) - array([ 1.5, 2.5, 3.5]) + array([1.5, 2.5, 3.5]) >>> np.trapz(a, axis=1) - array([ 2., 8.]) + array([2., 8.]) """ y = asanyarray(y) @@ -4156,17 +4158,17 @@ def meshgrid(*xi, **kwargs): >>> y = np.linspace(0, 1, ny) >>> xv, yv = np.meshgrid(x, y) >>> xv - array([[ 0. , 0.5, 1. ], - [ 0. , 0.5, 1. ]]) + array([[0. , 0.5, 1. ], + [0. , 0.5, 1. ]]) >>> yv - array([[ 0., 0., 0.], - [ 1., 1., 1.]]) + array([[0., 0., 0.], + [1., 1., 1.]]) >>> xv, yv = np.meshgrid(x, y, sparse=True) # make sparse output arrays >>> xv - array([[ 0. , 0.5, 1. ]]) + array([[0. , 0.5, 1. ]]) >>> yv - array([[ 0.], - [ 1.]]) + array([[0.], + [1.]]) `meshgrid` is very useful to evaluate functions on a grid. @@ -4228,7 +4230,7 @@ def delete(arr, obj, axis=None): arr : array_like Input array. obj : slice, int or array of ints - Indicate which sub-arrays to remove. + Indicate indices of sub-arrays to remove along the specified axis. axis : int, optional The axis along which to delete the subarray defined by `obj`. If `axis` is None, `obj` is applied to the flattened array. @@ -4249,6 +4251,7 @@ def delete(arr, obj, axis=None): ----- Often it is preferable to use a boolean mask. For example: + >>> arr = np.arange(12) + 1 >>> mask = np.ones(len(arr), dtype=bool) >>> mask[[0,2,4]] = False >>> result = arr[mask,...] @@ -4480,7 +4483,7 @@ def insert(arr, obj, values, axis=None): [2, 2], [3, 3]]) >>> np.insert(a, 1, 5) - array([1, 5, 1, 2, 2, 3, 3]) + array([1, 5, 1, ..., 2, 3, 3]) >>> np.insert(a, 1, 5, axis=1) array([[1, 5, 1], [2, 5, 2], @@ -4500,13 +4503,13 @@ def insert(arr, obj, values, axis=None): >>> b array([1, 1, 2, 2, 3, 3]) >>> np.insert(b, [2, 2], [5, 6]) - array([1, 1, 5, 6, 2, 2, 3, 3]) + array([1, 1, 5, ..., 2, 3, 3]) >>> np.insert(b, slice(2, 4), [5, 6]) - array([1, 1, 5, 2, 6, 2, 3, 3]) + array([1, 1, 5, ..., 2, 3, 3]) >>> np.insert(b, [2, 2], [7.13, False]) # type casting - array([1, 1, 7, 0, 2, 2, 3, 3]) + array([1, 1, 7, ..., 2, 3, 3]) >>> x = np.arange(8).reshape(2, 4) >>> idx = (1, 3) @@ -4670,7 +4673,7 @@ def append(arr, values, axis=None): Examples -------- >>> np.append([1, 2, 3], [[4, 5, 6], [7, 8, 9]]) - array([1, 2, 3, 4, 5, 6, 7, 8, 9]) + array([1, 2, 3, ..., 7, 8, 9]) When `axis` is specified, `values` must have the correct shape. @@ -4680,8 +4683,8 @@ def append(arr, values, axis=None): [7, 8, 9]]) >>> np.append([[1, 2, 3], [4, 5, 6]], [7, 8, 9], axis=0) Traceback (most recent call last): - ... - ValueError: arrays must have same number of dimensions + ... + ValueError: all the input arrays must have same number of dimensions """ arr = asanyarray(arr) diff --git a/numpy/lib/histograms.py b/numpy/lib/histograms.py index 1ff25b81f..7b229cc89 100644 --- a/numpy/lib/histograms.py +++ b/numpy/lib/histograms.py @@ -3,21 +3,25 @@ Histogram-related functions """ from __future__ import division, absolute_import, print_function +import functools import operator import warnings import numpy as np from numpy.compat.py3k import basestring -from numpy.core.overrides import array_function_dispatch +from numpy.core import overrides __all__ = ['histogram', 'histogramdd', 'histogram_bin_edges'] +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + # range is a keyword argument to many functions, so save the builtin so they can # use it. _range = range -def _hist_bin_sqrt(x): +def _hist_bin_sqrt(x, range): """ Square root histogram bin estimator. @@ -34,10 +38,11 @@ def _hist_bin_sqrt(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return x.ptp() / np.sqrt(x.size) -def _hist_bin_sturges(x): +def _hist_bin_sturges(x, range): """ Sturges histogram bin estimator. @@ -56,10 +61,11 @@ def _hist_bin_sturges(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return x.ptp() / (np.log2(x.size) + 1.0) -def _hist_bin_rice(x): +def _hist_bin_rice(x, range): """ Rice histogram bin estimator. @@ -79,10 +85,11 @@ def _hist_bin_rice(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return x.ptp() / (2.0 * x.size ** (1.0 / 3)) -def _hist_bin_scott(x): +def _hist_bin_scott(x, range): """ Scott histogram bin estimator. @@ -100,10 +107,52 @@ def _hist_bin_scott(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return (24.0 * np.pi**0.5 / x.size)**(1.0 / 3.0) * np.std(x) -def _hist_bin_doane(x): +def _hist_bin_stone(x, range): + """ + Histogram bin estimator based on minimizing the estimated integrated squared error (ISE). + + The number of bins is chosen by minimizing the estimated ISE against the unknown true distribution. + The ISE is estimated using cross-validation and can be regarded as a generalization of Scott's rule. + https://en.wikipedia.org/wiki/Histogram#Scott.27s_normal_reference_rule + + This paper by Stone appears to be the origination of this rule. + http://digitalassets.lib.berkeley.edu/sdtr/ucb/text/34.pdf + + Parameters + ---------- + x : array_like + Input data that is to be histogrammed, trimmed to range. May not + be empty. + range : (float, float) + The lower and upper range of the bins. + + Returns + ------- + h : An estimate of the optimal bin width for the given data. + """ + + n = x.size + ptp_x = np.ptp(x) + if n <= 1 or ptp_x == 0: + return 0 + + def jhat(nbins): + hh = ptp_x / nbins + p_k = np.histogram(x, bins=nbins, range=range)[0] / n + return (2 - (n + 1) * p_k.dot(p_k)) / hh + + nbins_upper_bound = max(100, int(np.sqrt(n))) + nbins = min(_range(1, nbins_upper_bound + 1), key=jhat) + if nbins == nbins_upper_bound: + warnings.warn("The number of bins estimated may be suboptimal.", RuntimeWarning, stacklevel=2) + return ptp_x / nbins + + +def _hist_bin_doane(x, range): """ Doane's histogram bin estimator. @@ -121,6 +170,7 @@ def _hist_bin_doane(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused if x.size > 2: sg1 = np.sqrt(6.0 * (x.size - 2) / ((x.size + 1.0) * (x.size + 3))) sigma = np.std(x) @@ -137,7 +187,7 @@ def _hist_bin_doane(x): return 0.0 -def _hist_bin_fd(x): +def _hist_bin_fd(x, range): """ The Freedman-Diaconis histogram bin estimator. @@ -162,11 +212,12 @@ def _hist_bin_fd(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused iqr = np.subtract(*np.percentile(x, [75, 25])) return 2.0 * iqr * x.size ** (-1.0 / 3.0) -def _hist_bin_auto(x): +def _hist_bin_auto(x, range): """ Histogram bin estimator that uses the minimum width of the Freedman-Diaconis and Sturges estimators if the FD bandwidth is non zero @@ -200,8 +251,9 @@ def _hist_bin_auto(x): -------- _hist_bin_fd, _hist_bin_sturges """ - fd_bw = _hist_bin_fd(x) - sturges_bw = _hist_bin_sturges(x) + fd_bw = _hist_bin_fd(x, range) + sturges_bw = _hist_bin_sturges(x, range) + del range # unused if fd_bw: return min(fd_bw, sturges_bw) else: @@ -209,7 +261,8 @@ def _hist_bin_auto(x): return sturges_bw # Private dict initialized at module load time -_hist_bin_selectors = {'auto': _hist_bin_auto, +_hist_bin_selectors = {'stone': _hist_bin_stone, + 'auto': _hist_bin_auto, 'doane': _hist_bin_doane, 'fd': _hist_bin_fd, 'rice': _hist_bin_rice, @@ -344,7 +397,7 @@ def _get_bin_edges(a, bins, range, weights): n_equal_bins = 1 else: # Do not call selectors on empty arrays - width = _hist_bin_selectors[bin_name](a) + width = _hist_bin_selectors[bin_name](a, (first_edge, last_edge)) if width: n_equal_bins = int(np.ceil(_unsigned_subtract(last_edge, first_edge) / width)) else: @@ -446,6 +499,11 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): Less robust estimator that that takes into account data variability and data size. + 'stone' + Estimator based on leave-one-out cross-validation estimate of + the integrated squared error. Can be regarded as a generalization + of Scott's rule. + 'rice' Estimator does not take variability into account, only data size. Commonly overestimates number of bins required. @@ -587,7 +645,7 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): >>> hist_0, bins_0 = np.histogram(arr[group_id == 0], bins='auto') >>> hist_1, bins_1 = np.histogram(arr[group_id == 1], bins='auto') - >>> hist_0; hist1 + >>> hist_0; hist_1 array([1, 1, 1]) array([2, 1, 1, 2]) >>> bins_0; bins_1 @@ -690,14 +748,14 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, >>> np.histogram([1, 2, 1], bins=[0, 1, 2, 3]) (array([0, 2, 1]), array([0, 1, 2, 3])) >>> np.histogram(np.arange(4), bins=np.arange(5), density=True) - (array([ 0.25, 0.25, 0.25, 0.25]), array([0, 1, 2, 3, 4])) + (array([0.25, 0.25, 0.25, 0.25]), array([0, 1, 2, 3, 4])) >>> np.histogram([[1, 2, 1], [1, 0, 1]], bins=[0,1,2,3]) (array([1, 4, 1]), array([0, 1, 2, 3])) >>> a = np.arange(5) >>> hist, bin_edges = np.histogram(a, density=True) >>> hist - array([ 0.5, 0. , 0.5, 0. , 0. , 0.5, 0. , 0.5, 0. , 0.5]) + array([0.5, 0. , 0.5, 0. , 0. , 0.5, 0. , 0.5, 0. , 0.5]) >>> hist.sum() 2.4999999999999996 >>> np.sum(hist * np.diff(bin_edges)) @@ -712,8 +770,9 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, >>> rng = np.random.RandomState(10) # deterministic random data >>> a = np.hstack((rng.normal(size=1000), ... rng.normal(loc=5, scale=2, size=1000))) - >>> plt.hist(a, bins='auto') # arguments are passed to np.histogram + >>> _ = plt.hist(a, bins='auto') # arguments are passed to np.histogram >>> plt.title("Histogram with 'auto' bins") + Text(0.5, 1.0, "Histogram with 'auto' bins") >>> plt.show() """ diff --git a/numpy/lib/index_tricks.py b/numpy/lib/index_tricks.py index 26243d231..64c491cfa 100644 --- a/numpy/lib/index_tricks.py +++ b/numpy/lib/index_tricks.py @@ -1,5 +1,6 @@ from __future__ import division, absolute_import, print_function +import functools import sys import math @@ -9,14 +10,18 @@ from numpy.core.numeric import ( ) from numpy.core.numerictypes import find_common_type, issubdtype -from . import function_base import numpy.matrixlib as matrixlib from .function_base import diff from numpy.core.multiarray import ravel_multi_index, unravel_index -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import set_module +from numpy.core import overrides, linspace from numpy.lib.stride_tricks import as_strided +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + __all__ = [ 'ravel_multi_index', 'unravel_index', 'mgrid', 'ogrid', 'r_', 'c_', 's_', 'index_exp', 'ix_', 'ndenumerate', 'ndindex', 'fill_diagonal', @@ -341,7 +346,7 @@ class AxisConcatenator(object): step = 1 if isinstance(step, complex): size = int(abs(step)) - newobj = function_base.linspace(start, stop, num=size) + newobj = linspace(start, stop, num=size) else: newobj = _nx.arange(start, stop, step) if ndmin > 1: @@ -473,7 +478,7 @@ class RClass(AxisConcatenator): Examples -------- >>> np.r_[np.array([1,2,3]), 0, 0, np.array([4,5,6])] - array([1, 2, 3, 0, 0, 4, 5, 6]) + array([1, 2, 3, ..., 4, 5, 6]) >>> np.r_[-1:1:6j, [0]*3, 5, 6] array([-1. , -0.6, -0.2, 0.2, 0.6, 1. , 0. , 0. , 0. , 5. , 6. ]) @@ -533,15 +538,18 @@ class CClass(AxisConcatenator): [2, 5], [3, 6]]) >>> np.c_[np.array([[1,2,3]]), 0, 0, np.array([[4,5,6]])] - array([[1, 2, 3, 0, 0, 4, 5, 6]]) + array([[1, 2, 3, ..., 4, 5, 6]]) """ def __init__(self): AxisConcatenator.__init__(self, -1, ndmin=2, trans1d=0) + c_ = CClass() + +@set_module('numpy') class ndenumerate(object): """ Multidimensional index iterator. @@ -592,6 +600,7 @@ class ndenumerate(object): next = __next__ +@set_module('numpy') class ndindex(object): """ An N-dimensional iterator object to index arrays. @@ -804,7 +813,7 @@ def fill_diagonal(a, val, wrap=False): >>> # tall matrices no wrap >>> a = np.zeros((5, 3),int) - >>> fill_diagonal(a, 4) + >>> np.fill_diagonal(a, 4) >>> a array([[4, 0, 0], [0, 4, 0], @@ -814,7 +823,7 @@ def fill_diagonal(a, val, wrap=False): >>> # tall matrices wrap >>> a = np.zeros((5, 3),int) - >>> fill_diagonal(a, 4, wrap=True) + >>> np.fill_diagonal(a, 4, wrap=True) >>> a array([[4, 0, 0], [0, 4, 0], @@ -824,7 +833,7 @@ def fill_diagonal(a, val, wrap=False): >>> # wide matrices >>> a = np.zeros((3, 5),int) - >>> fill_diagonal(a, 4, wrap=True) + >>> np.fill_diagonal(a, 4, wrap=True) >>> a array([[4, 0, 0, 0, 0], [0, 4, 0, 0, 0], @@ -852,6 +861,7 @@ def fill_diagonal(a, val, wrap=False): a.flat[:end:step] = val +@set_module('numpy') def diag_indices(n, ndim=2): """ Return the indices to access the main diagonal of an array. diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py index 0379ecb1a..52ad45b68 100644 --- a/numpy/lib/mixins.py +++ b/numpy/lib/mixins.py @@ -69,9 +69,6 @@ class NDArrayOperatorsMixin(object): deferring to the ``__array_ufunc__`` method, which subclasses must implement. - This class does not yet implement the special operators corresponding - to ``matmul`` (``@``), because ``np.matmul`` is not yet a NumPy ufunc. - It is useful for writing classes that do not inherit from `numpy.ndarray`, but that should support arithmetic and numpy universal functions like arrays as described in `A Mechanism for Overriding Ufuncs @@ -155,6 +152,8 @@ class NDArrayOperatorsMixin(object): __add__, __radd__, __iadd__ = _numeric_methods(um.add, 'add') __sub__, __rsub__, __isub__ = _numeric_methods(um.subtract, 'sub') __mul__, __rmul__, __imul__ = _numeric_methods(um.multiply, 'mul') + __matmul__, __rmatmul__, __imatmul__ = _numeric_methods( + um.matmul, 'matmul') if sys.version_info.major < 3: # Python 3 uses only __truediv__ and __floordiv__ __div__, __rdiv__, __idiv__ = _numeric_methods(um.divide, 'div') diff --git a/numpy/lib/nanfunctions.py b/numpy/lib/nanfunctions.py index 279c4c5c4..b3bf1880b 100644 --- a/numpy/lib/nanfunctions.py +++ b/numpy/lib/nanfunctions.py @@ -22,10 +22,15 @@ Functions """ from __future__ import division, absolute_import, print_function +import functools import warnings import numpy as np from numpy.lib import function_base -from numpy.core.overrides import array_function_dispatch +from numpy.core import overrides + + +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') __all__ = [ @@ -266,9 +271,9 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): >>> np.nanmin(a) 1.0 >>> np.nanmin(a, axis=0) - array([ 1., 2.]) + array([1., 2.]) >>> np.nanmin(a, axis=1) - array([ 1., 3.]) + array([1., 3.]) When positive infinity and negative infinity are present: @@ -379,9 +384,9 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): >>> np.nanmax(a) 3.0 >>> np.nanmax(a, axis=0) - array([ 3., 2.]) + array([3., 2.]) >>> np.nanmax(a, axis=1) - array([ 2., 3.]) + array([2., 3.]) When positive infinity and negative infinity are present: @@ -596,12 +601,15 @@ def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): >>> np.nansum(a) 3.0 >>> np.nansum(a, axis=0) - array([ 2., 1.]) + array([2., 1.]) >>> np.nansum([1, np.nan, np.inf]) inf >>> np.nansum([1, np.nan, np.NINF]) -inf - >>> np.nansum([1, np.nan, np.inf, -np.inf]) # both +/- infinity present + >>> from numpy.testing import suppress_warnings + >>> with suppress_warnings() as sup: + ... sup.filter(RuntimeWarning) + ... np.nansum([1, np.nan, np.inf, -np.inf]) # both +/- infinity present nan """ @@ -672,7 +680,7 @@ def nanprod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): >>> np.nanprod(a) 6.0 >>> np.nanprod(a, axis=0) - array([ 3., 2.]) + array([3., 2.]) """ a, mask = _replace_nan(a, 1) @@ -733,16 +741,16 @@ def nancumsum(a, axis=None, dtype=None, out=None): >>> np.nancumsum([1]) array([1]) >>> np.nancumsum([1, np.nan]) - array([ 1., 1.]) + array([1., 1.]) >>> a = np.array([[1, 2], [3, np.nan]]) >>> np.nancumsum(a) - array([ 1., 3., 6., 6.]) + array([1., 3., 6., 6.]) >>> np.nancumsum(a, axis=0) - array([[ 1., 2.], - [ 4., 2.]]) + array([[1., 2.], + [4., 2.]]) >>> np.nancumsum(a, axis=1) - array([[ 1., 3.], - [ 3., 3.]]) + array([[1., 3.], + [3., 3.]]) """ a, mask = _replace_nan(a, 0) @@ -800,16 +808,16 @@ def nancumprod(a, axis=None, dtype=None, out=None): >>> np.nancumprod([1]) array([1]) >>> np.nancumprod([1, np.nan]) - array([ 1., 1.]) + array([1., 1.]) >>> a = np.array([[1, 2], [3, np.nan]]) >>> np.nancumprod(a) - array([ 1., 2., 6., 6.]) + array([1., 2., 6., 6.]) >>> np.nancumprod(a, axis=0) - array([[ 1., 2.], - [ 3., 2.]]) + array([[1., 2.], + [3., 2.]]) >>> np.nancumprod(a, axis=1) - array([[ 1., 2.], - [ 3., 3.]]) + array([[1., 2.], + [3., 3.]]) """ a, mask = _replace_nan(a, 1) @@ -890,9 +898,9 @@ def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): >>> np.nanmean(a) 2.6666666666666665 >>> np.nanmean(a, axis=0) - array([ 2., 4.]) + array([2., 4.]) >>> np.nanmean(a, axis=1) - array([ 1., 3.5]) + array([1., 3.5]) # may vary """ arr, mask = _replace_nan(a, 0) @@ -1044,19 +1052,19 @@ def nanmedian(a, axis=None, out=None, overwrite_input=False, keepdims=np._NoValu >>> a = np.array([[10.0, 7, 4], [3, 2, 1]]) >>> a[0, 1] = np.nan >>> a - array([[ 10., nan, 4.], - [ 3., 2., 1.]]) + array([[10., nan, 4.], + [ 3., 2., 1.]]) >>> np.median(a) nan >>> np.nanmedian(a) 3.0 >>> np.nanmedian(a, axis=0) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> np.median(a, axis=1) - array([ 7., 2.]) + array([nan, 2.]) >>> b = a.copy() >>> np.nanmedian(b, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a==b) >>> b = a.copy() >>> np.nanmedian(b, axis=None, overwrite_input=True) @@ -1172,27 +1180,27 @@ def nanpercentile(a, q, axis=None, out=None, overwrite_input=False, >>> a = np.array([[10., 7., 4.], [3., 2., 1.]]) >>> a[0][1] = np.nan >>> a - array([[ 10., nan, 4.], - [ 3., 2., 1.]]) + array([[10., nan, 4.], + [ 3., 2., 1.]]) >>> np.percentile(a, 50) nan >>> np.nanpercentile(a, 50) - 3.5 + 3.0 >>> np.nanpercentile(a, 50, axis=0) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> np.nanpercentile(a, 50, axis=1, keepdims=True) - array([[ 7.], - [ 2.]]) + array([[7.], + [2.]]) >>> m = np.nanpercentile(a, 50, axis=0) >>> out = np.zeros_like(m) >>> np.nanpercentile(a, 50, axis=0, out=out) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> m - array([ 6.5, 2. , 2.5]) + array([6.5, 2. , 2.5]) >>> b = a.copy() >>> np.nanpercentile(b, 50, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a==b) """ @@ -1286,26 +1294,26 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, >>> a = np.array([[10., 7., 4.], [3., 2., 1.]]) >>> a[0][1] = np.nan >>> a - array([[ 10., nan, 4.], - [ 3., 2., 1.]]) + array([[10., nan, 4.], + [ 3., 2., 1.]]) >>> np.quantile(a, 0.5) nan >>> np.nanquantile(a, 0.5) - 3.5 + 3.0 >>> np.nanquantile(a, 0.5, axis=0) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> np.nanquantile(a, 0.5, axis=1, keepdims=True) - array([[ 7.], - [ 2.]]) + array([[7.], + [2.]]) >>> m = np.nanquantile(a, 0.5, axis=0) >>> out = np.zeros_like(m) >>> np.nanquantile(a, 0.5, axis=0, out=out) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> m - array([ 6.5, 2. , 2.5]) + array([6.5, 2. , 2.5]) >>> b = a.copy() >>> np.nanquantile(b, 0.5, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a==b) """ a = np.asanyarray(a) @@ -1460,12 +1468,12 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): Examples -------- >>> a = np.array([[1, np.nan], [3, 4]]) - >>> np.var(a) + >>> np.nanvar(a) 1.5555555555555554 >>> np.nanvar(a, axis=0) - array([ 1., 0.]) + array([1., 0.]) >>> np.nanvar(a, axis=1) - array([ 0., 0.25]) + array([0., 0.25]) # may vary """ arr, mask = _replace_nan(a, 0) @@ -1614,9 +1622,9 @@ def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): >>> np.nanstd(a) 1.247219128924647 >>> np.nanstd(a, axis=0) - array([ 1., 0.]) + array([1., 0.]) >>> np.nanstd(a, axis=1) - array([ 0., 0.5]) + array([0., 0.5]) # may vary """ var = nanvar(a, axis=axis, dtype=dtype, out=out, ddof=ddof, diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 62fc9c5b3..704fea108 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -3,6 +3,7 @@ from __future__ import division, absolute_import, print_function import sys import os import re +import functools import itertools import warnings import weakref @@ -11,7 +12,9 @@ from operator import itemgetter, index as opindex import numpy as np from . import format from ._datasource import DataSource +from numpy.core import overrides from numpy.core.multiarray import packbits, unpackbits +from numpy.core.overrides import set_module from numpy.core._internal import recursive from ._iotools import ( LineSplitter, NameValidator, StringConverter, ConverterError, @@ -21,7 +24,7 @@ from ._iotools import ( from numpy.compat import ( asbytes, asstr, asunicode, asbytes_nested, bytes, basestring, unicode, - is_pathlib_path + os_fspath, os_PathLike ) from numpy.core.numeric import pickle @@ -32,6 +35,7 @@ else: from collections import Mapping +@set_module('numpy') def loads(*args, **kwargs): # NumPy 1.15.0, 2017-12-10 warnings.warn( @@ -47,6 +51,10 @@ __all__ = [ ] +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + class BagObj(object): """ BagObj(obj) @@ -104,8 +112,8 @@ def zipfile_factory(file, *args, **kwargs): pathlib.Path objects. `args` and `kwargs` are passed to the zipfile.ZipFile constructor. """ - if is_pathlib_path(file): - file = str(file) + if not hasattr(file, 'read'): + file = os_fspath(file) import zipfile kwargs['allowZip64'] = True return zipfile.ZipFile(file, *args, **kwargs) @@ -160,13 +168,13 @@ class NpzFile(Mapping): >>> x = np.arange(10) >>> y = np.sin(x) >>> np.savez(outfile, x=x, y=y) - >>> outfile.seek(0) + >>> _ = outfile.seek(0) >>> npz = np.load(outfile) >>> isinstance(npz, np.lib.io.NpzFile) True - >>> npz.files - ['y', 'x'] + >>> sorted(npz.files) + ['x', 'y'] >>> npz['x'] # getitem access array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) >>> npz.f.x # attribute lookup @@ -276,6 +284,7 @@ class NpzFile(Mapping): return self.keys() +@set_module('numpy') def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, encoding='ASCII'): """ @@ -399,15 +408,12 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, pickle_kwargs = {} # TODO: Use contextlib.ExitStack once we drop Python 2 - if isinstance(file, basestring): - fid = open(file, "rb") - own_fid = True - elif is_pathlib_path(file): - fid = file.open("rb") - own_fid = True - else: + if hasattr(file, 'read'): fid = file own_fid = False + else: + fid = open(os_fspath(file), "rb") + own_fid = True try: # Code to distinguish from NumPy binary files and pickles. @@ -435,8 +441,8 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, else: # Try a pickle if not allow_pickle: - raise ValueError("allow_pickle=False, but file does not contain " - "non-pickled data") + raise ValueError("Cannot load file containing pickled data " + "when allow_pickle=False") try: return pickle.load(fid, **pickle_kwargs) except Exception: @@ -447,6 +453,11 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, fid.close() +def _save_dispatcher(file, arr, allow_pickle=None, fix_imports=None): + return (arr,) + + +@array_function_dispatch(_save_dispatcher) def save(file, arr, allow_pickle=True, fix_imports=True): """ Save an array to a binary file in NumPy ``.npy`` format. @@ -491,24 +502,20 @@ def save(file, arr, allow_pickle=True, fix_imports=True): >>> x = np.arange(10) >>> np.save(outfile, x) - >>> outfile.seek(0) # Only needed here to simulate closing & reopening file + >>> _ = outfile.seek(0) # Only needed here to simulate closing & reopening file >>> np.load(outfile) array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) """ own_fid = False - if isinstance(file, basestring): + if hasattr(file, 'read'): + fid = file + else: + file = os_fspath(file) if not file.endswith('.npy'): file = file + '.npy' fid = open(file, "wb") own_fid = True - elif is_pathlib_path(file): - if not file.name.endswith('.npy'): - file = file.parent / (file.name + '.npy') - fid = file.open("wb") - own_fid = True - else: - fid = file if sys.version_info[0] >= 3: pickle_kwargs = dict(fix_imports=fix_imports) @@ -525,6 +532,14 @@ def save(file, arr, allow_pickle=True, fix_imports=True): fid.close() +def _savez_dispatcher(file, *args, **kwds): + for a in args: + yield a + for v in kwds.values(): + yield v + + +@array_function_dispatch(_savez_dispatcher) def savez(file, *args, **kwds): """ Save several arrays into a single file in uncompressed ``.npz`` format. @@ -582,10 +597,10 @@ def savez(file, *args, **kwds): Using `savez` with \\*args, the arrays are saved with default names. >>> np.savez(outfile, x, y) - >>> outfile.seek(0) # Only needed here to simulate closing & reopening file + >>> _ = outfile.seek(0) # Only needed here to simulate closing & reopening file >>> npzfile = np.load(outfile) >>> npzfile.files - ['arr_1', 'arr_0'] + ['arr_0', 'arr_1'] >>> npzfile['arr_0'] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -593,10 +608,10 @@ def savez(file, *args, **kwds): >>> outfile = TemporaryFile() >>> np.savez(outfile, x=x, y=y) - >>> outfile.seek(0) + >>> _ = outfile.seek(0) >>> npzfile = np.load(outfile) - >>> npzfile.files - ['y', 'x'] + >>> sorted(npzfile.files) + ['x', 'y'] >>> npzfile['x'] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -604,6 +619,14 @@ def savez(file, *args, **kwds): _savez(file, args, kwds, False) +def _savez_compressed_dispatcher(file, *args, **kwds): + for a in args: + yield a + for v in kwds.values(): + yield v + + +@array_function_dispatch(_savez_compressed_dispatcher) def savez_compressed(file, *args, **kwds): """ Save several arrays into a single file in compressed ``.npz`` format. @@ -673,12 +696,10 @@ def _savez(file, args, kwds, compress, allow_pickle=True, pickle_kwargs=None): # component of the so-called standard library. import zipfile - if isinstance(file, basestring): + if not hasattr(file, 'read'): + file = os_fspath(file) if not file.endswith('.npz'): file = file + '.npz' - elif is_pathlib_path(file): - if not file.name.endswith('.npz'): - file = file.parent / (file.name + '.npz') namedict = kwds for i, val in enumerate(args): @@ -771,6 +792,8 @@ def _getconv(dtype): # amount of lines loadtxt reads in one chunk, can be overridden for testing _loadtxt_chunksize = 50000 + +@set_module('numpy') def loadtxt(fname, dtype=float, comments='#', delimiter=None, converters=None, skiprows=0, usecols=None, unpack=False, ndmin=0, encoding='bytes', max_rows=None): @@ -806,7 +829,7 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, `genfromtxt`): ``converters = {3: lambda s: float(s.strip() or 0)}``. Default: None. skiprows : int, optional - Skip the first `skiprows` lines; default: 0. + Skip the first `skiprows` lines, including comments; default: 0. usecols : int or sequence, optional Which columns to read, with 0 being the first. For example, ``usecols = (1,4,5)`` will extract the 2nd, 5th and 6th columns. @@ -868,21 +891,21 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, >>> from io import StringIO # StringIO behaves like a file object >>> c = StringIO(u"0 1\\n2 3") >>> np.loadtxt(c) - array([[ 0., 1.], - [ 2., 3.]]) + array([[0., 1.], + [2., 3.]]) >>> d = StringIO(u"M 21 72\\nF 35 58") >>> np.loadtxt(d, dtype={'names': ('gender', 'age', 'weight'), ... 'formats': ('S1', 'i4', 'f4')}) - array([('M', 21, 72.0), ('F', 35, 58.0)], - dtype=[('gender', '|S1'), ('age', '<i4'), ('weight', '<f4')]) + array([(b'M', 21, 72.), (b'F', 35, 58.)], + dtype=[('gender', 'S1'), ('age', '<i4'), ('weight', '<f4')]) >>> c = StringIO(u"1,0,2\\n3,0,4") >>> x, y = np.loadtxt(c, delimiter=',', usecols=(0, 2), unpack=True) >>> x - array([ 1., 3.]) + array([1., 3.]) >>> y - array([ 2., 4.]) + array([2., 4.]) """ # Type conversions for Py3 convenience @@ -926,8 +949,8 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, fown = False try: - if is_pathlib_path(fname): - fname = str(fname) + if isinstance(fname, os_PathLike): + fname = os_fspath(fname) if _is_string_like(fname): fh = np.lib._datasource.open(fname, 'rt', encoding=encoding) fencoding = getattr(fh, 'encoding', 'latin1') @@ -1154,6 +1177,13 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, return X +def _savetxt_dispatcher(fname, X, fmt=None, delimiter=None, newline=None, + header=None, footer=None, comments=None, + encoding=None): + return (X,) + + +@array_function_dispatch(_savetxt_dispatcher) def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', footer='', comments='# ', encoding=None): """ @@ -1315,8 +1345,8 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', self.write = self.write_bytes own_fh = False - if is_pathlib_path(fname): - fname = str(fname) + if isinstance(fname, os_PathLike): + fname = os_fspath(fname) if _is_string_like(fname): # datasource doesn't support creating a new file ... open(fname, 'wt').close() @@ -1404,6 +1434,7 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', fh.close() +@set_module('numpy') def fromregex(file, regexp, dtype, encoding=None): """ Construct an array from a text file, using regular expression parsing. @@ -1450,17 +1481,17 @@ def fromregex(file, regexp, dtype, encoding=None): Examples -------- >>> f = open('test.dat', 'w') - >>> f.write("1312 foo\\n1534 bar\\n444 qux") + >>> _ = f.write("1312 foo\\n1534 bar\\n444 qux") >>> f.close() >>> regexp = r"(\\d+)\\s+(...)" # match [digits, whitespace, anything] >>> output = np.fromregex('test.dat', regexp, ... [('num', np.int64), ('key', 'S3')]) >>> output - array([(1312L, 'foo'), (1534L, 'bar'), (444L, 'qux')], - dtype=[('num', '<i8'), ('key', '|S3')]) + array([(1312, b'foo'), (1534, b'bar'), ( 444, b'qux')], + dtype=[('num', '<i8'), ('key', 'S3')]) >>> output['num'] - array([1312, 1534, 444], dtype=int64) + array([1312, 1534, 444]) """ own_fh = False @@ -1502,6 +1533,7 @@ def fromregex(file, regexp, dtype, encoding=None): #####-------------------------------------------------------------------------- +@set_module('numpy') def genfromtxt(fname, dtype=float, comments='#', delimiter=None, skip_header=0, skip_footer=0, converters=None, missing_values=None, filling_values=None, usecols=None, @@ -1642,26 +1674,26 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, >>> data = np.genfromtxt(s, dtype=[('myint','i8'),('myfloat','f8'), ... ('mystring','S5')], delimiter=",") >>> data - array((1, 1.3, 'abcde'), - dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '|S5')]) + array((1, 1.3, b'abcde'), + dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')]) Using dtype = None - >>> s.seek(0) # needed for StringIO example only + >>> _ = s.seek(0) # needed for StringIO example only >>> data = np.genfromtxt(s, dtype=None, ... names = ['myint','myfloat','mystring'], delimiter=",") >>> data - array((1, 1.3, 'abcde'), - dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '|S5')]) + array((1, 1.3, b'abcde'), + dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')]) Specifying dtype and names - >>> s.seek(0) + >>> _ = s.seek(0) >>> data = np.genfromtxt(s, dtype="i8,f8,S5", ... names=['myint','myfloat','mystring'], delimiter=",") >>> data - array((1, 1.3, 'abcde'), - dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '|S5')]) + array((1, 1.3, b'abcde'), + dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')]) An example with fixed-width columns @@ -1669,8 +1701,8 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, >>> data = np.genfromtxt(s, dtype=None, names=['intvar','fltvar','strvar'], ... delimiter=[1,3,5]) >>> data - array((1, 1.3, 'abcde'), - dtype=[('intvar', '<i8'), ('fltvar', '<f8'), ('strvar', '|S5')]) + array((1, 1.3, b'abcde'), + dtype=[('intvar', '<i8'), ('fltvar', '<f8'), ('strvar', 'S5')]) """ if max_rows is not None: @@ -1699,8 +1731,8 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, # Initialize the filehandle, the LineSplitter and the NameValidator own_fhd = False try: - if is_pathlib_path(fname): - fname = str(fname) + if isinstance(fname, os_PathLike): + fname = os_fspath(fname) if isinstance(fname, basestring): fhd = iter(np.lib._datasource.open(fname, 'rt', encoding=encoding)) own_fhd = True @@ -2094,10 +2126,10 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, if names is None: # If the dtype is uniform (before sizing strings) - base = set([ + base = { c_type for c, c_type in zip(converters, column_types) - if c._checked]) + if c._checked} if len(base) == 1: uniform_type, = base (ddtype, mdtype) = (uniform_type, bool) diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 165fd1b95..b55764b5d 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -8,17 +8,26 @@ __all__ = ['poly', 'roots', 'polyint', 'polyder', 'polyadd', 'polysub', 'polymul', 'polydiv', 'polyval', 'poly1d', 'polyfit', 'RankWarning'] +import functools import re import warnings import numpy.core.numeric as NX from numpy.core import (isscalar, abs, finfo, atleast_1d, hstack, dot, array, ones) +from numpy.core import overrides +from numpy.core.overrides import set_module from numpy.lib.twodim_base import diag, vander from numpy.lib.function_base import trim_zeros from numpy.lib.type_check import iscomplex, real, imag, mintypecode from numpy.linalg import eigvals, lstsq, inv + +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + +@set_module('numpy') class RankWarning(UserWarning): """ Issued by `polyfit` when the Vandermonde matrix is rank deficient. @@ -29,6 +38,12 @@ class RankWarning(UserWarning): """ pass + +def _poly_dispatcher(seq_of_zeros): + return seq_of_zeros + + +@array_function_dispatch(_poly_dispatcher) def poly(seq_of_zeros): """ Find the coefficients of a polynomial with the given sequence of roots. @@ -95,7 +110,7 @@ def poly(seq_of_zeros): Given a sequence of a polynomial's zeros: >>> np.poly((0, 0, 0)) # Multiple root example - array([1, 0, 0, 0]) + array([1., 0., 0., 0.]) The line above represents z**3 + 0*z**2 + 0*z + 0. @@ -104,14 +119,14 @@ def poly(seq_of_zeros): The line above represents z**3 - z/4 - >>> np.poly((np.random.random(1.)[0], 0, np.random.random(1.)[0])) - array([ 1. , -0.77086955, 0.08618131, 0. ]) #random + >>> np.poly((np.random.random(1)[0], 0, np.random.random(1)[0])) + array([ 1. , -0.77086955, 0.08618131, 0. ]) # random Given a square array object: >>> P = np.array([[0, 1./3], [-1./2, 0]]) >>> np.poly(P) - array([ 1. , 0. , 0.16666667]) + array([1. , 0. , 0.16666667]) Note how in all cases the leading coefficient is always 1. @@ -145,6 +160,12 @@ def poly(seq_of_zeros): return a + +def _roots_dispatcher(p): + return p + + +@array_function_dispatch(_roots_dispatcher) def roots(p): """ Return the roots of a polynomial with coefficients given in p. @@ -229,6 +250,12 @@ def roots(p): roots = hstack((roots, NX.zeros(trailing_zeros, roots.dtype))) return roots + +def _polyint_dispatcher(p, m=None, k=None): + return (p,) + + +@array_function_dispatch(_polyint_dispatcher) def polyint(p, m=1, k=None): """ Return an antiderivative (indefinite integral) of a polynomial. @@ -245,7 +272,7 @@ def polyint(p, m=1, k=None): Parameters ---------- p : array_like or poly1d - Polynomial to differentiate. + Polynomial to integrate. A sequence is interpreted as polynomial coefficients, see `poly1d`. m : int, optional Order of the antiderivative. (Default: 1) @@ -268,7 +295,7 @@ def polyint(p, m=1, k=None): >>> p = np.poly1d([1,1,1]) >>> P = np.polyint(p) >>> P - poly1d([ 0.33333333, 0.5 , 1. , 0. ]) + poly1d([ 0.33333333, 0.5 , 1. , 0. ]) # may vary >>> np.polyder(P) == p True @@ -283,7 +310,7 @@ def polyint(p, m=1, k=None): 0.0 >>> P = np.polyint(p, 3, k=[6,5,3]) >>> P - poly1d([ 0.01666667, 0.04166667, 0.16666667, 3. , 5. , 3. ]) + poly1d([ 0.01666667, 0.04166667, 0.16666667, 3. , 5. , 3. ]) # may vary Note that 3 = 6 / 2!, and that the constants are given in the order of integrations. Constant of the highest-order polynomial term comes first: @@ -322,6 +349,12 @@ def polyint(p, m=1, k=None): return poly1d(val) return val + +def _polyder_dispatcher(p, m=None): + return (p,) + + +@array_function_dispatch(_polyder_dispatcher) def polyder(p, m=1): """ Return the derivative of the specified order of a polynomial. @@ -371,7 +404,7 @@ def polyder(p, m=1): >>> np.polyder(p, 3) poly1d([6]) >>> np.polyder(p, 4) - poly1d([ 0.]) + poly1d([0.]) """ m = int(m) @@ -390,6 +423,12 @@ def polyder(p, m=1): val = poly1d(val) return val + +def _polyfit_dispatcher(x, y, deg, rcond=None, full=None, w=None, cov=None): + return (x, y, w) + + +@array_function_dispatch(_polyfit_dispatcher) def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): """ Least squares polynomial fit. @@ -424,9 +463,14 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): w : array_like, shape (M,), optional Weights to apply to the y-coordinates of the sample points. For gaussian uncertainties, use 1/sigma (not 1/sigma**2). - cov : bool, optional - Return the estimate and the covariance matrix of the estimate - If full is True, then cov is not returned. + cov : bool or str, optional + If given and not `False`, return not just the estimate but also its + covariance matrix. By default, the covariance are scaled by + chi2/sqrt(N-dof), i.e., the weights are presumed to be unreliable + except in a relative sense and everything is scaled such that the + reduced chi2 is unity. This scaling is omitted if ``cov='unscaled'``, + as is relevant for the case that the weights are 1/sigma**2, with + sigma known to be a reliable estimate of the uncertainty. Returns ------- @@ -508,28 +552,29 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): >>> y = np.array([0.0, 0.8, 0.9, 0.1, -0.8, -1.0]) >>> z = np.polyfit(x, y, 3) >>> z - array([ 0.08703704, -0.81349206, 1.69312169, -0.03968254]) + array([ 0.08703704, -0.81349206, 1.69312169, -0.03968254]) # may vary It is convenient to use `poly1d` objects for dealing with polynomials: >>> p = np.poly1d(z) >>> p(0.5) - 0.6143849206349179 + 0.6143849206349179 # may vary >>> p(3.5) - -0.34732142857143039 + -0.34732142857143039 # may vary >>> p(10) - 22.579365079365115 + 22.579365079365115 # may vary High-order polynomials may oscillate wildly: >>> p30 = np.poly1d(np.polyfit(x, y, 30)) - /... RankWarning: Polyfit may be poorly conditioned... + ... + >>> # RankWarning: Polyfit may be poorly conditioned... >>> p30(4) - -0.80000000000000204 + -0.80000000000000204 # may vary >>> p30(5) - -0.99999999999999445 + -0.99999999999999445 # may vary >>> p30(4.5) - -0.10547061179440398 + -0.10547061179440398 # may vary Illustration: @@ -594,14 +639,17 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): elif cov: Vbase = inv(dot(lhs.T, lhs)) Vbase /= NX.outer(scale, scale) - # Some literature ignores the extra -2.0 factor in the denominator, but - # it is included here because the covariance of Multivariate Student-T - # (which is implied by a Bayesian uncertainty analysis) includes it. - # Plus, it gives a slightly more conservative estimate of uncertainty. - if len(x) <= order + 2: - raise ValueError("the number of data points must exceed order + 2 " - "for Bayesian estimate the covariance matrix") - fac = resids / (len(x) - order - 2.0) + if cov == "unscaled": + fac = 1 + else: + if len(x) <= order: + raise ValueError("the number of data points must exceed order " + "to scale the covariance matrix") + # note, this used to be: fac = resids / (len(x) - order - 2.0) + # it was deciced that the "- 2" (originally justified by "Bayesian + # uncertainty analysis") is not was the user expects + # (see gh-11196 and gh-11197) + fac = resids / (len(x) - order) if y.ndim == 1: return c, Vbase * fac else: @@ -610,6 +658,11 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): return c +def _polyval_dispatcher(p, x): + return (p, x) + + +@array_function_dispatch(_polyval_dispatcher) def polyval(p, x): """ Evaluate a polynomial at specific values. @@ -664,11 +717,11 @@ def polyval(p, x): >>> np.polyval([3,0,1], 5) # 3 * 5**2 + 0 * 5**1 + 1 76 >>> np.polyval([3,0,1], np.poly1d(5)) - poly1d([ 76.]) + poly1d([76.]) >>> np.polyval(np.poly1d([3,0,1]), 5) 76 >>> np.polyval(np.poly1d([3,0,1]), np.poly1d(5)) - poly1d([ 76.]) + poly1d([76.]) """ p = NX.asarray(p) @@ -681,6 +734,12 @@ def polyval(p, x): y = y * x + p[i] return y + +def _binary_op_dispatcher(a1, a2): + return (a1, a2) + + +@array_function_dispatch(_binary_op_dispatcher) def polyadd(a1, a2): """ Find the sum of two polynomials. @@ -741,6 +800,8 @@ def polyadd(a1, a2): val = poly1d(val) return val + +@array_function_dispatch(_binary_op_dispatcher) def polysub(a1, a2): """ Difference (subtraction) of two polynomials. @@ -788,6 +849,7 @@ def polysub(a1, a2): return val +@array_function_dispatch(_binary_op_dispatcher) def polymul(a1, a2): """ Find the product of two polynomials. @@ -844,6 +906,12 @@ def polymul(a1, a2): val = poly1d(val) return val + +def _polydiv_dispatcher(u, v): + return (u, v) + + +@array_function_dispatch(_polydiv_dispatcher) def polydiv(u, v): """ Returns the quotient and remainder of polynomial division. @@ -886,7 +954,7 @@ def polydiv(u, v): >>> x = np.array([3.0, 5.0, 2.0]) >>> y = np.array([2.0, 1.0]) >>> np.polydiv(x, y) - (array([ 1.5 , 1.75]), array([ 0.25])) + (array([1.5 , 1.75]), array([0.25])) """ truepoly = (isinstance(u, poly1d) or isinstance(u, poly1d)) @@ -937,6 +1005,7 @@ def _raise_power(astr, wrap=70): return output + astr[n:] +@set_module('numpy') class poly1d(object): """ A one-dimensional polynomial class. @@ -980,7 +1049,7 @@ class poly1d(object): >>> p.r array([-1.+1.41421356j, -1.-1.41421356j]) >>> p(p.r) - array([ -4.44089210e-16+0.j, -4.44089210e-16+0.j]) + array([ -4.44089210e-16+0.j, -4.44089210e-16+0.j]) # may vary These numbers in the previous line represent (0, 0) to machine precision @@ -1007,7 +1076,7 @@ class poly1d(object): poly1d([ 1, 4, 10, 12, 9]) >>> (p**3 + 4) / p - (poly1d([ 1., 4., 10., 12., 9.]), poly1d([ 4.])) + (poly1d([ 1., 4., 10., 12., 9.]), poly1d([4.])) ``asarray(p)`` gives the coefficient array, so polynomials can be used in all functions that accept arrays: @@ -1029,7 +1098,7 @@ class poly1d(object): Construct a polynomial from its roots: >>> np.poly1d([1, 2], True) - poly1d([ 1, -3, 2]) + poly1d([ 1., -3., 2.]) This is the same polynomial as obtained by: diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index b6453d5a2..5ff35f0bb 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -14,8 +14,10 @@ import numpy.ma as ma from numpy import ndarray, recarray from numpy.ma import MaskedArray from numpy.ma.mrecords import MaskedRecords +from numpy.core.overrides import array_function_dispatch from numpy.lib._iotools import _is_string_like from numpy.compat import basestring +from numpy.testing import suppress_warnings if sys.version_info[0] < 3: from future_builtins import zip @@ -31,6 +33,11 @@ __all__ = [ ] +def _recursive_fill_fields_dispatcher(input, output): + return (input, output) + + +@array_function_dispatch(_recursive_fill_fields_dispatcher) def recursive_fill_fields(input, output): """ Fills fields from output with fields from input, @@ -50,11 +57,10 @@ def recursive_fill_fields(input, output): Examples -------- >>> from numpy.lib import recfunctions as rfn - >>> a = np.array([(1, 10.), (2, 20.)], dtype=[('A', int), ('B', float)]) + >>> a = np.array([(1, 10.), (2, 20.)], dtype=[('A', np.int64), ('B', np.float64)]) >>> b = np.zeros((3,), dtype=a.dtype) >>> rfn.recursive_fill_fields(a, b) - array([(1, 10.0), (2, 20.0), (0, 0.0)], - dtype=[('A', '<i4'), ('B', '<f8')]) + array([(1, 10.), (2, 20.), (0, 0.)], dtype=[('A', '<i8'), ('B', '<f8')]) """ newdtype = output.dtype @@ -82,11 +88,11 @@ def get_fieldspec(dtype): Examples -------- - >>> dt = np.dtype([(('a', 'A'), int), ('b', float, 3)]) + >>> dt = np.dtype([(('a', 'A'), np.int64), ('b', np.double, 3)]) >>> dt.descr - [(('a', 'A'), '<i4'), ('b', '<f8', (3,))] + [(('a', 'A'), '<i8'), ('b', '<f8', (3,))] >>> get_fieldspec(dt) - [(('a', 'A'), dtype('int32')), ('b', dtype(('<f8', (3,))))] + [(('a', 'A'), dtype('int64')), ('b', dtype(('<f8', (3,))))] """ if dtype.names is None: @@ -96,7 +102,7 @@ def get_fieldspec(dtype): fields = ((name, dtype.fields[name]) for name in dtype.names) # keep any titles, if present return [ - (name if len(f) == 2 else (f[2], name), f[0]) + (name if len(f) == 2 else (f[2], name), f[0]) for name, f in fields ] @@ -113,10 +119,15 @@ def get_names(adtype): Examples -------- >>> from numpy.lib import recfunctions as rfn - >>> rfn.get_names(np.empty((1,), dtype=int)) is None - True + >>> rfn.get_names(np.empty((1,), dtype=int)) + Traceback (most recent call last): + ... + AttributeError: 'numpy.ndarray' object has no attribute 'names' + >>> rfn.get_names(np.empty((1,), dtype=[('A',int), ('B', float)])) - ('A', 'B') + Traceback (most recent call last): + ... + AttributeError: 'numpy.ndarray' object has no attribute 'names' >>> adtype = np.dtype([('a', int), ('b', [('ba', int), ('bb', int)])]) >>> rfn.get_names(adtype) ('a', ('b', ('ba', 'bb'))) @@ -146,9 +157,13 @@ def get_names_flat(adtype): -------- >>> from numpy.lib import recfunctions as rfn >>> rfn.get_names_flat(np.empty((1,), dtype=int)) is None - True + Traceback (most recent call last): + ... + AttributeError: 'numpy.ndarray' object has no attribute 'names' >>> rfn.get_names_flat(np.empty((1,), dtype=[('A',int), ('B', float)])) - ('A', 'B') + Traceback (most recent call last): + ... + AttributeError: 'numpy.ndarray' object has no attribute 'names' >>> adtype = np.dtype([('a', int), ('b', [('ba', int), ('bb', int)])]) >>> rfn.get_names_flat(adtype) ('a', 'b', 'ba', 'bb') @@ -189,6 +204,11 @@ def flatten_descr(ndtype): return tuple(descr) +def _zip_dtype_dispatcher(seqarrays, flatten=None): + return seqarrays + + +@array_function_dispatch(_zip_dtype_dispatcher) def zip_dtype(seqarrays, flatten=False): newdtype = [] if flatten: @@ -205,6 +225,7 @@ def zip_dtype(seqarrays, flatten=False): return np.dtype(newdtype) +@array_function_dispatch(_zip_dtype_dispatcher) def zip_descr(seqarrays, flatten=False): """ Combine the dtype description of a series of arrays. @@ -297,6 +318,11 @@ def _izip_fields(iterable): yield element +def _izip_records_dispatcher(seqarrays, fill_value=None, flatten=None): + return seqarrays + + +@array_function_dispatch(_izip_records_dispatcher) def izip_records(seqarrays, fill_value=None, flatten=True): """ Returns an iterator of concatenated items from a sequence of arrays. @@ -357,6 +383,12 @@ def _fix_defaults(output, defaults=None): return output +def _merge_arrays_dispatcher(seqarrays, fill_value=None, flatten=None, + usemask=None, asrecarray=None): + return seqarrays + + +@array_function_dispatch(_merge_arrays_dispatcher) def merge_arrays(seqarrays, fill_value=-1, flatten=False, usemask=False, asrecarray=False): """ @@ -379,20 +411,18 @@ def merge_arrays(seqarrays, fill_value=-1, flatten=False, -------- >>> from numpy.lib import recfunctions as rfn >>> rfn.merge_arrays((np.array([1, 2]), np.array([10., 20., 30.]))) - masked_array(data = [(1, 10.0) (2, 20.0) (--, 30.0)], - mask = [(False, False) (False, False) (True, False)], - fill_value = (999999, 1e+20), - dtype = [('f0', '<i4'), ('f1', '<f8')]) - - >>> rfn.merge_arrays((np.array([1, 2]), np.array([10., 20., 30.])), - ... usemask=False) - array([(1, 10.0), (2, 20.0), (-1, 30.0)], - dtype=[('f0', '<i4'), ('f1', '<f8')]) - >>> rfn.merge_arrays((np.array([1, 2]).view([('a', int)]), + array([( 1, 10.), ( 2, 20.), (-1, 30.)], + dtype=[('f0', '<i8'), ('f1', '<f8')]) + + >>> rfn.merge_arrays((np.array([1, 2], dtype=np.int64), + ... np.array([10., 20., 30.])), usemask=False) + array([(1, 10.0), (2, 20.0), (-1, 30.0)], + dtype=[('f0', '<i8'), ('f1', '<f8')]) + >>> rfn.merge_arrays((np.array([1, 2]).view([('a', np.int64)]), ... np.array([10., 20., 30.])), ... usemask=False, asrecarray=True) - rec.array([(1, 10.0), (2, 20.0), (-1, 30.0)], - dtype=[('a', '<i4'), ('f1', '<f8')]) + rec.array([( 1, 10.), ( 2, 20.), (-1, 30.)], + dtype=[('a', '<i8'), ('f1', '<f8')]) Notes ----- @@ -494,6 +524,11 @@ def merge_arrays(seqarrays, fill_value=-1, flatten=False, return output +def _drop_fields_dispatcher(base, drop_names, usemask=None, asrecarray=None): + return (base,) + + +@array_function_dispatch(_drop_fields_dispatcher) def drop_fields(base, drop_names, usemask=True, asrecarray=False): """ Return a new array with fields in `drop_names` dropped. @@ -518,16 +553,14 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): -------- >>> from numpy.lib import recfunctions as rfn >>> a = np.array([(1, (2, 3.0)), (4, (5, 6.0))], - ... dtype=[('a', int), ('b', [('ba', float), ('bb', int)])]) + ... dtype=[('a', np.int64), ('b', [('ba', np.double), ('bb', np.int64)])]) >>> rfn.drop_fields(a, 'a') - array([((2.0, 3),), ((5.0, 6),)], - dtype=[('b', [('ba', '<f8'), ('bb', '<i4')])]) + array([((2., 3),), ((5., 6),)], + dtype=[('b', [('ba', '<f8'), ('bb', '<i8')])]) >>> rfn.drop_fields(a, 'ba') - array([(1, (3,)), (4, (6,))], - dtype=[('a', '<i4'), ('b', [('bb', '<i4')])]) + array([(1, (3,)), (4, (6,))], dtype=[('a', '<i8'), ('b', [('bb', '<i8')])]) >>> rfn.drop_fields(a, ['ba', 'bb']) - array([(1,), (4,)], - dtype=[('a', '<i4')]) + array([(1,), (4,)], dtype=[('a', '<i8')]) """ if _is_string_like(drop_names): drop_names = [drop_names] @@ -583,6 +616,11 @@ def _keep_fields(base, keep_names, usemask=True, asrecarray=False): return _fix_output(output, usemask=usemask, asrecarray=asrecarray) +def _rec_drop_fields_dispatcher(base, drop_names): + return (base,) + + +@array_function_dispatch(_rec_drop_fields_dispatcher) def rec_drop_fields(base, drop_names): """ Returns a new numpy.recarray with fields in `drop_names` dropped. @@ -590,6 +628,11 @@ def rec_drop_fields(base, drop_names): return drop_fields(base, drop_names, usemask=False, asrecarray=True) +def _rename_fields_dispatcher(base, namemapper): + return (base,) + + +@array_function_dispatch(_rename_fields_dispatcher) def rename_fields(base, namemapper): """ Rename the fields from a flexible-datatype ndarray or recarray. @@ -609,8 +652,8 @@ def rename_fields(base, namemapper): >>> a = np.array([(1, (2, [3.0, 30.])), (4, (5, [6.0, 60.]))], ... dtype=[('a', int),('b', [('ba', float), ('bb', (float, 2))])]) >>> rfn.rename_fields(a, {'a':'A', 'bb':'BB'}) - array([(1, (2.0, [3.0, 30.0])), (4, (5.0, [6.0, 60.0]))], - dtype=[('A', '<i4'), ('b', [('ba', '<f8'), ('BB', '<f8', 2)])]) + array([(1, (2., [ 3., 30.])), (4, (5., [ 6., 60.]))], + dtype=[('A', '<i8'), ('b', [('ba', '<f8'), ('BB', '<f8', (2,))])]) """ def _recursive_rename_fields(ndtype, namemapper): @@ -629,6 +672,14 @@ def rename_fields(base, namemapper): return base.view(newdtype) +def _append_fields_dispatcher(base, names, data, dtypes=None, + fill_value=None, usemask=None, asrecarray=None): + yield base + for d in data: + yield d + + +@array_function_dispatch(_append_fields_dispatcher) def append_fields(base, names, data, dtypes=None, fill_value=-1, usemask=True, asrecarray=False): """ @@ -699,6 +750,13 @@ def append_fields(base, names, data, dtypes=None, return _fix_output(output, usemask=usemask, asrecarray=asrecarray) +def _rec_append_fields_dispatcher(base, names, data, dtypes=None): + yield base + for d in data: + yield d + + +@array_function_dispatch(_rec_append_fields_dispatcher) def rec_append_fields(base, names, data, dtypes=None): """ Add new fields to an existing array. @@ -732,6 +790,12 @@ def rec_append_fields(base, names, data, dtypes=None): return append_fields(base, names, data=data, dtypes=dtypes, asrecarray=True, usemask=False) + +def _repack_fields_dispatcher(a, align=None, recurse=None): + return (a,) + + +@array_function_dispatch(_repack_fields_dispatcher) def repack_fields(a, align=False, recurse=False): """ Re-pack the fields of a structured array or dtype in memory. @@ -774,18 +838,18 @@ def repack_fields(a, align=False, recurse=False): ... print("offsets:", [d.fields[name][1] for name in d.names]) ... print("itemsize:", d.itemsize) ... - >>> dt = np.dtype('u1,i4,f4', align=True) + >>> dt = np.dtype('u1,<i4,<f4', align=True) >>> dt - dtype({'names':['f0','f1','f2'], 'formats':['u1','<i4','<f8'], 'offsets':[0,4,8], 'itemsize':16}, align=True) + dtype({'names':['f0','f1','f2'], 'formats':['u1','<i8','<f8'], 'offsets':[0,8,16], 'itemsize':24}, align=True) >>> print_offsets(dt) - offsets: [0, 4, 8] - itemsize: 16 + offsets: [0, 8, 16] + itemsize: 24 >>> packed_dt = repack_fields(dt) >>> packed_dt - dtype([('f0', 'u1'), ('f1', '<i4'), ('f2', '<f8')]) + dtype([('f0', 'u1'), ('f1', '<i8'), ('f2', '<f8')]) >>> print_offsets(packed_dt) - offsets: [0, 1, 5] - itemsize: 13 + offsets: [0, 1, 9] + itemsize: 17 """ if not isinstance(a, np.dtype): @@ -811,6 +875,351 @@ def repack_fields(a, align=False, recurse=False): dt = np.dtype(fieldinfo, align=align) return np.dtype((a.type, dt)) +def _get_fields_and_offsets(dt, offset=0): + """ + Returns a flat list of (dtype, count, offset) tuples of all the + scalar fields in the dtype "dt", including nested fields, in left + to right order. + """ + fields = [] + for name in dt.names: + field = dt.fields[name] + if field[0].names is None: + count = 1 + for size in field[0].shape: + count *= size + fields.append((field[0], count, field[1] + offset)) + else: + fields.extend(_get_fields_and_offsets(field[0], field[1] + offset)) + return fields + + +def _structured_to_unstructured_dispatcher(arr, dtype=None, copy=None, + casting=None): + return (arr,) + +@array_function_dispatch(_structured_to_unstructured_dispatcher) +def structured_to_unstructured(arr, dtype=None, copy=False, casting='unsafe'): + """ + Converts and n-D structured array into an (n+1)-D unstructured array. + + The new array will have a new last dimension equal in size to the + number of field-elements of the input array. If not supplied, the output + datatype is determined from the numpy type promotion rules applied to all + the field datatypes. + + Nested fields, as well as each element of any subarray fields, all count + as a single field-elements. + + Parameters + ---------- + arr : ndarray + Structured array or dtype to convert. Cannot contain object datatype. + dtype : dtype, optional + The dtype of the output unstructured array. + copy : bool, optional + See copy argument to `ndarray.astype`. If true, always return a copy. + If false, and `dtype` requirements are satisfied, a view is returned. + casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional + See casting argument of `ndarray.astype`. Controls what kind of data + casting may occur. + + Returns + ------- + unstructured : ndarray + Unstructured array with one more dimension. + + Examples + -------- + + >>> a = np.zeros(4, dtype=[('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + >>> a + array([(0, (0., 0), [0., 0.]), (0, (0., 0), [0., 0.]), + (0, (0., 0), [0., 0.]), (0, (0., 0), [0., 0.])], + dtype=[('a', '<i4'), ('b', [('f0', '<f4'), ('f1', '<u2')]), ('c', '<f4', (2,))]) + >>> structured_to_unstructured(arr) + array([[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]]) + + >>> b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + ... dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + >>> np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) + array([ 3. , 5.5, 9. , 11. ]) + + """ + if arr.dtype.names is None: + raise ValueError('arr must be a structured array') + + fields = _get_fields_and_offsets(arr.dtype) + n_fields = len(fields) + dts, counts, offsets = zip(*fields) + names = ['f{}'.format(n) for n in range(n_fields)] + + if dtype is None: + out_dtype = np.result_type(*[dt.base for dt in dts]) + else: + out_dtype = dtype + + # Use a series of views and casts to convert to an unstructured array: + + # first view using flattened fields (doesn't work for object arrays) + # Note: dts may include a shape for subarrays + flattened_fields = np.dtype({'names': names, + 'formats': dts, + 'offsets': offsets, + 'itemsize': arr.dtype.itemsize}) + with suppress_warnings() as sup: # until 1.16 (gh-12447) + sup.filter(FutureWarning, "Numpy has detected") + arr = arr.view(flattened_fields) + + # next cast to a packed format with all fields converted to new dtype + packed_fields = np.dtype({'names': names, + 'formats': [(out_dtype, c) for c in counts]}) + arr = arr.astype(packed_fields, copy=copy, casting=casting) + + # finally is it safe to view the packed fields as the unstructured type + return arr.view((out_dtype, sum(counts))) + +def _unstructured_to_structured_dispatcher(arr, dtype=None, names=None, + align=None, copy=None, casting=None): + return (arr,) + +@array_function_dispatch(_unstructured_to_structured_dispatcher) +def unstructured_to_structured(arr, dtype=None, names=None, align=False, + copy=False, casting='unsafe'): + """ + Converts and n-D unstructured array into an (n-1)-D structured array. + + The last dimension of the input array is converted into a structure, with + number of field-elements equal to the size of the last dimension of the + input array. By default all output fields have the input array's dtype, but + an output structured dtype with an equal number of fields-elements can be + supplied instead. + + Nested fields, as well as each element of any subarray fields, all count + towards the number of field-elements. + + Parameters + ---------- + arr : ndarray + Unstructured array or dtype to convert. + dtype : dtype, optional + The structured dtype of the output array + names : list of strings, optional + If dtype is not supplied, this specifies the field names for the output + dtype, in order. The field dtypes will be the same as the input array. + align : boolean, optional + Whether to create an aligned memory layout. + copy : bool, optional + See copy argument to `ndarray.astype`. If true, always return a copy. + If false, and `dtype` requirements are satisfied, a view is returned. + casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional + See casting argument of `ndarray.astype`. Controls what kind of data + casting may occur. + + Returns + ------- + structured : ndarray + Structured array with fewer dimensions. + + Examples + -------- + + >>> dt = np.dtype([('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + >>> a = np.arange(20).reshape((4,5)) + >>> a + array([[ 0, 1, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, 13, 14], + [15, 16, 17, 18, 19]]) + >>> unstructured_to_structured(a, dt) + array([( 0, ( 1., 2), [ 3., 4.]), ( 5, ( 6., 7), [ 8., 9.]), + (10, (11., 12), [13., 14.]), (15, (16., 17), [18., 19.])], + dtype=[('a', '<i4'), ('b', [('f0', '<f4'), ('f1', '<u2')]), ('c', '<f4', (2,))]) + + """ + if arr.shape == (): + raise ValueError('arr must have at least one dimension') + n_elem = arr.shape[-1] + + if dtype is None: + if names is None: + names = ['f{}'.format(n) for n in range(n_elem)] + out_dtype = np.dtype([(n, arr.dtype) for n in names], align=align) + fields = _get_fields_and_offsets(out_dtype) + dts, counts, offsets = zip(*fields) + else: + if names is not None: + raise ValueError("don't supply both dtype and names") + # sanity check of the input dtype + fields = _get_fields_and_offsets(dtype) + dts, counts, offsets = zip(*fields) + if n_elem != sum(counts): + raise ValueError('The length of the last dimension of arr must ' + 'be equal to the number of fields in dtype') + out_dtype = dtype + if align and not out_dtype.isalignedstruct: + raise ValueError("align was True but dtype is not aligned") + + names = ['f{}'.format(n) for n in range(len(fields))] + + # Use a series of views and casts to convert to a structured array: + + # first view as a packed structured array of one dtype + packed_fields = np.dtype({'names': names, + 'formats': [(arr.dtype, c) for c in counts]}) + arr = np.ascontiguousarray(arr).view(packed_fields) + + # next cast to an unpacked but flattened format with varied dtypes + flattened_fields = np.dtype({'names': names, + 'formats': dts, + 'offsets': offsets, + 'itemsize': out_dtype.itemsize}) + arr = arr.astype(flattened_fields, copy=copy, casting=casting) + + # finally view as the final nested dtype and remove the last axis + return arr.view(out_dtype)[..., 0] + +def _apply_along_fields_dispatcher(func, arr): + return (arr,) + +@array_function_dispatch(_apply_along_fields_dispatcher) +def apply_along_fields(func, arr): + """ + Apply function 'func' as a reduction across fields of a structured array. + + This is similar to `apply_along_axis`, but treats the fields of a + structured array as an extra axis. The fields are all first cast to a + common type following the type-promotion rules from `numpy.result_type` + applied to the field's dtypes. + + Parameters + ---------- + func : function + Function to apply on the "field" dimension. This function must + support an `axis` argument, like np.mean, np.sum, etc. + arr : ndarray + Structured array for which to apply func. + + Returns + ------- + out : ndarray + Result of the recution operation + + Examples + -------- + + >>> b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + ... dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + >>> apply_along_fields(np.mean, b) + array([ 2.66666667, 5.33333333, 8.66666667, 11. ]) + >>> apply_along_fields(np.mean, b[['x', 'z']]) + array([ 3. , 5.5, 9. , 11. ]) + + """ + if arr.dtype.names is None: + raise ValueError('arr must be a structured array') + + uarr = structured_to_unstructured(arr) + return func(uarr, axis=-1) + # works and avoids axis requirement, but very, very slow: + #return np.apply_along_axis(func, -1, uarr) + +def _assign_fields_by_name_dispatcher(dst, src, zero_unassigned=None): + return dst, src + +@array_function_dispatch(_assign_fields_by_name_dispatcher) +def assign_fields_by_name(dst, src, zero_unassigned=True): + """ + Assigns values from one structured array to another by field name. + + Normally in numpy >= 1.14, assignment of one structured array to another + copies fields "by position", meaning that the first field from the src is + copied to the first field of the dst, and so on, regardless of field name. + + This function instead copies "by field name", such that fields in the dst + are assigned from the identically named field in the src. This applies + recursively for nested structures. This is how structure assignment worked + in numpy >= 1.6 to <= 1.13. + + Parameters + ---------- + dst : ndarray + src : ndarray + The source and destination arrays during assignment. + zero_unassigned : bool, optional + If True, fields in the dst for which there was no matching + field in the src are filled with the value 0 (zero). This + was the behavior of numpy <= 1.13. If False, those fields + are not modified. + """ + + if dst.dtype.names is None: + dst[...] = src + return + + for name in dst.dtype.names: + if name not in src.dtype.names: + if zero_unassigned: + dst[name] = 0 + else: + assign_fields_by_name(dst[name], src[name], + zero_unassigned) + +def _require_fields_dispatcher(array, required_dtype): + return (array,) + +@array_function_dispatch(_require_fields_dispatcher) +def require_fields(array, required_dtype): + """ + Casts a structured array to a new dtype using assignment by field-name. + + This function assigns from the old to the new array by name, so the + value of a field in the output array is the value of the field with the + same name in the source array. This has the effect of creating a new + ndarray containing only the fields "required" by the required_dtype. + + If a field name in the required_dtype does not exist in the + input array, that field is created and set to 0 in the output array. + + Parameters + ---------- + a : ndarray + array to cast + required_dtype : dtype + datatype for output array + + Returns + ------- + out : ndarray + array with the new dtype, with field values copied from the fields in + the input array with the same name + + Examples + -------- + + >>> a = np.ones(4, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) + >>> require_fields(a, [('b', 'f4'), ('c', 'u1')]) + array([(1., 1), (1., 1), (1., 1), (1., 1)], + dtype=[('b', '<f4'), ('c', 'u1')]) + >>> require_fields(a, [('b', 'f4'), ('newf', 'u1')]) + array([(1., 0), (1., 0), (1., 0), (1., 0)], + dtype=[('b', '<f4'), ('newf', 'u1')]) + + """ + out = np.empty(array.shape, dtype=required_dtype) + assign_fields_by_name(out, array) + return out + + +def _stack_arrays_dispatcher(arrays, defaults=None, usemask=None, + asrecarray=None, autoconvert=None): + return arrays + + +@array_function_dispatch(_stack_arrays_dispatcher) def stack_arrays(arrays, defaults=None, usemask=True, asrecarray=False, autoconvert=False): """ @@ -839,15 +1248,16 @@ def stack_arrays(arrays, defaults=None, usemask=True, asrecarray=False, True >>> z = np.array([('A', 1), ('B', 2)], dtype=[('A', '|S3'), ('B', float)]) >>> zz = np.array([('a', 10., 100.), ('b', 20., 200.), ('c', 30., 300.)], - ... dtype=[('A', '|S3'), ('B', float), ('C', float)]) + ... dtype=[('A', '|S3'), ('B', np.double), ('C', np.double)]) >>> test = rfn.stack_arrays((z,zz)) >>> test - masked_array(data = [('A', 1.0, --) ('B', 2.0, --) ('a', 10.0, 100.0) ('b', 20.0, 200.0) - ('c', 30.0, 300.0)], - mask = [(False, False, True) (False, False, True) (False, False, False) - (False, False, False) (False, False, False)], - fill_value = ('N/A', 1e+20, 1e+20), - dtype = [('A', '|S3'), ('B', '<f8'), ('C', '<f8')]) + masked_array(data=[(b'A', 1.0, --), (b'B', 2.0, --), (b'a', 10.0, 100.0), + (b'b', 20.0, 200.0), (b'c', 30.0, 300.0)], + mask=[(False, False, True), (False, False, True), + (False, False, False), (False, False, False), + (False, False, False)], + fill_value=(b'N/A', 1.e+20, 1.e+20), + dtype=[('A', 'S3'), ('B', '<f8'), ('C', '<f8')]) """ if isinstance(arrays, ndarray): @@ -897,6 +1307,12 @@ def stack_arrays(arrays, defaults=None, usemask=True, asrecarray=False, usemask=usemask, asrecarray=asrecarray) +def _find_duplicates_dispatcher( + a, key=None, ignoremask=None, return_index=None): + return (a,) + + +@array_function_dispatch(_find_duplicates_dispatcher) def find_duplicates(a, key=None, ignoremask=True, return_index=False): """ Find the duplicates in a structured array along a given key @@ -920,7 +1336,10 @@ def find_duplicates(a, key=None, ignoremask=True, return_index=False): >>> a = np.ma.array([1, 1, 1, 2, 2, 3, 3], ... mask=[0, 0, 1, 0, 0, 0, 1]).view(ndtype) >>> rfn.find_duplicates(a, ignoremask=True, return_index=True) - ... # XXX: judging by the output, the ignoremask flag has no effect + (masked_array(data=[(1,), (1,), (2,), (2,)], + mask=[(False,), (False,), (False,), (False,)], + fill_value=(999999,), + dtype=[('a', '<i8')]), array([0, 1, 3, 4])) """ a = np.asanyarray(a).ravel() # Get a dictionary of fields @@ -951,8 +1370,15 @@ def find_duplicates(a, key=None, ignoremask=True, return_index=False): return duplicates +def _join_by_dispatcher( + key, r1, r2, jointype=None, r1postfix=None, r2postfix=None, + defaults=None, usemask=None, asrecarray=None): + return (r1, r2) + + +@array_function_dispatch(_join_by_dispatcher) def join_by(key, r1, r2, jointype='inner', r1postfix='1', r2postfix='2', - defaults=None, usemask=True, asrecarray=False): + defaults=None, usemask=True, asrecarray=False): """ Join arrays `r1` and `r2` on key `key`. @@ -1130,6 +1556,13 @@ def join_by(key, r1, r2, jointype='inner', r1postfix='1', r2postfix='2', return _fix_output(_fix_defaults(output, defaults), **kwargs) +def _rec_join_dispatcher( + key, r1, r2, jointype=None, r1postfix=None, r2postfix=None, + defaults=None): + return (r1, r2) + + +@array_function_dispatch(_rec_join_dispatcher) def rec_join(key, r1, r2, jointype='inner', r1postfix='1', r2postfix='2', defaults=None): """ diff --git a/numpy/lib/scimath.py b/numpy/lib/scimath.py index f1838fee6..5ac790ce9 100644 --- a/numpy/lib/scimath.py +++ b/numpy/lib/scimath.py @@ -20,6 +20,7 @@ from __future__ import division, absolute_import, print_function import numpy.core.numeric as nx import numpy.core.numerictypes as nt from numpy.core.numeric import asarray, any +from numpy.core.overrides import array_function_dispatch from numpy.lib.type_check import isreal @@ -58,7 +59,7 @@ def _tocomplex(arr): >>> a = np.array([1,2,3],np.short) >>> ac = np.lib.scimath._tocomplex(a); ac - array([ 1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) + array([1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) >>> ac.dtype dtype('complex64') @@ -69,7 +70,7 @@ def _tocomplex(arr): >>> b = np.array([1,2,3],np.double) >>> bc = np.lib.scimath._tocomplex(b); bc - array([ 1.+0.j, 2.+0.j, 3.+0.j]) + array([1.+0.j, 2.+0.j, 3.+0.j]) >>> bc.dtype dtype('complex128') @@ -80,13 +81,13 @@ def _tocomplex(arr): >>> c = np.array([1,2,3],np.csingle) >>> cc = np.lib.scimath._tocomplex(c); cc - array([ 1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) + array([1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) >>> c *= 2; c - array([ 2.+0.j, 4.+0.j, 6.+0.j], dtype=complex64) + array([2.+0.j, 4.+0.j, 6.+0.j], dtype=complex64) >>> cc - array([ 1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) + array([1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) """ if issubclass(arr.dtype.type, (nt.single, nt.byte, nt.short, nt.ubyte, nt.ushort, nt.csingle)): @@ -94,6 +95,7 @@ def _tocomplex(arr): else: return arr.astype(nt.cdouble) + def _fix_real_lt_zero(x): """Convert `x` to complex if it has real, negative components. @@ -121,6 +123,7 @@ def _fix_real_lt_zero(x): x = _tocomplex(x) return x + def _fix_int_lt_zero(x): """Convert `x` to double if it has real, negative components. @@ -147,6 +150,7 @@ def _fix_int_lt_zero(x): x = x * 1.0 return x + def _fix_real_abs_gt_1(x): """Convert `x` to complex if it has real components x_i with abs(x_i)>1. @@ -166,13 +170,19 @@ def _fix_real_abs_gt_1(x): array([0, 1]) >>> np.lib.scimath._fix_real_abs_gt_1([0,2]) - array([ 0.+0.j, 2.+0.j]) + array([0.+0.j, 2.+0.j]) """ x = asarray(x) if any(isreal(x) & (abs(x) > 1)): x = _tocomplex(x) return x + +def _unary_dispatcher(x): + return (x,) + + +@array_function_dispatch(_unary_dispatcher) def sqrt(x): """ Compute the square root of x. @@ -202,19 +212,21 @@ def sqrt(x): >>> np.lib.scimath.sqrt(1) 1.0 >>> np.lib.scimath.sqrt([1, 4]) - array([ 1., 2.]) + array([1., 2.]) But it automatically handles negative inputs: >>> np.lib.scimath.sqrt(-1) - (0.0+1.0j) + 1j >>> np.lib.scimath.sqrt([-1,4]) - array([ 0.+1.j, 2.+0.j]) + array([0.+1.j, 2.+0.j]) """ x = _fix_real_lt_zero(x) return nx.sqrt(x) + +@array_function_dispatch(_unary_dispatcher) def log(x): """ Compute the natural logarithm of `x`. @@ -261,6 +273,8 @@ def log(x): x = _fix_real_lt_zero(x) return nx.log(x) + +@array_function_dispatch(_unary_dispatcher) def log10(x): """ Compute the logarithm base 10 of `x`. @@ -303,12 +317,18 @@ def log10(x): 1.0 >>> np.emath.log10([-10**1, -10**2, 10**2]) - array([ 1.+1.3644j, 2.+1.3644j, 2.+0.j ]) + array([1.+1.3644j, 2.+1.3644j, 2.+0.j ]) """ x = _fix_real_lt_zero(x) return nx.log10(x) + +def _logn_dispatcher(n, x): + return (n, x,) + + +@array_function_dispatch(_logn_dispatcher) def logn(n, x): """ Take log base n of x. @@ -318,8 +338,8 @@ def logn(n, x): Parameters ---------- - n : int - The base in which the log is taken. + n : array_like + The integer base(s) in which the log is taken. x : array_like The value(s) whose log base `n` is (are) required. @@ -334,15 +354,17 @@ def logn(n, x): >>> np.set_printoptions(precision=4) >>> np.lib.scimath.logn(2, [4, 8]) - array([ 2., 3.]) + array([2., 3.]) >>> np.lib.scimath.logn(2, [-4, -8, 8]) - array([ 2.+4.5324j, 3.+4.5324j, 3.+0.j ]) + array([2.+4.5324j, 3.+4.5324j, 3.+0.j ]) """ x = _fix_real_lt_zero(x) n = _fix_real_lt_zero(n) return nx.log(x)/nx.log(n) + +@array_function_dispatch(_unary_dispatcher) def log2(x): """ Compute the logarithm base 2 of `x`. @@ -383,12 +405,18 @@ def log2(x): >>> np.emath.log2(8) 3.0 >>> np.emath.log2([-4, -8, 8]) - array([ 2.+4.5324j, 3.+4.5324j, 3.+0.j ]) + array([2.+4.5324j, 3.+4.5324j, 3.+0.j ]) """ x = _fix_real_lt_zero(x) return nx.log2(x) + +def _power_dispatcher(x, p): + return (x, p) + + +@array_function_dispatch(_power_dispatcher) def power(x, p): """ Return x to the power p, (x**p). @@ -423,15 +451,17 @@ def power(x, p): >>> np.lib.scimath.power([2, 4], 2) array([ 4, 16]) >>> np.lib.scimath.power([2, 4], -2) - array([ 0.25 , 0.0625]) + array([0.25 , 0.0625]) >>> np.lib.scimath.power([-2, 4], 2) - array([ 4.+0.j, 16.+0.j]) + array([ 4.-0.j, 16.+0.j]) """ x = _fix_real_lt_zero(x) p = _fix_int_lt_zero(p) return nx.power(x, p) + +@array_function_dispatch(_unary_dispatcher) def arccos(x): """ Compute the inverse cosine of x. @@ -469,12 +499,14 @@ def arccos(x): 0.0 >>> np.emath.arccos([1,2]) - array([ 0.-0.j , 0.+1.317j]) + array([0.-0.j , 0.-1.317j]) """ x = _fix_real_abs_gt_1(x) return nx.arccos(x) + +@array_function_dispatch(_unary_dispatcher) def arcsin(x): """ Compute the inverse sine of x. @@ -513,12 +545,14 @@ def arcsin(x): 0.0 >>> np.emath.arcsin([0,1]) - array([ 0. , 1.5708]) + array([0. , 1.5708]) """ x = _fix_real_abs_gt_1(x) return nx.arcsin(x) + +@array_function_dispatch(_unary_dispatcher) def arctanh(x): """ Compute the inverse hyperbolic tangent of `x`. @@ -555,11 +589,14 @@ def arctanh(x): -------- >>> np.set_printoptions(precision=4) - >>> np.emath.arctanh(np.eye(2)) - array([[ Inf, 0.], - [ 0., Inf]]) + >>> from numpy.testing import suppress_warnings + >>> with suppress_warnings() as sup: + ... sup.filter(RuntimeWarning) + ... np.emath.arctanh(np.eye(2)) + array([[inf, 0.], + [ 0., inf]]) >>> np.emath.arctanh([1j]) - array([ 0.+0.7854j]) + array([0.+0.7854j]) """ x = _fix_real_abs_gt_1(x) diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py index 66f534734..e088a6c4a 100644 --- a/numpy/lib/shape_base.py +++ b/numpy/lib/shape_base.py @@ -1,5 +1,6 @@ from __future__ import division, absolute_import, print_function +import functools import warnings import numpy.core.numeric as _nx @@ -8,7 +9,10 @@ from numpy.core.numeric import ( ) from numpy.core.fromnumeric import product, reshape, transpose from numpy.core.multiarray import normalize_axis_index +from numpy.core import overrides from numpy.core import vstack, atleast_3d +from numpy.core.shape_base import ( + _arrays_for_stack_dispatcher, _warn_for_nonsequence) from numpy.lib.index_tricks import ndindex from numpy.matrixlib.defmatrix import matrix # this raises all the right alarm bells @@ -21,6 +25,10 @@ __all__ = [ ] +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + def _make_along_axis_idx(arr_shape, indices, axis): # compute dimensions to iterate over if not _nx.issubdtype(indices.dtype, _nx.integer): @@ -44,6 +52,11 @@ def _make_along_axis_idx(arr_shape, indices, axis): return tuple(fancy_index) +def _take_along_axis_dispatcher(arr, indices, axis): + return (arr, indices) + + +@array_function_dispatch(_take_along_axis_dispatcher) def take_along_axis(arr, indices, axis): """ Take values from the input array by matching 1d index and data slices. @@ -116,7 +129,7 @@ def take_along_axis(arr, indices, axis): [40, 50, 60]]) >>> ai = np.argsort(a, axis=1); ai array([[0, 2, 1], - [1, 2, 0]], dtype=int64) + [1, 2, 0]]) >>> np.take_along_axis(a, ai, axis=1) array([[10, 20, 30], [40, 50, 60]]) @@ -129,7 +142,7 @@ def take_along_axis(arr, indices, axis): >>> ai = np.expand_dims(np.argmax(a, axis=1), axis=1) >>> ai array([[1], - [0], dtype=int64) + [0]]) >>> np.take_along_axis(a, ai, axis=1) array([[30], [60]]) @@ -139,10 +152,10 @@ def take_along_axis(arr, indices, axis): >>> ai_min = np.expand_dims(np.argmin(a, axis=1), axis=1) >>> ai_max = np.expand_dims(np.argmax(a, axis=1), axis=1) - >>> ai = np.concatenate([ai_min, ai_max], axis=axis) - >> ai + >>> ai = np.concatenate([ai_min, ai_max], axis=1) + >>> ai array([[0, 1], - [1, 0]], dtype=int64) + [1, 0]]) >>> np.take_along_axis(a, ai, axis=1) array([[10, 30], [40, 60]]) @@ -160,6 +173,11 @@ def take_along_axis(arr, indices, axis): return arr[_make_along_axis_idx(arr_shape, indices, axis)] +def _put_along_axis_dispatcher(arr, indices, values, axis): + return (arr, indices, values) + + +@array_function_dispatch(_put_along_axis_dispatcher) def put_along_axis(arr, indices, values, axis): """ Put values into the destination array by matching 1d index and data slices. @@ -225,7 +243,7 @@ def put_along_axis(arr, indices, values, axis): >>> ai = np.expand_dims(np.argmax(a, axis=1), axis=1) >>> ai array([[1], - [0]], dtype=int64) + [0]]) >>> np.put_along_axis(a, ai, 99, axis=1) >>> a array([[10, 99, 20], @@ -245,6 +263,11 @@ def put_along_axis(arr, indices, values, axis): arr[_make_along_axis_idx(arr_shape, indices, axis)] = values +def _apply_along_axis_dispatcher(func1d, axis, arr, *args, **kwargs): + return (arr,) + + +@array_function_dispatch(_apply_along_axis_dispatcher) def apply_along_axis(func1d, axis, arr, *args, **kwargs): """ Apply a function to 1-D slices along the given axis. @@ -307,9 +330,9 @@ def apply_along_axis(func1d, axis, arr, *args, **kwargs): ... return (a[0] + a[-1]) * 0.5 >>> b = np.array([[1,2,3], [4,5,6], [7,8,9]]) >>> np.apply_along_axis(my_func, 0, b) - array([ 4., 5., 6.]) + array([4., 5., 6.]) >>> np.apply_along_axis(my_func, 1, b) - array([ 2., 5., 8.]) + array([2., 5., 8.]) For a function that returns a 1D array, the number of dimensions in `outarr` is the same as `arr`. @@ -392,6 +415,11 @@ def apply_along_axis(func1d, axis, arr, *args, **kwargs): return res.__array_wrap__(out_arr) +def _apply_over_axes_dispatcher(func, a, axes): + return (a,) + + +@array_function_dispatch(_apply_over_axes_dispatcher) def apply_over_axes(func, a, axes): """ Apply a function repeatedly over multiple axes. @@ -474,9 +502,15 @@ def apply_over_axes(func, a, axes): val = res else: raise ValueError("function is not returning " - "an array of the correct shape") + "an array of the correct shape") return val + +def _expand_dims_dispatcher(a, axis): + return (a,) + + +@array_function_dispatch(_expand_dims_dispatcher) def expand_dims(a, axis): """ Expand the shape of an array. @@ -554,8 +588,15 @@ def expand_dims(a, axis): # axis = normalize_axis_index(axis, a.ndim + 1) return a.reshape(shape[:axis] + (1,) + shape[axis:]) + row_stack = vstack + +def _column_stack_dispatcher(tup): + return _arrays_for_stack_dispatcher(tup) + + +@array_function_dispatch(_column_stack_dispatcher) def column_stack(tup): """ Stack 1-D arrays as columns into a 2-D array. @@ -589,6 +630,7 @@ def column_stack(tup): [3, 4]]) """ + _warn_for_nonsequence(tup) arrays = [] for v in tup: arr = array(v, copy=False, subok=True) @@ -597,6 +639,12 @@ def column_stack(tup): arrays.append(arr) return _nx.concatenate(arrays, 1) + +def _dstack_dispatcher(tup): + return _arrays_for_stack_dispatcher(tup) + + +@array_function_dispatch(_dstack_dispatcher) def dstack(tup): """ Stack arrays in sequence depth wise (along third axis). @@ -647,8 +695,10 @@ def dstack(tup): [[3, 4]]]) """ + _warn_for_nonsequence(tup) return _nx.concatenate([atleast_3d(_m) for _m in tup], 2) + def _replace_zero_by_x_arrays(sub_arys): for i in range(len(sub_arys)): if _nx.ndim(sub_arys[i]) == 0: @@ -657,6 +707,12 @@ def _replace_zero_by_x_arrays(sub_arys): sub_arys[i] = _nx.empty(0, dtype=sub_arys[i].dtype) return sub_arys + +def _array_split_dispatcher(ary, indices_or_sections, axis=None): + return (ary, indices_or_sections) + + +@array_function_dispatch(_array_split_dispatcher) def array_split(ary, indices_or_sections, axis=0): """ Split an array into multiple sub-arrays. @@ -676,11 +732,11 @@ def array_split(ary, indices_or_sections, axis=0): -------- >>> x = np.arange(8.0) >>> np.array_split(x, 3) - [array([ 0., 1., 2.]), array([ 3., 4., 5.]), array([ 6., 7.])] + [array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7.])] >>> x = np.arange(7.0) >>> np.array_split(x, 3) - [array([ 0., 1., 2.]), array([ 3., 4.]), array([ 5., 6.])] + [array([0., 1., 2.]), array([3., 4.]), array([5., 6.])] """ try: @@ -712,7 +768,12 @@ def array_split(ary, indices_or_sections, axis=0): return sub_arys -def split(ary,indices_or_sections,axis=0): +def _split_dispatcher(ary, indices_or_sections, axis=None): + return (ary, indices_or_sections) + + +@array_function_dispatch(_split_dispatcher) +def split(ary, indices_or_sections, axis=0): """ Split an array into multiple sub-arrays. @@ -767,14 +828,14 @@ def split(ary,indices_or_sections,axis=0): -------- >>> x = np.arange(9.0) >>> np.split(x, 3) - [array([ 0., 1., 2.]), array([ 3., 4., 5.]), array([ 6., 7., 8.])] + [array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7., 8.])] >>> x = np.arange(8.0) >>> np.split(x, [3, 5, 6, 10]) - [array([ 0., 1., 2.]), - array([ 3., 4.]), - array([ 5.]), - array([ 6., 7.]), + [array([0., 1., 2.]), + array([3., 4.]), + array([5.]), + array([6., 7.]), array([], dtype=float64)] """ @@ -789,6 +850,12 @@ def split(ary,indices_or_sections,axis=0): res = array_split(ary, indices_or_sections, axis) return res + +def _hvdsplit_dispatcher(ary, indices_or_sections): + return (ary, indices_or_sections) + + +@array_function_dispatch(_hvdsplit_dispatcher) def hsplit(ary, indices_or_sections): """ Split an array into multiple sub-arrays horizontally (column-wise). @@ -805,43 +872,43 @@ def hsplit(ary, indices_or_sections): -------- >>> x = np.arange(16.0).reshape(4, 4) >>> x - array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]]) + array([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) >>> np.hsplit(x, 2) [array([[ 0., 1.], [ 4., 5.], [ 8., 9.], - [ 12., 13.]]), + [12., 13.]]), array([[ 2., 3.], [ 6., 7.], - [ 10., 11.], - [ 14., 15.]])] + [10., 11.], + [14., 15.]])] >>> np.hsplit(x, np.array([3, 6])) - [array([[ 0., 1., 2.], - [ 4., 5., 6.], - [ 8., 9., 10.], - [ 12., 13., 14.]]), - array([[ 3.], - [ 7.], - [ 11.], - [ 15.]]), - array([], dtype=float64)] + [array([[ 0., 1., 2.], + [ 4., 5., 6.], + [ 8., 9., 10.], + [12., 13., 14.]]), + array([[ 3.], + [ 7.], + [11.], + [15.]]), + array([], shape=(4, 0), dtype=float64)] With a higher dimensional array the split is still along the second axis. >>> x = np.arange(8.0).reshape(2, 2, 2) >>> x - array([[[ 0., 1.], - [ 2., 3.]], - [[ 4., 5.], - [ 6., 7.]]]) + array([[[0., 1.], + [2., 3.]], + [[4., 5.], + [6., 7.]]]) >>> np.hsplit(x, 2) - [array([[[ 0., 1.]], - [[ 4., 5.]]]), - array([[[ 2., 3.]], - [[ 6., 7.]]])] + [array([[[0., 1.]], + [[4., 5.]]]), + array([[[2., 3.]], + [[6., 7.]]])] """ if _nx.ndim(ary) == 0: @@ -851,6 +918,8 @@ def hsplit(ary, indices_or_sections): else: return split(ary, indices_or_sections, 0) + +@array_function_dispatch(_hvdsplit_dispatcher) def vsplit(ary, indices_or_sections): """ Split an array into multiple sub-arrays vertically (row-wise). @@ -867,41 +936,39 @@ def vsplit(ary, indices_or_sections): -------- >>> x = np.arange(16.0).reshape(4, 4) >>> x - array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]]) + array([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) >>> np.vsplit(x, 2) - [array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.]]), - array([[ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]])] + [array([[0., 1., 2., 3.], + [4., 5., 6., 7.]]), array([[ 8., 9., 10., 11.], + [12., 13., 14., 15.]])] >>> np.vsplit(x, np.array([3, 6])) - [array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.]]), - array([[ 12., 13., 14., 15.]]), - array([], dtype=float64)] + [array([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.]]), array([[12., 13., 14., 15.]]), array([], shape=(0, 4), dtype=float64)] With a higher dimensional array the split is still along the first axis. >>> x = np.arange(8.0).reshape(2, 2, 2) >>> x - array([[[ 0., 1.], - [ 2., 3.]], - [[ 4., 5.], - [ 6., 7.]]]) + array([[[0., 1.], + [2., 3.]], + [[4., 5.], + [6., 7.]]]) >>> np.vsplit(x, 2) - [array([[[ 0., 1.], - [ 2., 3.]]]), - array([[[ 4., 5.], - [ 6., 7.]]])] + [array([[[0., 1.], + [2., 3.]]]), array([[[4., 5.], + [6., 7.]]])] """ if _nx.ndim(ary) < 2: raise ValueError('vsplit only works on arrays of 2 or more dimensions') return split(ary, indices_or_sections, 0) + +@array_function_dispatch(_hvdsplit_dispatcher) def dsplit(ary, indices_or_sections): """ Split array into multiple sub-arrays along the 3rd axis (depth). @@ -918,30 +985,28 @@ def dsplit(ary, indices_or_sections): -------- >>> x = np.arange(16.0).reshape(2, 2, 4) >>> x - array([[[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.]], - [[ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]]]) + array([[[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.]], + [[ 8., 9., 10., 11.], + [12., 13., 14., 15.]]]) >>> np.dsplit(x, 2) - [array([[[ 0., 1.], - [ 4., 5.]], - [[ 8., 9.], - [ 12., 13.]]]), - array([[[ 2., 3.], - [ 6., 7.]], - [[ 10., 11.], - [ 14., 15.]]])] + [array([[[ 0., 1.], + [ 4., 5.]], + [[ 8., 9.], + [12., 13.]]]), array([[[ 2., 3.], + [ 6., 7.]], + [[10., 11.], + [14., 15.]]])] >>> np.dsplit(x, np.array([3, 6])) - [array([[[ 0., 1., 2.], - [ 4., 5., 6.]], - [[ 8., 9., 10.], - [ 12., 13., 14.]]]), - array([[[ 3.], - [ 7.]], - [[ 11.], - [ 15.]]]), - array([], dtype=float64)] - + [array([[[ 0., 1., 2.], + [ 4., 5., 6.]], + [[ 8., 9., 10.], + [12., 13., 14.]]]), + array([[[ 3.], + [ 7.]], + [[11.], + [15.]]]), + array([], shape=(2, 2, 0), dtype=float64)] """ if _nx.ndim(ary) < 3: raise ValueError('dsplit only works on arrays of 3 or more dimensions') @@ -971,6 +1036,12 @@ def get_array_wrap(*args): return wrappers[-1][-1] return None + +def _kron_dispatcher(a, b): + return (a, b) + + +@array_function_dispatch(_kron_dispatcher) def kron(a, b): """ Kronecker product of two arrays. @@ -1015,15 +1086,15 @@ def kron(a, b): Examples -------- >>> np.kron([1,10,100], [5,6,7]) - array([ 5, 6, 7, 50, 60, 70, 500, 600, 700]) + array([ 5, 6, 7, ..., 500, 600, 700]) >>> np.kron([5,6,7], [1,10,100]) - array([ 5, 50, 500, 6, 60, 600, 7, 70, 700]) + array([ 5, 50, 500, ..., 7, 70, 700]) >>> np.kron(np.eye(2), np.ones((2,2))) - array([[ 1., 1., 0., 0.], - [ 1., 1., 0., 0.], - [ 0., 0., 1., 1.], - [ 0., 0., 1., 1.]]) + array([[1., 1., 0., 0.], + [1., 1., 0., 0.], + [0., 0., 1., 1.], + [0., 0., 1., 1.]]) >>> a = np.arange(100).reshape((2,5,2,5)) >>> b = np.arange(24).reshape((2,3,4)) @@ -1070,6 +1141,11 @@ def kron(a, b): return result +def _tile_dispatcher(A, reps): + return (A, reps) + + +@array_function_dispatch(_tile_dispatcher) def tile(A, reps): """ Construct an array by repeating A the number of times given by reps. diff --git a/numpy/lib/stride_tricks.py b/numpy/lib/stride_tricks.py index ca13738c1..0dc36e41c 100644 --- a/numpy/lib/stride_tricks.py +++ b/numpy/lib/stride_tricks.py @@ -8,6 +8,7 @@ NumPy reference guide. from __future__ import division, absolute_import, print_function import numpy as np +from numpy.core.overrides import array_function_dispatch __all__ = ['broadcast_to', 'broadcast_arrays'] @@ -135,6 +136,11 @@ def _broadcast_to(array, shape, subok, readonly): return result +def _broadcast_to_dispatcher(array, shape, subok=None): + return (array,) + + +@array_function_dispatch(_broadcast_to_dispatcher, module='numpy') def broadcast_to(array, shape, subok=False): """Broadcast an array to a new shape. @@ -195,6 +201,11 @@ def _broadcast_shape(*args): return b.shape +def _broadcast_arrays_dispatcher(*args, **kwargs): + return args + + +@array_function_dispatch(_broadcast_arrays_dispatcher, module='numpy') def broadcast_arrays(*args, **kwargs): """ Broadcast any number of arrays against each other. diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index 1df8bebf6..8eac16b58 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -361,3 +361,18 @@ class TestOpenFunc(object): fp = datasource.open(local_file) assert_(fp) fp.close() + +def test_del_attr_handling(): + # DataSource __del__ can be called + # even if __init__ fails when the + # Exception object is caught by the + # caller as happens in refguide_check + # is_deprecated() function + + ds = datasource.DataSource() + # simulate failed __init__ by removing key attribute + # produced within __init__ and expected by __del__ + del ds._istmpdest + # should not raise an AttributeError if __del__ + # gracefully handles failed __init__: + ds.__del__() diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index b4888f1bd..e04fdc808 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -1,6 +1,5 @@ from __future__ import division, absolute_import, print_function -import sys import time from datetime import date @@ -246,7 +245,7 @@ class TestStringConverter(object): converter = StringConverter(int, default=0, missing_values="N/A") assert_equal( - converter.missing_values, set(['', 'N/A'])) + converter.missing_values, {'', 'N/A'}) def test_int64_dtype(self): "Check that int64 integer types can be specified" diff --git a/numpy/lib/tests/test_arraypad.py b/numpy/lib/tests/test_arraypad.py index e62fccaa0..20f6e4a1b 100644 --- a/numpy/lib/tests/test_arraypad.py +++ b/numpy/lib/tests/test_arraypad.py @@ -9,6 +9,91 @@ import numpy as np from numpy.testing import (assert_array_equal, assert_raises, assert_allclose, assert_equal) from numpy.lib import pad +from numpy.lib.arraypad import _as_pairs + + +class TestAsPairs(object): + + def test_single_value(self): + """Test casting for a single value.""" + expected = np.array([[3, 3]] * 10) + for x in (3, [3], [[3]]): + result = _as_pairs(x, 10) + assert_equal(result, expected) + # Test with dtype=object + obj = object() + assert_equal( + _as_pairs(obj, 10), + np.array([[obj, obj]] * 10) + ) + + def test_two_values(self): + """Test proper casting for two different values.""" + # Broadcasting in the first dimension with numbers + expected = np.array([[3, 4]] * 10) + for x in ([3, 4], [[3, 4]]): + result = _as_pairs(x, 10) + assert_equal(result, expected) + # and with dtype=object + obj = object() + assert_equal( + _as_pairs(["a", obj], 10), + np.array([["a", obj]] * 10) + ) + + # Broadcasting in the second / last dimension with numbers + assert_equal( + _as_pairs([[3], [4]], 2), + np.array([[3, 3], [4, 4]]) + ) + # and with dtype=object + assert_equal( + _as_pairs([["a"], [obj]], 2), + np.array([["a", "a"], [obj, obj]]) + ) + + def test_with_none(self): + expected = ((None, None), (None, None), (None, None)) + assert_equal( + _as_pairs(None, 3, as_index=False), + expected + ) + assert_equal( + _as_pairs(None, 3, as_index=True), + expected + ) + + def test_pass_through(self): + """Test if `x` already matching desired output are passed through.""" + expected = np.arange(12).reshape((6, 2)) + assert_equal( + _as_pairs(expected, 6), + expected + ) + + def test_as_index(self): + """Test results if `as_index=True`.""" + assert_equal( + _as_pairs([2.6, 3.3], 10, as_index=True), + np.array([[3, 3]] * 10, dtype=np.intp) + ) + assert_equal( + _as_pairs([2.6, 4.49], 10, as_index=True), + np.array([[3, 4]] * 10, dtype=np.intp) + ) + for x in (-3, [-3], [[-3]], [-3, 4], [3, -4], [[-3, 4]], [[4, -3]], + [[1, 2]] * 9 + [[1, -2]]): + with pytest.raises(ValueError, match="negative values"): + _as_pairs(x, 10, as_index=True) + + def test_exceptions(self): + """Ensure faulty usage is discovered.""" + with pytest.raises(ValueError, match="more dimensions than allowed"): + _as_pairs([[[3]]], 10) + with pytest.raises(ValueError, match="could not be broadcast"): + _as_pairs([[1, 2], [3, 4]], 3) + with pytest.raises(ValueError, match="could not be broadcast"): + _as_pairs(np.ones((2, 3)), 3) class TestConditionalShortcuts(object): @@ -535,6 +620,7 @@ class TestConstant(object): assert_array_equal(arr, expected) + class TestLinearRamp(object): def test_check_simple(self): a = np.arange(100).astype('f') diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index fef06ba53..a17fc66e5 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -4,7 +4,6 @@ from __future__ import division, absolute_import, print_function import numpy as np -import sys from numpy.testing import (assert_array_equal, assert_equal, assert_raises, assert_raises_regex) diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 3185e32ac..077507082 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -287,7 +287,6 @@ from io import BytesIO import numpy as np from numpy.testing import ( assert_, assert_array_equal, assert_raises, assert_raises_regex, - raises ) from numpy.lib import format @@ -524,6 +523,30 @@ def test_compressed_roundtrip(): assert_array_equal(arr, arr1) +# aligned +dt1 = np.dtype('i1, i4, i1', align=True) +# non-aligned, explicit offsets +dt2 = np.dtype({'names': ['a', 'b'], 'formats': ['i4', 'i4'], + 'offsets': [1, 6]}) +# nested struct-in-struct +dt3 = np.dtype({'names': ['c', 'd'], 'formats': ['i4', dt2]}) +# field with '' name +dt4 = np.dtype({'names': ['a', '', 'b'], 'formats': ['i4']*3}) +# titles +dt5 = np.dtype({'names': ['a', 'b'], 'formats': ['i4', 'i4'], + 'offsets': [1, 6], 'titles': ['aa', 'bb']}) + +@pytest.mark.parametrize("dt", [dt1, dt2, dt3, dt4, dt5]) +def test_load_padded_dtype(dt): + arr = np.zeros(3, dt) + for i in range(3): + arr[i] = i + 5 + npz_file = os.path.join(tempdir, 'aligned.npz') + np.savez(npz_file, arr=arr) + arr1 = np.load(npz_file)['arr'] + assert_array_equal(arr, arr1) + + def test_python2_python3_interoperability(): if sys.version_info[0] >= 3: fname = 'win64python2.npy' @@ -533,7 +556,6 @@ def test_python2_python3_interoperability(): data = np.load(path) assert_array_equal(data, np.ones(2)) - def test_pickle_python2_python3(): # Test that loading object arrays saved on Python 2 works both on # Python 2 and Python 3 and vice versa diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 40cca1dbb..3d4b0e3b2 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -11,17 +11,15 @@ from numpy import ma from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_raises, assert_allclose, - assert_array_max_ulp, assert_warns, assert_raises_regex, suppress_warnings, - HAS_REFCOUNT, + assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT, ) import numpy.lib.function_base as nfb from numpy.random import rand from numpy.lib import ( add_newdoc_ufunc, angle, average, bartlett, blackman, corrcoef, cov, delete, diff, digitize, extract, flipud, gradient, hamming, hanning, - histogram, histogramdd, i0, insert, interp, kaiser, meshgrid, msort, - piecewise, place, rot90, select, setxor1d, sinc, split, trapz, trim_zeros, - unwrap, unique, vectorize + i0, insert, interp, kaiser, meshgrid, msort, piecewise, place, rot90, + select, setxor1d, sinc, trapz, trim_zeros, unwrap, unique, vectorize ) from numpy.compat import long @@ -3114,3 +3112,29 @@ class TestAdd_newdoc(object): assert_equal(np.core.flatiter.index.__doc__[:len(tgt)], tgt) assert_(len(np.core.ufunc.identity.__doc__) > 300) assert_(len(np.lib.index_tricks.mgrid.__doc__) > 300) + +class TestSortComplex(object): + + @pytest.mark.parametrize("type_in, type_out", [ + ('l', 'D'), + ('h', 'F'), + ('H', 'F'), + ('b', 'F'), + ('B', 'F'), + ('g', 'G'), + ]) + def test_sort_real(self, type_in, type_out): + # sort_complex() type casting for real input types + a = np.array([5, 3, 6, 2, 1], dtype=type_in) + actual = np.sort_complex(a) + expected = np.sort(a).astype(type_out) + assert_equal(actual, expected) + assert_equal(actual.dtype, expected.dtype) + + def test_sort_complex(self): + # sort_complex() handling of complex input + a = np.array([2 + 3j, 1 - 2j, 1 - 3j, 2 + 1j], dtype='D') + expected = np.array([1 - 3j, 1 - 2j, 2 + 1j, 2 + 3j], dtype='D') + actual = np.sort_complex(a) + assert_equal(actual, expected) + assert_equal(actual.dtype, expected.dtype) diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py index 1b5a71d0e..c96b01d42 100644 --- a/numpy/lib/tests/test_histograms.py +++ b/numpy/lib/tests/test_histograms.py @@ -6,7 +6,7 @@ from numpy.lib.histograms import histogram, histogramdd, histogram_bin_edges from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_raises, assert_allclose, - assert_array_max_ulp, assert_warns, assert_raises_regex, suppress_warnings, + assert_array_max_ulp, assert_raises_regex, suppress_warnings, ) @@ -289,13 +289,13 @@ class TestHistogram(object): def test_object_array_of_0d(self): # gh-7864 assert_raises(ValueError, - histogram, [np.array([0.4]) for i in range(10)] + [-np.inf]) + histogram, [np.array(0.4) for i in range(10)] + [-np.inf]) assert_raises(ValueError, - histogram, [np.array([0.4]) for i in range(10)] + [np.inf]) + histogram, [np.array(0.4) for i in range(10)] + [np.inf]) # these should not crash - np.histogram([np.array([0.5]) for i in range(10)] + [.500000000000001]) - np.histogram([np.array([0.5]) for i in range(10)] + [.5]) + np.histogram([np.array(0.5) for i in range(10)] + [.500000000000001]) + np.histogram([np.array(0.5) for i in range(10)] + [.5]) def test_some_nan_values(self): # gh-7503 @@ -431,7 +431,7 @@ class TestHistogramOptimBinNums(object): def test_empty(self): estimator_list = ['fd', 'scott', 'rice', 'sturges', - 'doane', 'sqrt', 'auto'] + 'doane', 'sqrt', 'auto', 'stone'] # check it can deal with empty data for estimator in estimator_list: a, b = histogram([], bins=estimator) @@ -447,11 +447,11 @@ class TestHistogramOptimBinNums(object): # Some basic sanity checking, with some fixed data. # Checking for the correct number of bins basic_test = {50: {'fd': 4, 'scott': 4, 'rice': 8, 'sturges': 7, - 'doane': 8, 'sqrt': 8, 'auto': 7}, + 'doane': 8, 'sqrt': 8, 'auto': 7, 'stone': 2}, 500: {'fd': 8, 'scott': 8, 'rice': 16, 'sturges': 10, - 'doane': 12, 'sqrt': 23, 'auto': 10}, + 'doane': 12, 'sqrt': 23, 'auto': 10, 'stone': 9}, 5000: {'fd': 17, 'scott': 17, 'rice': 35, 'sturges': 14, - 'doane': 17, 'sqrt': 71, 'auto': 17}} + 'doane': 17, 'sqrt': 71, 'auto': 17, 'stone': 20}} for testlen, expectedResults in basic_test.items(): # Create some sort of non uniform data to test with @@ -471,11 +471,11 @@ class TestHistogramOptimBinNums(object): precalculated. """ small_dat = {1: {'fd': 1, 'scott': 1, 'rice': 1, 'sturges': 1, - 'doane': 1, 'sqrt': 1}, + 'doane': 1, 'sqrt': 1, 'stone': 1}, 2: {'fd': 2, 'scott': 1, 'rice': 3, 'sturges': 2, - 'doane': 1, 'sqrt': 2}, + 'doane': 1, 'sqrt': 2, 'stone': 1}, 3: {'fd': 2, 'scott': 2, 'rice': 3, 'sturges': 3, - 'doane': 3, 'sqrt': 2}} + 'doane': 3, 'sqrt': 2, 'stone': 1}} for testlen, expectedResults in small_dat.items(): testdat = np.arange(testlen) @@ -499,7 +499,7 @@ class TestHistogramOptimBinNums(object): """ novar_dataset = np.ones(100) novar_resultdict = {'fd': 1, 'scott': 1, 'rice': 1, 'sturges': 1, - 'doane': 1, 'sqrt': 1, 'auto': 1} + 'doane': 1, 'sqrt': 1, 'auto': 1, 'stone': 1} for estimator, numbins in novar_resultdict.items(): a, b = np.histogram(novar_dataset, estimator) @@ -538,12 +538,32 @@ class TestHistogramOptimBinNums(object): xcenter = np.linspace(-10, 10, 50) outlier_dataset = np.hstack((np.linspace(-110, -100, 5), xcenter)) - outlier_resultdict = {'fd': 21, 'scott': 5, 'doane': 11} + outlier_resultdict = {'fd': 21, 'scott': 5, 'doane': 11, 'stone': 6} for estimator, numbins in outlier_resultdict.items(): a, b = np.histogram(outlier_dataset, estimator) assert_equal(len(a), numbins) + def test_scott_vs_stone(self): + """Verify that Scott's rule and Stone's rule converges for normally distributed data""" + + def nbins_ratio(seed, size): + rng = np.random.RandomState(seed) + x = rng.normal(loc=0, scale=2, size=size) + a, b = len(np.histogram(x, 'stone')[0]), len(np.histogram(x, 'scott')[0]) + return a / (a + b) + + ll = [[nbins_ratio(seed, size) for size in np.geomspace(start=10, stop=100, num=4).round().astype(int)] + for seed in range(256)] + + # the average difference between the two methods decreases as the dataset size increases. + assert_almost_equal(abs(np.mean(ll, axis=0) - 0.5), + [0.1065248, + 0.0968844, + 0.0331818, + 0.0178057], + decimal=3) + def test_simple_range(self): """ Straightforward testing with a mixture of linspace data (for @@ -555,11 +575,11 @@ class TestHistogramOptimBinNums(object): # Checking for the correct number of bins basic_test = { 50: {'fd': 8, 'scott': 8, 'rice': 15, - 'sturges': 14, 'auto': 14}, + 'sturges': 14, 'auto': 14, 'stone': 8}, 500: {'fd': 15, 'scott': 16, 'rice': 32, - 'sturges': 20, 'auto': 20}, + 'sturges': 20, 'auto': 20, 'stone': 80}, 5000: {'fd': 33, 'scott': 33, 'rice': 69, - 'sturges': 27, 'auto': 33} + 'sturges': 27, 'auto': 33, 'stone': 80} } for testlen, expectedResults in basic_test.items(): @@ -794,3 +814,20 @@ class TestHistogramdd(object): hist_dd, edges_dd = histogramdd((v,), (bins,), density=True) assert_equal(hist, hist_dd) assert_equal(edges, edges_dd[0]) + + def test_density_via_normed(self): + # normed should simply alias to density argument + v = np.arange(10) + bins = np.array([0, 1, 3, 6, 10]) + hist, edges = histogram(v, bins, density=True) + hist_dd, edges_dd = histogramdd((v,), (bins,), normed=True) + assert_equal(hist, hist_dd) + assert_equal(edges, edges_dd[0]) + + def test_density_normed_redundancy(self): + v = np.arange(10) + bins = np.array([0, 1, 3, 6, 10]) + with assert_raises_regex(TypeError, "Cannot specify both"): + hist_dd, edges_dd = histogramdd((v,), (bins,), + density=True, + normed=True) diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py index 76d9b403e..3246f68ff 100644 --- a/numpy/lib/tests/test_index_tricks.py +++ b/numpy/lib/tests/test_index_tricks.py @@ -226,6 +226,11 @@ class TestConcatenator(object): g = r_[-10.1, np.array([1]), np.array([2, 3, 4]), 10.0] assert_(g.dtype == 'f8') + def test_complex_step(self): + # Regression test for #12262 + g = r_[0:36:100j] + assert_(g.shape == (100,)) + def test_2d(self): b = np.random.rand(5, 5) c = np.random.rand(5, 5) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 08800ff97..7ef25538b 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -6,7 +6,6 @@ import os import threading import time import warnings -import gc import io import re import pytest @@ -18,7 +17,7 @@ import locale import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConversionWarning -from numpy.compat import asbytes, bytes, unicode, Path +from numpy.compat import asbytes, bytes, Path from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, @@ -355,6 +354,16 @@ class TestSaveTxt(object): c.seek(0) assert_equal(c.readlines(), [b'1 2\n', b'3 4\n']) + @pytest.mark.skipif(Path is None, reason="No pathlib.Path") + def test_multifield_view(self): + a = np.ones(1, dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'f4')]) + v = a[['x', 'z']] + with temppath(suffix='.npy') as path: + path = Path(path) + np.save(path, v) + data = np.load(path) + assert_array_equal(data, v) + def test_delimiter(self): a = np.array([[1., 2.], [3., 4.]]) c = BytesIO() @@ -2049,7 +2058,6 @@ M 33 21.99 def test_utf8_file(self): utf8 = b"\xcf\x96" - latin1 = b"\xf6\xfc\xf6" with temppath() as path: with open(path, "wb") as f: f.write((b"test1,testNonethe" + utf8 + b",test3\n") * 2) @@ -2295,11 +2303,35 @@ class TestPathUsage(object): assert_array_equal(x, a) def test_save_load(self): - # Test that pathlib.Path instances can be used with savez. + # Test that pathlib.Path instances can be used with save. + with temppath(suffix='.npy') as path: + path = Path(path) + a = np.array([[1, 2], [3, 4]], int) + np.save(path, a) + data = np.load(path) + assert_array_equal(data, a) + + def test_save_load_memmap(self): + # Test that pathlib.Path instances can be loaded mem-mapped. + with temppath(suffix='.npy') as path: + path = Path(path) + a = np.array([[1, 2], [3, 4]], int) + np.save(path, a) + data = np.load(path, mmap_mode='r') + assert_array_equal(data, a) + # close the mem-mapped file + del data + + def test_save_load_memmap_readwrite(self): + # Test that pathlib.Path instances can be written mem-mapped. with temppath(suffix='.npy') as path: path = Path(path) a = np.array([[1, 2], [3, 4]], int) np.save(path, a) + b = np.load(path, mmap_mode='r+') + a[0][0] = 5 + b[0][0] = 5 + del b # closes the file data = np.load(path) assert_array_equal(data, a) diff --git a/numpy/lib/tests/test_mixins.py b/numpy/lib/tests/test_mixins.py index f2d915502..3dd5346b6 100644 --- a/numpy/lib/tests/test_mixins.py +++ b/numpy/lib/tests/test_mixins.py @@ -199,6 +199,17 @@ class TestNDArrayOperatorsMixin(object): err_msg = 'failed for operator {}'.format(op) _assert_equal_type_and_value(expected, actual, err_msg=err_msg) + def test_matmul(self): + array = np.array([1, 2], dtype=np.float64) + array_like = ArrayLike(array) + expected = ArrayLike(np.float64(5)) + _assert_equal_type_and_value(expected, np.matmul(array_like, array)) + if not PY2: + _assert_equal_type_and_value( + expected, operator.matmul(array_like, array)) + _assert_equal_type_and_value( + expected, operator.matmul(array, array_like)) + def test_ufunc_at(self): array = ArrayLike(np.array([1, 2, 3, 4])) assert_(np.negative.at(array, np.array([0, 1])) is None) diff --git a/numpy/lib/tests/test_polynomial.py b/numpy/lib/tests/test_polynomial.py index 9f7c117a2..77414ba7c 100644 --- a/numpy/lib/tests/test_polynomial.py +++ b/numpy/lib/tests/test_polynomial.py @@ -3,7 +3,7 @@ from __future__ import division, absolute_import, print_function import numpy as np from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, - assert_array_almost_equal, assert_raises + assert_array_almost_equal, assert_raises, assert_allclose ) @@ -122,27 +122,34 @@ class TestPolynomial(object): weights = np.arange(8, 1, -1)**2/7.0 # Check exception when too few points for variance estimate. Note that - # the Bayesian estimate requires the number of data points to exceed - # degree + 3. + # the estimate requires the number of data points to exceed + # degree + 1 assert_raises(ValueError, np.polyfit, - [0, 1, 3], [0, 1, 3], deg=0, cov=True) + [1], [1], deg=0, cov=True) # check 1D case m, cov = np.polyfit(x, y+err, 2, cov=True) est = [3.8571, 0.2857, 1.619] assert_almost_equal(est, m, decimal=4) - val0 = [[2.9388, -5.8776, 1.6327], - [-5.8776, 12.7347, -4.2449], - [1.6327, -4.2449, 2.3220]] + val0 = [[ 1.4694, -2.9388, 0.8163], + [-2.9388, 6.3673, -2.1224], + [ 0.8163, -2.1224, 1.161 ]] assert_almost_equal(val0, cov, decimal=4) m2, cov2 = np.polyfit(x, y+err, 2, w=weights, cov=True) assert_almost_equal([4.8927, -1.0177, 1.7768], m2, decimal=4) - val = [[8.7929, -10.0103, 0.9756], - [-10.0103, 13.6134, -1.8178], - [0.9756, -1.8178, 0.6674]] + val = [[ 4.3964, -5.0052, 0.4878], + [-5.0052, 6.8067, -0.9089], + [ 0.4878, -0.9089, 0.3337]] assert_almost_equal(val, cov2, decimal=4) + m3, cov3 = np.polyfit(x, y+err, 2, w=weights, cov="unscaled") + assert_almost_equal([4.8927, -1.0177, 1.7768], m3, decimal=4) + val = [[ 0.1473, -0.1677, 0.0163], + [-0.1677, 0.228 , -0.0304], + [ 0.0163, -0.0304, 0.0112]] + assert_almost_equal(val, cov3, decimal=4) + # check 2D (n,1) case y = y[:, np.newaxis] c = c[:, np.newaxis] @@ -158,6 +165,29 @@ class TestPolynomial(object): assert_almost_equal(val0, cov[:, :, 0], decimal=4) assert_almost_equal(val0, cov[:, :, 1], decimal=4) + # check order 1 (deg=0) case, were the analytic results are simple + np.random.seed(123) + y = np.random.normal(size=(4, 10000)) + mean, cov = np.polyfit(np.zeros(y.shape[0]), y, deg=0, cov=True) + # Should get sigma_mean = sigma/sqrt(N) = 1./sqrt(4) = 0.5. + assert_allclose(mean.std(), 0.5, atol=0.01) + assert_allclose(np.sqrt(cov.mean()), 0.5, atol=0.01) + # Without scaling, since reduced chi2 is 1, the result should be the same. + mean, cov = np.polyfit(np.zeros(y.shape[0]), y, w=np.ones(y.shape[0]), + deg=0, cov="unscaled") + assert_allclose(mean.std(), 0.5, atol=0.01) + assert_almost_equal(np.sqrt(cov.mean()), 0.5) + # If we estimate our errors wrong, no change with scaling: + w = np.full(y.shape[0], 1./0.5) + mean, cov = np.polyfit(np.zeros(y.shape[0]), y, w=w, deg=0, cov=True) + assert_allclose(mean.std(), 0.5, atol=0.01) + assert_allclose(np.sqrt(cov.mean()), 0.5, atol=0.01) + # But if we do not scale, our estimate for the error in the mean will + # differ. + mean, cov = np.polyfit(np.zeros(y.shape[0]), y, w=w, deg=0, cov="unscaled") + assert_allclose(mean.std(), 0.5, atol=0.01) + assert_almost_equal(np.sqrt(cov.mean()), 0.25) + def test_objects(self): from decimal import Decimal p = np.poly1d([Decimal('4.0'), Decimal('3.0'), Decimal('2.0')]) diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 5585a95f9..11f8a5afa 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -10,7 +10,8 @@ from numpy.testing import assert_, assert_raises from numpy.lib.recfunctions import ( drop_fields, rename_fields, get_fieldstructure, recursive_fill_fields, find_duplicates, merge_arrays, append_fields, stack_arrays, join_by, - repack_fields) + repack_fields, unstructured_to_structured, structured_to_unstructured, + apply_along_fields, require_fields, assign_fields_by_name) get_names = np.lib.recfunctions.get_names get_names_flat = np.lib.recfunctions.get_names_flat zip_descr = np.lib.recfunctions.zip_descr @@ -204,6 +205,77 @@ class TestRecFunctions(object): dt = np.dtype((np.record, dt)) assert_(repack_fields(dt).type is np.record) + def test_structured_to_unstructured(self): + a = np.zeros(4, dtype=[('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + out = structured_to_unstructured(a) + assert_equal(out, np.zeros((4,5), dtype='f8')) + + b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + out = np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) + assert_equal(out, np.array([ 3. , 5.5, 9. , 11. ])) + + c = np.arange(20).reshape((4,5)) + out = unstructured_to_structured(c, a.dtype) + want = np.array([( 0, ( 1., 2), [ 3., 4.]), + ( 5, ( 6., 7), [ 8., 9.]), + (10, (11., 12), [13., 14.]), + (15, (16., 17), [18., 19.])], + dtype=[('a', '<i4'), + ('b', [('f0', '<f4'), ('f1', '<u2')]), + ('c', '<f4', (2,))]) + assert_equal(out, want) + + d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + assert_equal(apply_along_fields(np.mean, d), + np.array([ 8.0/3, 16.0/3, 26.0/3, 11. ])) + assert_equal(apply_along_fields(np.mean, d[['x', 'z']]), + np.array([ 3. , 5.5, 9. , 11. ])) + + # check that for uniform field dtypes we get a view, not a copy: + d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'i4')]) + dd = structured_to_unstructured(d) + ddd = unstructured_to_structured(dd, d.dtype) + assert_(dd.base is d) + assert_(ddd.base is d) + + # test that nested fields with identical names don't break anything + point = np.dtype([('x', int), ('y', int)]) + triangle = np.dtype([('a', point), ('b', point), ('c', point)]) + arr = np.zeros(10, triangle) + res = structured_to_unstructured(arr, dtype=int) + assert_equal(res, np.zeros((10, 6), dtype=int)) + + + def test_field_assignment_by_name(self): + a = np.ones(2, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) + newdt = [('b', 'f4'), ('c', 'u1')] + + assert_equal(require_fields(a, newdt), np.ones(2, newdt)) + + b = np.array([(1,2), (3,4)], dtype=newdt) + assign_fields_by_name(a, b, zero_unassigned=False) + assert_equal(a, np.array([(1,1,2),(1,3,4)], dtype=a.dtype)) + assign_fields_by_name(a, b) + assert_equal(a, np.array([(0,1,2),(0,3,4)], dtype=a.dtype)) + + # test nested fields + a = np.ones(2, dtype=[('a', [('b', 'f8'), ('c', 'u1')])]) + newdt = [('a', [('c', 'u1')])] + assert_equal(require_fields(a, newdt), np.ones(2, newdt)) + b = np.array([((2,),), ((3,),)], dtype=newdt) + assign_fields_by_name(a, b, zero_unassigned=False) + assert_equal(a, np.array([((1,2),), ((1,3),)], dtype=a.dtype)) + assign_fields_by_name(a, b) + assert_equal(a, np.array([((0,2),), ((0,3),)], dtype=a.dtype)) + + # test unstructured code path for 0d arrays + a, b = np.array(3), np.array(0) + assign_fields_by_name(b, a) + assert_equal(b[()], 3) + class TestRecursiveFillFields(object): # Test recursive_fill_fields. diff --git a/numpy/lib/tests/test_shape_base.py b/numpy/lib/tests/test_shape_base.py index a7f5ca7db..01ea028bb 100644 --- a/numpy/lib/tests/test_shape_base.py +++ b/numpy/lib/tests/test_shape_base.py @@ -260,8 +260,8 @@ class TestApplyAlongAxis(object): def test_with_iterable_object(self): # from issue 5248 d = np.array([ - [set([1, 11]), set([2, 22]), set([3, 33])], - [set([4, 44]), set([5, 55]), set([6, 66])] + [{1, 11}, {2, 22}, {3, 33}], + [{4, 44}, {5, 55}, {6, 66}] ]) actual = np.apply_along_axis(lambda a: set.union(*a), 0, d) expected = np.array([{1, 11, 4, 44}, {2, 22, 5, 55}, {3, 33, 6, 66}]) @@ -457,6 +457,7 @@ class TestSplit(object): a = np.arange(10) assert_raises(ValueError, split, a, 3) + class TestColumnStack(object): def test_non_iterable(self): assert_raises(TypeError, column_stack, 1) @@ -481,6 +482,10 @@ class TestColumnStack(object): actual = np.column_stack((a, b)) assert_equal(actual, expected) + def test_generator(self): + with assert_warns(FutureWarning): + column_stack((np.arange(3) for _ in range(2))) + class TestDstack(object): def test_non_iterable(self): @@ -514,6 +519,10 @@ class TestDstack(object): desired = np.array([[[1, 1], [2, 2]]]) assert_array_equal(res, desired) + def test_generator(self): + with assert_warns(FutureWarning): + dstack((np.arange(3) for _ in range(2))) + # array_split has more comprehensive test of splitting. # only do simple test on hsplit, vsplit, and dsplit diff --git a/numpy/lib/twodim_base.py b/numpy/lib/twodim_base.py index 98efba191..e165c9b02 100644 --- a/numpy/lib/twodim_base.py +++ b/numpy/lib/twodim_base.py @@ -3,11 +3,15 @@ """ from __future__ import division, absolute_import, print_function +import functools + from numpy.core.numeric import ( absolute, asanyarray, arange, zeros, greater_equal, multiply, ones, asarray, where, int8, int16, int32, int64, empty, promote_types, diagonal, nonzero ) +from numpy.core.overrides import set_module +from numpy.core import overrides from numpy.core import iinfo, transpose @@ -17,6 +21,10 @@ __all__ = [ 'tril_indices_from', 'triu_indices', 'triu_indices_from', ] +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + i1 = iinfo(int8) i2 = iinfo(int16) i4 = iinfo(int32) @@ -33,6 +41,11 @@ def _min_int(low, high): return int64 +def _flip_dispatcher(m): + return (m,) + + +@array_function_dispatch(_flip_dispatcher) def fliplr(m): """ Flip array in the left/right direction. @@ -64,13 +77,13 @@ def fliplr(m): -------- >>> A = np.diag([1.,2.,3.]) >>> A - array([[ 1., 0., 0.], - [ 0., 2., 0.], - [ 0., 0., 3.]]) + array([[1., 0., 0.], + [0., 2., 0.], + [0., 0., 3.]]) >>> np.fliplr(A) - array([[ 0., 0., 1.], - [ 0., 2., 0.], - [ 3., 0., 0.]]) + array([[0., 0., 1.], + [0., 2., 0.], + [3., 0., 0.]]) >>> A = np.random.randn(2,3,5) >>> np.all(np.fliplr(A) == A[:,::-1,...]) @@ -83,6 +96,7 @@ def fliplr(m): return m[:, ::-1] +@array_function_dispatch(_flip_dispatcher) def flipud(m): """ Flip array in the up/down direction. @@ -115,13 +129,13 @@ def flipud(m): -------- >>> A = np.diag([1.0, 2, 3]) >>> A - array([[ 1., 0., 0.], - [ 0., 2., 0.], - [ 0., 0., 3.]]) + array([[1., 0., 0.], + [0., 2., 0.], + [0., 0., 3.]]) >>> np.flipud(A) - array([[ 0., 0., 3.], - [ 0., 2., 0.], - [ 1., 0., 0.]]) + array([[0., 0., 3.], + [0., 2., 0.], + [1., 0., 0.]]) >>> A = np.random.randn(2,3,5) >>> np.all(np.flipud(A) == A[::-1,...]) @@ -137,6 +151,7 @@ def flipud(m): return m[::-1, ...] +@set_module('numpy') def eye(N, M=None, k=0, dtype=float, order='C'): """ Return a 2-D array with ones on the diagonal and zeros elsewhere. @@ -176,9 +191,9 @@ def eye(N, M=None, k=0, dtype=float, order='C'): array([[1, 0], [0, 1]]) >>> np.eye(3, k=1) - array([[ 0., 1., 0.], - [ 0., 0., 1.], - [ 0., 0., 0.]]) + array([[0., 1., 0.], + [0., 0., 1.], + [0., 0., 0.]]) """ if M is None: @@ -194,6 +209,11 @@ def eye(N, M=None, k=0, dtype=float, order='C'): return m +def _diag_dispatcher(v, k=None): + return (v,) + + +@array_function_dispatch(_diag_dispatcher) def diag(v, k=0): """ Extract a diagonal or construct a diagonal array. @@ -265,6 +285,7 @@ def diag(v, k=0): raise ValueError("Input must be 1- or 2-d.") +@array_function_dispatch(_diag_dispatcher) def diagflat(v, k=0): """ Create a two-dimensional array with the flattened input as a diagonal. @@ -324,6 +345,7 @@ def diagflat(v, k=0): return wrap(res) +@set_module('numpy') def tri(N, M=None, k=0, dtype=float): """ An array with ones at and below the given diagonal and zeros elsewhere. @@ -356,9 +378,9 @@ def tri(N, M=None, k=0, dtype=float): [1, 1, 1, 1, 1]]) >>> np.tri(3, 5, -1) - array([[ 0., 0., 0., 0., 0.], - [ 1., 0., 0., 0., 0.], - [ 1., 1., 0., 0., 0.]]) + array([[0., 0., 0., 0., 0.], + [1., 0., 0., 0., 0.], + [1., 1., 0., 0., 0.]]) """ if M is None: @@ -373,6 +395,11 @@ def tri(N, M=None, k=0, dtype=float): return m +def _trilu_dispatcher(m, k=None): + return (m,) + + +@array_function_dispatch(_trilu_dispatcher) def tril(m, k=0): """ Lower triangle of an array. @@ -411,6 +438,7 @@ def tril(m, k=0): return where(mask, m, zeros(1, m.dtype)) +@array_function_dispatch(_trilu_dispatcher) def triu(m, k=0): """ Upper triangle of an array. @@ -439,7 +467,12 @@ def triu(m, k=0): return where(mask, zeros(1, m.dtype), m) +def _vander_dispatcher(x, N=None, increasing=None): + return (x,) + + # Originally borrowed from John Hunter and matplotlib +@array_function_dispatch(_vander_dispatcher) def vander(x, N=None, increasing=False): """ Generate a Vandermonde matrix. @@ -507,7 +540,7 @@ def vander(x, N=None, increasing=False): of the differences between the values of the input vector: >>> np.linalg.det(np.vander(x)) - 48.000000000000043 + 48.000000000000043 # may vary >>> (5-3)*(5-2)*(5-1)*(3-2)*(3-1)*(2-1) 48 @@ -530,6 +563,12 @@ def vander(x, N=None, increasing=False): return v +def _histogram2d_dispatcher(x, y, bins=None, range=None, normed=None, + weights=None, density=None): + return (x, y, bins, weights) + + +@array_function_dispatch(_histogram2d_dispatcher) def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, density=None): """ @@ -605,7 +644,7 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, Examples -------- - >>> import matplotlib as mpl + >>> from matplotlib.image import NonUniformImage >>> import matplotlib.pyplot as plt Construct a 2-D histogram with variable bin width. First define the bin @@ -627,6 +666,7 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, >>> ax = fig.add_subplot(131, title='imshow: square bins') >>> plt.imshow(H, interpolation='nearest', origin='low', ... extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]]) + <matplotlib.image.AxesImage object at 0x...> :func:`pcolormesh <matplotlib.pyplot.pcolormesh>` can display actual edges: @@ -634,13 +674,14 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, ... aspect='equal') >>> X, Y = np.meshgrid(xedges, yedges) >>> ax.pcolormesh(X, Y, H) + <matplotlib.collections.QuadMesh object at 0x...> :class:`NonUniformImage <matplotlib.image.NonUniformImage>` can be used to display actual bin edges with interpolation: >>> ax = fig.add_subplot(133, title='NonUniformImage: interpolated', ... aspect='equal', xlim=xedges[[0, -1]], ylim=yedges[[0, -1]]) - >>> im = mpl.image.NonUniformImage(ax, interpolation='bilinear') + >>> im = NonUniformImage(ax, interpolation='bilinear') >>> xcenters = (xedges[:-1] + xedges[1:]) / 2 >>> ycenters = (yedges[:-1] + yedges[1:]) / 2 >>> im.set_data(xcenters, ycenters, H) @@ -662,6 +703,7 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, return hist, edges[0], edges[1] +@set_module('numpy') def mask_indices(n, mask_func, k=0): """ Return the indices to access (n, n) arrays, given a masking function. @@ -732,6 +774,7 @@ def mask_indices(n, mask_func, k=0): return nonzero(a != 0) +@set_module('numpy') def tril_indices(n, k=0, m=None): """ Return the indices for the lower-triangle of an (n, m) array. @@ -788,7 +831,7 @@ def tril_indices(n, k=0, m=None): Both for indexing: >>> a[il1] - array([ 0, 4, 5, 8, 9, 10, 12, 13, 14, 15]) + array([ 0, 4, 5, ..., 13, 14, 15]) And for assigning values: @@ -812,6 +855,11 @@ def tril_indices(n, k=0, m=None): return nonzero(tri(n, m, k=k, dtype=bool)) +def _trilu_indices_form_dispatcher(arr, k=None): + return (arr,) + + +@array_function_dispatch(_trilu_indices_form_dispatcher) def tril_indices_from(arr, k=0): """ Return the indices for the lower-triangle of arr. @@ -840,6 +888,7 @@ def tril_indices_from(arr, k=0): return tril_indices(arr.shape[-2], k=k, m=arr.shape[-1]) +@set_module('numpy') def triu_indices(n, k=0, m=None): """ Return the indices for the upper-triangle of an (n, m) array. @@ -897,7 +946,7 @@ def triu_indices(n, k=0, m=None): Both for indexing: >>> a[iu1] - array([ 0, 1, 2, 3, 5, 6, 7, 10, 11, 15]) + array([ 0, 1, 2, ..., 10, 11, 15]) And for assigning values: @@ -922,6 +971,7 @@ def triu_indices(n, k=0, m=None): return nonzero(~tri(n, m, k=k-1, dtype=bool)) +@array_function_dispatch(_trilu_indices_form_dispatcher) def triu_indices_from(arr, k=0): """ Return the indices for the upper-triangle of arr. diff --git a/numpy/lib/type_check.py b/numpy/lib/type_check.py index 603da8567..f55517732 100644 --- a/numpy/lib/type_check.py +++ b/numpy/lib/type_check.py @@ -2,6 +2,7 @@ """ from __future__ import division, absolute_import, print_function +import functools import warnings __all__ = ['iscomplexobj', 'isrealobj', 'imag', 'iscomplex', @@ -10,12 +11,21 @@ __all__ = ['iscomplexobj', 'isrealobj', 'imag', 'iscomplex', 'common_type'] import numpy.core.numeric as _nx -from numpy.core.numeric import asarray, asanyarray, array, isnan, zeros +from numpy.core.numeric import asarray, asanyarray, isnan, zeros +from numpy.core.overrides import set_module +from numpy.core import overrides from .ufunclike import isneginf, isposinf + +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + _typecodes_by_elsize = 'GDFgdfQqLlIiHhBb?' -def mintypecode(typechars,typeset='GDFgdf',default='d'): + +@set_module('numpy') +def mintypecode(typechars, typeset='GDFgdf', default='d'): """ Return the character for the minimum-size type to which given types can be safely cast. @@ -65,13 +75,16 @@ def mintypecode(typechars,typeset='GDFgdf',default='d'): return default if 'F' in intersection and 'd' in intersection: return 'D' - l = [] - for t in intersection: - i = _typecodes_by_elsize.index(t) - l.append((i, t)) + l = [(_typecodes_by_elsize.index(t), t) for t in intersection] l.sort() return l[0][1] + +def _asfarray_dispatcher(a, dtype=None): + return (a,) + + +@array_function_dispatch(_asfarray_dispatcher) def asfarray(a, dtype=_nx.float_): """ Return an array converted to a float type. @@ -92,11 +105,11 @@ def asfarray(a, dtype=_nx.float_): Examples -------- >>> np.asfarray([2, 3]) - array([ 2., 3.]) + array([2., 3.]) >>> np.asfarray([2, 3], dtype='float') - array([ 2., 3.]) + array([2., 3.]) >>> np.asfarray([2, 3], dtype='int8') - array([ 2., 3.]) + array([2., 3.]) """ if not _nx.issubdtype(dtype, _nx.inexact): @@ -104,6 +117,11 @@ def asfarray(a, dtype=_nx.float_): return asarray(a, dtype=dtype) +def _real_dispatcher(val): + return (val,) + + +@array_function_dispatch(_real_dispatcher) def real(val): """ Return the real part of the complex argument. @@ -128,13 +146,13 @@ def real(val): -------- >>> a = np.array([1+2j, 3+4j, 5+6j]) >>> a.real - array([ 1., 3., 5.]) + array([1., 3., 5.]) >>> a.real = 9 >>> a - array([ 9.+2.j, 9.+4.j, 9.+6.j]) + array([9.+2.j, 9.+4.j, 9.+6.j]) >>> a.real = np.array([9, 8, 7]) >>> a - array([ 9.+2.j, 8.+4.j, 7.+6.j]) + array([9.+2.j, 8.+4.j, 7.+6.j]) >>> np.real(1 + 1j) 1.0 @@ -145,6 +163,11 @@ def real(val): return asanyarray(val).real +def _imag_dispatcher(val): + return (val,) + + +@array_function_dispatch(_imag_dispatcher) def imag(val): """ Return the imaginary part of the complex argument. @@ -169,10 +192,10 @@ def imag(val): -------- >>> a = np.array([1+2j, 3+4j, 5+6j]) >>> a.imag - array([ 2., 4., 6.]) + array([2., 4., 6.]) >>> a.imag = np.array([8, 10, 12]) >>> a - array([ 1. +8.j, 3.+10.j, 5.+12.j]) + array([1. +8.j, 3.+10.j, 5.+12.j]) >>> np.imag(1 + 1j) 1.0 @@ -183,6 +206,11 @@ def imag(val): return asanyarray(val).imag +def _is_type_dispatcher(x): + return (x,) + + +@array_function_dispatch(_is_type_dispatcher) def iscomplex(x): """ Returns a bool array, where True if input element is complex. @@ -218,6 +246,8 @@ def iscomplex(x): res = zeros(ax.shape, bool) return res[()] # convert to scalar if needed + +@array_function_dispatch(_is_type_dispatcher) def isreal(x): """ Returns a bool array, where True if input element is real. @@ -248,6 +278,8 @@ def isreal(x): """ return imag(x) == 0 + +@array_function_dispatch(_is_type_dispatcher) def iscomplexobj(x): """ Check for a complex type or an array of complex numbers. @@ -288,6 +320,7 @@ def iscomplexobj(x): return issubclass(type_, _nx.complexfloating) +@array_function_dispatch(_is_type_dispatcher) def isrealobj(x): """ Return True if x is a not complex type or an array of complex numbers. @@ -329,6 +362,12 @@ def _getmaxmin(t): f = getlimits.finfo(t) return f.max, f.min + +def _nan_to_num_dispatcher(x, copy=None): + return (x,) + + +@array_function_dispatch(_nan_to_num_dispatcher) def nan_to_num(x, copy=True): """ Replace NaN with zero and infinity with large finite numbers. @@ -383,11 +422,13 @@ def nan_to_num(x, copy=True): 0.0 >>> x = np.array([np.inf, -np.inf, np.nan, -128, 128]) >>> np.nan_to_num(x) - array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, - -1.28000000e+002, 1.28000000e+002]) + array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary + -1.28000000e+002, 1.28000000e+002]) >>> y = np.array([complex(np.inf, np.nan), np.nan, complex(np.nan, np.inf)]) + array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary + -1.28000000e+002, 1.28000000e+002]) >>> np.nan_to_num(y) - array([ 1.79769313e+308 +0.00000000e+000j, + array([ 1.79769313e+308 +0.00000000e+000j, # may vary 0.00000000e+000 +0.00000000e+000j, 0.00000000e+000 +1.79769313e+308j]) """ @@ -411,7 +452,12 @@ def nan_to_num(x, copy=True): #----------------------------------------------------------------------------- -def real_if_close(a,tol=100): +def _real_if_close_dispatcher(a, tol=None): + return (a,) + + +@array_function_dispatch(_real_if_close_dispatcher) +def real_if_close(a, tol=100): """ If complex input returns a real array if complex parts are close to zero. @@ -446,12 +492,12 @@ def real_if_close(a,tol=100): Examples -------- >>> np.finfo(float).eps - 2.2204460492503131e-16 + 2.2204460492503131e-16 # may vary >>> np.real_if_close([2.1 + 4e-14j], tol=1000) - array([ 2.1]) + array([2.1]) >>> np.real_if_close([2.1 + 4e-13j], tol=1000) - array([ 2.1 +4.00000000e-13j]) + array([2.1+4.e-13j]) """ a = asanyarray(a) @@ -466,6 +512,11 @@ def real_if_close(a,tol=100): return a +def _asscalar_dispatcher(a): + return (a,) + + +@array_function_dispatch(_asscalar_dispatcher) def asscalar(a): """ Convert an array of size 1 to its scalar equivalent. @@ -489,7 +540,6 @@ def asscalar(a): -------- >>> np.asscalar(np.array([24])) 24 - """ # 2018-10-10, 1.16 @@ -523,6 +573,7 @@ _namefromtype = {'S1': 'character', 'O': 'object' } +@set_module('numpy') def typename(char): """ Return a description for the given data type code. @@ -586,6 +637,13 @@ array_precision = {_nx.half: 0, _nx.csingle: 1, _nx.cdouble: 2, _nx.clongdouble: 3} + + +def _common_type_dispatcher(*arrays): + return arrays + + +@array_function_dispatch(_common_type_dispatcher) def common_type(*arrays): """ Return a scalar type which is common to the input arrays. @@ -615,11 +673,11 @@ def common_type(*arrays): Examples -------- >>> np.common_type(np.arange(2, dtype=np.float32)) - <type 'numpy.float32'> + <class 'numpy.float32'> >>> np.common_type(np.arange(2, dtype=np.float32), np.arange(2)) - <type 'numpy.float64'> + <class 'numpy.float64'> >>> np.common_type(np.arange(4), np.array([45, 6.j]), np.array([45.0])) - <type 'numpy.complex128'> + <class 'numpy.complex128'> """ is_complex = False diff --git a/numpy/lib/ufunclike.py b/numpy/lib/ufunclike.py index 6259c5445..5c411e8c8 100644 --- a/numpy/lib/ufunclike.py +++ b/numpy/lib/ufunclike.py @@ -8,6 +8,7 @@ from __future__ import division, absolute_import, print_function __all__ = ['fix', 'isneginf', 'isposinf'] import numpy.core.numeric as nx +from numpy.core.overrides import array_function_dispatch, ENABLE_ARRAY_FUNCTION import warnings import functools @@ -37,7 +38,34 @@ def _deprecate_out_named_y(f): return func +def _fix_out_named_y(f): + """ + Allow the out argument to be passed as the name `y` (deprecated) + + This decorator should only be used if _deprecate_out_named_y is used on + a corresponding dispatcher fucntion. + """ + @functools.wraps(f) + def func(x, out=None, **kwargs): + if 'y' in kwargs: + # we already did error checking in _deprecate_out_named_y + out = kwargs.pop('y') + return f(x, out=out, **kwargs) + + return func + + +if not ENABLE_ARRAY_FUNCTION: + _fix_out_named_y = _deprecate_out_named_y + + @_deprecate_out_named_y +def _dispatcher(x, out=None): + return (x, out) + + +@array_function_dispatch(_dispatcher, verify=False, module='numpy') +@_fix_out_named_y def fix(x, out=None): """ Round to nearest integer towards zero. @@ -83,7 +111,8 @@ def fix(x, out=None): return res -@_deprecate_out_named_y +@array_function_dispatch(_dispatcher, verify=False, module='numpy') +@_fix_out_named_y def isposinf(x, out=None): """ Test element-wise for positive infinity, return result as bool array. @@ -125,11 +154,11 @@ def isposinf(x, out=None): Examples -------- >>> np.isposinf(np.PINF) - array(True, dtype=bool) + True >>> np.isposinf(np.inf) - array(True, dtype=bool) + True >>> np.isposinf(np.NINF) - array(False, dtype=bool) + False >>> np.isposinf([-np.inf, 0., np.inf]) array([False, False, True]) @@ -151,7 +180,8 @@ def isposinf(x, out=None): return nx.logical_and(is_inf, signbit, out) -@_deprecate_out_named_y +@array_function_dispatch(_dispatcher, verify=False, module='numpy') +@_fix_out_named_y def isneginf(x, out=None): """ Test element-wise for negative infinity, return result as bool array. @@ -194,11 +224,11 @@ def isneginf(x, out=None): Examples -------- >>> np.isneginf(np.NINF) - array(True, dtype=bool) + True >>> np.isneginf(np.inf) - array(False, dtype=bool) + False >>> np.isneginf(np.PINF) - array(False, dtype=bool) + False >>> np.isneginf([-np.inf, 0., np.inf]) array([ True, False, False]) diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index 249873654..5a4cae235 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -7,6 +7,7 @@ import re import warnings from numpy.core.numerictypes import issubclass_, issubsctype, issubdtype +from numpy.core.overrides import set_module from numpy.core import ndarray, ufunc, asarray import numpy as np @@ -149,10 +150,8 @@ def deprecate(*args, **kwargs): Warning: >>> olduint = np.deprecate(np.uint) + DeprecationWarning: `uint64` is deprecated! # may vary >>> olduint(6) - /usr/lib/python2.5/site-packages/numpy/lib/utils.py:114: - DeprecationWarning: uint32 is deprecated - warnings.warn(str1, DeprecationWarning, stacklevel=2) 6 """ @@ -164,13 +163,6 @@ def deprecate(*args, **kwargs): fn = args[0] args = args[1:] - # backward compatibility -- can be removed - # after next release - if 'newname' in kwargs: - kwargs['new_name'] = kwargs.pop('newname') - if 'oldname' in kwargs: - kwargs['old_name'] = kwargs.pop('oldname') - return _Deprecate(*args, **kwargs)(fn) else: return _Deprecate(*args, **kwargs) @@ -207,8 +199,8 @@ def byte_bounds(a): >>> low, high = np.byte_bounds(I) >>> high - low == I.size*I.itemsize True - >>> I = np.eye(2, dtype='G'); I.dtype - dtype('complex192') + >>> I = np.eye(2); I.dtype + dtype('float64') >>> low, high = np.byte_bounds(I) >>> high - low == I.size*I.itemsize True @@ -269,17 +261,17 @@ def who(vardict=None): >>> np.who() Name Shape Bytes Type =========================================================== - a 10 40 int32 + a 10 80 int64 b 20 160 float64 - Upper bound on total bytes = 200 + Upper bound on total bytes = 240 >>> d = {'x': np.arange(2.0), 'y': np.arange(3.0), 'txt': 'Some str', ... 'idx':5} >>> np.who(d) Name Shape Bytes Type =========================================================== - y 3 24 float64 x 2 16 float64 + y 3 24 float64 Upper bound on total bytes = 40 """ @@ -439,6 +431,7 @@ def _info(obj, output=sys.stdout): print("type: %s" % obj.dtype, file=output) +@set_module('numpy') def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): """ Get help information for a function, class, or module. @@ -644,6 +637,7 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): print(inspect.getdoc(object), file=output) +@set_module('numpy') def source(object, output=sys.stdout): """ Print or write to a file the source code for a NumPy object. @@ -701,6 +695,8 @@ _lookfor_caches = {} # signature _function_signature_re = re.compile(r"[a-z0-9_]+\(.*[,=].*\)", re.I) + +@set_module('numpy') def lookfor(what, module=None, import_modules=True, regenerate=False, output=None): """ @@ -735,7 +731,7 @@ def lookfor(what, module=None, import_modules=True, regenerate=False, Examples -------- - >>> np.lookfor('binary representation') + >>> np.lookfor('binary representation') # doctest: +SKIP Search results for 'binary representation' ------------------------------------------ numpy.binary_repr @@ -1106,7 +1102,7 @@ def safe_eval(source): >>> np.safe_eval('open("/home/user/.ssh/id_dsa").read()') Traceback (most recent call last): ... - SyntaxError: Unsupported source construct: compiler.ast.CallFunc + ValueError: malformed node or string: <_ast.Call object at 0x...> """ # Local import to speed up numpy's import time. diff --git a/numpy/linalg/linalg.py b/numpy/linalg/linalg.py index 59923f3c5..92fa6cb73 100644 --- a/numpy/linalg/linalg.py +++ b/numpy/linalg/linalg.py @@ -16,6 +16,7 @@ __all__ = ['matrix_power', 'solve', 'tensorsolve', 'tensorinv', 'inv', 'svd', 'eig', 'eigh', 'lstsq', 'norm', 'qr', 'cond', 'matrix_rank', 'LinAlgError', 'multi_dot'] +import functools import operator import warnings @@ -28,10 +29,16 @@ from numpy.core import ( swapaxes, divide, count_nonzero, isnan ) from numpy.core.multiarray import normalize_axis_index -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import set_module +from numpy.core import overrides from numpy.lib.twodim_base import triu, eye from numpy.linalg import lapack_lite, _umath_linalg + +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy.linalg') + + # For Python2/3 compatibility _N = b'N' _V = b'V' @@ -41,7 +48,8 @@ _L = b'L' fortran_int = intc -# Error object + +@set_module('numpy.linalg') class LinAlgError(Exception): """ Generic Python-exception-derived object raised by linalg functions. @@ -69,7 +77,6 @@ class LinAlgError(Exception): numpy.linalg.LinAlgError: Singular matrix """ - pass def _determine_error_states(): @@ -370,7 +377,7 @@ def solve(a, b): >>> b = np.array([9,8]) >>> x = np.linalg.solve(a, b) >>> x - array([ 2., 3.]) + array([2., 3.]) Check that the solution is correct: @@ -528,10 +535,10 @@ def inv(a): >>> a = np.array([[[1., 2.], [3., 4.]], [[1, 3], [3, 5]]]) >>> inv(a) - array([[[-2. , 1. ], - [ 1.5, -0.5]], - [[-5. , 2. ], - [ 3. , -1. ]]]) + array([[[-2. , 1. ], + [ 1.5 , -0.5 ]], + [[-1.25, 0.75], + [ 0.75, -0.25]]]) """ a, wrap = _makearray(a) @@ -723,21 +730,21 @@ def cholesky(a): -------- >>> A = np.array([[1,-2j],[2j,5]]) >>> A - array([[ 1.+0.j, 0.-2.j], + array([[ 1.+0.j, -0.-2.j], [ 0.+2.j, 5.+0.j]]) >>> L = np.linalg.cholesky(A) >>> L - array([[ 1.+0.j, 0.+0.j], - [ 0.+2.j, 1.+0.j]]) + array([[1.+0.j, 0.+0.j], + [0.+2.j, 1.+0.j]]) >>> np.dot(L, L.T.conj()) # verify that L * L.H = A - array([[ 1.+0.j, 0.-2.j], - [ 0.+2.j, 5.+0.j]]) + array([[1.+0.j, 0.-2.j], + [0.+2.j, 5.+0.j]]) >>> A = [[1,-2j],[2j,5]] # what happens if A is only array_like? >>> np.linalg.cholesky(A) # an ndarray object is returned - array([[ 1.+0.j, 0.+0.j], - [ 0.+2.j, 1.+0.j]]) + array([[1.+0.j, 0.+0.j], + [0.+2.j, 1.+0.j]]) >>> # But a matrix object is returned if A is a matrix object - >>> LA.cholesky(np.matrix(A)) + >>> np.linalg.cholesky(np.matrix(A)) matrix([[ 1.+0.j, 0.+0.j], [ 0.+2.j, 1.+0.j]]) @@ -871,9 +878,9 @@ def qr(a, mode='reduced'): [1, 1], [2, 1]]) >>> b = np.array([1, 0, 2, 1]) - >>> q, r = LA.qr(A) + >>> q, r = np.linalg.qr(A) >>> p = np.dot(q.T, b) - >>> np.dot(LA.inv(r), p) + >>> np.dot(np.linalg.inv(r), p) array([ 1.1e-16, 1.0e+00]) """ @@ -1042,7 +1049,7 @@ def eigvals(a): >>> A = np.dot(Q, D) >>> A = np.dot(A, Q.T) >>> LA.eigvals(A) - array([ 1., -1.]) + array([ 1., -1.]) # random """ a, wrap = _makearray(a) @@ -1124,24 +1131,24 @@ def eigvalsh(a, UPLO='L'): >>> from numpy import linalg as LA >>> a = np.array([[1, -2j], [2j, 5]]) >>> LA.eigvalsh(a) - array([ 0.17157288, 5.82842712]) + array([ 0.17157288, 5.82842712]) # may vary >>> # demonstrate the treatment of the imaginary part of the diagonal >>> a = np.array([[5+2j, 9-2j], [0+2j, 2-1j]]) >>> a - array([[ 5.+2.j, 9.-2.j], - [ 0.+2.j, 2.-1.j]]) + array([[5.+2.j, 9.-2.j], + [0.+2.j, 2.-1.j]]) >>> # with UPLO='L' this is numerically equivalent to using LA.eigvals() >>> # with: >>> b = np.array([[5.+0.j, 0.-2.j], [0.+2.j, 2.-0.j]]) >>> b - array([[ 5.+0.j, 0.-2.j], - [ 0.+2.j, 2.+0.j]]) + array([[5.+0.j, 0.-2.j], + [0.+2.j, 2.+0.j]]) >>> wa = LA.eigvalsh(a) >>> wb = LA.eigvals(b) >>> wa; wb - array([ 1., 6.]) - array([ 6.+0.j, 1.+0.j]) + array([1., 6.]) + array([6.+0.j, 1.+0.j]) """ UPLO = UPLO.upper() @@ -1257,19 +1264,19 @@ def eig(a): >>> w, v = LA.eig(np.diag((1, 2, 3))) >>> w; v - array([ 1., 2., 3.]) - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([1., 2., 3.]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) Real matrix possessing complex e-values and e-vectors; note that the e-values are complex conjugates of each other. >>> w, v = LA.eig(np.array([[1, -1], [1, 1]])) >>> w; v - array([ 1. + 1.j, 1. - 1.j]) - array([[ 0.70710678+0.j , 0.70710678+0.j ], - [ 0.00000000-0.70710678j, 0.00000000+0.70710678j]]) + array([1.+1.j, 1.-1.j]) + array([[0.70710678+0.j , 0.70710678-0.j ], + [0. -0.70710678j, 0. +0.70710678j]]) Complex-valued matrix with real e-values (but complex-valued e-vectors); note that a.conj().T = a, i.e., a is Hermitian. @@ -1277,9 +1284,9 @@ def eig(a): >>> a = np.array([[1, 1j], [-1j, 1]]) >>> w, v = LA.eig(a) >>> w; v - array([ 2.00000000e+00+0.j, 5.98651912e-36+0.j]) # i.e., {2, 0} - array([[ 0.00000000+0.70710678j, 0.70710678+0.j ], - [ 0.70710678+0.j , 0.00000000+0.70710678j]]) + array([2.+0.j, 0.+0.j]) + array([[ 0. +0.70710678j, 0.70710678+0.j ], # may vary + [ 0.70710678+0.j , -0. +0.70710678j]]) Be careful about round-off error! @@ -1287,9 +1294,9 @@ def eig(a): >>> # Theor. e-values are 1 +/- 1e-9 >>> w, v = LA.eig(a) >>> w; v - array([ 1., 1.]) - array([[ 1., 0.], - [ 0., 1.]]) + array([1., 1.]) + array([[1., 0.], + [0., 1.]]) """ a, wrap = _makearray(a) @@ -1385,49 +1392,49 @@ def eigh(a, UPLO='L'): >>> from numpy import linalg as LA >>> a = np.array([[1, -2j], [2j, 5]]) >>> a - array([[ 1.+0.j, 0.-2.j], + array([[ 1.+0.j, -0.-2.j], [ 0.+2.j, 5.+0.j]]) >>> w, v = LA.eigh(a) >>> w; v - array([ 0.17157288, 5.82842712]) - array([[-0.92387953+0.j , -0.38268343+0.j ], - [ 0.00000000+0.38268343j, 0.00000000-0.92387953j]]) + array([0.17157288, 5.82842712]) + array([[-0.92387953+0.j , -0.38268343+0.j ], # may vary + [ 0. +0.38268343j, 0. -0.92387953j]]) >>> np.dot(a, v[:, 0]) - w[0] * v[:, 0] # verify 1st e-val/vec pair - array([2.77555756e-17 + 0.j, 0. + 1.38777878e-16j]) + array([5.55111512e-17+0.0000000e+00j, 0.00000000e+00+1.2490009e-16j]) >>> np.dot(a, v[:, 1]) - w[1] * v[:, 1] # verify 2nd e-val/vec pair - array([ 0.+0.j, 0.+0.j]) + array([0.+0.j, 0.+0.j]) >>> A = np.matrix(a) # what happens if input is a matrix object >>> A - matrix([[ 1.+0.j, 0.-2.j], + matrix([[ 1.+0.j, -0.-2.j], [ 0.+2.j, 5.+0.j]]) >>> w, v = LA.eigh(A) >>> w; v - array([ 0.17157288, 5.82842712]) - matrix([[-0.92387953+0.j , -0.38268343+0.j ], - [ 0.00000000+0.38268343j, 0.00000000-0.92387953j]]) + array([0.17157288, 5.82842712]) + matrix([[-0.92387953+0.j , -0.38268343+0.j ], # may vary + [ 0. +0.38268343j, 0. -0.92387953j]]) >>> # demonstrate the treatment of the imaginary part of the diagonal >>> a = np.array([[5+2j, 9-2j], [0+2j, 2-1j]]) >>> a - array([[ 5.+2.j, 9.-2.j], - [ 0.+2.j, 2.-1.j]]) + array([[5.+2.j, 9.-2.j], + [0.+2.j, 2.-1.j]]) >>> # with UPLO='L' this is numerically equivalent to using LA.eig() with: >>> b = np.array([[5.+0.j, 0.-2.j], [0.+2.j, 2.-0.j]]) >>> b - array([[ 5.+0.j, 0.-2.j], - [ 0.+2.j, 2.+0.j]]) + array([[5.+0.j, 0.-2.j], + [0.+2.j, 2.+0.j]]) >>> wa, va = LA.eigh(a) >>> wb, vb = LA.eig(b) >>> wa; wb - array([ 1., 6.]) - array([ 6.+0.j, 1.+0.j]) + array([1., 6.]) + array([6.+0.j, 1.+0.j]) >>> va; vb - array([[-0.44721360-0.j , -0.89442719+0.j ], - [ 0.00000000+0.89442719j, 0.00000000-0.4472136j ]]) - array([[ 0.89442719+0.j , 0.00000000-0.4472136j], - [ 0.00000000-0.4472136j, 0.89442719+0.j ]]) + array([[-0.4472136 +0.j , -0.89442719+0.j ], # may vary + [ 0. +0.89442719j, 0. -0.4472136j ]]) + array([[ 0.89442719+0.j , -0. +0.4472136j], + [-0. +0.4472136j, 0.89442719+0.j ]]) """ UPLO = UPLO.upper() if UPLO not in ('L', 'U'): @@ -1698,9 +1705,9 @@ def cond(x, p=None): >>> LA.cond(a, 2) 1.4142135623730951 >>> LA.cond(a, -2) - 0.70710678118654746 + 0.70710678118654746 # may vary >>> min(LA.svd(a, compute_uv=0))*min(LA.svd(LA.inv(a), compute_uv=0)) - 0.70710678118654746 + 0.70710678118654746 # may vary """ x = asarray(x) # in case we have a matrix @@ -1995,7 +2002,7 @@ def slogdet(a): >>> a = np.array([[1, 2], [3, 4]]) >>> (sign, logdet) = np.linalg.slogdet(a) >>> (sign, logdet) - (-1, 0.69314718055994529) + (-1, 0.69314718055994529) # may vary >>> sign * np.exp(logdet) -2.0 @@ -2067,7 +2074,7 @@ def det(a): >>> a = np.array([[1, 2], [3, 4]]) >>> np.linalg.det(a) - -2.0 + -2.0 # may vary Computing determinants for a stack of matrices: @@ -2174,15 +2181,15 @@ def lstsq(a, b, rcond="warn"): [ 3., 1.]]) >>> m, c = np.linalg.lstsq(A, y, rcond=None)[0] - >>> print(m, c) - 1.0 -0.95 + >>> m, c + (1.0 -0.95) # may vary Plot the data along with the fitted line: >>> import matplotlib.pyplot as plt - >>> plt.plot(x, y, 'o', label='Original data', markersize=10) - >>> plt.plot(x, m*x + c, 'r', label='Fitted line') - >>> plt.legend() + >>> _ = plt.plot(x, y, 'o', label='Original data', markersize=10) + >>> _ = plt.plot(x, m*x + c, 'r', label='Fitted line') + >>> _ = plt.legend() >>> plt.show() """ @@ -2198,6 +2205,7 @@ def lstsq(a, b, rcond="warn"): raise LinAlgError('Incompatible dimensions') t, result_t = _commonType(a, b) + # FIXME: real_t is unused real_t = _linalgRealType(t) result_real_t = _realType(result_t) @@ -2359,7 +2367,7 @@ def norm(x, ord=None, axis=None, keepdims=False): >>> from numpy import linalg as LA >>> a = np.arange(9) - 4 >>> a - array([-4, -3, -2, -1, 0, 1, 2, 3, 4]) + array([-4, -3, -2, ..., 2, 3, 4]) >>> b = a.reshape((3, 3)) >>> b array([[-4, -3, -2], @@ -2395,13 +2403,13 @@ def norm(x, ord=None, axis=None, keepdims=False): 7.3484692283495345 >>> LA.norm(a, -2) - nan + 0.0 >>> LA.norm(b, -2) - 1.8570331885190563e-016 + 1.8570331885190563e-016 # may vary >>> LA.norm(a, 3) - 5.8480354764257312 + 5.8480354764257312 # may vary >>> LA.norm(a, -3) - nan + 0.0 Using the `axis` argument to compute vector norms: @@ -2576,18 +2584,20 @@ def multi_dot(arrays): >>> from numpy.linalg import multi_dot >>> # Prepare some data - >>> A = np.random.random(10000, 100) - >>> B = np.random.random(100, 1000) - >>> C = np.random.random(1000, 5) - >>> D = np.random.random(5, 333) + >>> A = np.random.random((10000, 100)) + >>> B = np.random.random((100, 1000)) + >>> C = np.random.random((1000, 5)) + >>> D = np.random.random((5, 333)) >>> # the actual dot multiplication - >>> multi_dot([A, B, C, D]) + >>> _ = multi_dot([A, B, C, D]) instead of:: - >>> np.dot(np.dot(np.dot(A, B), C), D) + >>> _ = np.dot(np.dot(np.dot(A, B), C), D) + ... >>> # or - >>> A.dot(B).dot(C).dot(D) + >>> _ = A.dot(B).dot(C).dot(D) + ... Notes ----- diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index 0e94c2633..235488c6e 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -13,13 +13,14 @@ import pytest import numpy as np from numpy import array, single, double, csingle, cdouble, dot, identity, matmul -from numpy import multiply, atleast_2d, inf, asarray, matrix +from numpy import multiply, atleast_2d, inf, asarray from numpy import linalg from numpy.linalg import matrix_power, norm, matrix_rank, multi_dot, LinAlgError from numpy.linalg.linalg import _multi_dot_matrix_chain_order from numpy.testing import ( assert_, assert_equal, assert_raises, assert_array_equal, - assert_almost_equal, assert_allclose, suppress_warnings + assert_almost_equal, assert_allclose, suppress_warnings, + assert_raises_regex, ) @@ -931,6 +932,14 @@ class TestLstsq(LstsqCases): assert_equal(rank, min(m, n)) assert_equal(s.shape, (min(m, n),)) + def test_incompatible_dims(self): + # use modified version of docstring example + x = np.array([0, 1, 2, 3]) + y = np.array([-1, 0.2, 0.9, 2.1, 3.3]) + A = np.vstack([x, np.ones(len(x))]).T + with assert_raises_regex(LinAlgError, "Incompatible dimensions"): + linalg.lstsq(A, y, rcond=None) + @pytest.mark.parametrize('dt', [np.dtype(c) for c in '?bBhHiIqQefdgFDGO']) class TestMatrixPower(object): @@ -946,7 +955,6 @@ class TestMatrixPower(object): dtnoinv = [object, np.dtype('e'), np.dtype('g'), np.dtype('G')] def test_large_power(self, dt): - power = matrix_power rshft = self.rshft_1.astype(dt) assert_equal( matrix_power(rshft, 2**100 + 2**10 + 2**5 + 0), self.rshft_0) @@ -1610,8 +1618,6 @@ class TestQR(object): def test_qr_empty(self, m, n): k = min(m, n) a = np.empty((m, n)) - a_type = type(a) - a_dtype = a.dtype self.check_qr(a) @@ -1915,3 +1921,44 @@ class TestMultiDot(object): def test_too_few_input_arrays(self): assert_raises(ValueError, multi_dot, []) assert_raises(ValueError, multi_dot, [np.random.random((3, 3))]) + + +class TestTensorinv(object): + + @pytest.mark.parametrize("arr, ind", [ + (np.ones((4, 6, 8, 2)), 2), + (np.ones((3, 3, 2)), 1), + ]) + def test_non_square_handling(self, arr, ind): + with assert_raises(LinAlgError): + linalg.tensorinv(arr, ind=ind) + + @pytest.mark.parametrize("shape, ind", [ + # examples from docstring + ((4, 6, 8, 3), 2), + ((24, 8, 3), 1), + ]) + def test_tensorinv_shape(self, shape, ind): + a = np.eye(24) + a.shape = shape + ainv = linalg.tensorinv(a=a, ind=ind) + expected = a.shape[ind:] + a.shape[:ind] + actual = ainv.shape + assert_equal(actual, expected) + + @pytest.mark.parametrize("ind", [ + 0, -2, + ]) + def test_tensorinv_ind_limit(self, ind): + a = np.eye(24) + a.shape = (4, 6, 8, 3) + with assert_raises(ValueError): + linalg.tensorinv(a=a, ind=ind) + + def test_tensorinv_result(self): + # mimic a docstring example + a = np.eye(24) + a.shape = (24, 8, 3) + ainv = linalg.tensorinv(a, ind=1) + b = np.ones(24) + assert_allclose(np.tensordot(ainv, b, 1), np.linalg.tensorsolve(a, b)) diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 9259aa4c7..63a61599c 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -446,36 +446,37 @@ def _check_fill_value(fill_value, ndtype): If fill_value is not None, its value is forced to the given dtype. The result is always a 0d array. + """ ndtype = np.dtype(ndtype) - fields = ndtype.fields if fill_value is None: fill_value = default_fill_value(ndtype) - elif fields: - fdtype = [(_[0], _[1]) for _ in ndtype.descr] + elif ndtype.names is not None: if isinstance(fill_value, (ndarray, np.void)): try: - fill_value = np.array(fill_value, copy=False, dtype=fdtype) + fill_value = np.array(fill_value, copy=False, dtype=ndtype) except ValueError: err_msg = "Unable to transform %s to dtype %s" - raise ValueError(err_msg % (fill_value, fdtype)) + raise ValueError(err_msg % (fill_value, ndtype)) else: fill_value = np.asarray(fill_value, dtype=object) fill_value = np.array(_recursive_set_fill_value(fill_value, ndtype), dtype=ndtype) else: if isinstance(fill_value, basestring) and (ndtype.char not in 'OSVU'): + # Note this check doesn't work if fill_value is not a scalar err_msg = "Cannot set fill value of string with array of dtype %s" raise TypeError(err_msg % ndtype) else: # In case we want to convert 1e20 to int. + # Also in case of converting string arrays. try: fill_value = np.array(fill_value, copy=False, dtype=ndtype) - except OverflowError: - # Raise TypeError instead of OverflowError. OverflowError - # is seldom used, and the real problem here is that the - # passed fill_value is not compatible with the ndtype. - err_msg = "Fill value %s overflows dtype %s" + except (OverflowError, ValueError): + # Raise TypeError instead of OverflowError or ValueError. + # OverflowError is seldom used, and the real problem here is + # that the passed fill_value is not compatible with the ndtype. + err_msg = "Cannot convert fill_value %s to dtype %s" raise TypeError(err_msg % (fill_value, ndtype)) return np.array(fill_value) @@ -515,18 +516,18 @@ def set_fill_value(a, fill_value): array([0, 1, 2, 3, 4]) >>> a = ma.masked_where(a < 3, a) >>> a - masked_array(data = [-- -- -- 3 4], - mask = [ True True True False False], - fill_value=999999) + masked_array(data=[--, --, --, 3, 4], + mask=[ True, True, True, False, False], + fill_value=999999) >>> ma.set_fill_value(a, -999) >>> a - masked_array(data = [-- -- -- 3 4], - mask = [ True True True False False], - fill_value=-999) + masked_array(data=[--, --, --, 3, 4], + mask=[ True, True, True, False, False], + fill_value=-999) Nothing happens if `a` is not a masked array. - >>> a = range(5) + >>> a = list(range(5)) >>> a [0, 1, 2, 3, 4] >>> ma.set_fill_value(a, 100) @@ -688,13 +689,12 @@ def getdata(a, subok=True): >>> import numpy.ma as ma >>> a = ma.masked_equal([[1,2],[3,4]], 2) >>> a - masked_array(data = - [[1 --] - [3 4]], - mask = - [[False True] - [False False]], - fill_value=999999) + masked_array( + data=[[1, --], + [3, 4]], + mask=[[False, True], + [False, False]], + fill_value=2) >>> ma.getdata(a) array([[1, 2], [3, 4]]) @@ -751,20 +751,19 @@ def fix_invalid(a, mask=nomask, copy=True, fill_value=None): -------- >>> x = np.ma.array([1., -1, np.nan, np.inf], mask=[1] + [0]*3) >>> x - masked_array(data = [-- -1.0 nan inf], - mask = [ True False False False], - fill_value = 1e+20) + masked_array(data=[--, -1.0, nan, inf], + mask=[ True, False, False, False], + fill_value=1e+20) >>> np.ma.fix_invalid(x) - masked_array(data = [-- -1.0 -- --], - mask = [ True False True True], - fill_value = 1e+20) + masked_array(data=[--, -1.0, --, --], + mask=[ True, False, True, True], + fill_value=1e+20) >>> fixed = np.ma.fix_invalid(x) >>> fixed.data - array([ 1.00000000e+00, -1.00000000e+00, 1.00000000e+20, - 1.00000000e+20]) + array([ 1.e+00, -1.e+00, 1.e+20, 1.e+20]) >>> x.data - array([ 1., -1., NaN, Inf]) + array([ 1., -1., nan, inf]) """ a = masked_array(a, copy=copy, mask=mask, subok=True) @@ -777,6 +776,10 @@ def fix_invalid(a, mask=nomask, copy=True, fill_value=None): a._data[invalid] = fill_value return a +def is_string_or_list_of_strings(val): + return (isinstance(val, basestring) or + (isinstance(val, list) and val and + builtins.all(isinstance(s, basestring) for s in val))) ############################################################################### # Ufuncs # @@ -1341,9 +1344,9 @@ def make_mask_descr(ndtype): -------- >>> import numpy.ma as ma >>> dtype = np.dtype({'names':['foo', 'bar'], - 'formats':[np.float32, int]}) + ... 'formats':[np.float32, np.int64]}) >>> dtype - dtype([('foo', '<f4'), ('bar', '<i4')]) + dtype([('foo', '<f4'), ('bar', '<i8')]) >>> ma.make_mask_descr(dtype) dtype([('foo', '|b1'), ('bar', '|b1')]) >>> ma.make_mask_descr(np.float32) @@ -1376,13 +1379,12 @@ def getmask(a): >>> import numpy.ma as ma >>> a = ma.masked_equal([[1,2],[3,4]], 2) >>> a - masked_array(data = - [[1 --] - [3 4]], - mask = - [[False True] - [False False]], - fill_value=999999) + masked_array( + data=[[1, --], + [3, 4]], + mask=[[False, True], + [False, False]], + fill_value=2) >>> ma.getmask(a) array([[False, True], [False, False]]) @@ -1397,12 +1399,11 @@ def getmask(a): >>> b = ma.masked_array([[1,2],[3,4]]) >>> b - masked_array(data = - [[1 2] - [3 4]], - mask = - False, - fill_value=999999) + masked_array( + data=[[1, 2], + [3, 4]], + mask=False, + fill_value=999999) >>> ma.nomask False >>> ma.getmask(b) == ma.nomask @@ -1440,13 +1441,12 @@ def getmaskarray(arr): >>> import numpy.ma as ma >>> a = ma.masked_equal([[1,2],[3,4]], 2) >>> a - masked_array(data = - [[1 --] - [3 4]], - mask = - [[False True] - [False False]], - fill_value=999999) + masked_array( + data=[[1, --], + [3, 4]], + mask=[[False, True], + [False, False]], + fill_value=2) >>> ma.getmaskarray(a) array([[False, True], [False, False]]) @@ -1455,13 +1455,12 @@ def getmaskarray(arr): >>> b = ma.masked_array([[1,2],[3,4]]) >>> b - masked_array(data = - [[1 2] - [3 4]], - mask = - False, - fill_value=999999) - >>> >ma.getmaskarray(b) + masked_array( + data=[[1, 2], + [3, 4]], + mask=False, + fill_value=999999) + >>> ma.getmaskarray(b) array([[False, False], [False, False]]) @@ -1499,9 +1498,9 @@ def is_mask(m): >>> import numpy.ma as ma >>> m = ma.masked_equal([0, 1, 0, 2, 3], 0) >>> m - masked_array(data = [-- 1 -- 2 3], - mask = [ True False True False False], - fill_value=999999) + masked_array(data=[--, 1, --, 2, 3], + mask=[ True, False, True, False, False], + fill_value=0) >>> ma.is_mask(m) False >>> ma.is_mask(m.mask) @@ -1522,14 +1521,14 @@ def is_mask(m): Arrays with complex dtypes don't return True. >>> dtype = np.dtype({'names':['monty', 'pithon'], - 'formats':[bool, bool]}) + ... 'formats':[bool, bool]}) >>> dtype dtype([('monty', '|b1'), ('pithon', '|b1')]) >>> m = np.array([(True, False), (False, True), (True, False)], - dtype=dtype) + ... dtype=dtype) >>> m - array([(True, False), (False, True), (True, False)], - dtype=[('monty', '|b1'), ('pithon', '|b1')]) + array([( True, False), (False, True), ( True, False)], + dtype=[('monty', '?'), ('pithon', '?')]) >>> ma.is_mask(m) False @@ -1595,7 +1594,7 @@ def make_mask(m, copy=False, shrink=True, dtype=MaskType): >>> m = np.zeros(4) >>> m - array([ 0., 0., 0., 0.]) + array([0., 0., 0., 0.]) >>> ma.make_mask(m) False >>> ma.make_mask(m, shrink=False) @@ -1611,11 +1610,11 @@ def make_mask(m, copy=False, shrink=True, dtype=MaskType): >>> arr [(1, 0), (0, 1), (1, 0), (1, 0)] >>> dtype = np.dtype({'names':['man', 'mouse'], - 'formats':[int, int]}) + ... 'formats':[np.int64, np.int64]}) >>> arr = np.array(arr, dtype=dtype) >>> arr array([(1, 0), (0, 1), (1, 0), (1, 0)], - dtype=[('man', '<i4'), ('mouse', '<i4')]) + dtype=[('man', '<i8'), ('mouse', '<i8')]) >>> ma.make_mask(arr, dtype=dtype) array([(True, False), (False, True), (True, False), (True, False)], dtype=[('man', '|b1'), ('mouse', '|b1')]) @@ -1674,9 +1673,9 @@ def make_mask_none(newshape, dtype=None): Defining a more complex dtype. >>> dtype = np.dtype({'names':['foo', 'bar'], - 'formats':[np.float32, int]}) + ... 'formats':[np.float32, np.int64]}) >>> dtype - dtype([('foo', '<f4'), ('bar', '<i4')]) + dtype([('foo', '<f4'), ('bar', '<i8')]) >>> ma.make_mask_none((3,), dtype=dtype) array([(False, False), (False, False), (False, False)], dtype=[('foo', '|b1'), ('bar', '|b1')]) @@ -1774,16 +1773,16 @@ def flatten_mask(mask): Examples -------- >>> mask = np.array([0, 0, 1]) - >>> flatten_mask(mask) + >>> np.ma.flatten_mask(mask) array([False, False, True]) >>> mask = np.array([(0, 0), (0, 1)], dtype=[('a', bool), ('b', bool)]) - >>> flatten_mask(mask) + >>> np.ma.flatten_mask(mask) array([False, False, False, True]) >>> mdtype = [('a', bool), ('b', [('ba', bool), ('bb', bool)])] >>> mask = np.array([(0, (0, 0)), (0, (0, 1))], dtype=mdtype) - >>> flatten_mask(mask) + >>> np.ma.flatten_mask(mask) array([False, False, False, False, False, True]) """ @@ -1868,38 +1867,39 @@ def masked_where(condition, a, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_where(a <= 2, a) - masked_array(data = [-- -- -- 3], - mask = [ True True True False], - fill_value=999999) + masked_array(data=[--, --, --, 3], + mask=[ True, True, True, False], + fill_value=999999) Mask array `b` conditional on `a`. >>> b = ['a', 'b', 'c', 'd'] >>> ma.masked_where(a == 2, b) - masked_array(data = [a b -- d], - mask = [False False True False], - fill_value=N/A) + masked_array(data=['a', 'b', --, 'd'], + mask=[False, False, True, False], + fill_value='N/A', + dtype='<U1') Effect of the `copy` argument. >>> c = ma.masked_where(a <= 2, a) >>> c - masked_array(data = [-- -- -- 3], - mask = [ True True True False], - fill_value=999999) + masked_array(data=[--, --, --, 3], + mask=[ True, True, True, False], + fill_value=999999) >>> c[0] = 99 >>> c - masked_array(data = [99 -- -- 3], - mask = [False True True False], - fill_value=999999) + masked_array(data=[99, --, --, 3], + mask=[False, True, True, False], + fill_value=999999) >>> a array([0, 1, 2, 3]) >>> c = ma.masked_where(a <= 2, a, copy=False) >>> c[0] = 99 >>> c - masked_array(data = [99 -- -- 3], - mask = [False True True False], - fill_value=999999) + masked_array(data=[99, --, --, 3], + mask=[False, True, True, False], + fill_value=999999) >>> a array([99, 1, 2, 3]) @@ -1908,19 +1908,19 @@ def masked_where(condition, a, copy=True): >>> a = np.arange(4) >>> a = ma.masked_where(a == 2, a) >>> a - masked_array(data = [0 1 -- 3], - mask = [False False True False], - fill_value=999999) + masked_array(data=[0, 1, --, 3], + mask=[False, False, True, False], + fill_value=999999) >>> b = np.arange(4) >>> b = ma.masked_where(b == 0, b) >>> b - masked_array(data = [-- 1 2 3], - mask = [ True False False False], - fill_value=999999) + masked_array(data=[--, 1, 2, 3], + mask=[ True, False, False, False], + fill_value=999999) >>> ma.masked_where(a == 3, b) - masked_array(data = [-- 1 -- --], - mask = [ True False True True], - fill_value=999999) + masked_array(data=[--, 1, --, --], + mask=[ True, False, True, True], + fill_value=999999) """ # Make sure that condition is a valid standard-type mask. @@ -1960,9 +1960,9 @@ def masked_greater(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_greater(a, 2) - masked_array(data = [0 1 2 --], - mask = [False False False True], - fill_value=999999) + masked_array(data=[0, 1, 2, --], + mask=[False, False, False, True], + fill_value=999999) """ return masked_where(greater(x, value), x, copy=copy) @@ -1986,9 +1986,9 @@ def masked_greater_equal(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_greater_equal(a, 2) - masked_array(data = [0 1 -- --], - mask = [False False True True], - fill_value=999999) + masked_array(data=[0, 1, --, --], + mask=[False, False, True, True], + fill_value=999999) """ return masked_where(greater_equal(x, value), x, copy=copy) @@ -2012,9 +2012,9 @@ def masked_less(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_less(a, 2) - masked_array(data = [-- -- 2 3], - mask = [ True True False False], - fill_value=999999) + masked_array(data=[--, --, 2, 3], + mask=[ True, True, False, False], + fill_value=999999) """ return masked_where(less(x, value), x, copy=copy) @@ -2038,9 +2038,9 @@ def masked_less_equal(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_less_equal(a, 2) - masked_array(data = [-- -- -- 3], - mask = [ True True True False], - fill_value=999999) + masked_array(data=[--, --, --, 3], + mask=[ True, True, True, False], + fill_value=999999) """ return masked_where(less_equal(x, value), x, copy=copy) @@ -2064,9 +2064,9 @@ def masked_not_equal(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_not_equal(a, 2) - masked_array(data = [-- -- 2 --], - mask = [ True True False True], - fill_value=999999) + masked_array(data=[--, --, 2, --], + mask=[ True, True, False, True], + fill_value=999999) """ return masked_where(not_equal(x, value), x, copy=copy) @@ -2092,9 +2092,9 @@ def masked_equal(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_equal(a, 2) - masked_array(data = [0 1 -- 3], - mask = [False False True False], - fill_value=999999) + masked_array(data=[0, 1, --, 3], + mask=[False, False, True, False], + fill_value=2) """ output = masked_where(equal(x, value), x, copy=copy) @@ -2123,16 +2123,16 @@ def masked_inside(x, v1, v2, copy=True): >>> import numpy.ma as ma >>> x = [0.31, 1.2, 0.01, 0.2, -0.4, -1.1] >>> ma.masked_inside(x, -0.3, 0.3) - masked_array(data = [0.31 1.2 -- -- -0.4 -1.1], - mask = [False False True True False False], - fill_value=1e+20) + masked_array(data=[0.31, 1.2, --, --, -0.4, -1.1], + mask=[False, False, True, True, False, False], + fill_value=1e+20) The order of `v1` and `v2` doesn't matter. >>> ma.masked_inside(x, 0.3, -0.3) - masked_array(data = [0.31 1.2 -- -- -0.4 -1.1], - mask = [False False True True False False], - fill_value=1e+20) + masked_array(data=[0.31, 1.2, --, --, -0.4, -1.1], + mask=[False, False, True, True, False, False], + fill_value=1e+20) """ if v2 < v1: @@ -2163,16 +2163,16 @@ def masked_outside(x, v1, v2, copy=True): >>> import numpy.ma as ma >>> x = [0.31, 1.2, 0.01, 0.2, -0.4, -1.1] >>> ma.masked_outside(x, -0.3, 0.3) - masked_array(data = [-- -- 0.01 0.2 -- --], - mask = [ True True False False True True], - fill_value=1e+20) + masked_array(data=[--, --, 0.01, 0.2, --, --], + mask=[ True, True, False, False, True, True], + fill_value=1e+20) The order of `v1` and `v2` doesn't matter. >>> ma.masked_outside(x, 0.3, -0.3) - masked_array(data = [-- -- 0.01 0.2 -- --], - mask = [ True True False False True True], - fill_value=1e+20) + masked_array(data=[--, --, 0.01, 0.2, --, --], + mask=[ True, True, False, False, True, True], + fill_value=1e+20) """ if v2 < v1: @@ -2217,20 +2217,27 @@ def masked_object(x, value, copy=True, shrink=True): >>> food = np.array(['green_eggs', 'ham'], dtype=object) >>> # don't eat spoiled food >>> eat = ma.masked_object(food, 'green_eggs') - >>> print(eat) - [-- ham] + >>> eat + masked_array(data=[--, 'ham'], + mask=[ True, False], + fill_value='green_eggs', + dtype=object) >>> # plain ol` ham is boring >>> fresh_food = np.array(['cheese', 'ham', 'pineapple'], dtype=object) >>> eat = ma.masked_object(fresh_food, 'green_eggs') - >>> print(eat) - [cheese ham pineapple] + >>> eat + masked_array(data=['cheese', 'ham', 'pineapple'], + mask=False, + fill_value='green_eggs', + dtype=object) Note that `mask` is set to ``nomask`` if possible. >>> eat - masked_array(data = [cheese ham pineapple], - mask = False, - fill_value=?) + masked_array(data=['cheese', 'ham', 'pineapple'], + mask=False, + fill_value='green_eggs', + dtype=object) """ if isMaskedArray(x): @@ -2285,16 +2292,16 @@ def masked_values(x, value, rtol=1e-5, atol=1e-8, copy=True, shrink=True): >>> import numpy.ma as ma >>> x = np.array([1, 1.1, 2, 1.1, 3]) >>> ma.masked_values(x, 1.1) - masked_array(data = [1.0 -- 2.0 -- 3.0], - mask = [False True False True False], - fill_value=1.1) + masked_array(data=[1.0, --, 2.0, --, 3.0], + mask=[False, True, False, True, False], + fill_value=1.1) Note that `mask` is set to ``nomask`` if possible. >>> ma.masked_values(x, 1.5) - masked_array(data = [ 1. 1.1 2. 1.1 3. ], - mask = False, - fill_value=1.5) + masked_array(data=[1. , 1.1, 2. , 1.1, 3. ], + mask=False, + fill_value=1.5) For integers, the fill value will be different in general to the result of ``masked_equal``. @@ -2303,13 +2310,13 @@ def masked_values(x, value, rtol=1e-5, atol=1e-8, copy=True, shrink=True): >>> x array([0, 1, 2, 3, 4]) >>> ma.masked_values(x, 2) - masked_array(data = [0 1 -- 3 4], - mask = [False False True False False], - fill_value=2) + masked_array(data=[0, 1, --, 3, 4], + mask=[False, False, True, False, False], + fill_value=2) >>> ma.masked_equal(x, 2) - masked_array(data = [0 1 -- 3 4], - mask = [False False True False False], - fill_value=999999) + masked_array(data=[0, 1, --, 3, 4], + mask=[False, False, True, False, False], + fill_value=2) """ xnew = filled(x, value) @@ -2343,11 +2350,11 @@ def masked_invalid(a, copy=True): >>> a[2] = np.NaN >>> a[3] = np.PINF >>> a - array([ 0., 1., NaN, Inf, 4.]) + array([ 0., 1., nan, inf, 4.]) >>> ma.masked_invalid(a) - masked_array(data = [0.0 1.0 -- -- 4.0], - mask = [False False True True False], - fill_value=1e+20) + masked_array(data=[0.0, 1.0, --, --, 4.0], + mask=[False, False, True, True, False], + fill_value=1e+20) """ a = np.array(a, copy=copy, subok=True) @@ -2508,7 +2515,7 @@ def flatten_structured_array(a): -------- >>> ndtype = [('a', int), ('b', float)] >>> a = np.array([(1, 1), (2, 2)], dtype=ndtype) - >>> flatten_structured_array(a) + >>> np.ma.flatten_structured_array(a) array([[1., 1.], [2., 2.]]) @@ -2679,9 +2686,7 @@ class MaskedIterator(object): >>> fl.next() 3 >>> fl.next() - masked_array(data = --, - mask = True, - fill_value = 1e+20) + masked >>> fl.next() Traceback (most recent call last): File "<stdin>", line 1, in <module> @@ -3242,7 +3247,7 @@ class MaskedArray(ndarray): # Inherit attributes from self dout._update_from(self) # Check the fill_value - if isinstance(indx, basestring): + if is_string_or_list_of_strings(indx): if self._fill_value is not None: dout._fill_value = self._fill_value[indx] @@ -3546,6 +3551,11 @@ class MaskedArray(ndarray): array([[False, False], [False, False]]) >>> x.shrink_mask() + masked_array( + data=[[1, 2], + [3, 4]], + mask=False, + fill_value=999999) >>> x.mask False @@ -3634,7 +3644,7 @@ class MaskedArray(ndarray): -inf >>> x.set_fill_value(np.pi) >>> x.fill_value - 3.1415926535897931 + 3.1415926535897931 # may vary Reset to default: @@ -3683,9 +3693,9 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999) >>> x.filled() - array([1, 2, -999, 4, -999]) + array([ 1, 2, -999, 4, -999]) >>> type(x.filled()) - <type 'numpy.ndarray'> + <class 'numpy.ndarray'> Subclassing is preserved. This means that if, e.g., the data part of the masked array is a recarray, `filled` returns a recarray: @@ -3750,7 +3760,7 @@ class MaskedArray(ndarray): >>> x.compressed() array([0, 1]) >>> type(x.compressed()) - <type 'numpy.ndarray'> + <class 'numpy.ndarray'> """ data = ndarray.ravel(self._data) @@ -3792,25 +3802,29 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) >>> x.compress([1, 0, 1]) - masked_array(data = [1 3], - mask = [False False], - fill_value=999999) + masked_array(data=[1, 3], + mask=[False, False], + fill_value=999999) >>> x.compress([1, 0, 1], axis=1) - masked_array(data = - [[1 3] - [-- --] - [7 9]], - mask = - [[False False] - [ True True] - [False False]], - fill_value=999999) + masked_array( + data=[[1, 3], + [--, --], + [7, 9]], + mask=[[False, False], + [ True, True], + [False, False]], + fill_value=999999) """ # Get the basic components @@ -4014,6 +4028,16 @@ class MaskedArray(ndarray): check = check.view(type(self)) check._update_from(self) check._mask = mask + + # Cast fill value to bool_ if needed. If it cannot be cast, the + # default boolean fill value is used. + if check._fill_value is not None: + try: + fill = _check_fill_value(check._fill_value, np.bool_) + except (TypeError, ValueError): + fill = _check_fill_value(None, np.bool_) + check._fill_value = fill + return check def __eq__(self, other): @@ -4333,9 +4357,9 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array([1+1.j, -2j, 3.45+1.6j], mask=[False, True, False]) >>> x.get_imag() - masked_array(data = [1.0 -- 1.6], - mask = [False True False], - fill_value = 1e+20) + masked_array(data=[1.0, --, 1.6], + mask=[False, True, False], + fill_value=1e+20) """ result = self._data.imag.view(type(self)) @@ -4368,9 +4392,9 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array([1+1.j, -2j, 3.45+1.6j], mask=[False, True, False]) >>> x.get_real() - masked_array(data = [1.0 -- 3.45], - mask = [False True False], - fill_value = 1e+20) + masked_array(data=[1.0, --, 3.45], + mask=[False, True, False], + fill_value=1e+20) """ result = self._data.real.view(type(self)) @@ -4416,13 +4440,12 @@ class MaskedArray(ndarray): >>> a = ma.arange(6).reshape((2, 3)) >>> a[1, :] = ma.masked >>> a - masked_array(data = - [[0 1 2] - [-- -- --]], - mask = - [[False False False] - [ True True True]], - fill_value = 999999) + masked_array( + data=[[0, 1, 2], + [--, --, --]], + mask=[[False, False, False], + [ True, True, True]], + fill_value=999999) >>> a.count() 3 @@ -4507,12 +4530,20 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] - >>> print(x.ravel()) - [1 -- 3 -- 5 -- 7 -- 9] + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) + >>> x.ravel() + masked_array(data=[1, --, 3, --, 5, --, 7, --, 9], + mask=[False, True, False, True, False, True, False, True, + False], + fill_value=999999) """ r = ndarray.ravel(self._data, order=order).view(type(self)) @@ -4561,15 +4592,25 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2],[3,4]], mask=[1,0,0,1]) - >>> print(x) - [[-- 2] - [3 --]] + >>> x + masked_array( + data=[[--, 2], + [3, --]], + mask=[[ True, False], + [False, True]], + fill_value=999999) >>> x = x.reshape((4,1)) - >>> print(x) - [[--] - [2] - [3] - [--]] + >>> x + masked_array( + data=[[--], + [2], + [3], + [--]], + mask=[[ True], + [False], + [False], + [ True]], + fill_value=999999) """ kwargs.update(order=kwargs.get('order', 'C')) @@ -4626,21 +4667,36 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) >>> x.put([0,4,8],[10,20,30]) - >>> print(x) - [[10 -- 3] - [-- 20 --] - [7 -- 30]] + >>> x + masked_array( + data=[[10, --, 3], + [--, 20, --], + [7, --, 30]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) >>> x.put(4,999) - >>> print(x) - [[10 -- 3] - [-- 999 --] - [7 -- 30]] + >>> x + masked_array( + data=[[10, --, 3], + [--, 999, --], + [7, --, 30]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) """ # Hard mask: Get rid of the values/indices that fall on masked data @@ -4680,14 +4736,14 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array([1, 2, 3], mask=[0, 1, 1]) >>> x.ids() - (166670640, 166659832) + (166670640, 166659832) # may vary If the array has no mask, the address of `nomask` is returned. This address is typically not close to the data in memory: >>> x = np.ma.array([1, 2, 3]) >>> x.ids() - (166691080, 3083169284L) + (166691080, 3083169284L) # may vary """ if self._mask is nomask: @@ -4836,13 +4892,12 @@ class MaskedArray(ndarray): >>> import numpy.ma as ma >>> x = ma.array(np.eye(3)) >>> x - masked_array(data = - [[ 1. 0. 0.] - [ 0. 1. 0.] - [ 0. 0. 1.]], - mask = - False, - fill_value=1e+20) + masked_array( + data=[[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]], + mask=False, + fill_value=1e+20) >>> x.nonzero() (array([0, 1, 2]), array([0, 1, 2])) @@ -4850,15 +4905,14 @@ class MaskedArray(ndarray): >>> x[1, 1] = ma.masked >>> x - masked_array(data = - [[1.0 0.0 0.0] - [0.0 -- 0.0] - [0.0 0.0 1.0]], - mask = - [[False False False] - [False True False] - [False False False]], - fill_value=1e+20) + masked_array( + data=[[1.0, 0.0, 0.0], + [0.0, --, 0.0], + [0.0, 0.0, 1.0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1e+20) >>> x.nonzero() (array([0, 2]), array([0, 2])) @@ -4875,13 +4929,12 @@ class MaskedArray(ndarray): >>> a = ma.array([[1,2,3],[4,5,6],[7,8,9]]) >>> a > 3 - masked_array(data = - [[False False False] - [ True True True] - [ True True True]], - mask = - False, - fill_value=999999) + masked_array( + data=[[False, False, False], + [ True, True, True], + [ True, True, True]], + mask=False, + fill_value=True) >>> ma.nonzero(a > 3) (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) @@ -4963,18 +5016,27 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] - >>> print(x.sum()) + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) + >>> x.sum() 25 - >>> print(x.sum(axis=1)) - [4 5 16] - >>> print(x.sum(axis=0)) - [8 5 12] + >>> x.sum(axis=1) + masked_array(data=[4, 5, 16], + mask=[False, False, False], + fill_value=999999) + >>> x.sum(axis=0) + masked_array(data=[8, 5, 12], + mask=[False, False, False], + fill_value=999999) >>> print(type(x.sum(axis=0, dtype=np.int64)[0])) - <type 'numpy.int64'> + <class 'numpy.int64'> """ kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims} @@ -5025,8 +5087,11 @@ class MaskedArray(ndarray): Examples -------- >>> marr = np.ma.array(np.arange(10), mask=[0,0,0,1,1,1,0,0,0,0]) - >>> print(marr.cumsum()) - [0 1 3 -- -- -- 9 16 24 33] + >>> marr.cumsum() + masked_array(data=[0, 1, 3, --, --, --, 9, 16, 24, 33], + mask=[False, False, False, True, True, True, False, False, + False, False], + fill_value=999999) """ result = self.filled(0).cumsum(axis=axis, dtype=dtype, out=out) @@ -5130,9 +5195,9 @@ class MaskedArray(ndarray): -------- >>> a = np.ma.array([1,2,3], mask=[False, False, True]) >>> a - masked_array(data = [1 2 --], - mask = [False False True], - fill_value = 999999) + masked_array(data=[1, 2, --], + mask=[False, False, True], + fill_value=999999) >>> a.mean() 1.5 @@ -5185,9 +5250,9 @@ class MaskedArray(ndarray): -------- >>> a = np.ma.array([1,2,3]) >>> a.anom() - masked_array(data = [-1. 0. 1.], - mask = False, - fill_value = 1e+20) + masked_array(data=[-1., 0., 1.], + mask=False, + fill_value=1e+20) """ m = self.mean(axis, dtype) @@ -5367,9 +5432,9 @@ class MaskedArray(ndarray): -------- >>> a = np.ma.array([3,2,1], mask=[False, False, True]) >>> a - masked_array(data = [3 2 --], - mask = [False False True], - fill_value = 999999) + masked_array(data=[3, 2, --], + mask=[False, False, True], + fill_value=999999) >>> a.argsort() array([1, 0, 2]) @@ -5417,15 +5482,19 @@ class MaskedArray(ndarray): Examples -------- - >>> x = np.ma.array(arange(4), mask=[1,1,0,0]) + >>> x = np.ma.array(np.arange(4), mask=[1,1,0,0]) >>> x.shape = (2,2) - >>> print(x) - [[-- --] - [2 3]] - >>> print(x.argmin(axis=0, fill_value=-1)) - [0 0] - >>> print(x.argmin(axis=0, fill_value=9)) - [1 1] + >>> x + masked_array( + data=[[--, --], + [2, 3]], + mask=[[ True, True], + [False, False]], + fill_value=999999) + >>> x.argmin(axis=0, fill_value=-1) + array([0, 0]) + >>> x.argmin(axis=0, fill_value=9) + array([1, 1]) """ if fill_value is None: @@ -5516,23 +5585,29 @@ class MaskedArray(ndarray): Examples -------- - >>> a = ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) + >>> a = np.ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) >>> # Default >>> a.sort() - >>> print(a) - [1 3 5 -- --] + >>> a + masked_array(data=[1, 3, 5, --, --], + mask=[False, False, False, True, True], + fill_value=999999) - >>> a = ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) + >>> a = np.ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) >>> # Put missing values in the front >>> a.sort(endwith=False) - >>> print(a) - [-- -- 1 3 5] + >>> a + masked_array(data=[--, --, 1, 3, 5], + mask=[ True, True, False, False, False], + fill_value=999999) - >>> a = ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) + >>> a = np.ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) >>> # fill_value takes over endwith >>> a.sort(endwith=False, fill_value=3) - >>> print(a) - [1 -- -- 3 5] + >>> a + masked_array(data=[1, --, --, 3, 5], + mask=[False, True, True, False, False], + fill_value=999999) """ if self._mask is nomask: @@ -5638,27 +5713,36 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array(np.arange(6), mask=[0 ,1, 0, 0, 0 ,1]).reshape(3, 2) - >>> print(x) - [[0 --] - [2 3] - [4 --]] + >>> x + masked_array( + data=[[0, --], + [2, 3], + [4, --]], + mask=[[False, True], + [False, False], + [False, True]], + fill_value=999999) >>> x.mini() - 0 + masked_array(data=0, + mask=False, + fill_value=999999) >>> x.mini(axis=0) - masked_array(data = [0 3], - mask = [False False], - fill_value = 999999) - >>> print(x.mini(axis=1)) - [0 2 4] + masked_array(data=[0, 3], + mask=[False, False], + fill_value=999999) + >>> x.mini(axis=1) + masked_array(data=[0, 2, 4], + mask=[False, False, False], + fill_value=999999) There is a small difference between `mini` and `min`: >>> x[:,1].mini(axis=0) - masked_array(data = --, - mask = True, - fill_value = 999999) + masked_array(data=3, + mask=False, + fill_value=999999) >>> x[:,1].min(axis=0) - masked + 3 """ # 2016-04-13, 1.13.0, gh-8764 @@ -5911,7 +5995,7 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array(np.array([[1, 2], [3, 4]]), mask=[[0, 1], [1, 0]]) >>> x.tobytes() - '\\x01\\x00\\x00\\x00?B\\x0f\\x00?B\\x0f\\x00\\x04\\x00\\x00\\x00' + b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00?B\\x0f\\x00\\x00\\x00\\x00\\x00?B\\x0f\\x00\\x00\\x00\\x00\\x00\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00' """ return self.filled(fill_value).tobytes(order=order) @@ -5959,14 +6043,20 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] - >>> print(x.toflex()) - [[(1, False) (2, True) (3, False)] - [(4, True) (5, False) (6, True)] - [(7, False) (8, True) (9, False)]] + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) + >>> x.toflex() + array([[(1, False), (2, True), (3, False)], + [(4, True), (5, False), (6, True)], + [(7, False), (8, True), (9, False)]], + dtype=[('_data', '<i8'), ('_mask', '?')]) """ # Get the basic dtype. @@ -6213,15 +6303,14 @@ def isMaskedArray(x): [ 0., 0., 1.]]) >>> m = ma.masked_values(a, 0) >>> m - masked_array(data = - [[1.0 -- --] - [-- 1.0 --] - [-- -- 1.0]], - mask = - [[False True True] - [ True False True] - [ True True False]], - fill_value=0.0) + masked_array( + data=[[1.0, --, --], + [--, 1.0, --], + [--, --, 1.0]], + mask=[[False, True, True], + [ True, False, True], + [ True, True, False]], + fill_value=0.0) >>> ma.isMaskedArray(a) False >>> ma.isMaskedArray(m) @@ -6325,7 +6414,7 @@ class MaskedConstant(MaskedArray): def __copy__(self): return self - + def __deepcopy__(self, memo): return self @@ -6385,16 +6474,16 @@ def is_masked(x): >>> import numpy.ma as ma >>> x = ma.masked_equal([0, 1, 0, 2, 3], 0) >>> x - masked_array(data = [-- 1 -- 2 3], - mask = [ True False True False False], - fill_value=999999) + masked_array(data=[--, 1, --, 2, 3], + mask=[ True, False, True, False, False], + fill_value=0) >>> ma.is_masked(x) True >>> x = ma.masked_equal([0, 1, 0, 2, 3], 42) >>> x - masked_array(data = [0 1 0 2 3], - mask = False, - fill_value=999999) + masked_array(data=[0, 1, 0, 2, 3], + mask=False, + fill_value=42) >>> ma.is_masked(x) False @@ -6744,17 +6833,17 @@ def concatenate(arrays, axis=0): >>> a[1] = ma.masked >>> b = ma.arange(2, 5) >>> a - masked_array(data = [0 -- 2], - mask = [False True False], - fill_value = 999999) + masked_array(data=[0, --, 2], + mask=[False, True, False], + fill_value=999999) >>> b - masked_array(data = [2 3 4], - mask = False, - fill_value = 999999) + masked_array(data=[2, 3, 4], + mask=False, + fill_value=999999) >>> ma.concatenate([a, b]) - masked_array(data = [0 -- 2 2 3 4], - mask = [False True False False False False], - fill_value = 999999) + masked_array(data=[0, --, 2, 2, 3, 4], + mask=[False, True, False, False, False, False], + fill_value=999999) """ d = np.concatenate([getdata(a) for a in arrays], axis) @@ -6909,24 +6998,21 @@ def transpose(a, axes=None): >>> import numpy.ma as ma >>> x = ma.arange(4).reshape((2,2)) >>> x[1, 1] = ma.masked - >>>> x - masked_array(data = - [[0 1] - [2 --]], - mask = - [[False False] - [False True]], - fill_value = 999999) + >>> x + masked_array( + data=[[0, 1], + [2, --]], + mask=[[False, False], + [False, True]], + fill_value=999999) >>> ma.transpose(x) - masked_array(data = - [[0 2] - [1 --]], - mask = - [[False False] - [False True]], - fill_value = 999999) - + masked_array( + data=[[0, 2], + [1, --]], + mask=[[False, False], + [False, True]], + fill_value=999999) """ # We can't use 'frommethod', as 'transpose' doesn't take keywords try: @@ -6973,39 +7059,39 @@ def resize(x, new_shape): >>> a = ma.array([[1, 2] ,[3, 4]]) >>> a[0, 1] = ma.masked >>> a - masked_array(data = - [[1 --] - [3 4]], - mask = - [[False True] - [False False]], - fill_value = 999999) + masked_array( + data=[[1, --], + [3, 4]], + mask=[[False, True], + [False, False]], + fill_value=999999) >>> np.resize(a, (3, 3)) - array([[1, 2, 3], - [4, 1, 2], - [3, 4, 1]]) + masked_array( + data=[[1, 2, 3], + [4, 1, 2], + [3, 4, 1]], + mask=False, + fill_value=999999) >>> ma.resize(a, (3, 3)) - masked_array(data = - [[1 -- 3] - [4 1 --] - [3 4 1]], - mask = - [[False True False] - [False False True] - [False False False]], - fill_value = 999999) + masked_array( + data=[[1, --, 3], + [4, 1, --], + [3, 4, 1]], + mask=[[False, True, False], + [False, False, True], + [False, False, False]], + fill_value=999999) A MaskedArray is always returned, regardless of the input type. >>> a = np.array([[1, 2] ,[3, 4]]) >>> ma.resize(a, (3, 3)) - masked_array(data = - [[1 2 3] - [4 1 2] - [3 4 1]], - mask = - False, - fill_value = 999999) + masked_array( + data=[[1, 2, 3], + [4, 1, 2], + [3, 4, 1]], + mask=False, + fill_value=999999) """ # We can't use _frommethods here, as N.resize is notoriously whiny. @@ -7074,7 +7160,7 @@ def where(condition, x=_NoValue, y=_NoValue): Parameters ---------- condition : array_like, bool - Where True, yield `x`, otherwise yield `y`. + Where True, yield `x`, otherwise yield `y`. x, y : array_like, optional Values from which to choose. `x`, `y` and `condition` need to be broadcastable to some shape. @@ -7096,14 +7182,24 @@ def where(condition, x=_NoValue, y=_NoValue): >>> x = np.ma.array(np.arange(9.).reshape(3, 3), mask=[[0, 1, 0], ... [1, 0, 1], ... [0, 1, 0]]) - >>> print(x) - [[0.0 -- 2.0] - [-- 4.0 --] - [6.0 -- 8.0]] - >>> print(np.ma.where(x > 5, x, -3.1416)) - [[-3.1416 -- -3.1416] - [-- -3.1416 --] - [6.0 -- 8.0]] + >>> x + masked_array( + data=[[0.0, --, 2.0], + [--, 4.0, --], + [6.0, --, 8.0]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=1e+20) + >>> np.ma.where(x > 5, x, -3.1416) + masked_array( + data=[[-3.1416, --, -3.1416], + [--, -3.1416, --], + [6.0, --, 8.0]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=1e+20) """ @@ -7183,9 +7279,9 @@ def choose(indices, choices, out=None, mode='raise'): >>> choice = np.array([[1,1,1], [2,2,2], [3,3,3]]) >>> a = np.array([2, 1, 0]) >>> np.ma.choose(a, choice) - masked_array(data = [3 2 1], - mask = False, - fill_value=999999) + masked_array(data=[3, 2, 1], + mask=False, + fill_value=999999) """ def fmask(x): @@ -7308,25 +7404,23 @@ def mask_rowcols(a, axis=None): [0, 0, 0]]) >>> a = ma.masked_equal(a, 1) >>> a - masked_array(data = - [[0 0 0] - [0 -- 0] - [0 0 0]], - mask = - [[False False False] - [False True False] - [False False False]], - fill_value=999999) + masked_array( + data=[[0, 0, 0], + [0, --, 0], + [0, 0, 0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1) >>> ma.mask_rowcols(a) - masked_array(data = - [[0 -- 0] - [-- -- --] - [0 -- 0]], - mask = - [[False True False] - [ True True True] - [False True False]], - fill_value=999999) + masked_array( + data=[[0, --, 0], + [--, --, --], + [0, --, 0]], + mask=[[False, True, False], + [ True, True, True], + [False, True, False]], + fill_value=1) """ a = array(a, subok=False) @@ -7387,24 +7481,22 @@ def dot(a, b, strict=False, out=None): Examples -------- - >>> a = ma.array([[1, 2, 3], [4, 5, 6]], mask=[[1, 0, 0], [0, 0, 0]]) - >>> b = ma.array([[1, 2], [3, 4], [5, 6]], mask=[[1, 0], [0, 0], [0, 0]]) + >>> a = np.ma.array([[1, 2, 3], [4, 5, 6]], mask=[[1, 0, 0], [0, 0, 0]]) + >>> b = np.ma.array([[1, 2], [3, 4], [5, 6]], mask=[[1, 0], [0, 0], [0, 0]]) >>> np.ma.dot(a, b) - masked_array(data = - [[21 26] - [45 64]], - mask = - [[False False] - [False False]], - fill_value = 999999) + masked_array( + data=[[21, 26], + [45, 64]], + mask=[[False, False], + [False, False]], + fill_value=999999) >>> np.ma.dot(a, b, strict=True) - masked_array(data = - [[-- --] - [-- 64]], - mask = - [[ True True] - [ True False]], - fill_value = 999999) + masked_array( + data=[[--, --], + [--, 64]], + mask=[[ True, True], + [ True, False]], + fill_value=999999) """ # !!!: Works only with 2D arrays. There should be a way to get it to run @@ -7572,18 +7664,18 @@ def allequal(a, b, fill_value=True): Examples -------- - >>> a = ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1]) + >>> a = np.ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1]) >>> a - masked_array(data = [10000000000.0 1e-07 --], - mask = [False False True], - fill_value=1e+20) + masked_array(data=[10000000000.0, 1e-07, --], + mask=[False, False, True], + fill_value=1e+20) - >>> b = array([1e10, 1e-7, -42.0]) + >>> b = np.array([1e10, 1e-7, -42.0]) >>> b array([ 1.00000000e+10, 1.00000000e-07, -4.20000000e+01]) - >>> ma.allequal(a, b, fill_value=False) + >>> np.ma.allequal(a, b, fill_value=False) False - >>> ma.allequal(a, b) + >>> np.ma.allequal(a, b) True """ @@ -7649,29 +7741,29 @@ def allclose(a, b, masked_equal=True, rtol=1e-5, atol=1e-8): Examples -------- - >>> a = ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1]) + >>> a = np.ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1]) >>> a - masked_array(data = [10000000000.0 1e-07 --], - mask = [False False True], - fill_value = 1e+20) - >>> b = ma.array([1e10, 1e-8, -42.0], mask=[0, 0, 1]) - >>> ma.allclose(a, b) + masked_array(data=[10000000000.0, 1e-07, --], + mask=[False, False, True], + fill_value=1e+20) + >>> b = np.ma.array([1e10, 1e-8, -42.0], mask=[0, 0, 1]) + >>> np.ma.allclose(a, b) False - >>> a = ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1]) - >>> b = ma.array([1.00001e10, 1e-9, -42.0], mask=[0, 0, 1]) - >>> ma.allclose(a, b) + >>> a = np.ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1]) + >>> b = np.ma.array([1.00001e10, 1e-9, -42.0], mask=[0, 0, 1]) + >>> np.ma.allclose(a, b) True - >>> ma.allclose(a, b, masked_equal=False) + >>> np.ma.allclose(a, b, masked_equal=False) False Masked values are not compared directly. - >>> a = ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1]) - >>> b = ma.array([1.00001e10, 1e-9, 42.0], mask=[0, 0, 1]) - >>> ma.allclose(a, b) + >>> a = np.ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1]) + >>> b = np.ma.array([1.00001e10, 1e-9, 42.0], mask=[0, 0, 1]) + >>> np.ma.allclose(a, b) True - >>> ma.allclose(a, b, masked_equal=False) + >>> np.ma.allclose(a, b, masked_equal=False) False """ @@ -7738,15 +7830,14 @@ def asarray(a, dtype=None, order=None): -------- >>> x = np.arange(10.).reshape(2, 5) >>> x - array([[ 0., 1., 2., 3., 4.], - [ 5., 6., 7., 8., 9.]]) + array([[0., 1., 2., 3., 4.], + [5., 6., 7., 8., 9.]]) >>> np.ma.asarray(x) - masked_array(data = - [[ 0. 1. 2. 3. 4.] - [ 5. 6. 7. 8. 9.]], - mask = - False, - fill_value = 1e+20) + masked_array( + data=[[0., 1., 2., 3., 4.], + [5., 6., 7., 8., 9.]], + mask=False, + fill_value=1e+20) >>> type(np.ma.asarray(x)) <class 'numpy.ma.core.MaskedArray'> @@ -7786,15 +7877,14 @@ def asanyarray(a, dtype=None): -------- >>> x = np.arange(10.).reshape(2, 5) >>> x - array([[ 0., 1., 2., 3., 4.], - [ 5., 6., 7., 8., 9.]]) + array([[0., 1., 2., 3., 4.], + [5., 6., 7., 8., 9.]]) >>> np.ma.asanyarray(x) - masked_array(data = - [[ 0. 1. 2. 3. 4.] - [ 5. 6. 7. 8. 9.]], - mask = - False, - fill_value = 1e+20) + masked_array( + data=[[0., 1., 2., 3., 4.], + [5., 6., 7., 8., 9.]], + mask=False, + fill_value=1e+20) >>> type(np.ma.asanyarray(x)) <class 'numpy.ma.core.MaskedArray'> @@ -7938,39 +8028,38 @@ def fromflex(fxarray): >>> x = np.ma.array(np.arange(9).reshape(3, 3), mask=[0] + [1, 0] * 4) >>> rec = x.toflex() >>> rec - array([[(0, False), (1, True), (2, False)], - [(3, True), (4, False), (5, True)], - [(6, False), (7, True), (8, False)]], - dtype=[('_data', '<i4'), ('_mask', '|b1')]) + array([[(0, False), (1, True), (2, False)], + [(3, True), (4, False), (5, True)], + [(6, False), (7, True), (8, False)]], + dtype=[('_data', '<i8'), ('_mask', '?')]) >>> x2 = np.ma.fromflex(rec) >>> x2 - masked_array(data = - [[0 -- 2] - [-- 4 --] - [6 -- 8]], - mask = - [[False True False] - [ True False True] - [False True False]], - fill_value = 999999) + masked_array( + data=[[0, --, 2], + [--, 4, --], + [6, --, 8]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) Extra fields can be present in the structured array but are discarded: >>> dt = [('_data', '<i4'), ('_mask', '|b1'), ('field3', '<f4')] >>> rec2 = np.zeros((2, 2), dtype=dt) >>> rec2 - array([[(0, False, 0.0), (0, False, 0.0)], - [(0, False, 0.0), (0, False, 0.0)]], - dtype=[('_data', '<i4'), ('_mask', '|b1'), ('field3', '<f4')]) + array([[(0, False, 0.), (0, False, 0.)], + [(0, False, 0.), (0, False, 0.)]], + dtype=[('_data', '<i4'), ('_mask', '?'), ('field3', '<f4')]) >>> y = np.ma.fromflex(rec2) >>> y - masked_array(data = - [[0 0] - [0 0]], - mask = - [[False False] - [False False]], - fill_value = 999999) + masked_array( + data=[[0, 0], + [0, 0]], + mask=[[False, False], + [False, False]], + fill_value=999999, + dtype=int32) """ return masked_array(fxarray['_data'], mask=fxarray['_mask']) @@ -8071,7 +8160,10 @@ def append(a, b, axis=None): >>> import numpy.ma as ma >>> a = ma.masked_values([1, 2, 3], 2) >>> b = ma.masked_values([[4, 5, 6], [7, 8, 9]], 7) - >>> print(ma.append(a, b)) - [1 -- 3 4 5 6 -- 8 9] + >>> ma.append(a, b) + masked_array(data=[1, --, 3, 4, 5, 6, --, 8, 9], + mask=[False, True, False, False, False, False, True, False, + False], + fill_value=999999) """ return concatenate([a, b], axis) diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py index 3be4d3625..2e3b84e1c 100644 --- a/numpy/ma/extras.py +++ b/numpy/ma/extras.py @@ -81,15 +81,14 @@ def count_masked(arr, axis=None): >>> a[1, 2] = ma.masked >>> a[2, 1] = ma.masked >>> a - masked_array(data = - [[0 1 2] - [-- 4 --] - [6 -- 8]], - mask = - [[False False False] - [ True False True] - [False True False]], - fill_value=999999) + masked_array( + data=[[0, 1, 2], + [--, 4, --], + [6, --, 8]], + mask=[[False, False, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) >>> ma.count_masked(a) 3 @@ -132,15 +131,15 @@ def masked_all(shape, dtype=float): -------- >>> import numpy.ma as ma >>> ma.masked_all((3, 3)) - masked_array(data = - [[-- -- --] - [-- -- --] - [-- -- --]], - mask = - [[ True True True] - [ True True True] - [ True True True]], - fill_value=1e+20) + masked_array( + data=[[--, --, --], + [--, --, --], + [--, --, --]], + mask=[[ True, True, True], + [ True, True, True], + [ True, True, True]], + fill_value=1e+20, + dtype=float64) The `dtype` parameter defines the underlying data type. @@ -188,16 +187,16 @@ def masked_all_like(arr): >>> import numpy.ma as ma >>> arr = np.zeros((2, 3), dtype=np.float32) >>> arr - array([[ 0., 0., 0.], - [ 0., 0., 0.]], dtype=float32) + array([[0., 0., 0.], + [0., 0., 0.]], dtype=float32) >>> ma.masked_all_like(arr) - masked_array(data = - [[-- -- --] - [-- -- --]], - mask = - [[ True True True] - [ True True True]], - fill_value=1e+20) + masked_array( + data=[[--, --, --], + [--, --, --]], + mask=[[ True, True, True], + [ True, True, True]], + fill_value=1e+20, + dtype=float32) The dtype of the masked array matches the dtype of `arr`. @@ -492,28 +491,45 @@ if apply_over_axes.__doc__ is not None: Examples -------- - >>> a = ma.arange(24).reshape(2,3,4) - >>> a[:,0,1] = ma.masked - >>> a[:,1,:] = ma.masked - >>> print(a) - [[[0 -- 2 3] - [-- -- -- --] - [8 9 10 11]] - - [[12 -- 14 15] - [-- -- -- --] - [20 21 22 23]]] - >>> print(ma.apply_over_axes(ma.sum, a, [0,2])) - [[[46] - [--] - [124]]] + >>> a = np.ma.arange(24).reshape(2,3,4) + >>> a[:,0,1] = np.ma.masked + >>> a[:,1,:] = np.ma.masked + >>> a + masked_array( + data=[[[0, --, 2, 3], + [--, --, --, --], + [8, 9, 10, 11]], + [[12, --, 14, 15], + [--, --, --, --], + [20, 21, 22, 23]]], + mask=[[[False, True, False, False], + [ True, True, True, True], + [False, False, False, False]], + [[False, True, False, False], + [ True, True, True, True], + [False, False, False, False]]], + fill_value=999999) + >>> np.ma.apply_over_axes(np.ma.sum, a, [0,2]) + masked_array( + data=[[[46], + [--], + [124]]], + mask=[[[False], + [ True], + [False]]], + fill_value=999999) Tuple axis arguments to ufuncs are equivalent: - >>> print(ma.sum(a, axis=(0,2)).reshape((1,-1,1))) - [[[46] - [--] - [124]]] + >>> np.ma.sum(a, axis=(0,2)).reshape((1,-1,1)) + masked_array( + data=[[[46], + [--], + [124]]], + mask=[[[False], + [ True], + [False]]], + fill_value=999999) """ @@ -558,14 +574,19 @@ def average(a, axis=None, weights=None, returned=False): 1.25 >>> x = np.ma.arange(6.).reshape(3, 2) - >>> print(x) - [[ 0. 1.] - [ 2. 3.] - [ 4. 5.]] + >>> x + masked_array( + data=[[0., 1.], + [2., 3.], + [4., 5.]], + mask=False, + fill_value=1e+20) >>> avg, sumweights = np.ma.average(x, axis=0, weights=[1, 2, 3], ... returned=True) - >>> print(avg) - [2.66666666667 3.66666666667] + >>> avg + masked_array(data=[2.6666666666666665, 3.6666666666666665], + mask=[False, False], + fill_value=1e+20) """ a = asarray(a) @@ -676,9 +697,9 @@ def median(a, axis=None, out=None, overwrite_input=False, keepdims=False): >>> np.ma.median(x) 2.5 >>> np.ma.median(x, axis=-1, overwrite_input=True) - masked_array(data = [ 2. 5.], - mask = False, - fill_value = 1e+20) + masked_array(data=[2.0, 5.0], + mask=[False, False], + fill_value=1e+20) """ if not hasattr(a, 'mask'): @@ -856,15 +877,14 @@ def compress_rowcols(x, axis=None): ... [1, 0, 0], ... [0, 0, 0]]) >>> x - masked_array(data = - [[-- 1 2] - [-- 4 5] - [6 7 8]], - mask = - [[ True False False] - [ True False False] - [False False False]], - fill_value = 999999) + masked_array( + data=[[--, 1, 2], + [--, 4, 5], + [6, 7, 8]], + mask=[[ True, False, False], + [ True, False, False], + [False, False, False]], + fill_value=999999) >>> np.ma.compress_rowcols(x) array([[7, 8]]) @@ -937,25 +957,24 @@ def mask_rows(a, axis=None): [0, 0, 0]]) >>> a = ma.masked_equal(a, 1) >>> a - masked_array(data = - [[0 0 0] - [0 -- 0] - [0 0 0]], - mask = - [[False False False] - [False True False] - [False False False]], - fill_value=999999) + masked_array( + data=[[0, 0, 0], + [0, --, 0], + [0, 0, 0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1) + >>> ma.mask_rows(a) - masked_array(data = - [[0 0 0] - [-- -- --] - [0 0 0]], - mask = - [[False False False] - [ True True True] - [False False False]], - fill_value=999999) + masked_array( + data=[[0, 0, 0], + [--, --, --], + [0, 0, 0]], + mask=[[False, False, False], + [ True, True, True], + [False, False, False]], + fill_value=1) """ return mask_rowcols(a, 0) @@ -982,25 +1001,23 @@ def mask_cols(a, axis=None): [0, 0, 0]]) >>> a = ma.masked_equal(a, 1) >>> a - masked_array(data = - [[0 0 0] - [0 -- 0] - [0 0 0]], - mask = - [[False False False] - [False True False] - [False False False]], - fill_value=999999) + masked_array( + data=[[0, 0, 0], + [0, --, 0], + [0, 0, 0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1) >>> ma.mask_cols(a) - masked_array(data = - [[0 -- 0] - [0 -- 0] - [0 -- 0]], - mask = - [[False True False] - [False True False] - [False True False]], - fill_value=999999) + masked_array( + data=[[0, --, 0], + [0, --, 0], + [0, --, 0]], + mask=[[False, True, False], + [False, True, False], + [False, True, False]], + fill_value=1) """ return mask_rowcols(a, 1) @@ -1078,12 +1095,12 @@ def intersect1d(ar1, ar2, assume_unique=False): Examples -------- - >>> x = array([1, 3, 3, 3], mask=[0, 0, 0, 1]) - >>> y = array([3, 1, 1, 1], mask=[0, 0, 0, 1]) - >>> intersect1d(x, y) - masked_array(data = [1 3 --], - mask = [False False True], - fill_value = 999999) + >>> x = np.ma.array([1, 3, 3, 3], mask=[0, 0, 0, 1]) + >>> y = np.ma.array([3, 1, 1, 1], mask=[0, 0, 0, 1]) + >>> np.ma.intersect1d(x, y) + masked_array(data=[1, 3, --], + mask=[False, False, True], + fill_value=999999) """ if assume_unique: @@ -1216,9 +1233,9 @@ def setdiff1d(ar1, ar2, assume_unique=False): -------- >>> x = np.ma.array([1, 2, 3, 4], mask=[0, 1, 0, 1]) >>> np.ma.setdiff1d(x, [1, 2]) - masked_array(data = [3 --], - mask = [False True], - fill_value = 999999) + masked_array(data=[3, --], + mask=[False, True], + fill_value=999999) """ if assume_unique: @@ -1483,7 +1500,9 @@ class mr_class(MAxisConcatenator): Examples -------- >>> np.ma.mr_[np.ma.array([1,2,3]), 0, 0, np.ma.array([4,5,6])] - array([1, 2, 3, 0, 0, 4, 5, 6]) + masked_array(data=[1, 2, 3, ..., 4, 5, 6], + mask=False, + fill_value=999999) """ def __init__(self): @@ -1524,19 +1543,19 @@ def flatnotmasked_edges(a): Examples -------- >>> a = np.ma.arange(10) - >>> flatnotmasked_edges(a) - [0,-1] + >>> np.ma.flatnotmasked_edges(a) + array([0, 9]) >>> mask = (a < 3) | (a > 8) | (a == 5) >>> a[mask] = np.ma.masked >>> np.array(a[~a.mask]) array([3, 4, 6, 7, 8]) - >>> flatnotmasked_edges(a) + >>> np.ma.flatnotmasked_edges(a) array([3, 8]) >>> a[:] = np.ma.masked - >>> print(flatnotmasked_edges(ma)) + >>> print(np.ma.flatnotmasked_edges(a)) None """ @@ -1588,7 +1607,7 @@ def notmasked_edges(a, axis=None): >>> np.array(am[~am.mask]) array([0, 1, 2, 3, 6]) - >>> np.ma.notmasked_edges(ma) + >>> np.ma.notmasked_edges(am) array([0, 6]) """ @@ -1709,15 +1728,11 @@ def notmasked_contiguous(a, axis=None): [slice(0, 1, None), slice(2, 4, None), slice(7, 9, None), slice(11, 12, None)] >>> np.ma.notmasked_contiguous(ma, axis=0) - [[slice(0, 1, None), slice(2, 3, None)], # column broken into two segments - [], # fully masked column - [slice(0, 1, None)], - [slice(0, 3, None)]] + [[slice(0, 1, None), slice(2, 3, None)], [], [slice(0, 1, None)], [slice(0, 3, None)]] >>> np.ma.notmasked_contiguous(ma, axis=1) - [[slice(0, 1, None), slice(2, 4, None)], # row broken into two segments - [slice(3, 4, None)], - [slice(0, 1, None), slice(3, 4, None)]] + [[slice(0, 1, None), slice(2, 4, None)], [slice(3, 4, None)], [slice(0, 1, None), slice(3, 4, None)]] + """ a = asarray(a) nd = a.ndim diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 50ab6e1de..e0dbf1b1a 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -26,7 +26,7 @@ from numpy.testing import ( assert_raises, assert_warns, suppress_warnings ) from numpy import ndarray -from numpy.compat import asbytes, asbytes_nested +from numpy.compat import asbytes from numpy.ma.testutils import ( assert_, assert_array_equal, assert_equal, assert_almost_equal, assert_equal_records, fail_if_equal, assert_not_equal, @@ -60,6 +60,11 @@ suppress_copy_mask_on_assignment.filter( "setting an item on a masked array which has a shared mask will not copy") +# For parametrized numeric testing +num_dts = [np.dtype(dt_) for dt_ in '?bhilqBHILQefdgFD'] +num_ids = [dt_.char for dt_ in num_dts] + + class TestMaskedArray(object): # Base test class for MaskedArrays. @@ -228,7 +233,7 @@ class TestMaskedArray(object): x = np.array([('A', 0)], dtype={'names':['f0','f1'], 'formats':['S4','i8'], 'offsets':[0,8]}) - data = array(x) # used to fail due to 'V' padding field in x.dtype.descr + array(x) # used to fail due to 'V' padding field in x.dtype.descr def test_asarray(self): (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d @@ -1415,23 +1420,34 @@ class TestMaskedArrayArithmetic(object): # Test the equality of structured arrays ndtype = [('A', int), ('B', int)] a = array([(1, 1), (2, 2)], mask=[(0, 1), (0, 0)], dtype=ndtype) + test = (a == a) assert_equal(test.data, [True, True]) assert_equal(test.mask, [False, False]) + assert_(test.fill_value == True) + test = (a == a[0]) assert_equal(test.data, [True, False]) assert_equal(test.mask, [False, False]) + assert_(test.fill_value == True) + b = array([(1, 1), (2, 2)], mask=[(1, 0), (0, 0)], dtype=ndtype) test = (a == b) assert_equal(test.data, [False, True]) assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + test = (a[0] == b) assert_equal(test.data, [False, False]) assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + b = array([(1, 1), (2, 2)], mask=[(0, 1), (1, 0)], dtype=ndtype) test = (a == b) assert_equal(test.data, [True, True]) assert_equal(test.mask, [False, False]) + assert_(test.fill_value == True) + # complicated dtype, 2-dimensional array. ndtype = [('A', int), ('B', [('BA', int), ('BB', int)])] a = array([[(1, (1, 1)), (2, (2, 2))], @@ -1441,28 +1457,40 @@ class TestMaskedArrayArithmetic(object): test = (a[0, 0] == a) assert_equal(test.data, [[True, False], [False, False]]) assert_equal(test.mask, [[False, False], [False, True]]) + assert_(test.fill_value == True) def test_ne_on_structured(self): # Test the equality of structured arrays ndtype = [('A', int), ('B', int)] a = array([(1, 1), (2, 2)], mask=[(0, 1), (0, 0)], dtype=ndtype) + test = (a != a) assert_equal(test.data, [False, False]) assert_equal(test.mask, [False, False]) + assert_(test.fill_value == True) + test = (a != a[0]) assert_equal(test.data, [False, True]) assert_equal(test.mask, [False, False]) + assert_(test.fill_value == True) + b = array([(1, 1), (2, 2)], mask=[(1, 0), (0, 0)], dtype=ndtype) test = (a != b) assert_equal(test.data, [True, False]) assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + test = (a[0] != b) assert_equal(test.data, [True, True]) assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + b = array([(1, 1), (2, 2)], mask=[(0, 1), (1, 0)], dtype=ndtype) test = (a != b) assert_equal(test.data, [False, False]) assert_equal(test.mask, [False, False]) + assert_(test.fill_value == True) + # complicated dtype, 2-dimensional array. ndtype = [('A', int), ('B', [('BA', int), ('BB', int)])] a = array([[(1, (1, 1)), (2, (2, 2))], @@ -1472,6 +1500,7 @@ class TestMaskedArrayArithmetic(object): test = (a[0, 0] != a) assert_equal(test.data, [[False, True], [True, True]]) assert_equal(test.mask, [[False, False], [False, True]]) + assert_(test.fill_value == True) def test_eq_ne_structured_extra(self): # ensure simple examples are symmetric and make sense. @@ -1507,6 +1536,120 @@ class TestMaskedArrayArithmetic(object): el_by_el = [m1[name] != m2[name] for name in dt.names] assert_equal(array(el_by_el, dtype=bool).any(), ne_expected) + @pytest.mark.parametrize('dt', ['S', 'U']) + @pytest.mark.parametrize('fill', [None, 'A']) + def test_eq_for_strings(self, dt, fill): + # Test the equality of structured arrays + a = array(['a', 'b'], dtype=dt, mask=[0, 1], fill_value=fill) + + test = (a == a) + assert_equal(test.data, [True, True]) + assert_equal(test.mask, [False, True]) + assert_(test.fill_value == True) + + test = (a == a[0]) + assert_equal(test.data, [True, False]) + assert_equal(test.mask, [False, True]) + assert_(test.fill_value == True) + + b = array(['a', 'b'], dtype=dt, mask=[1, 0], fill_value=fill) + test = (a == b) + assert_equal(test.data, [False, False]) + assert_equal(test.mask, [True, True]) + assert_(test.fill_value == True) + + # test = (a[0] == b) # doesn't work in Python2 + test = (b == a[0]) + assert_equal(test.data, [False, False]) + assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + + @pytest.mark.parametrize('dt', ['S', 'U']) + @pytest.mark.parametrize('fill', [None, 'A']) + def test_ne_for_strings(self, dt, fill): + # Test the equality of structured arrays + a = array(['a', 'b'], dtype=dt, mask=[0, 1], fill_value=fill) + + test = (a != a) + assert_equal(test.data, [False, False]) + assert_equal(test.mask, [False, True]) + assert_(test.fill_value == True) + + test = (a != a[0]) + assert_equal(test.data, [False, True]) + assert_equal(test.mask, [False, True]) + assert_(test.fill_value == True) + + b = array(['a', 'b'], dtype=dt, mask=[1, 0], fill_value=fill) + test = (a != b) + assert_equal(test.data, [True, True]) + assert_equal(test.mask, [True, True]) + assert_(test.fill_value == True) + + # test = (a[0] != b) # doesn't work in Python2 + test = (b != a[0]) + assert_equal(test.data, [True, True]) + assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + + @pytest.mark.parametrize('dt1', num_dts, ids=num_ids) + @pytest.mark.parametrize('dt2', num_dts, ids=num_ids) + @pytest.mark.parametrize('fill', [None, 1]) + def test_eq_for_numeric(self, dt1, dt2, fill): + # Test the equality of structured arrays + a = array([0, 1], dtype=dt1, mask=[0, 1], fill_value=fill) + + test = (a == a) + assert_equal(test.data, [True, True]) + assert_equal(test.mask, [False, True]) + assert_(test.fill_value == True) + + test = (a == a[0]) + assert_equal(test.data, [True, False]) + assert_equal(test.mask, [False, True]) + assert_(test.fill_value == True) + + b = array([0, 1], dtype=dt2, mask=[1, 0], fill_value=fill) + test = (a == b) + assert_equal(test.data, [False, False]) + assert_equal(test.mask, [True, True]) + assert_(test.fill_value == True) + + # test = (a[0] == b) # doesn't work in Python2 + test = (b == a[0]) + assert_equal(test.data, [False, False]) + assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + + @pytest.mark.parametrize('dt1', num_dts, ids=num_ids) + @pytest.mark.parametrize('dt2', num_dts, ids=num_ids) + @pytest.mark.parametrize('fill', [None, 1]) + def test_ne_for_numeric(self, dt1, dt2, fill): + # Test the equality of structured arrays + a = array([0, 1], dtype=dt1, mask=[0, 1], fill_value=fill) + + test = (a != a) + assert_equal(test.data, [False, False]) + assert_equal(test.mask, [False, True]) + assert_(test.fill_value == True) + + test = (a != a[0]) + assert_equal(test.data, [False, True]) + assert_equal(test.mask, [False, True]) + assert_(test.fill_value == True) + + b = array([0, 1], dtype=dt2, mask=[1, 0], fill_value=fill) + test = (a != b) + assert_equal(test.data, [True, True]) + assert_equal(test.mask, [True, True]) + assert_(test.fill_value == True) + + # test = (a[0] != b) # doesn't work in Python2 + test = (b != a[0]) + assert_equal(test.data, [True, True]) + assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + def test_eq_with_None(self): # Really, comparisons with None should not be done, but check them # anyway. Note that pep8 will flag these tests. @@ -1886,6 +2029,17 @@ class TestFillingValues(object): assert_equal(x.fill_value, 999.) assert_equal(x._fill_value, np.array(999.)) + def test_subarray_fillvalue(self): + # gh-10483 test multi-field index fill value + fields = array([(1, 1, 1)], + dtype=[('i', int), ('s', '|S8'), ('f', float)]) + with suppress_warnings() as sup: + sup.filter(FutureWarning, "Numpy has detected") + subfields = fields[['i', 'f']] + assert_equal(tuple(subfields.fill_value), (999999, 1.e+20)) + # test comparison does not raise: + subfields[1:] == subfields[:-1] + def test_fillvalue_exotic_dtype(self): # Tests yet more exotic flexible dtypes _check_fill_value = np.ma.core._check_fill_value @@ -2247,9 +2401,9 @@ class TestMaskedArrayInPlaceArithmetics(object): assert_equal(xm, y + 1) (x, _, xm) = self.floatdata - id1 = x.data.ctypes._data + id1 = x.data.ctypes.data x += 1. - assert_(id1 == x.data.ctypes._data) + assert_(id1 == x.data.ctypes.data) assert_equal(x, y + 1.) def test_inplace_addition_array(self): @@ -5017,11 +5171,8 @@ def test_astype_mask_ordering(): assert_(x_f2.mask.flags.f_contiguous) -dts = [np.dtype(dt_) for dt_ in '?bhilqBHILQefdgFD'] -ids = [dt_.char for dt_ in dts] - -@pytest.mark.parametrize('dt1', dts, ids=ids) -@pytest.mark.parametrize('dt2', dts, ids=ids) +@pytest.mark.parametrize('dt1', num_dts, ids=num_ids) +@pytest.mark.parametrize('dt2', num_dts, ids=num_ids) @pytest.mark.filterwarnings('ignore::numpy.ComplexWarning') def test_astype_basic(dt1, dt2): # See gh-12070 diff --git a/numpy/ma/tests/test_extras.py b/numpy/ma/tests/test_extras.py index c29bec2bd..5243cf714 100644 --- a/numpy/ma/tests/test_extras.py +++ b/numpy/ma/tests/test_extras.py @@ -14,7 +14,7 @@ import itertools import numpy as np from numpy.testing import ( - assert_warns, suppress_warnings, assert_raises, + assert_warns, suppress_warnings ) from numpy.ma.testutils import ( assert_, assert_array_equal, assert_equal, assert_almost_equal @@ -29,9 +29,8 @@ from numpy.ma.extras import ( ediff1d, apply_over_axes, apply_along_axis, compress_nd, compress_rowcols, mask_rowcols, clump_masked, clump_unmasked, flatnotmasked_contiguous, notmasked_contiguous, notmasked_edges, masked_all, masked_all_like, isin, - diagflat, stack, vstack, hstack + diagflat, stack, vstack ) -import numpy.ma.extras as mae class TestGeneric(object): diff --git a/numpy/ma/tests/test_mrecords.py b/numpy/ma/tests/test_mrecords.py index 8b9e3fbc9..dbbf1c8a1 100644 --- a/numpy/ma/tests/test_mrecords.py +++ b/numpy/ma/tests/test_mrecords.py @@ -7,8 +7,6 @@ """ from __future__ import division, absolute_import, print_function -import warnings - import numpy as np import numpy.ma as ma from numpy import recarray diff --git a/numpy/ma/tests/test_old_ma.py b/numpy/ma/tests/test_old_ma.py index 807121184..2978be22c 100644 --- a/numpy/ma/tests/test_old_ma.py +++ b/numpy/ma/tests/test_old_ma.py @@ -8,7 +8,6 @@ import numpy.core.fromnumeric as fromnumeric from numpy.testing import ( assert_, assert_raises, assert_equal, ) -from numpy.ma.testutils import assert_array_equal from numpy.ma import ( MaskType, MaskedArray, absolute, add, all, allclose, allequal, alltrue, arange, arccos, arcsin, arctan, arctan2, array, average, choose, diff --git a/numpy/ma/tests/test_regression.py b/numpy/ma/tests/test_regression.py index 96c418a51..54f1bda7d 100644 --- a/numpy/ma/tests/test_regression.py +++ b/numpy/ma/tests/test_regression.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import warnings - import numpy as np from numpy.testing import ( assert_, assert_array_equal, assert_allclose, suppress_warnings @@ -84,3 +82,8 @@ class TestRegression(object): assert_(a.mask.shape == (2,)) assert_(b.shape == (2, 2)) assert_(b.mask.shape == (2, 2)) + + def test_empty_list_on_structured(self): + # See gh-12464. Indexing with empty list should give empty result. + ma = np.ma.MaskedArray([(1, 1.), (2, 2.), (3, 3.)], dtype='i4,f4') + assert_array_equal(ma[[]], ma[:0]) diff --git a/numpy/matlib.py b/numpy/matlib.py index 004e5f0c8..9e115943a 100644 --- a/numpy/matlib.py +++ b/numpy/matlib.py @@ -39,11 +39,11 @@ def empty(shape, dtype=None, order='C'): -------- >>> import numpy.matlib >>> np.matlib.empty((2, 2)) # filled with random data - matrix([[ 6.76425276e-320, 9.79033856e-307], - [ 7.39337286e-309, 3.22135945e-309]]) #random + matrix([[ 6.76425276e-320, 9.79033856e-307], # random + [ 7.39337286e-309, 3.22135945e-309]]) >>> np.matlib.empty((2, 2), dtype=int) - matrix([[ 6600475, 0], - [ 6586976, 22740995]]) #random + matrix([[ 6600475, 0], # random + [ 6586976, 22740995]]) """ return ndarray.__new__(matrix, shape, dtype, order=order) @@ -82,11 +82,11 @@ def ones(shape, dtype=None, order='C'): Examples -------- >>> np.matlib.ones((2,3)) - matrix([[ 1., 1., 1.], - [ 1., 1., 1.]]) + matrix([[1., 1., 1.], + [1., 1., 1.]]) >>> np.matlib.ones(2) - matrix([[ 1., 1.]]) + matrix([[1., 1.]]) """ a = ndarray.__new__(matrix, shape, dtype, order=order) @@ -126,11 +126,11 @@ def zeros(shape, dtype=None, order='C'): -------- >>> import numpy.matlib >>> np.matlib.zeros((2, 3)) - matrix([[ 0., 0., 0.], - [ 0., 0., 0.]]) + matrix([[0., 0., 0.], + [0., 0., 0.]]) >>> np.matlib.zeros(2) - matrix([[ 0., 0.]]) + matrix([[0., 0.]]) """ a = ndarray.__new__(matrix, shape, dtype, order=order) @@ -210,9 +210,9 @@ def eye(n,M=None, k=0, dtype=float, order='C'): -------- >>> import numpy.matlib >>> np.matlib.eye(3, k=1, dtype=float) - matrix([[ 0., 1., 0.], - [ 0., 0., 1.], - [ 0., 0., 0.]]) + matrix([[0., 1., 0.], + [0., 0., 1.], + [0., 0., 0.]]) """ return asmatrix(np.eye(n, M=M, k=k, dtype=dtype, order=order)) @@ -243,19 +243,20 @@ def rand(*args): Examples -------- + >>> np.random.seed(123) >>> import numpy.matlib >>> np.matlib.rand(2, 3) - matrix([[ 0.68340382, 0.67926887, 0.83271405], - [ 0.00793551, 0.20468222, 0.95253525]]) #random + matrix([[0.69646919, 0.28613933, 0.22685145], + [0.55131477, 0.71946897, 0.42310646]]) >>> np.matlib.rand((2, 3)) - matrix([[ 0.84682055, 0.73626594, 0.11308016], - [ 0.85429008, 0.3294825 , 0.89139555]]) #random + matrix([[0.9807642 , 0.68482974, 0.4809319 ], + [0.39211752, 0.34317802, 0.72904971]]) If the first argument is a tuple, other arguments are ignored: >>> np.matlib.rand((2, 3), 4) - matrix([[ 0.46898646, 0.15163588, 0.95188261], - [ 0.59208621, 0.09561818, 0.00583606]]) #random + matrix([[0.43857224, 0.0596779 , 0.39804426], + [0.73799541, 0.18249173, 0.17545176]]) """ if isinstance(args[0], tuple): @@ -294,18 +295,19 @@ def randn(*args): Examples -------- + >>> np.random.seed(123) >>> import numpy.matlib >>> np.matlib.randn(1) - matrix([[-0.09542833]]) #random + matrix([[-1.0856306]]) >>> np.matlib.randn(1, 2, 3) - matrix([[ 0.16198284, 0.0194571 , 0.18312985], - [-0.7509172 , 1.61055 , 0.45298599]]) #random + matrix([[ 0.99734545, 0.2829785 , -1.50629471], + [-0.57860025, 1.65143654, -2.42667924]]) Two-by-four matrix of samples from :math:`N(3, 6.25)`: >>> 2.5 * np.matlib.randn((2, 4)) + 3 - matrix([[ 4.74085004, 8.89381862, 4.09042411, 4.83721922], - [ 7.52373709, 5.07933944, -2.64043543, 0.45610557]]) #random + matrix([[1.92771843, 6.16484065, 0.83314899, 1.30278462], + [2.76322758, 6.72847407, 1.40274501, 1.8900451 ]]) """ if isinstance(args[0], tuple): diff --git a/numpy/matrixlib/defmatrix.py b/numpy/matrixlib/defmatrix.py index 7baa401a8..6f8eadf86 100644 --- a/numpy/matrixlib/defmatrix.py +++ b/numpy/matrixlib/defmatrix.py @@ -7,6 +7,7 @@ import warnings import ast import numpy.core.numeric as N from numpy.core.numeric import concatenate, isscalar +from numpy.core.overrides import set_module # While not in __all__, matrix_power used to be defined here, so we import # it for backward compatibility. from numpy.linalg import matrix_power @@ -33,6 +34,8 @@ def _convert_from_string(data): newdata.append(newrow) return newdata + +@set_module('numpy') def asmatrix(data, dtype=None): """ Interpret the input as a matrix. @@ -67,6 +70,8 @@ def asmatrix(data, dtype=None): """ return matrix(data, dtype=dtype, copy=False) + +@set_module('numpy') class matrix(N.ndarray): """ matrix(data, dtype=None, copy=True) @@ -99,9 +104,9 @@ class matrix(N.ndarray): Examples -------- >>> a = np.matrix('1 2; 3 4') - >>> print(a) - [[1 2] - [3 4]] + >>> a + matrix([[1, 2], + [3, 4]]) >>> np.matrix([[1, 2], [3, 4]]) matrix([[1, 2], @@ -305,12 +310,12 @@ class matrix(N.ndarray): matrix([[3], [7]]) >>> x.sum(axis=1, dtype='float') - matrix([[ 3.], - [ 7.]]) - >>> out = np.zeros((1, 2), dtype='float') - >>> x.sum(axis=1, dtype='float', out=out) - matrix([[ 3.], - [ 7.]]) + matrix([[3.], + [7.]]) + >>> out = np.zeros((2, 1), dtype='float') + >>> x.sum(axis=1, dtype='float', out=np.asmatrix(out)) + matrix([[3.], + [7.]]) """ return N.ndarray.sum(self, axis, dtype, out, keepdims=True)._collapse(axis) @@ -432,7 +437,7 @@ class matrix(N.ndarray): >>> x.mean() 5.5 >>> x.mean(0) - matrix([[ 4., 5., 6., 7.]]) + matrix([[4., 5., 6., 7.]]) >>> x.mean(1) matrix([[ 1.5], [ 5.5], @@ -464,9 +469,9 @@ class matrix(N.ndarray): [ 4, 5, 6, 7], [ 8, 9, 10, 11]]) >>> x.std() - 3.4520525295346629 + 3.4520525295346629 # may vary >>> x.std(0) - matrix([[ 3.26598632, 3.26598632, 3.26598632, 3.26598632]]) + matrix([[ 3.26598632, 3.26598632, 3.26598632, 3.26598632]]) # may vary >>> x.std(1) matrix([[ 1.11803399], [ 1.11803399], @@ -500,11 +505,11 @@ class matrix(N.ndarray): >>> x.var() 11.916666666666666 >>> x.var(0) - matrix([[ 10.66666667, 10.66666667, 10.66666667, 10.66666667]]) + matrix([[ 10.66666667, 10.66666667, 10.66666667, 10.66666667]]) # may vary >>> x.var(1) - matrix([[ 1.25], - [ 1.25], - [ 1.25]]) + matrix([[1.25], + [1.25], + [1.25]]) """ return N.ndarray.var(self, axis, dtype, out, ddof, keepdims=True)._collapse(axis) @@ -819,7 +824,7 @@ class matrix(N.ndarray): matrix([[-2. , 1. ], [ 1.5, -0.5]]) >>> m.getI() * m - matrix([[ 1., 0.], + matrix([[ 1., 0.], # may vary [ 0., 1.]]) """ @@ -881,7 +886,8 @@ class matrix(N.ndarray): [ 4, 5, 6, 7], [ 8, 9, 10, 11]]) >>> x.getA1() - array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + array([ 0, 1, 2, ..., 9, 10, 11]) + """ return self.__array__().ravel() @@ -981,10 +987,10 @@ class matrix(N.ndarray): [ 4. -4.j, 5. -5.j, 6. -6.j, 7. -7.j], [ 8. -8.j, 9. -9.j, 10.-10.j, 11.-11.j]]) >>> z.getH() - matrix([[ 0. +0.j, 4. +4.j, 8. +8.j], - [ 1. +1.j, 5. +5.j, 9. +9.j], - [ 2. +2.j, 6. +6.j, 10.+10.j], - [ 3. +3.j, 7. +7.j, 11.+11.j]]) + matrix([[ 0. -0.j, 4. +4.j, 8. +8.j], + [ 1. +1.j, 5. +5.j, 9. +9.j], + [ 2. +2.j, 6. +6.j, 10.+10.j], + [ 3. +3.j, 7. +7.j, 11.+11.j]]) """ if issubclass(self.dtype.type, N.complexfloating): @@ -1023,6 +1029,7 @@ def _from_string(str, gdict, ldict): return concatenate(rowtup, axis=0) +@set_module('numpy') def bmat(obj, ldict=None, gdict=None): """ Build a matrix object from a string, nested sequence, or array. diff --git a/numpy/matrixlib/tests/test_defmatrix.py b/numpy/matrixlib/tests/test_defmatrix.py index f8a8ad511..aa6e08d64 100644 --- a/numpy/matrixlib/tests/test_defmatrix.py +++ b/numpy/matrixlib/tests/test_defmatrix.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - try: # Accessing collections abstract classes from collections # has been deprecated since Python 3.3 diff --git a/numpy/matrixlib/tests/test_masked_matrix.py b/numpy/matrixlib/tests/test_masked_matrix.py index 1751020db..52fd18577 100644 --- a/numpy/matrixlib/tests/test_masked_matrix.py +++ b/numpy/matrixlib/tests/test_masked_matrix.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.ma.testutils import (assert_, assert_equal, assert_raises, assert_array_equal) diff --git a/numpy/matrixlib/tests/test_matrix_linalg.py b/numpy/matrixlib/tests/test_matrix_linalg.py index 8d31ec5b0..6fc733c2e 100644 --- a/numpy/matrixlib/tests/test_matrix_linalg.py +++ b/numpy/matrixlib/tests/test_matrix_linalg.py @@ -1,8 +1,6 @@ """ Test functions for linalg module using the matrix class.""" from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.linalg.tests.test_linalg import ( diff --git a/numpy/matrixlib/tests/test_multiarray.py b/numpy/matrixlib/tests/test_multiarray.py index 8de0a7c6a..6d84bd477 100644 --- a/numpy/matrixlib/tests/test_multiarray.py +++ b/numpy/matrixlib/tests/test_multiarray.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.testing import assert_, assert_equal, assert_array_equal diff --git a/numpy/matrixlib/tests/test_numeric.py b/numpy/matrixlib/tests/test_numeric.py index e9f44e747..95e1c8001 100644 --- a/numpy/matrixlib/tests/test_numeric.py +++ b/numpy/matrixlib/tests/test_numeric.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.testing import assert_equal diff --git a/numpy/matrixlib/tests/test_regression.py b/numpy/matrixlib/tests/test_regression.py index 88654c76a..70e147279 100644 --- a/numpy/matrixlib/tests/test_regression.py +++ b/numpy/matrixlib/tests/test_regression.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.testing import assert_, assert_equal, assert_raises diff --git a/numpy/polynomial/chebyshev.py b/numpy/polynomial/chebyshev.py index f1ddc9b06..e0734e1b8 100644 --- a/numpy/polynomial/chebyshev.py +++ b/numpy/polynomial/chebyshev.py @@ -361,12 +361,12 @@ def poly2cheb(pol): >>> from numpy import polynomial as P >>> p = P.Polynomial(range(4)) >>> p - Polynomial([ 0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) + Polynomial([0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) >>> c = p.convert(kind=P.Chebyshev) >>> c - Chebyshev([ 1. , 3.25, 1. , 0.75], domain=[-1, 1], window=[-1, 1]) + Chebyshev([1. , 3.25, 1. , 0.75], domain=[-1., 1.], window=[-1., 1.]) >>> P.chebyshev.poly2cheb(range(4)) - array([ 1. , 3.25, 1. , 0.75]) + array([1. , 3.25, 1. , 0.75]) """ [pol] = pu.as_series([pol]) @@ -413,12 +413,12 @@ def cheb2poly(c): >>> from numpy import polynomial as P >>> c = P.Chebyshev(range(4)) >>> c - Chebyshev([ 0., 1., 2., 3.], [-1., 1.]) + Chebyshev([0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) >>> p = c.convert(kind=P.Polynomial) >>> p - Polynomial([ -2., -8., 4., 12.], [-1., 1.]) + Polynomial([-2., -8., 4., 12.], domain=[-1., 1.], window=[-1., 1.]) >>> P.chebyshev.cheb2poly(range(4)) - array([ -2., -8., 4., 12.]) + array([-2., -8., 4., 12.]) """ from .polynomial import polyadd, polysub, polymulx @@ -538,7 +538,7 @@ def chebfromroots(roots): array([ 0. , -0.25, 0. , 0.25]) >>> j = complex(0,1) >>> C.chebfromroots((-j,j)) # x^2 + 1 relative to the standard basis - array([ 1.5+0.j, 0.0+0.j, 0.5+0.j]) + array([1.5+0.j, 0. +0.j, 0.5+0.j]) """ if len(roots) == 0: @@ -594,7 +594,7 @@ def chebadd(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> C.chebadd(c1,c2) - array([ 4., 4., 4.]) + array([4., 4., 4.]) """ # c1, c2 are trimmed copies @@ -688,7 +688,7 @@ def chebmulx(c): -------- >>> from numpy.polynomial import chebyshev as C >>> C.chebmulx([1,2,3]) - array([ 1., 2.5, 3., 1.5, 2.]) + array([1. , 2.5, 1. , 1.5]) """ # c is a trimmed copy @@ -796,10 +796,10 @@ def chebdiv(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> C.chebdiv(c1,c2) # quotient "intuitive," remainder not - (array([ 3.]), array([-8., -4.])) + (array([3.]), array([-8., -4.])) >>> c2 = (0,1,2,3) >>> C.chebdiv(c2,c1) # neither "intuitive" - (array([ 0., 2.]), array([-2., -4.])) + (array([0., 2.]), array([-2., -4.])) """ # c1, c2 are trimmed copies @@ -853,7 +853,7 @@ def chebpow(c, pow, maxpower=16): -------- >>> from numpy.polynomial import chebyshev as C >>> C.chebpow([1, 2, 3, 4], 2) - array([15.5, 22. , 16. , 14. , 12.5, 12. , 8. ]) + array([15.5, 22. , 16. , ..., 12.5, 12. , 8. ]) """ # c is a trimmed copy @@ -928,13 +928,13 @@ def chebder(c, m=1, scl=1, axis=0): >>> from numpy.polynomial import chebyshev as C >>> c = (1,2,3,4) >>> C.chebder(c) - array([ 14., 12., 24.]) + array([14., 12., 24.]) >>> C.chebder(c,3) - array([ 96.]) + array([96.]) >>> C.chebder(c,scl=-1) array([-14., -12., -24.]) >>> C.chebder(c,2,-1) - array([ 12., 96.]) + array([12., 96.]) """ c = np.array(c, ndmin=1, copy=1) @@ -1048,8 +1048,8 @@ def chebint(c, m=1, k=[], lbnd=0, scl=1, axis=0): >>> C.chebint(c) array([ 0.5, -0.5, 0.5, 0.5]) >>> C.chebint(c,3) - array([ 0.03125 , -0.1875 , 0.04166667, -0.05208333, 0.01041667, - 0.00625 ]) + array([ 0.03125 , -0.1875 , 0.04166667, -0.05208333, 0.01041667, # may vary + 0.00625 ]) >>> C.chebint(c, k=3) array([ 3.5, -0.5, 0.5, 0.5]) >>> C.chebint(c,lbnd=-2) @@ -1096,7 +1096,7 @@ def chebint(c, m=1, k=[], lbnd=0, scl=1, axis=0): if n > 1: tmp[2] = c[1]/4 for j in range(2, n): - t = c[j]/(2*j + 1) + t = c[j]/(2*j + 1) # FIXME: t never used tmp[j + 1] = c[j]/(2*(j + 1)) tmp[j - 1] -= c[j]/(2*(j - 1)) tmp[0] += k[i] - chebval(lbnd, tmp) @@ -1674,7 +1674,7 @@ def chebfit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1885,7 +1885,7 @@ def chebroots(c): -------- >>> import numpy.polynomial.chebyshev as cheb >>> cheb.chebroots((-1, 1,-1, 1)) # T3 - T2 + T1 - T0 has real roots - array([ -5.00000000e-01, 2.60860684e-17, 1.00000000e+00]) + array([ -5.00000000e-01, 2.60860684e-17, 1.00000000e+00]) # may vary """ # c is a trimmed copy diff --git a/numpy/polynomial/hermite.py b/numpy/polynomial/hermite.py index 2aed4b34f..93c9fc564 100644 --- a/numpy/polynomial/hermite.py +++ b/numpy/polynomial/hermite.py @@ -114,7 +114,7 @@ def poly2herm(pol): -------- >>> from numpy.polynomial.hermite import poly2herm >>> poly2herm(np.arange(4)) - array([ 1. , 2.75 , 0.5 , 0.375]) + array([1. , 2.75 , 0.5 , 0.375]) """ [pol] = pu.as_series([pol]) @@ -160,7 +160,7 @@ def herm2poly(c): -------- >>> from numpy.polynomial.hermite import herm2poly >>> herm2poly([ 1. , 2.75 , 0.5 , 0.375]) - array([ 0., 1., 2., 3.]) + array([0., 1., 2., 3.]) """ from .polynomial import polyadd, polysub, polymulx @@ -280,10 +280,10 @@ def hermfromroots(roots): >>> from numpy.polynomial.hermite import hermfromroots, hermval >>> coef = hermfromroots((-1, 0, 1)) >>> hermval((-1, 0, 1), coef) - array([ 0., 0., 0.]) + array([0., 0., 0.]) >>> coef = hermfromroots((-1j, 1j)) >>> hermval((-1j, 1j), coef) - array([ 0.+0.j, 0.+0.j]) + array([0.+0.j, 0.+0.j]) """ if len(roots) == 0: @@ -337,7 +337,7 @@ def hermadd(c1, c2): -------- >>> from numpy.polynomial.hermite import hermadd >>> hermadd([1, 2, 3], [1, 2, 3, 4]) - array([ 2., 4., 6., 4.]) + array([2., 4., 6., 4.]) """ # c1, c2 are trimmed copies @@ -385,7 +385,7 @@ def hermsub(c1, c2): -------- >>> from numpy.polynomial.hermite import hermsub >>> hermsub([1, 2, 3, 4], [1, 2, 3]) - array([ 0., 0., 0., 4.]) + array([0., 0., 0., 4.]) """ # c1, c2 are trimmed copies @@ -435,7 +435,7 @@ def hermmulx(c): -------- >>> from numpy.polynomial.hermite import hermmulx >>> hermmulx([1, 2, 3]) - array([ 2. , 6.5, 1. , 1.5]) + array([2. , 6.5, 1. , 1.5]) """ # c is a trimmed copy @@ -488,7 +488,7 @@ def hermmul(c1, c2): -------- >>> from numpy.polynomial.hermite import hermmul >>> hermmul([1, 2, 3], [0, 1, 2]) - array([ 52., 29., 52., 7., 6.]) + array([52., 29., 52., 7., 6.]) """ # s1, s2 are trimmed copies @@ -557,11 +557,11 @@ def hermdiv(c1, c2): -------- >>> from numpy.polynomial.hermite import hermdiv >>> hermdiv([ 52., 29., 52., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 0.])) + (array([1., 2., 3.]), array([0.])) >>> hermdiv([ 54., 31., 52., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 2., 2.])) + (array([1., 2., 3.]), array([2., 2.])) >>> hermdiv([ 53., 30., 52., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 1., 1.])) + (array([1., 2., 3.]), array([1., 1.])) """ # c1, c2 are trimmed copies @@ -617,7 +617,7 @@ def hermpow(c, pow, maxpower=16): -------- >>> from numpy.polynomial.hermite import hermpow >>> hermpow([1, 2, 3], 2) - array([ 81., 52., 82., 12., 9.]) + array([81., 52., 82., 12., 9.]) """ # c is a trimmed copy @@ -690,9 +690,9 @@ def hermder(c, m=1, scl=1, axis=0): -------- >>> from numpy.polynomial.hermite import hermder >>> hermder([ 1. , 0.5, 0.5, 0.5]) - array([ 1., 2., 3.]) + array([1., 2., 3.]) >>> hermder([-0.5, 1./2., 1./8., 1./12., 1./16.], m=2) - array([ 1., 2., 3.]) + array([1., 2., 3.]) """ c = np.array(c, ndmin=1, copy=1) @@ -799,15 +799,15 @@ def hermint(c, m=1, k=[], lbnd=0, scl=1, axis=0): -------- >>> from numpy.polynomial.hermite import hermint >>> hermint([1,2,3]) # integrate once, value 0 at 0. - array([ 1. , 0.5, 0.5, 0.5]) + array([1. , 0.5, 0.5, 0.5]) >>> hermint([1,2,3], m=2) # integrate twice, value & deriv 0 at 0 - array([-0.5 , 0.5 , 0.125 , 0.08333333, 0.0625 ]) + array([-0.5 , 0.5 , 0.125 , 0.08333333, 0.0625 ]) # may vary >>> hermint([1,2,3], k=1) # integrate once, value 1 at 0. - array([ 2. , 0.5, 0.5, 0.5]) + array([2. , 0.5, 0.5, 0.5]) >>> hermint([1,2,3], lbnd=-1) # integrate once, value 0 at -1 array([-2. , 0.5, 0.5, 0.5]) >>> hermint([1,2,3], m=2, k=[1,2], lbnd=-1) - array([ 1.66666667, -0.5 , 0.125 , 0.08333333, 0.0625 ]) + array([ 1.66666667, -0.5 , 0.125 , 0.08333333, 0.0625 ]) # may vary """ c = np.array(c, ndmin=1, copy=1) @@ -918,8 +918,8 @@ def hermval(x, c, tensor=True): >>> hermval(1, coef) 11.0 >>> hermval([[1,2],[3,4]], coef) - array([[ 11., 51.], - [ 115., 203.]]) + array([[ 11., 51.], + [115., 203.]]) """ c = np.array(c, ndmin=1, copy=0) @@ -1437,7 +1437,7 @@ def hermfit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1490,7 +1490,7 @@ def hermfit(x, y, deg, rcond=None, full=False, w=None): >>> err = np.random.randn(len(x))/10 >>> y = hermval(x, [1, 2, 3]) + err >>> hermfit(x, y, 2) - array([ 0.97902637, 1.99849131, 3.00006 ]) + array([1.0218, 1.9986, 2.9999]) # may vary """ x = np.asarray(x) + 0.0 @@ -1656,9 +1656,9 @@ def hermroots(c): >>> from numpy.polynomial.hermite import hermroots, hermfromroots >>> coef = hermfromroots([-1, 0, 1]) >>> coef - array([ 0. , 0.25 , 0. , 0.125]) + array([0. , 0.25 , 0. , 0.125]) >>> hermroots(coef) - array([ -1.00000000e+00, -1.38777878e-17, 1.00000000e+00]) + array([-1.00000000e+00, -1.38777878e-17, 1.00000000e+00]) """ # c is a trimmed copy @@ -1704,7 +1704,7 @@ def _normed_hermite_n(x, n): """ if n == 0: - return np.ones(x.shape)/np.sqrt(np.sqrt(np.pi)) + return np.full(x.shape, 1/np.sqrt(np.sqrt(np.pi))) c0 = 0. c1 = 1./np.sqrt(np.sqrt(np.pi)) diff --git a/numpy/polynomial/hermite_e.py b/numpy/polynomial/hermite_e.py index d4520ad6c..bafb4b997 100644 --- a/numpy/polynomial/hermite_e.py +++ b/numpy/polynomial/hermite_e.py @@ -161,7 +161,7 @@ def herme2poly(c): -------- >>> from numpy.polynomial.hermite_e import herme2poly >>> herme2poly([ 2., 10., 2., 3.]) - array([ 0., 1., 2., 3.]) + array([0., 1., 2., 3.]) """ from .polynomial import polyadd, polysub, polymulx @@ -281,10 +281,10 @@ def hermefromroots(roots): >>> from numpy.polynomial.hermite_e import hermefromroots, hermeval >>> coef = hermefromroots((-1, 0, 1)) >>> hermeval((-1, 0, 1), coef) - array([ 0., 0., 0.]) + array([0., 0., 0.]) >>> coef = hermefromroots((-1j, 1j)) >>> hermeval((-1j, 1j), coef) - array([ 0.+0.j, 0.+0.j]) + array([0.+0.j, 0.+0.j]) """ if len(roots) == 0: @@ -338,7 +338,7 @@ def hermeadd(c1, c2): -------- >>> from numpy.polynomial.hermite_e import hermeadd >>> hermeadd([1, 2, 3], [1, 2, 3, 4]) - array([ 2., 4., 6., 4.]) + array([2., 4., 6., 4.]) """ # c1, c2 are trimmed copies @@ -386,7 +386,7 @@ def hermesub(c1, c2): -------- >>> from numpy.polynomial.hermite_e import hermesub >>> hermesub([1, 2, 3, 4], [1, 2, 3]) - array([ 0., 0., 0., 4.]) + array([0., 0., 0., 4.]) """ # c1, c2 are trimmed copies @@ -432,7 +432,7 @@ def hermemulx(c): -------- >>> from numpy.polynomial.hermite_e import hermemulx >>> hermemulx([1, 2, 3]) - array([ 2., 7., 2., 3.]) + array([2., 7., 2., 3.]) """ # c is a trimmed copy @@ -485,7 +485,7 @@ def hermemul(c1, c2): -------- >>> from numpy.polynomial.hermite_e import hermemul >>> hermemul([1, 2, 3], [0, 1, 2]) - array([ 14., 15., 28., 7., 6.]) + array([14., 15., 28., 7., 6.]) """ # s1, s2 are trimmed copies @@ -554,9 +554,9 @@ def hermediv(c1, c2): -------- >>> from numpy.polynomial.hermite_e import hermediv >>> hermediv([ 14., 15., 28., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 0.])) + (array([1., 2., 3.]), array([0.])) >>> hermediv([ 15., 17., 28., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 1., 2.])) + (array([1., 2., 3.]), array([1., 2.])) """ # c1, c2 are trimmed copies @@ -612,7 +612,7 @@ def hermepow(c, pow, maxpower=16): -------- >>> from numpy.polynomial.hermite_e import hermepow >>> hermepow([1, 2, 3], 2) - array([ 23., 28., 46., 12., 9.]) + array([23., 28., 46., 12., 9.]) """ # c is a trimmed copy @@ -685,9 +685,9 @@ def hermeder(c, m=1, scl=1, axis=0): -------- >>> from numpy.polynomial.hermite_e import hermeder >>> hermeder([ 1., 1., 1., 1.]) - array([ 1., 2., 3.]) + array([1., 2., 3.]) >>> hermeder([-0.25, 1., 1./2., 1./3., 1./4 ], m=2) - array([ 1., 2., 3.]) + array([1., 2., 3.]) """ c = np.array(c, ndmin=1, copy=1) @@ -794,15 +794,15 @@ def hermeint(c, m=1, k=[], lbnd=0, scl=1, axis=0): -------- >>> from numpy.polynomial.hermite_e import hermeint >>> hermeint([1, 2, 3]) # integrate once, value 0 at 0. - array([ 1., 1., 1., 1.]) + array([1., 1., 1., 1.]) >>> hermeint([1, 2, 3], m=2) # integrate twice, value & deriv 0 at 0 - array([-0.25 , 1. , 0.5 , 0.33333333, 0.25 ]) + array([-0.25 , 1. , 0.5 , 0.33333333, 0.25 ]) # may vary >>> hermeint([1, 2, 3], k=1) # integrate once, value 1 at 0. - array([ 2., 1., 1., 1.]) + array([2., 1., 1., 1.]) >>> hermeint([1, 2, 3], lbnd=-1) # integrate once, value 0 at -1 array([-1., 1., 1., 1.]) >>> hermeint([1, 2, 3], m=2, k=[1, 2], lbnd=-1) - array([ 1.83333333, 0. , 0.5 , 0.33333333, 0.25 ]) + array([ 1.83333333, 0. , 0.5 , 0.33333333, 0.25 ]) # may vary """ c = np.array(c, ndmin=1, copy=1) @@ -913,8 +913,8 @@ def hermeval(x, c, tensor=True): >>> hermeval(1, coef) 3.0 >>> hermeval([[1,2],[3,4]], coef) - array([[ 3., 14.], - [ 31., 54.]]) + array([[ 3., 14.], + [31., 54.]]) """ c = np.array(c, ndmin=1, copy=0) @@ -1430,7 +1430,7 @@ def hermefit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1480,10 +1480,11 @@ def hermefit(x, y, deg, rcond=None, full=False, w=None): -------- >>> from numpy.polynomial.hermite_e import hermefit, hermeval >>> x = np.linspace(-10, 10) + >>> np.random.seed(123) >>> err = np.random.randn(len(x))/10 >>> y = hermeval(x, [1, 2, 3]) + err >>> hermefit(x, y, 2) - array([ 1.01690445, 1.99951418, 2.99948696]) + array([ 1.01690445, 1.99951418, 2.99948696]) # may vary """ x = np.asarray(x) + 0.0 @@ -1650,9 +1651,9 @@ def hermeroots(c): >>> from numpy.polynomial.hermite_e import hermeroots, hermefromroots >>> coef = hermefromroots([-1, 0, 1]) >>> coef - array([ 0., 2., 0., 1.]) + array([0., 2., 0., 1.]) >>> hermeroots(coef) - array([-1., 0., 1.]) + array([-1., 0., 1.]) # may vary """ # c is a trimmed copy @@ -1698,7 +1699,7 @@ def _normed_hermite_e_n(x, n): """ if n == 0: - return np.ones(x.shape)/np.sqrt(np.sqrt(2*np.pi)) + return np.full(x.shape, 1/np.sqrt(np.sqrt(2*np.pi))) c0 = 0. c1 = 1./np.sqrt(np.sqrt(2*np.pi)) diff --git a/numpy/polynomial/laguerre.py b/numpy/polynomial/laguerre.py index a116d20a7..9207c9afe 100644 --- a/numpy/polynomial/laguerre.py +++ b/numpy/polynomial/laguerre.py @@ -160,7 +160,7 @@ def lag2poly(c): -------- >>> from numpy.polynomial.laguerre import lag2poly >>> lag2poly([ 23., -63., 58., -18.]) - array([ 0., 1., 2., 3.]) + array([0., 1., 2., 3.]) """ from .polynomial import polyadd, polysub, polymulx @@ -277,10 +277,10 @@ def lagfromroots(roots): >>> from numpy.polynomial.laguerre import lagfromroots, lagval >>> coef = lagfromroots((-1, 0, 1)) >>> lagval((-1, 0, 1), coef) - array([ 0., 0., 0.]) + array([0., 0., 0.]) >>> coef = lagfromroots((-1j, 1j)) >>> lagval((-1j, 1j), coef) - array([ 0.+0.j, 0.+0.j]) + array([0.+0.j, 0.+0.j]) """ if len(roots) == 0: @@ -334,7 +334,7 @@ def lagadd(c1, c2): -------- >>> from numpy.polynomial.laguerre import lagadd >>> lagadd([1, 2, 3], [1, 2, 3, 4]) - array([ 2., 4., 6., 4.]) + array([2., 4., 6., 4.]) """ @@ -383,7 +383,7 @@ def lagsub(c1, c2): -------- >>> from numpy.polynomial.laguerre import lagsub >>> lagsub([1, 2, 3, 4], [1, 2, 3]) - array([ 0., 0., 0., 4.]) + array([0., 0., 0., 4.]) """ # c1, c2 are trimmed copies @@ -433,7 +433,7 @@ def lagmulx(c): -------- >>> from numpy.polynomial.laguerre import lagmulx >>> lagmulx([1, 2, 3]) - array([ -1., -1., 11., -9.]) + array([-1., -1., 11., -9.]) """ # c is a trimmed copy @@ -556,9 +556,9 @@ def lagdiv(c1, c2): -------- >>> from numpy.polynomial.laguerre import lagdiv >>> lagdiv([ 8., -13., 38., -51., 36.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 0.])) + (array([1., 2., 3.]), array([0.])) >>> lagdiv([ 9., -12., 38., -51., 36.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 1., 1.])) + (array([1., 2., 3.]), array([1., 1.])) """ # c1, c2 are trimmed copies @@ -687,9 +687,9 @@ def lagder(c, m=1, scl=1, axis=0): -------- >>> from numpy.polynomial.laguerre import lagder >>> lagder([ 1., 1., 1., -3.]) - array([ 1., 2., 3.]) + array([1., 2., 3.]) >>> lagder([ 1., 0., 0., -4., 3.], m=2) - array([ 1., 2., 3.]) + array([1., 2., 3.]) """ c = np.array(c, ndmin=1, copy=1) @@ -805,9 +805,9 @@ def lagint(c, m=1, k=[], lbnd=0, scl=1, axis=0): >>> lagint([1,2,3], k=1) array([ 2., 1., 1., -3.]) >>> lagint([1,2,3], lbnd=-1) - array([ 11.5, 1. , 1. , -3. ]) + array([11.5, 1. , 1. , -3. ]) >>> lagint([1,2], m=2, k=[1,2], lbnd=-1) - array([ 11.16666667, -5. , -3. , 2. ]) + array([ 11.16666667, -5. , -3. , 2. ]) # may vary """ c = np.array(c, ndmin=1, copy=1) @@ -1436,7 +1436,7 @@ def lagfit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1489,7 +1489,7 @@ def lagfit(x, y, deg, rcond=None, full=False, w=None): >>> err = np.random.randn(len(x))/10 >>> y = lagval(x, [1, 2, 3]) + err >>> lagfit(x, y, 2) - array([ 0.96971004, 2.00193749, 3.00288744]) + array([ 0.96971004, 2.00193749, 3.00288744]) # may vary """ x = np.asarray(x) + 0.0 @@ -1656,7 +1656,7 @@ def lagroots(c): >>> coef array([ 2., -8., 12., -6.]) >>> lagroots(coef) - array([ -4.44089210e-16, 1.00000000e+00, 2.00000000e+00]) + array([-4.4408921e-16, 1.0000000e+00, 2.0000000e+00]) """ # c is a trimmed copy diff --git a/numpy/polynomial/legendre.py b/numpy/polynomial/legendre.py index e9c24594b..f81bc002c 100644 --- a/numpy/polynomial/legendre.py +++ b/numpy/polynomial/legendre.py @@ -136,10 +136,10 @@ def poly2leg(pol): >>> from numpy import polynomial as P >>> p = P.Polynomial(np.arange(4)) >>> p - Polynomial([ 0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) + Polynomial([0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) >>> c = P.Legendre(P.legendre.poly2leg(p.coef)) >>> c - Legendre([ 1. , 3.25, 1. , 0.75], domain=[-1, 1], window=[-1, 1]) + Legendre([ 1. , 3.25, 1. , 0.75], domain=[-1, 1], window=[-1, 1]) # may vary """ [pol] = pu.as_series([pol]) @@ -183,12 +183,13 @@ def leg2poly(c): Examples -------- + >>> from numpy import polynomial as P >>> c = P.Legendre(range(4)) >>> c - Legendre([ 0., 1., 2., 3.], [-1., 1.]) + Legendre([0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) >>> p = c.convert(kind=P.Polynomial) >>> p - Polynomial([-1. , -3.5, 3. , 7.5], [-1., 1.]) + Polynomial([-1. , -3.5, 3. , 7.5], domain=[-1., 1.], window=[-1., 1.]) >>> P.leg2poly(range(4)) array([-1. , -3.5, 3. , 7.5]) @@ -310,7 +311,7 @@ def legfromroots(roots): array([ 0. , -0.4, 0. , 0.4]) >>> j = complex(0,1) >>> L.legfromroots((-j,j)) # x^2 + 1 relative to the standard basis - array([ 1.33333333+0.j, 0.00000000+0.j, 0.66666667+0.j]) + array([ 1.33333333+0.j, 0.00000000+0.j, 0.66666667+0.j]) # may vary """ if len(roots) == 0: @@ -366,7 +367,7 @@ def legadd(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> L.legadd(c1,c2) - array([ 4., 4., 4.]) + array([4., 4., 4.]) """ # c1, c2 are trimmed copies @@ -468,7 +469,7 @@ def legmulx(c): -------- >>> from numpy.polynomial import legendre as L >>> L.legmulx([1,2,3]) - array([ 0.66666667, 2.2, 1.33333333, 1.8]) + array([ 0.66666667, 2.2, 1.33333333, 1.8]) # may vary """ # c is a trimmed copy @@ -525,8 +526,8 @@ def legmul(c1, c2): >>> from numpy.polynomial import legendre as L >>> c1 = (1,2,3) >>> c2 = (3,2) - >>> P.legmul(c1,c2) # multiplication requires "reprojection" - array([ 4.33333333, 10.4 , 11.66666667, 3.6 ]) + >>> L.legmul(c1,c2) # multiplication requires "reprojection" + array([ 4.33333333, 10.4 , 11.66666667, 3.6 ]) # may vary """ # s1, s2 are trimmed copies @@ -597,10 +598,10 @@ def legdiv(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> L.legdiv(c1,c2) # quotient "intuitive," remainder not - (array([ 3.]), array([-8., -4.])) + (array([3.]), array([-8., -4.])) >>> c2 = (0,1,2,3) >>> L.legdiv(c2,c1) # neither "intuitive" - (array([-0.07407407, 1.66666667]), array([-1.03703704, -2.51851852])) + (array([-0.07407407, 1.66666667]), array([-1.03703704, -2.51851852])) # may vary """ # c1, c2 are trimmed copies @@ -729,7 +730,7 @@ def legder(c, m=1, scl=1, axis=0): >>> L.legder(c) array([ 6., 9., 20.]) >>> L.legder(c, 3) - array([ 60.]) + array([60.]) >>> L.legder(c, scl=-1) array([ -6., -9., -20.]) >>> L.legder(c, 2,-1) @@ -845,16 +846,16 @@ def legint(c, m=1, k=[], lbnd=0, scl=1, axis=0): >>> from numpy.polynomial import legendre as L >>> c = (1,2,3) >>> L.legint(c) - array([ 0.33333333, 0.4 , 0.66666667, 0.6 ]) + array([ 0.33333333, 0.4 , 0.66666667, 0.6 ]) # may vary >>> L.legint(c, 3) - array([ 1.66666667e-02, -1.78571429e-02, 4.76190476e-02, - -1.73472348e-18, 1.90476190e-02, 9.52380952e-03]) + array([ 1.66666667e-02, -1.78571429e-02, 4.76190476e-02, # may vary + -1.73472348e-18, 1.90476190e-02, 9.52380952e-03]) >>> L.legint(c, k=3) - array([ 3.33333333, 0.4 , 0.66666667, 0.6 ]) + array([ 3.33333333, 0.4 , 0.66666667, 0.6 ]) # may vary >>> L.legint(c, lbnd=-2) - array([ 7.33333333, 0.4 , 0.66666667, 0.6 ]) + array([ 7.33333333, 0.4 , 0.66666667, 0.6 ]) # may vary >>> L.legint(c, scl=2) - array([ 0.66666667, 0.8 , 1.33333333, 1.2 ]) + array([ 0.66666667, 0.8 , 1.33333333, 1.2 ]) # may vary """ c = np.array(c, ndmin=1, copy=1) @@ -1476,7 +1477,7 @@ def legfit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1686,7 +1687,7 @@ def legroots(c): -------- >>> import numpy.polynomial.legendre as leg >>> leg.legroots((1, 2, 3, 4)) # 4L_3 + 3L_2 + 2L_1 + 1L_0, all real roots - array([-0.85099543, -0.11407192, 0.51506735]) + array([-0.85099543, -0.11407192, 0.51506735]) # may vary """ # c is a trimmed copy diff --git a/numpy/polynomial/polynomial.py b/numpy/polynomial/polynomial.py index 259cd31f5..69599e3fd 100644 --- a/numpy/polynomial/polynomial.py +++ b/numpy/polynomial/polynomial.py @@ -185,7 +185,7 @@ def polyfromroots(roots): array([ 0., -1., 0., 1.]) >>> j = complex(0,1) >>> P.polyfromroots((-j,j)) # complex returned, though values are real - array([ 1.+0.j, 0.+0.j, 1.+0.j]) + array([1.+0.j, 0.+0.j, 1.+0.j]) """ if len(roots) == 0: @@ -233,7 +233,7 @@ def polyadd(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> sum = P.polyadd(c1,c2); sum - array([ 4., 4., 4.]) + array([4., 4., 4.]) >>> P.polyval(2, sum) # 4 + 4(2) + 4(2**2) 28.0 @@ -401,9 +401,9 @@ def polydiv(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> P.polydiv(c1,c2) - (array([ 3.]), array([-8., -4.])) + (array([3.]), array([-8., -4.])) >>> P.polydiv(c2,c1) - (array([ 0.33333333]), array([ 2.66666667, 1.33333333])) + (array([ 0.33333333]), array([ 2.66666667, 1.33333333])) # may vary """ # c1, c2 are trimmed copies @@ -529,7 +529,7 @@ def polyder(c, m=1, scl=1, axis=0): >>> P.polyder(c) # (d/dx)(c) = 2 + 6x + 12x**2 array([ 2., 6., 12.]) >>> P.polyder(c,3) # (d**3/dx**3)(c) = 24 - array([ 24.]) + array([24.]) >>> P.polyder(c,scl=-1) # (d/d(-x))(c) = -2 - 6x - 12x**2 array([ -2., -6., -12.]) >>> P.polyder(c,2,-1) # (d**2/d(-x)**2)(c) = 6 + 24x @@ -636,14 +636,14 @@ def polyint(c, m=1, k=[], lbnd=0, scl=1, axis=0): >>> from numpy.polynomial import polynomial as P >>> c = (1,2,3) >>> P.polyint(c) # should return array([0, 1, 1, 1]) - array([ 0., 1., 1., 1.]) + array([0., 1., 1., 1.]) >>> P.polyint(c,3) # should return array([0, 0, 0, 1/6, 1/12, 1/20]) - array([ 0. , 0. , 0. , 0.16666667, 0.08333333, - 0.05 ]) + array([ 0. , 0. , 0. , 0.16666667, 0.08333333, # may vary + 0.05 ]) >>> P.polyint(c,k=3) # should return array([3, 1, 1, 1]) - array([ 3., 1., 1., 1.]) + array([3., 1., 1., 1.]) >>> P.polyint(c,lbnd=-2) # should return array([6, 1, 1, 1]) - array([ 6., 1., 1., 1.]) + array([6., 1., 1., 1.]) >>> P.polyint(c,scl=-2) # should return array([0, -2, -2, -2]) array([ 0., -2., -2., -2.]) @@ -761,17 +761,17 @@ def polyval(x, c, tensor=True): array([[0, 1], [2, 3]]) >>> polyval(a, [1,2,3]) - array([[ 1., 6.], - [ 17., 34.]]) + array([[ 1., 6.], + [17., 34.]]) >>> coef = np.arange(4).reshape(2,2) # multidimensional coefficients >>> coef array([[0, 1], [2, 3]]) >>> polyval([1,2], coef, tensor=True) - array([[ 2., 4.], - [ 4., 7.]]) + array([[2., 4.], + [4., 7.]]) >>> polyval([1,2], coef, tensor=False) - array([ 2., 7.]) + array([2., 7.]) """ c = np.array(c, ndmin=1, copy=0) @@ -851,8 +851,8 @@ def polyvalfromroots(x, r, tensor=True): array([[0, 1], [2, 3]]) >>> polyvalfromroots(a, [-1, 0, 1]) - array([[ -0., 0.], - [ 6., 24.]]) + array([[-0., 0.], + [ 6., 24.]]) >>> r = np.arange(-2, 2).reshape(2,2) # multidimensional coefficients >>> r # each column of r defines one polynomial array([[-2, -1], @@ -1363,7 +1363,7 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None): be turned off by: >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1410,26 +1410,27 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None): Examples -------- + >>> np.random.seed(123) >>> from numpy.polynomial import polynomial as P >>> x = np.linspace(-1,1,51) # x "data": [-1, -0.96, ..., 0.96, 1] >>> y = x**3 - x + np.random.randn(len(x)) # x^3 - x + N(0,1) "noise" >>> c, stats = P.polyfit(x,y,3,full=True) + >>> np.random.seed(123) >>> c # c[0], c[2] should be approx. 0, c[1] approx. -1, c[3] approx. 1 - array([ 0.01909725, -1.30598256, -0.00577963, 1.02644286]) + array([ 0.01909725, -1.30598256, -0.00577963, 1.02644286]) # may vary >>> stats # note the large SSR, explaining the rather poor results - [array([ 38.06116253]), 4, array([ 1.38446749, 1.32119158, 0.50443316, - 0.28853036]), 1.1324274851176597e-014] + [array([ 38.06116253]), 4, array([ 1.38446749, 1.32119158, 0.50443316, # may vary + 0.28853036]), 1.1324274851176597e-014] Same thing without the added noise >>> y = x**3 - x >>> c, stats = P.polyfit(x,y,3,full=True) >>> c # c[0], c[2] should be "very close to 0", c[1] ~= -1, c[3] ~= 1 - array([ -1.73362882e-17, -1.00000000e+00, -2.67471909e-16, - 1.00000000e+00]) + array([-6.36925336e-18, -1.00000000e+00, -4.08053781e-16, 1.00000000e+00]) >>> stats # note the minuscule SSR - [array([ 7.46346754e-31]), 4, array([ 1.38446749, 1.32119158, - 0.50443316, 0.28853036]), 1.1324274851176597e-014] + [array([ 7.46346754e-31]), 4, array([ 1.38446749, 1.32119158, # may vary + 0.50443316, 0.28853036]), 1.1324274851176597e-014] """ x = np.asarray(x) + 0.0 @@ -1591,7 +1592,7 @@ def polyroots(c): dtype('float64') >>> j = complex(0,1) >>> poly.polyroots(poly.polyfromroots((-j,0,j))) - array([ 0.00000000e+00+0.j, 0.00000000e+00+1.j, 2.77555756e-17-1.j]) + array([ 0.00000000e+00+0.j, 0.00000000e+00+1.j, 2.77555756e-17-1.j]) # may vary """ # c is a trimmed copy diff --git a/numpy/polynomial/polyutils.py b/numpy/polynomial/polyutils.py index c1ed0c9b3..eff4a8ee1 100644 --- a/numpy/polynomial/polyutils.py +++ b/numpy/polynomial/polyutils.py @@ -156,19 +156,19 @@ def as_series(alist, trim=True): >>> from numpy.polynomial import polyutils as pu >>> a = np.arange(4) >>> pu.as_series(a) - [array([ 0.]), array([ 1.]), array([ 2.]), array([ 3.])] + [array([0.]), array([1.]), array([2.]), array([3.])] >>> b = np.arange(6).reshape((2,3)) >>> pu.as_series(b) - [array([ 0., 1., 2.]), array([ 3., 4., 5.])] + [array([0., 1., 2.]), array([3., 4., 5.])] >>> pu.as_series((1, np.arange(3), np.arange(2, dtype=np.float16))) - [array([ 1.]), array([ 0., 1., 2.]), array([ 0., 1.])] + [array([1.]), array([0., 1., 2.]), array([0., 1.])] >>> pu.as_series([2, [1.1, 0.]]) - [array([ 2.]), array([ 1.1])] + [array([2.]), array([1.1])] >>> pu.as_series([2, [1.1, 0.]], trim=False) - [array([ 2.]), array([ 1.1, 0. ])] + [array([2.]), array([1.1, 0. ])] """ arrays = [np.array(a, ndmin=1, copy=0) for a in alist] @@ -233,12 +233,12 @@ def trimcoef(c, tol=0): -------- >>> from numpy.polynomial import polyutils as pu >>> pu.trimcoef((0,0,3,0,5,0,0)) - array([ 0., 0., 3., 0., 5.]) + array([0., 0., 3., 0., 5.]) >>> pu.trimcoef((0,0,1e-3,0,1e-5,0,0),1e-3) # item == tol is trimmed - array([ 0.]) + array([0.]) >>> i = complex(0,1) # works for complex >>> pu.trimcoef((3e-4,1e-3*(1-i),5e-4,2e-5*(1+i)), 1e-3) - array([ 0.0003+0.j , 0.0010-0.001j]) + array([0.0003+0.j , 0.001 -0.001j]) """ if tol < 0: @@ -332,10 +332,10 @@ def mapparms(old, new): >>> pu.mapparms((-1,1),(-1,1)) (0.0, 1.0) >>> pu.mapparms((1,-1),(-1,1)) - (0.0, -1.0) + (-0.0, -1.0) >>> i = complex(0,1) >>> pu.mapparms((-i,-1),(1,i)) - ((1+1j), (1+0j)) + ((1+1j), (1-0j)) """ oldlen = old[1] - old[0] @@ -390,10 +390,10 @@ def mapdomain(x, old, new): >>> x = np.linspace(-1,1,6); x array([-1. , -0.6, -0.2, 0.2, 0.6, 1. ]) >>> x_out = pu.mapdomain(x, old_domain, new_domain); x_out - array([ 0. , 1.25663706, 2.51327412, 3.76991118, 5.02654825, + array([ 0. , 1.25663706, 2.51327412, 3.76991118, 5.02654825, # may vary 6.28318531]) >>> x - pu.mapdomain(x_out, new_domain, old_domain) - array([ 0., 0., 0., 0., 0., 0.]) + array([0., 0., 0., 0., 0., 0.]) Also works for complex numbers (and thus can be used to map any line in the complex plane to any other line therein). @@ -402,9 +402,9 @@ def mapdomain(x, old, new): >>> old = (-1 - i, 1 + i) >>> new = (-1 + i, 1 - i) >>> z = np.linspace(old[0], old[1], 6); z - array([-1.0-1.j , -0.6-0.6j, -0.2-0.2j, 0.2+0.2j, 0.6+0.6j, 1.0+1.j ]) - >>> new_z = P.mapdomain(z, old, new); new_z - array([-1.0+1.j , -0.6+0.6j, -0.2+0.2j, 0.2-0.2j, 0.6-0.6j, 1.0-1.j ]) + array([-1. -1.j , -0.6-0.6j, -0.2-0.2j, 0.2+0.2j, 0.6+0.6j, 1. +1.j ]) + >>> new_z = pu.mapdomain(z, old, new); new_z + array([-1.0+1.j , -0.6+0.6j, -0.2+0.2j, 0.2-0.2j, 0.6-0.6j, 1.0-1.j ]) # may vary """ x = np.asanyarray(x) diff --git a/numpy/random/mtrand/distributions.c b/numpy/random/mtrand/distributions.c index b7e157915..2548a646e 100644 --- a/numpy/random/mtrand/distributions.c +++ b/numpy/random/mtrand/distributions.c @@ -650,6 +650,9 @@ double rk_pareto(rk_state *state, double a) double rk_weibull(rk_state *state, double a) { + if (a == 0.0) { + return 0.0; + } return pow(rk_standard_exponential(state), 1./a); } diff --git a/numpy/random/mtrand/mtrand.pyx b/numpy/random/mtrand/mtrand.pyx index 6b054a20f..134bc2e09 100644 --- a/numpy/random/mtrand/mtrand.pyx +++ b/numpy/random/mtrand/mtrand.pyx @@ -1,3 +1,5 @@ +# cython: language_level=3 + # mtrand.pyx -- A Pyrex wrapper of Jean-Sebastien Roy's RandomKit # # Copyright 2005 Robert Kern (robert.kern@gmail.com) @@ -844,16 +846,16 @@ cdef class RandomState: Examples -------- >>> np.random.random_sample() - 0.47108547995356098 + 0.47108547995356098 # random >>> type(np.random.random_sample()) - <type 'float'> + <class 'float'> >>> np.random.random_sample((5,)) - array([ 0.30220482, 0.86820401, 0.1654503 , 0.11659149, 0.54323428]) + array([ 0.30220482, 0.86820401, 0.1654503 , 0.11659149, 0.54323428]) # random Three-by-two array of random numbers from [-5, 0): >>> 5 * np.random.random_sample((3, 2)) - 5 - array([[-3.99149989, -0.52338984], + array([[-3.99149989, -0.52338984], # random [-2.99091858, -0.79479508], [-1.23204345, -1.75224494]]) @@ -954,14 +956,14 @@ cdef class RandomState: Examples -------- >>> np.random.randint(2, size=10) - array([1, 0, 0, 0, 1, 1, 0, 0, 1, 0]) + array([1, 0, 0, 0, 1, 1, 0, 0, 1, 0]) # random >>> np.random.randint(1, size=10) array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) Generate a 2 x 4 array of ints between 0 and 4, inclusive: >>> np.random.randint(5, size=(2, 4)) - array([[4, 0, 2, 1], + array([[4, 0, 2, 1], # random [3, 2, 2, 0]]) """ @@ -1076,34 +1078,34 @@ cdef class RandomState: Generate a uniform random sample from np.arange(5) of size 3: >>> np.random.choice(5, 3) - array([0, 3, 4]) + array([0, 3, 4]) # random >>> #This is equivalent to np.random.randint(0,5,3) Generate a non-uniform random sample from np.arange(5) of size 3: >>> np.random.choice(5, 3, p=[0.1, 0, 0.3, 0.6, 0]) - array([3, 3, 0]) + array([3, 3, 0]) # random Generate a uniform random sample from np.arange(5) of size 3 without replacement: >>> np.random.choice(5, 3, replace=False) - array([3,1,0]) + array([3,1,0]) # random >>> #This is equivalent to np.random.permutation(np.arange(5))[:3] Generate a non-uniform random sample from np.arange(5) of size 3 without replacement: >>> np.random.choice(5, 3, replace=False, p=[0.1, 0, 0.3, 0.6, 0]) - array([2, 3, 0]) + array([2, 3, 0]) # random Any of the above can be repeated with an arbitrary array-like instead of just integers. For instance: >>> aa_milne_arr = ['pooh', 'rabbit', 'piglet', 'Christopher'] >>> np.random.choice(aa_milne_arr, 5, p=[0.5, 0.1, 0.1, 0.3]) - array(['pooh', 'pooh', 'pooh', 'Christopher', 'piglet'], - dtype='|S11') + array(['pooh', 'pooh', 'pooh', 'Christopher', 'piglet'], # random + dtype='<U11') """ @@ -1470,11 +1472,11 @@ cdef class RandomState: Examples -------- >>> np.random.random_integers(5) - 4 + 4 # random >>> type(np.random.random_integers(5)) - <type 'int'> + <class 'numpy.int64'> >>> np.random.random_integers(5, size=(3,2)) - array([[5, 4], + array([[5, 4], # random [3, 3], [4, 5]]) @@ -1483,7 +1485,7 @@ cdef class RandomState: :math:`{0, 5/8, 10/8, 15/8, 20/8}`): >>> 2.5 * (np.random.random_integers(5, size=(5,)) - 1) / 4. - array([ 0.625, 1.25 , 0.625, 0.625, 2.5 ]) + array([ 0.625, 1.25 , 0.625, 0.625, 2.5 ]) # random Roll two six sided dice 1000 times and sum the results: @@ -1679,9 +1681,9 @@ cdef class RandomState: Parameters ---------- a : float or array_like of floats - Alpha, non-negative. + Alpha, positive (>0). b : float or array_like of floats - Beta, non-negative. + Beta, positive (>0). size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. If size is ``None`` (default), @@ -2068,8 +2070,8 @@ cdef class RandomState: The lower bound for the top 1% of the samples is : - >>> sort(s)[-10] - 7.61988120985 + >>> np.sort(s)[-10] + 7.61988120985 # random So there is about a 1% chance that the F statistic will exceed 7.62, the measured value is 36, so the null hypothesis is rejected at the 1% @@ -2166,6 +2168,7 @@ cdef class RandomState: >>> NF = np.histogram(nc_vals, bins=50, density=True) >>> c_vals = np.random.f(dfnum, dfden, 1000000) >>> F = np.histogram(c_vals, bins=50, density=True) + >>> import matplotlib.pyplot as plt >>> plt.plot(F[1][1:], F[0]) >>> plt.plot(NF[1][1:], NF[0]) >>> plt.show() @@ -2261,7 +2264,7 @@ cdef class RandomState: Examples -------- >>> np.random.chisquare(2,4) - array([ 1.89920014, 9.00867716, 3.13710533, 5.62318272]) + array([ 1.89920014, 9.00867716, 3.13710533, 5.62318272]) # random """ cdef ndarray odf @@ -2443,6 +2446,7 @@ cdef class RandomState: -------- Draw samples and plot the distribution: + >>> import matplotlib.pyplot as plt >>> s = np.random.standard_cauchy(1000000) >>> s = s[(s>-25) & (s<25)] # truncate distribution so it plots well >>> plt.hist(s, bins=100) @@ -2785,7 +2789,7 @@ cdef class RandomState: Parameters ---------- a : float or array_like of floats - Shape of the distribution. Should be greater than zero. + Shape parameter of the distribution. Must be nonnegative. size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. If size is ``None`` (default), @@ -3279,12 +3283,13 @@ cdef class RandomState: >>> loc, scale = 10, 1 >>> s = np.random.logistic(loc, scale, 10000) + >>> import matplotlib.pyplot as plt >>> count, bins, ignored = plt.hist(s, bins=50) # plot against distribution >>> def logist(x, loc, scale): - ... return exp((loc-x)/scale)/(scale*(1+exp((loc-x)/scale))**2) + ... return np.exp((loc-x)/scale)/(scale*(1+np.exp((loc-x)/scale))**2) >>> plt.plot(bins, logist(bins, loc, scale)*count.max()/\\ ... logist(bins, loc, scale).max()) >>> plt.show() @@ -3479,6 +3484,7 @@ cdef class RandomState: -------- Draw values from the distribution and plot the histogram + >>> from matplotlib.pyplot import hist >>> values = hist(np.random.rayleigh(3, 100000), bins=200, density=True) Wave heights tend to follow a Rayleigh distribution. If the mean wave @@ -3492,7 +3498,7 @@ cdef class RandomState: The percentage of waves larger than 3 meters is: >>> 100.*sum(s>3)/1000000. - 0.087300000000000003 + 0.087300000000000003 # random """ cdef ndarray oscale @@ -3873,9 +3879,9 @@ cdef class RandomState: single success after drilling 5 wells, after 6 wells, etc.? >>> s = np.random.negative_binomial(1, 0.1, 100000) - >>> for i in range(1, 11): + >>> for i in range(1, 11): # doctest: +SKIP ... probability = sum(s<i) / 100000. - ... print i, "wells drilled, probability of one success =", probability + ... print(i, "wells drilled, probability of one success =", probability) """ cdef ndarray on @@ -4233,6 +4239,7 @@ cdef class RandomState: >>> ngood, nbad, nsamp = 100, 2, 10 # number of good, number of bad, and number of samples >>> s = np.random.hypergeometric(ngood, nbad, nsamp, 1000) + >>> from matplotlib.pyplot import hist >>> hist(s) # note that it is very unlikely to grab both bad items @@ -4342,14 +4349,15 @@ cdef class RandomState: >>> a = .6 >>> s = np.random.logseries(a, 10000) + >>> import matplotlib.pyplot as plt >>> count, bins, ignored = plt.hist(s) # plot against distribution >>> def logseries(k, p): - ... return -p**k/(k*log(1-p)) + ... return -p**k/(k*np.log(1-p)) >>> plt.plot(bins, logseries(bins, a)*count.max()/ - logseries(bins, a).max(), 'r') + ... logseries(bins, a).max(), 'r') >>> plt.show() """ @@ -4474,7 +4482,7 @@ cdef class RandomState: standard deviation: >>> list((x[0,0,:] - mean) < 0.6) - [True, True] + [True, True] # random """ from numpy.dual import svd @@ -4580,14 +4588,14 @@ cdef class RandomState: Throw a dice 20 times: >>> np.random.multinomial(20, [1/6.]*6, size=1) - array([[4, 1, 7, 5, 2, 1]]) + array([[4, 1, 7, 5, 2, 1]]) # random It landed 4 times on 1, once on 2, etc. Now, throw the dice 20 times, and 20 times again: >>> np.random.multinomial(20, [1/6.]*6, size=2) - array([[3, 4, 3, 3, 4, 3], + array([[3, 4, 3, 3, 4, 3], # random [2, 4, 3, 4, 0, 7]]) For the first run, we threw 3 times 1, 4 times 2, etc. For the second, @@ -4596,7 +4604,7 @@ cdef class RandomState: A loaded die is more likely to land on number 6: >>> np.random.multinomial(100, [1/7.]*5 + [2/7.]) - array([11, 16, 14, 17, 16, 26]) + array([11, 16, 14, 17, 16, 26]) # random The probability inputs should be normalized. As an implementation detail, the value of the last entry is ignored and assumed to take @@ -4605,7 +4613,7 @@ cdef class RandomState: other should be sampled like so: >>> np.random.multinomial(100, [1.0 / 3, 2.0 / 3]) # RIGHT - array([38, 62]) + array([38, 62]) # random not like: @@ -4659,8 +4667,9 @@ cdef class RandomState: Draw `size` samples of dimension k from a Dirichlet distribution. A Dirichlet-distributed random variable can be seen as a multivariate - generalization of a Beta distribution. Dirichlet pdf is the conjugate - prior of a multinomial in Bayesian inference. + generalization of a Beta distribution. The Dirichlet distribution + is a conjugate prior of a multinomial distribution in Bayesian + inference. Parameters ---------- @@ -4684,13 +4693,24 @@ cdef class RandomState: Notes ----- - .. math:: X \\approx \\prod_{i=1}^{k}{x^{\\alpha_i-1}_i} - Uses the following property for computation: for each dimension, - draw a random sample y_i from a standard gamma generator of shape - `alpha_i`, then - :math:`X = \\frac{1}{\\sum_{i=1}^k{y_i}} (y_1, \\ldots, y_n)` is - Dirichlet distributed. + The Dirichlet distribution is a distribution over vectors + :math:`x` that fulfil the conditions :math:`x_i>0` and + :math:`\\sum_{i=1}^k x_i = 1`. + + The probability density function :math:`p` of a + Dirichlet-distributed random vector :math:`X` is + proportional to + + .. math:: p(x) \\propto \\prod_{i=1}^{k}{x^{\\alpha_i-1}_i}, + + where :math:`\\alpha` is a vector containing the positive + concentration parameters. + + The method uses the following property for computation: let :math:`Y` + be a random vector which has components that follow a standard gamma + distribution, then :math:`X = \\frac{1}{\\sum_{i=1}^k{Y_i}} Y` + is Dirichlet-distributed References ---------- @@ -4710,6 +4730,7 @@ cdef class RandomState: >>> s = np.random.dirichlet((10, 5, 3), 20).transpose() + >>> import matplotlib.pyplot as plt >>> plt.barh(range(20), s[0]) >>> plt.barh(range(20), s[1], left=s[0], color='g') >>> plt.barh(range(20), s[2], left=s[0]+s[1], color='r') @@ -4798,14 +4819,14 @@ cdef class RandomState: >>> arr = np.arange(10) >>> np.random.shuffle(arr) >>> arr - [1 7 5 2 9 4 3 6 0 8] + [1 7 5 2 9 4 3 6 0 8] # random Multi-dimensional arrays are only shuffled along the first axis: >>> arr = np.arange(9).reshape((3, 3)) >>> np.random.shuffle(arr) >>> arr - array([[3, 4, 5], + array([[3, 4, 5], # random [6, 7, 8], [0, 1, 2]]) @@ -4885,14 +4906,14 @@ cdef class RandomState: Examples -------- >>> np.random.permutation(10) - array([1, 7, 4, 3, 0, 9, 2, 5, 8, 6]) + array([1, 7, 4, 3, 0, 9, 2, 5, 8, 6]) # random >>> np.random.permutation([1, 4, 9, 12, 15]) - array([15, 1, 9, 4, 12]) + array([15, 1, 9, 4, 12]) # random >>> arr = np.arange(9).reshape((3, 3)) >>> np.random.permutation(arr) - array([[6, 7, 8], + array([[6, 7, 8], # random [0, 1, 2], [3, 4, 5]]) diff --git a/numpy/random/mtrand/numpy.pxd b/numpy/random/mtrand/numpy.pxd index 9092fa113..1b4fe6c10 100644 --- a/numpy/random/mtrand/numpy.pxd +++ b/numpy/random/mtrand/numpy.pxd @@ -1,3 +1,5 @@ +# cython: language_level=3 + # :Author: Travis Oliphant from cpython.exc cimport PyErr_Print diff --git a/numpy/random/tests/test_random.py b/numpy/random/tests/test_random.py index 276517363..d0bb92a73 100644 --- a/numpy/random/tests/test_random.py +++ b/numpy/random/tests/test_random.py @@ -942,7 +942,8 @@ class TestRandomDist(object): assert_array_almost_equal(actual, desired, decimal=15) def test_weibull_0(self): - assert_equal(np.random.weibull(a=0), 0) + np.random.seed(self.seed) + assert_equal(np.random.weibull(a=0, size=12), np.zeros(12)) assert_raises(ValueError, np.random.weibull, a=-0.) def test_zipf(self): diff --git a/numpy/testing/_private/nosetester.py b/numpy/testing/_private/nosetester.py index 1728d9d1f..19569a509 100644 --- a/numpy/testing/_private/nosetester.py +++ b/numpy/testing/_private/nosetester.py @@ -92,7 +92,7 @@ def run_module_suite(file_to_run=None, argv=None): Alternatively, calling:: - >>> run_module_suite(file_to_run="numpy/tests/test_matlib.py") + >>> run_module_suite(file_to_run="numpy/tests/test_matlib.py") # doctest: +SKIP from an interpreter will run all the test routine in 'test_matlib.py'. """ diff --git a/numpy/testing/_private/parameterized.py b/numpy/testing/_private/parameterized.py index 53e67517d..a5fa4fb5e 100644 --- a/numpy/testing/_private/parameterized.py +++ b/numpy/testing/_private/parameterized.py @@ -190,7 +190,7 @@ def parameterized_argument_value_pairs(func, p): in zip(named_args, argspec.defaults or []) ]) - seen_arg_names = set([ n for (n, _) in result ]) + seen_arg_names = {n for (n, _) in result} keywords = QuietOrderedDict(sorted([ (name, p.kwargs[name]) for name in p.kwargs diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index a3832fcde..1f7b516b3 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -318,8 +318,9 @@ def assert_equal(actual, desired, err_msg='', verbose=True): Examples -------- >>> np.testing.assert_equal([4,5], [4,6]) - ... - <type 'exceptions.AssertionError'>: + Traceback (most recent call last): + ... + AssertionError: Items are not equal: item=1 ACTUAL: 5 @@ -352,7 +353,7 @@ def assert_equal(actual, desired, err_msg='', verbose=True): # XXX: catch ValueError for subclasses of ndarray where iscomplex fail try: usecomplex = iscomplexobj(actual) or iscomplexobj(desired) - except ValueError: + except (ValueError, TypeError): usecomplex = False if usecomplex: @@ -510,21 +511,24 @@ def assert_almost_equal(actual,desired,decimal=7,err_msg='',verbose=True): >>> import numpy.testing as npt >>> npt.assert_almost_equal(2.3333333333333, 2.33333334) >>> npt.assert_almost_equal(2.3333333333333, 2.33333334, decimal=10) - ... - <type 'exceptions.AssertionError'>: - Items are not equal: - ACTUAL: 2.3333333333333002 - DESIRED: 2.3333333399999998 + Traceback (most recent call last): + ... + AssertionError: + Arrays are not almost equal to 10 decimals + ACTUAL: 2.3333333333333 + DESIRED: 2.33333334 >>> npt.assert_almost_equal(np.array([1.0,2.3333333333333]), ... np.array([1.0,2.33333334]), decimal=9) - ... - <type 'exceptions.AssertionError'>: - Arrays are not almost equal - <BLANKLINE> - (mismatch 50.0%) - x: array([ 1. , 2.33333333]) - y: array([ 1. , 2.33333334]) + Traceback (most recent call last): + ... + AssertionError: + Arrays are not almost equal to 9 decimals + Mismatch: 50% + Max absolute difference: 6.66669964e-09 + Max relative difference: 2.85715698e-09 + x: array([1. , 2.333333333]) + y: array([1. , 2.33333334]) """ __tracebackhide__ = True # Hide traceback for py.test @@ -626,14 +630,15 @@ def assert_approx_equal(actual,desired,significant=7,err_msg='',verbose=True): -------- >>> np.testing.assert_approx_equal(0.12345677777777e-20, 0.1234567e-20) >>> np.testing.assert_approx_equal(0.12345670e-20, 0.12345671e-20, - significant=8) + ... significant=8) >>> np.testing.assert_approx_equal(0.12345670e-20, 0.12345672e-20, - significant=8) - ... - <type 'exceptions.AssertionError'>: + ... significant=8) + Traceback (most recent call last): + ... + AssertionError: Items are not equal to 8 significant digits: - ACTUAL: 1.234567e-021 - DESIRED: 1.2345672000000001e-021 + ACTUAL: 1.234567e-21 + DESIRED: 1.2345672e-21 the evaluated condition that raises the exception is @@ -660,10 +665,10 @@ def assert_approx_equal(actual,desired,significant=7,err_msg='',verbose=True): sc_actual = actual/scale except ZeroDivisionError: sc_actual = 0.0 - msg = build_err_msg([actual, desired], err_msg, - header='Items are not equal to %d significant digits:' % - significant, - verbose=verbose) + msg = build_err_msg( + [actual, desired], err_msg, + header='Items are not equal to %d significant digits:' % significant, + verbose=verbose) try: # If one of desired/actual is not finite, handle it specially here: # check that both are nan if any is a nan, and test for equality @@ -686,12 +691,14 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, header='', precision=6, equal_nan=True, equal_inf=True): __tracebackhide__ = True # Hide traceback for py.test - from numpy.core import array, isnan, inf, bool_ - from numpy.core.fromnumeric import all as npall + from numpy.core import array, array2string, isnan, inf, bool_, errstate x = array(x, copy=False, subok=True) y = array(y, copy=False, subok=True) + # original array for output formating + ox, oy = x, y + def isnumber(x): return x.dtype.char in '?bhilqpBHILQPefdgFDG' @@ -705,15 +712,20 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, at the same locations. """ - # Both the != True comparison here and the cast to bool_ at the end are - # done to deal with `masked`, which cannot be compared usefully, and - # for which np.all yields masked. The use of the function np.all is - # for back compatibility with ndarray subclasses that changed the - # return values of the all method. We are not committed to supporting - # such subclasses, but some used to work. x_id = func(x) y_id = func(y) - if npall(x_id == y_id) != True: + # We include work-arounds here to handle three types of slightly + # pathological ndarray subclasses: + # (1) all() on `masked` array scalars can return masked arrays, so we + # use != True + # (2) __eq__ on some ndarray subclasses returns Python booleans + # instead of element-wise comparisons, so we cast to bool_() and + # use isinstance(..., bool) checks + # (3) subclasses with bare-bones __array_function__ implemenations may + # not implement np.all(), so favor using the .all() method + # We are not committed to supporting such subclasses, but it's nice to + # support them if possible. + if bool_(x_id == y_id).all() != True: msg = build_err_msg([x, y], err_msg + '\nx and y %s location mismatch:' % (hasval), verbose=verbose, header=header, @@ -721,9 +733,9 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, raise AssertionError(msg) # If there is a scalar, then here we know the array has the same # flag as it everywhere, so we should return the scalar flag. - if x_id.ndim == 0: + if isinstance(x_id, bool) or x_id.ndim == 0: return bool_(x_id) - elif y_id.ndim == 0: + elif isinstance(x_id, bool) or y_id.ndim == 0: return bool_(y_id) else: return y_id @@ -775,15 +787,31 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, reduced = val.ravel() cond = reduced.all() reduced = reduced.tolist() + # The below comparison is a hack to ensure that fully masked # results, for which val.ravel().all() returns np.ma.masked, # do not trigger a failure (np.ma.masked != True evaluates as # np.ma.masked, which is falsy). if cond != True: - match = 100-100.0*reduced.count(1)/len(reduced) - msg = build_err_msg([x, y], - err_msg - + '\n(mismatch %s%%)' % (match,), + mismatch = 100.0 * reduced.count(0) / ox.size + remarks = ['Mismatch: {:.3g}%'.format(mismatch)] + + with errstate(invalid='ignore', divide='ignore'): + # ignore errors for non-numeric types + with contextlib.suppress(TypeError): + error = abs(x - y) + max_abs_error = error.max() + remarks.append('Max absolute difference: ' + + array2string(max_abs_error)) + + # note: this definition of relative error matches that one + # used by assert_allclose (found in np.isclose) + max_rel_error = (error / abs(y)).max() + remarks.append('Max relative difference: ' + + array2string(max_rel_error)) + + err_msg += '\n' + '\n'.join(remarks) + msg = build_err_msg([ox, oy], err_msg, verbose=verbose, header=header, names=('x', 'y'), precision=precision) raise AssertionError(msg) @@ -843,14 +871,15 @@ def assert_array_equal(x, y, err_msg='', verbose=True): >>> np.testing.assert_array_equal([1.0,np.pi,np.nan], ... [1, np.sqrt(np.pi)**2, np.nan]) - ... - <type 'exceptions.ValueError'>: + Traceback (most recent call last): + ... AssertionError: Arrays are not equal - <BLANKLINE> - (mismatch 50.0%) - x: array([ 1. , 3.14159265, NaN]) - y: array([ 1. , 3.14159265, NaN]) + Mismatch: 33.3% + Max absolute difference: 4.4408921e-16 + Max relative difference: 1.41357986e-16 + x: array([1. , 3.141593, nan]) + y: array([1. , 3.141593, nan]) Use `assert_allclose` or one of the nulp (number of floating point values) functions for these cases instead: @@ -915,30 +944,33 @@ def assert_array_almost_equal(x, y, decimal=6, err_msg='', verbose=True): the first assert does not raise an exception >>> np.testing.assert_array_almost_equal([1.0,2.333,np.nan], - [1.0,2.333,np.nan]) + ... [1.0,2.333,np.nan]) >>> np.testing.assert_array_almost_equal([1.0,2.33333,np.nan], ... [1.0,2.33339,np.nan], decimal=5) - ... - <type 'exceptions.AssertionError'>: + Traceback (most recent call last): + ... AssertionError: - Arrays are not almost equal - <BLANKLINE> - (mismatch 50.0%) - x: array([ 1. , 2.33333, NaN]) - y: array([ 1. , 2.33339, NaN]) + Arrays are not almost equal to 5 decimals + Mismatch: 33.3% + Max absolute difference: 6.e-05 + Max relative difference: 2.57136612e-05 + x: array([1. , 2.33333, nan]) + y: array([1. , 2.33339, nan]) >>> np.testing.assert_array_almost_equal([1.0,2.33333,np.nan], ... [1.0,2.33333, 5], decimal=5) - <type 'exceptions.ValueError'>: - ValueError: - Arrays are not almost equal - x: array([ 1. , 2.33333, NaN]) - y: array([ 1. , 2.33333, 5. ]) + Traceback (most recent call last): + ... + AssertionError: + Arrays are not almost equal to 5 decimals + x and y nan location mismatch: + x: array([1. , 2.33333, nan]) + y: array([1. , 2.33333, 5. ]) """ __tracebackhide__ = True # Hide traceback for py.test - from numpy.core import around, number, float_, result_type, array + from numpy.core import number, float_, result_type, array from numpy.core.numerictypes import issubdtype from numpy.core.fromnumeric import any as npany @@ -1015,27 +1047,34 @@ def assert_array_less(x, y, err_msg='', verbose=True): -------- >>> np.testing.assert_array_less([1.0, 1.0, np.nan], [1.1, 2.0, np.nan]) >>> np.testing.assert_array_less([1.0, 1.0, np.nan], [1, 2.0, np.nan]) - ... - <type 'exceptions.ValueError'>: + Traceback (most recent call last): + ... + AssertionError: Arrays are not less-ordered - (mismatch 50.0%) - x: array([ 1., 1., NaN]) - y: array([ 1., 2., NaN]) + Mismatch: 33.3% + Max absolute difference: 1. + Max relative difference: 0.5 + x: array([ 1., 1., nan]) + y: array([ 1., 2., nan]) >>> np.testing.assert_array_less([1.0, 4.0], 3) - ... - <type 'exceptions.ValueError'>: + Traceback (most recent call last): + ... + AssertionError: Arrays are not less-ordered - (mismatch 50.0%) - x: array([ 1., 4.]) + Mismatch: 50% + Max absolute difference: 2. + Max relative difference: 0.66666667 + x: array([1., 4.]) y: array(3) >>> np.testing.assert_array_less([1.0, 2.0, 3.0], [4]) - ... - <type 'exceptions.ValueError'>: + Traceback (most recent call last): + ... + AssertionError: Arrays are not less-ordered (shapes (3,), (1,) mismatch) - x: array([ 1., 2., 3.]) + x: array([1., 2., 3.]) y: array([4]) """ @@ -1140,7 +1179,7 @@ def rundocs(filename=None, raise_on_error=True): argument to the ``test()`` call. For example, to run all tests (including doctests) for `numpy.lib`: - >>> np.lib.test(doctests=True) #doctest: +SKIP + >>> np.lib.test(doctests=True) # doctest: +SKIP """ from numpy.compat import npy_load_module import doctest @@ -1322,7 +1361,7 @@ def decorate_methods(cls, decorator, testmatch=None): return -def measure(code_str,times=1,label=None): +def measure(code_str, times=1, label=None): """ Return elapsed time for executing code in the namespace of the caller. @@ -1349,9 +1388,9 @@ def measure(code_str,times=1,label=None): Examples -------- - >>> etime = np.testing.measure('for i in range(1000): np.sqrt(i**2)', - ... times=times) - >>> print("Time for a single execution : ", etime / times, "s") + >>> times = 10 + >>> etime = np.testing.measure('for i in range(1000): np.sqrt(i**2)', times=times) + >>> print("Time for a single execution : ", etime / times, "s") # doctest: +SKIP Time for a single execution : 0.005 s """ @@ -1436,7 +1475,7 @@ def assert_allclose(actual, desired, rtol=1e-7, atol=0, equal_nan=True, -------- >>> x = [1e-5, 1e-3, 1e-1] >>> y = np.arccos(np.cos(x)) - >>> assert_allclose(x, y, rtol=1e-5, atol=0) + >>> np.testing.assert_allclose(x, y, rtol=1e-5, atol=0) """ __tracebackhide__ = True # Hide traceback for py.test @@ -1890,7 +1929,8 @@ class clear_and_catch_warnings(warnings.catch_warnings): Examples -------- >>> import warnings - >>> with clear_and_catch_warnings(modules=[np.core.fromnumeric]): + >>> with np.testing.clear_and_catch_warnings( + ... modules=[np.core.fromnumeric]): ... warnings.simplefilter('always') ... warnings.filterwarnings('ignore', module='np.core.fromnumeric') ... # do something that raises a warning but ignore those in @@ -1971,25 +2011,28 @@ class suppress_warnings(object): Examples -------- - >>> with suppress_warnings() as sup: - ... sup.filter(DeprecationWarning, "Some text") - ... sup.filter(module=np.ma.core) - ... log = sup.record(FutureWarning, "Does this occur?") - ... command_giving_warnings() - ... # The FutureWarning was given once, the filtered warnings were - ... # ignored. All other warnings abide outside settings (may be - ... # printed/error) - ... assert_(len(log) == 1) - ... assert_(len(sup.log) == 1) # also stored in log attribute - - Or as a decorator: - - >>> sup = suppress_warnings() - >>> sup.filter(module=np.ma.core) # module must match exact - >>> @sup - >>> def some_function(): - ... # do something which causes a warning in np.ma.core - ... pass + + With a context manager:: + + with np.testing.suppress_warnings() as sup: + sup.filter(DeprecationWarning, "Some text") + sup.filter(module=np.ma.core) + log = sup.record(FutureWarning, "Does this occur?") + command_giving_warnings() + # The FutureWarning was given once, the filtered warnings were + # ignored. All other warnings abide outside settings (may be + # printed/error) + assert_(len(log) == 1) + assert_(len(sup.log) == 1) # also stored in log attribute + + Or as a decorator:: + + sup = np.testing.suppress_warnings() + sup.filter(module=np.ma.core) # module must match exactly + @sup + def some_function(): + # do something which causes a warning in np.ma.core + pass """ def __init__(self, forwarding_rule="always"): self._entered = False diff --git a/numpy/testing/tests/test_decorators.py b/numpy/testing/tests/test_decorators.py index b8283d9de..bb3ea1acb 100644 --- a/numpy/testing/tests/test_decorators.py +++ b/numpy/testing/tests/test_decorators.py @@ -13,7 +13,7 @@ from numpy.testing import ( try: - import nose + import nose # noqa: F401 except ImportError: HAVE_NOSE = False else: diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py index e0d3414f7..c376a3852 100644 --- a/numpy/testing/tests/test_utils.py +++ b/numpy/testing/tests/test_utils.py @@ -158,6 +158,41 @@ class TestArrayEqual(_GenericTest): self._test_equal(a, b) self._test_equal(b, a) + def test_subclass_that_overrides_eq(self): + # While we cannot guarantee testing functions will always work for + # subclasses, the tests should ideally rely only on subclasses having + # comparison operators, not on them being able to store booleans + # (which, e.g., astropy Quantity cannot usefully do). See gh-8452. + class MyArray(np.ndarray): + def __eq__(self, other): + return bool(np.equal(self, other).all()) + + def __ne__(self, other): + return not self == other + + a = np.array([1., 2.]).view(MyArray) + b = np.array([2., 3.]).view(MyArray) + assert_(type(a == a), bool) + assert_(a == a) + assert_(a != b) + self._test_equal(a, a) + self._test_not_equal(a, b) + self._test_not_equal(b, a) + + def test_subclass_that_does_not_implement_npall(self): + class MyArray(np.ndarray): + def __array_function__(self, *args, **kwargs): + return NotImplemented + + a = np.array([1., 2.]).view(MyArray) + b = np.array([2., 3.]).view(MyArray) + if np.core.overrides.ENABLE_ARRAY_FUNCTION: + with assert_raises(TypeError): + np.all(a) + self._test_equal(a, a) + self._test_not_equal(a, b) + self._test_not_equal(b, a) + class TestBuildErrorMessage(object): @@ -292,24 +327,22 @@ class TestEqual(TestArrayEqual): self._test_not_equal(x, y) def test_error_message(self): - try: + with pytest.raises(AssertionError) as exc_info: self._assert_func(np.array([1, 2]), np.array([[1, 2]])) - except AssertionError as e: - msg = str(e) - msg2 = msg.replace("shapes (2L,), (1L, 2L)", "shapes (2,), (1, 2)") - msg_reference = textwrap.dedent("""\ + msg = str(exc_info.value) + msg2 = msg.replace("shapes (2L,), (1L, 2L)", "shapes (2,), (1, 2)") + msg_reference = textwrap.dedent("""\ - Arrays are not equal + Arrays are not equal - (shapes (2,), (1, 2) mismatch) - x: array([1, 2]) - y: array([[1, 2]])""") - try: - assert_equal(msg, msg_reference) - except AssertionError: - assert_equal(msg2, msg_reference) - else: - raise AssertionError("Did not raise") + (shapes (2,), (1, 2) mismatch) + x: array([1, 2]) + y: array([[1, 2]])""") + + try: + assert_equal(msg, msg_reference) + except AssertionError: + assert_equal(msg2, msg_reference) class TestArrayAlmostEqual(_GenericTest): @@ -469,29 +502,58 @@ class TestAlmostEqual(_GenericTest): self._test_not_equal(x, z) def test_error_message(self): - """Check the message is formatted correctly for the decimal value""" + """Check the message is formatted correctly for the decimal value. + Also check the message when input includes inf or nan (gh12200)""" x = np.array([1.00000000001, 2.00000000002, 3.00003]) y = np.array([1.00000000002, 2.00000000003, 3.00004]) - # test with a different amount of decimal digits - # note that we only check for the formatting of the arrays themselves - b = ('x: array([1.00000000001, 2.00000000002, 3.00003 ' - ' ])\n y: array([1.00000000002, 2.00000000003, 3.00004 ])') - try: + # Test with a different amount of decimal digits + with pytest.raises(AssertionError) as exc_info: self._assert_func(x, y, decimal=12) - except AssertionError as e: - # remove anything that's not the array string - assert_equal(str(e).split('%)\n ')[1], b) - - # with the default value of decimal digits, only the 3rd element differs - # note that we only check for the formatting of the arrays themselves - b = ('x: array([1. , 2. , 3.00003])\n y: array([1. , ' - '2. , 3.00004])') - try: + msgs = str(exc_info.value).split('\n') + assert_equal(msgs[3], 'Mismatch: 100%') + assert_equal(msgs[4], 'Max absolute difference: 1.e-05') + assert_equal(msgs[5], 'Max relative difference: 3.33328889e-06') + assert_equal( + msgs[6], + ' x: array([1.00000000001, 2.00000000002, 3.00003 ])') + assert_equal( + msgs[7], + ' y: array([1.00000000002, 2.00000000003, 3.00004 ])') + + # With the default value of decimal digits, only the 3rd element + # differs. Note that we only check for the formatting of the arrays + # themselves. + with pytest.raises(AssertionError) as exc_info: self._assert_func(x, y) - except AssertionError as e: - # remove anything that's not the array string - assert_equal(str(e).split('%)\n ')[1], b) + msgs = str(exc_info.value).split('\n') + assert_equal(msgs[3], 'Mismatch: 33.3%') + assert_equal(msgs[4], 'Max absolute difference: 1.e-05') + assert_equal(msgs[5], 'Max relative difference: 3.33328889e-06') + assert_equal(msgs[6], ' x: array([1. , 2. , 3.00003])') + assert_equal(msgs[7], ' y: array([1. , 2. , 3.00004])') + + # Check the error message when input includes inf + x = np.array([np.inf, 0]) + y = np.array([np.inf, 1]) + with pytest.raises(AssertionError) as exc_info: + self._assert_func(x, y) + msgs = str(exc_info.value).split('\n') + assert_equal(msgs[3], 'Mismatch: 50%') + assert_equal(msgs[4], 'Max absolute difference: 1.') + assert_equal(msgs[5], 'Max relative difference: 1.') + assert_equal(msgs[6], ' x: array([inf, 0.])') + assert_equal(msgs[7], ' y: array([inf, 1.])') + + # Check the error message when dividing by zero + x = np.array([1, 2]) + y = np.array([0, 0]) + with pytest.raises(AssertionError) as exc_info: + self._assert_func(x, y) + msgs = str(exc_info.value).split('\n') + assert_equal(msgs[3], 'Mismatch: 100%') + assert_equal(msgs[4], 'Max absolute difference: 2') + assert_equal(msgs[5], 'Max relative difference: inf') def test_subclass_that_cannot_be_bool(self): # While we cannot guarantee testing functions will always work for @@ -780,12 +842,12 @@ class TestAssertAllclose(object): def test_report_fail_percentage(self): a = np.array([1, 1, 1, 1]) b = np.array([1, 1, 1, 2]) - try: + + with pytest.raises(AssertionError) as exc_info: assert_allclose(a, b) - msg = '' - except AssertionError as exc: - msg = exc.args[0] - assert_("mismatch 25.0%" in msg) + msg = str(exc_info.value) + assert_('Mismatch: 25%\nMax absolute difference: 1\n' + 'Max relative difference: 0.5' in msg) def test_equal_nan(self): a = np.array([np.nan]) @@ -1068,16 +1130,14 @@ class TestStringEqual(object): assert_string_equal("hello", "hello") assert_string_equal("hello\nmultiline", "hello\nmultiline") - try: + with pytest.raises(AssertionError) as exc_info: assert_string_equal("foo\nbar", "hello\nbar") - except AssertionError as exc: - assert_equal(str(exc), "Differences in strings:\n- foo\n+ hello") - else: - raise AssertionError("exception not raised") + msg = str(exc_info.value) + assert_equal(msg, "Differences in strings:\n- foo\n+ hello") assert_raises(AssertionError, lambda: assert_string_equal("foo", "hello")) - + def test_regex(self): assert_string_equal("a+*b", "a+*b") @@ -1383,7 +1443,7 @@ def test_tempdir(): def test_temppath(): with temppath() as fpath: - with open(fpath, 'w') as f: + with open(fpath, 'w'): pass assert_(not os.path.isfile(fpath)) diff --git a/numpy/tests/test_ctypeslib.py b/numpy/tests/test_ctypeslib.py index 675f8d242..d389b37a8 100644 --- a/numpy/tests/test_ctypeslib.py +++ b/numpy/tests/test_ctypeslib.py @@ -2,6 +2,7 @@ from __future__ import division, absolute_import, print_function import sys import pytest +import weakref import numpy as np from numpy.ctypeslib import ndpointer, load_library, as_array @@ -9,20 +10,30 @@ from numpy.distutils.misc_util import get_shared_lib_extension from numpy.testing import assert_, assert_array_equal, assert_raises, assert_equal try: + import ctypes +except ImportError: + ctypes = None +else: cdll = None + test_cdll = None if hasattr(sys, 'gettotalrefcount'): try: cdll = load_library('_multiarray_umath_d', np.core._multiarray_umath.__file__) except OSError: pass + try: + test_cdll = load_library('_multiarray_tests', np.core._multiarray_tests.__file__) + except OSError: + pass if cdll is None: cdll = load_library('_multiarray_umath', np.core._multiarray_umath.__file__) - _HAS_CTYPE = True -except ImportError: - _HAS_CTYPE = False + if test_cdll is None: + test_cdll = load_library('_multiarray_tests', np.core._multiarray_tests.__file__) + + c_forward_pointer = test_cdll.forward_pointer -@pytest.mark.skipif(not _HAS_CTYPE, +@pytest.mark.skipif(ctypes is None, reason="ctypes not available in this python") @pytest.mark.skipif(sys.platform == 'cygwin', reason="Known to fail on cygwin") @@ -108,12 +119,72 @@ class TestNdpointer(object): assert_raises(TypeError, p.from_param, np.array([[1, 2], [3, 4]])) def test_cache(self): - a1 = ndpointer(dtype=np.float64) - a2 = ndpointer(dtype=np.float64) - assert_(a1 == a2) + assert_(ndpointer(dtype=np.float64) is ndpointer(dtype=np.float64)) + # shapes are normalized + assert_(ndpointer(shape=2) is ndpointer(shape=(2,))) -@pytest.mark.skipif(not _HAS_CTYPE, + # 1.12 <= v < 1.16 had a bug that made these fail + assert_(ndpointer(shape=2) is not ndpointer(ndim=2)) + assert_(ndpointer(ndim=2) is not ndpointer(shape=2)) + +@pytest.mark.skipif(ctypes is None, + reason="ctypes not available on this python installation") +class TestNdpointerCFunc(object): + def test_arguments(self): + """ Test that arguments are coerced from arrays """ + c_forward_pointer.restype = ctypes.c_void_p + c_forward_pointer.argtypes = (ndpointer(ndim=2),) + + c_forward_pointer(np.zeros((2, 3))) + # too many dimensions + assert_raises( + ctypes.ArgumentError, c_forward_pointer, np.zeros((2, 3, 4))) + + @pytest.mark.parametrize( + 'dt', [ + float, + np.dtype(dict( + formats=['<i4', '<i4'], + names=['a', 'b'], + offsets=[0, 2], + itemsize=6 + )) + ], ids=[ + 'float', + 'overlapping-fields' + ] + ) + def test_return(self, dt): + """ Test that return values are coerced to arrays """ + arr = np.zeros((2, 3), dt) + ptr_type = ndpointer(shape=arr.shape, dtype=arr.dtype) + + c_forward_pointer.restype = ptr_type + c_forward_pointer.argtypes = (ptr_type,) + + # check that the arrays are equivalent views on the same data + arr2 = c_forward_pointer(arr) + assert_equal(arr2.dtype, arr.dtype) + assert_equal(arr2.shape, arr.shape) + assert_equal( + arr2.__array_interface__['data'], + arr.__array_interface__['data'] + ) + + def test_vague_return_value(self): + """ Test that vague ndpointer return values do not promote to arrays """ + arr = np.zeros((2, 3)) + ptr_type = ndpointer(dtype=arr.dtype) + + c_forward_pointer.restype = ptr_type + c_forward_pointer.argtypes = (ptr_type,) + + ret = c_forward_pointer(arr) + assert_(isinstance(ret, ptr_type)) + + +@pytest.mark.skipif(ctypes is None, reason="ctypes not available on this python installation") class TestAsArray(object): def test_array(self): @@ -190,3 +261,15 @@ class TestAsArray(object): b = np.ctypeslib.as_array(newpnt, (N,)) # now delete both, which should cleanup both objects del newpnt, b + + def test_segmentation_fault(self): + arr = np.zeros((224, 224, 3)) + c_arr = np.ctypeslib.as_ctypes(arr) + arr_ref = weakref.ref(arr) + del arr + + # check the reference wasn't cleaned up + assert_(arr_ref() is not None) + + # check we avoid the segfault + c_arr[0][0][0] diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py new file mode 100644 index 000000000..194f8ecbb --- /dev/null +++ b/numpy/tests/test_public_api.py @@ -0,0 +1,89 @@ +from __future__ import division, absolute_import, print_function + +import sys + +import numpy as np +import pytest +try: + import ctypes +except ImportError: + ctypes = None + +def check_dir(module, module_name=None): + """Returns a mapping of all objects with the wrong __module__ attribute.""" + if module_name is None: + module_name = module.__name__ + results = {} + for name in dir(module): + item = getattr(module, name) + if (hasattr(item, '__module__') and hasattr(item, '__name__') + and item.__module__ != module_name): + results[name] = item.__module__ + '.' + item.__name__ + return results + + +@pytest.mark.skipif( + sys.version_info[0] < 3, + reason="NumPy exposes slightly different functions on Python 2") +def test_numpy_namespace(): + # None of these objects are publicly documented. + undocumented = { + 'Tester': 'numpy.testing._private.nosetester.NoseTester', + '_add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc', + 'add_docstring': 'numpy.core._multiarray_umath.add_docstring', + 'add_newdoc': 'numpy.core.function_base.add_newdoc', + 'add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc', + 'byte_bounds': 'numpy.lib.utils.byte_bounds', + 'compare_chararrays': 'numpy.core._multiarray_umath.compare_chararrays', + 'deprecate': 'numpy.lib.utils.deprecate', + 'deprecate_with_doc': 'numpy.lib.utils.<lambda>', + 'disp': 'numpy.lib.function_base.disp', + 'fastCopyAndTranspose': 'numpy.core._multiarray_umath._fastCopyAndTranspose', + 'get_array_wrap': 'numpy.lib.shape_base.get_array_wrap', + 'get_include': 'numpy.lib.utils.get_include', + 'int_asbuffer': 'numpy.core._multiarray_umath.int_asbuffer', + 'mafromtxt': 'numpy.lib.npyio.mafromtxt', + 'maximum_sctype': 'numpy.core.numerictypes.maximum_sctype', + 'ndfromtxt': 'numpy.lib.npyio.ndfromtxt', + 'recfromcsv': 'numpy.lib.npyio.recfromcsv', + 'recfromtxt': 'numpy.lib.npyio.recfromtxt', + 'safe_eval': 'numpy.lib.utils.safe_eval', + 'set_string_function': 'numpy.core.arrayprint.set_string_function', + 'show_config': 'numpy.__config__.show', + 'who': 'numpy.lib.utils.who', + } + # These built-in types are re-exported by numpy. + builtins = { + 'bool': 'builtins.bool', + 'complex': 'builtins.complex', + 'float': 'builtins.float', + 'int': 'builtins.int', + 'long': 'builtins.int', + 'object': 'builtins.object', + 'str': 'builtins.str', + 'unicode': 'builtins.str', + } + whitelist = dict(undocumented, **builtins) + bad_results = check_dir(np) + # pytest gives better error messages with the builtin assert than with + # assert_equal + assert bad_results == whitelist + + +def test_numpy_linalg(): + bad_results = check_dir(np.linalg) + assert bad_results == {} + + +def test_numpy_fft(): + bad_results = check_dir(np.fft) + assert bad_results == {} + +@pytest.mark.skipif(ctypes is None, + reason="ctypes not available in this python") +def test_NPY_NO_EXPORT(): + cdll = ctypes.CDLL(np.core._multiarray_tests.__file__) + # Make sure an arbitrary NPY_NO_EXPORT function is actually hidden + f = getattr(cdll, 'test_not_exported', None) + assert f is None, ("'test_not_exported' is mistakenly exported, " + "NPY_NO_EXPORT does not work") diff --git a/numpy/tests/test_scripts.py b/numpy/tests/test_scripts.py index 33210cc42..e42dc25f9 100644 --- a/numpy/tests/test_scripts.py +++ b/numpy/tests/test_scripts.py @@ -7,8 +7,8 @@ from __future__ import division, print_function, absolute_import import sys import os import pytest -from os.path import join as pathjoin, isfile, dirname, basename -from subprocess import Popen, PIPE +from os.path import join as pathjoin, isfile, dirname +import subprocess import numpy as np from numpy.compat.py3k import basestring @@ -17,74 +17,13 @@ from numpy.testing import assert_, assert_equal is_inplace = isfile(pathjoin(dirname(np.__file__), '..', 'setup.py')) -def run_command(cmd, check_code=True): - """ Run command sequence `cmd` returning exit code, stdout, stderr - - Parameters - ---------- - cmd : str or sequence - string with command name or sequence of strings defining command - check_code : {True, False}, optional - If True, raise error for non-zero return code - - Returns - ------- - returncode : int - return code from execution of `cmd` - stdout : bytes (python 3) or str (python 2) - stdout from `cmd` - stderr : bytes (python 3) or str (python 2) - stderr from `cmd` - - Raises - ------ - RuntimeError - If `check_code` is True, and return code !=0 - """ - cmd = [cmd] if isinstance(cmd, basestring) else list(cmd) - if os.name == 'nt': - # Quote any arguments with spaces. The quotes delimit the arguments - # on Windows, and the arguments might be file paths with spaces. - # On Unix the list elements are each separate arguments. - cmd = ['"{0}"'.format(c) if ' ' in c else c for c in cmd] - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - stdout, stderr = proc.communicate() - if proc.poll() is None: - proc.terminate() - if check_code and proc.returncode != 0: - raise RuntimeError('\n'.join( - ['Command "{0}" failed with', - 'stdout', '------', '{1}', '', - 'stderr', '------', '{2}']).format(cmd, stdout, stderr)) - return proc.returncode, stdout, stderr - - -@pytest.mark.skipif(is_inplace, reason="Cannot test f2py command inplace") -@pytest.mark.xfail(reason="Test is unreliable") -def test_f2py(): - # test that we can run f2py script - - def try_f2py_commands(cmds): - success = 0 - for f2py_cmd in cmds: - try: - code, stdout, stderr = run_command([f2py_cmd, '-v']) - assert_equal(stdout.strip(), b'2') - success += 1 - except Exception: - pass - return success - +def find_f2py_commands(): if sys.platform == 'win32': - # Only the single 'f2py' script is installed in windows. exe_dir = dirname(sys.executable) if exe_dir.endswith('Scripts'): # virtualenv - f2py_cmds = [os.path.join(exe_dir, 'f2py')] + return [os.path.join(exe_dir, 'f2py')] else: - f2py_cmds = [os.path.join(exe_dir, "Scripts", 'f2py')] - success = try_f2py_commands(f2py_cmds) - msg = "Warning: f2py not found in path" - assert_(success == 1, msg) + return [os.path.join(exe_dir, "Scripts", 'f2py')] else: # Three scripts are installed in Unix-like systems: # 'f2py', 'f2py{major}', and 'f2py{major.minor}'. For example, @@ -93,7 +32,18 @@ def test_f2py(): version = sys.version_info major = str(version.major) minor = str(version.minor) - f2py_cmds = ('f2py', 'f2py' + major, 'f2py' + major + '.' + minor) - success = try_f2py_commands(f2py_cmds) - msg = "Warning: not all of %s, %s, and %s are found in path" % f2py_cmds - assert_(success == 3, msg) + return ['f2py', 'f2py' + major, 'f2py' + major + '.' + minor] + + +@pytest.mark.skipif(is_inplace, reason="Cannot test f2py command inplace") +@pytest.mark.xfail(reason="Test is unreliable") +@pytest.mark.parametrize('f2py_cmd', find_f2py_commands()) +def test_f2py(f2py_cmd): + # test that we can run f2py script + stdout = subprocess.check_output([f2py_cmd, '-v']) + assert_equal(stdout.strip(), b'2') + + +def test_pep338(): + stdout = subprocess.check_output([sys.executable, '-mnumpy.f2py', '-v']) + assert_equal(stdout.strip(), b'2') |