diff options
Diffstat (limited to 'numpy')
184 files changed, 7419 insertions, 4631 deletions
diff --git a/numpy/_globals.py b/numpy/_globals.py index 9a7b458f1..f5c0761b5 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -17,7 +17,6 @@ motivated this module. """ from __future__ import division, absolute_import, print_function - __ALL__ = [ 'ModuleDeprecationWarning', 'VisibleDeprecationWarning', '_NoValue' ] @@ -39,7 +38,9 @@ class ModuleDeprecationWarning(DeprecationWarning): nose tester will let pass without making tests fail. """ - pass + + +ModuleDeprecationWarning.__module__ = 'numpy' class VisibleDeprecationWarning(UserWarning): @@ -50,7 +51,10 @@ class VisibleDeprecationWarning(UserWarning): the usage is most likely a user bug. """ - pass + + +VisibleDeprecationWarning.__module__ = 'numpy' + class _NoValueType(object): """Special keyword value. @@ -73,4 +77,5 @@ class _NoValueType(object): def __repr__(self): return "<no value>" + _NoValue = _NoValueType() diff --git a/numpy/_pytesttester.py b/numpy/_pytesttester.py index 30ecc69c7..8d1a3811c 100644 --- a/numpy/_pytesttester.py +++ b/numpy/_pytesttester.py @@ -105,24 +105,18 @@ class PytestTester(object): Notes ----- - Each NumPy module exposes `test` in its namespace to run all tests for it. - For example, to run all tests for numpy.lib: + Each NumPy module exposes `test` in its namespace to run all tests for + it. For example, to run all tests for numpy.lib: >>> np.lib.test() #doctest: +SKIP Examples -------- >>> result = np.lib.test() #doctest: +SKIP - Running unit tests for numpy.lib ... - Ran 976 tests in 3.933s - - OK - - >>> result.errors #doctest: +SKIP - [] - >>> result.knownfail #doctest: +SKIP - [] + 1023 passed, 2 skipped, 6 deselected, 1 xfailed in 10.39 seconds + >>> result + True """ import pytest diff --git a/numpy/compat/_inspect.py b/numpy/compat/_inspect.py index 76bf544a5..439d0d2c2 100644 --- a/numpy/compat/_inspect.py +++ b/numpy/compat/_inspect.py @@ -184,9 +184,8 @@ def formatargvalues(args, varargs, varkw, locals, def convert(name, locals=locals, formatarg=formatarg, formatvalue=formatvalue): return formatarg(name) + formatvalue(locals[name]) - specs = [] - for i in range(len(args)): - specs.append(strseq(args[i], convert, join)) + specs = [strseq(arg, convert, join) for arg in args] + if varargs: specs.append(formatvarargs(varargs) + formatvalue(locals[varargs])) if varkw: diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index ea472f1b3..513415e09 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -49,7 +49,7 @@ add_newdoc('numpy.core', 'flatiter', >>> x = np.arange(6).reshape(2, 3) >>> fl = x.flat >>> type(fl) - <type 'numpy.flatiter'> + <class 'numpy.flatiter'> >>> for item in fl: ... print(item) ... @@ -320,71 +320,68 @@ add_newdoc('numpy.core', 'nditer', Here is how we might write an ``iter_add`` function, using the Python iterator protocol:: - def iter_add_py(x, y, out=None): - addop = np.add - it = np.nditer([x, y, out], [], - [['readonly'], ['readonly'], ['writeonly','allocate']]) - with it: - for (a, b, c) in it: - addop(a, b, out=c) - return it.operands[2] + >>> def iter_add_py(x, y, out=None): + ... addop = np.add + ... it = np.nditer([x, y, out], [], + ... [['readonly'], ['readonly'], ['writeonly','allocate']]) + ... with it: + ... for (a, b, c) in it: + ... addop(a, b, out=c) + ... return it.operands[2] Here is the same function, but following the C-style pattern:: - def iter_add(x, y, out=None): - addop = np.add - - it = np.nditer([x, y, out], [], - [['readonly'], ['readonly'], ['writeonly','allocate']]) - with it: - while not it.finished: - addop(it[0], it[1], out=it[2]) - it.iternext() - - return it.operands[2] + >>> def iter_add(x, y, out=None): + ... addop = np.add + ... it = np.nditer([x, y, out], [], + ... [['readonly'], ['readonly'], ['writeonly','allocate']]) + ... with it: + ... while not it.finished: + ... addop(it[0], it[1], out=it[2]) + ... it.iternext() + ... return it.operands[2] Here is an example outer product function:: - def outer_it(x, y, out=None): - mulop = np.multiply - - it = np.nditer([x, y, out], ['external_loop'], - [['readonly'], ['readonly'], ['writeonly', 'allocate']], - op_axes=[list(range(x.ndim)) + [-1] * y.ndim, - [-1] * x.ndim + list(range(y.ndim)), - None]) - with it: - for (a, b, c) in it: - mulop(a, b, out=c) - return it.operands[2] - - >>> a = np.arange(2)+1 - >>> b = np.arange(3)+1 - >>> outer_it(a,b) - array([[1, 2, 3], - [2, 4, 6]]) + >>> def outer_it(x, y, out=None): + ... mulop = np.multiply + ... it = np.nditer([x, y, out], ['external_loop'], + ... [['readonly'], ['readonly'], ['writeonly', 'allocate']], + ... op_axes=[list(range(x.ndim)) + [-1] * y.ndim, + ... [-1] * x.ndim + list(range(y.ndim)), + ... None]) + ... with it: + ... for (a, b, c) in it: + ... mulop(a, b, out=c) + ... return it.operands[2] + + >>> a = np.arange(2)+1 + >>> b = np.arange(3)+1 + >>> outer_it(a,b) + array([[1, 2, 3], + [2, 4, 6]]) Here is an example function which operates like a "lambda" ufunc:: - def luf(lamdaexpr, *args, **kwargs): - "luf(lambdaexpr, op1, ..., opn, out=None, order='K', casting='safe', buffersize=0)" - nargs = len(args) - op = (kwargs.get('out',None),) + args - it = np.nditer(op, ['buffered','external_loop'], - [['writeonly','allocate','no_broadcast']] + - [['readonly','nbo','aligned']]*nargs, - order=kwargs.get('order','K'), - casting=kwargs.get('casting','safe'), - buffersize=kwargs.get('buffersize',0)) - while not it.finished: - it[0] = lamdaexpr(*it[1:]) - it.iternext() - return it.operands[0] - - >>> a = np.arange(5) - >>> b = np.ones(5) - >>> luf(lambda i,j:i*i + j/2, a, b) - array([ 0.5, 1.5, 4.5, 9.5, 16.5]) + >>> def luf(lamdaexpr, *args, **kwargs): + ... '''luf(lambdaexpr, op1, ..., opn, out=None, order='K', casting='safe', buffersize=0)''' + ... nargs = len(args) + ... op = (kwargs.get('out',None),) + args + ... it = np.nditer(op, ['buffered','external_loop'], + ... [['writeonly','allocate','no_broadcast']] + + ... [['readonly','nbo','aligned']]*nargs, + ... order=kwargs.get('order','K'), + ... casting=kwargs.get('casting','safe'), + ... buffersize=kwargs.get('buffersize',0)) + ... while not it.finished: + ... it[0] = lamdaexpr(*it[1:]) + ... it.iternext() + ... return it.operands[0] + + >>> a = np.arange(5) + >>> b = np.ones(5) + >>> luf(lambda i,j:i*i + j/2, a, b) + array([ 0.5, 1.5, 4.5, 9.5, 16.5]) If operand flags `"writeonly"` or `"readwrite"` are used the operands may be views into the original data with the `WRITEBACKIFCOPY` flag. In this case @@ -393,16 +390,16 @@ add_newdoc('numpy.core', 'nditer', data will be written back to the original data when the `__exit__` function is called but not before: - >>> a = np.arange(6, dtype='i4')[::-2] - >>> with nditer(a, [], - ... [['writeonly', 'updateifcopy']], - ... casting='unsafe', - ... op_dtypes=[np.dtype('f4')]) as i: - ... x = i.operands[0] - ... x[:] = [-1, -2, -3] - ... # a still unchanged here - >>> a, x - array([-1, -2, -3]), array([-1, -2, -3]) + >>> a = np.arange(6, dtype='i4')[::-2] + >>> with np.nditer(a, [], + ... [['writeonly', 'updateifcopy']], + ... casting='unsafe', + ... op_dtypes=[np.dtype('f4')]) as i: + ... x = i.operands[0] + ... x[:] = [-1, -2, -3] + ... # a still unchanged here + >>> a, x + (array([-1, -2, -3], dtype=int32), array([-1., -2., -3.], dtype=float32)) It is important to note that once the iterator is exited, dangling references (like `x` in the example) may or may not share data with @@ -428,10 +425,10 @@ add_newdoc('numpy.core', 'nditer', ('copy', >>> x = np.arange(10) >>> y = x + 1 >>> it = np.nditer([x, y]) - >>> it.next() + >>> next(it) (array(0), array(1)) >>> it2 = it.copy() - >>> it2.next() + >>> next(it2) (array(1), array(2)) """)) @@ -544,7 +541,6 @@ add_newdoc('numpy.core', 'nested_iters', ... print(i.multi_index) ... for y in j: ... print('', j.multi_index, y) - (0,) (0, 0) 0 (0, 1) 1 @@ -617,9 +613,9 @@ add_newdoc('numpy.core', 'broadcast', >>> out = np.empty(b.shape) >>> out.flat = [u+v for (u,v) in b] >>> out - array([[ 5., 6., 7.], - [ 6., 7., 8.], - [ 7., 8., 9.]]) + array([[5., 6., 7.], + [6., 7., 8.], + [7., 8., 9.]]) Compare against built-in broadcasting: @@ -643,7 +639,7 @@ add_newdoc('numpy.core', 'broadcast', ('index', >>> b = np.broadcast(x, y) >>> b.index 0 - >>> b.next(), b.next(), b.next() + >>> next(b), next(b), next(b) ((1, 4), (1, 5), (1, 6)) >>> b.index 3 @@ -762,11 +758,11 @@ add_newdoc('numpy.core', 'broadcast', ('reset', Examples -------- >>> x = np.array([1, 2, 3]) - >>> y = np.array([[4], [5], [6]] + >>> y = np.array([[4], [5], [6]]) >>> b = np.broadcast(x, y) >>> b.index 0 - >>> b.next(), b.next(), b.next() + >>> next(b), next(b), next(b) ((1, 4), (2, 4), (3, 4)) >>> b.index 3 @@ -1072,6 +1068,43 @@ add_newdoc('numpy.core.multiarray', 'fromstring', """) +add_newdoc('numpy.core.multiarray', 'compare_chararrays', + """ + compare_chararrays(a, b, cmp_op, rstrip) + + Performs element-wise comparison of two string arrays using the + comparison operator specified by `cmp_op`. + + Parameters + ---------- + a, b : array_like + Arrays to be compared. + cmp_op : {"<", "<=", "==", ">=", ">", "!="} + Type of comparison. + rstrip : Boolean + If True, the spaces at the end of Strings are removed before the comparison. + + Returns + ------- + out : ndarray + The output array of type Boolean with the same shape as a and b. + + Raises + ------ + ValueError + If `cmp_op` is not valid. + TypeError + If at least one of `a` or `b` is a non-string array + + Examples + -------- + >>> a = np.array(["a", "b", "cde"]) + >>> b = np.array(["a", "a", "dec"]) + >>> np.compare_chararrays(a, b, ">", True) + array([False, True, False]) + + """) + add_newdoc('numpy.core.multiarray', 'fromiter', """ fromiter(iterable, dtype, count=-1) @@ -1152,32 +1185,32 @@ add_newdoc('numpy.core.multiarray', 'fromfile', -------- Construct an ndarray: - >>> dt = np.dtype([('time', [('min', int), ('sec', int)]), + >>> dt = np.dtype([('time', [('min', np.int64), ('sec', np.int64)]), ... ('temp', float)]) >>> x = np.zeros((1,), dtype=dt) >>> x['time']['min'] = 10; x['temp'] = 98.25 >>> x array([((10, 0), 98.25)], - dtype=[('time', [('min', '<i4'), ('sec', '<i4')]), ('temp', '<f8')]) + dtype=[('time', [('min', '<i8'), ('sec', '<i8')]), ('temp', '<f8')]) Save the raw data to disk: - >>> import os - >>> fname = os.tmpnam() + >>> import tempfile + >>> fname = tempfile.mkstemp()[1] >>> x.tofile(fname) Read the raw data from disk: >>> np.fromfile(fname, dtype=dt) array([((10, 0), 98.25)], - dtype=[('time', [('min', '<i4'), ('sec', '<i4')]), ('temp', '<f8')]) + dtype=[('time', [('min', '<i8'), ('sec', '<i8')]), ('temp', '<f8')]) The recommended way to store and load data: >>> np.save(fname, x) >>> np.load(fname + '.npy') array([((10, 0), 98.25)], - dtype=[('time', [('min', '<i4'), ('sec', '<i4')]), ('temp', '<f8')]) + dtype=[('time', [('min', '<i8'), ('sec', '<i8')]), ('temp', '<f8')]) """) @@ -1205,17 +1238,16 @@ add_newdoc('numpy.core.multiarray', 'frombuffer', >>> dt = np.dtype(int) >>> dt = dt.newbyteorder('>') - >>> np.frombuffer(buf, dtype=dt) + >>> np.frombuffer(buf, dtype=dt) # doctest: +SKIP The data of the resulting array will not be byteswapped, but will be interpreted correctly. Examples -------- - >>> s = 'hello world' + >>> s = b'hello world' >>> np.frombuffer(s, dtype='S1', count=5, offset=6) - array(['w', 'o', 'r', 'l', 'd'], - dtype='|S1') + array([b'w', b'o', b'r', b'l', b'd'], dtype='|S1') >>> np.frombuffer(b'\\x01\\x02', dtype=np.uint8) array([1, 2], dtype=uint8) @@ -1320,6 +1352,12 @@ add_newdoc('numpy.core.multiarray', 'set_numeric_ops', Set numerical operators for array objects. + .. deprecated:: 1.16 + + For the general case, use :c:func:`PyUFunc_ReplaceLoopBySignature`. + For ndarray subclasses, define the ``__array_ufunc__`` method and + override the relevant ufunc. + Parameters ---------- op1, op2, ... : callable @@ -1470,134 +1508,11 @@ add_newdoc('numpy.core.multiarray', 'getbuffer', """) -add_newdoc('numpy.core', 'matmul', - """ - matmul(a, b, out=None) - - Matrix product of two arrays. - - The behavior depends on the arguments in the following way. - - - If both arguments are 2-D they are multiplied like conventional - matrices. - - If either argument is N-D, N > 2, it is treated as a stack of - matrices residing in the last two indexes and broadcast accordingly. - - If the first argument is 1-D, it is promoted to a matrix by - prepending a 1 to its dimensions. After matrix multiplication - the prepended 1 is removed. - - If the second argument is 1-D, it is promoted to a matrix by - appending a 1 to its dimensions. After matrix multiplication - the appended 1 is removed. - - Multiplication by a scalar is not allowed, use ``*`` instead. Note that - multiplying a stack of matrices with a vector will result in a stack of - vectors, but matmul will not recognize it as such. - - ``matmul`` differs from ``dot`` in two important ways. - - - Multiplication by scalars is not allowed. - - Stacks of matrices are broadcast together as if the matrices - were elements. - - .. warning:: - This function is preliminary and included in NumPy 1.10.0 for testing - and documentation. Its semantics will not change, but the number and - order of the optional arguments will. - - .. versionadded:: 1.10.0 - - Parameters - ---------- - a : array_like - First argument. - b : array_like - Second argument. - out : ndarray, optional - Output argument. This must have the exact kind that would be returned - if it was not used. In particular, it must have the right type, must be - C-contiguous, and its dtype must be the dtype that would be returned - for `dot(a,b)`. This is a performance feature. Therefore, if these - conditions are not met, an exception is raised, instead of attempting - to be flexible. - - Returns - ------- - output : ndarray - Returns the dot product of `a` and `b`. If `a` and `b` are both - 1-D arrays then a scalar is returned; otherwise an array is - returned. If `out` is given, then it is returned. - - Raises - ------ - ValueError - If the last dimension of `a` is not the same size as - the second-to-last dimension of `b`. - - If scalar value is passed. - - See Also - -------- - vdot : Complex-conjugating dot product. - tensordot : Sum products over arbitrary axes. - einsum : Einstein summation convention. - dot : alternative matrix product with different broadcasting rules. - - Notes - ----- - The matmul function implements the semantics of the `@` operator introduced - in Python 3.5 following PEP465. - - Examples - -------- - For 2-D arrays it is the matrix product: - - >>> a = [[1, 0], [0, 1]] - >>> b = [[4, 1], [2, 2]] - >>> np.matmul(a, b) - array([[4, 1], - [2, 2]]) - - For 2-D mixed with 1-D, the result is the usual. - - >>> a = [[1, 0], [0, 1]] - >>> b = [1, 2] - >>> np.matmul(a, b) - array([1, 2]) - >>> np.matmul(b, a) - array([1, 2]) - - - Broadcasting is conventional for stacks of arrays - - >>> a = np.arange(2*2*4).reshape((2,2,4)) - >>> b = np.arange(2*2*4).reshape((2,4,2)) - >>> np.matmul(a,b).shape - (2, 2, 2) - >>> np.matmul(a,b)[0,1,1] - 98 - >>> sum(a[0,1,:] * b[0,:,1]) - 98 - - Vector, vector returns the scalar inner product, but neither argument - is complex-conjugated: - - >>> np.matmul([2j, 3j], [2j, 3j]) - (-13+0j) - - Scalar multiplication raises an error. - - >>> np.matmul([1,2], 3) - Traceback (most recent call last): - ... - ValueError: Scalar operands are not allowed, use '*' instead - - """) - add_newdoc('numpy.core.multiarray', 'c_einsum', """ c_einsum(subscripts, *operands, out=None, dtype=None, order='K', casting='safe') - + *This documentation shadows that of the native python implementation of the `einsum` function, except all references and examples related to the `optimize` argument (v 0.12.0) have been removed.* @@ -2021,8 +1936,8 @@ add_newdoc('numpy.core.multiarray', 'ndarray', First mode, `buffer` is None: >>> np.ndarray(shape=(2,2), dtype=float, order='F') - array([[ -1.13698227e+002, 4.25087011e-303], - [ 2.88528414e-306, 3.27025015e-309]]) #random + array([[0.0e+000, 0.0e+000], # random + [ nan, 2.5e-323]]) Second mode: @@ -2113,7 +2028,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('ctypes', ----- Below are the public attributes of this object which were documented in "Guide to NumPy" (we have omitted undocumented public attributes, - as well as documented private attributes): + as well as documented private attributes): .. autoattribute:: numpy.core._internal._ctypes.data @@ -2127,14 +2042,6 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('ctypes', .. automethod:: numpy.core._internal._ctypes.strides_as - Be careful using the ctypes attribute - especially on temporary - arrays or arrays constructed on the fly. For example, calling - ``(a+b).ctypes.data_as(ctypes.c_void_p)`` returns a pointer to memory - that is invalid because the array created as (a+b) is deallocated - before the next Python statement. You can avoid this problem using - either ``c=a+b`` or ``ct=(a+b).ctypes``. In the latter case, ct will - hold a reference to the array until ct is deleted or re-assigned. - If the ctypes module is not available, then the ctypes attribute of array objects still returns something useful, but ctypes objects are not returned and errors may be raised instead. In particular, @@ -2336,7 +2243,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('flat', >>> x.T.flat[3] 5 >>> type(x.flat) - <type 'numpy.flatiter'> + <class 'numpy.flatiter'> An assignment example: @@ -2455,7 +2362,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('size', Notes ----- - `a.size` returns a standard arbitrary precision Python integer. This + `a.size` returns a standard arbitrary precision Python integer. This may not be the case with other methods of obtaining the same value (like the suggested ``np.prod(a.shape)``, which returns an instance of ``np.int_``), and may be relevant if the value is used further in @@ -2786,7 +2693,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('astype', -------- >>> x = np.array([1, 2, 2.5]) >>> x - array([ 1. , 2. , 2.5]) + array([1. , 2. , 2.5]) >>> x.astype(int) array([1, 2, 2]) @@ -2817,19 +2724,20 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('byteswap', Examples -------- >>> A = np.array([1, 256, 8755], dtype=np.int16) - >>> map(hex, A) + >>> list(map(hex, A)) ['0x1', '0x100', '0x2233'] >>> A.byteswap(inplace=True) array([ 256, 1, 13090], dtype=int16) - >>> map(hex, A) + >>> list(map(hex, A)) ['0x100', '0x1', '0x3322'] Arrays of strings are not swapped >>> A = np.array(['ceg', 'fac']) >>> A.byteswap() - array(['ceg', 'fac'], - dtype='|S3') + Traceback (most recent call last): + ... + UnicodeDecodeError: ... """)) @@ -3017,14 +2925,14 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('dot', >>> a = np.eye(2) >>> b = np.ones((2, 2)) * 2 >>> a.dot(b) - array([[ 2., 2.], - [ 2., 2.]]) + array([[2., 2.], + [2., 2.]]) This array method can be conveniently chained: >>> a.dot(b).dot(b) - array([[ 8., 8.], - [ 8., 8.]]) + array([[8., 8.], + [8., 8.]]) """)) @@ -3077,7 +2985,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('fill', >>> a = np.empty(2) >>> a.fill(1) >>> a - array([ 1., 1.]) + array([1., 1.]) """)) @@ -3146,18 +3054,18 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('getfield', >>> x = np.diag([1.+1.j]*2) >>> x[1, 1] = 2 + 4.j >>> x - array([[ 1.+1.j, 0.+0.j], - [ 0.+0.j, 2.+4.j]]) + array([[1.+1.j, 0.+0.j], + [0.+0.j, 2.+4.j]]) >>> x.getfield(np.float64) - array([[ 1., 0.], - [ 0., 2.]]) + array([[1., 0.], + [0., 2.]]) By choosing an offset of 8 bytes we can select the complex part of the array for our view: >>> x.getfield(np.float64, offset=8) - array([[ 1., 0.], - [ 0., 4.]]) + array([[1., 0.], + [0., 4.]]) """)) @@ -3203,19 +3111,20 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('item', Examples -------- + >>> np.random.seed(123) >>> x = np.random.randint(9, size=(3, 3)) >>> x - array([[3, 1, 7], - [2, 8, 3], - [8, 5, 3]]) + array([[2, 2, 6], + [1, 3, 6], + [1, 0, 1]]) >>> x.item(3) - 2 + 1 >>> x.item(7) - 5 + 0 >>> x.item((0, 1)) - 1 + 2 >>> x.item((2, 2)) - 3 + 1 """)) @@ -3251,17 +3160,18 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('itemset', Examples -------- + >>> np.random.seed(123) >>> x = np.random.randint(9, size=(3, 3)) >>> x - array([[3, 1, 7], - [2, 8, 3], - [8, 5, 3]]) + array([[2, 2, 6], + [1, 3, 6], + [1, 0, 1]]) >>> x.itemset(4, 0) >>> x.itemset((2, 2), 9) >>> x - array([[3, 1, 7], - [2, 0, 3], - [8, 5, 9]]) + array([[2, 2, 6], + [1, 0, 6], + [1, 0, 9]]) """)) @@ -3702,7 +3612,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('resize', >>> a.resize((1, 1)) Traceback (most recent call last): ... - ValueError: cannot resize an array that has been referenced ... + ValueError: cannot resize an array that references or is referenced ... Unless `refcheck` is False: @@ -3775,23 +3685,23 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('setfield', -------- >>> x = np.eye(3) >>> x.getfield(np.float64) - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) >>> x.setfield(3, np.int32) >>> x.getfield(np.int32) array([[3, 3, 3], [3, 3, 3], - [3, 3, 3]]) + [3, 3, 3]], dtype=int32) >>> x - array([[ 1.00000000e+000, 1.48219694e-323, 1.48219694e-323], - [ 1.48219694e-323, 1.00000000e+000, 1.48219694e-323], - [ 1.48219694e-323, 1.48219694e-323, 1.00000000e+000]]) + array([[1.0e+000, 1.5e-323, 1.5e-323], + [1.5e-323, 1.0e+000, 1.5e-323], + [1.5e-323, 1.5e-323, 1.0e+000]]) >>> x.setfield(np.eye(3), np.int32) >>> x - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) """)) @@ -3844,6 +3754,9 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('setflags', Examples -------- + >>> y = np.array([[3, 1, 7], + ... [2, 0, 0], + ... [8, 5, 9]]) >>> y array([[3, 1, 7], [2, 0, 0], @@ -3923,8 +3836,8 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('sort', >>> a = np.array([('a', 2), ('c', 1)], dtype=[('x', 'S1'), ('y', int)]) >>> a.sort(order='y') >>> a - array([('c', 1), ('a', 2)], - dtype=[('x', '|S1'), ('y', '<i4')]) + array([(b'c', 1), (b'a', 2)], + dtype=[('x', 'S1'), ('y', '<i8')]) """)) @@ -3980,6 +3893,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('partition', array([2, 1, 3, 4]) >>> a.partition((1, 3)) + >>> a array([1, 2, 3, 4]) """)) @@ -4090,7 +4004,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('tofile', machines with different endianness. Some of these problems can be overcome by outputting the data as text files, at the expense of speed and file size. - + When fid is a file object, array contents are directly written to the file, bypassing the file object's ``write`` method. As a result, tofile cannot be used with files objects supporting compression (e.g., GzipFile) @@ -4161,13 +4075,13 @@ tobytesdoc = """ Examples -------- - >>> x = np.array([[0, 1], [2, 3]]) + >>> x = np.array([[0, 1], [2, 3]], dtype='<u2') >>> x.tobytes() - b'\\x00\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x03\\x00\\x00\\x00' + b'\\x00\\x00\\x01\\x00\\x02\\x00\\x03\\x00' >>> x.tobytes('C') == x.tobytes() True >>> x.tobytes('F') - b'\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x03\\x00\\x00\\x00' + b'\\x00\\x00\\x02\\x00\\x01\\x00\\x03\\x00' """ @@ -4317,7 +4231,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('view', >>> y matrix([[513]], dtype=int16) >>> print(type(y)) - <class 'numpy.matrixlib.defmatrix.matrix'> + <class 'numpy.matrix'> Creating a view on a structured array so it can be used in calculations @@ -4327,19 +4241,19 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('view', array([[1, 2], [3, 4]], dtype=int8) >>> xv.mean(0) - array([ 2., 3.]) + array([2., 3.]) Making changes to the view changes the underlying array >>> xv[0,1] = 20 - >>> print(x) - [(1, 20) (3, 4)] + >>> x + array([(1, 20), (3, 4)], dtype=[('a', 'i1'), ('b', 'i1')]) Using a view to convert an array to a recarray: >>> z = x.view(np.recarray) >>> z.a - array([1], dtype=int8) + array([1, 3], dtype=int8) Views share data: @@ -4357,8 +4271,8 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('view', [4, 5]], dtype=int16) >>> y.view(dtype=[('width', np.int16), ('length', np.int16)]) Traceback (most recent call last): - File "<stdin>", line 1, in <module> - ValueError: new type not compatible with array. + ... + ValueError: To change to a dtype of a different size, the array must be C-contiguous >>> z = y.copy() >>> z.view(dtype=[('width', np.int16), ('length', np.int16)]) array([[(1, 2)], @@ -4409,10 +4323,9 @@ add_newdoc('numpy.core.umath', 'frompyfunc', >>> oct_array = np.frompyfunc(oct, 1, 1) >>> oct_array(np.array((10, 30, 100))) - array([012, 036, 0144], dtype=object) + array(['0o12', '0o36', '0o144'], dtype=object) >>> np.array((oct(10), oct(30), oct(100))) # for comparison - array(['012', '036', '0144'], - dtype='|S4') + array(['0o12', '0o36', '0o144'], dtype='<U5') """) @@ -4474,7 +4387,7 @@ add_newdoc('numpy.core.umath', 'geterrobj', >>> np.base_repr(np.geterrobj()[1], 8) '0' >>> old_err = np.seterr(divide='warn', over='log', under='call', - invalid='print') + ... invalid='print') >>> np.base_repr(np.geterrobj()[1], 8) '4351' @@ -4546,184 +4459,6 @@ add_newdoc('numpy.core.umath', 'seterrobj', # ############################################################################## -add_newdoc('numpy.core.multiarray', 'bincount', - """ - bincount(x, weights=None, minlength=0) - - Count number of occurrences of each value in array of non-negative ints. - - The number of bins (of size 1) is one larger than the largest value in - `x`. If `minlength` is specified, there will be at least this number - of bins in the output array (though it will be longer if necessary, - depending on the contents of `x`). - Each bin gives the number of occurrences of its index value in `x`. - If `weights` is specified the input array is weighted by it, i.e. if a - value ``n`` is found at position ``i``, ``out[n] += weight[i]`` instead - of ``out[n] += 1``. - - Parameters - ---------- - x : array_like, 1 dimension, nonnegative ints - Input array. - weights : array_like, optional - Weights, array of the same shape as `x`. - minlength : int, optional - A minimum number of bins for the output array. - - .. versionadded:: 1.6.0 - - Returns - ------- - out : ndarray of ints - The result of binning the input array. - The length of `out` is equal to ``np.amax(x)+1``. - - Raises - ------ - ValueError - If the input is not 1-dimensional, or contains elements with negative - values, or if `minlength` is negative. - TypeError - If the type of the input is float or complex. - - See Also - -------- - histogram, digitize, unique - - Examples - -------- - >>> np.bincount(np.arange(5)) - array([1, 1, 1, 1, 1]) - >>> np.bincount(np.array([0, 1, 1, 3, 2, 1, 7])) - array([1, 3, 1, 1, 0, 0, 0, 1]) - - >>> x = np.array([0, 1, 1, 3, 2, 1, 7, 23]) - >>> np.bincount(x).size == np.amax(x)+1 - True - - The input array needs to be of integer dtype, otherwise a - TypeError is raised: - - >>> np.bincount(np.arange(5, dtype=float)) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: array cannot be safely cast to required type - - A possible use of ``bincount`` is to perform sums over - variable-size chunks of an array, using the ``weights`` keyword. - - >>> w = np.array([0.3, 0.5, 0.2, 0.7, 1., -0.6]) # weights - >>> x = np.array([0, 1, 1, 2, 2, 2]) - >>> np.bincount(x, weights=w) - array([ 0.3, 0.7, 1.1]) - - """) - -add_newdoc('numpy.core.multiarray', 'ravel_multi_index', - """ - ravel_multi_index(multi_index, dims, mode='raise', order='C') - - Converts a tuple of index arrays into an array of flat - indices, applying boundary modes to the multi-index. - - Parameters - ---------- - multi_index : tuple of array_like - A tuple of integer arrays, one array for each dimension. - dims : tuple of ints - The shape of array into which the indices from ``multi_index`` apply. - mode : {'raise', 'wrap', 'clip'}, optional - Specifies how out-of-bounds indices are handled. Can specify - either one mode or a tuple of modes, one mode per index. - - * 'raise' -- raise an error (default) - * 'wrap' -- wrap around - * 'clip' -- clip to the range - - In 'clip' mode, a negative index which would normally - wrap will clip to 0 instead. - order : {'C', 'F'}, optional - Determines whether the multi-index should be viewed as - indexing in row-major (C-style) or column-major - (Fortran-style) order. - - Returns - ------- - raveled_indices : ndarray - An array of indices into the flattened version of an array - of dimensions ``dims``. - - See Also - -------- - unravel_index - - Notes - ----- - .. versionadded:: 1.6.0 - - Examples - -------- - >>> arr = np.array([[3,6,6],[4,5,1]]) - >>> np.ravel_multi_index(arr, (7,6)) - array([22, 41, 37]) - >>> np.ravel_multi_index(arr, (7,6), order='F') - array([31, 41, 13]) - >>> np.ravel_multi_index(arr, (4,6), mode='clip') - array([22, 23, 19]) - >>> np.ravel_multi_index(arr, (4,4), mode=('clip','wrap')) - array([12, 13, 13]) - - >>> np.ravel_multi_index((3,1,4,1), (6,7,8,9)) - 1621 - """) - -add_newdoc('numpy.core.multiarray', 'unravel_index', - """ - unravel_index(indices, shape, order='C') - - Converts a flat index or array of flat indices into a tuple - of coordinate arrays. - - Parameters - ---------- - indices : array_like - An integer array whose elements are indices into the flattened - version of an array of dimensions ``shape``. Before version 1.6.0, - this function accepted just one index value. - shape : tuple of ints - The shape of the array to use for unraveling ``indices``. - - .. versionchanged:: 1.16.0 - Renamed from ``dims`` to ``shape``. - - order : {'C', 'F'}, optional - Determines whether the indices should be viewed as indexing in - row-major (C-style) or column-major (Fortran-style) order. - - .. versionadded:: 1.6.0 - - Returns - ------- - unraveled_coords : tuple of ndarray - Each array in the tuple has the same shape as the ``indices`` - array. - - See Also - -------- - ravel_multi_index - - Examples - -------- - >>> np.unravel_index([22, 41, 37], (7,6)) - (array([3, 6, 6]), array([4, 5, 1])) - >>> np.unravel_index([31, 41, 13], (7,6), order='F') - (array([3, 6, 6]), array([4, 5, 1])) - - >>> np.unravel_index(1621, (6,7,8,9)) - (3, 1, 4, 1) - - """) - add_newdoc('numpy.core.multiarray', 'add_docstring', """ add_docstring(obj, docstring) @@ -4798,7 +4533,10 @@ add_newdoc('numpy.core.multiarray', 'packbits', ... [0,0,1]]]) >>> b = np.packbits(a, axis=-1) >>> b - array([[[160],[64]],[[192],[32]]], dtype=uint8) + array([[[160], + [ 64]], + [[192], + [ 32]]], dtype=uint8) Note that in binary 160 = 1010 0000, 64 = 0100 0000, 192 = 1100 0000, and 32 = 0010 0000. @@ -5200,7 +4938,7 @@ add_newdoc('numpy.core', 'ufunc', ('reduce', to None - otherwise it defaults to ufunc.identity. If ``None`` is given, the first element of the reduction is used, and an error is thrown if the reduction is empty. - + .. versionadded:: 1.15.0 Returns @@ -5233,18 +4971,18 @@ add_newdoc('numpy.core', 'ufunc', ('reduce', >>> np.add.reduce(X, 2) array([[ 1, 5], [ 9, 13]]) - + You can use the ``initial`` keyword argument to initialize the reduction with a different value. - + >>> np.add.reduce([10], initial=5) 15 - >>> np.add.reduce(np.ones((2, 2, 2)), axis=(0, 2), initializer=10) + >>> np.add.reduce(np.ones((2, 2, 2)), axis=(0, 2), initial=10) array([14., 14.]) - + Allows reductions of empty arrays where they would normally fail, i.e. for ufuncs without an identity. - + >>> np.minimum.reduce([], initial=np.inf) inf >>> np.minimum.reduce([]) @@ -5312,23 +5050,23 @@ add_newdoc('numpy.core', 'ufunc', ('accumulate', >>> I = np.eye(2) >>> I - array([[ 1., 0.], - [ 0., 1.]]) + array([[1., 0.], + [0., 1.]]) Accumulate along axis 0 (rows), down columns: >>> np.add.accumulate(I, 0) - array([[ 1., 0.], - [ 1., 1.]]) + array([[1., 0.], + [1., 1.]]) >>> np.add.accumulate(I) # no axis specified = axis zero - array([[ 1., 0.], - [ 1., 1.]]) + array([[1., 0.], + [1., 1.]]) Accumulate along axis 1 (columns), through rows: >>> np.add.accumulate(I, 1) - array([[ 1., 1.], - [ 0., 1.]]) + array([[1., 1.], + [0., 1.]]) """)) @@ -5405,10 +5143,10 @@ add_newdoc('numpy.core', 'ufunc', ('reduceat', >>> x = np.linspace(0, 15, 16).reshape(4,4) >>> x - array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]]) + array([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) :: @@ -5420,11 +5158,11 @@ add_newdoc('numpy.core', 'ufunc', ('reduceat', # [row1 + row2 + row3 + row4] >>> np.add.reduceat(x, [0, 3, 1, 2, 0]) - array([[ 12., 15., 18., 21.], - [ 12., 13., 14., 15.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.], - [ 24., 28., 32., 36.]]) + array([[12., 15., 18., 21.], + [12., 13., 14., 15.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [24., 28., 32., 36.]]) :: @@ -5432,10 +5170,10 @@ add_newdoc('numpy.core', 'ufunc', ('reduceat', # [col1 * col2 * col3, col4] >>> np.multiply.reduceat(x, [0, 3], 1) - array([[ 0., 3.], - [ 120., 7.], - [ 720., 11.], - [ 2184., 15.]]) + array([[ 0., 3.], + [ 120., 7.], + [ 720., 11.], + [2184., 15.]]) """)) @@ -5534,14 +5272,14 @@ add_newdoc('numpy.core', 'ufunc', ('at', >>> a = np.array([1, 2, 3, 4]) >>> np.negative.at(a, [0, 1]) - >>> print(a) - array([-1, -2, 3, 4]) + >>> a + array([-1, -2, 3, 4]) Increment items 0 and 1, and increment item 2 twice: >>> a = np.array([1, 2, 3, 4]) >>> np.add.at(a, [0, 1, 2, 2], 1) - >>> print(a) + >>> a array([2, 3, 5, 4]) Add items 0 and 1 in first array to second array, @@ -5550,7 +5288,7 @@ add_newdoc('numpy.core', 'ufunc', ('at', >>> a = np.array([1, 2, 3, 4]) >>> b = np.array([1, 2]) >>> np.add.at(a, [0, 1], b) - >>> print(a) + >>> a array([2, 4, 3, 4]) """)) @@ -5615,13 +5353,13 @@ add_newdoc('numpy.core.multiarray', 'dtype', Structured type, two fields: the first field contains an unsigned int, the second an int32: - >>> np.dtype([('f1', np.uint), ('f2', np.int32)]) - dtype([('f1', '<u4'), ('f2', '<i4')]) + >>> np.dtype([('f1', np.uint64), ('f2', np.int32)]) + dtype([('f1', '<u8'), ('f2', '<i4')]) Using array-protocol type strings: >>> np.dtype([('a','f8'),('b','S10')]) - dtype([('a', '<f8'), ('b', '|S10')]) + dtype([('a', '<f8'), ('b', 'S10')]) Using comma-separated field formats. The shape is (2,3): @@ -5631,24 +5369,24 @@ add_newdoc('numpy.core.multiarray', 'dtype', Using tuples. ``int`` is a fixed type, 3 the field's shape. ``void`` is a flexible type, here of size 10: - >>> np.dtype([('hello',(int,3)),('world',np.void,10)]) - dtype([('hello', '<i4', 3), ('world', '|V10')]) + >>> np.dtype([('hello',(np.int64,3)),('world',np.void,10)]) + dtype([('hello', '<i8', (3,)), ('world', 'V10')]) Subdivide ``int16`` into 2 ``int8``'s, called x and y. 0 and 1 are the offsets in bytes: >>> np.dtype((np.int16, {'x':(np.int8,0), 'y':(np.int8,1)})) - dtype(('<i2', [('x', '|i1'), ('y', '|i1')])) + dtype((numpy.int16, [('x', 'i1'), ('y', 'i1')])) Using dictionaries. Two fields named 'gender' and 'age': >>> np.dtype({'names':['gender','age'], 'formats':['S1',np.uint8]}) - dtype([('gender', '|S1'), ('age', '|u1')]) + dtype([('gender', 'S1'), ('age', 'u1')]) Offsets in bytes, here 0 and 25: >>> np.dtype({'surname':('S25',0),'age':(np.uint8,25)}) - dtype([('surname', '|S25'), ('age', '|u1')]) + dtype([('surname', 'S25'), ('age', 'u1')]) """) @@ -5714,13 +5452,13 @@ add_newdoc('numpy.core.multiarray', 'dtype', ('char', add_newdoc('numpy.core.multiarray', 'dtype', ('descr', """ - PEP3118 interface description of the data-type. + `__array_interface__` description of the data-type. The format is that required by the 'descr' key in the - PEP3118 `__array_interface__` attribute. + `__array_interface__` attribute. - Warning: This attribute exists specifically for PEP3118 compliance, and - is not a datatype description compatible with `np.dtype`. + Warning: This attribute exists specifically for `__array_interface__`, + and is not a datatype description compatible with `np.dtype`. """)) add_newdoc('numpy.core.multiarray', 'dtype', ('fields', @@ -6052,7 +5790,7 @@ add_newdoc('numpy.core.multiarray', 'busdaycalendar', ... holidays=['2011-07-01', '2011-07-04', '2011-07-17']) >>> # Default is Monday to Friday weekdays ... bdd.weekmask - array([ True, True, True, True, True, False, False], dtype='bool') + array([ True, True, True, True, True, False, False]) >>> # Any holidays already on the weekend are removed ... bdd.holidays array(['2011-07-01', '2011-07-04'], dtype='datetime64[D]') @@ -6149,7 +5887,7 @@ add_newdoc('numpy.core.multiarray', 'datetime_data', as a timedelta >>> np.datetime64('2010', np.datetime_data(dt_25s)) - numpy.datetime64('2010-01-01T00:00:00', '25s') + numpy.datetime64('2010-01-01T00:00:00','25s') """) diff --git a/numpy/core/_dtype_ctypes.py b/numpy/core/_dtype_ctypes.py index f10b4e99f..0852b1ef2 100644 --- a/numpy/core/_dtype_ctypes.py +++ b/numpy/core/_dtype_ctypes.py @@ -33,17 +33,65 @@ def _from_ctypes_array(t): def _from_ctypes_structure(t): - # TODO: gh-10533, gh-10532 - fields = [] for item in t._fields_: if len(item) > 2: raise TypeError( "ctypes bitfields have no dtype equivalent") - fname, ftyp = item - fields.append((fname, dtype_from_ctypes_type(ftyp))) - # by default, ctypes structs are aligned - return np.dtype(fields, align=True) + if hasattr(t, "_pack_"): + formats = [] + offsets = [] + names = [] + current_offset = 0 + for fname, ftyp in t._fields_: + names.append(fname) + formats.append(dtype_from_ctypes_type(ftyp)) + # Each type has a default offset, this is platform dependent for some types. + effective_pack = min(t._pack_, ctypes.alignment(ftyp)) + current_offset = ((current_offset + effective_pack - 1) // effective_pack) * effective_pack + offsets.append(current_offset) + current_offset += ctypes.sizeof(ftyp) + + return np.dtype(dict( + formats=formats, + offsets=offsets, + names=names, + itemsize=ctypes.sizeof(t))) + else: + fields = [] + for fname, ftyp in t._fields_: + fields.append((fname, dtype_from_ctypes_type(ftyp))) + + # by default, ctypes structs are aligned + return np.dtype(fields, align=True) + + +def _from_ctypes_scalar(t): + """ + Return the dtype type with endianness included if it's the case + """ + if getattr(t, '__ctype_be__', None) is t: + return np.dtype('>' + t._type_) + elif getattr(t, '__ctype_le__', None) is t: + return np.dtype('<' + t._type_) + else: + return np.dtype(t._type_) + + +def _from_ctypes_union(t): + formats = [] + offsets = [] + names = [] + for fname, ftyp in t._fields_: + names.append(fname) + formats.append(dtype_from_ctypes_type(ftyp)) + offsets.append(0) # Union fields are offset to 0 + + return np.dtype(dict( + formats=formats, + offsets=offsets, + names=names, + itemsize=ctypes.sizeof(t))) def dtype_from_ctypes_type(t): @@ -57,12 +105,9 @@ def dtype_from_ctypes_type(t): elif issubclass(t, _ctypes.Structure): return _from_ctypes_structure(t) elif issubclass(t, _ctypes.Union): - # TODO - raise NotImplementedError( - "conversion from ctypes.Union types like {} to dtype" - .format(t.__name__)) - elif isinstance(t._type_, str): - return np.dtype(t._type_) + return _from_ctypes_union(t) + elif isinstance(getattr(t, '_type_', None), str): + return _from_ctypes_scalar(t) else: raise NotImplementedError( "Unknown ctypes type {}".format(t.__name__)) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 30069f0ca..1d3bb5584 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -10,6 +10,7 @@ import re import sys from numpy.compat import unicode +from numpy.core.overrides import set_module from .multiarray import dtype, array, ndarray try: import ctypes @@ -237,19 +238,68 @@ _getintp_ctype.cache = None class _missing_ctypes(object): def cast(self, num, obj): - return num + return num.value + + class c_void_p(object): + def __init__(self, ptr): + self.value = ptr + + +class _unsafe_first_element_pointer(object): + """ + Helper to allow viewing an array as a ctypes pointer to the first element + + This avoids: + * dealing with strides + * `.view` rejecting object-containing arrays + * `memoryview` not supporting overlapping fields + """ + def __init__(self, arr): + self.base = arr + + @property + def __array_interface__(self): + i = dict( + shape=(), + typestr='|V0', + data=(self.base.__array_interface__['data'][0], False), + strides=(), + version=3, + ) + return i + + +def _get_void_ptr(arr): + """ + Get a `ctypes.c_void_p` to arr.data, that keeps a reference to the array + """ + import numpy as np + # convert to a 0d array that has a data pointer referrign to the start + # of arr. This holds a reference to arr. + simple_arr = np.asarray(_unsafe_first_element_pointer(arr)) + + # create a `char[0]` using the same memory. + c_arr = (ctypes.c_char * 0).from_buffer(simple_arr) + + # finally cast to void* + return ctypes.cast(ctypes.pointer(c_arr), ctypes.c_void_p) - def c_void_p(self, num): - return num class _ctypes(object): def __init__(self, array, ptr=None): + self._arr = array + if ctypes: self._ctypes = ctypes + # get a void pointer to the buffer, which keeps the array alive + self._data = _get_void_ptr(array) + assert self._data.value == ptr else: + # fake a pointer-like object that holds onto the reference self._ctypes = _missing_ctypes() - self._arr = array - self._data = ptr + self._data = self._ctypes.c_void_p(ptr) + self._data._objects = array + if self._arr.ndim == 0: self._zerod = True else: @@ -262,6 +312,8 @@ class _ctypes(object): ``self.data_as(ctypes.c_void_p)``. Perhaps you want to use the data as a pointer to a ctypes array of floating-point data: ``self.data_as(ctypes.POINTER(ctypes.c_double))``. + + The returned pointer will keep a reference to the array. """ return self._ctypes.cast(self._data, obj) @@ -283,7 +335,8 @@ class _ctypes(object): return None return (obj*self._arr.ndim)(*self._arr.strides) - def get_data(self): + @property + def data(self): """ A pointer to the memory area of the array as a Python integer. This memory area may contain data that is not aligned, or not in correct @@ -292,10 +345,16 @@ class _ctypes(object): attribute to arbitrary C-code to avoid trouble that can include Python crashing. User Beware! The value of this attribute is exactly the same as ``self._array_interface_['data'][0]``. + + Note that unlike `data_as`, a reference will not be kept to the array: + code like ``ctypes.c_void_p((a + b).ctypes.data)`` will result in a + pointer to a deallocated array, and should be spelt + ``(a + b).ctypes.data_as(ctypes.c_void_p)`` """ - return self._data + return self._data.value - def get_shape(self): + @property + def shape(self): """ (c_intp*self.ndim): A ctypes array of length self.ndim where the basetype is the C-integer corresponding to ``dtype('p')`` on this @@ -306,7 +365,8 @@ class _ctypes(object): """ return self.shape_as(_getintp_ctype()) - def get_strides(self): + @property + def strides(self): """ (c_intp*self.ndim): A ctypes array of length self.ndim where the basetype is the same as for the shape attribute. This ctypes array @@ -316,13 +376,20 @@ class _ctypes(object): """ return self.strides_as(_getintp_ctype()) - def get_as_parameter(self): - return self._ctypes.c_void_p(self._data) + @property + def _as_parameter_(self): + """ + Overrides the ctypes semi-magic method + + Enables `c_func(some_array.ctypes)` + """ + return self._data - data = property(get_data) - shape = property(get_shape) - strides = property(get_strides) - _as_parameter_ = property(get_as_parameter, None, doc="_as parameter_") + # kept for compatibility + get_data = data.fget + get_shape = shape.fget + get_strides = strides.fget + get_as_parameter = _as_parameter_.fget def _newnames(datatype, order): @@ -482,6 +549,12 @@ _pep3118_standard_map = { } _pep3118_standard_typechars = ''.join(_pep3118_standard_map.keys()) +_pep3118_unsupported_map = { + 'u': 'UCS-2 strings', + '&': 'pointers', + 't': 'bitfields', + 'X': 'function pointers', +} class _Stream(object): def __init__(self, s): @@ -593,6 +666,11 @@ def __dtype_from_pep3118(stream, is_subdtype): stream.byteorder, stream.byteorder) value = dtype(numpy_byteorder + dtypechar) align = value.alignment + elif stream.next in _pep3118_unsupported_map: + desc = _pep3118_unsupported_map[stream.next] + raise NotImplementedError( + "Unrepresentable PEP 3118 data type {!r} ({})" + .format(stream.next, desc)) else: raise ValueError("Unknown PEP 3118 data type specifier %r" % stream.s) @@ -718,9 +796,11 @@ def _lcm(a, b): return a // _gcd(a, b) * b # Exception used in shares_memory() +@set_module('numpy') class TooHardError(RuntimeError): pass +@set_module('numpy') class AxisError(ValueError, IndexError): """ Axis supplied was invalid. """ def __init__(self, axis, ndim=None, msg_prefix=None): @@ -750,6 +830,13 @@ def array_ufunc_errmsg_formatter(dummy, ufunc, method, *inputs, **kwargs): .format(ufunc, method, args_string, types_string)) +def array_function_errmsg_formatter(public_api, types): + """ Format the error message for when __array_ufunc__ gives up. """ + func_name = '{}.{}'.format(public_api.__module__, public_api.__name__) + return ("no implementation found for '{}' on types that implement " + '__array_function__: {}'.format(func_name, list(types))) + + def _ufunc_doc_signature_formatter(ufunc): """ Builds a signature string which resembles PEP 457 diff --git a/numpy/core/_methods.py b/numpy/core/_methods.py index 8974f0ce1..33f6d01a8 100644 --- a/numpy/core/_methods.py +++ b/numpy/core/_methods.py @@ -154,18 +154,3 @@ def _ptp(a, axis=None, out=None, keepdims=False): umr_minimum(a, axis, None, None, keepdims), out ) - -_NDARRAY_ARRAY_FUNCTION = mu.ndarray.__array_function__ - -def _array_function(self, func, types, args, kwargs): - # TODO: rewrite this in C - # Cannot handle items that have __array_function__ other than our own. - for t in types: - if t is not mu.ndarray: - method = getattr(t, '__array_function__', _NDARRAY_ARRAY_FUNCTION) - if method is not _NDARRAY_ARRAY_FUNCTION: - return NotImplemented - - # Arguments contain no overrides, so we can safely call the - # overloaded function again. - return func(*args, **kwargs) diff --git a/numpy/core/_type_aliases.py b/numpy/core/_type_aliases.py index cce6c0425..d6e1a1fb7 100644 --- a/numpy/core/_type_aliases.py +++ b/numpy/core/_type_aliases.py @@ -60,7 +60,7 @@ for k, v in typeinfo.items(): else: _concrete_typeinfo[k] = v -_concrete_types = set(v.type for k, v in _concrete_typeinfo.items()) +_concrete_types = {v.type for k, v in _concrete_typeinfo.items()} def _bits_of(obj): diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index ccc1468c4..7d8785c32 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -26,6 +26,7 @@ __docformat__ = 'restructuredtext' import sys import functools +import numbers if sys.version_info[0] >= 3: try: from _thread import get_ident @@ -48,7 +49,7 @@ from .fromnumeric import ravel, any from .numeric import concatenate, asarray, errstate from .numerictypes import (longlong, intc, int_, float_, complex_, bool_, flexible) -from .overrides import array_function_dispatch +from .overrides import array_function_dispatch, set_module import warnings import contextlib @@ -86,9 +87,15 @@ def _make_options_dict(precision=None, threshold=None, edgeitems=None, if legacy not in [None, False, '1.13']: warnings.warn("legacy printing option can currently only be '1.13' or " "`False`", stacklevel=3) - + if threshold is not None: + # forbid the bad threshold arg suggested by stack overflow, gh-12351 + if not isinstance(threshold, numbers.Number) or np.isnan(threshold): + raise ValueError("threshold must be numeric and non-NAN, try " + "sys.maxsize for untruncated representation") return options + +@set_module('numpy') def set_printoptions(precision=None, threshold=None, edgeitems=None, linewidth=None, suppress=None, nanstr=None, infstr=None, formatter=None, sign=None, floatmode=None, **kwarg): @@ -194,21 +201,21 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, Floating point precision can be set: >>> np.set_printoptions(precision=4) - >>> print(np.array([1.123456789])) - [ 1.1235] + >>> np.array([1.123456789]) + [1.1235] Long arrays can be summarised: >>> np.set_printoptions(threshold=5) - >>> print(np.arange(10)) - [0 1 2 ..., 7 8 9] + >>> np.arange(10) + array([0, 1, 2, ..., 7, 8, 9]) Small results can be suppressed: >>> eps = np.finfo(float).eps >>> x = np.arange(4.) >>> x**2 - (x + eps)**2 - array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) + array([-4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) >>> np.set_printoptions(suppress=True) >>> x**2 - (x + eps)**2 array([-0., -0., 0., 0.]) @@ -250,6 +257,7 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, set_legacy_print_mode(0) +@set_module('numpy') def get_printoptions(): """ Return the current print options. @@ -279,6 +287,7 @@ def get_printoptions(): return _format_options.copy() +@set_module('numpy') @contextlib.contextmanager def printoptions(*args, **kwargs): """Context manager for setting print options. @@ -290,9 +299,10 @@ def printoptions(*args, **kwargs): Examples -------- + >>> from numpy.testing import assert_equal >>> with np.printoptions(precision=2): - ... print(np.array([2.0])) / 3 - [0.67] + ... np.array([2.0]) / 3 + array([0.67]) The `as`-clause of the `with`-statement gives the current print options: @@ -635,9 +645,9 @@ def array2string(a, max_line_width=None, precision=None, Examples -------- >>> x = np.array([1e-16,1,2,3]) - >>> print(np.array2string(x, precision=2, separator=',', - ... suppress_small=True)) - [ 0., 1., 2., 3.] + >>> np.array2string(x, precision=2, separator=',', + ... suppress_small=True) + '[0.,1.,2.,3.]' >>> x = np.arange(3.) >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x}) @@ -645,7 +655,7 @@ def array2string(a, max_line_width=None, precision=None, >>> x = np.arange(3) >>> np.array2string(x, formatter={'int':lambda x: hex(x)}) - '[0x0L 0x1L 0x2L]' + '[0x0 0x1 0x2]' """ legacy = kwarg.pop('legacy', None) @@ -976,6 +986,8 @@ class LongFloatFormat(FloatingFormat): DeprecationWarning, stacklevel=2) super(LongFloatFormat, self).__init__(*args, **kwargs) + +@set_module('numpy') def format_float_scientific(x, precision=None, unique=True, trim='k', sign=False, pad_left=None, exp_digits=None): """ @@ -1043,6 +1055,8 @@ def format_float_scientific(x, precision=None, unique=True, trim='k', trim=trim, sign=sign, pad_left=pad_left, exp_digits=exp_digits) + +@set_module('numpy') def format_float_positional(x, precision=None, unique=True, fractional=True, trim='k', sign=False, pad_left=None, pad_right=None): @@ -1344,7 +1358,7 @@ def dtype_is_implied(dtype): >>> np.core.arrayprint.dtype_is_implied(np.int8) False >>> np.array([1, 2, 3], np.int8) - array([1, 2, 3], dtype=np.int8) + array([1, 2, 3], dtype=int8) """ dtype = np.dtype(dtype) if _format_options['legacy'] == '1.13' and dtype.type == bool_: @@ -1364,6 +1378,7 @@ def dtype_short_repr(dtype): The intent is roughly that the following holds >>> from numpy import * + >>> dt = np.int64([1, 2]).dtype >>> assert eval(dtype_short_repr(dt)) == dt """ if dtype.names is not None: @@ -1467,13 +1482,13 @@ def array_repr(arr, max_line_width=None, precision=None, suppress_small=None): >>> np.array_repr(np.array([1,2])) 'array([1, 2])' >>> np.array_repr(np.ma.array([0.])) - 'MaskedArray([ 0.])' + 'MaskedArray([0.])' >>> np.array_repr(np.array([], np.int32)) 'array([], dtype=int32)' >>> x = np.array([1e-6, 4e-7, 2, 3]) >>> np.array_repr(x, precision=6, suppress_small=True) - 'array([ 0.000001, 0. , 2. , 3. ])' + 'array([0.000001, 0. , 2. , 3. ])' """ return _array_repr_implementation( @@ -1547,10 +1562,12 @@ def array_str(a, max_line_width=None, precision=None, suppress_small=None): a, max_line_width, precision, suppress_small) +# needed if __array_function__ is disabled +_array2string_impl = getattr(array2string, '__wrapped__', array2string) _default_array_str = functools.partial(_array_str_implementation, - array2string=array2string.__wrapped__) + array2string=_array2string_impl) _default_array_repr = functools.partial(_array_repr_implementation, - array2string=array2string.__wrapped__) + array2string=_array2string_impl) def set_string_function(f, repr=True): @@ -1582,8 +1599,8 @@ def set_string_function(f, repr=True): >>> a = np.arange(10) >>> a HA! - What are you going to do now? - >>> print(a) - [0 1 2 3 4 5 6 7 8 9] + >>> _ = a + >>> # [0 1 2 3 4 5 6 7 8 9] We can reset the function to the default: @@ -1601,7 +1618,7 @@ def set_string_function(f, repr=True): >>> x.__str__() 'random' >>> x.__repr__() - 'array([ 0, 1, 2, 3])' + 'array([0, 1, 2, 3])' """ if f is None: diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index c8b998bfc..00f10df57 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -39,9 +39,12 @@ 0x0000000b = edb1ba83730c650fd9bc5772a919cda7 # Version 12 (NumPy 1.14) Added PyArray_ResolveWritebackIfCopy, -# Version 12 (NumPy 1.15) No change. # PyArray_SetWritebackIfCopyBase and deprecated PyArray_SetUpdateIfCopyBase. +# Version 12 (NumPy 1.15) No change. 0x0000000c = a1bc756c5782853ec2e3616cf66869d8 -# Version 13 (Numpy 1.16) Added fields core_dim_flags and core_dim_sizes to PyUFuncObject -0x0000000d = a1bc756c5782853ec2e3616cf66869d8 +# Version 13 (NumPy 1.16) +# Deprecate PyArray_SetNumericOps and PyArray_GetNumericOps, +# Add fields core_dim_flags and core_dim_sizes to PyUFuncObject. +# Add PyUFunc_FromFuncAndDataAndSignatureAndIdentity to ufunc_funcs_api. +0x0000000d = 5b0e8bbded00b166125974fc71e80a33 diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 42c564a97..4aca2373c 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -19,6 +19,7 @@ __docformat__ = 'restructuredtext' # The files under src/ that are scanned for API functions API_FILES = [join('multiarray', 'alloc.c'), + join('multiarray', 'arrayfunction_override.c'), join('multiarray', 'array_assign_array.c'), join('multiarray', 'array_assign_scalar.c'), join('multiarray', 'arrayobject.c'), @@ -163,9 +164,7 @@ def skip_brackets(s, lbrac, rbrac): def split_arguments(argstr): arguments = [] - bracket_counts = {'(': 0, '[': 0} current_argument = [] - state = 0 i = 0 def finish_arg(): if current_argument: @@ -400,9 +399,7 @@ class FunctionApi(object): return " (void *) %s" % self.name def internal_define(self): - annstr = [] - for a in self.annotations: - annstr.append(str(a)) + annstr = [str(a) for a in self.annotations] annstr = ' '.join(annstr) astr = """\ NPY_NO_EXPORT %s %s %s \\\n (%s);""" % (annstr, self.return_type, @@ -463,10 +460,7 @@ def get_api_functions(tagname, api_dict): functions = [] for f in API_FILES: functions.extend(find_functions(f, tagname)) - dfunctions = [] - for func in functions: - o = api_dict[func.name][0] - dfunctions.append( (o, func) ) + dfunctions = [(api_dict[func.name][0], func) for func in functions] dfunctions.sort() return [a[1] for a in dfunctions] diff --git a/numpy/core/code_generators/generate_numpy_api.py b/numpy/core/code_generators/generate_numpy_api.py index 7f2541667..5e04fb86d 100644 --- a/numpy/core/code_generators/generate_numpy_api.py +++ b/numpy/core/code_generators/generate_numpy_api.py @@ -50,7 +50,6 @@ _import_array(void) PyObject *c_api = NULL; if (numpy == NULL) { - PyErr_SetString(PyExc_ImportError, "numpy.core._multiarray_umath failed to import"); return -1; } c_api = PyObject_GetAttrString(numpy, "_ARRAY_API"); @@ -193,7 +192,9 @@ def do_generate_api(targets, sources): genapi.check_api_dict(multiarray_api_index) numpyapi_list = genapi.get_api_functions('NUMPY_API', - multiarray_funcs) + multiarray_funcs) + + # FIXME: ordered_funcs_api is unused ordered_funcs_api = genapi.order_dict(multiarray_funcs) # Create dict name -> *Api instance diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index 199ad831b..f5ee02c42 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -10,11 +10,14 @@ sys.path.insert(0, os.path.dirname(__file__)) import ufunc_docstrings as docstrings sys.path.pop(0) -Zero = "PyUFunc_Zero" -One = "PyUFunc_One" -None_ = "PyUFunc_None" -AllOnes = "PyUFunc_MinusOne" -ReorderableNone = "PyUFunc_ReorderableNone" +Zero = "PyInt_FromLong(0)" +One = "PyInt_FromLong(1)" +True_ = "(Py_INCREF(Py_True), Py_True)" +False_ = "(Py_INCREF(Py_False), Py_False)" +None_ = object() +AllOnes = "PyInt_FromLong(-1)" +MinusInfinity = 'PyFloat_FromDouble(-NPY_INFINITY)' +ReorderableNone = "(Py_INCREF(Py_None), Py_None)" # Sentinel value to specify using the full type description in the # function name @@ -74,10 +77,7 @@ class TypeDescription(object): _fdata_map = dict(e='npy_%sf', f='npy_%sf', d='npy_%s', g='npy_%sl', F='nc_%sf', D='nc_%s', G='nc_%sl') def build_func_data(types, f): - func_data = [] - for t in types: - d = _fdata_map.get(t, '%s') % (f,) - func_data.append(d) + func_data = [_fdata_map.get(t, '%s') % (f,) for t in types] return func_data def TD(types, f=None, astype=None, in_=None, out=None, simd=None): @@ -124,7 +124,7 @@ class Ufunc(object): type_descriptions : list of TypeDescription objects """ def __init__(self, nin, nout, identity, docstring, typereso, - *type_descriptions): + *type_descriptions, **kwargs): self.nin = nin self.nout = nout if identity is None: @@ -133,10 +133,13 @@ class Ufunc(object): self.docstring = docstring self.typereso = typereso self.type_descriptions = [] + self.signature = kwargs.pop('signature', None) for td in type_descriptions: self.type_descriptions.extend(td) for td in self.type_descriptions: td.finish_signature(self.nin, self.nout) + if kwargs: + raise ValueError('unknown kwargs %r' % str(kwargs)) # String-handling utilities to avoid locale-dependence. @@ -458,7 +461,7 @@ defdict = { [TypeDescription('O', FullTypeDescr, 'OO', 'O')], ), 'logical_and': - Ufunc(2, 1, One, + Ufunc(2, 1, True_, docstrings.get('numpy.core.umath.logical_and'), 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(nodatetime_or_obj, out='?', simd=[('avx2', ints)]), @@ -472,14 +475,14 @@ defdict = { TD(O, f='npy_ObjectLogicalNot'), ), 'logical_or': - Ufunc(2, 1, Zero, + Ufunc(2, 1, False_, docstrings.get('numpy.core.umath.logical_or'), 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(nodatetime_or_obj, out='?', simd=[('avx2', ints)]), TD(O, f='npy_ObjectLogicalOr'), ), 'logical_xor': - Ufunc(2, 1, Zero, + Ufunc(2, 1, False_, docstrings.get('numpy.core.umath.logical_xor'), 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(nodatetime_or_obj, out='?'), @@ -514,7 +517,7 @@ defdict = { TD(O, f='npy_ObjectMin') ), 'logaddexp': - Ufunc(2, 1, None, + Ufunc(2, 1, MinusInfinity, docstrings.get('numpy.core.umath.logaddexp'), None, TD(flts, f="logaddexp", astype={'e':'f'}) @@ -901,7 +904,14 @@ defdict = { "PyUFunc_SimpleBinaryOperationTypeResolver", TD(ints), TD('O', f='npy_ObjectLCM'), - ) + ), +'matmul' : + Ufunc(2, 1, None, + docstrings.get('numpy.core.umath.matmul'), + "PyUFunc_SimpleBinaryOperationTypeResolver", + TD(notimes_or_obj), + signature='(n?,k),(k,m?)->(n?,m?)', + ), } if sys.version_info[0] >= 3: @@ -1047,19 +1057,44 @@ def make_ufuncs(funcdict): # string literal in C code. We split at endlines because textwrap.wrap # do not play well with \n docstring = '\\n\"\"'.join(docstring.split(r"\n")) + if uf.signature is None: + sig = "NULL" + else: + sig = '"{}"'.format(uf.signature) fmt = textwrap.dedent("""\ - f = PyUFunc_FromFuncAndData( + identity = {identity_expr}; + if ({has_identity} && identity == NULL) {{ + return -1; + }} + f = PyUFunc_FromFuncAndDataAndSignatureAndIdentity( {name}_functions, {name}_data, {name}_signatures, {nloops}, {nin}, {nout}, {identity}, "{name}", - "{doc}", 0 + "{doc}", 0, {sig}, identity ); + if ({has_identity}) {{ + Py_DECREF(identity); + }} if (f == NULL) {{ return -1; - }}""") - mlist.append(fmt.format( + }} + """) + args = dict( name=name, nloops=len(uf.type_descriptions), - nin=uf.nin, nout=uf.nout, identity=uf.identity, doc=docstring - )) + nin=uf.nin, nout=uf.nout, + has_identity='0' if uf.identity is None_ else '1', + identity='PyUFunc_IdentityValue', + identity_expr=uf.identity, + doc=docstring, + sig=sig, + ) + + # Only PyUFunc_None means don't reorder - we pass this using the old + # argument + if uf.identity is None_: + args['identity'] = 'PyUFunc_None' + args['identity_expr'] = 'NULL' + + mlist.append(fmt.format(**args)) if uf.typereso is not None: mlist.append( r"((PyUFuncObject *)f)->type_resolver = &%s;" % uf.typereso) @@ -1083,11 +1118,13 @@ def make_code(funcdict, filename): #include "cpuid.h" #include "ufunc_object.h" #include "ufunc_type_resolution.h" + #include "loops.h" + #include "matmul.h" %s static int InitOperators(PyObject *dictionary) { - PyObject *f; + PyObject *f, *identity; %s %s diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index d8a9ee6b4..a71c236fd 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -402,6 +402,8 @@ ufunc_funcs_api = { # End 1.7 API 'PyUFunc_RegisterLoopForDescr': (41,), # End 1.8 API + 'PyUFunc_FromFuncAndDataAndSignatureAndIdentity': (42,), + # End 1.16 API } # List of all the dicts which define the C API diff --git a/numpy/core/code_generators/ufunc_docstrings.py b/numpy/core/code_generators/ufunc_docstrings.py index 13231de29..267e63b2d 100644 --- a/numpy/core/code_generators/ufunc_docstrings.py +++ b/numpy/core/code_generators/ufunc_docstrings.py @@ -39,7 +39,8 @@ subst = { def add_newdoc(place, name, doc): doc = textwrap.dedent(doc).strip() - if name[0] != '_': + if name[0] != '_' and name != 'matmul': + # matmul is special, it does not use the OUT_SCALAR replacement strings if '\nx :' in doc: assert '$OUT_SCALAR_1' in doc, "in {}".format(name) elif '\nx2 :' in doc or '\nx1, x2 :' in doc: @@ -647,8 +648,8 @@ add_newdoc('numpy.core.umath', 'bitwise_or', array([ 6, 5, 255]) >>> np.array([2, 5, 255]) | np.array([4, 4, 4]) array([ 6, 5, 255]) - >>> np.bitwise_or(np.array([2, 5, 255, 2147483647L], dtype=np.int32), - ... np.array([4, 4, 4, 2147483647L], dtype=np.int32)) + >>> np.bitwise_or(np.array([2, 5, 255, 2147483647], dtype=np.int32), + ... np.array([4, 4, 4, 2147483647], dtype=np.int32)) array([ 6, 5, 255, 2147483647]) >>> np.bitwise_or([True, True], [False, True]) array([ True, True]) @@ -836,6 +837,7 @@ add_newdoc('numpy.core.umath', 'cos', array([ 1.00000000e+00, 6.12303177e-17, -1.00000000e+00]) >>> >>> # Example of providing the optional output parameter + >>> out1 = np.array([0], dtype='d') >>> out2 = np.cos([0.1], out1) >>> out2 is out1 True @@ -844,7 +846,7 @@ add_newdoc('numpy.core.umath', 'cos', >>> np.cos(np.zeros((3,3)),np.zeros((2,2))) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: invalid return array shape + ValueError: operands could not be broadcast together with shapes (3,3) (2,2) """) @@ -911,7 +913,7 @@ add_newdoc('numpy.core.umath', 'degrees', 270., 300., 330.]) >>> out = np.zeros((rad.shape)) - >>> r = degrees(rad, out) + >>> r = np.degrees(rad, out) >>> np.all(r == out) True @@ -1558,33 +1560,31 @@ add_newdoc('numpy.core.umath', 'invert', We've seen that 13 is represented by ``00001101``. The invert or bit-wise NOT of 13 is then: - >>> np.invert(np.array([13], dtype=uint8)) - array([242], dtype=uint8) + >>> x = np.invert(np.array(13, dtype=np.uint8)) + >>> x + 242 >>> np.binary_repr(x, width=8) - '00001101' - >>> np.binary_repr(242, width=8) '11110010' The result depends on the bit-width: - >>> np.invert(np.array([13], dtype=uint16)) - array([65522], dtype=uint16) + >>> x = np.invert(np.array(13, dtype=np.uint16)) + >>> x + 65522 >>> np.binary_repr(x, width=16) - '0000000000001101' - >>> np.binary_repr(65522, width=16) '1111111111110010' When using signed integer types the result is the two's complement of the result for the unsigned type: - >>> np.invert(np.array([13], dtype=int8)) + >>> np.invert(np.array([13], dtype=np.int8)) array([-14], dtype=int8) >>> np.binary_repr(-14, width=8) '11110010' Booleans are accepted as well: - >>> np.invert(array([True, False])) + >>> np.invert(np.array([True, False])) array([False, True]) """) @@ -1968,7 +1968,7 @@ add_newdoc('numpy.core.umath', 'log10', Examples -------- >>> np.log10([1e-15, -3.]) - array([-15., NaN]) + array([-15., nan]) """) @@ -2360,7 +2360,7 @@ add_newdoc('numpy.core.umath', 'maximum', [ 0.5, 2. ]]) >>> np.maximum([np.nan, 0, np.nan], [0, np.nan, np.nan]) - array([ NaN, NaN, NaN]) + array([nan, nan, nan]) >>> np.maximum(np.Inf, 1) inf @@ -2419,7 +2419,7 @@ add_newdoc('numpy.core.umath', 'minimum', [ 0. , 1. ]]) >>> np.minimum([np.nan, 0, np.nan],[0, np.nan, np.nan]) - array([ NaN, NaN, NaN]) + array([nan, nan, nan]) >>> np.minimum(-np.Inf, 1) -inf @@ -2479,7 +2479,7 @@ add_newdoc('numpy.core.umath', 'fmax', [ 0.5, 2. ]]) >>> np.fmax([np.nan, 0, np.nan],[0, np.nan, np.nan]) - array([ 0., 0., NaN]) + array([ 0., 0., nan]) """) @@ -2537,10 +2537,133 @@ add_newdoc('numpy.core.umath', 'fmin', [ 0. , 1. ]]) >>> np.fmin([np.nan, 0, np.nan],[0, np.nan, np.nan]) - array([ 0., 0., NaN]) + array([ 0., 0., nan]) """) +add_newdoc('numpy.core.umath', 'matmul', + """ + Matrix product of two arrays. + + Parameters + ---------- + x1, x2 : array_like + Input arrays, scalars not allowed. + out : ndarray, optional + A location into which the result is stored. If provided, it must have + a shape that matches the signature `(n,k),(k,m)->(n,m)`. If not + provided or `None`, a freshly-allocated array is returned. + **kwargs + For other keyword-only arguments, see the + :ref:`ufunc docs <ufuncs.kwargs>`. + + ..versionadded:: 1.16 + Now handles ufunc kwargs + + Returns + ------- + y : ndarray + The matrix product of the inputs. + This is a scalar only when both x1, x2 are 1-d vectors. + + Raises + ------ + ValueError + If the last dimension of `a` is not the same size as + the second-to-last dimension of `b`. + + If a scalar value is passed in. + + See Also + -------- + vdot : Complex-conjugating dot product. + tensordot : Sum products over arbitrary axes. + einsum : Einstein summation convention. + dot : alternative matrix product with different broadcasting rules. + + Notes + ----- + + The behavior depends on the arguments in the following way. + + - If both arguments are 2-D they are multiplied like conventional + matrices. + - If either argument is N-D, N > 2, it is treated as a stack of + matrices residing in the last two indexes and broadcast accordingly. + - If the first argument is 1-D, it is promoted to a matrix by + prepending a 1 to its dimensions. After matrix multiplication + the prepended 1 is removed. + - If the second argument is 1-D, it is promoted to a matrix by + appending a 1 to its dimensions. After matrix multiplication + the appended 1 is removed. + + ``matmul`` differs from ``dot`` in two important ways: + + - Multiplication by scalars is not allowed, use ``*`` instead. + - Stacks of matrices are broadcast together as if the matrices + were elements, respecting the signature ``(n,k),(k,m)->(n,m)``: + + >>> a = np.ones([9, 5, 7, 4]) + >>> c = np.ones([9, 5, 4, 3]) + >>> np.dot(a, c).shape + (9, 5, 7, 9, 5, 3) + >>> np.matmul(a, c).shape + (9, 5, 7, 3) + >>> # n is 7, k is 4, m is 3 + + The matmul function implements the semantics of the `@` operator introduced + in Python 3.5 following PEP465. + + Examples + -------- + For 2-D arrays it is the matrix product: + + >>> a = np.array([[1, 0], + ... [0, 1]]) + >>> b = np.array([[4, 1], + ... [2, 2]]) + >>> np.matmul(a, b) + array([[4, 1], + [2, 2]]) + + For 2-D mixed with 1-D, the result is the usual. + + >>> a = np.array([[1, 0], + ... [0, 1]]) + >>> b = np.array([1, 2]) + >>> np.matmul(a, b) + array([1, 2]) + >>> np.matmul(b, a) + array([1, 2]) + + + Broadcasting is conventional for stacks of arrays + + >>> a = np.arange(2 * 2 * 4).reshape((2, 2, 4)) + >>> b = np.arange(2 * 2 * 4).reshape((2, 4, 2)) + >>> np.matmul(a,b).shape + (2, 2, 2) + >>> np.matmul(a, b)[0, 1, 1] + 98 + >>> sum(a[0, 1, :] * b[0 , :, 1]) + 98 + + Vector, vector returns the scalar inner product, but neither argument + is complex-conjugated: + + >>> np.matmul([2j, 3j], [2j, 3j]) + (-13+0j) + + Scalar multiplication raises an error. + + >>> np.matmul([1,2], 3) + Traceback (most recent call last): + ... + ValueError: matmul: Input operand 1 does not have enough dimensions ... + + .. versionadded:: 1.10.0 + """) + add_newdoc('numpy.core.umath', 'modf', """ Return the fractional and integral parts of an array, element-wise. @@ -3352,6 +3475,7 @@ add_newdoc('numpy.core.umath', 'sinh', >>> # Discrepancy due to vagaries of floating point arithmetic. >>> # Example of providing the optional output parameter + >>> out1 = np.array([0], dtype='d') >>> out2 = np.sinh([0.1], out1) >>> out2 is out1 True @@ -3360,7 +3484,7 @@ add_newdoc('numpy.core.umath', 'sinh', >>> np.sinh(np.zeros((3,3)),np.zeros((2,2))) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: invalid return array shape + ValueError: operands could not be broadcast together with shapes (3,3) (2,2) """) @@ -3405,8 +3529,8 @@ add_newdoc('numpy.core.umath', 'sqrt', >>> np.sqrt([4, -1, -3+4J]) array([ 2.+0.j, 0.+1.j, 1.+2.j]) - >>> np.sqrt([4, -1, numpy.inf]) - array([ 2., NaN, Inf]) + >>> np.sqrt([4, -1, np.inf]) + array([ 2., nan, inf]) """) @@ -3537,6 +3661,7 @@ add_newdoc('numpy.core.umath', 'tan', >>> >>> # Example of providing the optional output parameter illustrating >>> # that what is returned is a reference to said parameter + >>> out1 = np.array([0], dtype='d') >>> out2 = np.cos([0.1], out1) >>> out2 is out1 True @@ -3545,7 +3670,7 @@ add_newdoc('numpy.core.umath', 'tan', >>> np.cos(np.zeros((3,3)),np.zeros((2,2))) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: invalid return array shape + ValueError: operands could not be broadcast together with shapes (3,3) (2,2) """) @@ -3588,6 +3713,7 @@ add_newdoc('numpy.core.umath', 'tanh', >>> # Example of providing the optional output parameter illustrating >>> # that what is returned is a reference to said parameter + >>> out1 = np.array([0], dtype='d') >>> out2 = np.tanh([0.1], out1) >>> out2 is out1 True @@ -3596,7 +3722,7 @@ add_newdoc('numpy.core.umath', 'tanh', >>> np.tanh(np.zeros((3,3)),np.zeros((2,2))) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: invalid return array shape + ValueError: operands could not be broadcast together with shapes (3,3) (2,2) """) @@ -3638,8 +3764,6 @@ add_newdoc('numpy.core.umath', 'true_divide', >>> np.true_divide(x, 4) array([ 0. , 0.25, 0.5 , 0.75, 1. ]) - >>> x/4 - array([0, 0, 0, 0, 1]) >>> x//4 array([0, 0, 0, 0, 1]) @@ -3735,7 +3859,7 @@ add_newdoc('numpy.core.umath', 'ldexp', Examples -------- >>> np.ldexp(5, np.arange(4)) - array([ 5., 10., 20., 40.], dtype=float32) + array([ 5., 10., 20., 40.], dtype=float16) >>> x = np.arange(6) >>> np.ldexp(*np.frexp(x)) diff --git a/numpy/core/defchararray.py b/numpy/core/defchararray.py index e86086012..007fc6186 100644 --- a/numpy/core/defchararray.py +++ b/numpy/core/defchararray.py @@ -23,6 +23,7 @@ from .numerictypes import string_, unicode_, integer, object_, bool_, character from .numeric import ndarray, compare_chararrays from .numeric import array as narray from numpy.core.multiarray import _vec_string +from numpy.core.overrides import set_module from numpy.core import overrides from numpy.compat import asbytes, long import numpy @@ -497,8 +498,7 @@ def count(a, sub, start=0, end=None): -------- >>> c = np.array(['aAaAaA', ' aA ', 'abBABba']) >>> c - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') >>> np.char.count(c, 'A') array([3, 1, 1]) >>> np.char.count(c, 'aA') @@ -551,8 +551,7 @@ def decode(a, encoding=None, errors=None): -------- >>> c = np.array(['aAaAaA', ' aA ', 'abBABba']) >>> c - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') >>> np.char.encode(c, encoding='cp037') array(['\\x81\\xc1\\x81\\xc1\\x81\\xc1', '@@\\x81\\xc1@@', '\\x81\\x82\\xc2\\xc1\\xc2\\x82\\x81'], @@ -636,8 +635,7 @@ def endswith(a, suffix, start=0, end=None): >>> s[0] = 'foo' >>> s[1] = 'bar' >>> s - array(['foo', 'bar'], - dtype='|S3') + array(['foo', 'bar'], dtype='<U3') >>> np.char.endswith(s, 'ar') array([False, True]) >>> np.char.endswith(s, 'a', start=1, end=2) @@ -1035,11 +1033,9 @@ def lower(a): Examples -------- >>> c = np.array(['A1B C', '1BCA', 'BCA1']); c - array(['A1B C', '1BCA', 'BCA1'], - dtype='|S5') + array(['A1B C', '1BCA', 'BCA1'], dtype='<U5') >>> np.char.lower(c) - array(['a1b c', '1bca', 'bca1'], - dtype='|S5') + array(['a1b c', '1bca', 'bca1'], dtype='<U5') """ a_arr = numpy.asarray(a) @@ -1083,23 +1079,20 @@ def lstrip(a, chars=None): -------- >>> c = np.array(['aAaAaA', ' aA ', 'abBABba']) >>> c - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') The 'a' variable is unstripped from c[1] because whitespace leading. >>> np.char.lstrip(c, 'a') - array(['AaAaA', ' aA ', 'bBABba'], - dtype='|S7') + array(['AaAaA', ' aA ', 'bBABba'], dtype='<U7') >>> np.char.lstrip(c, 'A') # leaves c unchanged - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') >>> (np.char.lstrip(c, ' ') == np.char.lstrip(c, '')).all() - ... # XXX: is this a regression? this line now returns False + ... # XXX: is this a regression? This used to return True ... # np.char.lstrip(c,'') does not modify c at all. - True + False >>> (np.char.lstrip(c, ' ') == np.char.lstrip(c, None)).all() True @@ -1399,10 +1392,10 @@ def rstrip(a, chars=None): >>> c = np.array(['aAaAaA', 'abBABba'], dtype='S7'); c array(['aAaAaA', 'abBABba'], dtype='|S7') - >>> np.char.rstrip(c, 'a') + >>> np.char.rstrip(c, b'a') array(['aAaAaA', 'abBABb'], dtype='|S7') - >>> np.char.rstrip(c, 'A') + >>> np.char.rstrip(c, b'A') array(['aAaAa', 'abBABba'], dtype='|S7') @@ -1548,17 +1541,13 @@ def strip(a, chars=None): -------- >>> c = np.array(['aAaAaA', ' aA ', 'abBABba']) >>> c - array(['aAaAaA', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAaA', ' aA ', 'abBABba'], dtype='<U7') >>> np.char.strip(c) - array(['aAaAaA', 'aA', 'abBABba'], - dtype='|S7') + array(['aAaAaA', 'aA', 'abBABba'], dtype='<U7') >>> np.char.strip(c, 'a') # 'a' unstripped from c[1] because whitespace leads - array(['AaAaA', ' aA ', 'bBABb'], - dtype='|S7') + array(['AaAaA', ' aA ', 'bBABb'], dtype='<U7') >>> np.char.strip(c, 'A') # 'A' unstripped from c[1] because (unprinted) ws trails - array(['aAaAa', ' aA ', 'abBABba'], - dtype='|S7') + array(['aAaAa', ' aA ', 'abBABba'], dtype='<U7') """ a_arr = numpy.asarray(a) @@ -1710,11 +1699,9 @@ def upper(a): Examples -------- >>> c = np.array(['a1b c', '1bca', 'bca1']); c - array(['a1b c', '1bca', 'bca1'], - dtype='|S5') + array(['a1b c', '1bca', 'bca1'], dtype='<U5') >>> np.char.upper(c) - array(['A1B C', '1BCA', 'BCA1'], - dtype='|S5') + array(['A1B C', '1BCA', 'BCA1'], dtype='<U5') """ a_arr = numpy.asarray(a) @@ -1820,6 +1807,7 @@ def isdecimal(a): return _vec_string(a, bool_, 'isdecimal') +@set_module('numpy') class chararray(ndarray): """ chararray(shape, itemsize=1, unicode=False, buffer=None, offset=0, @@ -1948,18 +1936,16 @@ class chararray(ndarray): >>> charar = np.chararray((3, 3)) >>> charar[:] = 'a' >>> charar - chararray([['a', 'a', 'a'], - ['a', 'a', 'a'], - ['a', 'a', 'a']], - dtype='|S1') + chararray([[b'a', b'a', b'a'], + [b'a', b'a', b'a'], + [b'a', b'a', b'a']], dtype='|S1') >>> charar = np.chararray(charar.shape, itemsize=5) >>> charar[:] = 'abc' >>> charar - chararray([['abc', 'abc', 'abc'], - ['abc', 'abc', 'abc'], - ['abc', 'abc', 'abc']], - dtype='|S5') + chararray([[b'abc', b'abc', b'abc'], + [b'abc', b'abc', b'abc'], + [b'abc', b'abc', b'abc']], dtype='|S5') """ def __new__(subtype, shape, itemsize=1, unicode=False, buffer=None, diff --git a/numpy/core/einsumfunc.py b/numpy/core/einsumfunc.py index d9f88cb1c..83b7d8287 100644 --- a/numpy/core/einsumfunc.py +++ b/numpy/core/einsumfunc.py @@ -41,10 +41,10 @@ def _flop_count(idx_contraction, inner, num_terms, size_dictionary): -------- >>> _flop_count('abc', False, 1, {'a': 2, 'b':3, 'c':5}) - 90 + 30 >>> _flop_count('abc', True, 2, {'a': 2, 'b':3, 'c':5}) - 270 + 60 """ @@ -169,9 +169,9 @@ def _optimal_path(input_sets, output_set, idx_dict, memory_limit): Examples -------- >>> isets = [set('abd'), set('ac'), set('bdc')] - >>> oset = set('') + >>> oset = set() >>> idx_sizes = {'a': 1, 'b':2, 'c':3, 'd':4} - >>> _path__optimal_path(isets, oset, idx_sizes, 5000) + >>> _optimal_path(isets, oset, idx_sizes, 5000) [(0, 2), (0, 1)] """ @@ -340,9 +340,9 @@ def _greedy_path(input_sets, output_set, idx_dict, memory_limit): Examples -------- >>> isets = [set('abd'), set('ac'), set('bdc')] - >>> oset = set('') + >>> oset = set() >>> idx_sizes = {'a': 1, 'b':2, 'c':3, 'd':4} - >>> _path__greedy_path(isets, oset, idx_sizes, 5000) + >>> _greedy_path(isets, oset, idx_sizes, 5000) [(0, 2), (0, 1)] """ @@ -539,13 +539,14 @@ def _parse_einsum_input(operands): -------- The operand list is simplified to reduce printing: + >>> np.random.seed(123) >>> a = np.random.rand(4, 4) >>> b = np.random.rand(4, 4, 4) - >>> __parse_einsum_input(('...a,...a->...', a, b)) - ('za,xza', 'xz', [a, b]) + >>> _parse_einsum_input(('...a,...a->...', a, b)) + ('za,xza', 'xz', [a, b]) # may vary - >>> __parse_einsum_input((a, [Ellipsis, 0], b, [Ellipsis, 0])) - ('za,xza', 'xz', [a, b]) + >>> _parse_einsum_input((a, [Ellipsis, 0], b, [Ellipsis, 0])) + ('za,xza', 'xz', [a, b]) # may vary """ if len(operands) == 0: @@ -763,6 +764,7 @@ def einsum_path(*operands, **kwargs): of the contraction and the remaining contraction ``(0, 1)`` is then completed. + >>> np.random.seed(123) >>> a = np.random.rand(2, 2) >>> b = np.random.rand(2, 5) >>> c = np.random.rand(5, 2) @@ -770,7 +772,7 @@ def einsum_path(*operands, **kwargs): >>> print(path_info[0]) ['einsum_path', (1, 2), (0, 1)] >>> print(path_info[1]) - Complete contraction: ij,jk,kl->il + Complete contraction: ij,jk,kl->il # may vary Naive scaling: 4 Optimized scaling: 3 Naive FLOP count: 1.600e+02 @@ -789,12 +791,12 @@ def einsum_path(*operands, **kwargs): >>> I = np.random.rand(10, 10, 10, 10) >>> C = np.random.rand(10, 10) >>> path_info = np.einsum_path('ea,fb,abcd,gc,hd->efgh', C, C, I, C, C, - optimize='greedy') + ... optimize='greedy') >>> print(path_info[0]) ['einsum_path', (0, 2), (0, 3), (0, 2), (0, 1)] - >>> print(path_info[1]) - Complete contraction: ea,fb,abcd,gc,hd->efgh + >>> print(path_info[1]) + Complete contraction: ea,fb,abcd,gc,hd->efgh # may vary Naive scaling: 8 Optimized scaling: 5 Naive FLOP count: 8.000e+08 @@ -849,7 +851,6 @@ def einsum_path(*operands, **kwargs): # Python side parsing input_subscripts, output_subscript, operands = _parse_einsum_input(operands) - subscripts = input_subscripts + '->' + output_subscript # Build a few useful list and sets input_list = input_subscripts.split(',') @@ -888,9 +889,8 @@ def einsum_path(*operands, **kwargs): broadcast_indices = [set(x) for x in broadcast_indices] # Compute size of each input array plus the output array - size_list = [] - for term in input_list + [output_subscript]: - size_list.append(_compute_size_by_dict(term, dimension_dict)) + size_list = [_compute_size_by_dict(term, dimension_dict) + for term in input_list + [output_subscript]] max_size = max(size_list) if memory_limit is None: @@ -1276,32 +1276,32 @@ def einsum(*operands, **kwargs): >>> a = np.arange(60.).reshape(3,4,5) >>> b = np.arange(24.).reshape(4,3,2) >>> np.einsum('ijk,jil->kl', a, b) - array([[ 4400., 4730.], - [ 4532., 4874.], - [ 4664., 5018.], - [ 4796., 5162.], - [ 4928., 5306.]]) + array([[4400., 4730.], + [4532., 4874.], + [4664., 5018.], + [4796., 5162.], + [4928., 5306.]]) >>> np.einsum(a, [0,1,2], b, [1,0,3], [2,3]) - array([[ 4400., 4730.], - [ 4532., 4874.], - [ 4664., 5018.], - [ 4796., 5162.], - [ 4928., 5306.]]) + array([[4400., 4730.], + [4532., 4874.], + [4664., 5018.], + [4796., 5162.], + [4928., 5306.]]) >>> np.tensordot(a,b, axes=([1,0],[0,1])) - array([[ 4400., 4730.], - [ 4532., 4874.], - [ 4664., 5018.], - [ 4796., 5162.], - [ 4928., 5306.]]) + array([[4400., 4730.], + [4532., 4874.], + [4664., 5018.], + [4796., 5162.], + [4928., 5306.]]) Writeable returned arrays (since version 1.10.0): >>> a = np.zeros((3, 3)) >>> np.einsum('ii->i', a)[:] = 1 >>> a - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) Example of ellipsis use: @@ -1324,19 +1324,27 @@ def einsum(*operands, **kwargs): particularly significant with larger arrays: >>> a = np.ones(64).reshape(2,4,8) - # Basic `einsum`: ~1520ms (benchmarked on 3.1GHz Intel i5.) + + Basic `einsum`: ~1520ms (benchmarked on 3.1GHz Intel i5.) + >>> for iteration in range(500): - ... np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a) - # Sub-optimal `einsum` (due to repeated path calculation time): ~330ms + ... _ = np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a) + + Sub-optimal `einsum` (due to repeated path calculation time): ~330ms + >>> for iteration in range(500): - ... np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='optimal') - # Greedy `einsum` (faster optimal path approximation): ~160ms + ... _ = np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='optimal') + + Greedy `einsum` (faster optimal path approximation): ~160ms + >>> for iteration in range(500): - ... np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='greedy') - # Optimal `einsum` (best usage pattern in some use cases): ~110ms + ... _ = np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='greedy') + + Optimal `einsum` (best usage pattern in some use cases): ~110ms + >>> path = np.einsum_path('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize='optimal')[0] >>> for iteration in range(500): - ... np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize=path) + ... _ = np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize=path) """ @@ -1375,9 +1383,7 @@ def einsum(*operands, **kwargs): # Start contraction loop for num, contraction in enumerate(contraction_list): inds, idx_rm, einsum_str, remaining, blas = contraction - tmp_operands = [] - for x in inds: - tmp_operands.append(operands.pop(x)) + tmp_operands = [operands.pop(x) for x in inds] # Do we need to deal with the output? handle_out = specified_out and ((num + 1) == len(contraction_list)) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 7dfb52fea..240eac6ce 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -67,10 +67,8 @@ def _wrapfunc(obj, method, *args, **kwds): def _wrapreduction(obj, ufunc, method, axis, dtype, out, **kwargs): - passkwargs = {} - for k, v in kwargs.items(): - if v is not np._NoValue: - passkwargs[k] = v + passkwargs = {k: v for k, v in kwargs.items() + if v is not np._NoValue} if type(obj) is not mu.ndarray: try: @@ -242,12 +240,16 @@ def reshape(a, newshape, order='C'): you should assign the new shape to the shape attribute of the array:: >>> a = np.zeros((10, 2)) + # A transpose makes the array non-contiguous >>> b = a.T + # Taking a view makes it possible to modify the shape without modifying # the initial object. >>> c = b.view() >>> c.shape = (20) + Traceback (most recent call last): + ... AttributeError: incompatible shape for a non-contiguous array The `order` keyword gives the index ordering both for *fetching* the values @@ -1646,21 +1648,21 @@ def ravel(a, order='C'): It is equivalent to ``reshape(-1, order=order)``. >>> x = np.array([[1, 2, 3], [4, 5, 6]]) - >>> print(np.ravel(x)) - [1 2 3 4 5 6] + >>> np.ravel(x) + array([1, 2, 3, 4, 5, 6]) - >>> print(x.reshape(-1)) - [1 2 3 4 5 6] + >>> x.reshape(-1) + array([1, 2, 3, 4, 5, 6]) - >>> print(np.ravel(x, order='F')) - [1 4 2 5 3 6] + >>> np.ravel(x, order='F') + array([1, 4, 2, 5, 3, 6]) When ``order`` is 'A', it will preserve the array's 'C' or 'F' ordering: - >>> print(np.ravel(x.T)) - [1 4 2 5 3 6] - >>> print(np.ravel(x.T, order='A')) - [1 2 3 4 5 6] + >>> np.ravel(x.T) + array([1, 4, 2, 5, 3, 6]) + >>> np.ravel(x.T, order='A') + array([1, 2, 3, 4, 5, 6]) When ``order`` is 'K', it will preserve orderings that are neither 'C' nor 'F', but won't reverse axes: @@ -1749,7 +1751,7 @@ def nonzero(a): array([[0, 0], [1, 1], [2, 0], - [2, 1]) + [2, 1]]) A common use for ``nonzero`` is to find the indices of an array, where a condition is True. Given an array `a`, the condition `a` > 3 is a @@ -2152,10 +2154,10 @@ def any(a, axis=None, out=None, keepdims=np._NoValue): >>> np.any(np.nan) True - >>> o=np.array([False]) + >>> o=np.array(False) >>> z=np.any([-1, 4, 5], out=o) >>> z, o - (array([ True]), array([ True])) + (array(True), array(True)) >>> # Check now that z is a reference to o >>> z is o True @@ -2238,10 +2240,10 @@ def all(a, axis=None, out=None, keepdims=np._NoValue): >>> np.all([1.0, np.nan]) True - >>> o=np.array([False]) + >>> o=np.array(False) >>> z=np.all([-1, 4, 5], out=o) - >>> id(z), id(o), z # doctest: +SKIP - (28293632, 28293632, array([ True])) + >>> id(z), id(o), z + (28293632, 28293632, array(True)) # may vary """ return _wrapreduction(a, np.logical_and, 'all', axis, None, out, keepdims=keepdims) @@ -2726,8 +2728,8 @@ def prod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, initial=np._N raised on overflow. That means that, on a 32-bit platform: >>> x = np.array([536870910, 536870910, 536870910, 536870910]) - >>> np.prod(x) # random - 16 + >>> np.prod(x) + 16 # may vary The product of an empty array is the neutral element 1: @@ -2995,11 +2997,11 @@ def around(a, decimals=0, out=None): Examples -------- >>> np.around([0.37, 1.64]) - array([ 0., 2.]) + array([0., 2.]) >>> np.around([0.37, 1.64], decimals=1) - array([ 0.4, 1.6]) + array([0.4, 1.6]) >>> np.around([.5, 1.5, 2.5, 3.5, 4.5]) # rounds to nearest even value - array([ 0., 2., 2., 4., 4.]) + array([0., 2., 2., 4., 4.]) >>> np.around([1,2,3,11], decimals=1) # ndarray of ints is returned array([ 1, 2, 3, 11]) >>> np.around([1,2,3,11], decimals=-1) @@ -3087,9 +3089,9 @@ def mean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): >>> np.mean(a) 2.5 >>> np.mean(a, axis=0) - array([ 2., 3.]) + array([2., 3.]) >>> np.mean(a, axis=1) - array([ 1.5, 3.5]) + array([1.5, 3.5]) In single precision, `mean` can be inaccurate: @@ -3102,7 +3104,7 @@ def mean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): Computing the mean in float64 is more accurate: >>> np.mean(a, dtype=np.float64) - 0.55000000074505806 + 0.55000000074505806 # may vary """ kwargs = {} @@ -3208,11 +3210,11 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): -------- >>> a = np.array([[1, 2], [3, 4]]) >>> np.std(a) - 1.1180339887498949 + 1.1180339887498949 # may vary >>> np.std(a, axis=0) - array([ 1., 1.]) + array([1., 1.]) >>> np.std(a, axis=1) - array([ 0.5, 0.5]) + array([0.5, 0.5]) In single precision, std() can be inaccurate: @@ -3225,7 +3227,7 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): Computing the standard deviation in float64 is more accurate: >>> np.std(a, dtype=np.float64) - 0.44999999925494177 + 0.44999999925494177 # may vary """ kwargs = {} @@ -3332,9 +3334,9 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): >>> np.var(a) 1.25 >>> np.var(a, axis=0) - array([ 1., 1.]) + array([1., 1.]) >>> np.var(a, axis=1) - array([ 0.25, 0.25]) + array([0.25, 0.25]) In single precision, var() can be inaccurate: @@ -3347,7 +3349,7 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): Computing the variance in float64 is more accurate: >>> np.var(a, dtype=np.float64) - 0.20249999932944759 + 0.20249999932944759 # may vary >>> ((1-0.55)**2 + (0.1-0.55)**2)/2 0.2025 diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 799b1418d..762328173 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -1,16 +1,22 @@ from __future__ import division, absolute_import, print_function +import functools import warnings import operator from . import numeric as _nx from .numeric import (result_type, NaN, shares_memory, MAY_SHARE_BOUNDS, - TooHardError,asanyarray) + TooHardError, asanyarray) from numpy.core.multiarray import add_docstring +from numpy.core import overrides __all__ = ['logspace', 'linspace', 'geomspace'] +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + def _index_deprecate(i, stacklevel=2): try: i = operator.index(i) @@ -23,7 +29,14 @@ def _index_deprecate(i, stacklevel=2): return i -def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): +def _linspace_dispatcher(start, stop, num=None, endpoint=None, retstep=None, + dtype=None, axis=None): + return (start, stop) + + +@array_function_dispatch(_linspace_dispatcher) +def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, + axis=0): """ Return evenly spaced numbers over a specified interval. @@ -32,11 +45,14 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): The endpoint of the interval can optionally be excluded. + .. versionchanged:: 1.16.0 + Non-scalar `start` and `stop` are now supported. + Parameters ---------- - start : scalar + start : array_like The starting value of the sequence. - stop : scalar + stop : array_like The end value of the sequence, unless `endpoint` is set to False. In that case, the sequence consists of all but the last of ``num + 1`` evenly spaced samples, so that `stop` is excluded. Note that the step @@ -55,6 +71,13 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): .. versionadded:: 1.9.0 + axis : int, optional + The axis in the result to store the samples. Relevant only if start + or stop are array-like. By default (0), the samples will be along a + new axis inserted at the beginning. Use -1 to get an axis at the end. + + .. versionadded:: 1.16.0 + Returns ------- samples : ndarray @@ -79,14 +102,17 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): Examples -------- >>> np.linspace(2.0, 3.0, num=5) - array([ 2. , 2.25, 2.5 , 2.75, 3. ]) + array([2. , 2.25, 2.5 , 2.75, 3. ]) >>> np.linspace(2.0, 3.0, num=5, endpoint=False) - array([ 2. , 2.2, 2.4, 2.6, 2.8]) + array([2. , 2.2, 2.4, 2.6, 2.8]) >>> np.linspace(2.0, 3.0, num=5, retstep=True) - (array([ 2. , 2.25, 2.5 , 2.75, 3. ]), 0.25) + (array([2. , 2.25, 2.5 , 2.75, 3. ]), 0.25) Graphical illustration: + >>> import matplotlib + >>> import matplotlib.pyplot + >>> matplotlib.pyplot.switch_backend('agg') >>> import matplotlib.pyplot as plt >>> N = 8 >>> y = np.zeros(N) @@ -116,16 +142,15 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): if dtype is None: dtype = dt - y = _nx.arange(0, num, dtype=dt) - delta = stop - start + y = _nx.arange(0, num, dtype=dt).reshape((-1,) + (1,) * delta.ndim) # In-place multiplication y *= delta/div is faster, but prevents the multiplicant # from overriding what class is produced, and thus prevents, e.g. use of Quantities, # see gh-7142. Hence, we multiply in place only for standard scalar types. - _mult_inplace = _nx.isscalar(delta) + _mult_inplace = _nx.isscalar(delta) if num > 1: step = delta / div - if step == 0: + if _nx.any(step == 0): # Special handling for denormal numbers, gh-5437 y /= div if _mult_inplace: @@ -148,13 +173,23 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): if endpoint and num > 1: y[-1] = stop + if axis != 0: + y = _nx.moveaxis(y, 0, axis) + if retstep: return y.astype(dtype, copy=False), step else: return y.astype(dtype, copy=False) -def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): +def _logspace_dispatcher(start, stop, num=None, endpoint=None, base=None, + dtype=None, axis=None): + return (start, stop) + + +@array_function_dispatch(_logspace_dispatcher) +def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, + axis=0): """ Return numbers spaced evenly on a log scale. @@ -162,11 +197,14 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): (`base` to the power of `start`) and ends with ``base ** stop`` (see `endpoint` below). + .. versionchanged:: 1.16.0 + Non-scalar `start` and `stop` are now supported. + Parameters ---------- - start : float + start : array_like ``base ** start`` is the starting value of the sequence. - stop : float + stop : array_like ``base ** stop`` is the final value of the sequence, unless `endpoint` is False. In that case, ``num + 1`` values are spaced over the interval in log-space, of which all but the last (a sequence of @@ -183,6 +221,13 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): dtype : dtype The type of the output array. If `dtype` is not given, infer the data type from the other input arguments. + axis : int, optional + The axis in the result to store the samples. Relevant only if start + or stop are array-like. By default (0), the samples will be along a + new axis inserted at the beginning. Use -1 to get an axis at the end. + + .. versionadded:: 1.16.0 + Returns ------- @@ -210,14 +255,17 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): Examples -------- >>> np.logspace(2.0, 3.0, num=4) - array([ 100. , 215.443469 , 464.15888336, 1000. ]) + array([ 100. , 215.443469 , 464.15888336, 1000. ]) >>> np.logspace(2.0, 3.0, num=4, endpoint=False) - array([ 100. , 177.827941 , 316.22776602, 562.34132519]) + array([100. , 177.827941 , 316.22776602, 562.34132519]) >>> np.logspace(2.0, 3.0, num=4, base=2.0) - array([ 4. , 5.0396842 , 6.34960421, 8. ]) + array([4. , 5.0396842 , 6.34960421, 8. ]) Graphical illustration: + >>> import matplotlib + >>> import matplotlib.pyplot + >>> matplotlib.pyplot.switch_backend('agg') >>> import matplotlib.pyplot as plt >>> N = 10 >>> x1 = np.logspace(0.1, 1, N, endpoint=True) @@ -232,24 +280,33 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None): >>> plt.show() """ - y = linspace(start, stop, num=num, endpoint=endpoint) + y = linspace(start, stop, num=num, endpoint=endpoint, axis=axis) if dtype is None: return _nx.power(base, y) - return _nx.power(base, y).astype(dtype) + return _nx.power(base, y).astype(dtype, copy=False) + + +def _geomspace_dispatcher(start, stop, num=None, endpoint=None, dtype=None, + axis=None): + return (start, stop) -def geomspace(start, stop, num=50, endpoint=True, dtype=None): +@array_function_dispatch(_geomspace_dispatcher) +def geomspace(start, stop, num=50, endpoint=True, dtype=None, axis=0): """ Return numbers spaced evenly on a log scale (a geometric progression). This is similar to `logspace`, but with endpoints specified directly. Each output sample is a constant multiple of the previous. + .. versionchanged:: 1.16.0 + Non-scalar `start` and `stop` are now supported. + Parameters ---------- - start : scalar + start : array_like The starting value of the sequence. - stop : scalar + stop : array_like The final value of the sequence, unless `endpoint` is False. In that case, ``num + 1`` values are spaced over the interval in log-space, of which all but the last (a sequence of @@ -262,6 +319,12 @@ def geomspace(start, stop, num=50, endpoint=True, dtype=None): dtype : dtype The type of the output array. If `dtype` is not given, infer the data type from the other input arguments. + axis : int, optional + The axis in the result to store the samples. Relevant only if start + or stop are array-like. By default (0), the samples will be along a + new axis inserted at the beginning. Use -1 to get an axis at the end. + + .. versionadded:: 1.16.0 Returns ------- @@ -304,62 +367,75 @@ def geomspace(start, stop, num=50, endpoint=True, dtype=None): Negative, decreasing, and complex inputs are allowed: >>> np.geomspace(1000, 1, num=4) - array([ 1000., 100., 10., 1.]) + array([1000., 100., 10., 1.]) >>> np.geomspace(-1000, -1, num=4) array([-1000., -100., -10., -1.]) >>> np.geomspace(1j, 1000j, num=4) # Straight line - array([ 0. +1.j, 0. +10.j, 0. +100.j, 0.+1000.j]) + array([0. +1.j, 0. +10.j, 0. +100.j, 0.+1000.j]) >>> np.geomspace(-1+0j, 1+0j, num=5) # Circle - array([-1.00000000+0.j , -0.70710678+0.70710678j, - 0.00000000+1.j , 0.70710678+0.70710678j, - 1.00000000+0.j ]) + array([-1.00000000e+00+1.22464680e-16j, -7.07106781e-01+7.07106781e-01j, + 6.12323400e-17+1.00000000e+00j, 7.07106781e-01+7.07106781e-01j, + 1.00000000e+00+0.00000000e+00j]) Graphical illustration of ``endpoint`` parameter: + >>> import matplotlib + >>> matplotlib.use('agg') >>> import matplotlib.pyplot as plt >>> N = 10 >>> y = np.zeros(N) >>> plt.semilogx(np.geomspace(1, 1000, N, endpoint=True), y + 1, 'o') + [<matplotlib.lines.Line2D object at 0x...>] >>> plt.semilogx(np.geomspace(1, 1000, N, endpoint=False), y + 2, 'o') + [<matplotlib.lines.Line2D object at 0x...>] >>> plt.axis([0.5, 2000, 0, 3]) + [0.5, 2000, 0, 3] >>> plt.grid(True, color='0.7', linestyle='-', which='both', axis='both') >>> plt.show() """ - if start == 0 or stop == 0: + start = asanyarray(start) + stop = asanyarray(stop) + if _nx.any(start == 0) or _nx.any(stop == 0): raise ValueError('Geometric sequence cannot include zero') - dt = result_type(start, stop, float(num)) + dt = result_type(start, stop, float(num), _nx.zeros((), dtype)) if dtype is None: dtype = dt else: # complex to dtype('complex128'), for instance dtype = _nx.dtype(dtype) + # Promote both arguments to the same dtype in case, for instance, one is + # complex and another is negative and log would produce NaN otherwise. + # Copy since we may change things in-place further down. + start = start.astype(dt, copy=True) + stop = stop.astype(dt, copy=True) + + out_sign = _nx.ones(_nx.broadcast(start, stop).shape, dt) # Avoid negligible real or imaginary parts in output by rotating to # positive real, calculating, then undoing rotation - out_sign = 1 - if start.real == stop.real == 0: - start, stop = start.imag, stop.imag - out_sign = 1j * out_sign - if _nx.sign(start) == _nx.sign(stop) == -1: - start, stop = -start, -stop - out_sign = -out_sign - - # Promote both arguments to the same dtype in case, for instance, one is - # complex and another is negative and log would produce NaN otherwise - start = start + (stop - stop) - stop = stop + (start - start) - if _nx.issubdtype(dtype, _nx.complexfloating): - start = start + 0j - stop = stop + 0j + if _nx.issubdtype(dt, _nx.complexfloating): + all_imag = (start.real == 0.) & (stop.real == 0.) + if _nx.any(all_imag): + start[all_imag] = start[all_imag].imag + stop[all_imag] = stop[all_imag].imag + out_sign[all_imag] = 1j + + both_negative = (_nx.sign(start) == -1) & (_nx.sign(stop) == -1) + if _nx.any(both_negative): + _nx.negative(start, out=start, where=both_negative) + _nx.negative(stop, out=stop, where=both_negative) + _nx.negative(out_sign, out=out_sign, where=both_negative) log_start = _nx.log10(start) log_stop = _nx.log10(stop) result = out_sign * logspace(log_start, log_stop, num=num, endpoint=endpoint, base=10.0, dtype=dtype) + if axis != 0: + result = _nx.moveaxis(result, 0, axis) - return result.astype(dtype) + return result.astype(dtype, copy=False) #always succeed diff --git a/numpy/core/getlimits.py b/numpy/core/getlimits.py index 389f16ff5..544b8b35f 100644 --- a/numpy/core/getlimits.py +++ b/numpy/core/getlimits.py @@ -8,6 +8,7 @@ __all__ = ['finfo', 'iinfo'] import warnings from .machar import MachAr +from .overrides import set_module from . import numeric from . import numerictypes as ntypes from .numeric import array, inf @@ -289,6 +290,7 @@ def _discovered_machar(ftype): params['title']) +@set_module('numpy') class finfo(object): """ finfo(dtype) @@ -439,6 +441,7 @@ class finfo(object): " max=%(_str_max)s, dtype=%(dtype)s)") % d) +@set_module('numpy') class iinfo(object): """ iinfo(type) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index da006909a..b0b749c80 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -505,7 +505,8 @@ typedef struct { PyArray_NonzeroFunc *nonzero; /* - * Used for arange. + * Used for arange. Should return 0 on success + * and -1 on failure. * Can be NULL. */ PyArray_FillFunc *fill; diff --git a/numpy/core/include/numpy/npy_1_7_deprecated_api.h b/numpy/core/include/numpy/npy_1_7_deprecated_api.h index 76b57b748..a6ee21219 100644 --- a/numpy/core/include/numpy/npy_1_7_deprecated_api.h +++ b/numpy/core/include/numpy/npy_1_7_deprecated_api.h @@ -5,6 +5,8 @@ #error "Should never include npy_*_*_deprecated_api directly." #endif +/* Emit a warning if the user did not specifically request the old API */ +#ifndef NPY_NO_DEPRECATED_API #if defined(_WIN32) #define _WARN___STR2__(x) #x #define _WARN___STR1__(x) _WARN___STR2__(x) @@ -16,6 +18,7 @@ "#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION" #endif /* TODO: How to do this warning message for other compilers? */ +#endif /* * This header exists to collect all dangerous/deprecated NumPy API diff --git a/numpy/core/include/numpy/npy_3kcompat.h b/numpy/core/include/numpy/npy_3kcompat.h index a3c69f44e..832bc0599 100644 --- a/numpy/core/include/numpy/npy_3kcompat.h +++ b/numpy/core/include/numpy/npy_3kcompat.h @@ -219,6 +219,7 @@ npy_PyFile_Dup2(PyObject *file, char *mode, npy_off_t *orig_pos) if (handle == NULL) { PyErr_SetString(PyExc_IOError, "Getting a FILE* from a Python file object failed"); + return NULL; } /* Record the original raw file handle position */ @@ -383,6 +384,36 @@ npy_PyFile_CloseFile(PyObject *file) } +/* This is a copy of _PyErr_ChainExceptions + */ +static NPY_INLINE void +npy_PyErr_ChainExceptions(PyObject *exc, PyObject *val, PyObject *tb) +{ + if (exc == NULL) + return; + + if (PyErr_Occurred()) { + /* only py3 supports this anyway */ + #ifdef NPY_PY3K + PyObject *exc2, *val2, *tb2; + PyErr_Fetch(&exc2, &val2, &tb2); + PyErr_NormalizeException(&exc, &val, &tb); + if (tb != NULL) { + PyException_SetTraceback(val, tb); + Py_DECREF(tb); + } + Py_DECREF(exc); + PyErr_NormalizeException(&exc2, &val2, &tb2); + PyException_SetContext(val2, val); + PyErr_Restore(exc2, val2, tb2); + #endif + } + else { + PyErr_Restore(exc, val, tb); + } +} + + /* This is a copy of _PyErr_ChainExceptions, with: * - a minimal implementation for python 2 * - __cause__ used instead of __context__ diff --git a/numpy/core/include/numpy/ufuncobject.h b/numpy/core/include/numpy/ufuncobject.h index 85f8a6c08..90d837a9b 100644 --- a/numpy/core/include/numpy/ufuncobject.h +++ b/numpy/core/include/numpy/ufuncobject.h @@ -223,7 +223,8 @@ typedef struct _tagPyUFuncObject { */ npy_uint32 *core_dim_flags; - + /* Identity for reduction, when identity == PyUFunc_IdentityValue */ + PyObject *identity_value; } PyUFuncObject; @@ -299,6 +300,12 @@ typedef struct _tagPyUFuncObject { * This case allows reduction with multiple axes at once. */ #define PyUFunc_ReorderableNone -2 +/* + * UFunc unit is in identity_value, and the order of operations can be reordered + * This case allows reduction with multiple axes at once. + */ +#define PyUFunc_IdentityValue -3 + #define UFUNC_REDUCE 0 #define UFUNC_ACCUMULATE 1 diff --git a/numpy/core/machar.py b/numpy/core/machar.py index 7578544fe..91fb4eda8 100644 --- a/numpy/core/machar.py +++ b/numpy/core/machar.py @@ -11,9 +11,11 @@ __all__ = ['MachAr'] from numpy.core.fromnumeric import any from numpy.core.numeric import errstate +from numpy.core.overrides import set_module # Need to speed this up...especially for longfloat +@set_module('numpy') class MachAr(object): """ Diagnosing machine parameters. diff --git a/numpy/core/memmap.py b/numpy/core/memmap.py index f5cc68bb9..9ba4817f4 100644 --- a/numpy/core/memmap.py +++ b/numpy/core/memmap.py @@ -5,6 +5,7 @@ from .numeric import uint8, ndarray, dtype from numpy.compat import ( long, basestring, os_fspath, contextlib_nullcontext, is_pathlib_path ) +from numpy.core.overrides import set_module __all__ = ['memmap'] @@ -19,6 +20,8 @@ mode_equivalents = { "write":"w+" } + +@set_module('numpy') class memmap(ndarray): """Create a memory-map to an array stored in a *binary* file on disk. @@ -132,9 +135,9 @@ class memmap(ndarray): >>> fp = np.memmap(filename, dtype='float32', mode='w+', shape=(3,4)) >>> fp - memmap([[ 0., 0., 0., 0.], - [ 0., 0., 0., 0.], - [ 0., 0., 0., 0.]], dtype=float32) + memmap([[0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.]], dtype=float32) Write data to memmap array: diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 25debd2f8..4c2715892 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -7,6 +7,7 @@ by importing from the extension module. """ import functools +import warnings from . import overrides from . import _multiarray_umath @@ -39,17 +40,38 @@ __all__ = [ 'tracemalloc_domain', 'typeinfo', 'unpackbits', 'unravel_index', 'vdot', 'where', 'zeros'] -array_function_dispatch = functools.partial( - overrides.array_function_dispatch, module='numpy') - - -def _empty_like_dispatcher(prototype, dtype=None, order=None, subok=None): - return (prototype,) +arange.__module__ = 'numpy' +array.__module__ = 'numpy' +datetime_data.__module__ = 'numpy' +empty.__module__ = 'numpy' +frombuffer.__module__ = 'numpy' +fromfile.__module__ = 'numpy' +fromiter.__module__ = 'numpy' +frompyfunc.__module__ = 'numpy' +fromstring.__module__ = 'numpy' +geterrobj.__module__ = 'numpy' +may_share_memory.__module__ = 'numpy' +nested_iters.__module__ = 'numpy' +promote_types.__module__ = 'numpy' +set_numeric_ops.__module__ = 'numpy' +seterrobj.__module__ = 'numpy' +zeros.__module__ = 'numpy' + + +# We can't verify dispatcher signatures because NumPy's C functions don't +# support introspection. +array_function_from_c_func_and_dispatcher = functools.partial( + overrides.array_function_from_dispatcher, + module='numpy', docs_from_dispatcher=True, verify=False) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.empty_like) +def empty_like(prototype, dtype=None, order=None, subok=None): + """ + empty_like(prototype, dtype=None, order='K', subok=True) -@array_function_dispatch(_empty_like_dispatcher) -def empty_like(prototype, dtype=None, order='K', subok=True): - """Return a new array with the same shape and type as a given array. + Return a new array with the same shape and type as a given array. Parameters ---------- @@ -95,25 +117,19 @@ def empty_like(prototype, dtype=None, order='K', subok=True): -------- >>> a = ([1,2,3], [4,5,6]) # a is array-like >>> np.empty_like(a) - array([[-1073741821, -1073741821, 3], #random + array([[-1073741821, -1073741821, 3], # random [ 0, 0, -1073741821]]) >>> a = np.array([[1., 2., 3.],[4.,5.,6.]]) >>> np.empty_like(a) - array([[ -2.00000715e+000, 1.48219694e-323, -2.00000572e+000],#random + array([[ -2.00000715e+000, 1.48219694e-323, -2.00000572e+000], # random [ 4.38791518e-305, -2.00000715e+000, 4.17269252e-309]]) """ - return _multiarray_umath.empty_like(prototype, dtype, order, subok) - - -def _concatenate_dispatcher(arrays, axis=None, out=None): - for array in arrays: - yield array - yield out + return (prototype,) -@array_function_dispatch(_concatenate_dispatcher) -def concatenate(arrays, axis=0, out=None): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.concatenate) +def concatenate(arrays, axis=None, out=None): """ concatenate((a1, a2, ...), axis=0, out=None) @@ -195,16 +211,18 @@ def concatenate(arrays, axis=0, out=None): fill_value=999999) """ - return _multiarray_umath.concatenate(arrays, axis, out) - - -def _inner_dispatcher(a, b): - return (a, b) + if out is not None: + # optimize for the typical case where only arrays is provided + arrays = list(arrays) + arrays.append(out) + return arrays -@array_function_dispatch(_inner_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.inner) def inner(a, b): """ + inner(a, b) + Inner product of two arrays. Ordinary inner product of vectors for 1-D arrays (without complex @@ -270,19 +288,15 @@ def inner(a, b): An example where `b` is a scalar: >>> np.inner(np.eye(2), 7) - array([[ 7., 0.], - [ 0., 7.]]) + array([[7., 0.], + [0., 7.]]) """ - return _multiarray_umath.inner(a, b) - - -def _where_dispatcher(condition, x=None, y=None): - return (condition, x, y) + return (a, b) -@array_function_dispatch(_where_dispatcher) -def where(condition, x=np._NoValue, y=np._NoValue): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.where) +def where(condition, x=None, y=None): """ where(condition, [x, y]) @@ -353,21 +367,14 @@ def where(condition, x=np._NoValue, y=np._NoValue): [ 0, 2, -1], [ 0, 3, -1]]) """ - # _multiarray_umath.where only accepts positional arguments - args = tuple(a for a in (x, y) if a is not np._NoValue) - return _multiarray_umath.where(condition, *args) - - -def _lexsort_dispatcher(keys, axis=None): - if isinstance(keys, tuple): - return keys - else: - return (keys,) + return (condition, x, y) -@array_function_dispatch(_lexsort_dispatcher) -def lexsort(keys, axis=-1): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.lexsort) +def lexsort(keys, axis=None): """ + lexsort(keys, axis=-1) + Perform an indirect stable sort using a sequence of keys. Given multiple sorting keys, which can be interpreted as columns in a @@ -416,8 +423,8 @@ def lexsort(keys, axis=-1): >>> a = [1,5,1,4,3,4,4] # First column >>> b = [9,4,0,4,0,2,1] # Second column >>> ind = np.lexsort((b,a)) # Sort by a, then by b - >>> print(ind) - [2 0 4 6 5 3 1] + >>> ind + array([2, 0, 4, 6, 5, 3, 1]) >>> [(a[i],b[i]) for i in ind] [(1, 0), (1, 9), (3, 0), (4, 1), (4, 2), (4, 4), (5, 4)] @@ -439,16 +446,17 @@ def lexsort(keys, axis=-1): array([2, 0, 4, 6, 5, 3, 1]) """ - return _multiarray_umath.lexsort(keys, axis) - - -def _can_cast_dispatcher(from_, to, casting=None): - return (from_,) + if isinstance(keys, tuple): + return keys + else: + return (keys,) -@array_function_dispatch(_can_cast_dispatcher) -def can_cast(from_, to, casting='safe'): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.can_cast) +def can_cast(from_, to, casting=None): """ + can_cast(from_, to, casting='safe') + Returns True if cast between data types can occur according to the casting rule. If from is a scalar or array scalar, also returns True if the scalar value can be cast without overflow or truncation @@ -552,16 +560,14 @@ def can_cast(from_, to, casting='safe'): True """ - return _multiarray_umath.can_cast(from_, to, casting) - - -def _min_scalar_type_dispatcher(a): - return (a,) + return (from_,) -@array_function_dispatch(_min_scalar_type_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.min_scalar_type) def min_scalar_type(a): """ + min_scalar_type(a) + For scalar ``a``, returns the data type with the smallest size and smallest scalar kind which can hold its value. For non-scalar array ``a``, returns the vector's dtype unmodified. @@ -605,16 +611,14 @@ def min_scalar_type(a): dtype('float64') """ - return _multiarray_umath.min_scalar_type(a) - - -def _result_type_dispatcher(*arrays_and_dtypes): - return arrays_and_dtypes + return (a,) -@array_function_dispatch(_result_type_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.result_type) def result_type(*arrays_and_dtypes): """ + result_type(*arrays_and_dtypes) + Returns the type that results from applying the NumPy type promotion rules to the arguments. @@ -679,16 +683,14 @@ def result_type(*arrays_and_dtypes): dtype('float64') """ - return _multiarray_umath.result_type(*arrays_and_dtypes) - - -def _dot_dispatcher(a, b, out=None): - return (a, b, out) + return arrays_and_dtypes -@array_function_dispatch(_dot_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.dot) def dot(a, b, out=None): """ + dot(a, b, out=None) + Dot product of two arrays. Specifically, - If both `a` and `b` are 1-D arrays, it is inner product of vectors @@ -769,16 +771,14 @@ def dot(a, b, out=None): 499128 """ - return _multiarray_umath.dot(a, b, out) - - -def _vdot_dispatcher(a, b): - return (a, b) + return (a, b, out) -@array_function_dispatch(_vdot_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.vdot) def vdot(a, b): """ + vdot(a, b) + Return the dot product of two vectors. The vdot(`a`, `b`) function handles complex numbers differently than @@ -829,17 +829,464 @@ def vdot(a, b): 30 """ - return _multiarray_umath.vdot(a, b) + return (a, b) -def _is_busday_dispatcher( - dates, weekmask=None, holidays=None, busdaycal=None, out=None): - return (dates, weekmask, holidays, out) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.bincount) +def bincount(x, weights=None, minlength=None): + """ + bincount(x, weights=None, minlength=0) + + Count number of occurrences of each value in array of non-negative ints. + + The number of bins (of size 1) is one larger than the largest value in + `x`. If `minlength` is specified, there will be at least this number + of bins in the output array (though it will be longer if necessary, + depending on the contents of `x`). + Each bin gives the number of occurrences of its index value in `x`. + If `weights` is specified the input array is weighted by it, i.e. if a + value ``n`` is found at position ``i``, ``out[n] += weight[i]`` instead + of ``out[n] += 1``. + + Parameters + ---------- + x : array_like, 1 dimension, nonnegative ints + Input array. + weights : array_like, optional + Weights, array of the same shape as `x`. + minlength : int, optional + A minimum number of bins for the output array. + + .. versionadded:: 1.6.0 + + Returns + ------- + out : ndarray of ints + The result of binning the input array. + The length of `out` is equal to ``np.amax(x)+1``. + + Raises + ------ + ValueError + If the input is not 1-dimensional, or contains elements with negative + values, or if `minlength` is negative. + TypeError + If the type of the input is float or complex. + + See Also + -------- + histogram, digitize, unique + + Examples + -------- + >>> np.bincount(np.arange(5)) + array([1, 1, 1, 1, 1]) + >>> np.bincount(np.array([0, 1, 1, 3, 2, 1, 7])) + array([1, 3, 1, 1, 0, 0, 0, 1]) + + >>> x = np.array([0, 1, 1, 3, 2, 1, 7, 23]) + >>> np.bincount(x).size == np.amax(x)+1 + True + + The input array needs to be of integer dtype, otherwise a + TypeError is raised: + + >>> np.bincount(np.arange(5, dtype=float)) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + TypeError: array cannot be safely cast to required type + + A possible use of ``bincount`` is to perform sums over + variable-size chunks of an array, using the ``weights`` keyword. + + >>> w = np.array([0.3, 0.5, 0.2, 0.7, 1., -0.6]) # weights + >>> x = np.array([0, 1, 1, 2, 2, 2]) + >>> np.bincount(x, weights=w) + array([ 0.3, 0.7, 1.1]) + + """ + return (x, weights) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.ravel_multi_index) +def ravel_multi_index(multi_index, dims, mode=None, order=None): + """ + ravel_multi_index(multi_index, dims, mode='raise', order='C') + + Converts a tuple of index arrays into an array of flat + indices, applying boundary modes to the multi-index. + + Parameters + ---------- + multi_index : tuple of array_like + A tuple of integer arrays, one array for each dimension. + dims : tuple of ints + The shape of array into which the indices from ``multi_index`` apply. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices are handled. Can specify + either one mode or a tuple of modes, one mode per index. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + In 'clip' mode, a negative index which would normally + wrap will clip to 0 instead. + order : {'C', 'F'}, optional + Determines whether the multi-index should be viewed as + indexing in row-major (C-style) or column-major + (Fortran-style) order. + + Returns + ------- + raveled_indices : ndarray + An array of indices into the flattened version of an array + of dimensions ``dims``. + + See Also + -------- + unravel_index + + Notes + ----- + .. versionadded:: 1.6.0 + + Examples + -------- + >>> arr = np.array([[3,6,6],[4,5,1]]) + >>> np.ravel_multi_index(arr, (7,6)) + array([22, 41, 37]) + >>> np.ravel_multi_index(arr, (7,6), order='F') + array([31, 41, 13]) + >>> np.ravel_multi_index(arr, (4,6), mode='clip') + array([22, 23, 19]) + >>> np.ravel_multi_index(arr, (4,4), mode=('clip','wrap')) + array([12, 13, 13]) + + >>> np.ravel_multi_index((3,1,4,1), (6,7,8,9)) + 1621 + """ + return multi_index + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.unravel_index) +def unravel_index(indices, shape=None, order=None, dims=None): + """ + unravel_index(indices, shape, order='C') + + Converts a flat index or array of flat indices into a tuple + of coordinate arrays. + + Parameters + ---------- + indices : array_like + An integer array whose elements are indices into the flattened + version of an array of dimensions ``shape``. Before version 1.6.0, + this function accepted just one index value. + shape : tuple of ints + The shape of the array to use for unraveling ``indices``. + + .. versionchanged:: 1.16.0 + Renamed from ``dims`` to ``shape``. + + order : {'C', 'F'}, optional + Determines whether the indices should be viewed as indexing in + row-major (C-style) or column-major (Fortran-style) order. + + .. versionadded:: 1.6.0 + + Returns + ------- + unraveled_coords : tuple of ndarray + Each array in the tuple has the same shape as the ``indices`` + array. + + See Also + -------- + ravel_multi_index + + Examples + -------- + >>> np.unravel_index([22, 41, 37], (7,6)) + (array([3, 6, 6]), array([4, 5, 1])) + >>> np.unravel_index([31, 41, 13], (7,6), order='F') + (array([3, 6, 6]), array([4, 5, 1])) + + >>> np.unravel_index(1621, (6,7,8,9)) + (3, 1, 4, 1) + + """ + if dims is not None: + warnings.warn("'shape' argument should be used instead of 'dims'", + DeprecationWarning, stacklevel=3) + return (indices,) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.copyto) +def copyto(dst, src, casting=None, where=None): + """ + copyto(dst, src, casting='same_kind', where=True) + + Copies values from one array to another, broadcasting as necessary. + + Raises a TypeError if the `casting` rule is violated, and if + `where` is provided, it selects which elements to copy. + + .. versionadded:: 1.7.0 + + Parameters + ---------- + dst : ndarray + The array into which values are copied. + src : array_like + The array from which values are copied. + casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional + Controls what kind of data casting may occur when copying. + + * 'no' means the data types should not be cast at all. + * 'equiv' means only byte-order changes are allowed. + * 'safe' means only casts which can preserve values are allowed. + * 'same_kind' means only safe casts or casts within a kind, + like float64 to float32, are allowed. + * 'unsafe' means any data conversions may be done. + where : array_like of bool, optional + A boolean array which is broadcasted to match the dimensions + of `dst`, and selects elements to copy from `src` to `dst` + wherever it contains the value True. + """ + return (dst, src, where) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.putmask) +def putmask(a, mask, values): + """ + putmask(a, mask, values) + + Changes elements of an array based on conditional and input values. + + Sets ``a.flat[n] = values[n]`` for each n where ``mask.flat[n]==True``. + + If `values` is not the same size as `a` and `mask` then it will repeat. + This gives behavior different from ``a[mask] = values``. + + Parameters + ---------- + a : array_like + Target array. + mask : array_like + Boolean mask array. It has to be the same shape as `a`. + values : array_like + Values to put into `a` where `mask` is True. If `values` is smaller + than `a` it will be repeated. + + See Also + -------- + place, put, take, copyto + + Examples + -------- + >>> x = np.arange(6).reshape(2, 3) + >>> np.putmask(x, x>2, x**2) + >>> x + array([[ 0, 1, 2], + [ 9, 16, 25]]) + + If `values` is smaller than `a` it is repeated: + + >>> x = np.arange(5) + >>> np.putmask(x, x>1, [-33, -44]) + >>> x + array([ 0, 1, -33, -44, -33]) + + """ + return (a, mask, values) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.packbits) +def packbits(myarray, axis=None): + """ + packbits(myarray, axis=None) + + Packs the elements of a binary-valued array into bits in a uint8 array. + + The result is padded to full bytes by inserting zero bits at the end. + Parameters + ---------- + myarray : array_like + An array of integers or booleans whose elements should be packed to + bits. + axis : int, optional + The dimension over which bit-packing is done. + ``None`` implies packing the flattened array. + + Returns + ------- + packed : ndarray + Array of type uint8 whose elements represent bits corresponding to the + logical (0 or nonzero) value of the input elements. The shape of + `packed` has the same number of dimensions as the input (unless `axis` + is None, in which case the output is 1-D). + + See Also + -------- + unpackbits: Unpacks elements of a uint8 array into a binary-valued output + array. + + Examples + -------- + >>> a = np.array([[[1,0,1], + ... [0,1,0]], + ... [[1,1,0], + ... [0,0,1]]]) + >>> b = np.packbits(a, axis=-1) + >>> b + array([[[160], + [ 64]], + [[192], + [ 32]]], dtype=uint8) + + Note that in binary 160 = 1010 0000, 64 = 0100 0000, 192 = 1100 0000, + and 32 = 0010 0000. -@array_function_dispatch(_is_busday_dispatcher) -def is_busday(dates, weekmask=None, holidays=None, busdaycal=None, - out=None): + """ + return (myarray,) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.unpackbits) +def unpackbits(myarray, axis=None): + """ + unpackbits(myarray, axis=None) + + Unpacks elements of a uint8 array into a binary-valued output array. + + Each element of `myarray` represents a bit-field that should be unpacked + into a binary-valued output array. The shape of the output array is either + 1-D (if `axis` is None) or the same shape as the input array with unpacking + done along the axis specified. + + Parameters + ---------- + myarray : ndarray, uint8 type + Input array. + axis : int, optional + The dimension over which bit-unpacking is done. + ``None`` implies unpacking the flattened array. + + Returns + ------- + unpacked : ndarray, uint8 type + The elements are binary-valued (0 or 1). + + See Also + -------- + packbits : Packs the elements of a binary-valued array into bits in a uint8 + array. + + Examples + -------- + >>> a = np.array([[2], [7], [23]], dtype=np.uint8) + >>> a + array([[ 2], + [ 7], + [23]], dtype=uint8) + >>> b = np.unpackbits(a, axis=1) + >>> b + array([[0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 0, 1, 1, 1]], dtype=uint8) + + """ + return (myarray,) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.shares_memory) +def shares_memory(a, b, max_work=None): + """ + shares_memory(a, b, max_work=None) + + Determine if two arrays share memory + + Parameters + ---------- + a, b : ndarray + Input arrays + max_work : int, optional + Effort to spend on solving the overlap problem (maximum number + of candidate solutions to consider). The following special + values are recognized: + + max_work=MAY_SHARE_EXACT (default) + The problem is solved exactly. In this case, the function returns + True only if there is an element shared between the arrays. + max_work=MAY_SHARE_BOUNDS + Only the memory bounds of a and b are checked. + + Raises + ------ + numpy.TooHardError + Exceeded max_work. + + Returns + ------- + out : bool + + See Also + -------- + may_share_memory + + Examples + -------- + >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9])) + False + + """ + return (a, b) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.may_share_memory) +def may_share_memory(a, b, max_work=None): + """ + may_share_memory(a, b, max_work=None) + + Determine if two arrays might share memory + + A return of True does not necessarily mean that the two arrays + share any element. It just means that they *might*. + + Only the memory bounds of a and b are checked by default. + + Parameters + ---------- + a, b : ndarray + Input arrays + max_work : int, optional + Effort to spend on solving the overlap problem. See + `shares_memory` for details. Default for ``may_share_memory`` + is to do a bounds check. + + Returns + ------- + out : bool + + See Also + -------- + shares_memory + + Examples + -------- + >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9])) + False + >>> x = np.zeros([3, 4]) + >>> np.may_share_memory(x[:,0], x[:,1]) + True + + """ + return (a, b) + + +@array_function_from_c_func_and_dispatcher(_multiarray_umath.is_busday) +def is_busday(dates, weekmask=None, holidays=None, busdaycal=None, out=None): """ is_busday(dates, weekmask='1111100', holidays=None, busdaycal=None, out=None) @@ -887,28 +1334,14 @@ def is_busday(dates, weekmask=None, holidays=None, busdaycal=None, >>> # The weekdays are Friday, Saturday, and Monday ... np.is_busday(['2011-07-01', '2011-07-02', '2011-07-18'], ... holidays=['2011-07-01', '2011-07-04', '2011-07-17']) - array([False, False, True], dtype='bool') + array([False, False, True]) """ - kwargs = {} - if weekmask is not None: - kwargs['weekmask'] = weekmask - if holidays is not None: - kwargs['holidays'] = holidays - if busdaycal is not None: - kwargs['busdaycal'] = busdaycal - if out is not None: - kwargs['out'] = out - return _multiarray_umath.is_busday(dates, **kwargs) - - -def _busday_offset_dispatcher(dates, offsets, roll=None, weekmask=None, - holidays=None, busdaycal=None, out=None): - return (dates, offsets, weekmask, holidays, out) + return (dates, weekmask, holidays, out) -@array_function_dispatch(_busday_offset_dispatcher) -def busday_offset(dates, offsets, roll='raise', weekmask=None, - holidays=None, busdaycal=None, out=None): +@array_function_from_c_func_and_dispatcher(_multiarray_umath.busday_offset) +def busday_offset(dates, offsets, roll=None, weekmask=None, holidays=None, + busdaycal=None, out=None): """ busday_offset(dates, offsets, roll='raise', weekmask='1111100', holidays=None, busdaycal=None, out=None) @@ -975,46 +1408,32 @@ def busday_offset(dates, offsets, roll='raise', weekmask=None, -------- >>> # First business day in October 2011 (not accounting for holidays) ... np.busday_offset('2011-10', 0, roll='forward') - numpy.datetime64('2011-10-03','D') + numpy.datetime64('2011-10-03') >>> # Last business day in February 2012 (not accounting for holidays) ... np.busday_offset('2012-03', -1, roll='forward') - numpy.datetime64('2012-02-29','D') + numpy.datetime64('2012-02-29') >>> # Third Wednesday in January 2011 ... np.busday_offset('2011-01', 2, roll='forward', weekmask='Wed') - numpy.datetime64('2011-01-19','D') + numpy.datetime64('2011-01-19') >>> # 2012 Mother's Day in Canada and the U.S. ... np.busday_offset('2012-05', 1, roll='forward', weekmask='Sun') - numpy.datetime64('2012-05-13','D') + numpy.datetime64('2012-05-13') >>> # First business day on or after a date ... np.busday_offset('2011-03-20', 0, roll='forward') - numpy.datetime64('2011-03-21','D') + numpy.datetime64('2011-03-21') >>> np.busday_offset('2011-03-22', 0, roll='forward') - numpy.datetime64('2011-03-22','D') + numpy.datetime64('2011-03-22') >>> # First business day after a date ... np.busday_offset('2011-03-20', 1, roll='backward') - numpy.datetime64('2011-03-21','D') + numpy.datetime64('2011-03-21') >>> np.busday_offset('2011-03-22', 1, roll='backward') - numpy.datetime64('2011-03-23','D') + numpy.datetime64('2011-03-23') """ - kwargs = {} - if weekmask is not None: - kwargs['weekmask'] = weekmask - if holidays is not None: - kwargs['holidays'] = holidays - if busdaycal is not None: - kwargs['busdaycal'] = busdaycal - if out is not None: - kwargs['out'] = out - return _multiarray_umath.busday_offset(dates, offsets, roll, **kwargs) - - -def _busday_count_dispatcher(begindates, enddates, weekmask=None, - holidays=None, busdaycal=None, out=None): - return (begindates, enddates, weekmask, holidays, out) + return (dates, offsets, weekmask, holidays, out) -@array_function_dispatch(_busday_count_dispatcher) +@array_function_from_c_func_and_dispatcher(_multiarray_umath.busday_count) def busday_count(begindates, enddates, weekmask=None, holidays=None, busdaycal=None, out=None): """ @@ -1073,32 +1492,21 @@ def busday_count(begindates, enddates, weekmask=None, holidays=None, ... np.busday_count('2011-01', '2011-02') 21 >>> # Number of weekdays in 2011 - ... np.busday_count('2011', '2012') + >>> np.busday_count('2011', '2012') 260 >>> # Number of Saturdays in 2011 ... np.busday_count('2011', '2012', weekmask='Sat') 53 """ - kwargs = {} - if weekmask is not None: - kwargs['weekmask'] = weekmask - if holidays is not None: - kwargs['holidays'] = holidays - if busdaycal is not None: - kwargs['busdaycal'] = busdaycal - if out is not None: - kwargs['out'] = out - return _multiarray_umath.busday_count(begindates, enddates, **kwargs) - - -def _datetime_as_string_dispatcher( - arr, unit=None, timezone=None, casting=None): - return (arr,) + return (begindates, enddates, weekmask, holidays, out) -@array_function_dispatch(_datetime_as_string_dispatcher) -def datetime_as_string(arr, unit=None, timezone='naive', casting='same_kind'): +@array_function_from_c_func_and_dispatcher( + _multiarray_umath.datetime_as_string) +def datetime_as_string(arr, unit=None, timezone=None, casting=None): """ + datetime_as_string(arr, unit=None, timezone='naive', casting='same_kind') + Convert an array of datetimes into an array of strings. Parameters @@ -1122,6 +1530,7 @@ def datetime_as_string(arr, unit=None, timezone='naive', casting='same_kind'): Examples -------- + >>> import pytz >>> d = np.arange('2002-10-27T04:30', 4*60, 60, dtype='M8[m]') >>> d array(['2002-10-27T04:30', '2002-10-27T05:30', '2002-10-27T06:30', @@ -1152,7 +1561,9 @@ def datetime_as_string(arr, unit=None, timezone='naive', casting='same_kind'): 'casting' can be used to specify whether precision can be changed >>> np.datetime_as_string(d, unit='h', casting='safe') + Traceback (most recent call last): + ... TypeError: Cannot create a datetime string as units 'h' from a NumPy datetime with units 'm' according to the rule 'safe' """ - return _multiarray_umath.datetime_as_string(arr, unit, timezone, casting) + return (arr,) diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 265c3636f..8a8efddf3 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -30,6 +30,7 @@ if sys.version_info[0] < 3: from . import overrides from . import umath +from .overrides import set_module from .umath import (multiply, invert, sin, UFUNC_BUFSIZE_DEFAULT, ERR_IGNORE, ERR_WARN, ERR_RAISE, ERR_CALL, ERR_PRINT, ERR_LOG, ERR_DEFAULT, PINF, NAN) @@ -92,6 +93,7 @@ if sys.version_info[0] < 3: __all__.extend(['getbuffer', 'newbuffer']) +@set_module('numpy') class ComplexWarning(RuntimeWarning): """ The warning raised when casting a complex dtype to a real dtype. @@ -158,9 +160,9 @@ def zeros_like(a, dtype=None, order='K', subok=True): >>> y = np.arange(3, dtype=float) >>> y - array([ 0., 1., 2.]) + array([0., 1., 2.]) >>> np.zeros_like(y) - array([ 0., 0., 0.]) + array([0., 0., 0.]) """ res = empty_like(a, dtype=dtype, order=order, subok=subok) @@ -170,6 +172,7 @@ def zeros_like(a, dtype=None, order='K', subok=True): return res +@set_module('numpy') def ones(shape, dtype=None, order='C'): """ Return a new array of given shape and type, filled with ones. @@ -202,19 +205,19 @@ def ones(shape, dtype=None, order='C'): Examples -------- >>> np.ones(5) - array([ 1., 1., 1., 1., 1.]) + array([1., 1., 1., 1., 1.]) >>> np.ones((5,), dtype=int) array([1, 1, 1, 1, 1]) >>> np.ones((2, 1)) - array([[ 1.], - [ 1.]]) + array([[1.], + [1.]]) >>> s = (2,2) >>> np.ones(s) - array([[ 1., 1.], - [ 1., 1.]]) + array([[1., 1.], + [1., 1.]]) """ a = empty(shape, dtype, order) @@ -277,9 +280,9 @@ def ones_like(a, dtype=None, order='K', subok=True): >>> y = np.arange(3, dtype=float) >>> y - array([ 0., 1., 2.]) + array([0., 1., 2.]) >>> np.ones_like(y) - array([ 1., 1., 1.]) + array([1., 1., 1.]) """ res = empty_like(a, dtype=dtype, order=order, subok=subok) @@ -287,6 +290,7 @@ def ones_like(a, dtype=None, order='K', subok=True): return res +@set_module('numpy') def full(shape, fill_value, dtype=None, order='C'): """ Return a new array of given shape and type, filled with `fill_value`. @@ -319,8 +323,8 @@ def full(shape, fill_value, dtype=None, order='C'): Examples -------- >>> np.full((2, 2), np.inf) - array([[ inf, inf], - [ inf, inf]]) + array([[inf, inf], + [inf, inf]]) >>> np.full((2, 2), 10) array([[10, 10], [10, 10]]) @@ -381,13 +385,13 @@ def full_like(a, fill_value, dtype=None, order='K', subok=True): >>> np.full_like(x, 0.1) array([0, 0, 0, 0, 0, 0]) >>> np.full_like(x, 0.1, dtype=np.double) - array([ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) + array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) >>> np.full_like(x, np.nan, dtype=np.double) - array([ nan, nan, nan, nan, nan, nan]) + array([nan, nan, nan, nan, nan, nan]) >>> y = np.arange(6, dtype=np.double) >>> np.full_like(y, 0.1) - array([ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) + array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) """ res = empty_like(a, dtype=dtype, order=order, subok=subok) @@ -462,6 +466,7 @@ def count_nonzero(a, axis=None): return a_bool.sum(axis=axis, dtype=np.intp) +@set_module('numpy') def asarray(a, dtype=None, order=None): """Convert the input to an array. @@ -533,6 +538,7 @@ def asarray(a, dtype=None, order=None): return array(a, dtype, copy=False, order=order) +@set_module('numpy') def asanyarray(a, dtype=None, order=None): """Convert the input to an ndarray, but pass ndarray subclasses through. @@ -585,6 +591,7 @@ def asanyarray(a, dtype=None, order=None): return array(a, dtype, copy=False, order=order, subok=True) +@set_module('numpy') def ascontiguousarray(a, dtype=None): """ Return a contiguous array (ndim >= 1) in memory (C order). @@ -613,8 +620,8 @@ def ascontiguousarray(a, dtype=None): -------- >>> x = np.arange(6).reshape(2,3) >>> np.ascontiguousarray(x, dtype=np.float32) - array([[ 0., 1., 2.], - [ 3., 4., 5.]], dtype=float32) + array([[0., 1., 2.], + [3., 4., 5.]], dtype=float32) >>> x.flags['C_CONTIGUOUS'] True @@ -625,6 +632,7 @@ def ascontiguousarray(a, dtype=None): return array(a, dtype, copy=False, order='C', ndmin=1) +@set_module('numpy') def asfortranarray(a, dtype=None): """ Return an array (ndim >= 1) laid out in Fortran order in memory. @@ -665,6 +673,7 @@ def asfortranarray(a, dtype=None): return array(a, dtype, copy=False, order='F', ndmin=1) +@set_module('numpy') def require(a, dtype=None, requirements=None): """ Return an ndarray of the provided type that satisfies requirements. @@ -736,7 +745,7 @@ def require(a, dtype=None, requirements=None): if not requirements: return asanyarray(a, dtype=dtype) else: - requirements = set(possible_flags[x.upper()] for x in requirements) + requirements = {possible_flags[x.upper()] for x in requirements} if 'E' in requirements: requirements.remove('E') @@ -745,7 +754,7 @@ def require(a, dtype=None, requirements=None): subok = True order = 'A' - if requirements >= set(['C', 'F']): + if requirements >= {'C', 'F'}: raise ValueError('Cannot specify both "C" and "F" order') elif 'F' in requirements: order = 'F' @@ -763,6 +772,7 @@ def require(a, dtype=None, requirements=None): return arr +@set_module('numpy') def isfortran(a): """ Returns True if the array is Fortran contiguous but *not* C contiguous. @@ -792,7 +802,7 @@ def isfortran(a): >>> np.isfortran(a) False - >>> b = np.array([[1, 2, 3], [4, 5, 6]], order='FORTRAN') + >>> b = np.array([[1, 2, 3], [4, 5, 6]], order='F') >>> b array([[1, 2, 3], [4, 5, 6]]) @@ -977,11 +987,11 @@ def correlate(a, v, mode='valid'): Examples -------- >>> np.correlate([1, 2, 3], [0, 1, 0.5]) - array([ 3.5]) + array([3.5]) >>> np.correlate([1, 2, 3], [0, 1, 0.5], "same") - array([ 2. , 3.5, 3. ]) + array([2. , 3.5, 3. ]) >>> np.correlate([1, 2, 3], [0, 1, 0.5], "full") - array([ 0.5, 2. , 3.5, 3. , 0. ]) + array([0.5, 2. , 3.5, 3. , 0. ]) Using complex sequences: @@ -1077,20 +1087,20 @@ def convolve(a, v, mode='full'): before "sliding" the two across one another: >>> np.convolve([1, 2, 3], [0, 1, 0.5]) - array([ 0. , 1. , 2.5, 4. , 1.5]) + array([0. , 1. , 2.5, 4. , 1.5]) Only return the middle values of the convolution. Contains boundary effects, where zeros are taken into account: >>> np.convolve([1,2,3],[0,1,0.5], 'same') - array([ 1. , 2.5, 4. ]) + array([1. , 2.5, 4. ]) The two arrays are of the same length, so there is only one position where they completely overlap: >>> np.convolve([1,2,3],[0,1,0.5], 'valid') - array([ 2.5]) + array([2.5]) """ a, v = array(a, copy=False, ndmin=1), array(v, copy=False, ndmin=1) @@ -1166,11 +1176,11 @@ def outer(a, b, out=None): [-2., -1., 0., 1., 2.]]) >>> im = np.outer(1j*np.linspace(2, -2, 5), np.ones((5,))) >>> im - array([[ 0.+2.j, 0.+2.j, 0.+2.j, 0.+2.j, 0.+2.j], - [ 0.+1.j, 0.+1.j, 0.+1.j, 0.+1.j, 0.+1.j], - [ 0.+0.j, 0.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.-1.j, 0.-1.j, 0.-1.j, 0.-1.j, 0.-1.j], - [ 0.-2.j, 0.-2.j, 0.-2.j, 0.-2.j, 0.-2.j]]) + array([[0.+2.j, 0.+2.j, 0.+2.j, 0.+2.j, 0.+2.j], + [0.+1.j, 0.+1.j, 0.+1.j, 0.+1.j, 0.+1.j], + [0.+0.j, 0.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], + [0.-1.j, 0.-1.j, 0.-1.j, 0.-1.j, 0.-1.j], + [0.-2.j, 0.-2.j, 0.-2.j, 0.-2.j, 0.-2.j]]) >>> grid = rl + im >>> grid array([[-2.+2.j, -1.+2.j, 0.+2.j, 1.+2.j, 2.+2.j], @@ -1183,9 +1193,9 @@ def outer(a, b, out=None): >>> x = np.array(['a', 'b', 'c'], dtype=object) >>> np.outer(x, [1, 2, 3]) - array([[a, aa, aaa], - [b, bb, bbb], - [c, cc, ccc]], dtype=object) + array([['a', 'aa', 'aaa'], + ['b', 'bb', 'bbb'], + ['c', 'cc', 'ccc']], dtype=object) """ a = asarray(a) @@ -1254,11 +1264,11 @@ def tensordot(a, b, axes=2): >>> c.shape (5, 2) >>> c - array([[ 4400., 4730.], - [ 4532., 4874.], - [ 4664., 5018.], - [ 4796., 5162.], - [ 4928., 5306.]]) + array([[4400., 4730.], + [4532., 4874.], + [4664., 5018.], + [4796., 5162.], + [4928., 5306.]]) >>> # A slower but equivalent way of computing the same... >>> d = np.zeros((5,2)) >>> for i in range(5): @@ -1284,40 +1294,40 @@ def tensordot(a, b, axes=2): [3, 4]], [[5, 6], [7, 8]]]) - array([[a, b], - [c, d]], dtype=object) + array([['a', 'b'], + ['c', 'd']], dtype=object) >>> np.tensordot(a, A) # third argument default is 2 for double-contraction - array([abbcccdddd, aaaaabbbbbbcccccccdddddddd], dtype=object) + array(['abbcccdddd', 'aaaaabbbbbbcccccccdddddddd'], dtype=object) >>> np.tensordot(a, A, 1) - array([[[acc, bdd], - [aaacccc, bbbdddd]], - [[aaaaacccccc, bbbbbdddddd], - [aaaaaaacccccccc, bbbbbbbdddddddd]]], dtype=object) + array([[['acc', 'bdd'], + ['aaacccc', 'bbbdddd']], + [['aaaaacccccc', 'bbbbbdddddd'], + ['aaaaaaacccccccc', 'bbbbbbbdddddddd']]], dtype=object) >>> np.tensordot(a, A, 0) # tensor product (result too long to incl.) - array([[[[[a, b], - [c, d]], + array([[[[['a', 'b'], + ['c', 'd']], ... >>> np.tensordot(a, A, (0, 1)) - array([[[abbbbb, cddddd], - [aabbbbbb, ccdddddd]], - [[aaabbbbbbb, cccddddddd], - [aaaabbbbbbbb, ccccdddddddd]]], dtype=object) + array([[['abbbbb', 'cddddd'], + ['aabbbbbb', 'ccdddddd']], + [['aaabbbbbbb', 'cccddddddd'], + ['aaaabbbbbbbb', 'ccccdddddddd']]], dtype=object) >>> np.tensordot(a, A, (2, 1)) - array([[[abb, cdd], - [aaabbbb, cccdddd]], - [[aaaaabbbbbb, cccccdddddd], - [aaaaaaabbbbbbbb, cccccccdddddddd]]], dtype=object) + array([[['abb', 'cdd'], + ['aaabbbb', 'cccdddd']], + [['aaaaabbbbbb', 'cccccdddddd'], + ['aaaaaaabbbbbbbb', 'cccccccdddddddd']]], dtype=object) >>> np.tensordot(a, A, ((0, 1), (0, 1))) - array([abbbcccccddddddd, aabbbbccccccdddddddd], dtype=object) + array(['abbbcccccddddddd', 'aabbbbccccccdddddddd'], dtype=object) >>> np.tensordot(a, A, ((2, 1), (1, 0))) - array([acccbbdddd, aaaaacccccccbbbbbbdddddddd], dtype=object) + array(['acccbbdddd', 'aaaaacccccccbbbbbbdddddddd'], dtype=object) """ try: @@ -1770,7 +1780,7 @@ def cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None): >>> x = [1,2] >>> y = [4,5] >>> np.cross(x, y) - -3 + array(-3) Multiple vector cross-products. Note that the direction of the cross product vector is defined by the `right-hand rule`. @@ -1889,6 +1899,7 @@ def cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None): little_endian = (sys.byteorder == 'little') +@set_module('numpy') def indices(dimensions, dtype=int): """ Return an array representing the indices of a grid. @@ -1960,6 +1971,7 @@ def indices(dimensions, dtype=int): return res +@set_module('numpy') def fromfunction(function, shape, **kwargs): """ Construct an array by executing a function over each coordinate. @@ -2020,6 +2032,7 @@ def _frombuffer(buf, dtype, shape, order): return frombuffer(buf, dtype=dtype).reshape(shape, order=order) +@set_module('numpy') def isscalar(num): """ Returns True if the type of `num` is a scalar type. @@ -2084,10 +2097,10 @@ def isscalar(num): NumPy supports PEP 3141 numbers: >>> from fractions import Fraction - >>> isscalar(Fraction(5, 17)) + >>> np.isscalar(Fraction(5, 17)) True >>> from numbers import Number - >>> isscalar(Number()) + >>> np.isscalar(Number()) True """ @@ -2096,6 +2109,7 @@ def isscalar(num): or isinstance(num, numbers.Number)) +@set_module('numpy') def binary_repr(num, width=None): """ Return the binary representation of the input number as a string. @@ -2206,6 +2220,7 @@ def binary_repr(num, width=None): return '1' * (outwidth - binwidth) + binary +@set_module('numpy') def base_repr(number, base=2, padding=0): """ Return a string representation of a number in the given base system. @@ -2300,6 +2315,7 @@ def _maketup(descr, val): return tuple(res) +@set_module('numpy') def identity(n, dtype=None): """ Return the identity array. @@ -2323,9 +2339,9 @@ def identity(n, dtype=None): Examples -------- >>> np.identity(3) - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) """ from numpy import eye @@ -2471,23 +2487,23 @@ def isclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False): Examples -------- >>> np.isclose([1e10,1e-7], [1.00001e10,1e-8]) - array([True, False]) + array([ True, False]) >>> np.isclose([1e10,1e-8], [1.00001e10,1e-9]) - array([True, True]) + array([ True, True]) >>> np.isclose([1e10,1e-8], [1.0001e10,1e-9]) - array([False, True]) + array([False, True]) >>> np.isclose([1.0, np.nan], [1.0, np.nan]) - array([True, False]) + array([ True, False]) >>> np.isclose([1.0, np.nan], [1.0, np.nan], equal_nan=True) - array([True, True]) + array([ True, True]) >>> np.isclose([1e-8, 1e-7], [0.0, 0.0]) - array([ True, False], dtype=bool) + array([ True, False]) >>> np.isclose([1e-100, 1e-7], [0.0, 0.0], atol=0.0) - array([False, False], dtype=bool) + array([False, False]) >>> np.isclose([1e-10, 1e-10], [1e-20, 0.0]) - array([ True, True], dtype=bool) + array([ True, True]) >>> np.isclose([1e-10, 1e-10], [1e-20, 0.999999e-10], atol=0.0) - array([False, True], dtype=bool) + array([False, True]) """ def within_tol(x, y, atol, rtol): with errstate(invalid='ignore'): @@ -2634,12 +2650,10 @@ _errdict = {"ignore": ERR_IGNORE, "print": ERR_PRINT, "log": ERR_LOG} -_errdict_rev = {} -for key in _errdict.keys(): - _errdict_rev[_errdict[key]] = key -del key +_errdict_rev = {value: key for key, value in _errdict.items()} +@set_module('numpy') def seterr(all=None, divide=None, over=None, under=None, invalid=None): """ Set how floating-point errors are handled. @@ -2696,11 +2710,9 @@ def seterr(all=None, divide=None, over=None, under=None, invalid=None): -------- >>> old_settings = np.seterr(all='ignore') #seterr to known value >>> np.seterr(over='raise') - {'over': 'ignore', 'divide': 'ignore', 'invalid': 'ignore', - 'under': 'ignore'} + {'divide': 'ignore', 'over': 'ignore', 'under': 'ignore', 'invalid': 'ignore'} >>> np.seterr(**old_settings) # reset to default - {'over': 'raise', 'divide': 'ignore', 'invalid': 'ignore', - 'under': 'ignore'} + {'divide': 'ignore', 'over': 'raise', 'under': 'ignore', 'invalid': 'ignore'} >>> np.int16(32000) * np.int16(3) 30464 @@ -2710,11 +2722,11 @@ def seterr(all=None, divide=None, over=None, under=None, invalid=None): File "<stdin>", line 1, in <module> FloatingPointError: overflow encountered in short_scalars + >>> from collections import OrderedDict >>> old_settings = np.seterr(all='print') - >>> np.geterr() - {'over': 'print', 'divide': 'print', 'invalid': 'print', 'under': 'print'} + >>> OrderedDict(np.geterr()) + OrderedDict([('divide', 'print'), ('over', 'print'), ('under', 'print'), ('invalid', 'print')]) >>> np.int16(32000) * np.int16(3) - Warning: overflow encountered in short_scalars 30464 """ @@ -2741,6 +2753,7 @@ def seterr(all=None, divide=None, over=None, under=None, invalid=None): return old +@set_module('numpy') def geterr(): """ Get the current way of handling floating-point errors. @@ -2764,18 +2777,17 @@ def geterr(): Examples -------- - >>> np.geterr() - {'over': 'warn', 'divide': 'warn', 'invalid': 'warn', - 'under': 'ignore'} + >>> from collections import OrderedDict + >>> sorted(np.geterr().items()) + [('divide', 'warn'), ('invalid', 'warn'), ('over', 'warn'), ('under', 'ignore')] >>> np.arange(3.) / np.arange(3.) - array([ NaN, 1., 1.]) + array([nan, 1., 1.]) >>> oldsettings = np.seterr(all='warn', over='raise') - >>> np.geterr() - {'over': 'raise', 'divide': 'warn', 'invalid': 'warn', 'under': 'warn'} + >>> OrderedDict(sorted(np.geterr().items())) + OrderedDict([('divide', 'warn'), ('invalid', 'warn'), ('over', 'raise'), ('under', 'warn')]) >>> np.arange(3.) / np.arange(3.) - __main__:1: RuntimeWarning: invalid value encountered in divide - array([ NaN, 1., 1.]) + array([nan, 1., 1.]) """ maskvalue = umath.geterrobj()[1] @@ -2792,6 +2804,7 @@ def geterr(): return res +@set_module('numpy') def setbufsize(size): """ Set the size of the buffer used in ufuncs. @@ -2816,6 +2829,7 @@ def setbufsize(size): return old +@set_module('numpy') def getbufsize(): """ Return the size of the buffer used in ufuncs. @@ -2829,6 +2843,7 @@ def getbufsize(): return umath.geterrobj()[0] +@set_module('numpy') def seterrcall(func): """ Set the floating-point error callback function or log object. @@ -2879,15 +2894,16 @@ def seterrcall(func): >>> saved_handler = np.seterrcall(err_handler) >>> save_err = np.seterr(all='call') + >>> from collections import OrderedDict >>> np.array([1, 2, 3]) / 0.0 Floating point error (divide by zero), with flag 1 - array([ Inf, Inf, Inf]) + array([inf, inf, inf]) >>> np.seterrcall(saved_handler) <function err_handler at 0x...> - >>> np.seterr(**save_err) - {'over': 'call', 'divide': 'call', 'invalid': 'call', 'under': 'call'} + >>> OrderedDict(sorted(np.seterr(**save_err).items())) + OrderedDict([('divide', 'call'), ('invalid', 'call'), ('over', 'call'), ('under', 'call')]) Log error message: @@ -2901,14 +2917,13 @@ def seterrcall(func): >>> save_err = np.seterr(all='log') >>> np.array([1, 2, 3]) / 0.0 - LOG: Warning: divide by zero encountered in divide - <BLANKLINE> - array([ Inf, Inf, Inf]) + LOG: Warning: divide by zero encountered in true_divide + array([inf, inf, inf]) >>> np.seterrcall(saved_handler) - <__main__.Log object at 0x...> - >>> np.seterr(**save_err) - {'over': 'log', 'divide': 'log', 'invalid': 'log', 'under': 'log'} + <numpy.core.numeric.Log object at 0x...> + >>> OrderedDict(sorted(np.seterr(**save_err).items())) + OrderedDict([('divide', 'log'), ('invalid', 'log'), ('over', 'log'), ('under', 'log')]) """ if func is not None and not isinstance(func, collections_abc.Callable): @@ -2921,6 +2936,7 @@ def seterrcall(func): return old +@set_module('numpy') def geterrcall(): """ Return the current callback function used on floating-point errors. @@ -2956,7 +2972,7 @@ def geterrcall(): >>> oldhandler = np.seterrcall(err_handler) >>> np.array([1, 2, 3]) / 0.0 Floating point error (divide by zero), with flag 1 - array([ Inf, Inf, Inf]) + array([inf, inf, inf]) >>> cur_handler = np.geterrcall() >>> cur_handler is err_handler @@ -2973,6 +2989,7 @@ class _unspecified(object): _Unspecified = _unspecified() +@set_module('numpy') class errstate(object): """ errstate(**kwargs) @@ -3003,15 +3020,14 @@ class errstate(object): Examples -------- + >>> from collections import OrderedDict >>> olderr = np.seterr(all='ignore') # Set error handling to known state. >>> np.arange(3) / 0. - array([ NaN, Inf, Inf]) + array([nan, inf, inf]) >>> with np.errstate(divide='warn'): ... np.arange(3) / 0. - ... - __main__:2: RuntimeWarning: divide by zero encountered in divide - array([ NaN, Inf, Inf]) + array([nan, inf, inf]) >>> np.sqrt(-1) nan @@ -3023,9 +3039,8 @@ class errstate(object): Outside the context the error handling behavior has not changed: - >>> np.geterr() - {'over': 'warn', 'divide': 'warn', 'invalid': 'warn', - 'under': 'ignore'} + >>> OrderedDict(sorted(np.geterr().items())) + OrderedDict([('divide', 'ignore'), ('invalid', 'ignore'), ('over', 'ignore'), ('under', 'ignore')]) """ # Note that we don't want to run the above doctests because they will fail diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 2fb841f7c..5bc37b73a 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -92,6 +92,7 @@ from numpy.core.multiarray import ( datetime_as_string, busday_offset, busday_count, is_busday, busdaycalendar ) +from numpy.core.overrides import set_module # we add more at the bottom __all__ = ['sctypeDict', 'sctypeNA', 'typeDict', 'typeNA', 'sctypes', @@ -162,19 +163,19 @@ def maximum_sctype(t): Examples -------- >>> np.maximum_sctype(int) - <type 'numpy.int64'> + <class 'numpy.int64'> >>> np.maximum_sctype(np.uint8) - <type 'numpy.uint64'> + <class 'numpy.uint64'> >>> np.maximum_sctype(complex) - <type 'numpy.complex192'> + <class 'numpy.complex256'> # may vary >>> np.maximum_sctype(str) - <type 'numpy.string_'> + <class 'numpy.str_'> >>> np.maximum_sctype('i2') - <type 'numpy.int64'> + <class 'numpy.int64'> >>> np.maximum_sctype('f4') - <type 'numpy.float96'> + <class 'numpy.float128'> # may vary """ g = obj2sctype(t) @@ -187,6 +188,8 @@ def maximum_sctype(t): else: return t + +@set_module('numpy') def issctype(rep): """ Determines whether the given object represents a scalar data-type. @@ -231,6 +234,8 @@ def issctype(rep): except Exception: return False + +@set_module('numpy') def obj2sctype(rep, default=None): """ Return the scalar dtype or NumPy equivalent of Python type of an object. @@ -255,19 +260,18 @@ def obj2sctype(rep, default=None): Examples -------- >>> np.obj2sctype(np.int32) - <type 'numpy.int32'> + <class 'numpy.int32'> >>> np.obj2sctype(np.array([1., 2.])) - <type 'numpy.float64'> + <class 'numpy.float64'> >>> np.obj2sctype(np.array([1.j])) - <type 'numpy.complex128'> + <class 'numpy.complex128'> >>> np.obj2sctype(dict) - <type 'numpy.object_'> + <class 'numpy.object_'> >>> np.obj2sctype('string') - <type 'numpy.string_'> >>> np.obj2sctype(1, default=list) - <type 'list'> + <class 'list'> """ # prevent abtract classes being upcast @@ -285,6 +289,7 @@ def obj2sctype(rep, default=None): return res.type +@set_module('numpy') def issubclass_(arg1, arg2): """ Determine if a class is a subclass of a second class. @@ -313,7 +318,7 @@ def issubclass_(arg1, arg2): Examples -------- >>> np.issubclass_(np.int32, int) - True + False # True on Python 2.7 >>> np.issubclass_(np.int32, float) False @@ -323,6 +328,8 @@ def issubclass_(arg1, arg2): except TypeError: return False + +@set_module('numpy') def issubsctype(arg1, arg2): """ Determine if the first argument is a subclass of the second argument. @@ -344,7 +351,7 @@ def issubsctype(arg1, arg2): Examples -------- >>> np.issubsctype('S8', str) - True + False >>> np.issubsctype(np.array([1]), int) True >>> np.issubsctype(np.array([1]), float) @@ -353,6 +360,8 @@ def issubsctype(arg1, arg2): """ return issubclass(obj2sctype(arg1), obj2sctype(arg2)) + +@set_module('numpy') def issubdtype(arg1, arg2): """ Returns True if first argument is a typecode lower/equal in type hierarchy. @@ -446,6 +455,8 @@ def _construct_lookups(): _construct_lookups() + +@set_module('numpy') def sctype2char(sctype): """ Return the string representation of a scalar dtype. @@ -473,9 +484,9 @@ def sctype2char(sctype): Examples -------- - >>> for sctype in [np.int32, float, complex, np.string_, np.ndarray]: + >>> for sctype in [np.int32, np.double, np.complex, np.string_, np.ndarray]: ... print(np.sctype2char(sctype)) - l + l # may vary d D S @@ -586,6 +597,8 @@ def _register_types(): _register_types() + +@set_module('numpy') def find_common_type(array_types, scalar_types): """ Determine common type following standard coercion rules. diff --git a/numpy/core/overrides.py b/numpy/core/overrides.py index 088c15e65..c55174ecd 100644 --- a/numpy/core/overrides.py +++ b/numpy/core/overrides.py @@ -1,79 +1,23 @@ -"""Preliminary implementation of NEP-18 - -TODO: rewrite this in C for performance. -""" +"""Implementation of __array_function__ overrides from NEP-18.""" import collections import functools +import os -from numpy.core._multiarray_umath import ndarray +from numpy.core._multiarray_umath import ( + add_docstring, implement_array_function, _get_implementing_args) from numpy.compat._inspect import getargspec -_NDARRAY_ARRAY_FUNCTION = ndarray.__array_function__ -_NDARRAY_ONLY = [ndarray] - - -def get_overloaded_types_and_args(relevant_args): - """Returns a list of arguments on which to call __array_function__. +ENABLE_ARRAY_FUNCTION = bool( + int(os.environ.get('NUMPY_EXPERIMENTAL_ARRAY_FUNCTION', 0))) - Parameters - ---------- - relevant_args : iterable of array-like - Iterable of array-like arguments to check for __array_function__ - methods. - Returns - ------- - overloaded_types : collection of types - Types of arguments from relevant_args with __array_function__ methods. - overloaded_args : list - Arguments from relevant_args on which to call __array_function__ - methods, in the order in which they should be called. +add_docstring( + implement_array_function, """ - # Runtime is O(num_arguments * num_unique_types) - overloaded_types = [] - overloaded_args = [] - for arg in relevant_args: - arg_type = type(arg) - # We only collect arguments if they have a unique type, which ensures - # reasonable performance even with a long list of possibly overloaded - # arguments. - if (arg_type not in overloaded_types and - hasattr(arg_type, '__array_function__')): - - # Create lists explicitly for the first type (usually the only one - # done) to avoid setting up the iterator for overloaded_args. - if overloaded_types: - overloaded_types.append(arg_type) - # By default, insert argument at the end, but if it is - # subclass of another argument, insert it before that argument. - # This ensures "subclasses before superclasses". - index = len(overloaded_args) - for i, old_arg in enumerate(overloaded_args): - if issubclass(arg_type, type(old_arg)): - index = i - break - overloaded_args.insert(index, arg) - else: - overloaded_types = [arg_type] - overloaded_args = [arg] - - # Short-cut for the common case of only ndarray. - if overloaded_types == _NDARRAY_ONLY: - return overloaded_types, [] - - # Special handling for ndarray.__array_function__ - overloaded_args = [ - arg for arg in overloaded_args - if type(arg).__array_function__ is not _NDARRAY_ARRAY_FUNCTION - ] - - return overloaded_types, overloaded_args - - -def array_function_implementation_or_override( - implementation, public_api, relevant_args, args, kwargs): - """Implement a function with checks for __array_function__ overrides. + Implement a function with checks for __array_function__ overrides. + + All arguments are required, and can only be passed by position. Arguments --------- @@ -88,37 +32,37 @@ def array_function_implementation_or_override( Iterable of arguments to check for __array_function__ methods. args : tuple Arbitrary positional arguments originally passed into ``public_api``. - kwargs : tuple + kwargs : dict Arbitrary keyword arguments originally passed into ``public_api``. Returns ------- - Result from calling `implementation()` or an `__array_function__` + Result from calling ``implementation()`` or an ``__array_function__`` method, as appropriate. Raises ------ TypeError : if no implementation is found. - """ - # Check for __array_function__ methods. - types, overloaded_args = get_overloaded_types_and_args(relevant_args) - if not overloaded_args: - return implementation(*args, **kwargs) + """) - # Call overrides - for overloaded_arg in overloaded_args: - # Use `public_api` instead of `implemenation` so __array_function__ - # implementations can do equality/identity comparisons. - result = overloaded_arg.__array_function__( - public_api, types, args, kwargs) - if result is not NotImplemented: - return result +# exposed for testing purposes; used internally by implement_array_function +add_docstring( + _get_implementing_args, + """ + Collect arguments on which to call __array_function__. + + Parameters + ---------- + relevant_args : iterable of array-like + Iterable of possibly array-like arguments to check for + __array_function__ methods. - func_name = '{}.{}'.format(public_api.__module__, public_api.__name__) - raise TypeError("no implementation found for '{}' on types that implement " - '__array_function__: {}' - .format(func_name, list(map(type, overloaded_args)))) + Returns + ------- + Sequence of arguments with __array_function__ methods, in the order in + which they should be called. + """) ArgSpec = collections.namedtuple('ArgSpec', 'args varargs keywords defaults') @@ -146,19 +90,78 @@ def verify_matching_signatures(implementation, dispatcher): 'default argument values') -def array_function_dispatch(dispatcher, module=None, verify=True): - """Decorator for adding dispatch with the __array_function__ protocol.""" +def set_module(module): + """Decorator for overriding __module__ on a function or class. + + Example usage:: + + @set_module('numpy') + def example(): + pass + + assert example.__module__ == 'numpy' + """ + def decorator(func): + if module is not None: + func.__module__ = module + return func + return decorator + + +def array_function_dispatch(dispatcher, module=None, verify=True, + docs_from_dispatcher=False): + """Decorator for adding dispatch with the __array_function__ protocol. + + See NEP-18 for example usage. + + Parameters + ---------- + dispatcher : callable + Function that when called like ``dispatcher(*args, **kwargs)`` with + arguments from the NumPy function call returns an iterable of + array-like arguments to check for ``__array_function__``. + module : str, optional + __module__ attribute to set on new function, e.g., ``module='numpy'``. + By default, module is copied from the decorated function. + verify : bool, optional + If True, verify the that the signature of the dispatcher and decorated + function signatures match exactly: all required and optional arguments + should appear in order with the same names, but the default values for + all optional arguments should be ``None``. Only disable verification + if the dispatcher's signature needs to deviate for some particular + reason, e.g., because the function has a signature like + ``func(*args, **kwargs)``. + docs_from_dispatcher : bool, optional + If True, copy docs from the dispatcher function onto the dispatched + function, rather than from the implementation. This is useful for + functions defined in C, which otherwise don't have docstrings. + + Returns + ------- + Function suitable for decorating the implementation of a NumPy function. + """ + + if not ENABLE_ARRAY_FUNCTION: + # __array_function__ requires an explicit opt-in for now + def decorator(implementation): + if module is not None: + implementation.__module__ = module + if docs_from_dispatcher: + add_docstring(implementation, dispatcher.__doc__) + return implementation + return decorator + def decorator(implementation): - # TODO: only do this check when the appropriate flag is enabled or for - # a dev install. We want this check for testing but don't want to - # slow down all numpy imports. if verify: verify_matching_signatures(implementation, dispatcher) + if docs_from_dispatcher: + add_docstring(implementation, dispatcher.__doc__) + @functools.wraps(implementation) def public_api(*args, **kwargs): relevant_args = dispatcher(*args, **kwargs) - return array_function_implementation_or_override( + return implement_array_function( implementation, public_api, relevant_args, args, kwargs) if module is not None: @@ -171,3 +174,14 @@ def array_function_dispatch(dispatcher, module=None, verify=True): return public_api return decorator + + +def array_function_from_dispatcher( + implementation, module=None, verify=True, docs_from_dispatcher=True): + """Like array_function_dispatcher, but with function arguments flipped.""" + + def decorator(dispatcher): + return array_function_dispatch( + dispatcher, module, verify=verify, + docs_from_dispatcher=docs_from_dispatcher)(implementation) + return decorator diff --git a/numpy/core/records.py b/numpy/core/records.py index 1b596e4de..4d18c5712 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -7,10 +7,9 @@ Most commonly, ndarrays contain elements of a single type, e.g. floats, integers, bools etc. However, it is possible for elements to be combinations of these using structured types, such as:: - >>> a = np.array([(1, 2.0), (1, 2.0)], dtype=[('x', int), ('y', float)]) + >>> a = np.array([(1, 2.0), (1, 2.0)], dtype=[('x', np.int64), ('y', np.float64)]) >>> a - array([(1, 2.0), (1, 2.0)], - dtype=[('x', '<i4'), ('y', '<f8')]) + array([(1, 2.), (1, 2.)], dtype=[('x', '<i8'), ('y', '<f8')]) Here, each element consists of two fields: x (and int), and y (a float). This is known as a structured array. The different fields are analogous @@ -21,7 +20,7 @@ one would a dictionary:: array([1, 1]) >>> a['y'] - array([ 2., 2.]) + array([2., 2.]) Record arrays allow us to access fields as properties:: @@ -31,7 +30,7 @@ Record arrays allow us to access fields as properties:: array([1, 1]) >>> ar.y - array([ 2., 2.]) + array([2., 2.]) """ from __future__ import division, absolute_import, print_function @@ -39,10 +38,12 @@ from __future__ import division, absolute_import, print_function import sys import os import warnings +from collections import Counter, OrderedDict from . import numeric as sb from . import numerictypes as nt from numpy.compat import isfileobj, bytes, long, unicode, os_fspath +from numpy.core.overrides import set_module from .arrayprint import get_printoptions # All of the functions allow formats to be a dtype @@ -73,15 +74,28 @@ _byteorderconv = {'b':'>', numfmt = nt.typeDict +# taken from OrderedDict recipes in the Python documentation +# https://docs.python.org/3.3/library/collections.html#ordereddict-examples-and-recipes +class _OrderedCounter(Counter, OrderedDict): + """Counter that remembers the order elements are first encountered""" + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, OrderedDict(self)) + + def __reduce__(self): + return self.__class__, (OrderedDict(self),) + + def find_duplicate(list): """Find duplication in a list, return a list of duplicated elements""" - dup = [] - for i in range(len(list)): - if (list[i] in list[i + 1:]): - if (list[i] not in dup): - dup.append(list[i]) - return dup + return [ + item + for item, counts in _OrderedCounter(list).items() + if counts > 1 + ] + +@set_module('numpy') class format_parser(object): """ Class to convert formats, names, titles description to a dtype. @@ -125,10 +139,9 @@ class format_parser(object): Examples -------- - >>> np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'], + >>> np.format_parser(['<f8', '<i4', '<a5'], ['col1', 'col2', 'col3'], ... ['T1', 'T2', 'T3']).dtype - dtype([(('T1', 'col1'), '<f8'), (('T2', 'col2'), '<i4'), - (('T3', 'col3'), '|S5')]) + dtype([(('T1', 'col1'), '<f8'), (('T2', 'col2'), '<i4'), (('T3', 'col3'), 'S5')]) `names` and/or `titles` can be empty lists. If `titles` is an empty list, titles will simply not appear. If `names` is empty, default field names @@ -136,9 +149,9 @@ class format_parser(object): >>> np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'], ... []).dtype - dtype([('col1', '<f8'), ('col2', '<i4'), ('col3', '|S5')]) - >>> np.format_parser(['f8', 'i4', 'a5'], [], []).dtype - dtype([('f0', '<f8'), ('f1', '<i4'), ('f2', '|S5')]) + dtype([('col1', '<f8'), ('col2', '<i4'), ('col3', '<S5')]) + >>> np.format_parser(['<f8', '<i4', '<a5'], [], []).dtype + dtype([('f0', '<f8'), ('f1', '<i4'), ('f2', 'S5')]) """ @@ -287,10 +300,8 @@ class record(nt.void): # pretty-print all fields names = self.dtype.names maxlen = max(len(name) for name in names) - rows = [] fmt = '%% %ds: %%s' % maxlen - for name in names: - rows.append(fmt % (name, getattr(self, name))) + rows = [fmt % (name, getattr(self, name)) for name in names] return "\n".join(rows) # The recarray is almost identical to a standard array (which supports @@ -379,20 +390,19 @@ class recarray(ndarray): -------- Create an array with two fields, ``x`` and ``y``: - >>> x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', float), ('y', int)]) + >>> x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', '<f8'), ('y', '<i8')]) >>> x - array([(1.0, 2), (3.0, 4)], - dtype=[('x', '<f8'), ('y', '<i4')]) + array([(1., 2), (3., 4)], dtype=[('x', '<f8'), ('y', '<i8')]) >>> x['x'] - array([ 1., 3.]) + array([1., 3.]) View the array as a record array: >>> x = x.view(np.recarray) >>> x.x - array([ 1., 3.]) + array([1., 3.]) >>> x.y array([2, 4]) @@ -579,7 +589,7 @@ def fromarrays(arrayList, dtype=None, shape=None, formats=None, >>> x3=np.array([1.1,2,3,4]) >>> r = np.core.records.fromarrays([x1,x2,x3],names='a,b,c') >>> print(r[1]) - (2, 'dd', 2.0) + (2, 'dd', 2.0) # may vary >>> x1[1]=34 >>> r.a array([1, 2, 3, 4]) @@ -658,11 +668,11 @@ def fromrecords(recList, dtype=None, shape=None, formats=None, names=None, >>> r.col1 array([456, 2]) >>> r.col2 - array(['dbe', 'de'], - dtype='|S3') + array(['dbe', 'de'], dtype='<U3') >>> import pickle - >>> print(pickle.loads(pickle.dumps(r))) - [(456, 'dbe', 1.2) (2, 'de', 1.3)] + >>> pickle.loads(pickle.dumps(r)) + rec.array([(456, 'dbe', 1.2), ( 2, 'de', 1.3)], + dtype=[('col1', '<i8'), ('col2', '<U3'), ('col3', '<f8')]) """ if formats is None and dtype is None: # slower @@ -749,7 +759,7 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, >>> a = a.newbyteorder('<') >>> a.tofile(fd) >>> - >>> fd.seek(0) + >>> _ = fd.seek(0) >>> r=np.core.records.fromfile(fd, formats='f8,i4,a5', shape=10, ... byteorder='<') >>> print(r[5]) @@ -782,13 +792,13 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, itemsize = descr.itemsize - shapeprod = sb.array(shape).prod() + shapeprod = sb.array(shape).prod(dtype=nt.intp) shapesize = shapeprod * itemsize if shapesize < 0: shape = list(shape) - shape[shape.index(-1)] = size / -shapesize + shape[shape.index(-1)] = size // -shapesize shape = tuple(shape) - shapeprod = sb.array(shape).prod() + shapeprod = sb.array(shape).prod(dtype=nt.intp) nbytes = shapeprod * itemsize diff --git a/numpy/core/setup.py b/numpy/core/setup.py index a4429cee2..9ccca629e 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -379,8 +379,9 @@ def check_mathlib(config_cmd): def visibility_define(config): """Return the define value to use for NPY_VISIBILITY_HIDDEN (may be empty string).""" - if config.check_compiler_gcc4(): - return '__attribute__((visibility("hidden")))' + hide = '__attribute__((visibility("hidden")))' + if config.check_gcc_function_attribute(hide, 'hideme'): + return hide else: return '' @@ -677,7 +678,7 @@ def configuration(parent_package='',top_path=None): join('src', 'npymath', 'npy_math_complex.c.src'), join('src', 'npymath', 'halffloat.c') ] - + # Must be true for CRT compilers but not MinGW/cygwin. See gh-9977. is_msvc = platform.system() == 'Windows' config.add_installed_library('npymath', @@ -697,7 +698,8 @@ def configuration(parent_package='',top_path=None): ####################################################################### # This library is created for the build but it is not installed - npysort_sources = [join('src', 'npysort', 'quicksort.c.src'), + npysort_sources = [join('src', 'common', 'npy_sort.h.src'), + join('src', 'npysort', 'quicksort.c.src'), join('src', 'npysort', 'mergesort.c.src'), join('src', 'npysort', 'heapsort.c.src'), join('src', 'common', 'npy_partition.h.src'), @@ -730,6 +732,7 @@ def configuration(parent_package='',top_path=None): join('src', 'common', 'cblasfuncs.h'), join('src', 'common', 'lowlevel_strided_loops.h'), join('src', 'common', 'mem_overlap.h'), + join('src', 'common', 'npy_cblas.h'), join('src', 'common', 'npy_config.h'), join('src', 'common', 'npy_ctypes.h'), join('src', 'common', 'npy_extint128.h'), @@ -772,6 +775,7 @@ def configuration(parent_package='',top_path=None): multiarray_deps = [ join('src', 'multiarray', 'arrayobject.h'), join('src', 'multiarray', 'arraytypes.h'), + join('src', 'multiarray', 'arrayfunction_override.h'), join('src', 'multiarray', 'buffer.h'), join('src', 'multiarray', 'calculation.h'), join('src', 'multiarray', 'common.h'), @@ -824,6 +828,7 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'arraytypes.c.src'), join('src', 'multiarray', 'array_assign_scalar.c'), join('src', 'multiarray', 'array_assign_array.c'), + join('src', 'multiarray', 'arrayfunction_override.c'), join('src', 'multiarray', 'buffer.c'), join('src', 'multiarray', 'calculation.c'), join('src', 'multiarray', 'compiled_base.c'), @@ -890,6 +895,8 @@ def configuration(parent_package='',top_path=None): join('src', 'umath', 'simd.inc.src'), join('src', 'umath', 'loops.h.src'), join('src', 'umath', 'loops.c.src'), + join('src', 'umath', 'matmul.h.src'), + join('src', 'umath', 'matmul.c.src'), join('src', 'umath', 'ufunc_object.c'), join('src', 'umath', 'extobj.c'), join('src', 'umath', 'cpuid.c'), @@ -903,14 +910,15 @@ def configuration(parent_package='',top_path=None): join('include', 'numpy', 'npy_math.h'), join('include', 'numpy', 'halffloat.h'), join('src', 'multiarray', 'common.h'), + join('src', 'multiarray', 'number.h'), join('src', 'common', 'templ_common.h.src'), join('src', 'umath', 'simd.inc.src'), join('src', 'umath', 'override.h'), join(codegen_dir, 'generate_ufunc_api.py'), - ] + ] config.add_extension('_multiarray_umath', - sources=multiarray_src + umath_src + + sources=multiarray_src + umath_src + npymath_sources + common_src + [generate_config_h, generate_numpyconfig_h, @@ -920,7 +928,7 @@ def configuration(parent_package='',top_path=None): generate_umath_c, generate_ufunc_api, ], - depends=deps + multiarray_deps + umath_deps + + depends=deps + multiarray_deps + umath_deps + common_deps, libraries=['npymath', 'npysort'], extra_info=extra_info) diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 3edf0824e..f8332c362 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -48,13 +48,13 @@ def atleast_1d(*arys): Examples -------- >>> np.atleast_1d(1.0) - array([ 1.]) + array([1.]) >>> x = np.arange(9.0).reshape(3,3) >>> np.atleast_1d(x) - array([[ 0., 1., 2.], - [ 3., 4., 5.], - [ 6., 7., 8.]]) + array([[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]) >>> np.atleast_1d(x) is x True @@ -106,11 +106,11 @@ def atleast_2d(*arys): Examples -------- >>> np.atleast_2d(3.0) - array([[ 3.]]) + array([[3.]]) >>> x = np.arange(3.0) >>> np.atleast_2d(x) - array([[ 0., 1., 2.]]) + array([[0., 1., 2.]]) >>> np.atleast_2d(x).base is x True @@ -166,7 +166,7 @@ def atleast_3d(*arys): Examples -------- >>> np.atleast_3d(3.0) - array([[[ 3.]]]) + array([[[3.]]]) >>> x = np.arange(3.0) >>> np.atleast_3d(x).shape @@ -179,7 +179,7 @@ def atleast_3d(*arys): True >>> for arr in np.atleast_3d([1, 2], [[1, 2]], [[[1, 2]]]): - ... print(arr, arr.shape) + ... print(arr, arr.shape) # doctest: +SKIP ... [[[1] [2]]] (1, 2, 1) @@ -217,6 +217,11 @@ def _arrays_for_stack_dispatcher(arrays, stacklevel=4): return arrays +def _warn_for_nonsequence(arrays): + if not overrides.ENABLE_ARRAY_FUNCTION: + _arrays_for_stack_dispatcher(arrays, stacklevel=4) + + def _vhstack_dispatcher(tup): return _arrays_for_stack_dispatcher(tup) @@ -274,6 +279,7 @@ def vstack(tup): [4]]) """ + _warn_for_nonsequence(tup) return _nx.concatenate([atleast_2d(_m) for _m in tup], 0) @@ -325,6 +331,7 @@ def hstack(tup): [3, 4]]) """ + _warn_for_nonsequence(tup) arrs = [atleast_1d(_m) for _m in tup] # As a special case, dimension 0 of 1-dimensional arrays is "horizontal" if arrs and arrs[0].ndim == 1: @@ -335,10 +342,11 @@ def hstack(tup): def _stack_dispatcher(arrays, axis=None, out=None): arrays = _arrays_for_stack_dispatcher(arrays, stacklevel=6) - for a in arrays: - yield a if out is not None: - yield out + # optimize for the typical case where only arrays is provided + arrays = list(arrays) + arrays.append(out) + return arrays @array_function_dispatch(_stack_dispatcher) @@ -398,11 +406,12 @@ def stack(arrays, axis=0, out=None): [3, 4]]) """ + _warn_for_nonsequence(arrays) arrays = [asanyarray(arr) for arr in arrays] if not arrays: raise ValueError('need at least one array to stack') - shapes = set(arr.shape for arr in arrays) + shapes = {arr.shape for arr in arrays} if len(shapes) != 1: raise ValueError('all input arrays must have the same shape') @@ -752,11 +761,11 @@ def block(arrays): ... [A, np.zeros((2, 3))], ... [np.ones((3, 2)), B ] ... ]) - array([[ 2., 0., 0., 0., 0.], - [ 0., 2., 0., 0., 0.], - [ 1., 1., 3., 0., 0.], - [ 1., 1., 0., 3., 0.], - [ 1., 1., 0., 0., 3.]]) + array([[2., 0., 0., 0., 0.], + [0., 2., 0., 0., 0.], + [1., 1., 3., 0., 0.], + [1., 1., 0., 3., 0.], + [1., 1., 0., 0., 3.]]) With a list of depth 1, `block` can be used as `hstack` @@ -766,7 +775,7 @@ def block(arrays): >>> a = np.array([1, 2, 3]) >>> b = np.array([2, 3, 4]) >>> np.block([a, b, 10]) # hstack([a, b, 10]) - array([1, 2, 3, 2, 3, 4, 10]) + array([ 1, 2, 3, 2, 3, 4, 10]) >>> A = np.ones((2, 2), int) >>> B = 2 * A diff --git a/numpy/core/src/common/cblasfuncs.c b/numpy/core/src/common/cblasfuncs.c index 6460c5db1..39572fed4 100644 --- a/numpy/core/src/common/cblasfuncs.c +++ b/numpy/core/src/common/cblasfuncs.c @@ -182,12 +182,13 @@ _select_matrix_shape(PyArrayObject *array) * This also makes sure that the data segment is aligned with * an itemsize address as well by returning one if not true. */ -static int +NPY_NO_EXPORT int _bad_strides(PyArrayObject *ap) { int itemsize = PyArray_ITEMSIZE(ap); int i, N=PyArray_NDIM(ap); npy_intp *strides = PyArray_STRIDES(ap); + npy_intp *dims = PyArray_DIMS(ap); if (((npy_intp)(PyArray_DATA(ap)) % itemsize) != 0) { return 1; @@ -196,12 +197,14 @@ _bad_strides(PyArrayObject *ap) if ((strides[i] < 0) || (strides[i] % itemsize) != 0) { return 1; } + if ((strides[i] == 0 && dims[i] > 1)) { + return 1; + } } return 0; } - /* * dot(a,b) * Returns the dot product of a and b for arrays of floating point types. diff --git a/numpy/core/src/common/get_attr_string.h b/numpy/core/src/common/get_attr_string.h index bec87c5ed..d458d9550 100644 --- a/numpy/core/src/common/get_attr_string.h +++ b/numpy/core/src/common/get_attr_string.h @@ -103,7 +103,6 @@ PyArray_LookupSpecial(PyObject *obj, char *name) if (_is_basic_python_type(tp)) { return NULL; } - return maybe_get_attr((PyObject *)tp, name); } diff --git a/numpy/core/src/common/npy_sort.h b/numpy/core/src/common/npy_sort.h deleted file mode 100644 index 8c6f05623..000000000 --- a/numpy/core/src/common/npy_sort.h +++ /dev/null @@ -1,204 +0,0 @@ -#ifndef __NPY_SORT_H__ -#define __NPY_SORT_H__ - -/* Python include is for future object sorts */ -#include <Python.h> -#include <numpy/npy_common.h> -#include <numpy/ndarraytypes.h> - -#define NPY_ENOMEM 1 -#define NPY_ECOMP 2 - -static NPY_INLINE int npy_get_msb(npy_uintp unum) -{ - int depth_limit = 0; - while (unum >>= 1) { - depth_limit++; - } - return depth_limit; -} - -int quicksort_bool(void *vec, npy_intp cnt, void *null); -int heapsort_bool(void *vec, npy_intp cnt, void *null); -int mergesort_bool(void *vec, npy_intp cnt, void *null); -int aquicksort_bool(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_bool(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_bool(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_byte(void *vec, npy_intp cnt, void *null); -int heapsort_byte(void *vec, npy_intp cnt, void *null); -int mergesort_byte(void *vec, npy_intp cnt, void *null); -int aquicksort_byte(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_byte(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_byte(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_ubyte(void *vec, npy_intp cnt, void *null); -int heapsort_ubyte(void *vec, npy_intp cnt, void *null); -int mergesort_ubyte(void *vec, npy_intp cnt, void *null); -int aquicksort_ubyte(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_ubyte(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_ubyte(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_short(void *vec, npy_intp cnt, void *null); -int heapsort_short(void *vec, npy_intp cnt, void *null); -int mergesort_short(void *vec, npy_intp cnt, void *null); -int aquicksort_short(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_short(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_short(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_ushort(void *vec, npy_intp cnt, void *null); -int heapsort_ushort(void *vec, npy_intp cnt, void *null); -int mergesort_ushort(void *vec, npy_intp cnt, void *null); -int aquicksort_ushort(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_ushort(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_ushort(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_int(void *vec, npy_intp cnt, void *null); -int heapsort_int(void *vec, npy_intp cnt, void *null); -int mergesort_int(void *vec, npy_intp cnt, void *null); -int aquicksort_int(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_int(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_int(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_uint(void *vec, npy_intp cnt, void *null); -int heapsort_uint(void *vec, npy_intp cnt, void *null); -int mergesort_uint(void *vec, npy_intp cnt, void *null); -int aquicksort_uint(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_uint(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_uint(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_long(void *vec, npy_intp cnt, void *null); -int heapsort_long(void *vec, npy_intp cnt, void *null); -int mergesort_long(void *vec, npy_intp cnt, void *null); -int aquicksort_long(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_long(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_long(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_ulong(void *vec, npy_intp cnt, void *null); -int heapsort_ulong(void *vec, npy_intp cnt, void *null); -int mergesort_ulong(void *vec, npy_intp cnt, void *null); -int aquicksort_ulong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_ulong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_ulong(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_longlong(void *vec, npy_intp cnt, void *null); -int heapsort_longlong(void *vec, npy_intp cnt, void *null); -int mergesort_longlong(void *vec, npy_intp cnt, void *null); -int aquicksort_longlong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_longlong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_longlong(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_ulonglong(void *vec, npy_intp cnt, void *null); -int heapsort_ulonglong(void *vec, npy_intp cnt, void *null); -int mergesort_ulonglong(void *vec, npy_intp cnt, void *null); -int aquicksort_ulonglong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_ulonglong(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_ulonglong(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_half(void *vec, npy_intp cnt, void *null); -int heapsort_half(void *vec, npy_intp cnt, void *null); -int mergesort_half(void *vec, npy_intp cnt, void *null); -int aquicksort_half(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_half(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_half(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_float(void *vec, npy_intp cnt, void *null); -int heapsort_float(void *vec, npy_intp cnt, void *null); -int mergesort_float(void *vec, npy_intp cnt, void *null); -int aquicksort_float(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_float(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_float(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_double(void *vec, npy_intp cnt, void *null); -int heapsort_double(void *vec, npy_intp cnt, void *null); -int mergesort_double(void *vec, npy_intp cnt, void *null); -int aquicksort_double(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_double(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_double(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_longdouble(void *vec, npy_intp cnt, void *null); -int heapsort_longdouble(void *vec, npy_intp cnt, void *null); -int mergesort_longdouble(void *vec, npy_intp cnt, void *null); -int aquicksort_longdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_longdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_longdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_cfloat(void *vec, npy_intp cnt, void *null); -int heapsort_cfloat(void *vec, npy_intp cnt, void *null); -int mergesort_cfloat(void *vec, npy_intp cnt, void *null); -int aquicksort_cfloat(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_cfloat(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_cfloat(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_cdouble(void *vec, npy_intp cnt, void *null); -int heapsort_cdouble(void *vec, npy_intp cnt, void *null); -int mergesort_cdouble(void *vec, npy_intp cnt, void *null); -int aquicksort_cdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_cdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_cdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_clongdouble(void *vec, npy_intp cnt, void *null); -int heapsort_clongdouble(void *vec, npy_intp cnt, void *null); -int mergesort_clongdouble(void *vec, npy_intp cnt, void *null); -int aquicksort_clongdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_clongdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_clongdouble(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_string(void *vec, npy_intp cnt, void *arr); -int heapsort_string(void *vec, npy_intp cnt, void *arr); -int mergesort_string(void *vec, npy_intp cnt, void *arr); -int aquicksort_string(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int aheapsort_string(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int amergesort_string(void *vec, npy_intp *ind, npy_intp cnt, void *arr); - - -int quicksort_unicode(void *vec, npy_intp cnt, void *arr); -int heapsort_unicode(void *vec, npy_intp cnt, void *arr); -int mergesort_unicode(void *vec, npy_intp cnt, void *arr); -int aquicksort_unicode(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int aheapsort_unicode(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int amergesort_unicode(void *vec, npy_intp *ind, npy_intp cnt, void *arr); - - -int quicksort_datetime(void *vec, npy_intp cnt, void *null); -int heapsort_datetime(void *vec, npy_intp cnt, void *null); -int mergesort_datetime(void *vec, npy_intp cnt, void *null); -int aquicksort_datetime(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_datetime(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_datetime(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int quicksort_timedelta(void *vec, npy_intp cnt, void *null); -int heapsort_timedelta(void *vec, npy_intp cnt, void *null); -int mergesort_timedelta(void *vec, npy_intp cnt, void *null); -int aquicksort_timedelta(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int aheapsort_timedelta(void *vec, npy_intp *ind, npy_intp cnt, void *null); -int amergesort_timedelta(void *vec, npy_intp *ind, npy_intp cnt, void *null); - - -int npy_quicksort(void *vec, npy_intp cnt, void *arr); -int npy_heapsort(void *vec, npy_intp cnt, void *arr); -int npy_mergesort(void *vec, npy_intp cnt, void *arr); -int npy_aquicksort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int npy_aheapsort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); -int npy_amergesort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); - -#endif diff --git a/numpy/core/src/common/npy_sort.h.src b/numpy/core/src/common/npy_sort.h.src new file mode 100644 index 000000000..c31a82764 --- /dev/null +++ b/numpy/core/src/common/npy_sort.h.src @@ -0,0 +1,83 @@ +#ifndef __NPY_SORT_H__ +#define __NPY_SORT_H__ + +/* Python include is for future object sorts */ +#include <Python.h> +#include <numpy/npy_common.h> +#include <numpy/ndarraytypes.h> + +#define NPY_ENOMEM 1 +#define NPY_ECOMP 2 + +static NPY_INLINE int npy_get_msb(npy_uintp unum) +{ + int depth_limit = 0; + while (unum >>= 1) { + depth_limit++; + } + return depth_limit; +} + + +/* + ***************************************************************************** + ** NUMERIC SORTS ** + ***************************************************************************** + */ + + +/**begin repeat + * + * #suff = bool, byte, ubyte, short, ushort, int, uint, long, ulong, + * longlong, ulonglong, half, float, double, longdouble, + * cfloat, cdouble, clongdouble, datetime, timedelta# + */ + +int quicksort_@suff@(void *vec, npy_intp cnt, void *null); +int heapsort_@suff@(void *vec, npy_intp cnt, void *null); +int mergesort_@suff@(void *vec, npy_intp cnt, void *null); +int aquicksort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *null); +int aheapsort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *null); +int amergesort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *null); + +/**end repeat**/ + + + +/* + ***************************************************************************** + ** STRING SORTS ** + ***************************************************************************** + */ + + +/**begin repeat + * + * #suff = string, unicode# + */ + +int quicksort_@suff@(void *vec, npy_intp cnt, void *arr); +int heapsort_@suff@(void *vec, npy_intp cnt, void *arr); +int mergesort_@suff@(void *vec, npy_intp cnt, void *arr); +int aquicksort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *arr); +int aheapsort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *arr); +int amergesort_@suff@(void *vec, npy_intp *ind, npy_intp cnt, void *arr); + +/**end repeat**/ + + +/* + ***************************************************************************** + ** GENERIC SORT ** + ***************************************************************************** + */ + + +int npy_quicksort(void *vec, npy_intp cnt, void *arr); +int npy_heapsort(void *vec, npy_intp cnt, void *arr); +int npy_mergesort(void *vec, npy_intp cnt, void *arr); +int npy_aquicksort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); +int npy_aheapsort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); +int npy_amergesort(void *vec, npy_intp *ind, npy_intp cnt, void *arr); + +#endif diff --git a/numpy/core/src/common/ufunc_override.c b/numpy/core/src/common/ufunc_override.c index 33b54c665..89f08a9cb 100644 --- a/numpy/core/src/common/ufunc_override.c +++ b/numpy/core/src/common/ufunc_override.c @@ -1,10 +1,9 @@ #define NPY_NO_DEPRECATED_API NPY_API_VERSION -#define NO_IMPORT_ARRAY +#define _MULTIARRAYMODULE #include "npy_pycompat.h" #include "get_attr_string.h" #include "npy_import.h" - #include "ufunc_override.h" /* @@ -12,45 +11,39 @@ * is not the default, i.e., the object is not an ndarray, and its * __array_ufunc__ is not the same as that of ndarray. * - * Returns a new reference, the value of type(obj).__array_ufunc__ - * - * If the __array_ufunc__ matches that of ndarray, or does not exist, return - * NULL. - * - * Note that since this module is used with both multiarray and umath, we do - * not have access to PyArray_Type and therewith neither to PyArray_CheckExact - * nor to the default __array_ufunc__ method, so instead we import locally. - * TODO: Can this really not be done more smartly? + * Returns a new reference, the value of type(obj).__array_ufunc__ if it + * exists and is different from that of ndarray, and NULL otherwise. */ NPY_NO_EXPORT PyObject * -get_non_default_array_ufunc(PyObject *obj) +PyUFuncOverride_GetNonDefaultArrayUfunc(PyObject *obj) { - static PyObject *ndarray = NULL; static PyObject *ndarray_array_ufunc = NULL; PyObject *cls_array_ufunc; - /* on first entry, import and cache ndarray and its __array_ufunc__ */ - if (ndarray == NULL) { - npy_cache_import("numpy.core.multiarray", "ndarray", &ndarray); - ndarray_array_ufunc = PyObject_GetAttrString(ndarray, + /* On first entry, cache ndarray's __array_ufunc__ */ + if (ndarray_array_ufunc == NULL) { + ndarray_array_ufunc = PyObject_GetAttrString((PyObject *)&PyArray_Type, "__array_ufunc__"); } /* Fast return for ndarray */ - if ((PyObject *)Py_TYPE(obj) == ndarray) { + if (PyArray_CheckExact(obj)) { return NULL; } - /* does the class define __array_ufunc__? */ + /* + * Does the class define __array_ufunc__? (Note that LookupSpecial has fast + * return for basic python types, so no need to worry about those here) + */ cls_array_ufunc = PyArray_LookupSpecial(obj, "__array_ufunc__"); if (cls_array_ufunc == NULL) { return NULL; } - /* is it different from ndarray.__array_ufunc__? */ - if (cls_array_ufunc != ndarray_array_ufunc) { - return cls_array_ufunc; + /* Ignore if the same as ndarray.__array_ufunc__ */ + if (cls_array_ufunc == ndarray_array_ufunc) { + Py_DECREF(cls_array_ufunc); + return NULL; } - Py_DECREF(cls_array_ufunc); - return NULL; + return cls_array_ufunc; } /* @@ -62,9 +55,9 @@ get_non_default_array_ufunc(PyObject *obj) */ NPY_NO_EXPORT int -has_non_default_array_ufunc(PyObject * obj) +PyUFunc_HasOverride(PyObject * obj) { - PyObject *method = get_non_default_array_ufunc(obj); + PyObject *method = PyUFuncOverride_GetNonDefaultArrayUfunc(obj); if (method) { Py_DECREF(method); return 1; @@ -78,164 +71,51 @@ has_non_default_array_ufunc(PyObject * obj) * Get possible out argument from kwds, and returns the number of outputs * contained within it: if a tuple, the number of elements in it, 1 otherwise. * The out argument itself is returned in out_kwd_obj, and the outputs - * in the out_obj array (all as borrowed references). + * in the out_obj array (as borrowed references). * - * Returns -1 if kwds is not a dict, 0 if no outputs found. + * Returns 0 if no outputs found, -1 if kwds is not a dict (with an error set). */ -static int -get_out_objects(PyObject *kwds, PyObject **out_kwd_obj, PyObject ***out_objs) +NPY_NO_EXPORT int +PyUFuncOverride_GetOutObjects(PyObject *kwds, PyObject **out_kwd_obj, PyObject ***out_objs) { if (kwds == NULL) { + Py_INCREF(Py_None); + *out_kwd_obj = Py_None; return 0; } if (!PyDict_CheckExact(kwds)) { PyErr_SetString(PyExc_TypeError, - "Internal Numpy error: call to PyUFunc_WithOverride " + "Internal Numpy error: call to PyUFuncOverride_GetOutObjects " "with non-dict kwds"); + *out_kwd_obj = NULL; return -1; } /* borrowed reference */ *out_kwd_obj = PyDict_GetItemString(kwds, "out"); if (*out_kwd_obj == NULL) { + Py_INCREF(Py_None); + *out_kwd_obj = Py_None; return 0; } if (PyTuple_CheckExact(*out_kwd_obj)) { - *out_objs = PySequence_Fast_ITEMS(*out_kwd_obj); - return PySequence_Fast_GET_SIZE(*out_kwd_obj); - } - else { - *out_objs = out_kwd_obj; - return 1; - } -} - -/* - * For each positional argument and each argument in a possible "out" - * keyword, look for overrides of the standard ufunc behaviour, i.e., - * non-default __array_ufunc__ methods. - * - * Returns the number of overrides, setting corresponding objects - * in PyObject array ``with_override`` and the corresponding - * __array_ufunc__ methods in ``methods`` (both using new references). - * - * Only the first override for a given class is returned. - * - * returns -1 on failure. - */ -NPY_NO_EXPORT int -PyUFunc_WithOverride(PyObject *args, PyObject *kwds, - PyObject **with_override, PyObject **methods) -{ - int i; - int num_override_args = 0; - int narg, nout = 0; - PyObject *out_kwd_obj; - PyObject **arg_objs, **out_objs; - - narg = PyTuple_Size(args); - if (narg < 0) { - return -1; - } - arg_objs = PySequence_Fast_ITEMS(args); - - nout = get_out_objects(kwds, &out_kwd_obj, &out_objs); - if (nout < 0) { - return -1; - } - - for (i = 0; i < narg + nout; ++i) { - PyObject *obj; - int j; - int new_class = 1; - - if (i < narg) { - obj = arg_objs[i]; - } - else { - obj = out_objs[i - narg]; - } /* - * Have we seen this class before? If so, ignore. + * The C-API recommends calling PySequence_Fast before any of the other + * PySequence_Fast* functions. This is required for PyPy */ - for (j = 0; j < num_override_args; j++) { - new_class = (Py_TYPE(obj) != Py_TYPE(with_override[j])); - if (!new_class) { - break; - } - } - if (new_class) { - /* - * Now see if the object provides an __array_ufunc__. However, we should - * ignore the base ndarray.__ufunc__, so we skip any ndarray as well as - * any ndarray subclass instances that did not override __array_ufunc__. - */ - PyObject *method = get_non_default_array_ufunc(obj); - if (method == NULL) { - continue; - } - if (method == Py_None) { - PyErr_Format(PyExc_TypeError, - "operand '%.200s' does not support ufuncs " - "(__array_ufunc__=None)", - obj->ob_type->tp_name); - Py_DECREF(method); - goto fail; - } - Py_INCREF(obj); - with_override[num_override_args] = obj; - methods[num_override_args] = method; - ++num_override_args; + PyObject *seq; + seq = PySequence_Fast(*out_kwd_obj, + "Could not convert object to sequence"); + if (seq == NULL) { + *out_kwd_obj = NULL; + return -1; } + *out_objs = PySequence_Fast_ITEMS(seq); + *out_kwd_obj = seq; + return PySequence_Fast_GET_SIZE(seq); } - return num_override_args; - -fail: - for (i = 0; i < num_override_args; i++) { - Py_DECREF(with_override[i]); - Py_DECREF(methods[i]); - } - return -1; -} - -/* - * Check whether any of a set of input and output args have a non-default - * __array_ufunc__ method. Return 1 if so, 0 if not. - * - * This function primarily exists to help ndarray.__array_ufunc__ determine - * whether it can support a ufunc (which is the case only if none of the - * operands have an override). Thus, unlike in PyUFunc_CheckOverride, the - * actual overrides are not needed and one can stop looking once one is found. - * - * TODO: move this function and has_non_default_array_ufunc closer to ndarray. - */ -NPY_NO_EXPORT int -PyUFunc_HasOverride(PyObject *args, PyObject *kwds) -{ - int i; - int nin, nout; - PyObject *out_kwd_obj; - PyObject **in_objs, **out_objs; - - /* check inputs */ - nin = PyTuple_Size(args); - if (nin < 0) { - return -1; - } - in_objs = PySequence_Fast_ITEMS(args); - for (i = 0; i < nin; ++i) { - if (has_non_default_array_ufunc(in_objs[i])) { - return 1; - } - } - /* check outputs, if any */ - nout = get_out_objects(kwds, &out_kwd_obj, &out_objs); - if (nout < 0) { - return -1; - } - for (i = 0; i < nout; i++) { - if (has_non_default_array_ufunc(out_objs[i])) { - return 1; - } + else { + Py_INCREF(*out_kwd_obj); + *out_objs = out_kwd_obj; + return 1; } - return 0; } diff --git a/numpy/core/src/common/ufunc_override.h b/numpy/core/src/common/ufunc_override.h index 5b269d270..bf86865c9 100644 --- a/numpy/core/src/common/ufunc_override.h +++ b/numpy/core/src/common/ufunc_override.h @@ -8,18 +8,11 @@ * is not the default, i.e., the object is not an ndarray, and its * __array_ufunc__ is not the same as that of ndarray. * - * Returns a new reference, the value of type(obj).__array_ufunc__ - * - * If the __array_ufunc__ matches that of ndarray, or does not exist, return - * NULL. - * - * Note that since this module is used with both multiarray and umath, we do - * not have access to PyArray_Type and therewith neither to PyArray_CheckExact - * nor to the default __array_ufunc__ method, so instead we import locally. - * TODO: Can this really not be done more smartly? + * Returns a new reference, the value of type(obj).__array_ufunc__ if it + * exists and is different from that of ndarray, and NULL otherwise. */ NPY_NO_EXPORT PyObject * -get_non_default_array_ufunc(PyObject *obj); +PyUFuncOverride_GetNonDefaultArrayUfunc(PyObject *obj); /* * Check whether an object has __array_ufunc__ defined on its class and it @@ -29,18 +22,16 @@ get_non_default_array_ufunc(PyObject *obj); * Returns 1 if this is the case, 0 if not. */ NPY_NO_EXPORT int -has_non_default_array_ufunc(PyObject * obj); +PyUFunc_HasOverride(PyObject *obj); /* - * Check whether a set of input and output args have a non-default - * `__array_ufunc__` method. Returns the number of overrides, setting - * corresponding objects in PyObject array with_override (if not NULL). - * returns -1 on failure. + * Get possible out argument from kwds, and returns the number of outputs + * contained within it: if a tuple, the number of elements in it, 1 otherwise. + * The out argument itself is returned in out_kwd_obj, and the outputs + * in the out_obj array (as borrowed references). + * + * Returns 0 if no outputs found, -1 if kwds is not a dict (with an error set). */ NPY_NO_EXPORT int -PyUFunc_WithOverride(PyObject *args, PyObject *kwds, - PyObject **with_override, PyObject **methods); - -NPY_NO_EXPORT int -PyUFunc_HasOverride(PyObject *args, PyObject *kwds); +PyUFuncOverride_GetOutObjects(PyObject *kwds, PyObject **out_kwd_obj, PyObject ***out_objs); #endif diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index 6c4d49bd1..2a8275572 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -11,6 +11,13 @@ #include "npy_extint128.h" #include "common.h" + +#if defined(MS_WIN32) || defined(__CYGWIN__) +#define EXPORT(x) __declspec(dllexport) x +#else +#define EXPORT(x) x +#endif + #define ARRAY_SIZE(a) (sizeof(a)/sizeof(a[0])) /* test PyArray_IsPythonScalar, before including private py3 compat header */ @@ -31,6 +38,12 @@ IsPythonScalar(PyObject * dummy, PyObject *args) #include "npy_pycompat.h" +/** Function to test calling via ctypes */ +EXPORT(void*) forward_pointer(void *x) +{ + return x; +} + /* * TODO: * - Handle mode @@ -1855,6 +1868,16 @@ printf_float_g(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) return PrintFloat_Printf_g(obj, precision); } +static PyObject * +getset_numericops(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) +{ + PyObject * ops = PyArray_GetNumericOps(); + if (ops == NULL) { + return NULL; + } + return PyLong_FromLong(PyArray_SetNumericOps(ops)); +} + static PyMethodDef Multiarray_TestsMethods[] = { {"IsPythonScalar", IsPythonScalar, @@ -1963,6 +1986,9 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"get_fpu_mode", get_fpu_mode, METH_VARARGS, get_fpu_mode_doc}, + {"getset_numericops", + getset_numericops, + METH_NOARGS, NULL}, /**begin repeat * #name = cabs, carg# */ @@ -2040,3 +2066,9 @@ init_multiarray_tests(void) } return RETVAL; } + +NPY_NO_EXPORT int +test_not_exported(void) +{ + return 1; +} diff --git a/numpy/core/src/multiarray/arrayfunction_override.c b/numpy/core/src/multiarray/arrayfunction_override.c new file mode 100644 index 000000000..e62b32ab2 --- /dev/null +++ b/numpy/core/src/multiarray/arrayfunction_override.c @@ -0,0 +1,376 @@ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE + +#include "npy_pycompat.h" +#include "get_attr_string.h" +#include "npy_import.h" +#include "multiarraymodule.h" + + +/* Return the ndarray.__array_function__ method. */ +static PyObject * +get_ndarray_array_function(void) +{ + PyObject* method = PyObject_GetAttrString((PyObject *)&PyArray_Type, + "__array_function__"); + assert(method != NULL); + return method; +} + + +/* + * Get an object's __array_function__ method in the fastest way possible. + * Never raises an exception. Returns NULL if the method doesn't exist. + */ +static PyObject * +get_array_function(PyObject *obj) +{ + static PyObject *ndarray_array_function = NULL; + + if (ndarray_array_function == NULL) { + ndarray_array_function = get_ndarray_array_function(); + } + + /* Fast return for ndarray */ + if (PyArray_CheckExact(obj)) { + Py_INCREF(ndarray_array_function); + return ndarray_array_function; + } + + return PyArray_LookupSpecial(obj, "__array_function__"); +} + + +/* + * Like list.insert(), but for C arrays of PyObject*. Skips error checking. + */ +static void +pyobject_array_insert(PyObject **array, int length, int index, PyObject *item) +{ + int j; + + for (j = length; j > index; j--) { + array[j] = array[j - 1]; + } + array[index] = item; +} + + +/* + * Collects arguments with __array_function__ and their corresponding methods + * in the order in which they should be tried (i.e., skipping redundant types). + * `relevant_args` is expected to have been produced by PySequence_Fast. + * Returns the number of arguments, or -1 on failure. + */ +static int +get_implementing_args_and_methods(PyObject *relevant_args, + PyObject **implementing_args, + PyObject **methods) +{ + int num_implementing_args = 0; + Py_ssize_t i; + int j; + + PyObject **items = PySequence_Fast_ITEMS(relevant_args); + Py_ssize_t length = PySequence_Fast_GET_SIZE(relevant_args); + + for (i = 0; i < length; i++) { + int new_class = 1; + PyObject *argument = items[i]; + + /* Have we seen this type before? */ + for (j = 0; j < num_implementing_args; j++) { + if (Py_TYPE(argument) == Py_TYPE(implementing_args[j])) { + new_class = 0; + break; + } + } + if (new_class) { + PyObject *method = get_array_function(argument); + + if (method != NULL) { + int arg_index; + + if (num_implementing_args >= NPY_MAXARGS) { + PyErr_Format( + PyExc_TypeError, + "maximum number (%d) of distinct argument types " \ + "implementing __array_function__ exceeded", + NPY_MAXARGS); + Py_DECREF(method); + goto fail; + } + + /* "subclasses before superclasses, otherwise left to right" */ + arg_index = num_implementing_args; + for (j = 0; j < num_implementing_args; j++) { + PyObject *other_type; + other_type = (PyObject *)Py_TYPE(implementing_args[j]); + if (PyObject_IsInstance(argument, other_type)) { + arg_index = j; + break; + } + } + Py_INCREF(argument); + pyobject_array_insert(implementing_args, num_implementing_args, + arg_index, argument); + pyobject_array_insert(methods, num_implementing_args, + arg_index, method); + ++num_implementing_args; + } + } + } + return num_implementing_args; + +fail: + for (j = 0; j < num_implementing_args; j++) { + Py_DECREF(implementing_args[j]); + Py_DECREF(methods[j]); + } + return -1; +} + + +/* + * Is this object ndarray.__array_function__? + */ +static int +is_default_array_function(PyObject *obj) +{ + static PyObject *ndarray_array_function = NULL; + + if (ndarray_array_function == NULL) { + ndarray_array_function = get_ndarray_array_function(); + } + return obj == ndarray_array_function; +} + + +/* + * Core implementation of ndarray.__array_function__. This is exposed + * separately so we can avoid the overhead of a Python method call from + * within `implement_array_function`. + */ +NPY_NO_EXPORT PyObject * +array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, + PyObject *kwargs) +{ + Py_ssize_t j; + PyObject *implementation, *result; + + PyObject **items = PySequence_Fast_ITEMS(types); + Py_ssize_t length = PySequence_Fast_GET_SIZE(types); + + for (j = 0; j < length; j++) { + int is_subclass = PyObject_IsSubclass( + items[j], (PyObject *)&PyArray_Type); + if (is_subclass == -1) { + return NULL; + } + if (!is_subclass) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + } + + implementation = PyObject_GetAttr(func, npy_ma_str_wrapped); + if (implementation == NULL) { + return NULL; + } + result = PyObject_Call(implementation, args, kwargs); + Py_DECREF(implementation); + return result; +} + + +/* + * Calls __array_function__ on the provided argument, with a fast-path for + * ndarray. + */ +static PyObject * +call_array_function(PyObject* argument, PyObject* method, + PyObject* public_api, PyObject* types, + PyObject* args, PyObject* kwargs) +{ + if (is_default_array_function(method)) { + return array_function_method_impl(public_api, types, args, kwargs); + } + else { + return PyObject_CallFunctionObjArgs( + method, argument, public_api, types, args, kwargs, NULL); + } +} + + +/* + * Implements the __array_function__ protocol for a function, as described in + * in NEP-18. See numpy.core.overrides for a full docstring. + */ +NPY_NO_EXPORT PyObject * +array_implement_array_function( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args) +{ + PyObject *implementation, *public_api, *relevant_args, *args, *kwargs; + + PyObject *types = NULL; + PyObject *implementing_args[NPY_MAXARGS]; + PyObject *array_function_methods[NPY_MAXARGS]; + + int j, any_overrides; + int num_implementing_args = 0; + PyObject *result = NULL; + + static PyObject *errmsg_formatter = NULL; + + if (!PyArg_UnpackTuple( + positional_args, "implement_array_function", 5, 5, + &implementation, &public_api, &relevant_args, &args, &kwargs)) { + return NULL; + } + + relevant_args = PySequence_Fast( + relevant_args, + "dispatcher for __array_function__ did not return an iterable"); + if (relevant_args == NULL) { + return NULL; + } + + /* Collect __array_function__ implementations */ + num_implementing_args = get_implementing_args_and_methods( + relevant_args, implementing_args, array_function_methods); + if (num_implementing_args == -1) { + goto cleanup; + } + + /* + * Handle the typical case of no overrides. This is merely an optimization + * if some arguments are ndarray objects, but is also necessary if no + * arguments implement __array_function__ at all (e.g., if they are all + * built-in types). + */ + any_overrides = 0; + for (j = 0; j < num_implementing_args; j++) { + if (!is_default_array_function(array_function_methods[j])) { + any_overrides = 1; + break; + } + } + if (!any_overrides) { + result = PyObject_Call(implementation, args, kwargs); + goto cleanup; + } + + /* + * Create a Python object for types. + * We use a tuple, because it's the fastest Python collection to create + * and has the bonus of being immutable. + */ + types = PyTuple_New(num_implementing_args); + if (types == NULL) { + goto cleanup; + } + for (j = 0; j < num_implementing_args; j++) { + PyObject *arg_type = (PyObject *)Py_TYPE(implementing_args[j]); + Py_INCREF(arg_type); + PyTuple_SET_ITEM(types, j, arg_type); + } + + /* Call __array_function__ methods */ + for (j = 0; j < num_implementing_args; j++) { + PyObject *argument = implementing_args[j]; + PyObject *method = array_function_methods[j]; + + /* + * We use `public_api` instead of `implementation` here so + * __array_function__ implementations can do equality/identity + * comparisons. + */ + result = call_array_function( + argument, method, public_api, types, args, kwargs); + + if (result == Py_NotImplemented) { + /* Try the next one */ + Py_DECREF(result); + result = NULL; + } + else { + /* Either a good result, or an exception was raised. */ + goto cleanup; + } + } + + /* No acceptable override found, raise TypeError. */ + npy_cache_import("numpy.core._internal", + "array_function_errmsg_formatter", + &errmsg_formatter); + if (errmsg_formatter != NULL) { + PyObject *errmsg = PyObject_CallFunctionObjArgs( + errmsg_formatter, public_api, types, NULL); + if (errmsg != NULL) { + PyErr_SetObject(PyExc_TypeError, errmsg); + Py_DECREF(errmsg); + } + } + +cleanup: + for (j = 0; j < num_implementing_args; j++) { + Py_DECREF(implementing_args[j]); + Py_DECREF(array_function_methods[j]); + } + Py_XDECREF(types); + Py_DECREF(relevant_args); + return result; +} + + +/* + * Python wrapper for get_implementing_args_and_methods, for testing purposes. + */ +NPY_NO_EXPORT PyObject * +array__get_implementing_args( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args) +{ + PyObject *relevant_args; + int j; + int num_implementing_args = 0; + PyObject *implementing_args[NPY_MAXARGS]; + PyObject *array_function_methods[NPY_MAXARGS]; + PyObject *result = NULL; + + if (!PyArg_ParseTuple(positional_args, "O:array__get_implementing_args", + &relevant_args)) { + return NULL; + } + + relevant_args = PySequence_Fast( + relevant_args, + "dispatcher for __array_function__ did not return an iterable"); + if (relevant_args == NULL) { + return NULL; + } + + num_implementing_args = get_implementing_args_and_methods( + relevant_args, implementing_args, array_function_methods); + if (num_implementing_args == -1) { + goto cleanup; + } + + /* create a Python object for implementing_args */ + result = PyList_New(num_implementing_args); + if (result == NULL) { + goto cleanup; + } + for (j = 0; j < num_implementing_args; j++) { + PyObject *argument = implementing_args[j]; + Py_INCREF(argument); + PyList_SET_ITEM(result, j, argument); + } + +cleanup: + for (j = 0; j < num_implementing_args; j++) { + Py_DECREF(implementing_args[j]); + Py_DECREF(array_function_methods[j]); + } + Py_DECREF(relevant_args); + return result; +} diff --git a/numpy/core/src/multiarray/arrayfunction_override.h b/numpy/core/src/multiarray/arrayfunction_override.h new file mode 100644 index 000000000..0d224e2b6 --- /dev/null +++ b/numpy/core/src/multiarray/arrayfunction_override.h @@ -0,0 +1,16 @@ +#ifndef _NPY_PRIVATE__ARRAYFUNCTION_OVERRIDE_H +#define _NPY_PRIVATE__ARRAYFUNCTION_OVERRIDE_H + +NPY_NO_EXPORT PyObject * +array_implement_array_function( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args); + +NPY_NO_EXPORT PyObject * +array__get_implementing_args( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args); + +NPY_NO_EXPORT PyObject * +array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, + PyObject *kwargs); + +#endif diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 936a30426..97aaee93d 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -656,11 +656,9 @@ array_might_be_written(PyArrayObject *obj) { const char *msg = "Numpy has detected that you (may be) writing to an array returned\n" - "by numpy.diagonal or by selecting multiple fields in a structured\n" - "array. This code will likely break in a future numpy release --\n" - "see numpy.diagonal or arrays.indexing reference docs for details.\n" - "The quick fix is to make an explicit copy (e.g., do\n" - "arr.diagonal().copy() or arr[['f0','f1']].copy())."; + "by numpy.diagonal. This code will likely break in a future numpy\n" + "release -- see numpy.diagonal docs for details. The quick fix is\n" + "to make an explicit copy (e.g., do arr.diagonal().copy())."; if (PyArray_FLAGS(obj) & NPY_ARRAY_WARN_ON_WRITE) { /* 2012-07-17, 1.7 */ if (DEPRECATE_FUTUREWARNING(msg) < 0) { diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index d3aa1bd92..823ee7115 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -3594,9 +3594,10 @@ OBJECT_dot(char *ip1, npy_intp is1, char *ip2, npy_intp is2, char *op, npy_intp #define BOOL_fill NULL /* this requires buffer to be filled with objects or NULL */ -static void +static int OBJECT_fill(PyObject **buffer, npy_intp length, void *NPY_UNUSED(ignored)) { + int retval = 0; npy_intp i; PyObject *start = buffer[0]; PyObject *delta = buffer[1]; @@ -3604,27 +3605,31 @@ OBJECT_fill(PyObject **buffer, npy_intp length, void *NPY_UNUSED(ignored)) delta = PyNumber_Subtract(delta, start); if (!delta) { - return; + return -1; } second = start = PyNumber_Add(start, delta); if (!start) { - goto finish; + goto error; } buffer += 2; for (i = 2; i < length; i++, buffer++) { start = PyNumber_Add(start, delta); if (!start) { - goto finish; + goto error; } Py_XDECREF(*buffer); *buffer = start; } + goto finish; + +error: + retval = -1; finish: Py_XDECREF(second); Py_DECREF(delta); - return; + return retval; } /**begin repeat @@ -3638,7 +3643,7 @@ finish: * npy_float, npy_double, npy_longdouble, * npy_datetime, npy_timedelta# */ -static void +static int @NAME@_fill(@type@ *buffer, npy_intp length, void *NPY_UNUSED(ignored)) { npy_intp i; @@ -3649,10 +3654,11 @@ static void for (i = 2; i < length; ++i) { buffer[i] = start + i*delta; } + return 0; } /**end repeat**/ -static void +static int HALF_fill(npy_half *buffer, npy_intp length, void *NPY_UNUSED(ignored)) { npy_intp i; @@ -3663,6 +3669,7 @@ HALF_fill(npy_half *buffer, npy_intp length, void *NPY_UNUSED(ignored)) for (i = 2; i < length; ++i) { buffer[i] = npy_float_to_half(start + i*delta); } + return 0; } /**begin repeat @@ -3670,7 +3677,7 @@ HALF_fill(npy_half *buffer, npy_intp length, void *NPY_UNUSED(ignored)) * #NAME = CFLOAT, CDOUBLE, CLONGDOUBLE# * #type = npy_cfloat, npy_cdouble, npy_clongdouble# */ -static void +static int @NAME@_fill(@type@ *buffer, npy_intp length, void *NPY_UNUSED(ignore)) { npy_intp i; @@ -3688,6 +3695,7 @@ static void buffer->real = start.real + i*delta.real; buffer->imag = start.imag + i*delta.imag; } + return 0; } /**end repeat**/ @@ -4167,6 +4175,53 @@ small_correlate(const char * d_, npy_intp dstride, } /* +*/ + +/* A clone function for the datetime dtype c_metadata */ +static NpyAuxData * +_datetime_dtype_metadata_clone(NpyAuxData *data) +{ + PyArray_DatetimeDTypeMetaData *newdata = + (PyArray_DatetimeDTypeMetaData *)PyArray_malloc( + sizeof(*newdata)); + if (newdata == NULL) { + PyErr_NoMemory(); + return NULL; + } + + memcpy(newdata, data, sizeof(*newdata)); + + return (NpyAuxData *)newdata; +} + +/* + * Allcoate and initialize a PyArray_DatetimeDTypeMetaData object + */ +static NpyAuxData* +_create_datetime_metadata(NPY_DATETIMEUNIT base, int num) +{ + PyArray_DatetimeDTypeMetaData *data; + + /* Allocate memory for the metadata */ + data = PyArray_malloc(sizeof(*data)); + if (data == NULL) { + PyErr_NoMemory(); + return NULL; + } + + /* Initialize the base aux data */ + memset(data, 0, sizeof(PyArray_DatetimeDTypeMetaData)); + data->base.free = (NpyAuxData_FreeFunc *)PyArray_free; + data->base.clone = _datetime_dtype_metadata_clone; + + data->meta.base = base; + data->meta.num = num; + + return (NpyAuxData*)data; +} + + +/* ***************************************************************************** ** SETUP FUNCTION POINTERS ** ***************************************************************************** @@ -4522,66 +4577,6 @@ PyArray_DescrFromType(int type) return ret; } -/* A clone function for the datetime dtype metadata */ -static NpyAuxData * -datetime_dtype_metadata_clone(NpyAuxData *data) -{ - PyArray_DatetimeDTypeMetaData *newdata = - (PyArray_DatetimeDTypeMetaData *)PyArray_malloc( - sizeof(PyArray_DatetimeDTypeMetaData)); - if (newdata == NULL) { - return NULL; - } - - memcpy(newdata, data, sizeof(PyArray_DatetimeDTypeMetaData)); - - return (NpyAuxData *)newdata; -} - -/* - * Initializes the c_metadata field for the _builtin_descrs DATETIME - * and TIMEDELTA. - * - * must not be static, gcc 4.1.2 on redhat 5 then miscompiles this function - * see gh-5163 - * - */ -NPY_NO_EXPORT int -initialize_builtin_datetime_metadata(void) -{ - PyArray_DatetimeDTypeMetaData *data1, *data2; - - /* Allocate memory for the metadata */ - data1 = PyArray_malloc(sizeof(PyArray_DatetimeDTypeMetaData)); - if (data1 == NULL) { - return -1; - } - data2 = PyArray_malloc(sizeof(PyArray_DatetimeDTypeMetaData)); - if (data2 == NULL) { - PyArray_free(data1); - return -1; - } - - /* Initialize the base aux data */ - memset(data1, 0, sizeof(PyArray_DatetimeDTypeMetaData)); - memset(data2, 0, sizeof(PyArray_DatetimeDTypeMetaData)); - data1->base.free = (NpyAuxData_FreeFunc *)PyArray_free; - data2->base.free = (NpyAuxData_FreeFunc *)PyArray_free; - data1->base.clone = datetime_dtype_metadata_clone; - data2->base.clone = datetime_dtype_metadata_clone; - - /* Set to the default metadata */ - data1->meta.base = NPY_DATETIME_DEFAULTUNIT; - data1->meta.num = 1; - data2->meta.base = NPY_DATETIME_DEFAULTUNIT; - data2->meta.num = 1; - - _builtin_descrs[NPY_DATETIME]->c_metadata = (NpyAuxData *)data1; - _builtin_descrs[NPY_TIMEDELTA]->c_metadata = (NpyAuxData *)data2; - - return 0; -} - /* ***************************************************************************** ** SETUP TYPE INFO ** @@ -4650,7 +4645,14 @@ set_typeinfo(PyObject *dict) /**end repeat**/ - if (initialize_builtin_datetime_metadata() < 0) { + _builtin_descrs[NPY_DATETIME]->c_metadata = _create_datetime_metadata( + NPY_DATETIME_DEFAULTUNIT, 1); + if (_builtin_descrs[NPY_DATETIME]->c_metadata == NULL) { + return -1; + } + _builtin_descrs[NPY_TIMEDELTA]->c_metadata = _create_datetime_metadata( + NPY_DATETIME_DEFAULTUNIT, 1); + if (_builtin_descrs[NPY_DATETIME]->c_metadata == NULL) { return -1; } diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 65a290770..2e51cee7e 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -440,12 +440,18 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, return 0; } - /* Recursive case, first check the sequence contains only one type */ + /* + * The C-API recommends calling PySequence_Fast before any of the other + * PySequence_Fast* functions. This is required for PyPy + */ seq = PySequence_Fast(obj, "Could not convert object to sequence"); if (seq == NULL) { goto fail; } + + /* Recursive case, first check the sequence contains only one type */ size = PySequence_Fast_GET_SIZE(seq); + /* objects is borrowed, do not release seq */ objects = PySequence_Fast_ITEMS(seq); common_type = size > 0 ? Py_TYPE(objects[0]) : NULL; for (i = 1; i < size; ++i) { @@ -609,12 +615,6 @@ _IsWriteable(PyArrayObject *ap) * If it is a writeable array, then return TRUE * If we can find an array object * or a writeable buffer object as the final base object - * or a string object (for pickling support memory savings). - * - this last could be removed if a proper pickleable - * buffer was added to Python. - * - * MW: I think it would better to disallow switching from READONLY - * to WRITEABLE like this... */ while(PyArray_Check(base)) { @@ -623,15 +623,6 @@ _IsWriteable(PyArrayObject *ap) } base = PyArray_BASE((PyArrayObject *)base); } - - /* - * here so pickle support works seamlessly - * and unpickled array can be set and reset writeable - * -- could be abused -- - */ - if (PyString_Check(base)) { - return NPY_TRUE; - } #if defined(NPY_PY3K) if (PyObject_GetBuffer(base, &view, PyBUF_WRITABLE|PyBUF_SIMPLE) < 0) { PyErr_Clear(); diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index e8380e3bc..10e3478e2 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -1158,13 +1158,18 @@ arr_unravel_index(PyObject *self, PyObject *args, PyObject *kwds) char *kwlist[] = {"indices", "shape", "order", NULL}; - /* Continue to support the older "dims" argument in place + /* + * TODO: remove this in favor of warning raised in the dispatcher when + * __array_function__ is enabled by default. + */ + + /* + * Continue to support the older "dims" argument in place * of the "shape" argument. Issue an appropriate warning * if "dims" is detected in keywords, then replace it with - * the new "shape" argument and continue processing as usual */ - - - if (kwds) { + * the new "shape" argument and continue processing as usual. + */ + if (kwds) { PyObject *dims_item, *shape_item; dims_item = PyDict_GetItemString(kwds, "dims"); shape_item = PyDict_GetItemString(kwds, "shape"); diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c index e88582a51..7db467308 100644 --- a/numpy/core/src/multiarray/convert.c +++ b/numpy/core/src/multiarray/convert.c @@ -614,22 +614,6 @@ PyArray_View(PyArrayObject *self, PyArray_Descr *type, PyTypeObject *pytype) } dtype = PyArray_DESCR(self); - - if (type != NULL && !PyArray_EquivTypes(dtype, type) && - (PyArray_FLAGS(self) & NPY_ARRAY_WARN_ON_WRITE)) { - const char *msg = - "Numpy has detected that you may be viewing or writing to an array " - "returned by selecting multiple fields in a structured array. \n\n" - "This code may break in numpy 1.16 because this will return a view " - "instead of a copy -- see release notes for details."; - /* 2016-09-19, 1.12 */ - if (DEPRECATE_FUTUREWARNING(msg) < 0) { - return NULL; - } - /* Only warn once per array */ - PyArray_CLEARFLAGS(self, NPY_ARRAY_WARN_ON_WRITE); - } - flags = PyArray_FLAGS(self); Py_INCREF(dtype); diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 23a8dcea2..f77e414da 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -2024,7 +2024,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) newtype = oldtype; Py_INCREF(oldtype); } - if (PyDataType_ISUNSIZED(newtype)) { + else if (PyDataType_ISUNSIZED(newtype)) { PyArray_DESCR_REPLACE(newtype); if (newtype == NULL) { return NULL; diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index b9be3c09f..3038e4dea 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -257,6 +257,9 @@ _convert_from_tuple(PyObject *obj, int align) return NULL; } PyArray_DESCR_REPLACE(type); + if (type == NULL) { + return NULL; + } if (type->type_num == NPY_UNICODE) { type->elsize = itemsize << 2; } @@ -1651,6 +1654,9 @@ finish: if (PyDataType_ISUNSIZED(*at) && (*at)->elsize != elsize) { PyArray_DESCR_REPLACE(*at); + if (*at == NULL) { + goto error; + } (*at)->elsize = elsize; } if (endian != '=' && PyArray_ISNBO(endian)) { @@ -1659,6 +1665,9 @@ finish: if (endian != '=' && (*at)->byteorder != '|' && (*at)->byteorder != endian) { PyArray_DESCR_REPLACE(*at); + if (*at == NULL) { + goto error; + } (*at)->byteorder = endian; } return NPY_SUCCEED; diff --git a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src index 159bb4103..896e466c8 100644 --- a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src +++ b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src @@ -121,8 +121,8 @@ static void { #if @is_aligned@ && @elsize@ != 16 /* sanity check */ - assert(npy_is_aligned(dst, _ALIGN(@type@))); - assert(npy_is_aligned(src, _ALIGN(@type@))); + assert(N == 0 || npy_is_aligned(dst, _ALIGN(@type@))); + assert(N == 0 || npy_is_aligned(src, _ALIGN(@type@))); #endif /*printf("fn @prefix@_@oper@_size@elsize@\n");*/ while (N > 0) { @@ -201,8 +201,8 @@ static NPY_GCC_OPT_3 void } #if @is_aligned@ && @elsize@ != 16 /* sanity check */ - assert(npy_is_aligned(dst, _ALIGN(@type@))); - assert(npy_is_aligned(src, _ALIGN(@type@))); + assert(N == 0 || npy_is_aligned(dst, _ALIGN(@type@))); + assert(N == 0 || npy_is_aligned(src, _ALIGN(@type@))); #endif #if @elsize@ == 1 && @dst_contig@ memset(dst, *src, N); @@ -809,10 +809,10 @@ static NPY_GCC_OPT_3 void #if @aligned@ /* sanity check */ # if !@is_complex1@ - assert(npy_is_aligned(src, _ALIGN(_TYPE1))); + assert(N == 0 || npy_is_aligned(src, _ALIGN(_TYPE1))); # endif # if !@is_complex2@ - assert(npy_is_aligned(dst, _ALIGN(_TYPE2))); + assert(N == 0 || npy_is_aligned(dst, _ALIGN(_TYPE2))); # endif #endif diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index d371ae762..1b05faeeb 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -1389,54 +1389,14 @@ array_subscript_asarray(PyArrayObject *self, PyObject *op) } /* - * Helper function for _get_field_view which turns a multifield - * view into a "packed" copy, as done in numpy 1.15 and before. - * In numpy 1.16 this function should be removed. - */ -NPY_NO_EXPORT int -_multifield_view_to_copy(PyArrayObject **view) { - static PyObject *copyfunc = NULL; - PyObject *viewcopy; - - /* return a repacked copy of the view */ - npy_cache_import("numpy.lib.recfunctions", "repack_fields", ©func); - if (copyfunc == NULL) { - goto view_fail; - } - - PyArray_CLEARFLAGS(*view, NPY_ARRAY_WARN_ON_WRITE); - viewcopy = PyObject_CallFunction(copyfunc, "O", *view); - if (viewcopy == NULL) { - goto view_fail; - } - Py_DECREF(*view); - *view = (PyArrayObject*)viewcopy; - - /* warn when writing to the copy */ - PyArray_ENABLEFLAGS(*view, NPY_ARRAY_WARN_ON_WRITE); - return 0; - -view_fail: - Py_DECREF(*view); - *view = NULL; - return 0; -} - -/* * Attempts to subscript an array using a field name or list of field names. * * If an error occurred, return 0 and set view to NULL. If the subscript is not * a string or list of strings, return -1 and set view to NULL. Otherwise * return 0 and set view to point to a new view into arr for the given fields. - * - * In numpy 1.15 and before, in the case of a list of field names the returned - * view will actually be a copy by default, with fields packed together. - * The `force_view` argument causes a view to be returned. This argument can be - * removed in 1.16 when we plan to return a view always. */ NPY_NO_EXPORT int -_get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view, - int force_view) +_get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view) { *view = NULL; @@ -1597,11 +1557,7 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view, return 0; } - /* the code below can be replaced by "return 0" in 1.16 */ - if (force_view) { - return 0; - } - return _multifield_view_to_copy(view); + return 0; } return -1; } @@ -1629,7 +1585,7 @@ array_subscript(PyArrayObject *self, PyObject *op) /* return fields if op is a string index */ if (PyDataType_HASFIELDS(PyArray_DESCR(self))) { PyArrayObject *view; - int ret = _get_field_view(self, op, &view, 0); + int ret = _get_field_view(self, op, &view); if (ret == 0){ if (view == NULL) { return NULL; @@ -1911,7 +1867,7 @@ array_assign_subscript(PyArrayObject *self, PyObject *ind, PyObject *op) /* field access */ if (PyDataType_HASFIELDS(PyArray_DESCR(self))){ PyArrayObject *view; - int ret = _get_field_view(self, ind, &view, 1); + int ret = _get_field_view(self, ind, &view); if (ret == 0){ if (view == NULL) { return -1; diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 23b0bfd24..085bc00c0 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -8,6 +8,7 @@ #include "numpy/arrayobject.h" #include "numpy/arrayscalars.h" +#include "arrayfunction_override.h" #include "npy_config.h" #include "npy_pycompat.h" #include "npy_import.h" @@ -187,7 +188,7 @@ array_reshape(PyArrayObject *self, PyObject *args, PyObject *kwds) } if (n <= 1) { - if (PyTuple_GET_ITEM(args, 0) == Py_None) { + if (n != 0 && PyTuple_GET_ITEM(args, 0) == Py_None) { return PyArray_View(self, NULL, NULL); } if (!PyArg_ParseTuple(args, "O&:reshape", PyArray_IntpConverter, @@ -988,8 +989,58 @@ array_getarray(PyArrayObject *self, PyObject *args) } } +/* + * Check whether any of a set of input and output args have a non-default + * __array_ufunc__ method. Return 1 if so, 0 if not, and -1 on error. + * + * This function primarily exists to help ndarray.__array_ufunc__ determine + * whether it can support a ufunc (which is the case only if none of the + * operands have an override). Thus, unlike in umath/override.c, the + * actual overrides are not needed and one can stop looking once one is found. + */ +static int +any_array_ufunc_overrides(PyObject *args, PyObject *kwds) +{ + int i; + int nin, nout; + PyObject *out_kwd_obj; + PyObject *fast; + PyObject **in_objs, **out_objs; -static PyObject * + /* check inputs */ + nin = PyTuple_Size(args); + if (nin < 0) { + return -1; + } + fast = PySequence_Fast(args, "Could not convert object to sequence"); + if (fast == NULL) { + return -1; + } + in_objs = PySequence_Fast_ITEMS(fast); + for (i = 0; i < nin; ++i) { + if (PyUFunc_HasOverride(in_objs[i])) { + Py_DECREF(fast); + return 1; + } + } + Py_DECREF(fast); + /* check outputs, if any */ + nout = PyUFuncOverride_GetOutObjects(kwds, &out_kwd_obj, &out_objs); + if (nout < 0) { + return -1; + } + for (i = 0; i < nout; i++) { + if (PyUFunc_HasOverride(out_objs[i])) { + Py_DECREF(out_kwd_obj); + return 1; + } + } + Py_DECREF(out_kwd_obj); + return 0; +} + + +NPY_NO_EXPORT PyObject * array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) { PyObject *ufunc, *method_name, *normal_args, *ufunc_method; @@ -1009,7 +1060,7 @@ array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } /* ndarray cannot handle overrides itself */ - has_override = PyUFunc_HasOverride(normal_args, kwds); + has_override = any_array_ufunc_overrides(normal_args, kwds); if (has_override < 0) { goto cleanup; } @@ -1038,13 +1089,29 @@ cleanup: return result; } - static PyObject * -array_function(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_function(PyArrayObject *self, PyObject *c_args, PyObject *c_kwds) { - NPY_FORWARD_NDARRAY_METHOD("_array_function"); -} + PyObject *func, *types, *args, *kwargs, *result; + static char *kwlist[] = {"func", "types", "args", "kwargs", NULL}; + if (!PyArg_ParseTupleAndKeywords( + c_args, c_kwds, "OOOO:__array_function__", kwlist, + &func, &types, &args, &kwargs)) { + return NULL; + } + + types = PySequence_Fast( + types, + "types argument to ndarray.__array_function__ must be iterable"); + if (types == NULL) { + return NULL; + } + + result = array_function_method_impl(func, types, args, kwargs); + Py_DECREF(types); + return result; +} static PyObject * array_copy(PyArrayObject *self, PyObject *args, PyObject *kwds) diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 8f782cff6..62345d2b0 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -34,6 +34,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; /* Internal APIs */ +#include "arrayfunction_override.h" #include "arraytypes.h" #include "arrayobject.h" #include "hashdescr.h" @@ -72,10 +73,10 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; ***************************************************************************** */ #include "funcs.inc" -#include "loops.h" #include "umathmodule.h" NPY_NO_EXPORT int initscalarmath(PyObject *); +NPY_NO_EXPORT int set_matmul_flags(PyObject *d); /* in ufunc_object.c */ /* * global variable to determine if legacy printing is enabled, accessible from @@ -833,7 +834,10 @@ PyArray_InnerProduct(PyObject *op1, PyObject *op2) typenum = PyArray_ObjectType(op2, typenum); typec = PyArray_DescrFromType(typenum); if (typec == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot find a common data type."); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "Cannot find a common data type."); + } goto fail; } @@ -919,7 +923,10 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) typenum = PyArray_ObjectType(op2, typenum); typec = PyArray_DescrFromType(typenum); if (typec == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot find a common data type."); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "Cannot find a common data type."); + } return NULL; } @@ -976,7 +983,7 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) for (i = 0; i < PyArray_NDIM(ap2) - 2; i++) { dimensions[j++] = PyArray_DIMS(ap2)[i]; } - if(PyArray_NDIM(ap2) > 1) { + if (PyArray_NDIM(ap2) > 1) { dimensions[j++] = PyArray_DIMS(ap2)[PyArray_NDIM(ap2)-1]; } @@ -1312,7 +1319,7 @@ PyArray_Correlate2(PyObject *op1, PyObject *op2, int mode) */ if (inverted) { st = _pyarray_revert(ret); - if(st) { + if (st) { goto clean_ret; } } @@ -1359,7 +1366,7 @@ PyArray_Correlate(PyObject *op1, PyObject *op2, int mode) } ret = _pyarray_correlate(ap1, ap2, typenum, mode, &unused); - if(ret == NULL) { + if (ret == NULL) { goto fail; } Py_DECREF(ap1); @@ -1648,7 +1655,7 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) } full_path: - if(!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&i:array", kwd, + if (!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&i:array", kwd, &op, PyArray_DescrConverter2, &type, PyArray_BoolConverter, ©, @@ -2044,6 +2051,7 @@ static PyObject * array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) { PyObject *file = NULL, *ret; + PyObject *err_type = NULL, *err_value = NULL, *err_traceback = NULL; char *sep = ""; Py_ssize_t nin = -1; static char *kwlist[] = {"file", "dtype", "count", "sep", NULL}; @@ -2079,18 +2087,26 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) } ret = PyArray_FromFile(fp, type, (npy_intp) nin, sep); + /* If an exception is thrown in the call to PyArray_FromFile + * we need to clear it, and restore it later to ensure that + * we can cleanup the duplicated file descriptor properly. + */ + PyErr_Fetch(&err_type, &err_value, &err_traceback); if (npy_PyFile_DupClose2(file, fp, orig_pos) < 0) { + npy_PyErr_ChainExceptions(err_type, err_value, err_traceback); goto fail; } if (own && npy_PyFile_CloseFile(file) < 0) { + npy_PyErr_ChainExceptions(err_type, err_value, err_traceback); goto fail; } + PyErr_Restore(err_type, err_value, err_traceback); Py_DECREF(file); return ret; fail: Py_DECREF(file); - Py_DECREF(ret); + Py_XDECREF(ret); return NULL; } @@ -2303,154 +2319,6 @@ fail: return NULL; } - - -/* - * matmul - * - * Implements the protocol used by the '@' operator defined in PEP 364. - * Not in the NUMPY API at this time, maybe later. - * - * - * in1: Left hand side operand - * in2: Right hand side operand - * out: Either NULL, or an array into which the output should be placed. - * - * Returns NULL on error. - */ -static PyObject * -array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) -{ - PyObject *in1, *in2, *out = NULL; - char* kwlist[] = {"a", "b", "out", NULL }; - PyArrayObject *ap1, *ap2, *ret = NULL; - NPY_ORDER order = NPY_KEEPORDER; - NPY_CASTING casting = NPY_SAFE_CASTING; - PyArray_Descr *dtype; - int nd1, nd2, typenum; - char *subscripts; - PyArrayObject *ops[2]; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matmul", kwlist, - &in1, &in2, &out)) { - return NULL; - } - - if (out != NULL) { - if (out == Py_None) { - out = NULL; - } - else if (!PyArray_Check(out)) { - PyErr_SetString(PyExc_TypeError, "'out' must be an array"); - return NULL; - } - } - - dtype = PyArray_DescrFromObject(in1, NULL); - dtype = PyArray_DescrFromObject(in2, dtype); - if (dtype == NULL) { - PyErr_SetString(PyExc_ValueError, "Cannot find a common data type."); - return NULL; - } - typenum = dtype->type_num; - - if (typenum == NPY_OBJECT) { - /* matmul is not currently implemented for object arrays */ - PyErr_SetString(PyExc_TypeError, - "Object arrays are not currently supported"); - Py_DECREF(dtype); - return NULL; - } - - ap1 = (PyArrayObject *)PyArray_FromAny(in1, dtype, 0, 0, - NPY_ARRAY_ALIGNED, NULL); - if (ap1 == NULL) { - return NULL; - } - - Py_INCREF(dtype); - ap2 = (PyArrayObject *)PyArray_FromAny(in2, dtype, 0, 0, - NPY_ARRAY_ALIGNED, NULL); - if (ap2 == NULL) { - Py_DECREF(ap1); - return NULL; - } - - if (PyArray_NDIM(ap1) == 0 || PyArray_NDIM(ap2) == 0) { - /* Scalars are rejected */ - PyErr_SetString(PyExc_ValueError, - "Scalar operands are not allowed, use '*' instead"); - return NULL; - } - - nd1 = PyArray_NDIM(ap1); - nd2 = PyArray_NDIM(ap2); - -#if defined(HAVE_CBLAS) - if (nd1 <= 2 && nd2 <= 2 && - (NPY_DOUBLE == typenum || NPY_CDOUBLE == typenum || - NPY_FLOAT == typenum || NPY_CFLOAT == typenum)) { - return cblas_matrixproduct(typenum, ap1, ap2, (PyArrayObject *)out); - } -#endif - - /* - * Use einsum for the stacked cases. This is a quick implementation - * to avoid setting up the proper iterators. Einsum broadcasts, so - * we need to check dimensions before the call. - */ - if (nd1 == 1 && nd2 == 1) { - /* vector vector */ - if (PyArray_DIM(ap1, 0) != PyArray_DIM(ap2, 0)) { - dot_alignment_error(ap1, 0, ap2, 0); - goto fail; - } - subscripts = "i, i"; - } - else if (nd1 == 1) { - /* vector matrix */ - if (PyArray_DIM(ap1, 0) != PyArray_DIM(ap2, nd2 - 2)) { - dot_alignment_error(ap1, 0, ap2, nd2 - 2); - goto fail; - } - subscripts = "i, ...ij"; - } - else if (nd2 == 1) { - /* matrix vector */ - if (PyArray_DIM(ap1, nd1 - 1) != PyArray_DIM(ap2, 0)) { - dot_alignment_error(ap1, nd1 - 1, ap2, 0); - goto fail; - } - subscripts = "...i, i"; - } - else { - /* matrix * matrix */ - if (PyArray_DIM(ap1, nd1 - 1) != PyArray_DIM(ap2, nd2 - 2)) { - dot_alignment_error(ap1, nd1 - 1, ap2, nd2 - 2); - goto fail; - } - subscripts = "...ij, ...jk"; - } - ops[0] = ap1; - ops[1] = ap2; - ret = PyArray_EinsteinSum(subscripts, 2, ops, NULL, order, casting, - (PyArrayObject *)out); - Py_DECREF(ap1); - Py_DECREF(ap2); - - /* If no output was supplied, possibly convert to a scalar */ - if (ret != NULL && out == NULL) { - return PyArray_Return((PyArrayObject *)ret); - } - return (PyObject *)ret; - -fail: - Py_XDECREF(ap1); - Py_XDECREF(ap2); - return NULL; -} - - static int einsum_sub_op_from_str(PyObject *args, PyObject **str_obj, char **subscripts, PyArrayObject **op) @@ -2622,7 +2490,7 @@ einsum_sub_op_from_lists(PyObject *args, "operand and a subscripts list to einsum"); return -1; } - else if(nop >= NPY_MAXARGS) { + else if (nop >= NPY_MAXARGS) { PyErr_SetString(PyExc_ValueError, "too many operands"); return -1; } @@ -2857,7 +2725,7 @@ array_arange(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) { static char *kwd[]= {"start", "stop", "step", "dtype", NULL}; PyArray_Descr *typecode = NULL; - if(!PyArg_ParseTupleAndKeywords(args, kws, "O|OOO&:arange", kwd, + if (!PyArg_ParseTupleAndKeywords(args, kws, "O|OOO&:arange", kwd, &o_start, &o_stop, &o_step, @@ -2895,7 +2763,7 @@ array__get_ndarray_c_version(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObje { static char *kwlist[] = {NULL}; - if(!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist )) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist )) { return NULL; } return PyInt_FromLong( (long) PyArray_GetNDArrayCVersion() ); @@ -2968,7 +2836,7 @@ array_set_string_function(PyObject *NPY_UNUSED(self), PyObject *args, int repr = 1; static char *kwlist[] = {"f", "repr", NULL}; - if(!PyArg_ParseTupleAndKeywords(args, kwds, "|Oi:set_string_function", kwlist, &op, &repr)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oi:set_string_function", kwlist, &op, &repr)) { return NULL; } /* reset the array_repr function to built-in */ @@ -2990,7 +2858,7 @@ array_set_ops_function(PyObject *NPY_UNUSED(self), PyObject *NPY_UNUSED(args), { PyObject *oldops = NULL; - if ((oldops = PyArray_GetNumericOps()) == NULL) { + if ((oldops = _PyArray_GetNumericOps()) == NULL) { return NULL; } /* @@ -3000,8 +2868,10 @@ array_set_ops_function(PyObject *NPY_UNUSED(self), PyObject *NPY_UNUSED(args), */ if (kwds && PyArray_SetNumericOps(kwds) == -1) { Py_DECREF(oldops); - PyErr_SetString(PyExc_ValueError, + if (PyErr_Occurred() == NULL) { + PyErr_SetString(PyExc_ValueError, "one or more objects not callable"); + } return NULL; } return oldops; @@ -3276,7 +3146,7 @@ array_promote_types(PyObject *NPY_UNUSED(dummy), PyObject *args) PyArray_Descr *d1 = NULL; PyArray_Descr *d2 = NULL; PyObject *ret = NULL; - if(!PyArg_ParseTuple(args, "O&O&:promote_types", + if (!PyArg_ParseTuple(args, "O&O&:promote_types", PyArray_DescrConverter2, &d1, PyArray_DescrConverter2, &d2)) { goto finish; } @@ -3302,7 +3172,7 @@ array_min_scalar_type(PyObject *NPY_UNUSED(dummy), PyObject *args) PyArrayObject *array; PyObject *ret = NULL; - if(!PyArg_ParseTuple(args, "O:min_scalar_type", &array_in)) { + if (!PyArg_ParseTuple(args, "O:min_scalar_type", &array_in)) { return NULL; } @@ -3379,7 +3249,7 @@ array_datetime_data(PyObject *NPY_UNUSED(dummy), PyObject *args) PyArray_Descr *dtype; PyArray_DatetimeMetaData *meta; - if(!PyArg_ParseTuple(args, "O&:datetime_data", + if (!PyArg_ParseTuple(args, "O&:datetime_data", PyArray_DescrConverter, &dtype)) { return NULL; } @@ -3398,7 +3268,7 @@ new_buffer(PyObject *NPY_UNUSED(dummy), PyObject *args) { int size; - if(!PyArg_ParseTuple(args, "i:buffer", &size)) { + if (!PyArg_ParseTuple(args, "i:buffer", &size)) { return NULL; } return PyBuffer_New(size); @@ -4193,6 +4063,9 @@ normalize_axis_index(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) } static struct PyMethodDef array_module_methods[] = { + {"_get_implementing_args", + (PyCFunction)array__get_implementing_args, + METH_VARARGS, NULL}, {"_get_ndarray_c_version", (PyCFunction)array__get_ndarray_c_version, METH_VARARGS|METH_KEYWORDS, NULL}, @@ -4265,9 +4138,6 @@ static struct PyMethodDef array_module_methods[] = { {"vdot", (PyCFunction)array_vdot, METH_VARARGS | METH_KEYWORDS, NULL}, - {"matmul", - (PyCFunction)array_matmul, - METH_VARARGS | METH_KEYWORDS, NULL}, {"c_einsum", (PyCFunction)array_einsum, METH_VARARGS|METH_KEYWORDS, NULL}, @@ -4358,6 +4228,9 @@ static struct PyMethodDef array_module_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"_monotonicity", (PyCFunction)arr__monotonicity, METH_VARARGS | METH_KEYWORDS, NULL}, + {"implement_array_function", + (PyCFunction)array_implement_array_function, + METH_VARARGS, NULL}, {"interp", (PyCFunction)arr_interp, METH_VARARGS | METH_KEYWORDS, NULL}, {"interp_complex", (PyCFunction)arr_interp_complex, @@ -4610,6 +4483,7 @@ NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_array_wrap = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_array_finalize = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_buffer = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_ufunc = NULL; +NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_wrapped = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_order = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_copy = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_dtype = NULL; @@ -4626,6 +4500,7 @@ intern_strings(void) npy_ma_str_array_finalize = PyUString_InternFromString("__array_finalize__"); npy_ma_str_buffer = PyUString_InternFromString("__buffer__"); npy_ma_str_ufunc = PyUString_InternFromString("__array_ufunc__"); + npy_ma_str_wrapped = PyUString_InternFromString("__wrapped__"); npy_ma_str_order = PyUString_InternFromString("order"); npy_ma_str_copy = PyUString_InternFromString("copy"); npy_ma_str_dtype = PyUString_InternFromString("dtype"); @@ -4635,12 +4510,11 @@ intern_strings(void) return npy_ma_str_array && npy_ma_str_array_prepare && npy_ma_str_array_wrap && npy_ma_str_array_finalize && - npy_ma_str_buffer && npy_ma_str_ufunc && + npy_ma_str_buffer && npy_ma_str_ufunc && npy_ma_str_wrapped && npy_ma_str_order && npy_ma_str_copy && npy_ma_str_dtype && npy_ma_str_ndmin && npy_ma_str_axis1 && npy_ma_str_axis2; } - #if defined(NPY_PY3K) static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, @@ -4705,15 +4579,23 @@ PyMODINIT_FUNC init_multiarray_umath(void) { */ PyArray_Type.tp_hash = PyObject_HashNotImplemented; + if (PyType_Ready(&PyUFunc_Type) < 0) { + goto err; + } + /* Load the ufunc operators into the array module's namespace */ if (InitOperators(d) < 0) { goto err; } + if (set_matmul_flags(d) < 0) { + goto err; + } initialize_casting_tables(); initialize_numeric_types(); - if(initscalarmath(m) < 0) + if (initscalarmath(m) < 0) { goto err; + } if (PyType_Ready(&PyArray_Type) < 0) { goto err; diff --git a/numpy/core/src/multiarray/multiarraymodule.h b/numpy/core/src/multiarray/multiarraymodule.h index 3de68c549..60a3965c9 100644 --- a/numpy/core/src/multiarray/multiarraymodule.h +++ b/numpy/core/src/multiarray/multiarraymodule.h @@ -7,6 +7,7 @@ NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_array_wrap; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_array_finalize; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_buffer; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_ufunc; +NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_wrapped; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_order; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_copy; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_dtype; diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index dabbae064..d153a8a64 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -71,12 +71,8 @@ array_inplace_power(PyArrayObject *a1, PyObject *o2, PyObject *NPY_UNUSED(modulo n_ops.op = temp; \ } - -/*NUMPY_API - *Set internal structure with number functions that all arrays will use - */ NPY_NO_EXPORT int -PyArray_SetNumericOps(PyObject *dict) +_PyArray_SetNumericOps(PyObject *dict) { PyObject *temp = NULL; SET(add); @@ -116,19 +112,32 @@ PyArray_SetNumericOps(PyObject *dict) SET(minimum); SET(rint); SET(conjugate); + SET(matmul); return 0; } +/*NUMPY_API + *Set internal structure with number functions that all arrays will use + */ +NPY_NO_EXPORT int +PyArray_SetNumericOps(PyObject *dict) +{ + /* 2018-09-09, 1.16 */ + if (DEPRECATE("PyArray_SetNumericOps is deprecated. Use " + "PyUFunc_ReplaceLoopBySignature to replace ufunc inner loop functions " + "instead.") < 0) { + return -1; + } + return _PyArray_SetNumericOps(dict); +} + /* Note - macro contains goto */ #define GET(op) if (n_ops.op && \ (PyDict_SetItemString(dict, #op, n_ops.op)==-1)) \ goto fail; -/*NUMPY_API - Get dictionary showing number functions that all arrays will use -*/ NPY_NO_EXPORT PyObject * -PyArray_GetNumericOps(void) +_PyArray_GetNumericOps(void) { PyObject *dict; if ((dict = PyDict_New())==NULL) @@ -169,6 +178,7 @@ PyArray_GetNumericOps(void) GET(minimum); GET(rint); GET(conjugate); + GET(matmul); return dict; fail: @@ -176,6 +186,19 @@ PyArray_GetNumericOps(void) return NULL; } +/*NUMPY_API + Get dictionary showing number functions that all arrays will use +*/ +NPY_NO_EXPORT PyObject * +PyArray_GetNumericOps(void) +{ + /* 2018-09-09, 1.16 */ + if (DEPRECATE("PyArray_GetNumericOps is deprecated.") < 0) { + return NULL; + } + return _PyArray_GetNumericOps(); +} + static PyObject * _get_keywords(int rtype, PyArrayObject *out) { @@ -361,14 +384,8 @@ array_divmod(PyArrayObject *m1, PyObject *m2) static PyObject * array_matrix_multiply(PyArrayObject *m1, PyObject *m2) { - static PyObject *matmul = NULL; - - npy_cache_import("numpy.core.multiarray", "matmul", &matmul); - if (matmul == NULL) { - return NULL; - } BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_matrix_multiply, array_matrix_multiply); - return PyArray_GenericBinaryFunction(m1, m2, matmul); + return PyArray_GenericBinaryFunction(m1, m2, n_ops.matmul); } static PyObject * @@ -578,7 +595,7 @@ array_positive(PyArrayObject *m1) */ PyObject *exc, *val, *tb; PyErr_Fetch(&exc, &val, &tb); - if (has_non_default_array_ufunc((PyObject *)m1)) { + if (PyUFunc_HasOverride((PyObject *)m1)) { PyErr_Restore(exc, val, tb); return NULL; } diff --git a/numpy/core/src/multiarray/number.h b/numpy/core/src/multiarray/number.h index 99a2a722b..33a7cf872 100644 --- a/numpy/core/src/multiarray/number.h +++ b/numpy/core/src/multiarray/number.h @@ -39,6 +39,7 @@ typedef struct { PyObject *minimum; PyObject *rint; PyObject *conjugate; + PyObject *matmul; } NumericOps; extern NPY_NO_EXPORT NumericOps n_ops; @@ -48,10 +49,10 @@ NPY_NO_EXPORT PyObject * array_int(PyArrayObject *v); NPY_NO_EXPORT int -PyArray_SetNumericOps(PyObject *dict); +_PyArray_SetNumericOps(PyObject *dict); NPY_NO_EXPORT PyObject * -PyArray_GetNumericOps(void); +_PyArray_GetNumericOps(void); NPY_NO_EXPORT PyObject * PyArray_GenericBinaryFunction(PyArrayObject *m1, PyObject *m2, PyObject *op); diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 0f201b966..2f71c8ae9 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -1104,8 +1104,7 @@ static PyNumberMethods gentype_as_number = { (binaryfunc)gentype_add, /*nb_add*/ (binaryfunc)gentype_subtract, /*nb_subtract*/ (binaryfunc)gentype_multiply, /*nb_multiply*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) (binaryfunc)gentype_divide, /*nb_divide*/ #endif (binaryfunc)gentype_remainder, /*nb_remainder*/ @@ -1121,8 +1120,7 @@ static PyNumberMethods gentype_as_number = { (binaryfunc)gentype_and, /*nb_and*/ (binaryfunc)gentype_xor, /*nb_xor*/ (binaryfunc)gentype_or, /*nb_or*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) 0, /*nb_coerce*/ #endif (unaryfunc)gentype_int, /*nb_int*/ @@ -1132,16 +1130,14 @@ static PyNumberMethods gentype_as_number = { (unaryfunc)gentype_long, /*nb_long*/ #endif (unaryfunc)gentype_float, /*nb_float*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) (unaryfunc)gentype_oct, /*nb_oct*/ (unaryfunc)gentype_hex, /*nb_hex*/ #endif 0, /*inplace_add*/ 0, /*inplace_subtract*/ 0, /*inplace_multiply*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) 0, /*inplace_divide*/ #endif 0, /*inplace_remainder*/ @@ -1156,6 +1152,10 @@ static PyNumberMethods gentype_as_number = { 0, /*nb_inplace_floor_divide*/ 0, /*nb_inplace_true_divide*/ (unaryfunc)NULL, /*nb_index*/ +#if PY_VERSION_HEX >= 0x03050000 + 0, /*np_matmul*/ + 0, /*np_inplace_matmul*/ +#endif }; diff --git a/numpy/core/src/multiarray/temp_elide.c b/numpy/core/src/multiarray/temp_elide.c index 3d2f976f2..09b948218 100644 --- a/numpy/core/src/multiarray/temp_elide.c +++ b/numpy/core/src/multiarray/temp_elide.c @@ -166,7 +166,7 @@ check_callers(int * cannot) return 0; } /* get multiarray base address */ - if (dladdr(&PyArray_SetNumericOps, &info)) { + if (dladdr(&PyArray_INCREF, &info)) { pos_ma_start = info.dli_fbase; pos_ma_end = info.dli_fbase; } diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index f96e621b8..ae3ece77b 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -1861,7 +1861,8 @@ NPY_NO_EXPORT void if (!run_unary_reduce_simd_@kind@_@TYPE@(args, dimensions, steps)) { BINARY_REDUCE_LOOP(@type@) { const @type@ in2 = *(@type@ *)ip2; - io1 = (npy_isnan(io1) || io1 @OP@ in2) ? io1 : in2; + /* Order of operations important for MSVC 2015 */ + io1 = (io1 @OP@ in2 || npy_isnan(io1)) ? io1 : in2; } *((@type@ *)iop1) = io1; } @@ -1870,7 +1871,8 @@ NPY_NO_EXPORT void BINARY_LOOP { @type@ in1 = *(@type@ *)ip1; const @type@ in2 = *(@type@ *)ip2; - in1 = (npy_isnan(in1) || in1 @OP@ in2) ? in1 : in2; + /* Order of operations important for MSVC 2015 */ + in1 = (in1 @OP@ in2 || npy_isnan(in1)) ? in1 : in2; *((@type@ *)op1) = in1; } } @@ -1889,7 +1891,8 @@ NPY_NO_EXPORT void if (IS_BINARY_REDUCE) { BINARY_REDUCE_LOOP(@type@) { const @type@ in2 = *(@type@ *)ip2; - io1 = (npy_isnan(in2) || io1 @OP@ in2) ? io1 : in2; + /* Order of operations important for MSVC 2015 */ + io1 = (io1 @OP@ in2 || npy_isnan(in2)) ? io1 : in2; } *((@type@ *)iop1) = io1; } @@ -1897,7 +1900,8 @@ NPY_NO_EXPORT void BINARY_LOOP { const @type@ in1 = *(@type@ *)ip1; const @type@ in2 = *(@type@ *)ip2; - *((@type@ *)op1) = (npy_isnan(in2) || in1 @OP@ in2) ? in1 : in2; + /* Order of operations important for MSVC 2015 */ + *((@type@ *)op1) = (in1 @OP@ in2 || npy_isnan(in2)) ? in1 : in2; } } npy_clear_floatstatus_barrier((char*)dimensions); diff --git a/numpy/core/src/umath/matmul.c.src b/numpy/core/src/umath/matmul.c.src new file mode 100644 index 000000000..0cb3c82ad --- /dev/null +++ b/numpy/core/src/umath/matmul.c.src @@ -0,0 +1,402 @@ +/* -*- c -*- */ + +#define _UMATHMODULE +#define _MULTIARRAYMODULE +#define NPY_NO_DEPRECATED_API NPY_API_VERSION + +#include "Python.h" + +#include "npy_config.h" +#include "numpy/npy_common.h" +#include "numpy/arrayobject.h" +#include "numpy/ufuncobject.h" +#include "numpy/npy_math.h" +#include "numpy/halffloat.h" +#include "lowlevel_strided_loops.h" + +#include "npy_pycompat.h" + +#include "npy_cblas.h" +#include "arraytypes.h" /* For TYPE_dot functions */ + +#include <assert.h> + +/* + ***************************************************************************** + ** BASICS ** + ***************************************************************************** + */ + +/* + * -1 to be conservative, in case blas internally uses a for loop with an + * inclusive upper bound + */ +#define BLAS_MAXSIZE (NPY_MAX_INT - 1) + +/* + * Determine if a 2d matrix can be used by BLAS + * 1. Strides must not alias or overlap + * 2. The faster (second) axis must be contiguous + * 3. The slower (first) axis stride, in unit steps, must be larger than + * the faster axis dimension + */ +static NPY_INLINE npy_bool +is_blasable2d(npy_intp byte_stride1, npy_intp byte_stride2, + npy_intp d1, npy_intp d2, npy_intp itemsize) +{ + npy_intp unit_stride1 = byte_stride1 / itemsize; + if (byte_stride2 != itemsize) { + return NPY_FALSE; + } + if ((byte_stride1 % itemsize ==0) && + (unit_stride1 >= d2) && + (unit_stride1 <= BLAS_MAXSIZE)) + { + return NPY_TRUE; + } + return NPY_FALSE; +} + +#if defined(HAVE_CBLAS) +static const npy_cdouble oneD = {1.0, 0.0}, zeroD = {0.0, 0.0}; +static const npy_cfloat oneF = {1.0, 0.0}, zeroF = {0.0, 0.0}; + +/**begin repeat + * + * #name = FLOAT, DOUBLE, CFLOAT, CDOUBLE# + * #ctype = npy_float, npy_double, npy_cfloat, npy_cdouble# + * #typ = npy_float, npy_double, npy_cfloat, npy_cdouble# + * #prefix = s, d, c, z# + * #step1 = 1.F, 1., &oneF, &oneD# + * #step0 = 0.F, 0., &zeroF, &zeroD# + */ +NPY_NO_EXPORT void +@name@_gemv(void *ip1, npy_intp is1_m, npy_intp is1_n, + void *ip2, npy_intp is2_n, npy_intp NPY_UNUSED(is2_p), + void *op, npy_intp op_m, npy_intp NPY_UNUSED(op_p), + npy_intp m, npy_intp n, npy_intp NPY_UNUSED(p)) +{ + /* + * Vector matrix multiplication -- Level 2 BLAS + * arguments + * ip1: contiguous data, m*n shape + * ip2: data in c order, n*1 shape + * op: data in c order, m shape + */ + enum CBLAS_ORDER order; + int M, N, lda; + + assert(m <= BLAS_MAXSIZE && n <= BLAS_MAXSIZE); + assert (is_blasable2d(is2_n, sizeof(@typ@), n, 1, sizeof(@typ@))); + M = (int)m; + N = (int)n; + + if (is_blasable2d(is1_m, is1_n, m, n, sizeof(@typ@))) { + order = CblasColMajor; + lda = (int)(is1_m / sizeof(@typ@)); + } + else { + /* If not ColMajor, caller should have ensured we are RowMajor */ + /* will not assert in release mode */ + order = CblasRowMajor; + assert(is_blasable2d(is1_n, is1_m, n, m, sizeof(@typ@))); + lda = (int)(is1_n / sizeof(@typ@)); + } + cblas_@prefix@gemv(order, CblasTrans, N, M, @step1@, ip1, lda, ip2, + is2_n / sizeof(@typ@), @step0@, op, op_m / sizeof(@typ@)); +} + +NPY_NO_EXPORT void +@name@_matmul_matrixmatrix(void *ip1, npy_intp is1_m, npy_intp is1_n, + void *ip2, npy_intp is2_n, npy_intp is2_p, + void *op, npy_intp os_m, npy_intp os_p, + npy_intp m, npy_intp n, npy_intp p) +{ + /* + * matrix matrix multiplication -- Level 3 BLAS + */ + enum CBLAS_ORDER order = CblasRowMajor; + enum CBLAS_TRANSPOSE trans1, trans2; + int M, N, P, lda, ldb, ldc; + assert(m <= BLAS_MAXSIZE && n <= BLAS_MAXSIZE && p <= BLAS_MAXSIZE); + M = (int)m; + N = (int)n; + P = (int)p; + + assert(is_blasable2d(os_m, os_p, m, p, sizeof(@typ@))); + ldc = (int)(os_m / sizeof(@typ@)); + + if (is_blasable2d(is1_m, is1_n, m, n, sizeof(@typ@))) { + trans1 = CblasNoTrans; + lda = (int)(is1_m / sizeof(@typ@)); + } + else { + /* If not ColMajor, caller should have ensured we are RowMajor */ + /* will not assert in release mode */ + assert(is_blasable2d(is1_n, is1_m, n, m, sizeof(@typ@))); + trans1 = CblasTrans; + lda = (int)(is1_n / sizeof(@typ@)); + } + + if (is_blasable2d(is2_n, is2_p, n, p, sizeof(@typ@))) { + trans2 = CblasNoTrans; + ldb = (int)(is2_n / sizeof(@typ@)); + } + else { + /* If not ColMajor, caller should have ensured we are RowMajor */ + /* will not assert in release mode */ + assert(is_blasable2d(is2_p, is2_n, p, n, sizeof(@typ@))); + trans2 = CblasTrans; + ldb = (int)(is2_p / sizeof(@typ@)); + } + /* + * Use syrk if we have a case of a matrix times its transpose. + * Otherwise, use gemm for all other cases. + */ + if ( + (ip1 == ip2) && + (m == p) && + (is1_m == is2_p) && + (is1_n == is2_n) && + (trans1 != trans2) + ) { + npy_intp i,j; + if (trans1 == CblasNoTrans) { + cblas_@prefix@syrk(order, CblasUpper, trans1, P, N, @step1@, + ip1, lda, @step0@, op, ldc); + } + else { + cblas_@prefix@syrk(order, CblasUpper, trans1, P, N, @step1@, + ip1, ldb, @step0@, op, ldc); + } + /* Copy the triangle */ + for (i = 0; i < P; i++) { + for (j = i + 1; j < P; j++) { + ((@typ@*)op)[j * ldc + i] = ((@typ@*)op)[i * ldc + j]; + } + } + + } + else { + cblas_@prefix@gemm(order, trans1, trans2, M, P, N, @step1@, ip1, lda, + ip2, ldb, @step0@, op, ldc); + } +} + +/**end repeat**/ +#endif + +/* + * matmul loops + * signature is (m?,n),(n,p?)->(m?,p?) + */ + +/**begin repeat + * #TYPE = LONGDOUBLE, + * FLOAT, DOUBLE, HALF, + * CFLOAT, CDOUBLE, CLONGDOUBLE, + * UBYTE, USHORT, UINT, ULONG, ULONGLONG, + * BYTE, SHORT, INT, LONG, LONGLONG, + * BOOL# + * #typ = npy_longdouble, + * npy_float,npy_double,npy_half, + * npy_cfloat, npy_cdouble, npy_clongdouble, + * npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong, + * npy_byte, npy_short, npy_int, npy_long, npy_longlong, + * npy_bool# + * #IS_COMPLEX = 0, 0, 0, 0, 1, 1, 1, 0*11# + * #IS_HALF = 0, 0, 0, 1, 0*14# + */ + +NPY_NO_EXPORT void +@TYPE@_matmul_inner_noblas(void *_ip1, npy_intp is1_m, npy_intp is1_n, + void *_ip2, npy_intp is2_n, npy_intp is2_p, + void *_op, npy_intp os_m, npy_intp os_p, + npy_intp dm, npy_intp dn, npy_intp dp) + +{ + npy_intp m, n, p; + npy_intp ib1_n, ib2_n, ib2_p, ob_p; + char *ip1 = (char *)_ip1, *ip2 = (char *)_ip2, *op = (char *)_op; + + ib1_n = is1_n * dn; + ib2_n = is2_n * dn; + ib2_p = is2_p * dp; + ob_p = os_p * dp; + + for (m = 0; m < dm; m++) { + for (p = 0; p < dp; p++) { +#if @IS_COMPLEX@ == 1 + (*(@typ@ *)op).real = 0; + (*(@typ@ *)op).imag = 0; +#elif @IS_HALF@ + float sum = 0; +#else + *(@typ@ *)op = 0; +#endif + for (n = 0; n < dn; n++) { + @typ@ val1 = (*(@typ@ *)ip1); + @typ@ val2 = (*(@typ@ *)ip2); +#if @IS_HALF@ + sum += npy_half_to_float(val1) * npy_half_to_float(val2); +#elif @IS_COMPLEX@ == 1 + (*(@typ@ *)op).real += (val1.real * val2.real) - + (val1.imag * val2.imag); + (*(@typ@ *)op).imag += (val1.real * val2.imag) + + (val1.imag * val2.real); +#else + *(@typ@ *)op += val1 * val2; +#endif + ip2 += is2_n; + ip1 += is1_n; + } +#if @IS_HALF@ + *(@typ@ *)op = npy_float_to_half(sum); +#endif + ip1 -= ib1_n; + ip2 -= ib2_n; + op += os_p; + ip2 += is2_p; + } + op -= ob_p; + ip2 -= ib2_p; + ip1 += is1_m; + op += os_m; + } +} + +/**end repeat**/ + +/**begin repeat + * #TYPE = FLOAT, DOUBLE, LONGDOUBLE, HALF, + * CFLOAT, CDOUBLE, CLONGDOUBLE, + * UBYTE, USHORT, UINT, ULONG, ULONGLONG, + * BYTE, SHORT, INT, LONG, LONGLONG, + * BOOL# + * #typ = npy_float,npy_double,npy_longdouble, npy_half, + * npy_cfloat, npy_cdouble, npy_clongdouble, + * npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong, + * npy_byte, npy_short, npy_int, npy_long, npy_longlong, + * npy_bool# + * #IS_COMPLEX = 0, 0, 0, 0, 1, 1, 1, 0*11# + * #USEBLAS = 1, 1, 0, 0, 1, 1, 0*12# + */ + + +NPY_NO_EXPORT void +@TYPE@_matmul(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)) +{ + npy_intp dOuter = *dimensions++; + npy_intp iOuter; + npy_intp s0 = *steps++; + npy_intp s1 = *steps++; + npy_intp s2 = *steps++; + npy_intp dm = dimensions[0]; + npy_intp dn = dimensions[1]; + npy_intp dp = dimensions[2]; + npy_intp is1_m=steps[0], is1_n=steps[1], is2_n=steps[2], is2_p=steps[3], + os_m=steps[4], os_p=steps[5]; +#if @USEBLAS@ && defined(HAVE_CBLAS) + npy_intp sz = sizeof(@typ@); + npy_bool special_case = (dm == 1 || dn == 1 || dp == 1); + npy_bool any_zero_dim = (dm == 0 || dn == 0 || dp == 0); + npy_bool scalar_out = (dm == 1 && dp == 1); + npy_bool scalar_vec = (dn == 1 && (dp == 1 || dm == 1)); + npy_bool too_big_for_blas = (dm > BLAS_MAXSIZE || dn > BLAS_MAXSIZE || + dp > BLAS_MAXSIZE); + npy_bool i1_c_blasable = is_blasable2d(is1_m, is1_n, dm, dn, sz); + npy_bool i2_c_blasable = is_blasable2d(is2_n, is2_p, dn, dp, sz); + npy_bool i1_f_blasable = is_blasable2d(is1_n, is1_m, dn, dm, sz); + npy_bool i2_f_blasable = is_blasable2d(is2_p, is2_n, dp, dn, sz); + npy_bool i1blasable = i1_c_blasable || i1_f_blasable; + npy_bool i2blasable = i2_c_blasable || i2_f_blasable; + npy_bool o_c_blasable = is_blasable2d(os_m, os_p, dm, dp, sz); + npy_bool o_f_blasable = is_blasable2d(os_p, os_m, dp, dm, sz); + npy_bool vector_matrix = ((dm == 1) && i2blasable && + is_blasable2d(is1_n, sz, dn, 1, sz)); + npy_bool matrix_vector = ((dp == 1) && i1blasable && + is_blasable2d(is2_n, sz, dn, 1, sz)); +#endif + + for (iOuter = 0; iOuter < dOuter; iOuter++, + args[0] += s0, args[1] += s1, args[2] += s2) { + void *ip1=args[0], *ip2=args[1], *op=args[2]; +#if @USEBLAS@ && defined(HAVE_CBLAS) + /* + * TODO: refactor this out to a inner_loop_selector, in + * PyUFunc_MatmulLoopSelector. But that call does not have access to + * n, m, p and strides. + */ + if (too_big_for_blas || any_zero_dim) { + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + } + else if (special_case) { + /* Special case variants that have a 1 in the core dimensions */ + if (scalar_out) { + /* row @ column, 1,1 output */ + @TYPE@_dot(ip1, is1_n, ip2, is2_n, op, dn, NULL); + } else if (scalar_vec){ + /* + * 1,1d @ vector or vector @ 1,1d + * could use cblas_Xaxy, but that requires 0ing output + * and would not be faster (XXX prove it) + */ + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + } else if (vector_matrix) { + /* vector @ matrix, switch ip1, ip2, p and m */ + @TYPE@_gemv(ip2, is2_p, is2_n, ip1, is1_n, is1_m, + op, os_p, os_m, dp, dn, dm); + } else if (matrix_vector) { + /* matrix @ vector */ + @TYPE@_gemv(ip1, is1_m, is1_n, ip2, is2_n, is2_p, + + op, os_m, os_p, dm, dn, dp); + } else { + /* column @ row, 2d output, no blas needed or non-blas-able input */ + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + } + } else { + /* matrix @ matrix */ + if (i1blasable && i2blasable && o_c_blasable) { + @TYPE@_matmul_matrixmatrix(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, + dm, dn, dp); + } else if (i1blasable && i2blasable && o_f_blasable) { + /* + * Use transpose equivalence: + * matmul(a, b, o) == matmul(b.T, a.T, o.T) + */ + @TYPE@_matmul_matrixmatrix(ip2, is2_p, is2_n, + ip1, is1_n, is1_m, + op, os_p, os_m, + dp, dn, dm); + } else { + /* + * If parameters are castable to int and we copy the + * non-blasable (or non-ccontiguous output) + * we could still use BLAS, see gh-12365. + */ + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + } + } +#else + @TYPE@_matmul_inner_noblas(ip1, is1_m, is1_n, + ip2, is2_n, is2_p, + op, os_m, os_p, dm, dn, dp); + +#endif + } +} + +/**end repeat**/ + + diff --git a/numpy/core/src/umath/matmul.h.src b/numpy/core/src/umath/matmul.h.src new file mode 100644 index 000000000..16be7675b --- /dev/null +++ b/numpy/core/src/umath/matmul.h.src @@ -0,0 +1,12 @@ +/**begin repeat + * #TYPE = FLOAT, DOUBLE, LONGDOUBLE, HALF, + * CFLOAT, CDOUBLE, CLONGDOUBLE, + * UBYTE, USHORT, UINT, ULONG, ULONGLONG, + * BYTE, SHORT, INT, LONG, LONGLONG, + * BOOL# + **/ +NPY_NO_EXPORT void +@TYPE@_matmul(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); +/**end repeat**/ + + diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c index 4a381ba12..2ea23311b 100644 --- a/numpy/core/src/umath/override.c +++ b/numpy/core/src/umath/override.c @@ -5,8 +5,98 @@ #include "numpy/ufuncobject.h" #include "npy_import.h" -#include "ufunc_override.h" #include "override.h" +#include "ufunc_override.h" + +/* + * For each positional argument and each argument in a possible "out" + * keyword, look for overrides of the standard ufunc behaviour, i.e., + * non-default __array_ufunc__ methods. + * + * Returns the number of overrides, setting corresponding objects + * in PyObject array ``with_override`` and the corresponding + * __array_ufunc__ methods in ``methods`` (both using new references). + * + * Only the first override for a given class is returned. + * + * Returns -1 on failure. + */ +static int +get_array_ufunc_overrides(PyObject *args, PyObject *kwds, + PyObject **with_override, PyObject **methods) +{ + int i; + int num_override_args = 0; + int narg, nout = 0; + PyObject *out_kwd_obj; + PyObject **arg_objs, **out_objs; + + narg = PyTuple_Size(args); + if (narg < 0) { + return -1; + } + arg_objs = PySequence_Fast_ITEMS(args); + + nout = PyUFuncOverride_GetOutObjects(kwds, &out_kwd_obj, &out_objs); + if (nout < 0) { + return -1; + } + + for (i = 0; i < narg + nout; ++i) { + PyObject *obj; + int j; + int new_class = 1; + + if (i < narg) { + obj = arg_objs[i]; + } + else { + obj = out_objs[i - narg]; + } + /* + * Have we seen this class before? If so, ignore. + */ + for (j = 0; j < num_override_args; j++) { + new_class = (Py_TYPE(obj) != Py_TYPE(with_override[j])); + if (!new_class) { + break; + } + } + if (new_class) { + /* + * Now see if the object provides an __array_ufunc__. However, we should + * ignore the base ndarray.__ufunc__, so we skip any ndarray as well as + * any ndarray subclass instances that did not override __array_ufunc__. + */ + PyObject *method = PyUFuncOverride_GetNonDefaultArrayUfunc(obj); + if (method == NULL) { + continue; + } + if (method == Py_None) { + PyErr_Format(PyExc_TypeError, + "operand '%.200s' does not support ufuncs " + "(__array_ufunc__=None)", + obj->ob_type->tp_name); + Py_DECREF(method); + goto fail; + } + Py_INCREF(obj); + with_override[num_override_args] = obj; + methods[num_override_args] = method; + ++num_override_args; + } + } + Py_DECREF(out_kwd_obj); + return num_override_args; + +fail: + for (i = 0; i < num_override_args; i++) { + Py_DECREF(with_override[i]); + Py_DECREF(methods[i]); + } + Py_DECREF(out_kwd_obj); + return -1; +} /* * The following functions normalize ufunc arguments. The work done is similar @@ -359,7 +449,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, /* * Check inputs for overrides */ - num_override_args = PyUFunc_WithOverride( + num_override_args = get_array_ufunc_overrides( args, kwds, with_override, array_ufunc_methods); if (num_override_args == -1) { goto fail; diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index e98d9f865..a7987acda 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -1564,7 +1564,6 @@ static PyObject* } /**end repeat**/ - /**begin repeat * #name = byte, ubyte, short, ushort, int, uint, * long, ulong, longlong, ulonglong, @@ -1575,8 +1574,7 @@ static PyNumberMethods @name@_as_number = { (binaryfunc)@name@_add, /*nb_add*/ (binaryfunc)@name@_subtract, /*nb_subtract*/ (binaryfunc)@name@_multiply, /*nb_multiply*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) (binaryfunc)@name@_divide, /*nb_divide*/ #endif (binaryfunc)@name@_remainder, /*nb_remainder*/ @@ -1596,8 +1594,7 @@ static PyNumberMethods @name@_as_number = { (binaryfunc)@name@_and, /*nb_and*/ (binaryfunc)@name@_xor, /*nb_xor*/ (binaryfunc)@name@_or, /*nb_or*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) 0, /*nb_coerce*/ #endif (unaryfunc)@name@_int, /*nb_int*/ @@ -1607,16 +1604,14 @@ static PyNumberMethods @name@_as_number = { (unaryfunc)@name@_long, /*nb_long*/ #endif (unaryfunc)@name@_float, /*nb_float*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) (unaryfunc)@name@_oct, /*nb_oct*/ (unaryfunc)@name@_hex, /*nb_hex*/ #endif 0, /*inplace_add*/ 0, /*inplace_subtract*/ 0, /*inplace_multiply*/ -#if defined(NPY_PY3K) -#else +#if !defined(NPY_PY3K) 0, /*inplace_divide*/ #endif 0, /*inplace_remainder*/ @@ -1631,6 +1626,10 @@ static PyNumberMethods @name@_as_number = { 0, /*nb_inplace_floor_divide*/ 0, /*nb_inplace_true_divide*/ (unaryfunc)NULL, /*nb_index*/ +#if PY_VERSION_HEX >= 0x03050000 + 0, /*nb_matrix_multiply*/ + 0, /*nb_inplace_matrix_multiply*/ +#endif }; /**end repeat**/ diff --git a/numpy/core/src/umath/simd.inc.src b/numpy/core/src/umath/simd.inc.src index da0713b2b..4bb8569be 100644 --- a/numpy/core/src/umath/simd.inc.src +++ b/numpy/core/src/umath/simd.inc.src @@ -32,6 +32,8 @@ #include <float.h> #include <string.h> /* for memcpy */ +#define VECTOR_SIZE_BYTES 16 + static NPY_INLINE npy_uintp abs_ptrdiff(char *a, char *b) { @@ -144,7 +146,7 @@ static NPY_INLINE int run_@name@_simd_@func@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps) { #if @vector@ && defined NPY_HAVE_SSE2_INTRINSICS - if (@check@(sizeof(@type@), 16)) { + if (@check@(sizeof(@type@), VECTOR_SIZE_BYTES)) { sse2_@func@_@TYPE@((@type@*)args[1], (@type@*)args[0], dimensions[0]); return 1; } @@ -182,17 +184,24 @@ run_binary_simd_@kind@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps @type@ * ip2 = (@type@ *)args[1]; @type@ * op = (@type@ *)args[2]; npy_intp n = dimensions[0]; +#if defined __AVX512F__ + const npy_intp vector_size_bytes = 64; +#elif defined __AVX2__ + const npy_intp vector_size_bytes = 32; +#else + const npy_intp vector_size_bytes = 32; +#endif /* argument one scalar */ - if (IS_BLOCKABLE_BINARY_SCALAR1(sizeof(@type@), 16)) { + if (IS_BLOCKABLE_BINARY_SCALAR1(sizeof(@type@), vector_size_bytes)) { sse2_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } /* argument two scalar */ - else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), 16)) { + else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), vector_size_bytes)) { sse2_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } - else if (IS_BLOCKABLE_BINARY(sizeof(@type@), 16)) { + else if (IS_BLOCKABLE_BINARY(sizeof(@type@), vector_size_bytes)) { sse2_binary_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } @@ -232,16 +241,16 @@ run_binary_simd_@kind@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps npy_bool * op = (npy_bool *)args[2]; npy_intp n = dimensions[0]; /* argument one scalar */ - if (IS_BLOCKABLE_BINARY_SCALAR1_BOOL(sizeof(@type@), 16)) { + if (IS_BLOCKABLE_BINARY_SCALAR1_BOOL(sizeof(@type@), VECTOR_SIZE_BYTES)) { sse2_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } /* argument two scalar */ - else if (IS_BLOCKABLE_BINARY_SCALAR2_BOOL(sizeof(@type@), 16)) { + else if (IS_BLOCKABLE_BINARY_SCALAR2_BOOL(sizeof(@type@), VECTOR_SIZE_BYTES)) { sse2_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } - else if (IS_BLOCKABLE_BINARY_BOOL(sizeof(@type@), 16)) { + else if (IS_BLOCKABLE_BINARY_BOOL(sizeof(@type@), VECTOR_SIZE_BYTES)) { sse2_binary_@kind@_@TYPE@(op, ip1, ip2, n); return 1; } @@ -302,7 +311,8 @@ static NPY_INLINE int run_binary_simd_@kind@_BOOL(char **args, npy_intp *dimensions, npy_intp *steps) { #if defined NPY_HAVE_SSE2_INTRINSICS - if (sizeof(npy_bool) == 1 && IS_BLOCKABLE_BINARY(sizeof(npy_bool), 16)) { + if (sizeof(npy_bool) == 1 && + IS_BLOCKABLE_BINARY(sizeof(npy_bool), VECTOR_SIZE_BYTES)) { sse2_binary_@kind@_BOOL((npy_bool*)args[2], (npy_bool*)args[0], (npy_bool*)args[1], dimensions[0]); return 1; @@ -316,7 +326,8 @@ static NPY_INLINE int run_reduce_simd_@kind@_BOOL(char **args, npy_intp *dimensions, npy_intp *steps) { #if defined NPY_HAVE_SSE2_INTRINSICS - if (sizeof(npy_bool) == 1 && IS_BLOCKABLE_REDUCE(sizeof(npy_bool), 16)) { + if (sizeof(npy_bool) == 1 && + IS_BLOCKABLE_REDUCE(sizeof(npy_bool), VECTOR_SIZE_BYTES)) { sse2_reduce_@kind@_BOOL((npy_bool*)args[0], (npy_bool*)args[1], dimensions[0]); return 1; @@ -340,7 +351,8 @@ static NPY_INLINE int run_unary_simd_@kind@_BOOL(char **args, npy_intp *dimensions, npy_intp *steps) { #if defined NPY_HAVE_SSE2_INTRINSICS - if (sizeof(npy_bool) == 1 && IS_BLOCKABLE_UNARY(sizeof(npy_bool), 16)) { + if (sizeof(npy_bool) == 1 && + IS_BLOCKABLE_UNARY(sizeof(npy_bool), VECTOR_SIZE_BYTES)) { sse2_@kind@_BOOL((npy_bool*)args[1], (npy_bool*)args[0], dimensions[0]); return 1; } @@ -416,19 +428,20 @@ static void sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) { #ifdef __AVX512F__ - LOOP_BLOCK_ALIGN_VAR(op, @type@, 64) + const npy_intp vector_size_bytes = 64; + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[i] @OP@ ip2[i]; /* lots of specializations, to squeeze out max performance */ - if (npy_is_aligned(&ip1[i], 64) && npy_is_aligned(&ip2[i], 64)) { + if (npy_is_aligned(&ip1[i], vector_size_bytes) && npy_is_aligned(&ip2[i], vector_size_bytes)) { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, a); @vpre512@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @@ -436,16 +449,16 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } } - else if (npy_is_aligned(&ip1[i], 64)) { - LOOP_BLOCKED(@type@, 64) { + else if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); } } - else if (npy_is_aligned(&ip2[i], 64)) { - LOOP_BLOCKED(@type@, 64) { + else if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @@ -454,14 +467,14 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } else { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, a); @vpre512@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @@ -470,19 +483,21 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } #elif __AVX2__ - LOOP_BLOCK_ALIGN_VAR(op, @type@, 32) + const npy_intp vector_size_bytes = 32; + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[i] @OP@ ip2[i]; /* lots of specializations, to squeeze out max performance */ - if (npy_is_aligned(&ip1[i], 32) && npy_is_aligned(&ip2[i], 32)) { + if (npy_is_aligned(&ip1[i], vector_size_bytes) && + npy_is_aligned(&ip2[i], vector_size_bytes)) { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, a); @vpre256@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @@ -490,16 +505,16 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } } - else if (npy_is_aligned(&ip1[i], 32)) { - LOOP_BLOCKED(@type@, 32) { + else if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); } } - else if (npy_is_aligned(&ip2[i], 32)) { - LOOP_BLOCKED(@type@, 32) { + else if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @@ -508,14 +523,14 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } else { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, a); @vpre256@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @@ -524,19 +539,20 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } #else - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = ip1[i] @OP@ ip2[i]; /* lots of specializations, to squeeze out max performance */ - if (npy_is_aligned(&ip1[i], 16) && npy_is_aligned(&ip2[i], 16)) { + if (npy_is_aligned(&ip1[i], VECTOR_SIZE_BYTES) && + npy_is_aligned(&ip2[i], VECTOR_SIZE_BYTES)) { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a); @vpre@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @@ -544,16 +560,16 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } } } - else if (npy_is_aligned(&ip1[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + else if (npy_is_aligned(&ip1[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); } } - else if (npy_is_aligned(&ip2[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + else if (npy_is_aligned(&ip2[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @@ -562,14 +578,14 @@ sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) } else { if (ip1 == ip2) { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a); @vpre@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @@ -588,18 +604,19 @@ static void sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) { #ifdef __AVX512F__ + const npy_intp vector_size_bytes = 64; const @vtype512@ a = @vpre512@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 64) + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[0] @OP@ ip2[i]; - if (npy_is_aligned(&ip2[i], 64)) { - LOOP_BLOCKED(@type@, 64) { + if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); @@ -608,18 +625,19 @@ sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_i #elif __AVX2__ + const npy_intp vector_size_bytes = 32; const @vtype256@ a = @vpre256@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 32) + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[0] @OP@ ip2[i]; - if (npy_is_aligned(&ip2[i], 32)) { - LOOP_BLOCKED(@type@, 32) { + if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); @@ -627,17 +645,17 @@ sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_i } #else const @vtype@ a = @vpre@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = ip1[0] @OP@ ip2[i]; - if (npy_is_aligned(&ip2[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + if (npy_is_aligned(&ip2[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); @@ -654,18 +672,19 @@ static void sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) { #ifdef __AVX512F__ + const npy_intp vector_size_bytes = 64; const @vtype512@ b = @vpre512@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 64) + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[i] @OP@ ip2[0]; - if (npy_is_aligned(&ip1[i], 64)) { - LOOP_BLOCKED(@type@, 64) { + if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 64) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); @vpre512@_store_@vsuf@(&op[i], c); @@ -673,18 +692,19 @@ sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_i } #elif __AVX2__ + const npy_intp vector_size_bytes = 32; const @vtype256@ b = @vpre256@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 32) + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) op[i] = ip1[i] @OP@ ip2[0]; - if (npy_is_aligned(&ip1[i], 32)) { - LOOP_BLOCKED(@type@, 32) { + if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, vector_size_bytes) { @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); @vpre256@_store_@vsuf@(&op[i], c); @@ -692,17 +712,17 @@ sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_i } #else const @vtype@ b = @vpre@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = ip1[i] @OP@ ip2[0]; - if (npy_is_aligned(&ip1[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + if (npy_is_aligned(&ip1[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); @vpre@_store_@vsuf@(&op[i], c); @@ -742,10 +762,10 @@ sse2_compress4_to_byte_@TYPE@(@vtype@ r1, @vtype@ r2, @vtype@ r3, @vtype@ * r4, static void sse2_signbit_@TYPE@(npy_bool * op, @type@ * ip1, npy_intp n) { - LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { op[i] = npy_signbit(ip1[i]) != 0; } - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); int r = @vpre@_movemask_@vsuf@(a); if (sizeof(@type@) == 8) { @@ -783,14 +803,14 @@ sse2_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, npy_intp n) const @vtype@ fltmax = @vpre@_set1_@vsuf@(FLT_MAX); #endif #endif - LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { op[i] = npy_@kind@(ip1[i]) != 0; } - LOOP_BLOCKED(@type@, 64) { - @vtype@ a = @vpre@_load_@vsuf@(&ip1[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b = @vpre@_load_@vsuf@(&ip1[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c = @vpre@_load_@vsuf@(&ip1[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d = @vpre@_load_@vsuf@(&ip1[i + 3 * 16 / sizeof(@type@)]); + LOOP_BLOCKED(@type@, 4 * VECTOR_SIZE_BYTES) { + @vtype@ a = @vpre@_load_@vsuf@(&ip1[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b = @vpre@_load_@vsuf@(&ip1[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c = @vpre@_load_@vsuf@(&ip1[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d = @vpre@_load_@vsuf@(&ip1[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); @vtype@ r1, r2, r3, r4; #if @var@ != 0 /* isinf/isfinite */ /* fabs via masking of sign bit */ @@ -853,18 +873,18 @@ sse2_ordered_cmp_@kind@_@TYPE@(const @type@ a, const @type@ b) static void sse2_binary_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy_intp n) { - LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[i], ip2[i]); } - LOOP_BLOCKED(@type@, 64) { - @vtype@ a1 = @vpre@_load_@vsuf@(&ip1[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b1 = @vpre@_load_@vsuf@(&ip1[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c1 = @vpre@_load_@vsuf@(&ip1[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d1 = @vpre@_load_@vsuf@(&ip1[i + 3 * 16 / sizeof(@type@)]); - @vtype@ a2 = @vpre@_loadu_@vsuf@(&ip2[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b2 = @vpre@_loadu_@vsuf@(&ip2[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c2 = @vpre@_loadu_@vsuf@(&ip2[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d2 = @vpre@_loadu_@vsuf@(&ip2[i + 3 * 16 / sizeof(@type@)]); + LOOP_BLOCKED(@type@, 4 * VECTOR_SIZE_BYTES) { + @vtype@ a1 = @vpre@_load_@vsuf@(&ip1[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b1 = @vpre@_load_@vsuf@(&ip1[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c1 = @vpre@_load_@vsuf@(&ip1[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d1 = @vpre@_load_@vsuf@(&ip1[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ a2 = @vpre@_loadu_@vsuf@(&ip2[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b2 = @vpre@_loadu_@vsuf@(&ip2[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c2 = @vpre@_loadu_@vsuf@(&ip2[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d2 = @vpre@_loadu_@vsuf@(&ip2[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); @vtype@ r1 = @vpre@_@VOP@_@vsuf@(a1, a2); @vtype@ r2 = @vpre@_@VOP@_@vsuf@(b1, b2); @vtype@ r3 = @vpre@_@VOP@_@vsuf@(c1, c2); @@ -881,14 +901,14 @@ static void sse2_binary_scalar1_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy_intp n) { @vtype@ s = @vpre@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(ip2, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip2, @type@, VECTOR_SIZE_BYTES) { op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[0], ip2[i]); } - LOOP_BLOCKED(@type@, 64) { - @vtype@ a = @vpre@_load_@vsuf@(&ip2[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b = @vpre@_load_@vsuf@(&ip2[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c = @vpre@_load_@vsuf@(&ip2[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d = @vpre@_load_@vsuf@(&ip2[i + 3 * 16 / sizeof(@type@)]); + LOOP_BLOCKED(@type@, 4 * VECTOR_SIZE_BYTES) { + @vtype@ a = @vpre@_load_@vsuf@(&ip2[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b = @vpre@_load_@vsuf@(&ip2[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c = @vpre@_load_@vsuf@(&ip2[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d = @vpre@_load_@vsuf@(&ip2[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); @vtype@ r1 = @vpre@_@VOP@_@vsuf@(s, a); @vtype@ r2 = @vpre@_@VOP@_@vsuf@(s, b); @vtype@ r3 = @vpre@_@VOP@_@vsuf@(s, c); @@ -905,14 +925,14 @@ static void sse2_binary_scalar2_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy_intp n) { @vtype@ s = @vpre@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) { + LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[i], ip2[0]); } - LOOP_BLOCKED(@type@, 64) { - @vtype@ a = @vpre@_load_@vsuf@(&ip1[i + 0 * 16 / sizeof(@type@)]); - @vtype@ b = @vpre@_load_@vsuf@(&ip1[i + 1 * 16 / sizeof(@type@)]); - @vtype@ c = @vpre@_load_@vsuf@(&ip1[i + 2 * 16 / sizeof(@type@)]); - @vtype@ d = @vpre@_load_@vsuf@(&ip1[i + 3 * 16 / sizeof(@type@)]); + LOOP_BLOCKED(@type@, 4 * VECTOR_SIZE_BYTES) { + @vtype@ a = @vpre@_load_@vsuf@(&ip1[i + 0 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ b = @vpre@_load_@vsuf@(&ip1[i + 1 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ c = @vpre@_load_@vsuf@(&ip1[i + 2 * VECTOR_SIZE_BYTES / sizeof(@type@)]); + @vtype@ d = @vpre@_load_@vsuf@(&ip1[i + 3 * VECTOR_SIZE_BYTES / sizeof(@type@)]); @vtype@ r1 = @vpre@_@VOP@_@vsuf@(a, s); @vtype@ r2 = @vpre@_@VOP@_@vsuf@(b, s); @vtype@ r3 = @vpre@_@VOP@_@vsuf@(c, s); @@ -928,19 +948,20 @@ sse2_binary_scalar2_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy static void sse2_sqrt_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n) { - /* align output to 16 bytes */ - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) { + /* align output to VECTOR_SIZE_BYTES bytes */ + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) { op[i] = @scalarf@(ip[i]); } - assert(n < (16 / sizeof(@type@)) || npy_is_aligned(&op[i], 16)); - if (npy_is_aligned(&ip[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + assert(n < (VECTOR_SIZE_BYTES / sizeof(@type@)) || + npy_is_aligned(&op[i], VECTOR_SIZE_BYTES)); + if (npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ d = @vpre@_load_@vsuf@(&ip[i]); @vpre@_store_@vsuf@(&op[i], @vpre@_sqrt_@vsuf@(d)); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ d = @vpre@_loadu_@vsuf@(&ip[i]); @vpre@_store_@vsuf@(&op[i], @vpre@_sqrt_@vsuf@(d)); } @@ -979,19 +1000,20 @@ sse2_@kind@_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n) */ const @vtype@ mask = @vpre@_set1_@vsuf@(-0.@c@); - /* align output to 16 bytes */ - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) { + /* align output to VECTOR_SIZE_BYTES bytes */ + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) { op[i] = @scalar@_@type@(ip[i]); } - assert(n < (16 / sizeof(@type@)) || npy_is_aligned(&op[i], 16)); - if (npy_is_aligned(&ip[i], 16)) { - LOOP_BLOCKED(@type@, 16) { + assert(n < (VECTOR_SIZE_BYTES / sizeof(@type@)) || + npy_is_aligned(&op[i], VECTOR_SIZE_BYTES)); + if (npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_load_@vsuf@(&ip[i]); @vpre@_store_@vsuf@(&op[i], @vpre@_@VOP@_@vsuf@(mask, a)); } } else { - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vpre@_loadu_@vsuf@(&ip[i]); @vpre@_store_@vsuf@(&op[i], @vpre@_@VOP@_@vsuf@(mask, a)); } @@ -1012,11 +1034,12 @@ sse2_@kind@_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n) static void sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n) { - const npy_intp stride = 16 / (npy_intp)sizeof(@type@); - LOOP_BLOCK_ALIGN_VAR(ip, @type@, 16) { - *op = (npy_isnan(*op) || *op @OP@ ip[i]) ? *op : ip[i]; + const npy_intp stride = VECTOR_SIZE_BYTES / (npy_intp)sizeof(@type@); + LOOP_BLOCK_ALIGN_VAR(ip, @type@, VECTOR_SIZE_BYTES) { + /* Order of operations important for MSVC 2015 */ + *op = (*op @OP@ ip[i] || npy_isnan(*op)) ? *op : ip[i]; } - assert(n < (stride) || npy_is_aligned(&ip[i], 16)); + assert(n < (stride) || npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)); if (i + 3 * stride <= n) { /* load the first elements */ @vtype@ c1 = @vpre@_load_@vsuf@((@type@*)&ip[i]); @@ -1025,7 +1048,7 @@ sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n) /* minps/minpd will set invalid flag if nan is encountered */ npy_clear_floatstatus_barrier((char*)&c1); - LOOP_BLOCKED(@type@, 32) { + LOOP_BLOCKED(@type@, 2 * VECTOR_SIZE_BYTES) { @vtype@ v1 = @vpre@_load_@vsuf@((@type@*)&ip[i]); @vtype@ v2 = @vpre@_load_@vsuf@((@type@*)&ip[i + stride]); c1 = @vpre@_@VOP@_@vsuf@(c1, v1); @@ -1038,11 +1061,13 @@ sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n) } else { @type@ tmp = sse2_horizontal_@VOP@_@vtype@(c1); - *op = (npy_isnan(*op) || *op @OP@ tmp) ? *op : tmp; + /* Order of operations important for MSVC 2015 */ + *op = (*op @OP@ tmp || npy_isnan(*op)) ? *op : tmp; } } LOOP_BLOCKED_END { - *op = (npy_isnan(*op) || *op @OP@ ip[i]) ? *op : ip[i]; + /* Order of operations important for MSVC 2015 */ + *op = (*op @OP@ ip[i] || npy_isnan(*op)) ? *op : ip[i]; } npy_clear_floatstatus_barrier((char*)op); } @@ -1090,9 +1115,9 @@ static NPY_INLINE @vtype@ byte_to_true(@vtype@ v) static void sse2_binary_@kind@_BOOL(npy_bool * op, npy_bool * ip1, npy_bool * ip2, npy_intp n) { - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = ip1[i] @op@ ip2[i]; - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vloadu@((@vtype@*)&ip1[i]); @vtype@ b = @vloadu@((@vtype@*)&ip2[i]); #if @and@ @@ -1117,16 +1142,16 @@ static void sse2_reduce_@kind@_BOOL(npy_bool * op, npy_bool * ip, const npy_intp n) { const @vtype@ zero = @vpre@_setzero_@vsuf@(); - LOOP_BLOCK_ALIGN_VAR(ip, npy_bool, 16) { + LOOP_BLOCK_ALIGN_VAR(ip, npy_bool, VECTOR_SIZE_BYTES) { *op = *op @op@ ip[i]; if (*op @sc@ 0) { return; } } /* unrolled once to replace a slow movmsk with a fast pmaxb */ - LOOP_BLOCKED(npy_bool, 32) { + LOOP_BLOCKED(npy_bool, 2 * VECTOR_SIZE_BYTES) { @vtype@ v = @vload@((@vtype@*)&ip[i]); - @vtype@ v2 = @vload@((@vtype@*)&ip[i + 16]); + @vtype@ v2 = @vload@((@vtype@*)&ip[i + VECTOR_SIZE_BYTES]); v = @vpre@_cmpeq_epi8(v, zero); v2 = @vpre@_cmpeq_epi8(v2, zero); #if @and@ @@ -1164,9 +1189,9 @@ sse2_reduce_@kind@_BOOL(npy_bool * op, npy_bool * ip, const npy_intp n) static void sse2_@kind@_BOOL(@type@ * op, @type@ * ip, const npy_intp n) { - LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) + LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) op[i] = (ip[i] @op@ 0); - LOOP_BLOCKED(@type@, 16) { + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @vtype@ a = @vloadu@((@vtype@*)&ip[i]); #if @not@ const @vtype@ zero = @vpre@_setzero_@vsuf@(); @@ -1187,6 +1212,8 @@ sse2_@kind@_BOOL(@type@ * op, @type@ * ip, const npy_intp n) /**end repeat**/ +#undef VECTOR_SIZE_BYTES + #endif /* NPY_HAVE_SSE2_INTRINSICS */ #endif diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 8fb731fb7..ea0007a9d 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -308,6 +308,78 @@ _find_array_prepare(ufunc_full_args args, return; } +#define NPY_UFUNC_DEFAULT_INPUT_FLAGS \ + NPY_ITER_READONLY | \ + NPY_ITER_ALIGNED | \ + NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE + +#define NPY_UFUNC_DEFAULT_OUTPUT_FLAGS \ + NPY_ITER_ALIGNED | \ + NPY_ITER_ALLOCATE | \ + NPY_ITER_NO_BROADCAST | \ + NPY_ITER_NO_SUBTYPE | \ + NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE + +/* Called at module initialization to set the matmul ufunc output flags */ +NPY_NO_EXPORT int +set_matmul_flags(PyObject *d) +{ + PyObject *matmul = PyDict_GetItemString(d, "matmul"); + if (matmul == NULL) { + return -1; + } + /* + * The default output flag NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE allows + * perfectly overlapping input and output (in-place operations). While + * correct for the common mathematical operations, this assumption is + * incorrect in the general case and specifically in the case of matmul. + * + * NPY_ITER_UPDATEIFCOPY is added by default in + * PyUFunc_GeneralizedFunction, which is the variant called for gufuncs + * with a signature + * + * Enabling NPY_ITER_WRITEONLY can prevent a copy in some cases. + */ + ((PyUFuncObject *)matmul)->op_flags[2] = (NPY_ITER_WRITEONLY | + NPY_ITER_UPDATEIFCOPY | + NPY_UFUNC_DEFAULT_OUTPUT_FLAGS) & + ~NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; + return 0; +} + + +/* + * Set per-operand flags according to desired input or output flags. + * op_flags[i] for i in input (as determined by ufunc->nin) will be + * merged with op_in_flags, perhaps overriding per-operand flags set + * in previous stages. + * op_flags[i] for i in output will be set to op_out_flags only if previously + * unset. + * The input flag behavior preserves backward compatibility, while the + * output flag behaviour is the "correct" one for maximum flexibility. + */ +NPY_NO_EXPORT void +_ufunc_setup_flags(PyUFuncObject *ufunc, npy_uint32 op_in_flags, + npy_uint32 op_out_flags, npy_uint32 *op_flags) +{ + int nin = ufunc->nin; + int nout = ufunc->nout; + int nop = nin + nout, i; + /* Set up the flags */ + for (i = 0; i < nin; ++i) { + op_flags[i] = ufunc->op_flags[i] | op_in_flags; + /* + * If READWRITE flag has been set for this operand, + * then clear default READONLY flag + */ + if (op_flags[i] & (NPY_ITER_READWRITE | NPY_ITER_WRITEONLY)) { + op_flags[i] &= ~NPY_ITER_READONLY; + } + } + for (i = nin; i < nop; ++i) { + op_flags[i] = ufunc->op_flags[i] ? ufunc->op_flags[i] : op_out_flags; + } +} /* * This function analyzes the input arguments @@ -1394,11 +1466,11 @@ iterator_loop(PyUFuncObject *ufunc, PyObject **arr_prep, ufunc_full_args full_args, PyUFuncGenericFunction innerloop, - void *innerloopdata) + void *innerloopdata, + npy_uint32 *op_flags) { npy_intp i, nin = ufunc->nin, nout = ufunc->nout; npy_intp nop = nin + nout; - npy_uint32 op_flags[NPY_MAXARGS]; NpyIter *iter; char *baseptrs[NPY_MAXARGS]; @@ -1412,29 +1484,6 @@ iterator_loop(PyUFuncObject *ufunc, NPY_BEGIN_THREADS_DEF; - /* Set up the flags */ - for (i = 0; i < nin; ++i) { - op_flags[i] = NPY_ITER_READONLY | - NPY_ITER_ALIGNED | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - /* - * If READWRITE flag has been set for this operand, - * then clear default READONLY flag - */ - op_flags[i] |= ufunc->op_flags[i]; - if (op_flags[i] & (NPY_ITER_READWRITE | NPY_ITER_WRITEONLY)) { - op_flags[i] &= ~NPY_ITER_READONLY; - } - } - for (i = nin; i < nop; ++i) { - op_flags[i] = NPY_ITER_WRITEONLY | - NPY_ITER_ALIGNED | - NPY_ITER_ALLOCATE | - NPY_ITER_NO_BROADCAST | - NPY_ITER_NO_SUBTYPE | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - } - iter_flags = ufunc->iter_flags | NPY_ITER_EXTERNAL_LOOP | NPY_ITER_REFS_OK | @@ -1538,15 +1587,15 @@ iterator_loop(PyUFuncObject *ufunc, } /* + * ufunc - the ufunc to call * trivial_loop_ok - 1 if no alignment, data conversion, etc required - * nin - number of inputs - * nout - number of outputs - * op - the operands (nin + nout of them) + * op - the operands (ufunc->nin + ufunc->nout of them) + * dtypes - the dtype of each operand * order - the loop execution order/output memory order * buffersize - how big of a buffer to use * arr_prep - the __array_prepare__ functions for the outputs - * innerloop - the inner loop function - * innerloopdata - data to pass to the inner loop + * full_args - the original input, output PyObject * + * op_flags - per-operand flags, a combination of NPY_ITER_* constants */ static int execute_legacy_ufunc_loop(PyUFuncObject *ufunc, @@ -1556,7 +1605,8 @@ execute_legacy_ufunc_loop(PyUFuncObject *ufunc, NPY_ORDER order, npy_intp buffersize, PyObject **arr_prep, - ufunc_full_args full_args) + ufunc_full_args full_args, + npy_uint32 *op_flags) { npy_intp nin = ufunc->nin, nout = ufunc->nout; PyUFuncGenericFunction innerloop; @@ -1691,7 +1741,7 @@ execute_legacy_ufunc_loop(PyUFuncObject *ufunc, NPY_UF_DBG_PRINT("iterator loop\n"); if (iterator_loop(ufunc, op, dtypes, order, buffersize, arr_prep, full_args, - innerloop, innerloopdata) < 0) { + innerloop, innerloopdata, op_flags) < 0) { return -1; } @@ -1717,14 +1767,13 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, NPY_ORDER order, npy_intp buffersize, PyObject **arr_prep, - ufunc_full_args full_args) + ufunc_full_args full_args, + npy_uint32 *op_flags) { int i, nin = ufunc->nin, nout = ufunc->nout; int nop = nin + nout; - npy_uint32 op_flags[NPY_MAXARGS]; NpyIter *iter; int needs_api; - npy_intp default_op_in_flags = 0, default_op_out_flags = 0; NpyIter_IterNextFunc *iternext; char **dataptr; @@ -1734,48 +1783,10 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, PyArrayObject **op_it; npy_uint32 iter_flags; - if (wheremask != NULL) { - if (nop + 1 > NPY_MAXARGS) { - PyErr_SetString(PyExc_ValueError, - "Too many operands when including where= parameter"); - return -1; - } - op[nop] = wheremask; - dtypes[nop] = NULL; - default_op_out_flags |= NPY_ITER_WRITEMASKED; - } - - /* Set up the flags */ - for (i = 0; i < nin; ++i) { - op_flags[i] = default_op_in_flags | - NPY_ITER_READONLY | - NPY_ITER_ALIGNED | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - /* - * If READWRITE flag has been set for this operand, - * then clear default READONLY flag - */ - op_flags[i] |= ufunc->op_flags[i]; - if (op_flags[i] & (NPY_ITER_READWRITE | NPY_ITER_WRITEONLY)) { - op_flags[i] &= ~NPY_ITER_READONLY; - } - } for (i = nin; i < nop; ++i) { - /* - * We don't write to all elements, and the iterator may make - * UPDATEIFCOPY temporary copies. The output arrays (unless they are - * allocated by the iterator itself) must be considered READWRITE by the - * iterator, so that the elements we don't write to are copied to the - * possible temporary array. - */ - op_flags[i] = default_op_out_flags | - (op[i] != NULL ? NPY_ITER_READWRITE : NPY_ITER_WRITEONLY) | - NPY_ITER_ALIGNED | - NPY_ITER_ALLOCATE | - NPY_ITER_NO_BROADCAST | - NPY_ITER_NO_SUBTYPE | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; + op_flags[i] |= (op[i] != NULL ? NPY_ITER_READWRITE : NPY_ITER_WRITEONLY); } + if (wheremask != NULL) { op_flags[nop] = NPY_ITER_READONLY | NPY_ITER_ARRAYMASK; } @@ -2471,6 +2482,11 @@ _get_identity(PyUFuncObject *ufunc, npy_bool *reorderable) { *reorderable = 0; Py_RETURN_NONE; + case PyUFunc_IdentityValue: + *reorderable = 1; + Py_INCREF(ufunc->identity_value); + return ufunc->identity_value; + default: PyErr_Format(PyExc_ValueError, "ufunc %s has an invalid identity", ufunc_get_name_cstr(ufunc)); @@ -2785,6 +2801,18 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, if (retval < 0) { goto fail; } + /* + * We don't write to all elements, and the iterator may make + * UPDATEIFCOPY temporary copies. The output arrays (unless they are + * allocated by the iterator itself) must be considered READWRITE by the + * iterator, so that the elements we don't write to are copied to the + * possible temporary array. + */ + _ufunc_setup_flags(ufunc, NPY_ITER_COPY | NPY_UFUNC_DEFAULT_INPUT_FLAGS, + NPY_ITER_UPDATEIFCOPY | + NPY_ITER_READWRITE | + NPY_UFUNC_DEFAULT_OUTPUT_FLAGS, + op_flags); /* For the generalized ufunc, we get the loop right away too */ retval = ufunc->legacy_inner_loop_selector(ufunc, dtypes, &innerloop, &innerloopdata, &needs_api); @@ -2827,28 +2855,6 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, * Set up the iterator per-op flags. For generalized ufuncs, we * can't do buffering, so must COPY or UPDATEIFCOPY. */ - for (i = 0; i < nin; ++i) { - op_flags[i] = NPY_ITER_READONLY | - NPY_ITER_COPY | - NPY_ITER_ALIGNED | - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - /* - * If READWRITE flag has been set for this operand, - * then clear default READONLY flag - */ - op_flags[i] |= ufunc->op_flags[i]; - if (op_flags[i] & (NPY_ITER_READWRITE | NPY_ITER_WRITEONLY)) { - op_flags[i] &= ~NPY_ITER_READONLY; - } - } - for (i = nin; i < nop; ++i) { - op_flags[i] = NPY_ITER_READWRITE| - NPY_ITER_UPDATEIFCOPY| - NPY_ITER_ALIGNED| - NPY_ITER_ALLOCATE| - NPY_ITER_NO_BROADCAST| - NPY_ITER_OVERLAP_ASSUME_ELEMENTWISE; - } iter_flags = ufunc->iter_flags | NPY_ITER_MULTI_INDEX | @@ -2868,13 +2874,15 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, } /* Fill in any allocated outputs */ - for (i = nin; i < nop; ++i) { - if (op[i] == NULL) { - op[i] = NpyIter_GetOperandArray(iter)[i]; - Py_INCREF(op[i]); + { + PyArrayObject **operands = NpyIter_GetOperandArray(iter); + for (i = 0; i < nop; ++i) { + if (op[i] == NULL) { + op[i] = operands[i]; + Py_INCREF(op[i]); + } } } - /* * Set up the inner strides array. Because we're not doing * buffering, the strides are fixed throughout the looping. @@ -3097,7 +3105,8 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, int i, nop; const char *ufunc_name; int retval = -1, subok = 1; - int need_fancy = 0; + npy_uint32 op_flags[NPY_MAXARGS]; + npy_intp default_op_out_flags; PyArray_Descr *dtypes[NPY_MAXARGS]; @@ -3156,13 +3165,6 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, return retval; } - /* - * Use the masked loop if a wheremask was specified. - */ - if (wheremask != NULL) { - need_fancy = 1; - } - /* Get the buffersize and errormask */ if (_get_bufsize_errmask(extobj, ufunc_name, &buffersize, &errormask) < 0) { retval = -1; @@ -3177,16 +3179,20 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, goto fail; } - /* Only do the trivial loop check for the unmasked version. */ - if (!need_fancy) { - /* - * This checks whether a trivial loop is ok, making copies of - * scalar and one dimensional operands if that will help. - */ - trivial_loop_ok = check_for_trivial_loop(ufunc, op, dtypes, buffersize); - if (trivial_loop_ok < 0) { - goto fail; - } + if (wheremask != NULL) { + /* Set up the flags. */ + default_op_out_flags = NPY_ITER_NO_SUBTYPE | + NPY_ITER_WRITEMASKED | + NPY_UFUNC_DEFAULT_OUTPUT_FLAGS; + _ufunc_setup_flags(ufunc, NPY_UFUNC_DEFAULT_INPUT_FLAGS, + default_op_out_flags, op_flags); + } + else { + /* Set up the flags. */ + default_op_out_flags = NPY_ITER_WRITEONLY | + NPY_UFUNC_DEFAULT_OUTPUT_FLAGS; + _ufunc_setup_flags(ufunc, NPY_UFUNC_DEFAULT_INPUT_FLAGS, + default_op_out_flags, op_flags); } #if NPY_UF_DBG_TRACING @@ -3214,23 +3220,46 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, _find_array_prepare(full_args, arr_prep, nin, nout); } - /* Start with the floating-point exception flags cleared */ - npy_clear_floatstatus_barrier((char*)&ufunc); /* Do the ufunc loop */ - if (need_fancy) { + if (wheremask != NULL) { NPY_UF_DBG_PRINT("Executing fancy inner loop\n"); + if (nop + 1 > NPY_MAXARGS) { + PyErr_SetString(PyExc_ValueError, + "Too many operands when including where= parameter"); + return -1; + } + op[nop] = wheremask; + dtypes[nop] = NULL; + + /* Set up the flags */ + + npy_clear_floatstatus_barrier((char*)&ufunc); retval = execute_fancy_ufunc_loop(ufunc, wheremask, op, dtypes, order, - buffersize, arr_prep, full_args); + buffersize, arr_prep, full_args, op_flags); } else { NPY_UF_DBG_PRINT("Executing legacy inner loop\n"); + /* + * This checks whether a trivial loop is ok, making copies of + * scalar and one dimensional operands if that will help. + * Since it requires dtypes, it can only be called after + * ufunc->type_resolver + */ + trivial_loop_ok = check_for_trivial_loop(ufunc, op, dtypes, buffersize); + if (trivial_loop_ok < 0) { + goto fail; + } + + /* check_for_trivial_loop on half-floats can overflow */ + npy_clear_floatstatus_barrier((char*)&ufunc); + retval = execute_legacy_ufunc_loop(ufunc, trivial_loop_ok, op, dtypes, order, - buffersize, arr_prep, full_args); + buffersize, arr_prep, full_args, op_flags); } if (retval < 0) { goto fail; @@ -4840,6 +4869,20 @@ PyUFunc_FromFuncAndDataAndSignature(PyUFuncGenericFunction *func, void **data, const char *name, const char *doc, int unused, const char *signature) { + return PyUFunc_FromFuncAndDataAndSignatureAndIdentity( + func, data, types, ntypes, nin, nout, identity, name, doc, + unused, signature, NULL); +} + +/*UFUNC_API*/ +NPY_NO_EXPORT PyObject * +PyUFunc_FromFuncAndDataAndSignatureAndIdentity(PyUFuncGenericFunction *func, void **data, + char *types, int ntypes, + int nin, int nout, int identity, + const char *name, const char *doc, + int unused, const char *signature, + PyObject *identity_value) +{ PyUFuncObject *ufunc; if (nin + nout > NPY_MAXARGS) { PyErr_Format(PyExc_ValueError, @@ -4860,6 +4903,10 @@ PyUFunc_FromFuncAndDataAndSignature(PyUFuncGenericFunction *func, void **data, ufunc->nout = nout; ufunc->nargs = nin+nout; ufunc->identity = identity; + if (ufunc->identity == PyUFunc_IdentityValue) { + Py_INCREF(identity_value); + } + ufunc->identity_value = identity_value; ufunc->functions = func; ufunc->data = data; @@ -4881,6 +4928,7 @@ PyUFunc_FromFuncAndDataAndSignature(PyUFuncGenericFunction *func, void **data, ufunc->op_flags = PyArray_malloc(sizeof(npy_uint32)*ufunc->nargs); if (ufunc->op_flags == NULL) { + Py_DECREF(ufunc); return PyErr_NoMemory(); } memset(ufunc->op_flags, 0, sizeof(npy_uint32)*ufunc->nargs); @@ -5237,6 +5285,9 @@ ufunc_dealloc(PyUFuncObject *ufunc) PyArray_free(ufunc->op_flags); Py_XDECREF(ufunc->userloops); Py_XDECREF(ufunc->obj); + if (ufunc->identity == PyUFunc_IdentityValue) { + Py_DECREF(ufunc->identity_value); + } PyArray_free(ufunc); } diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index 6b042d837..ec60d9cfd 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -22,6 +22,11 @@ #include "ufunc_object.h" #include "common.h" +#include "mem_overlap.h" +#if defined(HAVE_CBLAS) +#include "cblasfuncs.h" +#endif + static const char * npy_casting_to_string(NPY_CASTING casting) { @@ -1299,7 +1304,6 @@ PyUFunc_MixedDivisionTypeResolver(PyUFuncObject *ufunc, type_tup, out_dtypes); } - static int find_userloop(PyUFuncObject *ufunc, PyArray_Descr **dtypes, diff --git a/numpy/core/src/umath/ufunc_type_resolution.h b/numpy/core/src/umath/ufunc_type_resolution.h index bb4823d24..2f37af753 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.h +++ b/numpy/core/src/umath/ufunc_type_resolution.h @@ -145,5 +145,4 @@ PyUFunc_DefaultMaskedInnerLoopSelector(PyUFuncObject *ufunc, NpyAuxData **out_innerloopdata, int *out_needs_api); - #endif diff --git a/numpy/core/src/umath/umathmodule.c b/numpy/core/src/umath/umathmodule.c index 20bd2b0a8..5de19fec2 100644 --- a/numpy/core/src/umath/umathmodule.c +++ b/numpy/core/src/umath/umathmodule.c @@ -29,6 +29,7 @@ #include "abstract.h" #include "numpy/npy_math.h" +#include "number.h" static PyUFuncGenericFunction pyfunc_functions[] = {PyUFunc_On_Om}; @@ -267,10 +268,6 @@ int initumath(PyObject *m) UFUNC_FLOATING_POINT_SUPPORT = 0; #endif - /* Initialize the types */ - if (PyType_Ready(&PyUFunc_Type) < 0) - return -1; - /* Add some symbolic constants to the module */ d = PyModule_GetDict(m); @@ -325,7 +322,7 @@ int initumath(PyObject *m) s2 = PyDict_GetItemString(d, "remainder"); /* Setup the array object's numerical structures with appropriate ufuncs in d*/ - PyArray_SetNumericOps(d); + _PyArray_SetNumericOps(d); PyDict_SetItemString(d, "conj", s); PyDict_SetItemString(d, "mod", s2); diff --git a/numpy/core/tests/test_arrayprint.py b/numpy/core/tests/test_arrayprint.py index 6522c6e8a..7a858d2e2 100644 --- a/numpy/core/tests/test_arrayprint.py +++ b/numpy/core/tests/test_arrayprint.py @@ -842,6 +842,10 @@ class TestPrintOptions(object): [[ 0.]]]])""") ) + def test_bad_args(self): + assert_raises(ValueError, np.set_printoptions, threshold='nan') + assert_raises(ValueError, np.set_printoptions, threshold=u'1') + assert_raises(ValueError, np.set_printoptions, threshold=b'1') def test_unicode_object_array(): import sys diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 10ef16800..edb5d5e46 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -13,8 +13,7 @@ import pytest import numpy as np from numpy.testing import ( - assert_raises, assert_warns, assert_no_warnings, assert_array_equal, - assert_ + assert_raises, assert_warns, assert_ ) try: @@ -523,3 +522,14 @@ class TestFromstring(_DeprecationTestCase): # 2017-10-19, 1.14 def test_fromstring(self): self.assert_deprecated(np.fromstring, args=('\x00'*80,)) + +class Test_GetSet_NumericOps(_DeprecationTestCase): + # 2018-09-20, 1.16.0 + def test_get_numeric_ops(self): + from numpy.core._multiarray_tests import getset_numericops + self.assert_deprecated(getset_numericops, num=2) + + # empty kwargs prevents any state actually changing which would break + # other tests. + self.assert_deprecated(np.set_numeric_ops, kwargs={}) + assert_raises(ValueError, np.set_numeric_ops, add='abc') diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index ecb51f72d..c55751e3c 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -156,9 +156,9 @@ class TestRecord(object): the dtype constructor. """ assert_raises(TypeError, np.dtype, - dict(names=set(['A', 'B']), formats=['f8', 'i4'])) + dict(names={'A', 'B'}, formats=['f8', 'i4'])) assert_raises(TypeError, np.dtype, - dict(names=['A', 'B'], formats=set(['f8', 'i4']))) + dict(names=['A', 'B'], formats={'f8', 'i4'})) def test_aligned_size(self): # Check that structured dtypes get padded to an aligned size @@ -807,9 +807,9 @@ class TestFromCTypes(object): p_uint8 = ctypes.POINTER(ctypes.c_uint8) assert_raises(TypeError, np.dtype, p_uint8) - @pytest.mark.xfail( - reason="Unions are not implemented", - raises=NotImplementedError) + def test_void_pointer(self): + self.check(ctypes.c_void_p, np.uintp) + def test_union(self): class Union(ctypes.Union): _fields_ = [ @@ -824,7 +824,52 @@ class TestFromCTypes(object): )) self.check(Union, expected) - @pytest.mark.xfail(reason="_pack_ is ignored - see gh-11651") + def test_union_with_struct_packed(self): + class Struct(ctypes.Structure): + _pack_ = 1 + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + + class Union(ctypes.Union): + _fields_ = [ + ('a', ctypes.c_uint8), + ('b', ctypes.c_uint16), + ('c', ctypes.c_uint32), + ('d', Struct), + ] + expected = np.dtype(dict( + names=['a', 'b', 'c', 'd'], + formats=['u1', np.uint16, np.uint32, [('one', 'u1'), ('two', np.uint32)]], + offsets=[0, 0, 0, 0], + itemsize=ctypes.sizeof(Union) + )) + self.check(Union, expected) + + def test_union_packed(self): + class Struct(ctypes.Structure): + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + _pack_ = 1 + class Union(ctypes.Union): + _pack_ = 1 + _fields_ = [ + ('a', ctypes.c_uint8), + ('b', ctypes.c_uint16), + ('c', ctypes.c_uint32), + ('d', Struct), + ] + expected = np.dtype(dict( + names=['a', 'b', 'c', 'd'], + formats=['u1', np.uint16, np.uint32, [('one', 'u1'), ('two', np.uint32)]], + offsets=[0, 0, 0, 0], + itemsize=ctypes.sizeof(Union) + )) + self.check(Union, expected) + def test_packed_structure(self): class PackedStructure(ctypes.Structure): _pack_ = 1 @@ -838,8 +883,45 @@ class TestFromCTypes(object): ]) self.check(PackedStructure, expected) - @pytest.mark.xfail(sys.byteorder != 'little', - reason="non-native endianness does not work - see gh-10533") + def test_large_packed_structure(self): + class PackedStructure(ctypes.Structure): + _pack_ = 2 + _fields_ = [ + ('a', ctypes.c_uint8), + ('b', ctypes.c_uint16), + ('c', ctypes.c_uint8), + ('d', ctypes.c_uint16), + ('e', ctypes.c_uint32), + ('f', ctypes.c_uint32), + ('g', ctypes.c_uint8) + ] + expected = np.dtype(dict( + formats=[np.uint8, np.uint16, np.uint8, np.uint16, np.uint32, np.uint32, np.uint8 ], + offsets=[0, 2, 4, 6, 8, 12, 16], + names=['a', 'b', 'c', 'd', 'e', 'f', 'g'], + itemsize=18)) + self.check(PackedStructure, expected) + + def test_big_endian_structure_packed(self): + class BigEndStruct(ctypes.BigEndianStructure): + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + _pack_ = 1 + expected = np.dtype([('one', 'u1'), ('two', '>u4')]) + self.check(BigEndStruct, expected) + + def test_little_endian_structure_packed(self): + class LittleEndStruct(ctypes.LittleEndianStructure): + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + _pack_ = 1 + expected = np.dtype([('one', 'u1'), ('two', '<u4')]) + self.check(LittleEndStruct, expected) + def test_little_endian_structure(self): class PaddedStruct(ctypes.LittleEndianStructure): _fields_ = [ @@ -852,8 +934,6 @@ class TestFromCTypes(object): ], align=True) self.check(PaddedStruct, expected) - @pytest.mark.xfail(sys.byteorder != 'big', - reason="non-native endianness does not work - see gh-10533") def test_big_endian_structure(self): class PaddedStruct(ctypes.BigEndianStructure): _fields_ = [ @@ -865,3 +945,9 @@ class TestFromCTypes(object): ('b', '>H') ], align=True) self.check(PaddedStruct, expected) + + def test_simple_endian_types(self): + self.check(ctypes.c_uint16.__ctype_le__, np.dtype('<u2')) + self.check(ctypes.c_uint16.__ctype_be__, np.dtype('>u2')) + self.check(ctypes.c_uint8.__ctype_le__, np.dtype('u1')) + self.check(ctypes.c_uint8.__ctype_be__, np.dtype('u1')) diff --git a/numpy/core/tests/test_einsum.py b/numpy/core/tests/test_einsum.py index 6b5b9c06e..3be4a8a26 100644 --- a/numpy/core/tests/test_einsum.py +++ b/numpy/core/tests/test_einsum.py @@ -11,9 +11,7 @@ from numpy.testing import ( # Setup for optimize einsum chars = 'abcdefghij' sizes = np.array([2, 3, 4, 5, 4, 3, 2, 6, 5, 4, 3]) -global_size_dict = {} -for size, char in zip(sizes, chars): - global_size_dict[char] = size +global_size_dict = dict(zip(chars, sizes)) class TestEinsum(object): diff --git a/numpy/core/tests/test_extint128.py b/numpy/core/tests/test_extint128.py index 0e9c07fd5..7c454a603 100644 --- a/numpy/core/tests/test_extint128.py +++ b/numpy/core/tests/test_extint128.py @@ -1,6 +1,5 @@ from __future__ import division, absolute_import, print_function -import sys import itertools import contextlib import operator @@ -8,7 +7,6 @@ import pytest import numpy as np import numpy.core._multiarray_tests as mt -from numpy.compat import long from numpy.testing import assert_raises, assert_equal diff --git a/numpy/core/tests/test_function_base.py b/numpy/core/tests/test_function_base.py index d0ff1c15f..459bacab0 100644 --- a/numpy/core/tests/test_function_base.py +++ b/numpy/core/tests/test_function_base.py @@ -2,7 +2,7 @@ from __future__ import division, absolute_import, print_function from numpy import ( logspace, linspace, geomspace, dtype, array, sctypes, arange, isnan, - ndarray, sqrt, nextafter + ndarray, sqrt, nextafter, stack ) from numpy.testing import ( assert_, assert_equal, assert_raises, assert_array_equal, assert_allclose, @@ -54,6 +54,20 @@ class TestLogspace(object): y = logspace(0, 6, num=7) assert_array_equal(y, [1, 10, 100, 1e3, 1e4, 1e5, 1e6]) + def test_start_stop_array(self): + start = array([0., 1.]) + stop = array([6., 7.]) + t1 = logspace(start, stop, 6) + t2 = stack([logspace(_start, _stop, 6) + for _start, _stop in zip(start, stop)], axis=1) + assert_equal(t1, t2) + t3 = logspace(start, stop[0], 6) + t4 = stack([logspace(_start, stop[0], 6) + for _start in start], axis=1) + assert_equal(t3, t4) + t5 = logspace(start, stop, 6, axis=-1) + assert_equal(t5, t2.T) + def test_dtype(self): y = logspace(0, 6, dtype='float32') assert_equal(y.dtype, dtype('float32')) @@ -156,7 +170,7 @@ class TestGeomspace(object): y = geomspace(1, 1e6, dtype=complex) assert_equal(y.dtype, dtype('complex')) - def test_array_scalar(self): + def test_start_stop_array_scalar(self): lim1 = array([120, 100], dtype="int8") lim2 = array([-120, -100], dtype="int8") lim3 = array([1200, 1000], dtype="uint16") @@ -172,6 +186,21 @@ class TestGeomspace(object): assert_allclose(t2, t5, rtol=1e-2) assert_allclose(t3, t6, rtol=1e-5) + def test_start_stop_array(self): + # Try to use all special cases. + start = array([1.e0, 32., 1j, -4j, 1+1j, -1]) + stop = array([1.e4, 2., 16j, -324j, 10000+10000j, 1]) + t1 = geomspace(start, stop, 5) + t2 = stack([geomspace(_start, _stop, 5) + for _start, _stop in zip(start, stop)], axis=1) + assert_equal(t1, t2) + t3 = geomspace(start, stop[0], 5) + t4 = stack([geomspace(_start, stop[0], 5) + for _start in start], axis=1) + assert_equal(t3, t4) + t5 = geomspace(start, stop, 5, axis=-1) + assert_equal(t5, t2.T) + def test_physical_quantities(self): a = PhysicalQuantity(1.0) b = PhysicalQuantity(5.0) @@ -227,7 +256,7 @@ class TestLinspace(object): y = linspace(0, 6, dtype='int32') assert_equal(y.dtype, dtype('int32')) - def test_array_scalar(self): + def test_start_stop_array_scalar(self): lim1 = array([-120, 100], dtype="int8") lim2 = array([120, -100], dtype="int8") lim3 = array([1200, 1000], dtype="uint16") @@ -241,6 +270,20 @@ class TestLinspace(object): assert_equal(t2, t5) assert_equal(t3, t6) + def test_start_stop_array(self): + start = array([-120, 120], dtype="int8") + stop = array([100, -100], dtype="int8") + t1 = linspace(start, stop, 5) + t2 = stack([linspace(_start, _stop, 5) + for _start, _stop in zip(start, stop)], axis=1) + assert_equal(t1, t2) + t3 = linspace(start, stop[0], 5) + t4 = stack([linspace(_start, stop[0], 5) + for _start in start], axis=1) + assert_equal(t3, t4) + t5 = linspace(start, stop, 5, axis=-1) + assert_equal(t5, t2.T) + def test_complex(self): lim1 = linspace(1 + 2j, 3 + 4j, 5) t1 = array([1.0+2.j, 1.5+2.5j, 2.0+3j, 2.5+3.5j, 3.0+4j]) @@ -285,9 +328,7 @@ class TestLinspace(object): @property def __array_interface__(self): - # Ideally should be `'shape': ()` but the current interface - # does not allow that - return {'shape': (1,), 'typestr': '<i4', 'data': self._data, + return {'shape': (), 'typestr': '<i4', 'data': self._data, 'version': 3} def __mul__(self, other): diff --git a/numpy/core/tests/test_half.py b/numpy/core/tests/test_half.py index 2f2d069e5..b28c933db 100644 --- a/numpy/core/tests/test_half.py +++ b/numpy/core/tests/test_half.py @@ -5,7 +5,7 @@ import pytest import numpy as np from numpy import uint16, float16, float32, float64 -from numpy.testing import assert_, assert_equal, suppress_warnings +from numpy.testing import assert_, assert_equal def assert_raises_fpe(strmatch, callable, *args, **kwargs): diff --git a/numpy/core/tests/test_mem_overlap.py b/numpy/core/tests/test_mem_overlap.py index f4ce6a84a..3c8e0e722 100644 --- a/numpy/core/tests/test_mem_overlap.py +++ b/numpy/core/tests/test_mem_overlap.py @@ -10,7 +10,7 @@ from numpy.core import _umath_tests from numpy.lib.stride_tricks import as_strided from numpy.compat import long from numpy.testing import ( - assert_, assert_raises, assert_equal, assert_array_equal, assert_allclose + assert_, assert_raises, assert_equal, assert_array_equal ) if sys.version_info[0] >= 3: diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 4b2a38990..951c01c6d 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -92,6 +92,46 @@ class TestFlags(object): self.a[0] = 5 self.a[0] = 0 + def test_writeable_from_readonly(self): + # gh-9440 - make sure fromstring, from buffer on readonly buffers + # set writeable False + data = b'\x00' * 100 + vals = np.frombuffer(data, 'B') + assert_raises(ValueError, vals.setflags, write=True) + types = np.dtype( [('vals', 'u1'), ('res3', 'S4')] ) + values = np.core.records.fromstring(data, types) + vals = values['vals'] + assert_raises(ValueError, vals.setflags, write=True) + + def test_writeable_from_buffer(self): + data = bytearray(b'\x00' * 100) + vals = np.frombuffer(data, 'B') + assert_(vals.flags.writeable) + vals.setflags(write=False) + assert_(vals.flags.writeable is False) + vals.setflags(write=True) + assert_(vals.flags.writeable) + types = np.dtype( [('vals', 'u1'), ('res3', 'S4')] ) + values = np.core.records.fromstring(data, types) + vals = values['vals'] + assert_(vals.flags.writeable) + vals.setflags(write=False) + assert_(vals.flags.writeable is False) + vals.setflags(write=True) + assert_(vals.flags.writeable) + + @pytest.mark.skipif(sys.version_info[0] < 3, reason="Python 2 always copies") + def test_writeable_pickle(self): + import pickle + # Small arrays will be copied without setting base. + # See condition for using PyArray_SetBaseObject in + # array_setstate. + a = np.arange(1000) + for v in range(pickle.HIGHEST_PROTOCOL): + vals = pickle.loads(pickle.dumps(a, v)) + assert_(vals.flags.writeable) + assert_(isinstance(vals.base, bytes)) + def test_otherflags(self): assert_equal(self.a.flags.carray, True) assert_equal(self.a.flags['C'], True) @@ -2595,7 +2635,8 @@ class TestMethods(object): assert_equal(x1.flatten('F'), y1f) assert_equal(x1.flatten('F'), x1.T.flatten()) - def test_dot(self): + @pytest.mark.parametrize('func', (np.dot, np.matmul)) + def test_arr_mult(self, func): a = np.array([[1, 0], [0, 1]]) b = np.array([[0, 1], [1, 0]]) c = np.array([[9, 1], [1, -9]]) @@ -2619,49 +2660,49 @@ class TestMethods(object): # gemm vs syrk optimizations for et in [np.float32, np.float64, np.complex64, np.complex128]: eaf = a.astype(et) - assert_equal(np.dot(eaf, eaf), eaf) - assert_equal(np.dot(eaf.T, eaf), eaf) - assert_equal(np.dot(eaf, eaf.T), eaf) - assert_equal(np.dot(eaf.T, eaf.T), eaf) - assert_equal(np.dot(eaf.T.copy(), eaf), eaf) - assert_equal(np.dot(eaf, eaf.T.copy()), eaf) - assert_equal(np.dot(eaf.T.copy(), eaf.T.copy()), eaf) + assert_equal(func(eaf, eaf), eaf) + assert_equal(func(eaf.T, eaf), eaf) + assert_equal(func(eaf, eaf.T), eaf) + assert_equal(func(eaf.T, eaf.T), eaf) + assert_equal(func(eaf.T.copy(), eaf), eaf) + assert_equal(func(eaf, eaf.T.copy()), eaf) + assert_equal(func(eaf.T.copy(), eaf.T.copy()), eaf) # syrk validations for et in [np.float32, np.float64, np.complex64, np.complex128]: eaf = a.astype(et) ebf = b.astype(et) - assert_equal(np.dot(ebf, ebf), eaf) - assert_equal(np.dot(ebf.T, ebf), eaf) - assert_equal(np.dot(ebf, ebf.T), eaf) - assert_equal(np.dot(ebf.T, ebf.T), eaf) + assert_equal(func(ebf, ebf), eaf) + assert_equal(func(ebf.T, ebf), eaf) + assert_equal(func(ebf, ebf.T), eaf) + assert_equal(func(ebf.T, ebf.T), eaf) # syrk - different shape, stride, and view validations for et in [np.float32, np.float64, np.complex64, np.complex128]: edf = d.astype(et) assert_equal( - np.dot(edf[::-1, :], edf.T), - np.dot(edf[::-1, :].copy(), edf.T.copy()) + func(edf[::-1, :], edf.T), + func(edf[::-1, :].copy(), edf.T.copy()) ) assert_equal( - np.dot(edf[:, ::-1], edf.T), - np.dot(edf[:, ::-1].copy(), edf.T.copy()) + func(edf[:, ::-1], edf.T), + func(edf[:, ::-1].copy(), edf.T.copy()) ) assert_equal( - np.dot(edf, edf[::-1, :].T), - np.dot(edf, edf[::-1, :].T.copy()) + func(edf, edf[::-1, :].T), + func(edf, edf[::-1, :].T.copy()) ) assert_equal( - np.dot(edf, edf[:, ::-1].T), - np.dot(edf, edf[:, ::-1].T.copy()) + func(edf, edf[:, ::-1].T), + func(edf, edf[:, ::-1].T.copy()) ) assert_equal( - np.dot(edf[:edf.shape[0] // 2, :], edf[::2, :].T), - np.dot(edf[:edf.shape[0] // 2, :].copy(), edf[::2, :].T.copy()) + func(edf[:edf.shape[0] // 2, :], edf[::2, :].T), + func(edf[:edf.shape[0] // 2, :].copy(), edf[::2, :].T.copy()) ) assert_equal( - np.dot(edf[::2, :], edf[:edf.shape[0] // 2, :].T), - np.dot(edf[::2, :].copy(), edf[:edf.shape[0] // 2, :].T.copy()) + func(edf[::2, :], edf[:edf.shape[0] // 2, :].T), + func(edf[::2, :].copy(), edf[:edf.shape[0] // 2, :].T.copy()) ) # syrk - different shape @@ -2669,9 +2710,43 @@ class TestMethods(object): edf = d.astype(et) eddtf = ddt.astype(et) edtdf = dtd.astype(et) - assert_equal(np.dot(edf, edf.T), eddtf) - assert_equal(np.dot(edf.T, edf), edtdf) + assert_equal(func(edf, edf.T), eddtf) + assert_equal(func(edf.T, edf), edtdf) + + @pytest.mark.parametrize('func', (np.dot, np.matmul)) + @pytest.mark.parametrize('dtype', 'ifdFD') + def test_no_dgemv(self, func, dtype): + # check vector arg for contiguous before gemv + # gh-12156 + a = np.arange(8.0, dtype=dtype).reshape(2, 4) + b = np.broadcast_to(1., (4, 1)) + ret1 = func(a, b) + ret2 = func(a, b.copy()) + assert_equal(ret1, ret2) + + ret1 = func(b.T, a.T) + ret2 = func(b.T.copy(), a.T) + assert_equal(ret1, ret2) + + # check for unaligned data + dt = np.dtype(dtype) + a = np.zeros(8 * dt.itemsize // 2 + 1, dtype='int16')[1:].view(dtype) + a = a.reshape(2, 4) + b = a[0] + # make sure it is not aligned + assert_(a.__array_interface__['data'][0] % dt.itemsize != 0) + ret1 = func(a, b) + ret2 = func(a.copy(), b.copy()) + assert_equal(ret1, ret2) + + ret1 = func(b.T, a.T) + ret2 = func(b.T.copy(), a.T.copy()) + assert_equal(ret1, ret2) + def test_dot(self): + a = np.array([[1, 0], [0, 1]]) + b = np.array([[0, 1], [1, 0]]) + c = np.array([[9, 1], [1, -9]]) # function versus methods assert_equal(np.dot(a, b), a.dot(b)) assert_equal(np.dot(np.dot(a, b), c), a.dot(b).dot(c)) @@ -2727,6 +2802,29 @@ class TestMethods(object): np.dot(a, b, out=out) np.matmul(a, b, out=out) + def test_dot_matmul_inner_array_casting_fails(self): + + class A(object): + def __array__(self, *args, **kwargs): + raise NotImplementedError + + # Don't override the error from calling __array__() + assert_raises(NotImplementedError, np.dot, A(), A()) + assert_raises(NotImplementedError, np.matmul, A(), A()) + assert_raises(NotImplementedError, np.inner, A(), A()) + + def test_matmul_out(self): + # overlapping memory + a = np.arange(18).reshape(2, 3, 3) + b = np.matmul(a, a) + c = np.matmul(a, a, out=a) + assert_(c is a) + assert_equal(c, b) + a = np.arange(18).reshape(2, 3, 3) + c = np.matmul(a, a, out=a[::-1, ...]) + assert_(c.base is a.base) + assert_equal(c, b) + def test_diagonal(self): a = np.arange(12).reshape((3, 4)) assert_equal(a.diagonal(), [0, 5, 10]) @@ -3136,6 +3234,8 @@ class TestBinop(object): # 'eq': (np.equal, False), # 'ne': (np.not_equal, False), } + if sys.version_info >= (3, 5): + ops['matmul'] = (np.matmul, False, float) class Coerced(Exception): pass @@ -3178,7 +3278,7 @@ class TestBinop(object): if issubclass(MyType, np.ndarray): # Use this range to avoid special case weirdnesses around # divide-by-0, pow(x, 2), overflow due to pow(big, big), etc. - return np.arange(3, 5).view(MyType) + return np.arange(3, 7).reshape(2, 2).view(MyType) else: return MyType() @@ -3187,7 +3287,7 @@ class TestBinop(object): for op, (ufunc, has_inplace, dtype) in ops.items(): err_msg = ('op: %s, ufunc: %s, has_inplace: %s, dtype: %s' % (op, ufunc, has_inplace, dtype)) - check_objs = [np.arange(3, 5, dtype=dtype)] + check_objs = [np.arange(3, 7, dtype=dtype).reshape(2, 2)] if check_scalar: check_objs.append(check_objs[0][0]) for arr in check_objs: @@ -3582,7 +3682,6 @@ class TestPickling(object): 'protocol 5 although it is not available')) def test_correct_protocol5_error_message(self): array = np.arange(10) - f = io.BytesIO() if sys.version_info[:2] in ((3, 6), (3, 7)): # For the specific case of python3.6 and 3.7, raise a clear import @@ -4541,6 +4640,19 @@ class TestIO(object): f.close() assert_equal(pos, 10, err_msg=err_msg) + def test_load_object_array_fromfile(self): + # gh-12300 + with open(self.filename, 'w') as f: + # Ensure we have a file with consistent contents + pass + + with open(self.filename, 'rb') as f: + assert_raises_regex(ValueError, "Cannot read into object array", + np.fromfile, f, dtype=object) + + assert_raises_regex(ValueError, "Cannot read into object array", + np.fromfile, self.filename, dtype=object) + def _check_from(self, s, value, **kw): if 'sep' not in kw: y = np.frombuffer(s, **kw) @@ -4843,11 +4955,11 @@ class TestRecord(object): assert_equal(dt.names, ['p', 'q']) def test_multiple_field_name_occurrence(self): - def test_assign(): - dtype = np.dtype([("A", "f8"), ("B", "f8"), ("A", "f8")]) + def test_dtype_init(): + np.dtype([("A", "f8"), ("B", "f8"), ("A", "f8")]) # Error raised when multiple fields have the same name - assert_raises(ValueError, test_assign) + assert_raises(ValueError, test_dtype_init) @pytest.mark.skipif(sys.version_info[0] < 3, reason="Not Python 3") def test_bytes_fields(self): @@ -4867,13 +4979,11 @@ class TestRecord(object): @pytest.mark.skipif(sys.version_info[0] < 3, reason="Not Python 3") def test_multiple_field_name_unicode(self): - def test_assign_unicode(): - dt = np.dtype([("\u20B9", "f8"), - ("B", "f8"), - ("\u20B9", "f8")]) + def test_dtype_unicode(): + np.dtype([("\u20B9", "f8"), ("B", "f8"), ("\u20B9", "f8")]) # Error raised when multiple fields have the same name(unicode included) - assert_raises(ValueError, test_assign_unicode) + assert_raises(ValueError, test_dtype_unicode) @pytest.mark.skipif(sys.version_info[0] >= 3, reason="Not Python 2") def test_unicode_field_titles(self): @@ -4955,25 +5065,9 @@ class TestRecord(object): fn2 = func('f2') b[fn2] = 3 - # In 1.16 code below can be replaced by: - # assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3)) - # assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2)) - # assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,))) - with suppress_warnings() as sup: - sup.filter(FutureWarning, - ".* selecting multiple fields .*") - - assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3)) - assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2)) - assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,))) - # view of subfield view/copy - assert_equal(b[['f1', 'f2']][0].view(('i4', 2)).tolist(), - (2, 3)) - assert_equal(b[['f2', 'f1']][0].view(('i4', 2)).tolist(), - (3, 2)) - view_dtype = [('f1', 'i4'), ('f3', [('', 'i4')])] - assert_equal(b[['f1', 'f3']][0].view(view_dtype).tolist(), - (2, (1,))) + assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3)) + assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2)) + assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,))) # non-ascii unicode field indexing is well behaved if not is_py3: @@ -4983,50 +5077,6 @@ class TestRecord(object): assert_raises(ValueError, a.__setitem__, u'\u03e0', 1) assert_raises(ValueError, a.__getitem__, u'\u03e0') - # can be removed in 1.16 - def test_field_names_deprecation(self): - - def collect_warnings(f, *args, **kwargs): - with warnings.catch_warnings(record=True) as log: - warnings.simplefilter("always") - f(*args, **kwargs) - return [w.category for w in log] - - a = np.zeros((1,), dtype=[('f1', 'i4'), - ('f2', 'i4'), - ('f3', [('sf1', 'i4')])]) - a['f1'][0] = 1 - a['f2'][0] = 2 - a['f3'][0] = (3,) - b = np.zeros((1,), dtype=[('f1', 'i4'), - ('f2', 'i4'), - ('f3', [('sf1', 'i4')])]) - b['f1'][0] = 1 - b['f2'][0] = 2 - b['f3'][0] = (3,) - - # All the different functions raise a warning, but not an error - assert_equal(collect_warnings(a[['f1', 'f2']].__setitem__, 0, (10, 20)), - [FutureWarning]) - # For <=1.12 a is not modified, but it will be in 1.13 - assert_equal(a, b) - - # Views also warn - subset = a[['f1', 'f2']] - subset_view = subset.view() - assert_equal(collect_warnings(subset_view['f1'].__setitem__, 0, 10), - [FutureWarning]) - # But the write goes through: - assert_equal(subset['f1'][0], 10) - # Only one warning per multiple field indexing, though (even if there - # are multiple views involved): - assert_equal(collect_warnings(subset['f1'].__setitem__, 0, 10), []) - - # make sure views of a multi-field index warn too - c = np.zeros(3, dtype='i8,i8,i8') - assert_equal(collect_warnings(c[['f0', 'f2']].view, 'i8,i8'), - [FutureWarning]) - def test_record_hash(self): a = np.array([(1, 2), (1, 2)], dtype='i1,i2') a.flags.writeable = False @@ -5050,6 +5100,16 @@ class TestRecord(object): np.array([(), (), (), (), ()], dtype={'names': [], 'formats': [], 'offsets': [], 'itemsize': 12}) + def test_multifield_indexing_view(self): + a = np.ones(3, dtype=[('a', 'i4'), ('b', 'f4'), ('c', 'u4')]) + v = a[['a', 'c']] + assert_(v.base is a) + assert_(v.dtype == np.dtype({'names': ['a', 'c'], + 'formats': ['i4', 'u4'], + 'offsets': [0, 8]})) + v[:] = (4,5) + assert_equal(a[0].item(), (4, 1, 5)) + class TestView(object): def test_basic(self): x = np.array([(1, 2, 3, 4), (5, 6, 7, 8)], @@ -5673,15 +5733,38 @@ class MatmulCommon(object): res = self.matmul(v, v) assert_(type(res) is np.dtype(dt).type) - def test_vector_vector_values(self): - vec = np.array([1, 2]) - tgt = 5 + def test_scalar_output(self): + vec1 = np.array([2]) + vec2 = np.array([3, 4]).reshape(1, -1) + tgt = np.array([6, 8]) for dt in self.types[1:]: - v1 = vec.astype(dt) - res = self.matmul(v1, v1) + v1 = vec1.astype(dt) + v2 = vec2.astype(dt) + res = self.matmul(v1, v2) + assert_equal(res, tgt) + res = self.matmul(v2.T, v1) assert_equal(res, tgt) # boolean type + vec = np.array([True, True], dtype='?').reshape(1, -1) + res = self.matmul(vec[:, 0], vec) + assert_equal(res, True) + + def test_vector_vector_values(self): + vec1 = np.array([1, 2]) + vec2 = np.array([3, 4]).reshape(-1, 1) + tgt1 = np.array([11]) + tgt2 = np.array([[3, 6], [4, 8]]) + for dt in self.types[1:]: + v1 = vec1.astype(dt) + v2 = vec2.astype(dt) + res = self.matmul(v1, v2) + assert_equal(res, tgt1) + # no broadcast, we must make v1 into a 2d ndarray + res = self.matmul(v2, v1.reshape(1, -1)) + assert_equal(res, tgt2) + + # boolean type vec = np.array([True, True], dtype='?') res = self.matmul(vec, vec) assert_equal(res, True) @@ -5810,44 +5893,96 @@ class TestMatmul(MatmulCommon): matmul = np.matmul def test_out_arg(self): - a = np.ones((2, 2), dtype=float) - b = np.ones((2, 2), dtype=float) - tgt = np.full((2,2), 2, dtype=float) + a = np.ones((5, 2), dtype=float) + b = np.array([[1, 3], [5, 7]], dtype=float) + tgt = np.dot(a, b) # test as positional argument msg = "out positional argument" - out = np.zeros((2, 2), dtype=float) + out = np.zeros((5, 2), dtype=float) self.matmul(a, b, out) assert_array_equal(out, tgt, err_msg=msg) # test as keyword argument msg = "out keyword argument" - out = np.zeros((2, 2), dtype=float) + out = np.zeros((5, 2), dtype=float) self.matmul(a, b, out=out) assert_array_equal(out, tgt, err_msg=msg) # test out with not allowed type cast (safe casting) - # einsum and cblas raise different error types, so - # use Exception. - msg = "out argument with illegal cast" - out = np.zeros((2, 2), dtype=np.int32) - assert_raises(Exception, self.matmul, a, b, out=out) - - # skip following tests for now, cblas does not allow non-contiguous - # outputs and consistency with dot would require same type, - # dimensions, subtype, and c_contiguous. - - # test out with allowed type cast - # msg = "out argument with allowed cast" - # out = np.zeros((2, 2), dtype=np.complex128) - # self.matmul(a, b, out=out) - # assert_array_equal(out, tgt, err_msg=msg) + msg = "Cannot cast ufunc matmul output" + out = np.zeros((5, 2), dtype=np.int32) + assert_raises_regex(TypeError, msg, self.matmul, a, b, out=out) + + # test out with type upcast to complex + out = np.zeros((5, 2), dtype=np.complex128) + c = self.matmul(a, b, out=out) + assert_(c is out) + with suppress_warnings() as sup: + sup.filter(np.ComplexWarning, '') + c = c.astype(tgt.dtype) + assert_array_equal(c, tgt) + + def test_out_contiguous(self): + a = np.ones((5, 2), dtype=float) + b = np.array([[1, 3], [5, 7]], dtype=float) + v = np.array([1, 3], dtype=float) + tgt = np.dot(a, b) + tgt_mv = np.dot(a, v) # test out non-contiguous - # msg = "out argument with non-contiguous layout" - # c = np.zeros((2, 2, 2), dtype=float) - # self.matmul(a, b, out=c[..., 0]) - # assert_array_equal(c, tgt, err_msg=msg) + out = np.ones((5, 2, 2), dtype=float) + c = self.matmul(a, b, out=out[..., 0]) + assert c.base is out + assert_array_equal(c, tgt) + c = self.matmul(a, v, out=out[:, 0, 0]) + assert_array_equal(c, tgt_mv) + c = self.matmul(v, a.T, out=out[:, 0, 0]) + assert_array_equal(c, tgt_mv) + + # test out contiguous in only last dim + out = np.ones((10, 2), dtype=float) + c = self.matmul(a, b, out=out[::2, :]) + assert_array_equal(c, tgt) + + # test transposes of out, args + out = np.ones((5, 2), dtype=float) + c = self.matmul(b.T, a.T, out=out.T) + assert_array_equal(out, tgt) + + m1 = np.arange(15.).reshape(5, 3) + m2 = np.arange(21.).reshape(3, 7) + m3 = np.arange(30.).reshape(5, 6)[:, ::2] # non-contiguous + vc = np.arange(10.) + vr = np.arange(6.) + m0 = np.zeros((3, 0)) + @pytest.mark.parametrize('args', ( + # matrix-matrix + (m1, m2), (m2.T, m1.T), (m2.T.copy(), m1.T), (m2.T, m1.T.copy()), + # matrix-matrix-transpose, contiguous and non + (m1, m1.T), (m1.T, m1), (m1, m3.T), (m3, m1.T), + (m3, m3.T), (m3.T, m3), + # matrix-matrix non-contiguous + (m3, m2), (m2.T, m3.T), (m2.T.copy(), m3.T), + # vector-matrix, matrix-vector, contiguous + (m1, vr[:3]), (vc[:5], m1), (m1.T, vc[:5]), (vr[:3], m1.T), + # vector-matrix, matrix-vector, vector non-contiguous + (m1, vr[::2]), (vc[::2], m1), (m1.T, vc[::2]), (vr[::2], m1.T), + # vector-matrix, matrix-vector, matrix non-contiguous + (m3, vr[:3]), (vc[:5], m3), (m3.T, vc[:5]), (vr[:3], m3.T), + # vector-matrix, matrix-vector, both non-contiguous + (m3, vr[::2]), (vc[::2], m3), (m3.T, vc[::2]), (vr[::2], m3.T), + # size == 0 + (m0, m0.T), (m0.T, m0), (m1, m0), (m0.T, m1.T), + )) + def test_dot_equivalent(self, args): + r1 = np.matmul(*args) + r2 = np.dot(*args) + assert_equal(r1, r2) + + r3 = np.matmul(args[0].copy(), args[1].copy()) + assert_equal(r1, r3) + if sys.version_info[:2] >= (3, 5): @@ -5871,6 +6006,11 @@ if sys.version_info[:2] >= (3, 5): assert_equal(self.matmul(a, b), "A") assert_equal(self.matmul(b, a), "A") + def test_matmul_raises(self): + assert_raises(TypeError, self.matmul, np.int8(5), np.int8(5)) + assert_raises(TypeError, self.matmul, np.void(b'abc'), np.void(b'abc')) + assert_raises(ValueError, self.matmul, np.arange(10), np.void(b'abc')) + def test_matmul_inplace(): # It would be nice to support in-place matmul eventually, but for now # we don't have a working implementation, so better just to error out @@ -5885,6 +6025,17 @@ if sys.version_info[:2] >= (3, 5): exec_ = getattr(builtins, "exec") assert_raises(TypeError, exec_, "a @= b", globals(), locals()) + def test_matmul_axes(): + a = np.arange(3*4*5).reshape(3, 4, 5) + c = np.matmul(a, a, axes=[(-2, -1), (-1, -2), (1, 2)]) + assert c.shape == (3, 4, 4) + d = np.matmul(a, a, axes=[(-2, -1), (-1, -2), (0, 1)]) + assert d.shape == (4, 4, 3) + e = np.swapaxes(d, 0, 2) + assert_array_equal(e, c) + f = np.matmul(a, np.arange(3), axes=[(1, 0), (0), (0)]) + assert f.shape == (4, 5) + class TestInner(object): @@ -6775,7 +6926,7 @@ class TestNewBufferProtocol(object): ValueError, "format string", np.array, m) - def test_error_message(self): + def test_error_message_unsupported(self): # wchar has no corresponding numpy type - if this changes in future, we # need a better way to construct an invalid memoryview format. t = ctypes.c_wchar * 4 @@ -6784,14 +6935,17 @@ class TestNewBufferProtocol(object): exc = cm.exception if sys.version_info.major > 2: - with assert_raises_regex(ValueError, "Unknown .* specifier 'u'"): + with assert_raises_regex( + NotImplementedError, + r"Unrepresentable .* 'u' \(UCS-2 strings\)" + ): raise exc.__cause__ def test_ctypes_integer_via_memoryview(self): # gh-11150, due to bpo-10746 for c_integer in {ctypes.c_int, ctypes.c_long, ctypes.c_longlong}: value = c_integer(42) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(record=True): warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning) np.asarray(value) @@ -6801,7 +6955,7 @@ class TestNewBufferProtocol(object): _fields_ = [('a', ctypes.c_uint8), ('b', ctypes.c_uint32)] f = foo(a=1, b=2) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(record=True): warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning) arr = np.asarray(f) @@ -7472,6 +7626,55 @@ class TestCTypes(object): finally: _internal.ctypes = ctypes + def _make_readonly(x): + x.flags.writeable = False + return x + + @pytest.mark.parametrize('arr', [ + np.array([1, 2, 3]), + np.array([['one', 'two'], ['three', 'four']]), + np.array((1, 2), dtype='i4,i4'), + np.zeros((2,), dtype= + np.dtype(dict( + formats=['<i4', '<i4'], + names=['a', 'b'], + offsets=[0, 2], + itemsize=6 + )) + ), + np.array([None], dtype=object), + np.array([]), + np.empty((0, 0)), + _make_readonly(np.array([1, 2, 3])), + ], ids=[ + '1d', + '2d', + 'structured', + 'overlapping', + 'object', + 'empty', + 'empty-2d', + 'readonly' + ]) + def test_ctypes_data_as_holds_reference(self, arr): + # gh-9647 + # create a copy to ensure that pytest does not mess with the refcounts + arr = arr.copy() + + arr_ref = weakref.ref(arr) + + ctypes_ptr = arr.ctypes.data_as(ctypes.c_void_p) + + # `ctypes_ptr` should hold onto `arr` + del arr + gc.collect() + assert_(arr_ref() is not None, "ctypes pointer did not hold onto a reference") + + # but when the `ctypes_ptr` object dies, so should `arr` + del ctypes_ptr + gc.collect() + assert_(arr_ref() is None, "unknowable whether ctypes pointer holds a reference") + class TestWritebackIfCopy(object): # all these tests use the WRITEBACKIFCOPY mechanism @@ -7491,6 +7694,7 @@ class TestWritebackIfCopy(object): mat = np.eye(5) out = np.eye(5, dtype='i2') res = np.clip(mat, a_min=-10, a_max=0, out=out) + assert_(res is out) assert_equal(np.sum(out), 0) def test_insert_noncontiguous(self): @@ -7768,6 +7972,44 @@ def test_uintalignment_and_alignment(): dst = np.zeros((2,2), dtype='c8') dst[:,1] = src[:,1] # assert in lowlevel_strided_loops fails? +class TestAlignment(object): + # adapted from scipy._lib.tests.test__util.test__aligned_zeros + # Checks that unusual memory alignments don't trip up numpy. + # In particular, check RELAXED_STRIDES don't trip alignment assertions in + # NDEBUG mode for size-0 arrays (gh-12503) + + def check(self, shape, dtype, order, align): + err_msg = repr((shape, dtype, order, align)) + x = _aligned_zeros(shape, dtype, order, align=align) + if align is None: + align = np.dtype(dtype).alignment + assert_equal(x.__array_interface__['data'][0] % align, 0) + if hasattr(shape, '__len__'): + assert_equal(x.shape, shape, err_msg) + else: + assert_equal(x.shape, (shape,), err_msg) + assert_equal(x.dtype, dtype) + if order == "C": + assert_(x.flags.c_contiguous, err_msg) + elif order == "F": + if x.size > 0: + assert_(x.flags.f_contiguous, err_msg) + elif order is None: + assert_(x.flags.c_contiguous, err_msg) + else: + raise ValueError() + + def test_various_alignments(self): + for align in [1, 2, 3, 4, 8, 16, 32, 64, None]: + for n in [0, 1, 3, 11]: + for order in ["C", "F", None]: + for dtype in np.typecodes["All"]: + if dtype == 'O': + # object dtype can't be misaligned + continue + for shape in [n, (1, 2, 3, n)]: + self.check(shape, np.dtype(dtype), order, align) + def test_getfield(): a = np.arange(32, dtype='uint16') if sys.byteorder == 'little': diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index 5e8165bc5..26fd9c346 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -1,14 +1,13 @@ from __future__ import division, absolute_import, print_function import sys -import warnings import pytest import numpy as np import numpy.core._multiarray_tests as _multiarray_tests from numpy import array, arange, nditer, all from numpy.testing import ( - assert_, assert_equal, assert_array_equal, assert_raises, assert_warns, + assert_, assert_equal, assert_array_equal, assert_raises, HAS_REFCOUNT, suppress_warnings ) @@ -2196,21 +2195,15 @@ class TestIterNested(object): a = arange(12).reshape(2, 3, 2) i, j = np.nested_iters(a, [[0], [1, 2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9, 10, 11]]) i, j = np.nested_iters(a, [[0, 1], [2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11]]) i, j = np.nested_iters(a, [[0, 2], [1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4], [1, 3, 5], [6, 8, 10], [7, 9, 11]]) def test_reorder(self): @@ -2219,40 +2212,28 @@ class TestIterNested(object): # In 'K' order (default), it gets reordered i, j = np.nested_iters(a, [[0], [2, 1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9, 10, 11]]) i, j = np.nested_iters(a, [[1, 0], [2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11]]) i, j = np.nested_iters(a, [[2, 0], [1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4], [1, 3, 5], [6, 8, 10], [7, 9, 11]]) # In 'C' order, it doesn't i, j = np.nested_iters(a, [[0], [2, 1]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4, 1, 3, 5], [6, 8, 10, 7, 9, 11]]) i, j = np.nested_iters(a, [[1, 0], [2]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1], [6, 7], [2, 3], [8, 9], [4, 5], [10, 11]]) i, j = np.nested_iters(a, [[2, 0], [1]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4], [6, 8, 10], [1, 3, 5], [7, 9, 11]]) def test_flip_axes(self): @@ -2261,40 +2242,28 @@ class TestIterNested(object): # In 'K' order (default), the axes all get flipped i, j = np.nested_iters(a, [[0], [1, 2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9, 10, 11]]) i, j = np.nested_iters(a, [[0, 1], [2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11]]) i, j = np.nested_iters(a, [[0, 2], [1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 2, 4], [1, 3, 5], [6, 8, 10], [7, 9, 11]]) # In 'C' order, flipping axes is disabled i, j = np.nested_iters(a, [[0], [1, 2]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[11, 10, 9, 8, 7, 6], [5, 4, 3, 2, 1, 0]]) i, j = np.nested_iters(a, [[0, 1], [2]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[11, 10], [9, 8], [7, 6], [5, 4], [3, 2], [1, 0]]) i, j = np.nested_iters(a, [[0, 2], [1]], order='C') - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[11, 9, 7], [10, 8, 6], [5, 3, 1], [4, 2, 0]]) def test_broadcast(self): @@ -2303,15 +2272,11 @@ class TestIterNested(object): b = arange(3).reshape(1, 3) i, j = np.nested_iters([a, b], [[0], [1]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[[0, 0], [0, 1], [0, 2]], [[1, 0], [1, 1], [1, 2]]]) i, j = np.nested_iters([a, b], [[1], [0]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[[0, 0], [1, 0]], [[0, 1], [1, 1]], [[0, 2], [1, 2]]]) def test_dtype_copy(self): @@ -2323,9 +2288,7 @@ class TestIterNested(object): op_flags=['readonly', 'copy'], op_dtypes='f8') assert_equal(j[0].dtype, np.dtype('f8')) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2], [3, 4, 5]]) vals = None @@ -2376,15 +2339,11 @@ class TestIterNested(object): def test_0d(self): a = np.arange(12).reshape(2, 3, 2) i, j = np.nested_iters(a, [[], [1, 0, 2]]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]]) i, j = np.nested_iters(a, [[1, 0, 2], []]) - vals = [] - for x in i: - vals.append([y for y in j]) + vals = [list(j) for _ in i] assert_equal(vals, [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11]]) i, j, k = np.nested_iters(a, [[2, 0], [], [1]]) @@ -2556,10 +2515,8 @@ def test_iter_buffering_reduction_reuse_reduce_loops(): op_flags=[['readonly'], ['readwrite']], buffersize=5) - bufsizes = [] with it: - for x, y in it: - bufsizes.append(x.shape[0]) + bufsizes = [x.shape[0] for x, y in it] assert_equal(bufsizes, [5, 2, 5, 2]) assert_equal(sum(bufsizes), a.size) diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index f264c4ab0..37534720a 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -13,7 +13,7 @@ from numpy.random import rand, randint, randn from numpy.testing import ( assert_, assert_equal, assert_raises, assert_raises_regex, assert_array_equal, assert_almost_equal, assert_array_almost_equal, - suppress_warnings, HAS_REFCOUNT + HAS_REFCOUNT ) diff --git a/numpy/core/tests/test_numerictypes.py b/numpy/core/tests/test_numerictypes.py index 27e4fdeec..71f7b7150 100644 --- a/numpy/core/tests/test_numerictypes.py +++ b/numpy/core/tests/test_numerictypes.py @@ -87,10 +87,8 @@ def normalize_descr(descr): else: nitem = (item[0], dtype) out.append(nitem) - elif isinstance(item[1], list): - l = [] - for j in normalize_descr(item[1]): - l.append(j) + elif isinstance(dtype, list): + l = normalize_descr(dtype) out.append((item[0], l)) else: raise ValueError("Expected a str or list and got %s" % diff --git a/numpy/core/tests/test_overrides.py b/numpy/core/tests/test_overrides.py index ee6d5da4a..8f1c16539 100644 --- a/numpy/core/tests/test_overrides.py +++ b/numpy/core/tests/test_overrides.py @@ -1,46 +1,57 @@ from __future__ import division, absolute_import, print_function +import inspect import sys import numpy as np from numpy.testing import ( assert_, assert_equal, assert_raises, assert_raises_regex) from numpy.core.overrides import ( - get_overloaded_types_and_args, array_function_dispatch, - verify_matching_signatures) + _get_implementing_args, array_function_dispatch, + verify_matching_signatures, ENABLE_ARRAY_FUNCTION) from numpy.core.numeric import pickle +import pytest -def _get_overloaded_args(relevant_args): - types, args = get_overloaded_types_and_args(relevant_args) - return args +requires_array_function = pytest.mark.skipif( + not ENABLE_ARRAY_FUNCTION, + reason="__array_function__ dispatch not enabled.") def _return_not_implemented(self, *args, **kwargs): return NotImplemented -class TestGetOverloadedTypesAndArgs(object): +# need to define this at the top level to test pickling +@array_function_dispatch(lambda array: (array,)) +def dispatched_one_arg(array): + """Docstring.""" + return 'original' + + +@array_function_dispatch(lambda array1, array2: (array1, array2)) +def dispatched_two_arg(array1, array2): + """Docstring.""" + return 'original' + + +@requires_array_function +class TestGetImplementingArgs(object): def test_ndarray(self): array = np.array(1) - types, args = get_overloaded_types_and_args([array]) - assert_equal(set(types), {np.ndarray}) - assert_equal(list(args), []) + args = _get_implementing_args([array]) + assert_equal(list(args), [array]) - types, args = get_overloaded_types_and_args([array, array]) - assert_equal(len(types), 1) - assert_equal(set(types), {np.ndarray}) - assert_equal(list(args), []) + args = _get_implementing_args([array, array]) + assert_equal(list(args), [array]) - types, args = get_overloaded_types_and_args([array, 1]) - assert_equal(set(types), {np.ndarray}) - assert_equal(list(args), []) + args = _get_implementing_args([array, 1]) + assert_equal(list(args), [array]) - types, args = get_overloaded_types_and_args([1, array]) - assert_equal(set(types), {np.ndarray}) - assert_equal(list(args), []) + args = _get_implementing_args([1, array]) + assert_equal(list(args), [array]) def test_ndarray_subclasses(self): @@ -54,18 +65,15 @@ class TestGetOverloadedTypesAndArgs(object): override_sub = np.array(1).view(OverrideSub) no_override_sub = np.array(1).view(NoOverrideSub) - types, args = get_overloaded_types_and_args([array, override_sub]) - assert_equal(set(types), {np.ndarray, OverrideSub}) - assert_equal(list(args), [override_sub]) + args = _get_implementing_args([array, override_sub]) + assert_equal(list(args), [override_sub, array]) - types, args = get_overloaded_types_and_args([array, no_override_sub]) - assert_equal(set(types), {np.ndarray, NoOverrideSub}) - assert_equal(list(args), []) + args = _get_implementing_args([array, no_override_sub]) + assert_equal(list(args), [no_override_sub, array]) - types, args = get_overloaded_types_and_args( + args = _get_implementing_args( [override_sub, no_override_sub]) - assert_equal(set(types), {OverrideSub, NoOverrideSub}) - assert_equal(list(args), [override_sub]) + assert_equal(list(args), [override_sub, no_override_sub]) def test_ndarray_and_duck_array(self): @@ -75,13 +83,11 @@ class TestGetOverloadedTypesAndArgs(object): array = np.array(1) other = Other() - types, args = get_overloaded_types_and_args([other, array]) - assert_equal(set(types), {np.ndarray, Other}) - assert_equal(list(args), [other]) + args = _get_implementing_args([other, array]) + assert_equal(list(args), [other, array]) - types, args = get_overloaded_types_and_args([array, other]) - assert_equal(set(types), {np.ndarray, Other}) - assert_equal(list(args), [other]) + args = _get_implementing_args([array, other]) + assert_equal(list(args), [array, other]) def test_ndarray_subclass_and_duck_array(self): @@ -95,10 +101,10 @@ class TestGetOverloadedTypesAndArgs(object): subarray = np.array(1).view(OverrideSub) other = Other() - assert_equal(_get_overloaded_args([array, subarray, other]), - [subarray, other]) - assert_equal(_get_overloaded_args([array, other, subarray]), - [subarray, other]) + assert_equal(_get_implementing_args([array, subarray, other]), + [subarray, array, other]) + assert_equal(_get_implementing_args([array, other, subarray]), + [subarray, array, other]) def test_many_duck_arrays(self): @@ -119,52 +125,88 @@ class TestGetOverloadedTypesAndArgs(object): c = C() d = D() - assert_equal(_get_overloaded_args([1]), []) - assert_equal(_get_overloaded_args([a]), [a]) - assert_equal(_get_overloaded_args([a, 1]), [a]) - assert_equal(_get_overloaded_args([a, a, a]), [a]) - assert_equal(_get_overloaded_args([a, d, a]), [a, d]) - assert_equal(_get_overloaded_args([a, b]), [b, a]) - assert_equal(_get_overloaded_args([b, a]), [b, a]) - assert_equal(_get_overloaded_args([a, b, c]), [b, c, a]) - assert_equal(_get_overloaded_args([a, c, b]), [c, b, a]) + assert_equal(_get_implementing_args([1]), []) + assert_equal(_get_implementing_args([a]), [a]) + assert_equal(_get_implementing_args([a, 1]), [a]) + assert_equal(_get_implementing_args([a, a, a]), [a]) + assert_equal(_get_implementing_args([a, d, a]), [a, d]) + assert_equal(_get_implementing_args([a, b]), [b, a]) + assert_equal(_get_implementing_args([b, a]), [b, a]) + assert_equal(_get_implementing_args([a, b, c]), [b, c, a]) + assert_equal(_get_implementing_args([a, c, b]), [c, b, a]) + + def test_too_many_duck_arrays(self): + namespace = dict(__array_function__=_return_not_implemented) + types = [type('A' + str(i), (object,), namespace) for i in range(33)] + relevant_args = [t() for t in types] + actual = _get_implementing_args(relevant_args[:32]) + assert_equal(actual, relevant_args[:32]) + with assert_raises_regex(TypeError, 'distinct argument types'): + _get_implementing_args(relevant_args) + + +@requires_array_function class TestNDArrayArrayFunction(object): def test_method(self): - class SubOverride(np.ndarray): + class Other(object): __array_function__ = _return_not_implemented class NoOverrideSub(np.ndarray): pass - array = np.array(1) + class OverrideSub(np.ndarray): + __array_function__ = _return_not_implemented - def func(): - return 'original' + array = np.array([1]) + other = Other() + no_override_sub = array.view(NoOverrideSub) + override_sub = array.view(OverrideSub) - result = array.__array_function__( - func=func, types=(np.ndarray,), args=(), kwargs={}) + result = array.__array_function__(func=dispatched_two_arg, + types=(np.ndarray,), + args=(array, 1.), kwargs={}) assert_equal(result, 'original') - result = array.__array_function__( - func=func, types=(np.ndarray, SubOverride), args=(), kwargs={}) + result = array.__array_function__(func=dispatched_two_arg, + types=(np.ndarray, Other), + args=(array, other), kwargs={}) assert_(result is NotImplemented) - result = array.__array_function__( - func=func, types=(np.ndarray, NoOverrideSub), args=(), kwargs={}) + result = array.__array_function__(func=dispatched_two_arg, + types=(np.ndarray, NoOverrideSub), + args=(array, no_override_sub), + kwargs={}) assert_equal(result, 'original') + result = array.__array_function__(func=dispatched_two_arg, + types=(np.ndarray, OverrideSub), + args=(array, override_sub), + kwargs={}) + assert_equal(result, 'original') -# need to define this at the top level to test pickling -@array_function_dispatch(lambda array: (array,)) -def dispatched_one_arg(array): - """Docstring.""" - return 'original' + with assert_raises_regex(TypeError, 'no implementation found'): + np.concatenate((array, other)) + expected = np.concatenate((array, array)) + result = np.concatenate((array, no_override_sub)) + assert_equal(result, expected.view(NoOverrideSub)) + result = np.concatenate((array, override_sub)) + assert_equal(result, expected.view(OverrideSub)) + + def test_no_wrapper(self): + array = np.array(1) + func = dispatched_one_arg.__wrapped__ + with assert_raises_regex(AttributeError, '__wrapped__'): + array.__array_function__(func=func, + types=(np.ndarray,), + args=(array,), kwargs={}) + +@requires_array_function class TestArrayFunctionDispatch(object): def test_pickle(self): @@ -204,6 +246,7 @@ class TestArrayFunctionDispatch(object): dispatched_one_arg(array) +@requires_array_function class TestVerifyMatchingSignatures(object): def test_verify_matching_signatures(self): @@ -256,6 +299,7 @@ def _new_duck_type_and_implements(): return (MyArray, implements) +@requires_array_function class TestArrayFunctionImplementation(object): def test_one_arg(self): @@ -302,6 +346,7 @@ class TestArrayFunctionImplementation(object): array = np.array(1) assert_(func(array) is array) + assert_equal(func.__module__, 'my') with assert_raises_regex( TypeError, "no implementation found for 'my.func'"): @@ -322,15 +367,21 @@ class TestNDArrayMethods(object): assert_equal(repr(array), 'MyArray(1)') assert_equal(str(array), '1') - + class TestNumPyFunctions(object): - def test_module(self): + def test_set_module(self): assert_equal(np.sum.__module__, 'numpy') assert_equal(np.char.equal.__module__, 'numpy.char') assert_equal(np.fft.fft.__module__, 'numpy.fft') assert_equal(np.linalg.solve.__module__, 'numpy.linalg') + @pytest.mark.skipif(sys.version_info[0] < 3, reason="Python 3 only") + def test_inspect_sum(self): + signature = inspect.signature(np.sum) + assert_('axis' in signature.parameters) + + @requires_array_function def test_override_sum(self): MyArray, implements = _new_duck_type_and_implements() diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py index af6c86b9e..c059ef510 100644 --- a/numpy/core/tests/test_records.py +++ b/numpy/core/tests/test_records.py @@ -7,7 +7,6 @@ try: import collections.abc as collections_abc except ImportError: import collections as collections_abc -import warnings import textwrap from os import path import pytest @@ -16,7 +15,7 @@ import numpy as np from numpy.compat import Path from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_array_almost_equal, - assert_raises, assert_warns, temppath + assert_raises, temppath ) from numpy.core.numeric import pickle @@ -332,15 +331,14 @@ class TestPathUsage(object): def test_tofile_fromfile(self): with temppath(suffix='.bin') as path: path = Path(path) - a = np.empty(10, dtype='f8,i4,a5') + np.random.seed(123) + a = np.random.rand(10).astype('f8,i4,a5') a[5] = (0.5,10,'abcde') - a.newbyteorder('<') with path.open("wb") as fd: a.tofile(fd) x = np.core.records.fromfile(path, formats='f8,i4,a5', - shape=10, - byteorder='<') + shape=10) assert_array_equal(x, a) @@ -380,7 +378,6 @@ class TestRecord(object): with assert_raises(ValueError): r.setfield([2,3], *r.dtype.fields['f']) - @pytest.mark.xfail(reason="See gh-10411, becomes real error in 1.16") def test_out_of_order_fields(self): # names in the same order, padding added to descr x = self.data[['col1', 'col2']] diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index a929b0efd..2421a1161 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -2,9 +2,7 @@ from __future__ import division, absolute_import, print_function import copy import sys -import platform import gc -import warnings import tempfile import pytest from os import path @@ -2409,7 +2407,7 @@ class TestRegression(object): t = np.dtype([((s, 'f1'), np.float64)]) data = np.zeros(10, t) for i in range(10): - v = str(data[['f1']]) + str(data[['f1']]) if HAS_REFCOUNT: assert_(base <= sys.getrefcount(s)) diff --git a/numpy/core/tests/test_scalarinherit.py b/numpy/core/tests/test_scalarinherit.py index 28436f6c7..9e32cf624 100644 --- a/numpy/core/tests/test_scalarinherit.py +++ b/numpy/core/tests/test_scalarinherit.py @@ -69,6 +69,7 @@ class TestCharacter(object): np_s = np.string_('abc') np_u = np.unicode_('abc') np_i = np.int(5) - res_np = np_s * np_i res_s = b'abc' * 5 - assert_(res_np == res_s) + res_u = u'abc' * 5 + assert_(np_s * np_i == res_s) + assert_(np_u * np_i == res_u) diff --git a/numpy/core/tests/test_scalarmath.py b/numpy/core/tests/test_scalarmath.py index a55f06b69..51bcf2b8d 100644 --- a/numpy/core/tests/test_scalarmath.py +++ b/numpy/core/tests/test_scalarmath.py @@ -9,7 +9,7 @@ import pytest import numpy as np from numpy.testing import ( - assert_, assert_equal, assert_raises, assert_almost_equal, assert_allclose, + assert_, assert_equal, assert_raises, assert_almost_equal, assert_array_equal, IS_PYPY, suppress_warnings, _gen_alignment_data, assert_warns ) @@ -184,7 +184,7 @@ class TestPower(object): a = 5 b = 4 c = 10 - expected = pow(a, b, c) + expected = pow(a, b, c) # noqa: F841 for t in (np.int32, np.float32, np.complex64): # note that 3-operand power only dispatches on the first argument assert_raises(TypeError, operator.pow, t(a), b, c) @@ -565,10 +565,10 @@ class TestMultiply(object): # Some of this behaviour may be controversial and could be open for # change. accepted_types = set(np.typecodes["AllInteger"]) - deprecated_types = set('?') + deprecated_types = {'?'} forbidden_types = ( set(np.typecodes["All"]) - accepted_types - deprecated_types) - forbidden_types -= set('V') # can't default-construct void scalars + forbidden_types -= {'V'} # can't default-construct void scalars for seq_type in (list, tuple): seq = seq_type([1, 2, 3]) diff --git a/numpy/core/tests/test_scalarprint.py b/numpy/core/tests/test_scalarprint.py index 472ff691d..cde1355aa 100644 --- a/numpy/core/tests/test_scalarprint.py +++ b/numpy/core/tests/test_scalarprint.py @@ -10,7 +10,7 @@ import pytest from tempfile import TemporaryFile import numpy as np -from numpy.testing import assert_, assert_equal, suppress_warnings, dec +from numpy.testing import assert_, assert_equal, suppress_warnings class TestRealScalars(object): def test_str(self): diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py index b2c610da6..b996321c2 100644 --- a/numpy/core/tests/test_shape_base.py +++ b/numpy/core/tests/test_shape_base.py @@ -1,6 +1,6 @@ from __future__ import division, absolute_import, print_function -import warnings +import pytest import sys import numpy as np from numpy.core import ( @@ -11,7 +11,7 @@ from numpy.core.shape_base import (_block_dispatcher, _block_setup, _block_concatenate, _block_slicing) from numpy.testing import ( assert_, assert_raises, assert_array_equal, assert_equal, - assert_raises_regex, assert_warns, assert_almost_equal + assert_raises_regex, assert_warns ) from numpy.compat import long @@ -373,6 +373,10 @@ def test_stack(): # empty arrays assert_(stack([[], [], []]).shape == (3, 0)) assert_(stack([[], [], []], axis=1).shape == (0, 3)) + # out + out = np.zeros_like(r1) + np.stack((a, b), out=out) + assert_array_equal(out, r1) # edge cases assert_raises_regex(ValueError, 'need at least one array', stack, []) assert_raises_regex(ValueError, 'must have the same shape', @@ -391,39 +395,32 @@ def test_stack(): assert_array_equal(result, np.array([0, 1, 2])) -# See for more information on how to parametrize a whole class -# https://docs.pytest.org/en/latest/example/parametrize.html#parametrizing-test-methods-through-per-class-configuration -def pytest_generate_tests(metafunc): - # called once per each test function - if hasattr(metafunc.cls, 'params'): - arglist = metafunc.cls.params - argnames = sorted(arglist[0]) - metafunc.parametrize(argnames, - [[funcargs[name] for name in argnames] - for funcargs in arglist]) - - -# blocking small arrays and large arrays go through different paths. -# the algorithm is triggered depending on the number of element -# copies required. -# We define a test fixture that forces most tests to go through -# both code paths. -# Ultimately, this should be removed if a single algorithm is found -# to be faster for both small and large arrays.s -def _block_force_concatenate(arrays): - arrays, list_ndim, result_ndim, _ = _block_setup(arrays) - return _block_concatenate(arrays, list_ndim, result_ndim) - - -def _block_force_slicing(arrays): - arrays, list_ndim, result_ndim, _ = _block_setup(arrays) - return _block_slicing(arrays, list_ndim, result_ndim) - - class TestBlock(object): - params = [dict(block=block), - dict(block=_block_force_concatenate), - dict(block=_block_force_slicing)] + @pytest.fixture(params=['block', 'force_concatenate', 'force_slicing']) + def block(self, request): + # blocking small arrays and large arrays go through different paths. + # the algorithm is triggered depending on the number of element + # copies required. + # We define a test fixture that forces most tests to go through + # both code paths. + # Ultimately, this should be removed if a single algorithm is found + # to be faster for both small and large arrays. + def _block_force_concatenate(arrays): + arrays, list_ndim, result_ndim, _ = _block_setup(arrays) + return _block_concatenate(arrays, list_ndim, result_ndim) + + def _block_force_slicing(arrays): + arrays, list_ndim, result_ndim, _ = _block_setup(arrays) + return _block_slicing(arrays, list_ndim, result_ndim) + + if request.param == 'force_concatenate': + return _block_force_concatenate + elif request.param == 'force_slicing': + return _block_force_slicing + elif request.param == 'block': + return block + else: + raise ValueError('Unknown blocking request. There is a typo in the tests.') def test_returns_copy(self, block): a = np.eye(3) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index bd7985dfb..2f8edebc0 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1,6 +1,5 @@ from __future__ import division, absolute_import, print_function -import sys import platform import warnings import fnmatch @@ -14,7 +13,7 @@ from numpy.testing import ( assert_, assert_equal, assert_raises, assert_raises_regex, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_allclose, assert_no_warnings, suppress_warnings, - _gen_alignment_data, assert_warns + _gen_alignment_data ) @@ -685,6 +684,10 @@ class TestLogAddExp(_FilterInvalids): assert_(np.isnan(np.logaddexp(0, np.nan))) assert_(np.isnan(np.logaddexp(np.nan, np.nan))) + def test_reduce(self): + assert_equal(np.logaddexp.identity, -np.inf) + assert_equal(np.logaddexp.reduce([]), -np.inf) + class TestLog1p(object): def test_log1p(self): @@ -1294,6 +1297,7 @@ class TestSign(object): # In reference to github issue #6229 def test_nan(): foo = np.array([np.nan]) + # FIXME: a not used a = np.sign(foo.astype(object)) assert_raises(TypeError, test_nan) @@ -2444,11 +2448,6 @@ class TestRationalFunctions(object): assert_equal(np.gcd(2**100, 3**100), 1) -def is_longdouble_finfo_bogus(): - info = np.finfo(np.longcomplex) - return not np.isfinite(np.log10(info.tiny/info.eps)) - - class TestComplexFunctions(object): funcs = [np.arcsin, np.arccos, np.arctan, np.arcsinh, np.arccosh, np.arctanh, np.sin, np.cos, np.tan, np.exp, @@ -2544,7 +2543,8 @@ class TestComplexFunctions(object): b = cfunc(p) assert_(abs(a - b) < atol, "%s %s: %s; cmath: %s" % (fname, p, a, b)) - def check_loss_of_precision(self, dtype): + @pytest.mark.parametrize('dtype', [np.complex64, np.complex_, np.longcomplex]) + def test_loss_of_precision(self, dtype): """Check loss of precision in complex arc* functions""" # Check against known-good functions @@ -2586,10 +2586,11 @@ class TestComplexFunctions(object): # It's not guaranteed that the system-provided arc functions # are accurate down to a few epsilons. (Eg. on Linux 64-bit) # So, give more leeway for long complex tests here: - check(x_series, 50*eps) + # Can use 2.1 for > Ubuntu LTS Trusty (2014), glibc = 2.19. + check(x_series, 50.0*eps) else: check(x_series, 2.1*eps) - check(x_basic, 2*eps/1e-3) + check(x_basic, 2.0*eps/1e-3) # Check a few points @@ -2629,15 +2630,6 @@ class TestComplexFunctions(object): check(func, pts, 1j) check(func, pts, 1+1j) - def test_loss_of_precision(self): - for dtype in [np.complex64, np.complex_]: - self.check_loss_of_precision(dtype) - - @pytest.mark.skipif(is_longdouble_finfo_bogus(), - reason="Bogus long double finfo") - def test_loss_of_precision_longcomplex(self): - self.check_loss_of_precision(np.longcomplex) - class TestAttributes(object): def test_attributes(self): diff --git a/numpy/ctypeslib.py b/numpy/ctypeslib.py index 24cfc6762..2e9781286 100644 --- a/numpy/ctypeslib.py +++ b/numpy/ctypeslib.py @@ -55,7 +55,9 @@ __all__ = ['load_library', 'ndpointer', 'test', 'ctypes_load_library', 'c_intp', 'as_ctypes', 'as_array'] import os -from numpy import integer, ndarray, dtype as _dtype, deprecate, array +from numpy import ( + integer, ndarray, dtype as _dtype, deprecate, array, frombuffer +) from numpy.core.multiarray import _flagdict, flagsobj try: @@ -91,7 +93,7 @@ else: def load_library(libname, loader_path): """ It is possible to load a library using - >>> lib = ctypes.cdll[<full_path_name>] + >>> lib = ctypes.cdll[<full_path_name>] # doctest: +SKIP But there are cross-platform considerations, such as library file extensions, plus the fact Windows will just load the first library it finds with that name. @@ -175,24 +177,6 @@ def _flags_fromnum(num): class _ndptr(_ndptr_base): - - def _check_retval_(self): - """This method is called when this class is used as the .restype - attribute for a shared-library function. It constructs a numpy - array from a void pointer.""" - return array(self) - - @property - def __array_interface__(self): - return {'descr': self._dtype_.descr, - '__ref': self, - 'strides': None, - 'shape': self._shape_, - 'version': 3, - 'typestr': self._dtype_.descr[0][1], - 'data': (self.value, False), - } - @classmethod def from_param(cls, obj): if not isinstance(obj, ndarray): @@ -213,6 +197,34 @@ class _ndptr(_ndptr_base): return obj.ctypes +class _concrete_ndptr(_ndptr): + """ + Like _ndptr, but with `_shape_` and `_dtype_` specified. + + Notably, this means the pointer has enough information to reconstruct + the array, which is not generally true. + """ + def _check_retval_(self): + """ + This method is called when this class is used as the .restype + attribute for a shared-library function, to automatically wrap the + pointer into an array. + """ + return self.contents + + @property + def contents(self): + """ + Get an ndarray viewing the data pointed to by this pointer. + + This mirrors the `contents` attribute of a normal ctypes pointer + """ + full_dtype = _dtype((self._dtype_, self._shape_)) + full_ctype = ctypes.c_char * full_dtype.itemsize + buffer = ctypes.cast(self, ctypes.POINTER(full_ctype)).contents + return frombuffer(buffer, dtype=full_dtype).squeeze(axis=0) + + # Factory for an array-checking class with from_param defined for # use with ctypes argtypes mechanism _pointer_type_cache = {} @@ -269,8 +281,11 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): """ + # normalize dtype to an Optional[dtype] if dtype is not None: dtype = _dtype(dtype) + + # normalize flags to an Optional[int] num = None if flags is not None: if isinstance(flags, str): @@ -287,10 +302,23 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): except Exception: raise TypeError("invalid flags specification") num = _num_fromflags(flags) + + # normalize shape to an Optional[tuple] + if shape is not None: + try: + shape = tuple(shape) + except TypeError: + # single integer -> 1-tuple + shape = (shape,) + + cache_key = (dtype, ndim, shape, num) + try: - return _pointer_type_cache[(dtype, ndim, shape, num)] + return _pointer_type_cache[cache_key] except KeyError: pass + + # produce a name for the new type if dtype is None: name = 'any' elif dtype.names: @@ -300,23 +328,21 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): if ndim is not None: name += "_%dd" % ndim if shape is not None: - try: - strshape = [str(x) for x in shape] - except TypeError: - strshape = [str(shape)] - shape = (shape,) - shape = tuple(shape) - name += "_"+"x".join(strshape) + name += "_"+"x".join(str(x) for x in shape) if flags is not None: name += "_"+"_".join(flags) + + if dtype is not None and shape is not None: + base = _concrete_ndptr else: - flags = [] - klass = type("ndpointer_%s"%name, (_ndptr,), + base = _ndptr + + klass = type("ndpointer_%s"%name, (base,), {"_dtype_": dtype, "_shape_" : shape, "_ndim_" : ndim, "_flags_" : num}) - _pointer_type_cache[(dtype, shape, ndim, num)] = klass + _pointer_type_cache[cache_key] = klass return klass @@ -375,5 +401,5 @@ if ctypes is not None: raise TypeError("readonly arrays unsupported") tp = _ctype_ndarray(_typecodes[ai["typestr"]], ai["shape"]) result = tp.from_address(addr) - result.__keep = ai + result.__keep = obj return result diff --git a/numpy/distutils/__init__.py b/numpy/distutils/__init__.py index 8dd326920..55514750e 100644 --- a/numpy/distutils/__init__.py +++ b/numpy/distutils/__init__.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import sys - from .__version__ import version as __version__ # Must import local ccompiler ASAP in order to get # customized CCompiler.spawn effective. diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py index b03fb96b2..100d0d069 100644 --- a/numpy/distutils/ccompiler.py +++ b/numpy/distutils/ccompiler.py @@ -6,6 +6,7 @@ import sys import types import shlex import time +import subprocess from copy import copy from distutils import ccompiler from distutils.ccompiler import * @@ -16,9 +17,11 @@ from distutils.version import LooseVersion from numpy.distutils import log from numpy.distutils.compat import get_exception -from numpy.distutils.exec_command import exec_command +from numpy.distutils.exec_command import ( + filepath_from_subprocess_output, forward_bytes_to_stdout +) from numpy.distutils.misc_util import cyg2win32, is_sequence, mingw32, \ - quote_args, get_num_build_jobs, \ + get_num_build_jobs, \ _commandline_dep_string # globals for parallel build management @@ -136,20 +139,37 @@ def CCompiler_spawn(self, cmd, display=None): if is_sequence(display): display = ' '.join(list(display)) log.info(display) - s, o = exec_command(cmd) - if s: - if is_sequence(cmd): - cmd = ' '.join(list(cmd)) - try: - print(o) - except UnicodeError: - # When installing through pip, `o` can contain non-ascii chars - pass - if re.search('Too many open files', o): - msg = '\nTry rerunning setup command until build succeeds.' - else: - msg = '' - raise DistutilsExecError('Command "%s" failed with exit status %d%s' % (cmd, s, msg)) + try: + subprocess.check_output(cmd) + except subprocess.CalledProcessError as exc: + o = exc.output + s = exc.returncode + except OSError: + # OSError doesn't have the same hooks for the exception + # output, but exec_command() historically would use an + # empty string for EnvironmentError (base class for + # OSError) + o = b'' + # status previously used by exec_command() for parent + # of OSError + s = 127 + else: + # use a convenience return here so that any kind of + # caught exception will execute the default code after the + # try / except block, which handles various exceptions + return None + + if is_sequence(cmd): + cmd = ' '.join(list(cmd)) + + forward_bytes_to_stdout(o) + + if re.search(b'Too many open files', o): + msg = '\nTry rerunning setup command until build succeeds.' + else: + msg = '' + raise DistutilsExecError('Command "%s" failed with exit status %d%s' % + (cmd, s, msg)) replace_method(CCompiler, 'spawn', CCompiler_spawn) @@ -404,10 +424,8 @@ def _compiler_to_string(compiler): v = getattr(compiler, key) mx = max(mx, len(key)) props.append((key, repr(v))) - lines = [] - format = '%-' + repr(mx+1) + 's = %s' - for prop in props: - lines.append(format % prop) + fmt = '%-' + repr(mx+1) + 's = %s' + lines = [fmt % prop for prop in props] return '\n'.join(lines) def CCompiler_show_customization(self): @@ -620,7 +638,21 @@ def CCompiler_get_version(self, force=False, ok_status=[0]): version = m.group('version') return version - status, output = exec_command(version_cmd, use_tee=0) + try: + output = subprocess.check_output(version_cmd) + except subprocess.CalledProcessError as exc: + output = exc.output + status = exc.returncode + except OSError: + # match the historical returns for a parent + # exception class caught by exec_command() + status = 127 + output = b'' + else: + # output isn't actually a filepath but we do this + # for now to match previous distutils behavior + output = filepath_from_subprocess_output(output) + status = 0 version = None if status in ok_status: @@ -738,8 +770,13 @@ ccompiler.new_compiler = new_compiler _distutils_gen_lib_options = gen_lib_options def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries): - library_dirs = quote_args(library_dirs) - runtime_library_dirs = quote_args(runtime_library_dirs) + # the version of this function provided by CPython allows the following + # to return lists, which are unpacked automatically: + # - compiler.runtime_library_dir_option + # our version extends the behavior to: + # - compiler.library_dir_option + # - compiler.library_option + # - compiler.find_library_file r = _distutils_gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) lib_opts = [] @@ -759,11 +796,6 @@ for _cc in ['msvc9', 'msvc', '_msvc', 'bcpp', 'cygwinc', 'emxc', 'unixc']: if _m is not None: setattr(_m, 'gen_lib_options', gen_lib_options) -_distutils_gen_preprocess_options = gen_preprocess_options -def gen_preprocess_options (macros, include_dirs): - include_dirs = quote_args(include_dirs) - return _distutils_gen_preprocess_options(macros, include_dirs) -ccompiler.gen_preprocess_options = gen_preprocess_options ##Fix distutils.util.split_quoted: # NOTE: I removed this fix in revision 4481 (see ticket #619), but it appears diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index 18d36480a..ab9d585a5 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -265,10 +265,10 @@ class build_ext (old_build_ext): # we blindly assume that both packages need all of the libraries, # resulting in a larger wheel than is required. This should be fixed, # but it's so rare that I won't bother to handle it. - pkg_roots = set( + pkg_roots = { self.get_ext_fullname(ext.name).split('.')[0] for ext in self.extensions - ) + } for pkg_root in pkg_roots: shared_lib_dir = os.path.join(pkg_root, '.libs') if not self.inplace: diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index 9def37822..668bc23fe 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -204,7 +204,6 @@ class build_src(build_ext.build_ext): def _build_npy_pkg_config(self, info, gd): - import shutil template, install_dir, subst_dict = info template_dir = os.path.dirname(template) for k, v in gd.items(): @@ -239,7 +238,6 @@ class build_src(build_ext.build_ext): if not install_cmd.finalized == 1: install_cmd.finalize_options() build_npkg = False - gd = {} if self.inplace == 1: top_prefix = '.' build_npkg = True diff --git a/numpy/distutils/command/config.py b/numpy/distutils/command/config.py index 6b904d6ef..d9b1e8488 100644 --- a/numpy/distutils/command/config.py +++ b/numpy/distutils/command/config.py @@ -95,7 +95,7 @@ Original exception was: %s, and the Compiler class was %s try: ret = mth(*((self,)+args)) except (DistutilsExecError, CompileError): - msg = str(get_exception()) + str(get_exception()) self.compiler = save_compiler raise CompileError self.compiler = save_compiler diff --git a/numpy/distutils/conv_template.py b/numpy/distutils/conv_template.py index 4a8746236..b33e315b4 100644 --- a/numpy/distutils/conv_template.py +++ b/numpy/distutils/conv_template.py @@ -206,10 +206,8 @@ def parse_loop_header(loophead) : dlist = [] if nsub is None : raise ValueError("No substitution variables found") - for i in range(nsub) : - tmp = {} - for name, vals in names : - tmp[name] = vals[i] + for i in range(nsub): + tmp = {name: vals[i] for name, vals in names} dlist.append(tmp) return dlist diff --git a/numpy/distutils/exec_command.py b/numpy/distutils/exec_command.py index af7810d75..ede347b03 100644 --- a/numpy/distutils/exec_command.py +++ b/numpy/distutils/exec_command.py @@ -67,8 +67,10 @@ def filepath_from_subprocess_output(output): Inherited from `exec_command`, and possibly incorrect. """ - output = output.decode(locale.getpreferredencoding(False), - errors='replace') + mylocale = locale.getpreferredencoding(False) + if mylocale is None: + mylocale = 'ascii' + output = output.decode(mylocale, errors='replace') output = output.replace('\r\n', '\n') # Another historical oddity if output[-1:] == '\n': @@ -79,6 +81,29 @@ def filepath_from_subprocess_output(output): output = output.encode('ascii', errors='replace') return output + +def forward_bytes_to_stdout(val): + """ + Forward bytes from a subprocess call to the console, without attempting to + decode them. + + The assumption is that the subprocess call already returned bytes in + a suitable encoding. + """ + if sys.version_info.major < 3: + # python 2 has binary output anyway + sys.stdout.write(val) + elif hasattr(sys.stdout, 'buffer'): + # use the underlying binary output if there is one + sys.stdout.buffer.write(val) + elif hasattr(sys.stdout, 'encoding'): + # round-trip the encoding if necessary + sys.stdout.write(val.decode(sys.stdout.encoding)) + else: + # make a best-guess at the encoding + sys.stdout.write(val.decode('utf8', errors='replace')) + + def temp_file_name(): fo, name = make_temp_file() fo.close() @@ -146,9 +171,7 @@ def find_executable(exe, path=None, _cache={}): def _preserve_environment( names ): log.debug('_preserve_environment(%r)' % (names)) - env = {} - for name in names: - env[name] = os.environ.get(name) + env = {name: os.environ.get(name) for name in names} return env def _update_environment( **env ): @@ -278,9 +301,10 @@ def _exec_command(command, use_shell=None, use_tee = None, **env): return 127, '' text, err = proc.communicate() - text = text.decode(locale.getpreferredencoding(False), - errors='replace') - + mylocale = locale.getpreferredencoding(False) + if mylocale is None: + mylocale = 'ascii' + text = text.decode(mylocale, errors='replace') text = text.replace('\r\n', '\n') # Another historical oddity if text[-1:] == '\n': diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py index 3bd8057b4..12b32832e 100644 --- a/numpy/distutils/fcompiler/__init__.py +++ b/numpy/distutils/fcompiler/__init__.py @@ -22,6 +22,7 @@ import os import sys import re import types +import shlex from numpy.compat import open_latin1 @@ -465,8 +466,10 @@ class FCompiler(CCompiler): noarch = self.distutils_vars.get('noarch', noopt) debug = self.distutils_vars.get('debug', False) - f77 = self.command_vars.compiler_f77 - f90 = self.command_vars.compiler_f90 + f77 = shlex.split(self.command_vars.compiler_f77, + posix=(os.name == 'posix')) + f90 = shlex.split(self.command_vars.compiler_f90, + posix=(os.name == 'posix')) f77flags = [] f90flags = [] @@ -480,6 +483,14 @@ class FCompiler(CCompiler): freeflags = self.flag_vars.free # XXX Assuming that free format is default for f90 compiler. fix = self.command_vars.compiler_fix + # NOTE: this and similar examples are probably just + # exluding --coverage flag when F90 = gfortran --coverage + # instead of putting that flag somewhere more appropriate + # this and similar examples where a Fortran compiler + # environment variable has been customized by CI or a user + # should perhaps eventually be more throughly tested and more + # robustly handled + fix = shlex.split(fix, posix=(os.name == 'posix')) if fix: fixflags = self.flag_vars.fix + f90flags @@ -506,11 +517,11 @@ class FCompiler(CCompiler): fflags = self.flag_vars.flags + dflags + oflags + aflags if f77: - self.set_commands(compiler_f77=[f77]+f77flags+fflags) + self.set_commands(compiler_f77=f77+f77flags+fflags) if f90: - self.set_commands(compiler_f90=[f90]+freeflags+f90flags+fflags) + self.set_commands(compiler_f90=f90+freeflags+f90flags+fflags) if fix: - self.set_commands(compiler_fix=[fix]+fixflags+fflags) + self.set_commands(compiler_fix=fix+fixflags+fflags) #XXX: Do we need LDSHARED->SOSHARED, LDFLAGS->SOFLAGS diff --git a/numpy/distutils/fcompiler/gnu.py b/numpy/distutils/fcompiler/gnu.py index f151809c7..81769e562 100644 --- a/numpy/distutils/fcompiler/gnu.py +++ b/numpy/distutils/fcompiler/gnu.py @@ -265,6 +265,10 @@ class GnuFCompiler(FCompiler): return [] def runtime_library_dir_option(self, dir): + if sys.platform[:3] == 'aix' or sys.platform == 'win32': + # Linux/Solaris/Unix support RPATH, Windows and AIX do not + raise NotImplementedError + sep = ',' if sys.platform == 'darwin' else '=' return '-Wl,-rpath%s"%s"' % (sep, dir) @@ -311,6 +315,12 @@ class Gnu95FCompiler(GnuFCompiler): module_dir_switch = '-J' module_include_switch = '-I' + if sys.platform[:3] == 'aix': + executables['linker_so'].append('-lpthread') + if platform.architecture()[0][:2] == '64': + for key in ['compiler_f77', 'compiler_f90','compiler_fix','linker_so', 'linker_exe']: + executables[key].append('-maix64') + g2c = 'gfortran' def _universal_flags(self, cmd): diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index eba0d9ba1..67a5f7234 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -1691,7 +1691,6 @@ class Configuration(object): """ if subst_dict is None: subst_dict = {} - basename = os.path.splitext(template)[0] template = os.path.join(self.package_path, template) if self.name in self.installed_pkg_config: @@ -2225,7 +2224,6 @@ def is_bootstrapping(): return True except AttributeError: return False - __NUMPY_SETUP__ = False ######################### diff --git a/numpy/distutils/npy_pkg_config.py b/numpy/distutils/npy_pkg_config.py index ea16e772d..bfe8b9f77 100644 --- a/numpy/distutils/npy_pkg_config.py +++ b/numpy/distutils/npy_pkg_config.py @@ -222,9 +222,7 @@ def parse_meta(config): if not config.has_section('meta'): raise FormatError("No meta section found !") - d = {} - for name, value in config.items('meta'): - d[name] = value + d = dict(config.items('meta')) for k in ['name', 'description', 'version']: if not k in d: diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 79adcc334..cd63cc849 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -1124,8 +1124,9 @@ class atlas_info(system_info): lapack = None atlas_1 = None for d in lib_dirs: - atlas = self.check_libs2(d, atlas_libs, []) + # FIXME: lapack_atlas is unused lapack_atlas = self.check_libs2(d, ['lapack_atlas'], []) + atlas = self.check_libs2(d, atlas_libs, []) if atlas is not None: lib_dirs2 = [d] + self.combine_paths(d, ['atlas*', 'ATLAS*']) lapack = self.check_libs2(lib_dirs2, lapack_libs, []) @@ -2107,17 +2108,17 @@ class numerix_info(system_info): if which[0] is None: which = "numpy", "defaulted" try: - import numpy + import numpy # noqa: F401 which = "numpy", "defaulted" except ImportError: msg1 = str(get_exception()) try: - import Numeric + import Numeric # noqa: F401 which = "numeric", "defaulted" except ImportError: msg2 = str(get_exception()) try: - import numarray + import numarray # noqa: F401 which = "numarray", "defaulted" except ImportError: msg3 = str(get_exception()) @@ -2437,7 +2438,6 @@ class umfpack_info(system_info): define_macros=[('SCIPY_UMFPACK_H', None)], swig_opts=['-I' + inc_dir]) - amd = get_info('amd') dict_append(info, **get_info('amd')) self.set_info(**info) @@ -2533,6 +2533,7 @@ def show_all(argv=None): del show_only[show_only.index(name)] conf = c() conf.verbosity = 2 + # FIXME: r not used r = conf.get_info() if show_only: log.info('Info classes not defined: %s', ','.join(show_only)) diff --git a/numpy/distutils/tests/test_misc_util.py b/numpy/distutils/tests/test_misc_util.py index 2a3294ddf..3e239cf48 100644 --- a/numpy/distutils/tests/test_misc_util.py +++ b/numpy/distutils/tests/test_misc_util.py @@ -79,3 +79,6 @@ def test_installed_npymath_ini(): # Regression test for gh-7707. If npymath.ini wasn't installed, then this # will give an error. info = get_info('npymath') + + assert isinstance(info, dict) + assert "define_macros" in info diff --git a/numpy/doc/glossary.py b/numpy/doc/glossary.py index a3b9423a8..a3707340d 100644 --- a/numpy/doc/glossary.py +++ b/numpy/doc/glossary.py @@ -270,13 +270,11 @@ Glossary masked_array(data = [-- 2.0 --], mask = [ True False True], fill_value = 1e+20) - <BLANKLINE> >>> x + [1, 2, 3] masked_array(data = [-- 4.0 --], mask = [ True False True], fill_value = 1e+20) - <BLANKLINE> Masked arrays are often used when operating on arrays containing diff --git a/numpy/doc/structured_arrays.py b/numpy/doc/structured_arrays.py index ab97c5df6..e92a06124 100644 --- a/numpy/doc/structured_arrays.py +++ b/numpy/doc/structured_arrays.py @@ -35,26 +35,24 @@ with the field name:: array([('Rex', 5, 81.0), ('Fido', 5, 27.0)], dtype=[('name', 'S10'), ('age', '<i4'), ('weight', '<f4')]) -Structured arrays are designed for low-level manipulation of structured data, -for example, for interpreting binary blobs. Structured datatypes are -designed to mimic 'structs' in the C language, making them also useful for -interfacing with C code. For these purposes, numpy supports specialized -features such as subarrays and nested datatypes, and allows manual control over -the memory layout of the structure. - -For simple manipulation of tabular data other pydata projects, such as pandas, -xarray, or DataArray, provide higher-level interfaces that may be more -suitable. These projects may also give better performance for tabular data -analysis because the C-struct-like memory layout of structured arrays can lead -to poor cache behavior. +Structured datatypes are designed to be able to mimic 'structs' in the C +language, and share a similar memory layout. They are meant for interfacing with +C code and for low-level manipulation of structured buffers, for example for +interpreting binary blobs. For these purposes they support specialized features +such as subarrays, nested datatypes, and unions, and allow control over the +memory layout of the structure. + +Users looking to manipulate tabular data, such as stored in csv files, may find +other pydata projects more suitable, such as xarray, pandas, or DataArray. +These provide a high-level interface for tabular data analysis and are better +optimized for that use. For instance, the C-struct-like memory layout of +structured arrays in numpy can lead to poor cache behavior in comparison. .. _defining-structured-types: Structured Datatypes ==================== -To use structured arrays one first needs to define a structured datatype. - A structured datatype can be thought of as a sequence of bytes of a certain length (the structure's :term:`itemsize`) which is interpreted as a collection of fields. Each field has a name, a datatype, and a byte offset within the @@ -180,7 +178,9 @@ values are tuples containing the dtype and byte offset of each field. :: mappingproxy({'x': (dtype('int64'), 0), 'y': (dtype('float32'), 8)}) Both the ``names`` and ``fields`` attributes will equal ``None`` for -unstructured arrays. +unstructured arrays. The recommended way to test if a dtype is structured is +with `if dt.names is not None` rather than `if dt.names`, to account for dtypes +with 0 fields. The string representation of a structured datatype is shown in the "list of tuples" form if possible, otherwise numpy falls back to using the more general @@ -397,6 +397,15 @@ typically a non-structured array, except in the case of nested structures. >>> y.dtype, y.shape, y.strides (dtype('float32'), (2,), (12,)) +If the accessed field is a subarray, the dimensions of the subarray +are appended to the shape of the result:: + + >>> x = np.zeros((2,2), dtype=[('a', np.int32), ('b', np.float64, (3,3))]) + >>> x['a'].shape + (2, 2) + >>> x['b'].shape + (2, 2, 3, 3) + Accessing Multiple Fields ``````````````````````````` @@ -404,11 +413,10 @@ One can index and assign to a structured array with a multi-field index, where the index is a list of field names. .. warning:: - The behavior of multi-field indexes will change from Numpy 1.15 to Numpy - 1.16. + The behavior of multi-field indexes changed from Numpy 1.15 to Numpy 1.16. -In Numpy 1.16, the result of indexing with a multi-field index will be a view -into the original array, as follows:: +The result of indexing with a multi-field index is a view into the original +array, as follows:: >>> a = np.zeros(3, dtype=[('a', 'i4'), ('b', 'i4'), ('c', 'f4')]) >>> a[['a', 'c']] @@ -420,32 +428,58 @@ in the order they were indexed. Note that unlike for single-field indexing, the view's dtype has the same itemsize as the original array, and has fields at the same offsets as in the original array, and unindexed fields are merely missing. -In Numpy 1.15, indexing an array with a multi-field index returns a copy of -the result above for 1.16, but with fields packed together in memory as if -passed through :func:`numpy.lib.recfunctions.repack_fields`. This is the -behavior since Numpy 1.7. - .. warning:: - The new behavior in Numpy 1.16 leads to extra "padding" bytes at the - location of unindexed fields. You will need to update any code which depends - on the data having a "packed" layout. For instance code such as:: + In Numpy 1.15, indexing an array with a multi-field index returned a copy of + the result above, but with fields packed together in memory as if + passed through :func:`numpy.lib.recfunctions.repack_fields`. + + The new behavior as of Numpy 1.16 leads to extra "padding" bytes at the + location of unindexed fields compared to 1.15. You will need to update any + code which depends on the data having a "packed" layout. For instance code + such as:: + + >>> a = np.zeros(3, dtype=[('a', 'i4'), ('b', 'i4'), ('c', 'f4')]) + >>> a[['a','c']].view('i8') # Fails in Numpy 1.16 + ValueError: When changing to a smaller dtype, its size must be a divisor of the size of original dtype + + will need to be changed. This code has raised a ``FutureWarning`` since + Numpy 1.12, and similar code has raised ``FutureWarning`` since 1.7. + + In 1.16 a number of functions have been introduced in the + :module:`numpy.lib.recfunctions` module to help users account for this + change. These are + :func:`numpy.lib.recfunctions.repack_fields`. + :func:`numpy.lib.recfunctions.structured_to_unstructured`, + :func:`numpy.lib.recfunctions.unstructured_to_structured`, + :func:`numpy.lib.recfunctions.apply_along_fields`, + :func:`numpy.lib.recfunctions.assign_fields_by_name`, and + :func:`numpy.lib.recfunctions.require_fields`. + + The function :func:`numpy.lib.recfunctions.repack_fields` can always be + used to reproduce the old behavior, as it will return a packed copy of the + structured array. The code above, for example, can be replaced with: + + >>> repack_fields(a[['a','c']]).view('i8') # supported in 1.16 + array([0, 0, 0]) + + Furthermore, numpy now provides a new function + :func:`numpy.lib.recfunctions.structured_to_unstructured` which is a safer + and more efficient alternative for users who wish to convert structured + arrays to unstructured arrays, as the view above is often indeded to do. + This function allows safe conversion to an unstructured type taking into + account padding, often avoids a copy, and also casts the datatypes + as needed, unlike the view. Code such as: - >>> a[['a','c']].view('i8') # will fail in Numpy 1.16 - ValueError: When changing to a smaller dtype, its size must be a divisor of the size of original dtype + >>> a = np.zeros(3, dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) + >>> a[['x', 'z']].view('f4') - will need to be changed. This code has raised a ``FutureWarning`` since - Numpy 1.12. + can be made safer by replacing with: - The following is a recommended fix, which will behave identically in Numpy - 1.15 and Numpy 1.16:: + >>> structured_to_unstructured(a[['x', 'z']]) + array([0, 0, 0]) - >>> from numpy.lib.recfunctions import repack_fields - >>> repack_fields(a[['a','c']]).view('i8') # supported 1.15 and 1.16 - array([0, 0, 0]) -Assigning to an array with a multi-field index will behave the same in Numpy -1.15 and Numpy 1.16. In both versions the assignment will modify the original -array:: +Assignment to an array with a multi-field index modifies the original array:: >>> a[['a', 'c']] = (2, 3) >>> a diff --git a/numpy/f2py/__main__.py b/numpy/f2py/__main__.py index 6eff41099..708f7f362 100644 --- a/numpy/f2py/__main__.py +++ b/numpy/f2py/__main__.py @@ -1,6 +1,6 @@ # See http://cens.ioc.ee/projects/f2py2e/ from __future__ import division, print_function -from f2py2e import main +from numpy.f2py.f2py2e import main main() diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index 8e63d3cff..c41dd77c6 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -718,10 +718,7 @@ def modsign2map(m): def cb_sign2map(a, var, index=None): ret = {'varname': a} - if index is None or 1: # disable 7712 patch - ret['varname_i'] = ret['varname'] - else: - ret['varname_i'] = ret['varname'] + '_' + str(index) + ret['varname_i'] = ret['varname'] ret['ctype'] = getctype(var) if ret['ctype'] in c2capi_map: ret['atype'] = c2capi_map[ret['ctype']] diff --git a/numpy/f2py/common_rules.py b/numpy/f2py/common_rules.py index 1940d4211..62c1ba207 100644 --- a/numpy/f2py/common_rules.py +++ b/numpy/f2py/common_rules.py @@ -31,11 +31,9 @@ from .crackfortran import rmbadname def findcommonblocks(block, top=1): ret = [] if hascommon(block): - for n in block['common'].keys(): - vars = {} - for v in block['common'][n]: - vars[v] = block['vars'][v] - ret.append((n, block['common'][n], vars)) + for key, value in block['common'].items(): + vars_ = {v: block['vars'][v] for v in value} + ret.append((key, value, vars_)) elif hasbody(block): for b in block['body']: ret = ret + findcommonblocks(b, 0) diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index 361203a57..2620fc9b7 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -1849,10 +1849,8 @@ def postcrack2(block, tab='', param_map=None): if not f90modulevars: return block if isinstance(block, list): - ret = [] - for g in block: - g = postcrack2(g, tab=tab + '\t', param_map=param_map) - ret.append(g) + ret = [postcrack2(g, tab=tab + '\t', param_map=param_map) + for g in block] return ret setmesstext(block) outmess('%sBlock: %s\n' % (tab, block['name']), 0) @@ -1870,10 +1868,8 @@ def postcrack2(block, tab='', param_map=None): val = kind['kind'] if val in param_map: kind['kind'] = param_map[val] - new_body = [] - for b in block['body']: - b = postcrack2(b, tab=tab + '\t', param_map=param_map) - new_body.append(b) + new_body = [postcrack2(b, tab=tab + '\t', param_map=param_map) + for b in block['body']] block['body'] = new_body return block @@ -3211,10 +3207,8 @@ def vars2fortran(block, vars, args, tab='', as_interface=False): vardef = '%s(kind=%s)' % (vardef, selector['kind']) c = ' ' if 'attrspec' in vars[a]: - attr = [] - for l in vars[a]['attrspec']: - if l not in ['external']: - attr.append(l) + attr = [l for l in vars[a]['attrspec'] + if l not in ['external']] if attr: vardef = '%s, %s' % (vardef, ','.join(attr)) c = ',' diff --git a/numpy/f2py/tests/test_block_docstring.py b/numpy/f2py/tests/test_block_docstring.py index b2981aa82..8fc072a5e 100644 --- a/numpy/f2py/tests/test_block_docstring.py +++ b/numpy/f2py/tests/test_block_docstring.py @@ -1,6 +1,5 @@ from __future__ import division, absolute_import, print_function -import textwrap import sys import pytest from . import util diff --git a/numpy/f2py/tests/test_parameter.py b/numpy/f2py/tests/test_parameter.py index a0bbd9460..6a378687a 100644 --- a/numpy/f2py/tests/test_parameter.py +++ b/numpy/f2py/tests/test_parameter.py @@ -1,7 +1,6 @@ from __future__ import division, absolute_import, print_function import os -import math import pytest import numpy as np diff --git a/numpy/f2py/tests/test_quoted_character.py b/numpy/f2py/tests/test_quoted_character.py index 38e380802..c9a1c36f5 100644 --- a/numpy/f2py/tests/test_quoted_character.py +++ b/numpy/f2py/tests/test_quoted_character.py @@ -4,13 +4,9 @@ from __future__ import division, absolute_import, print_function import sys -import os -import uuid from importlib import import_module import pytest -import numpy.f2py - from numpy.testing import assert_equal from . import util diff --git a/numpy/f2py/tests/test_regression.py b/numpy/f2py/tests/test_regression.py index d695de61b..3adae635d 100644 --- a/numpy/f2py/tests/test_regression.py +++ b/numpy/f2py/tests/test_regression.py @@ -1,7 +1,6 @@ from __future__ import division, absolute_import, print_function import os -import math import pytest import numpy as np diff --git a/numpy/f2py/tests/util.py b/numpy/f2py/tests/util.py index 73fc27b96..5fa5dadd2 100644 --- a/numpy/f2py/tests/util.py +++ b/numpy/f2py/tests/util.py @@ -15,9 +15,7 @@ import shutil import atexit import textwrap import re -import random import pytest -import numpy.f2py from numpy.compat import asbytes, asstr from numpy.testing import temppath @@ -26,7 +24,7 @@ from importlib import import_module try: from hashlib import md5 except ImportError: - from md5 import new as md5 + from md5 import new as md5 # noqa: F401 # # Maintaining a temporary module directory diff --git a/numpy/fft/fftpack.py b/numpy/fft/fftpack.py index de675936f..d0df6fb48 100644 --- a/numpy/fft/fftpack.py +++ b/numpy/fft/fftpack.py @@ -177,19 +177,17 @@ def fft(a, n=None, axis=-1, norm=None): Examples -------- >>> np.fft.fft(np.exp(2j * np.pi * np.arange(8) / 8)) - array([ -3.44505240e-16 +1.14383329e-17j, - 8.00000000e+00 -5.71092652e-15j, - 2.33482938e-16 +1.22460635e-16j, - 1.64863782e-15 +1.77635684e-15j, - 9.95839695e-17 +2.33482938e-16j, - 0.00000000e+00 +1.66837030e-15j, - 1.14383329e-17 +1.22460635e-16j, - -1.64863782e-15 +1.77635684e-15j]) + array([-2.33486982e-16+1.14423775e-17j, 8.00000000e+00-1.25557246e-15j, + 2.33486982e-16+2.33486982e-16j, 0.00000000e+00+1.22464680e-16j, + -1.14423775e-17+2.33486982e-16j, 0.00000000e+00+5.20784380e-16j, + 1.14423775e-17+1.14423775e-17j, 0.00000000e+00+1.22464680e-16j]) In this example, real input has an FFT which is Hermitian, i.e., symmetric in the real part and anti-symmetric in the imaginary part, as described in the `numpy.fft` documentation: + >>> import matplotlib + >>> matplotlib.use('Agg') >>> import matplotlib.pyplot as plt >>> t = np.arange(256) >>> sp = np.fft.fft(np.sin(t)) @@ -278,19 +276,21 @@ def ifft(a, n=None, axis=-1, norm=None): Examples -------- >>> np.fft.ifft([0, 4, 0, 0]) - array([ 1.+0.j, 0.+1.j, -1.+0.j, 0.-1.j]) + array([ 1.+0.j, 0.+1.j, -1.+0.j, 0.-1.j]) # may vary Create and plot a band-limited signal with random phases: + >>> import matplotlib + >>> matplotlib.use('agg') >>> import matplotlib.pyplot as plt >>> t = np.arange(400) >>> n = np.zeros((400,), dtype=complex) >>> n[40:60] = np.exp(1j*np.random.uniform(0, 2*np.pi, (20,))) >>> s = np.fft.ifft(n) >>> plt.plot(t, s.real, 'b-', t, s.imag, 'r--') - ... + [<matplotlib.lines.Line2D object at ...>, <matplotlib.lines.Line2D object at ...>] >>> plt.legend(('real', 'imaginary')) - ... + <matplotlib.legend.Legend object at ...> >>> plt.show() """ @@ -374,9 +374,9 @@ def rfft(a, n=None, axis=-1, norm=None): Examples -------- >>> np.fft.fft([0, 1, 0, 0]) - array([ 1.+0.j, 0.-1.j, -1.+0.j, 0.+1.j]) + array([ 1.+0.j, 0.-1.j, -1.+0.j, 0.+1.j]) # may vary >>> np.fft.rfft([0, 1, 0, 0]) - array([ 1.+0.j, 0.-1.j, -1.+0.j]) + array([ 1.+0.j, 0.-1.j, -1.+0.j]) # may vary Notice how the final element of the `fft` output is the complex conjugate of the second element, for real input. For `rfft`, this symmetry is @@ -465,9 +465,9 @@ def irfft(a, n=None, axis=-1, norm=None): Examples -------- >>> np.fft.ifft([1, -1j, -1, 1j]) - array([ 0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j]) + array([0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j]) # may vary >>> np.fft.irfft([1, -1j, -1]) - array([ 0., 1., 0., 0.]) + array([0., 1., 0., 0.]) Notice how the last term in the input to the ordinary `ifft` is the complex conjugate of the second term, and the output has zero imaginary @@ -543,16 +543,16 @@ def hfft(a, n=None, axis=-1, norm=None): -------- >>> signal = np.array([1, 2, 3, 4, 3, 2]) >>> np.fft.fft(signal) - array([ 15.+0.j, -4.+0.j, 0.+0.j, -1.-0.j, 0.+0.j, -4.+0.j]) + array([15.+0.j, -4.+0.j, 0.+0.j, -1.-0.j, 0.+0.j, -4.+0.j]) # may vary >>> np.fft.hfft(signal[:4]) # Input first half of signal - array([ 15., -4., 0., -1., 0., -4.]) + array([15., -4., 0., -1., 0., -4.]) >>> np.fft.hfft(signal, 6) # Input entire signal and truncate - array([ 15., -4., 0., -1., 0., -4.]) + array([15., -4., 0., -1., 0., -4.]) >>> signal = np.array([[1, 1.j], [-1.j, 2]]) >>> np.conj(signal.T) - signal # check Hermitian symmetry - array([[ 0.-0.j, 0.+0.j], + array([[ 0.-0.j, -0.+0.j], # may vary [ 0.+0.j, 0.-0.j]]) >>> freq_spectrum = np.fft.hfft(signal) >>> freq_spectrum @@ -616,9 +616,9 @@ def ihfft(a, n=None, axis=-1, norm=None): -------- >>> spectrum = np.array([ 15, -4, 0, -1, 0, -4]) >>> np.fft.ifft(spectrum) - array([ 1.+0.j, 2.-0.j, 3.+0.j, 4.+0.j, 3.+0.j, 2.-0.j]) + array([1.+0.j, 2.+0.j, 3.+0.j, 4.+0.j, 3.+0.j, 2.+0.j]) # may vary >>> np.fft.ihfft(spectrum) - array([ 1.-0.j, 2.-0.j, 3.-0.j, 4.-0.j]) + array([ 1.-0.j, 2.-0.j, 3.-0.j, 4.-0.j]) # may vary """ # The copy may be required for multithreading. @@ -732,17 +732,17 @@ def fftn(a, s=None, axes=None, norm=None): -------- >>> a = np.mgrid[:3, :3, :3][0] >>> np.fft.fftn(a, axes=(1, 2)) - array([[[ 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j]], - [[ 9.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j]], - [[ 18.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j]]]) + array([[[ 0.+0.j, 0.+0.j, 0.+0.j], # may vary + [ 0.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j]], + [[ 9.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j]], + [[18.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j], + [ 0.+0.j, 0.+0.j, 0.+0.j]]]) >>> np.fft.fftn(a, (2, 2), axes=(0, 1)) - array([[[ 2.+0.j, 2.+0.j, 2.+0.j], + array([[[ 2.+0.j, 2.+0.j, 2.+0.j], # may vary [ 0.+0.j, 0.+0.j, 0.+0.j]], [[-2.+0.j, -2.+0.j, -2.+0.j], [ 0.+0.j, 0.+0.j, 0.+0.j]]]) @@ -838,10 +838,10 @@ def ifftn(a, s=None, axes=None, norm=None): -------- >>> a = np.eye(4) >>> np.fft.ifftn(np.fft.fftn(a, axes=(0,)), axes=(1,)) - array([[ 1.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 1.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j, 1.+0.j]]) + array([[1.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], # may vary + [0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j], + [0.+0.j, 0.+0.j, 1.+0.j, 0.+0.j], + [0.+0.j, 0.+0.j, 0.+0.j, 1.+0.j]]) Create and plot an image with band-limited frequency content: @@ -934,16 +934,16 @@ def fft2(a, s=None, axes=(-2, -1), norm=None): -------- >>> a = np.mgrid[:5, :5][0] >>> np.fft.fft2(a) - array([[ 50.0 +0.j , 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ], - [-12.5+17.20477401j, 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ], - [-12.5 +4.0614962j , 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ], - [-12.5 -4.0614962j , 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ], - [-12.5-17.20477401j, 0.0 +0.j , 0.0 +0.j , - 0.0 +0.j , 0.0 +0.j ]]) + array([[ 50. +0.j , 0. +0.j , 0. +0.j , # may vary + 0. +0.j , 0. +0.j ], + [-12.5+17.20477401j, 0. +0.j , 0. +0.j , + 0. +0.j , 0. +0.j ], + [-12.5 +4.0614962j , 0. +0.j , 0. +0.j , + 0. +0.j , 0. +0.j ], + [-12.5 -4.0614962j , 0. +0.j , 0. +0.j , + 0. +0.j , 0. +0.j ], + [-12.5-17.20477401j, 0. +0.j , 0. +0.j , + 0. +0.j , 0. +0.j ]]) """ @@ -1028,10 +1028,10 @@ def ifft2(a, s=None, axes=(-2, -1), norm=None): -------- >>> a = 4 * np.eye(4) >>> np.fft.ifft2(a) - array([[ 1.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j, 0.+0.j, 1.+0.j], - [ 0.+0.j, 0.+0.j, 1.+0.j, 0.+0.j], - [ 0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j]]) + array([[1.+0.j, 0.+0.j, 0.+0.j, 0.+0.j], # may vary + [0.+0.j, 0.+0.j, 0.+0.j, 1.+0.j], + [0.+0.j, 0.+0.j, 1.+0.j, 0.+0.j], + [0.+0.j, 1.+0.j, 0.+0.j, 0.+0.j]]) """ @@ -1110,16 +1110,16 @@ def rfftn(a, s=None, axes=None, norm=None): -------- >>> a = np.ones((2, 2, 2)) >>> np.fft.rfftn(a) - array([[[ 8.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j]], - [[ 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j]]]) + array([[[8.+0.j, 0.+0.j], # may vary + [0.+0.j, 0.+0.j]], + [[0.+0.j, 0.+0.j], + [0.+0.j, 0.+0.j]]]) >>> np.fft.rfftn(a, axes=(2, 0)) - array([[[ 4.+0.j, 0.+0.j], - [ 4.+0.j, 0.+0.j]], - [[ 0.+0.j, 0.+0.j], - [ 0.+0.j, 0.+0.j]]]) + array([[[4.+0.j, 0.+0.j], # may vary + [4.+0.j, 0.+0.j]], + [[0.+0.j, 0.+0.j], + [0.+0.j, 0.+0.j]]]) """ # The copy may be required for multithreading. @@ -1247,12 +1247,12 @@ def irfftn(a, s=None, axes=None, norm=None): >>> a = np.zeros((3, 2, 2)) >>> a[0, 0, 0] = 3 * 2 * 2 >>> np.fft.irfftn(a) - array([[[ 1., 1.], - [ 1., 1.]], - [[ 1., 1.], - [ 1., 1.]], - [[ 1., 1.], - [ 1., 1.]]]) + array([[[1., 1.], + [1., 1.]], + [[1., 1.], + [1., 1.]], + [[1., 1.], + [1., 1.]]]) """ # The copy may be required for multithreading. diff --git a/numpy/fft/helper.py b/numpy/fft/helper.py index e65883651..9985f6d4c 100644 --- a/numpy/fft/helper.py +++ b/numpy/fft/helper.py @@ -11,7 +11,7 @@ except ImportError: import dummy_threading as threading from numpy.compat import integer_types from numpy.core import integer, empty, arange, asarray, roll -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import array_function_dispatch, set_module # Created by Pearu Peterson, September 2002 @@ -52,7 +52,7 @@ def fftshift(x, axes=None): -------- >>> freqs = np.fft.fftfreq(10, 0.1) >>> freqs - array([ 0., 1., 2., 3., 4., -5., -4., -3., -2., -1.]) + array([ 0., 1., 2., ..., -3., -2., -1.]) >>> np.fft.fftshift(freqs) array([-5., -4., -3., -2., -1., 0., 1., 2., 3., 4.]) @@ -128,6 +128,7 @@ def ifftshift(x, axes=None): return roll(x, shift, axes) +@set_module('numpy.fft') def fftfreq(n, d=1.0): """ Return the Discrete Fourier Transform sample frequencies. @@ -161,7 +162,7 @@ def fftfreq(n, d=1.0): >>> timestep = 0.1 >>> freq = np.fft.fftfreq(n, d=timestep) >>> freq - array([ 0. , 1.25, 2.5 , 3.75, -5. , -3.75, -2.5 , -1.25]) + array([ 0. , 1.25, 2.5 , ..., -3.75, -2.5 , -1.25]) """ if not isinstance(n, integer_types): @@ -174,9 +175,9 @@ def fftfreq(n, d=1.0): p2 = arange(-(n//2), 0, dtype=int) results[N:] = p2 return results * val - #return hstack((arange(0,(n-1)/2 + 1), arange(-(n/2),0))) / (n*d) +@set_module('numpy.fft') def rfftfreq(n, d=1.0): """ Return the Discrete Fourier Transform sample frequencies @@ -214,7 +215,7 @@ def rfftfreq(n, d=1.0): >>> sample_rate = 100 >>> freq = np.fft.fftfreq(n, d=1./sample_rate) >>> freq - array([ 0., 10., 20., 30., 40., -50., -40., -30., -20., -10.]) + array([ 0., 10., 20., ..., -30., -20., -10.]) >>> freq = np.fft.rfftfreq(n, d=1./sample_rate) >>> freq array([ 0., 10., 20., 30., 40., 50.]) diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index ab00b1444..3a0e67f60 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -20,17 +20,18 @@ gzip, bz2 and xz are supported. Example:: >>> # Create a DataSource, use os.curdir (default) for local storage. - >>> ds = datasource.DataSource() + >>> from numpy import DataSource + >>> ds = DataSource() >>> >>> # Open a remote file. >>> # DataSource downloads the file, stores it locally in: >>> # './www.google.com/index.html' >>> # opens the file and returns a file object. - >>> fp = ds.open('http://www.google.com/index.html') + >>> fp = ds.open('http://www.google.com/') # doctest: +SKIP >>> >>> # Use the file as you normally would - >>> fp.read() - >>> fp.close() + >>> fp.read() # doctest: +SKIP + >>> fp.close() # doctest: +SKIP """ from __future__ import division, absolute_import, print_function @@ -41,8 +42,12 @@ import warnings import shutil import io +from numpy.core.overrides import set_module + + _open = open + def _check_mode(mode, encoding, newline): """Check mode and that encoding and newline are compatible. @@ -152,6 +157,7 @@ class _FileOpeners(object): Examples -------- + >>> import gzip >>> np.lib._datasource._file_openers.keys() [None, '.bz2', '.gz', '.xz', '.lzma'] >>> np.lib._datasource._file_openers['.gz'] is gzip.open @@ -262,7 +268,8 @@ def open(path, mode='r', destpath=os.curdir, encoding=None, newline=None): return ds.open(path, mode, encoding=encoding, newline=newline) -class DataSource (object): +@set_module('numpy') +class DataSource(object): """ DataSource(destpath='.') @@ -285,7 +292,7 @@ class DataSource (object): URLs require a scheme string (``http://``) to be used, without it they will fail:: - >>> repos = DataSource() + >>> repos = np.DataSource() >>> repos.exists('www.google.com/index.html') False >>> repos.exists('http://www.google.com/index.html') @@ -297,17 +304,17 @@ class DataSource (object): -------- :: - >>> ds = DataSource('/home/guido') - >>> urlname = 'http://www.google.com/index.html' - >>> gfile = ds.open('http://www.google.com/index.html') # remote file + >>> ds = np.DataSource('/home/guido') + >>> urlname = 'http://www.google.com/' + >>> gfile = ds.open('http://www.google.com/') >>> ds.abspath(urlname) - '/home/guido/www.google.com/site/index.html' + '/home/guido/www.google.com/index.html' - >>> ds = DataSource(None) # use with temporary file + >>> ds = np.DataSource(None) # use with temporary file >>> ds.open('/home/guido/foobar.txt') <open file '/home/guido.foobar.txt', mode 'r' at 0x91d4430> >>> ds.abspath('/home/guido/foobar.txt') - '/tmp/tmpy4pgsP/home/guido/foobar.txt' + '/tmp/.../home/guido/foobar.txt' """ @@ -323,7 +330,7 @@ class DataSource (object): def __del__(self): # Remove temp directories - if self._istmpdest: + if hasattr(self, '_istmpdest') and self._istmpdest: shutil.rmtree(self._destpath) def _iszip(self, filename): diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index b604b8c52..0ebd39b8c 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -8,7 +8,7 @@ __docformat__ = "restructuredtext en" import sys import numpy as np import numpy.core.numeric as nx -from numpy.compat import asbytes, asunicode, bytes, asbytes_nested, basestring +from numpy.compat import asbytes, asunicode, bytes, basestring if sys.version_info[0] >= 3: from builtins import bool, int, float, complex, object, str @@ -146,11 +146,17 @@ def flatten_dtype(ndtype, flatten_base=False): >>> dt = np.dtype([('name', 'S4'), ('x', float), ('y', float), ... ('block', int, (2, 3))]) >>> np.lib._iotools.flatten_dtype(dt) - [dtype('|S4'), dtype('float64'), dtype('float64'), dtype('int32')] + [dtype('S4'), dtype('float64'), dtype('float64'), dtype('int64')] >>> np.lib._iotools.flatten_dtype(dt, flatten_base=True) - [dtype('|S4'), dtype('float64'), dtype('float64'), dtype('int32'), - dtype('int32'), dtype('int32'), dtype('int32'), dtype('int32'), - dtype('int32')] + [dtype('S4'), + dtype('float64'), + dtype('float64'), + dtype('int64'), + dtype('int64'), + dtype('int64'), + dtype('int64'), + dtype('int64'), + dtype('int64')] """ names = ndtype.names @@ -309,13 +315,13 @@ class NameValidator(object): -------- >>> validator = np.lib._iotools.NameValidator() >>> validator(['file', 'field2', 'with space', 'CaSe']) - ['file_', 'field2', 'with_space', 'CaSe'] + ('file_', 'field2', 'with_space', 'CaSe') >>> validator = np.lib._iotools.NameValidator(excludelist=['excl'], - deletechars='q', - case_sensitive='False') + ... deletechars='q', + ... case_sensitive=False) >>> validator(['excl', 'field2', 'no_q', 'with space', 'CaSe']) - ['excl_', 'field2', 'no_', 'with_space', 'case'] + ('EXCL', 'FIELD2', 'NO_Q', 'WITH_SPACE', 'CASE') """ # @@ -599,7 +605,7 @@ class StringConverter(object): -------- >>> import dateutil.parser >>> import datetime - >>> dateparser = datetustil.parser.parse + >>> dateparser = dateutil.parser.parse >>> defaultdate = datetime.date(2000, 1, 1) >>> StringConverter.upgrade_mapper(dateparser, default=defaultdate) """ @@ -693,7 +699,7 @@ class StringConverter(object): self.func = lambda x: int(float(x)) # Store the list of strings corresponding to missing values. if missing_values is None: - self.missing_values = set(['']) + self.missing_values = {''} else: if isinstance(missing_values, basestring): missing_values = missing_values.split(",") diff --git a/numpy/lib/_version.py b/numpy/lib/_version.py index c3563a7fa..8aa999fc9 100644 --- a/numpy/lib/_version.py +++ b/numpy/lib/_version.py @@ -47,9 +47,12 @@ class NumpyVersion(): >>> from numpy.lib import NumpyVersion >>> if NumpyVersion(np.__version__) < '1.7.0': ... print('skip') - skip + >>> # skip >>> NumpyVersion('1.7') # raises ValueError, add ".0" + Traceback (most recent call last): + ... + ValueError: Not a valid numpy version string """ diff --git a/numpy/lib/arraypad.py b/numpy/lib/arraypad.py index d27a3918f..b236cc449 100644 --- a/numpy/lib/arraypad.py +++ b/numpy/lib/arraypad.py @@ -886,105 +886,71 @@ def _pad_wrap(arr, pad_amt, axis=-1): return np.concatenate((wrap_chunk1, arr, wrap_chunk2), axis=axis) -def _normalize_shape(ndarray, shape, cast_to_int=True): +def _as_pairs(x, ndim, as_index=False): """ - Private function which does some checks and normalizes the possibly - much simpler representations of 'pad_width', 'stat_length', - 'constant_values', 'end_values'. + Broadcast `x` to an array with the shape (`ndim`, 2). - Parameters - ---------- - narray : ndarray - Input ndarray - shape : {sequence, array_like, float, int}, optional - The width of padding (pad_width), the number of elements on the - edge of the narray used for statistics (stat_length), the constant - value(s) to use when filling padded regions (constant_values), or the - endpoint target(s) for linear ramps (end_values). - ((before_1, after_1), ... (before_N, after_N)) unique number of - elements for each axis where `N` is rank of `narray`. - ((before, after),) yields same before and after constants for each - axis. - (constant,) or val is a shortcut for before = after = constant for - all axes. - cast_to_int : bool, optional - Controls if values in ``shape`` will be rounded and cast to int - before being returned. - - Returns - ------- - normalized_shape : tuple of tuples - val => ((val, val), (val, val), ...) - [[val1, val2], [val3, val4], ...] => ((val1, val2), (val3, val4), ...) - ((val1, val2), (val3, val4), ...) => no change - [[val1, val2], ] => ((val1, val2), (val1, val2), ...) - ((val1, val2), ) => ((val1, val2), (val1, val2), ...) - [[val , ], ] => ((val, val), (val, val), ...) - ((val , ), ) => ((val, val), (val, val), ...) - - """ - ndims = ndarray.ndim - - # Shortcut shape=None - if shape is None: - return ((None, None), ) * ndims - - # Convert any input `info` to a NumPy array - shape_arr = np.asarray(shape) - - try: - shape_arr = np.broadcast_to(shape_arr, (ndims, 2)) - except ValueError: - fmt = "Unable to create correctly shaped tuple from %s" - raise ValueError(fmt % (shape,)) - - # Cast if necessary - if cast_to_int is True: - shape_arr = np.round(shape_arr).astype(int) - - # Convert list of lists to tuple of tuples - return tuple(tuple(axis) for axis in shape_arr.tolist()) - - -def _validate_lengths(narray, number_elements): - """ - Private function which does some checks and reformats pad_width and - stat_length using _normalize_shape. + A helper function for `pad` that prepares and validates arguments like + `pad_width` for iteration in pairs. Parameters ---------- - narray : ndarray - Input ndarray - number_elements : {sequence, int}, optional - The width of padding (pad_width) or the number of elements on the edge - of the narray used for statistics (stat_length). - ((before_1, after_1), ... (before_N, after_N)) unique number of - elements for each axis. - ((before, after),) yields same before and after constants for each - axis. - (constant,) or int is a shortcut for before = after = constant for all - axes. + x : {None, scalar, array-like} + The object to broadcast to the shape (`ndim`, 2). + ndim : int + Number of pairs the broadcasted `x` will have. + as_index : bool, optional + If `x` is not None, try to round each element of `x` to an integer + (dtype `np.intp`) and ensure every element is positive. Returns ------- - _validate_lengths : tuple of tuples - int => ((int, int), (int, int), ...) - [[int1, int2], [int3, int4], ...] => ((int1, int2), (int3, int4), ...) - ((int1, int2), (int3, int4), ...) => no change - [[int1, int2], ] => ((int1, int2), (int1, int2), ...) - ((int1, int2), ) => ((int1, int2), (int1, int2), ...) - [[int , ], ] => ((int, int), (int, int), ...) - ((int , ), ) => ((int, int), (int, int), ...) - + pairs : nested iterables, shape (`ndim`, 2) + The broadcasted version of `x`. + + Raises + ------ + ValueError + If `as_index` is True and `x` contains negative elements. + Or if `x` is not broadcastable to the shape (`ndim`, 2). """ - normshp = _normalize_shape(narray, number_elements) - for i in normshp: - chk = [1 if x is None else x for x in i] - chk = [1 if x >= 0 else -1 for x in chk] - if (chk[0] < 0) or (chk[1] < 0): - fmt = "%s cannot contain negative values." - raise ValueError(fmt % (number_elements,)) - return normshp + if x is None: + # Pass through None as a special case, otherwise np.round(x) fails + # with an AttributeError + return ((None, None),) * ndim + + x = np.array(x) + if as_index: + x = np.round(x).astype(np.intp, copy=False) + + if x.ndim < 3: + # Optimization: Possibly use faster paths for cases where `x` has + # only 1 or 2 elements. `np.broadcast_to` could handle these as well + # but is currently slower + + if x.size == 1: + # x was supplied as a single value + x = x.ravel() # Ensure x[0] works for x.ndim == 0, 1, 2 + if as_index and x < 0: + raise ValueError("index can't contain negative values") + return ((x[0], x[0]),) * ndim + + if x.size == 2 and x.shape != (2, 1): + # x was supplied with a single value for each side + # but except case when each dimension has a single value + # which should be broadcasted to a pair, + # e.g. [[1], [2]] -> [[1, 1], [2, 2]] not [[1, 2], [1, 2]] + x = x.ravel() # Ensure x[0], x[1] works + if as_index and (x[0] < 0 or x[1] < 0): + raise ValueError("index can't contain negative values") + return ((x[0], x[1]),) * ndim + + if as_index and x.min() < 0: + raise ValueError("index can't contain negative values") + + # Converting the array with `tolist` seems to improve performance + # when iterating and indexing the result (see usage in `pad`) + return np.broadcast_to(x, (ndim, 2)).tolist() ############################################################################### @@ -1134,10 +1100,10 @@ def pad(array, pad_width, mode, **kwargs): -------- >>> a = [1, 2, 3, 4, 5] >>> np.pad(a, (2,3), 'constant', constant_values=(4, 6)) - array([4, 4, 1, 2, 3, 4, 5, 6, 6, 6]) + array([4, 4, 1, ..., 6, 6, 6]) >>> np.pad(a, (2, 3), 'edge') - array([1, 1, 1, 2, 3, 4, 5, 5, 5, 5]) + array([1, 1, 1, ..., 5, 5, 5]) >>> np.pad(a, (2, 3), 'linear_ramp', end_values=(5, -4)) array([ 5, 3, 1, 2, 3, 4, 5, 2, -1, -4]) @@ -1203,7 +1169,7 @@ def pad(array, pad_width, mode, **kwargs): raise TypeError('`pad_width` must be of integral type.') narray = np.array(array) - pad_width = _validate_lengths(narray, pad_width) + pad_width = _as_pairs(pad_width, narray.ndim, as_index=True) allowedkwargs = { 'constant': ['constant_values'], @@ -1239,10 +1205,9 @@ def pad(array, pad_width, mode, **kwargs): # Need to only normalize particular keywords. for i in kwargs: if i == 'stat_length': - kwargs[i] = _validate_lengths(narray, kwargs[i]) + kwargs[i] = _as_pairs(kwargs[i], narray.ndim, as_index=True) if i in ['end_values', 'constant_values']: - kwargs[i] = _normalize_shape(narray, kwargs[i], - cast_to_int=False) + kwargs[i] = _as_pairs(kwargs[i], narray.ndim) else: # Drop back to old, slower np.apply_along_axis mode for user-supplied # vector function diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index 850e20123..558150e48 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -82,7 +82,7 @@ def ediff1d(ary, to_end=None, to_begin=None): array([ 1, 2, 3, -7]) >>> np.ediff1d(x, to_begin=-99, to_end=np.array([88, 99])) - array([-99, 1, 2, 3, -7, 88, 99]) + array([-99, 1, 2, ..., -7, 88, 99]) The returned array is always 1D. @@ -241,13 +241,11 @@ def unique(ar, return_index=False, return_inverse=False, >>> a = np.array(['a', 'b', 'b', 'c', 'a']) >>> u, indices = np.unique(a, return_index=True) >>> u - array(['a', 'b', 'c'], - dtype='|S1') + array(['a', 'b', 'c'], dtype='<U1') >>> indices array([0, 1, 3]) >>> a[indices] - array(['a', 'b', 'c'], - dtype='|S1') + array(['a', 'b', 'c'], dtype='<U1') Reconstruct the input array from the unique values: @@ -256,9 +254,9 @@ def unique(ar, return_index=False, return_inverse=False, >>> u array([1, 2, 3, 4, 6]) >>> indices - array([0, 1, 4, 3, 1, 2, 1]) + array([0, 1, 4, ..., 1, 2, 1]) >>> u[indices] - array([1, 2, 6, 4, 2, 3, 2]) + array([1, 2, 6, ..., 2, 3, 2]) """ ar = np.asanyarray(ar) @@ -479,6 +477,11 @@ def setxor1d(ar1, ar2, assume_unique=False): return aux[flag[1:] & flag[:-1]] +def _in1d_dispatcher(ar1, ar2, assume_unique=None, invert=None): + return (ar1, ar2) + + +@array_function_dispatch(_in1d_dispatcher) def in1d(ar1, ar2, assume_unique=False, invert=False): """ Test whether each element of a 1-D array is also present in a second array. @@ -656,8 +659,8 @@ def isin(element, test_elements, assume_unique=False, invert=False): >>> test_elements = [1, 2, 4, 8] >>> mask = np.isin(element, test_elements) >>> mask - array([[ False, True], - [ True, False]]) + array([[False, True], + [ True, False]]) >>> element[mask] array([2, 4]) @@ -671,7 +674,7 @@ def isin(element, test_elements, assume_unique=False, invert=False): >>> mask = np.isin(element, test_elements, invert=True) >>> mask array([[ True, False], - [ False, True]]) + [False, True]]) >>> element[mask] array([0, 6]) @@ -680,14 +683,14 @@ def isin(element, test_elements, assume_unique=False, invert=False): >>> test_set = {1, 2, 4, 8} >>> np.isin(element, test_set) - array([[ False, False], - [ False, False]]) + array([[False, False], + [False, False]]) Casting the set to a list gives the expected result: >>> np.isin(element, list(test_set)) - array([[ False, True], - [ True, False]]) + array([[False, True], + [ True, False]]) """ element = np.asarray(element) return in1d(element, test_elements, assume_unique=assume_unique, diff --git a/numpy/lib/arrayterator.py b/numpy/lib/arrayterator.py index f2d4fe9fd..c16668582 100644 --- a/numpy/lib/arrayterator.py +++ b/numpy/lib/arrayterator.py @@ -80,9 +80,8 @@ class Arrayterator(object): >>> for subarr in a_itor: ... if not subarr.all(): - ... print(subarr, subarr.shape) - ... - [[[[0 1]]]] (1, 1, 1, 2) + ... print(subarr, subarr.shape) # doctest: +SKIP + >>> # [[[[0 1]]]] (1, 1, 1, 2) """ @@ -160,7 +159,7 @@ class Arrayterator(object): ... if not subarr: ... print(subarr, type(subarr)) ... - 0 <type 'numpy.int32'> + 0 <class 'numpy.int64'> """ for block in self: diff --git a/numpy/lib/financial.py b/numpy/lib/financial.py index e1e297492..216687475 100644 --- a/numpy/lib/financial.py +++ b/numpy/lib/financial.py @@ -127,7 +127,7 @@ def fv(rate, nper, pmt, pv, when='end'): >>> a = np.array((0.05, 0.06, 0.07))/12 >>> np.fv(a, 10*12, -100, -100) - array([ 15692.92889434, 16569.87435405, 17509.44688102]) + array([ 15692.92889434, 16569.87435405, 17509.44688102]) # may vary """ when = _convert_when(when) @@ -275,7 +275,7 @@ def nper(rate, pmt, pv, fv=0, when='end'): If you only had $150/month to pay towards the loan, how long would it take to pay-off a loan of $8,000 at 7% annual interest? - >>> print(round(np.nper(0.07/12, -150, 8000), 5)) + >>> print(np.round(np.nper(0.07/12, -150, 8000), 5)) 64.07335 So, over 64 months would be required to pay off the loan. @@ -286,10 +286,10 @@ def nper(rate, pmt, pv, fv=0, when='end'): >>> np.nper(*(np.ogrid[0.07/12: 0.08/12: 0.01/12, ... -150 : -99 : 50 , ... 8000 : 9001 : 1000])) - array([[[ 64.07334877, 74.06368256], - [ 108.07548412, 127.99022654]], - [[ 66.12443902, 76.87897353], - [ 114.70165583, 137.90124779]]]) + array([[[ 64.07334877, 74.06368256], + [108.07548412, 127.99022654]], + [[ 66.12443902, 76.87897353], + [114.70165583, 137.90124779]]]) """ when = _convert_when(when) @@ -539,7 +539,7 @@ def pv(rate, nper, pmt, fv=0, when='end'): >>> a = np.array((0.05, 0.04, 0.03))/12 >>> np.pv(a, 10*12, -100, 15692.93) - array([ -100.00067132, -649.26771385, -1273.78633713]) + array([ -100.00067132, -649.26771385, -1273.78633713]) # may vary So, to end up with the same $15692.93 under the same $100 per month "savings plan," for annual interest rates of 4% and 3%, one would @@ -704,15 +704,15 @@ def irr(values): Examples -------- - >>> round(irr([-100, 39, 59, 55, 20]), 5) + >>> round(np.irr([-100, 39, 59, 55, 20]), 5) 0.28095 - >>> round(irr([-100, 0, 0, 74]), 5) + >>> round(np.irr([-100, 0, 0, 74]), 5) -0.0955 - >>> round(irr([-100, 100, 0, -7]), 5) + >>> round(np.irr([-100, 100, 0, -7]), 5) -0.0833 - >>> round(irr([-100, 100, 0, 7]), 5) + >>> round(np.irr([-100, 100, 0, 7]), 5) 0.06206 - >>> round(irr([-5, 10.5, 1, -8, 1]), 5) + >>> round(np.irr([-5, 10.5, 1, -8, 1]), 5) 0.0886 (Compare with the Example given for numpy.lib.financial.npv) @@ -777,7 +777,7 @@ def npv(rate, values): Examples -------- >>> np.npv(0.281,[-100, 39, 59, 55, 20]) - -0.0084785916384548798 + -0.0084785916384548798 # may vary (Compare with the Example given for numpy.lib.financial.irr) diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 1ef3dca47..10945e5e8 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -259,6 +259,43 @@ def dtype_to_descr(dtype): else: return dtype.str +def descr_to_dtype(descr): + ''' + descr may be stored as dtype.descr, which is a list of + (name, format, [shape]) tuples. Offsets are not explicitly saved, rather + empty fields with name,format == '', '|Vn' are added as padding. + + This function reverses the process, eliminating the empty padding fields. + ''' + if isinstance(descr, (str, dict)): + # No padding removal needed + return numpy.dtype(descr) + + fields = [] + offset = 0 + for field in descr: + if len(field) == 2: + name, descr_str = field + dt = descr_to_dtype(descr_str) + else: + name, descr_str, shape = field + dt = numpy.dtype((descr_to_dtype(descr_str), shape)) + + # Ignore padding bytes, which will be void bytes with '' as name + # Once support for blank names is removed, only "if name == ''" needed) + is_pad = (name == '' and dt.type is numpy.void and dt.names is None) + if not is_pad: + fields.append((name, dt, offset)) + + offset += dt.itemsize + + names, formats, offsets = zip(*fields) + # names may be (title, names) tuples + nametups = (n if isinstance(n, tuple) else (None, n) for n in names) + titles, names = zip(*nametups) + return numpy.dtype({'names': names, 'formats': formats, 'titles': titles, + 'offsets': offsets, 'itemsize': offset}) + def header_data_from_array_1_0(array): """ Get the dictionary of header metadata from a numpy.ndarray. @@ -523,7 +560,7 @@ def _read_array_header(fp, version): msg = "fortran_order is not a valid bool: %r" raise ValueError(msg % (d['fortran_order'],)) try: - dtype = numpy.dtype(d['descr']) + dtype = descr_to_dtype(d['descr']) except TypeError as e: msg = "descr is not a valid dtype descriptor: %r" raise ValueError(msg % (d['descr'],)) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index fae6541bc..1ead375de 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -27,6 +27,7 @@ from numpy.core.fromnumeric import ( ravel, nonzero, partition, mean, any, sum ) from numpy.core.numerictypes import typecodes +from numpy.core.overrides import set_module from numpy.core import overrides from numpy.core.function_base import add_newdoc from numpy.lib.twodim_base import diag @@ -217,12 +218,12 @@ def flip(m, axis=None): [2, 3]], [[4, 5], [6, 7]]]) - >>> flip(A, 0) + >>> np.flip(A, 0) array([[[4, 5], [6, 7]], [[0, 1], [2, 3]]]) - >>> flip(A, 1) + >>> np.flip(A, 1) array([[[2, 3], [0, 1]], [[6, 7], @@ -238,7 +239,7 @@ def flip(m, axis=None): [[1, 0], [3, 2]]]) >>> A = np.random.randn(3,4,5) - >>> np.all(flip(A,2) == A[:,:,::-1,...]) + >>> np.all(np.flip(A,2) == A[:,:,::-1,...]) True """ if not hasattr(m, 'ndim'): @@ -254,6 +255,7 @@ def flip(m, axis=None): return m[indexer] +@set_module('numpy') def iterable(y): """ Check whether or not an object can be iterated over. @@ -357,7 +359,7 @@ def average(a, axis=None, weights=None, returned=False): Examples -------- - >>> data = range(1,5) + >>> data = list(range(1,5)) >>> data [1, 2, 3, 4] >>> np.average(data) @@ -371,11 +373,10 @@ def average(a, axis=None, weights=None, returned=False): [2, 3], [4, 5]]) >>> np.average(data, axis=1, weights=[1./4, 3./4]) - array([ 0.75, 2.75, 4.75]) + array([0.75, 2.75, 4.75]) >>> np.average(data, weights=[1./4, 3./4]) - Traceback (most recent call last): - ... + ... TypeError: Axis must be specified when shapes of a and weights differ. >>> a = np.ones(5, dtype=np.float128) @@ -429,6 +430,7 @@ def average(a, axis=None, weights=None, returned=False): return avg +@set_module('numpy') def asarray_chkfinite(a, dtype=None, order=None): """Convert the input to an array, checking for NaNs or Infs. @@ -583,7 +585,7 @@ def piecewise(x, condlist, funclist, *args, **kw): ``x >= 0``. >>> np.piecewise(x, [x < 0, x >= 0], [lambda x: -x, lambda x: x]) - array([ 2.5, 1.5, 0.5, 0.5, 1.5, 2.5]) + array([2.5, 1.5, 0.5, 0.5, 1.5, 2.5]) Apply the same function to a scalar value. @@ -668,7 +670,7 @@ def select(condlist, choicelist, default=0): >>> condlist = [x<3, x>5] >>> choicelist = [x, x**2] >>> np.select(condlist, choicelist) - array([ 0, 1, 2, 0, 0, 0, 36, 49, 64, 81]) + array([ 0, 1, 2, ..., 49, 64, 81]) """ # Check the size of condlist and choicelist are the same, or abort. @@ -851,9 +853,9 @@ def gradient(f, *varargs, **kwargs): -------- >>> f = np.array([1, 2, 4, 7, 11, 16], dtype=float) >>> np.gradient(f) - array([ 1. , 1.5, 2.5, 3.5, 4.5, 5. ]) + array([1. , 1.5, 2.5, 3.5, 4.5, 5. ]) >>> np.gradient(f, 2) - array([ 0.5 , 0.75, 1.25, 1.75, 2.25, 2.5 ]) + array([0.5 , 0.75, 1.25, 1.75, 2.25, 2.5 ]) Spacing can be also specified with an array that represents the coordinates of the values F along the dimensions. @@ -861,13 +863,13 @@ def gradient(f, *varargs, **kwargs): >>> x = np.arange(f.size) >>> np.gradient(f, x) - array([ 1. , 1.5, 2.5, 3.5, 4.5, 5. ]) + array([1. , 1.5, 2.5, 3.5, 4.5, 5. ]) Or a non uniform one: >>> x = np.array([0., 1., 1.5, 3.5, 4., 6.], dtype=float) >>> np.gradient(f, x) - array([ 1. , 3. , 3.5, 6.7, 6.9, 2.5]) + array([1. , 3. , 3.5, 6.7, 6.9, 2.5]) For two dimensional arrays, the return will be two arrays ordered by axis. In this example the first array stands for the gradient in @@ -875,8 +877,8 @@ def gradient(f, *varargs, **kwargs): >>> np.gradient(np.array([[1, 2, 6], [3, 4, 5]], dtype=float)) [array([[ 2., 2., -1.], - [ 2., 2., -1.]]), array([[ 1. , 2.5, 4. ], - [ 1. , 1. , 1. ]])] + [ 2., 2., -1.]]), array([[1. , 2.5, 4. ], + [1. , 1. , 1. ]])] In this example the spacing is also specified: uniform for axis=0 and non uniform for axis=1 @@ -885,17 +887,17 @@ def gradient(f, *varargs, **kwargs): >>> y = [1., 1.5, 3.5] >>> np.gradient(np.array([[1, 2, 6], [3, 4, 5]], dtype=float), dx, y) [array([[ 1. , 1. , -0.5], - [ 1. , 1. , -0.5]]), array([[ 2. , 2. , 2. ], - [ 2. , 1.7, 0.5]])] + [ 1. , 1. , -0.5]]), array([[2. , 2. , 2. ], + [2. , 1.7, 0.5]])] It is possible to specify how boundaries are treated using `edge_order` >>> x = np.array([0, 1, 2, 3, 4]) >>> f = x**2 >>> np.gradient(f, edge_order=1) - array([ 1., 2., 4., 6., 7.]) + array([1., 2., 4., 6., 7.]) >>> np.gradient(f, edge_order=2) - array([-0., 2., 4., 6., 8.]) + array([0., 2., 4., 6., 8.]) The `axis` keyword can be used to specify a subset of axes of which the gradient is calculated @@ -1197,7 +1199,7 @@ def diff(a, n=1, axis=-1, prepend=np._NoValue, append=np._NoValue): >>> np.diff(u8_arr) array([255], dtype=uint8) >>> u8_arr[1,...] - u8_arr[0,...] - array(255, np.uint8) + 255 If this is not desirable, then the array should be cast to a larger integer type first: @@ -1337,7 +1339,7 @@ def interp(x, xp, fp, left=None, right=None, period=None): >>> np.interp(2.5, xp, fp) 1.0 >>> np.interp([0, 1, 1.5, 2.72, 3.14], xp, fp) - array([ 3. , 3. , 2.5 , 0.56, 0. ]) + array([3. , 3. , 2.5 , 0.56, 0. ]) >>> UNDEF = -99.0 >>> np.interp(3.14, xp, fp, right=UNDEF) -99.0 @@ -1361,7 +1363,7 @@ def interp(x, xp, fp, left=None, right=None, period=None): >>> xp = [190, -190, 350, -350] >>> fp = [5, 10, 3, 4] >>> np.interp(x, xp, fp, period=360) - array([7.5, 5., 8.75, 6.25, 3., 3.25, 3.5, 3.75]) + array([7.5 , 5. , 8.75, 6.25, 3. , 3.25, 3.5 , 3.75]) Complex interpolation: @@ -1369,7 +1371,7 @@ def interp(x, xp, fp, left=None, right=None, period=None): >>> xp = [2,3,5] >>> fp = [1.0j, 0, 2+3j] >>> np.interp(x, xp, fp) - array([ 0.+1.j , 1.+1.5j]) + array([0.+1.j , 1.+1.5j]) """ @@ -1442,7 +1444,7 @@ def angle(z, deg=False): Examples -------- >>> np.angle([1.0, 1.0j, 1+1j]) # in radians - array([ 0. , 1.57079633, 0.78539816]) + array([ 0. , 1.57079633, 0.78539816]) # may vary >>> np.angle(1+1j, deg=True) # in degrees 45.0 @@ -1502,9 +1504,9 @@ def unwrap(p, discont=pi, axis=-1): >>> phase = np.linspace(0, np.pi, num=5) >>> phase[3:] += np.pi >>> phase - array([ 0. , 0.78539816, 1.57079633, 5.49778714, 6.28318531]) + array([ 0. , 0.78539816, 1.57079633, 5.49778714, 6.28318531]) # may vary >>> np.unwrap(phase) - array([ 0. , 0.78539816, 1.57079633, -0.78539816, 0. ]) + array([ 0. , 0.78539816, 1.57079633, -0.78539816, 0. ]) # may vary """ p = asarray(p) @@ -1544,10 +1546,10 @@ def sort_complex(a): Examples -------- >>> np.sort_complex([5, 3, 6, 2, 1]) - array([ 1.+0.j, 2.+0.j, 3.+0.j, 5.+0.j, 6.+0.j]) + array([1.+0.j, 2.+0.j, 3.+0.j, 5.+0.j, 6.+0.j]) >>> np.sort_complex([1 + 2j, 2 - 1j, 3 - 2j, 3 - 3j, 3 + 5j]) - array([ 1.+2.j, 2.-1.j, 3.-3.j, 3.-2.j, 3.+5.j]) + array([1.+2.j, 2.-1.j, 3.-3.j, 3.-2.j, 3.+5.j]) """ b = array(a, copy=True) @@ -1593,7 +1595,7 @@ def trim_zeros(filt, trim='fb'): array([1, 2, 3, 0, 2, 1]) >>> np.trim_zeros(a, 'b') - array([0, 0, 0, 1, 2, 3, 0, 2, 1]) + array([0, 0, 0, ..., 0, 2, 1]) The input data type is preserved, list/tuple in means list/tuple out. @@ -1618,25 +1620,6 @@ def trim_zeros(filt, trim='fb'): last = last - 1 return filt[first:last] - -@deprecate -def unique(x): - """ - This function is deprecated. Use numpy.lib.arraysetops.unique() - instead. - """ - try: - tmp = x.flatten() - if tmp.size == 0: - return tmp - tmp.sort() - idx = concatenate(([True], tmp[1:] != tmp[:-1])) - return tmp[idx] - except AttributeError: - items = sorted(set(x)) - return asarray(items) - - def _extract_dispatcher(condition, arr): return (condition, arr) @@ -1891,6 +1874,7 @@ def _create_arrays(broadcast_shape, dim_sizes, list_of_core_dims, dtypes): return arrays +@set_module('numpy') class vectorize(object): """ vectorize(pyfunc, otypes=None, doc=None, excluded=None, cache=False, @@ -1973,11 +1957,11 @@ class vectorize(object): >>> out = vfunc([1, 2, 3, 4], 2) >>> type(out[0]) - <type 'numpy.int32'> + <class 'numpy.int64'> >>> vfunc = np.vectorize(myfunc, otypes=[float]) >>> out = vfunc([1, 2, 3, 4], 2) >>> type(out[0]) - <type 'numpy.float64'> + <class 'numpy.float64'> The `excluded` argument can be used to prevent vectorizing over certain arguments. This can be useful for array-like arguments of a fixed length @@ -2005,18 +1989,18 @@ class vectorize(object): >>> import scipy.stats >>> pearsonr = np.vectorize(scipy.stats.pearsonr, - ... signature='(n),(n)->(),()') - >>> pearsonr([[0, 1, 2, 3]], [[1, 2, 3, 4], [4, 3, 2, 1]]) + ... signature='(n),(n)->(),()') + >>> pearsonr([[0, 1, 2, 3]], [[1, 2, 3, 4], [4, 3, 2, 1]]) (array([ 1., -1.]), array([ 0., 0.])) Or for a vectorized convolution: >>> convolve = np.vectorize(np.convolve, signature='(n),(m)->(k)') >>> convolve(np.eye(4), [1, 2, 1]) - array([[ 1., 2., 1., 0., 0., 0.], - [ 0., 1., 2., 1., 0., 0.], - [ 0., 0., 1., 2., 1., 0.], - [ 0., 0., 0., 1., 2., 1.]]) + array([[1., 2., 1., 0., 0., 0.], + [0., 1., 2., 1., 0., 0.], + [0., 0., 1., 2., 1., 0.], + [0., 0., 0., 1., 2., 1.]]) See Also -------- @@ -2326,10 +2310,14 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, array `m` and let ``f = fweights`` and ``a = aweights`` for brevity. The steps to compute the weighted covariance are as follows:: + >>> m = np.arange(10, dtype=np.float64) + >>> f = np.arange(10) * 2 + >>> a = np.arange(10) ** 2. + >>> ddof = 9 # N - 1 >>> w = f * a >>> v1 = np.sum(w) >>> v2 = np.sum(w * a) - >>> m -= np.sum(m * w, axis=1, keepdims=True) / v1 + >>> m -= np.sum(m * w, axis=None, keepdims=True) / v1 >>> cov = np.dot(m * w, m.T) * v1 / (v1**2 - ddof * v2) Note that when ``a == 1``, the normalization factor @@ -2361,14 +2349,14 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, >>> x = [-2.1, -1, 4.3] >>> y = [3, 1.1, 0.12] >>> X = np.stack((x, y), axis=0) - >>> print(np.cov(X)) - [[ 11.71 -4.286 ] - [ -4.286 2.14413333]] - >>> print(np.cov(x, y)) - [[ 11.71 -4.286 ] - [ -4.286 2.14413333]] - >>> print(np.cov(x)) - 11.71 + >>> np.cov(X) + array([[11.71 , -4.286 ], # may vary + [-4.286 , 2.144133]]) + >>> np.cov(x, y) + array([[11.71 , -4.286 ], # may vary + [-4.286 , 2.144133]]) + >>> np.cov(x) + array(11.71) """ # Check inputs @@ -2555,6 +2543,7 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): return c +@set_module('numpy') def blackman(M): """ Return the Blackman window. @@ -2604,12 +2593,14 @@ def blackman(M): Examples -------- + >>> import matplotlib + >>> matplotlib.use('agg') + >>> import matplotlib.pyplot as plt >>> np.blackman(12) - array([ -1.38777878e-17, 3.26064346e-02, 1.59903635e-01, - 4.14397981e-01, 7.36045180e-01, 9.67046769e-01, - 9.67046769e-01, 7.36045180e-01, 4.14397981e-01, - 1.59903635e-01, 3.26064346e-02, -1.38777878e-17]) - + array([-1.38777878e-17, 3.26064346e-02, 1.59903635e-01, # may vary + 4.14397981e-01, 7.36045180e-01, 9.67046769e-01, + 9.67046769e-01, 7.36045180e-01, 4.14397981e-01, + 1.59903635e-01, 3.26064346e-02, -1.38777878e-17]) Plot the window and the frequency response: @@ -2618,15 +2609,15 @@ def blackman(M): >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Blackman window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Blackman window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -2635,13 +2626,12 @@ def blackman(M): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of Blackman window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of Blackman window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> - >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + Text(0.5, 0, 'Normalized frequency [cycles per sample]') + >>> _ = plt.axis('tight') >>> plt.show() """ @@ -2653,6 +2643,7 @@ def blackman(M): return 0.42 - 0.5*cos(2.0*pi*n/(M-1)) + 0.08*cos(4.0*pi*n/(M-1)) +@set_module('numpy') def bartlett(M): """ Return the Bartlett window. @@ -2712,8 +2703,9 @@ def bartlett(M): Examples -------- + >>> import matplotlib.pyplot as plt >>> np.bartlett(12) - array([ 0. , 0.18181818, 0.36363636, 0.54545455, 0.72727273, + array([ 0. , 0.18181818, 0.36363636, 0.54545455, 0.72727273, # may vary 0.90909091, 0.90909091, 0.72727273, 0.54545455, 0.36363636, 0.18181818, 0. ]) @@ -2724,15 +2716,15 @@ def bartlett(M): >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Bartlett window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Bartlett window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -2741,13 +2733,12 @@ def bartlett(M): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of Bartlett window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of Bartlett window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> - >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + Text(0.5, 0, 'Normalized frequency [cycles per sample]') + >>> _ = plt.axis('tight') >>> plt.show() """ @@ -2759,6 +2750,7 @@ def bartlett(M): return where(less_equal(n, (M-1)/2.0), 2.0*n/(M-1), 2.0 - 2.0*n/(M-1)) +@set_module('numpy') def hanning(M): """ Return the Hanning window. @@ -2813,26 +2805,30 @@ def hanning(M): Examples -------- >>> np.hanning(12) - array([ 0. , 0.07937323, 0.29229249, 0.57115742, 0.82743037, - 0.97974649, 0.97974649, 0.82743037, 0.57115742, 0.29229249, - 0.07937323, 0. ]) + array([0. , 0.07937323, 0.29229249, 0.57115742, 0.82743037, + 0.97974649, 0.97974649, 0.82743037, 0.57115742, 0.29229249, + 0.07937323, 0. ]) Plot the window and its frequency response: + >>> import matplotlib + >>> import matplotlib.pyplot + >>> matplotlib.pyplot.switch_backend('agg') + >>> import matplotlib.pyplot as plt >>> from numpy.fft import fft, fftshift >>> window = np.hanning(51) >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Hann window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Hann window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -2841,13 +2837,13 @@ def hanning(M): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of the Hann window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of the Hann window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Normalized frequency [cycles per sample]') >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + ... >>> plt.show() """ @@ -2859,6 +2855,7 @@ def hanning(M): return 0.5 - 0.5*cos(2.0*pi*n/(M-1)) +@set_module('numpy') def hamming(M): """ Return the Hamming window. @@ -2911,26 +2908,30 @@ def hamming(M): Examples -------- >>> np.hamming(12) - array([ 0.08 , 0.15302337, 0.34890909, 0.60546483, 0.84123594, + array([ 0.08 , 0.15302337, 0.34890909, 0.60546483, 0.84123594, # may vary 0.98136677, 0.98136677, 0.84123594, 0.60546483, 0.34890909, 0.15302337, 0.08 ]) Plot the window and the frequency response: + >>> import matplotlib + >>> import matplotlib.pyplot + >>> matplotlib.pyplot.switch_backend('agg') + >>> import matplotlib.pyplot as plt >>> from numpy.fft import fft, fftshift >>> window = np.hamming(51) >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Hamming window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Hamming window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -2939,13 +2940,13 @@ def hamming(M): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of Hamming window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of Hamming window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Normalized frequency [cycles per sample]') >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + ... >>> plt.show() """ @@ -3094,9 +3095,9 @@ def i0(x): Examples -------- >>> np.i0([0.]) - array(1.0) + array(1.0) # may vary >>> np.i0([0., 1. + 2j]) - array([ 1.00000000+0.j , 0.18785373+0.64616944j]) + array([ 1.00000000+0.j , 0.18785373+0.64616944j]) # may vary """ x = atleast_1d(x).copy() @@ -3112,6 +3113,7 @@ def i0(x): ## End of cephes code for i0 +@set_module('numpy') def kaiser(M, beta): """ Return the Kaiser window. @@ -3190,11 +3192,14 @@ def kaiser(M, beta): Examples -------- + >>> import matplotlib + >>> matplotlib.use('agg') + >>> import matplotlib.pyplot as plt >>> np.kaiser(12, 14) - array([ 7.72686684e-06, 3.46009194e-03, 4.65200189e-02, - 2.29737120e-01, 5.99885316e-01, 9.45674898e-01, - 9.45674898e-01, 5.99885316e-01, 2.29737120e-01, - 4.65200189e-02, 3.46009194e-03, 7.72686684e-06]) + array([7.72686684e-06, 3.46009194e-03, 4.65200189e-02, # may vary + 2.29737120e-01, 5.99885316e-01, 9.45674898e-01, + 9.45674898e-01, 5.99885316e-01, 2.29737120e-01, + 4.65200189e-02, 3.46009194e-03, 7.72686684e-06]) Plot the window and the frequency response: @@ -3204,15 +3209,15 @@ def kaiser(M, beta): >>> plt.plot(window) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Kaiser window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Kaiser window') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("Sample") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Sample') >>> plt.show() >>> plt.figure() - <matplotlib.figure.Figure object at 0x...> + <Figure size 640x480 with 0 Axes> >>> A = fft(window, 2048) / 25.5 >>> mag = np.abs(fftshift(A)) >>> freq = np.linspace(-0.5, 0.5, len(A)) @@ -3221,13 +3226,13 @@ def kaiser(M, beta): >>> plt.plot(freq, response) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Frequency response of Kaiser window") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Frequency response of Kaiser window') >>> plt.ylabel("Magnitude [dB]") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Magnitude [dB]') >>> plt.xlabel("Normalized frequency [cycles per sample]") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'Normalized frequency [cycles per sample]') >>> plt.axis('tight') - (-0.5, 0.5, -100.0, ...) + (-0.5, 0.5, -100.0, ...) # may vary >>> plt.show() """ @@ -3283,31 +3288,33 @@ def sinc(x): Examples -------- + >>> import matplotlib + >>> import matplotlib.pyplot as plt >>> x = np.linspace(-4, 4, 41) >>> np.sinc(x) - array([ -3.89804309e-17, -4.92362781e-02, -8.40918587e-02, + array([-3.89804309e-17, -4.92362781e-02, -8.40918587e-02, # may vary -8.90384387e-02, -5.84680802e-02, 3.89804309e-17, - 6.68206631e-02, 1.16434881e-01, 1.26137788e-01, - 8.50444803e-02, -3.89804309e-17, -1.03943254e-01, + 6.68206631e-02, 1.16434881e-01, 1.26137788e-01, + 8.50444803e-02, -3.89804309e-17, -1.03943254e-01, -1.89206682e-01, -2.16236208e-01, -1.55914881e-01, - 3.89804309e-17, 2.33872321e-01, 5.04551152e-01, - 7.56826729e-01, 9.35489284e-01, 1.00000000e+00, - 9.35489284e-01, 7.56826729e-01, 5.04551152e-01, - 2.33872321e-01, 3.89804309e-17, -1.55914881e-01, - -2.16236208e-01, -1.89206682e-01, -1.03943254e-01, - -3.89804309e-17, 8.50444803e-02, 1.26137788e-01, - 1.16434881e-01, 6.68206631e-02, 3.89804309e-17, + 3.89804309e-17, 2.33872321e-01, 5.04551152e-01, + 7.56826729e-01, 9.35489284e-01, 1.00000000e+00, + 9.35489284e-01, 7.56826729e-01, 5.04551152e-01, + 2.33872321e-01, 3.89804309e-17, -1.55914881e-01, + -2.16236208e-01, -1.89206682e-01, -1.03943254e-01, + -3.89804309e-17, 8.50444803e-02, 1.26137788e-01, + 1.16434881e-01, 6.68206631e-02, 3.89804309e-17, -5.84680802e-02, -8.90384387e-02, -8.40918587e-02, -4.92362781e-02, -3.89804309e-17]) >>> plt.plot(x, np.sinc(x)) [<matplotlib.lines.Line2D object at 0x...>] >>> plt.title("Sinc Function") - <matplotlib.text.Text object at 0x...> + Text(0.5, 1.0, 'Sinc Function') >>> plt.ylabel("Amplitude") - <matplotlib.text.Text object at 0x...> + Text(0, 0.5, 'Amplitude') >>> plt.xlabel("X") - <matplotlib.text.Text object at 0x...> + Text(0.5, 0, 'X') >>> plt.show() It works in 2-D as well: @@ -3479,18 +3486,18 @@ def median(a, axis=None, out=None, overwrite_input=False, keepdims=False): >>> np.median(a) 3.5 >>> np.median(a, axis=0) - array([ 6.5, 4.5, 2.5]) + array([6.5, 4.5, 2.5]) >>> np.median(a, axis=1) - array([ 7., 2.]) + array([7., 2.]) >>> m = np.median(a, axis=0) >>> out = np.zeros_like(m) >>> np.median(a, axis=0, out=m) - array([ 6.5, 4.5, 2.5]) + array([6.5, 4.5, 2.5]) >>> m - array([ 6.5, 4.5, 2.5]) + array([6.5, 4.5, 2.5]) >>> b = a.copy() >>> np.median(b, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a==b) >>> b = a.copy() >>> np.median(b, axis=None, overwrite_input=True) @@ -3657,23 +3664,23 @@ def percentile(a, q, axis=None, out=None, >>> np.percentile(a, 50) 3.5 >>> np.percentile(a, 50, axis=0) - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> np.percentile(a, 50, axis=1) - array([ 7., 2.]) + array([7., 2.]) >>> np.percentile(a, 50, axis=1, keepdims=True) - array([[ 7.], - [ 2.]]) + array([[7.], + [2.]]) >>> m = np.percentile(a, 50, axis=0) >>> out = np.zeros_like(m) >>> np.percentile(a, 50, axis=0, out=out) - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> m - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> b = a.copy() >>> np.percentile(b, 50, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a == b) The different types of interpolation can be visualized graphically: @@ -3799,21 +3806,21 @@ def quantile(a, q, axis=None, out=None, >>> np.quantile(a, 0.5) 3.5 >>> np.quantile(a, 0.5, axis=0) - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> np.quantile(a, 0.5, axis=1) - array([ 7., 2.]) + array([7., 2.]) >>> np.quantile(a, 0.5, axis=1, keepdims=True) - array([[ 7.], - [ 2.]]) + array([[7.], + [2.]]) >>> m = np.quantile(a, 0.5, axis=0) >>> out = np.zeros_like(m) >>> np.quantile(a, 0.5, axis=0, out=out) - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> m - array([[ 6.5, 4.5, 2.5]]) + array([6.5, 4.5, 2.5]) >>> b = a.copy() >>> np.quantile(b, 0.5, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a == b) """ q = np.asanyarray(q) @@ -4042,9 +4049,9 @@ def trapz(y, x=None, dx=1.0, axis=-1): array([[0, 1, 2], [3, 4, 5]]) >>> np.trapz(a, axis=0) - array([ 1.5, 2.5, 3.5]) + array([1.5, 2.5, 3.5]) >>> np.trapz(a, axis=1) - array([ 2., 8.]) + array([2., 8.]) """ y = asanyarray(y) @@ -4162,17 +4169,17 @@ def meshgrid(*xi, **kwargs): >>> y = np.linspace(0, 1, ny) >>> xv, yv = np.meshgrid(x, y) >>> xv - array([[ 0. , 0.5, 1. ], - [ 0. , 0.5, 1. ]]) + array([[0. , 0.5, 1. ], + [0. , 0.5, 1. ]]) >>> yv - array([[ 0., 0., 0.], - [ 1., 1., 1.]]) + array([[0., 0., 0.], + [1., 1., 1.]]) >>> xv, yv = np.meshgrid(x, y, sparse=True) # make sparse output arrays >>> xv - array([[ 0. , 0.5, 1. ]]) + array([[0. , 0.5, 1. ]]) >>> yv - array([[ 0.], - [ 1.]]) + array([[0.], + [1.]]) `meshgrid` is very useful to evaluate functions on a grid. @@ -4255,6 +4262,7 @@ def delete(arr, obj, axis=None): ----- Often it is preferable to use a boolean mask. For example: + >>> arr = np.arange(12) + 1 >>> mask = np.ones(len(arr), dtype=bool) >>> mask[[0,2,4]] = False >>> result = arr[mask,...] @@ -4486,7 +4494,7 @@ def insert(arr, obj, values, axis=None): [2, 2], [3, 3]]) >>> np.insert(a, 1, 5) - array([1, 5, 1, 2, 2, 3, 3]) + array([1, 5, 1, ..., 2, 3, 3]) >>> np.insert(a, 1, 5, axis=1) array([[1, 5, 1], [2, 5, 2], @@ -4506,13 +4514,13 @@ def insert(arr, obj, values, axis=None): >>> b array([1, 1, 2, 2, 3, 3]) >>> np.insert(b, [2, 2], [5, 6]) - array([1, 1, 5, 6, 2, 2, 3, 3]) + array([1, 1, 5, ..., 2, 3, 3]) >>> np.insert(b, slice(2, 4), [5, 6]) - array([1, 1, 5, 2, 6, 2, 3, 3]) + array([1, 1, 5, ..., 2, 3, 3]) >>> np.insert(b, [2, 2], [7.13, False]) # type casting - array([1, 1, 7, 0, 2, 2, 3, 3]) + array([1, 1, 7, ..., 2, 3, 3]) >>> x = np.arange(8).reshape(2, 4) >>> idx = (1, 3) @@ -4676,7 +4684,7 @@ def append(arr, values, axis=None): Examples -------- >>> np.append([1, 2, 3], [[4, 5, 6], [7, 8, 9]]) - array([1, 2, 3, 4, 5, 6, 7, 8, 9]) + array([1, 2, 3, ..., 7, 8, 9]) When `axis` is specified, `values` must have the correct shape. @@ -4686,8 +4694,8 @@ def append(arr, values, axis=None): [7, 8, 9]]) >>> np.append([[1, 2, 3], [4, 5, 6]], [7, 8, 9], axis=0) Traceback (most recent call last): - ... - ValueError: arrays must have same number of dimensions + ... + ValueError: all the input arrays must have same number of dimensions """ arr = asanyarray(arr) diff --git a/numpy/lib/histograms.py b/numpy/lib/histograms.py index 1ff25b81f..7b229cc89 100644 --- a/numpy/lib/histograms.py +++ b/numpy/lib/histograms.py @@ -3,21 +3,25 @@ Histogram-related functions """ from __future__ import division, absolute_import, print_function +import functools import operator import warnings import numpy as np from numpy.compat.py3k import basestring -from numpy.core.overrides import array_function_dispatch +from numpy.core import overrides __all__ = ['histogram', 'histogramdd', 'histogram_bin_edges'] +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + # range is a keyword argument to many functions, so save the builtin so they can # use it. _range = range -def _hist_bin_sqrt(x): +def _hist_bin_sqrt(x, range): """ Square root histogram bin estimator. @@ -34,10 +38,11 @@ def _hist_bin_sqrt(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return x.ptp() / np.sqrt(x.size) -def _hist_bin_sturges(x): +def _hist_bin_sturges(x, range): """ Sturges histogram bin estimator. @@ -56,10 +61,11 @@ def _hist_bin_sturges(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return x.ptp() / (np.log2(x.size) + 1.0) -def _hist_bin_rice(x): +def _hist_bin_rice(x, range): """ Rice histogram bin estimator. @@ -79,10 +85,11 @@ def _hist_bin_rice(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return x.ptp() / (2.0 * x.size ** (1.0 / 3)) -def _hist_bin_scott(x): +def _hist_bin_scott(x, range): """ Scott histogram bin estimator. @@ -100,10 +107,52 @@ def _hist_bin_scott(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return (24.0 * np.pi**0.5 / x.size)**(1.0 / 3.0) * np.std(x) -def _hist_bin_doane(x): +def _hist_bin_stone(x, range): + """ + Histogram bin estimator based on minimizing the estimated integrated squared error (ISE). + + The number of bins is chosen by minimizing the estimated ISE against the unknown true distribution. + The ISE is estimated using cross-validation and can be regarded as a generalization of Scott's rule. + https://en.wikipedia.org/wiki/Histogram#Scott.27s_normal_reference_rule + + This paper by Stone appears to be the origination of this rule. + http://digitalassets.lib.berkeley.edu/sdtr/ucb/text/34.pdf + + Parameters + ---------- + x : array_like + Input data that is to be histogrammed, trimmed to range. May not + be empty. + range : (float, float) + The lower and upper range of the bins. + + Returns + ------- + h : An estimate of the optimal bin width for the given data. + """ + + n = x.size + ptp_x = np.ptp(x) + if n <= 1 or ptp_x == 0: + return 0 + + def jhat(nbins): + hh = ptp_x / nbins + p_k = np.histogram(x, bins=nbins, range=range)[0] / n + return (2 - (n + 1) * p_k.dot(p_k)) / hh + + nbins_upper_bound = max(100, int(np.sqrt(n))) + nbins = min(_range(1, nbins_upper_bound + 1), key=jhat) + if nbins == nbins_upper_bound: + warnings.warn("The number of bins estimated may be suboptimal.", RuntimeWarning, stacklevel=2) + return ptp_x / nbins + + +def _hist_bin_doane(x, range): """ Doane's histogram bin estimator. @@ -121,6 +170,7 @@ def _hist_bin_doane(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused if x.size > 2: sg1 = np.sqrt(6.0 * (x.size - 2) / ((x.size + 1.0) * (x.size + 3))) sigma = np.std(x) @@ -137,7 +187,7 @@ def _hist_bin_doane(x): return 0.0 -def _hist_bin_fd(x): +def _hist_bin_fd(x, range): """ The Freedman-Diaconis histogram bin estimator. @@ -162,11 +212,12 @@ def _hist_bin_fd(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused iqr = np.subtract(*np.percentile(x, [75, 25])) return 2.0 * iqr * x.size ** (-1.0 / 3.0) -def _hist_bin_auto(x): +def _hist_bin_auto(x, range): """ Histogram bin estimator that uses the minimum width of the Freedman-Diaconis and Sturges estimators if the FD bandwidth is non zero @@ -200,8 +251,9 @@ def _hist_bin_auto(x): -------- _hist_bin_fd, _hist_bin_sturges """ - fd_bw = _hist_bin_fd(x) - sturges_bw = _hist_bin_sturges(x) + fd_bw = _hist_bin_fd(x, range) + sturges_bw = _hist_bin_sturges(x, range) + del range # unused if fd_bw: return min(fd_bw, sturges_bw) else: @@ -209,7 +261,8 @@ def _hist_bin_auto(x): return sturges_bw # Private dict initialized at module load time -_hist_bin_selectors = {'auto': _hist_bin_auto, +_hist_bin_selectors = {'stone': _hist_bin_stone, + 'auto': _hist_bin_auto, 'doane': _hist_bin_doane, 'fd': _hist_bin_fd, 'rice': _hist_bin_rice, @@ -344,7 +397,7 @@ def _get_bin_edges(a, bins, range, weights): n_equal_bins = 1 else: # Do not call selectors on empty arrays - width = _hist_bin_selectors[bin_name](a) + width = _hist_bin_selectors[bin_name](a, (first_edge, last_edge)) if width: n_equal_bins = int(np.ceil(_unsigned_subtract(last_edge, first_edge) / width)) else: @@ -446,6 +499,11 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): Less robust estimator that that takes into account data variability and data size. + 'stone' + Estimator based on leave-one-out cross-validation estimate of + the integrated squared error. Can be regarded as a generalization + of Scott's rule. + 'rice' Estimator does not take variability into account, only data size. Commonly overestimates number of bins required. @@ -587,7 +645,7 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): >>> hist_0, bins_0 = np.histogram(arr[group_id == 0], bins='auto') >>> hist_1, bins_1 = np.histogram(arr[group_id == 1], bins='auto') - >>> hist_0; hist1 + >>> hist_0; hist_1 array([1, 1, 1]) array([2, 1, 1, 2]) >>> bins_0; bins_1 @@ -690,14 +748,14 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, >>> np.histogram([1, 2, 1], bins=[0, 1, 2, 3]) (array([0, 2, 1]), array([0, 1, 2, 3])) >>> np.histogram(np.arange(4), bins=np.arange(5), density=True) - (array([ 0.25, 0.25, 0.25, 0.25]), array([0, 1, 2, 3, 4])) + (array([0.25, 0.25, 0.25, 0.25]), array([0, 1, 2, 3, 4])) >>> np.histogram([[1, 2, 1], [1, 0, 1]], bins=[0,1,2,3]) (array([1, 4, 1]), array([0, 1, 2, 3])) >>> a = np.arange(5) >>> hist, bin_edges = np.histogram(a, density=True) >>> hist - array([ 0.5, 0. , 0.5, 0. , 0. , 0.5, 0. , 0.5, 0. , 0.5]) + array([0.5, 0. , 0.5, 0. , 0. , 0.5, 0. , 0.5, 0. , 0.5]) >>> hist.sum() 2.4999999999999996 >>> np.sum(hist * np.diff(bin_edges)) @@ -712,8 +770,9 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, >>> rng = np.random.RandomState(10) # deterministic random data >>> a = np.hstack((rng.normal(size=1000), ... rng.normal(loc=5, scale=2, size=1000))) - >>> plt.hist(a, bins='auto') # arguments are passed to np.histogram + >>> _ = plt.hist(a, bins='auto') # arguments are passed to np.histogram >>> plt.title("Histogram with 'auto' bins") + Text(0.5, 1.0, "Histogram with 'auto' bins") >>> plt.show() """ diff --git a/numpy/lib/index_tricks.py b/numpy/lib/index_tricks.py index ff2e00d3e..64c491cfa 100644 --- a/numpy/lib/index_tricks.py +++ b/numpy/lib/index_tricks.py @@ -13,6 +13,7 @@ from numpy.core.numerictypes import find_common_type, issubdtype import numpy.matrixlib as matrixlib from .function_base import diff from numpy.core.multiarray import ravel_multi_index, unravel_index +from numpy.core.overrides import set_module from numpy.core import overrides, linspace from numpy.lib.stride_tricks import as_strided @@ -477,7 +478,7 @@ class RClass(AxisConcatenator): Examples -------- >>> np.r_[np.array([1,2,3]), 0, 0, np.array([4,5,6])] - array([1, 2, 3, 0, 0, 4, 5, 6]) + array([1, 2, 3, ..., 4, 5, 6]) >>> np.r_[-1:1:6j, [0]*3, 5, 6] array([-1. , -0.6, -0.2, 0.2, 0.6, 1. , 0. , 0. , 0. , 5. , 6. ]) @@ -537,15 +538,18 @@ class CClass(AxisConcatenator): [2, 5], [3, 6]]) >>> np.c_[np.array([[1,2,3]]), 0, 0, np.array([[4,5,6]])] - array([[1, 2, 3, 0, 0, 4, 5, 6]]) + array([[1, 2, 3, ..., 4, 5, 6]]) """ def __init__(self): AxisConcatenator.__init__(self, -1, ndmin=2, trans1d=0) + c_ = CClass() + +@set_module('numpy') class ndenumerate(object): """ Multidimensional index iterator. @@ -596,6 +600,7 @@ class ndenumerate(object): next = __next__ +@set_module('numpy') class ndindex(object): """ An N-dimensional iterator object to index arrays. @@ -808,7 +813,7 @@ def fill_diagonal(a, val, wrap=False): >>> # tall matrices no wrap >>> a = np.zeros((5, 3),int) - >>> fill_diagonal(a, 4) + >>> np.fill_diagonal(a, 4) >>> a array([[4, 0, 0], [0, 4, 0], @@ -818,7 +823,7 @@ def fill_diagonal(a, val, wrap=False): >>> # tall matrices wrap >>> a = np.zeros((5, 3),int) - >>> fill_diagonal(a, 4, wrap=True) + >>> np.fill_diagonal(a, 4, wrap=True) >>> a array([[4, 0, 0], [0, 4, 0], @@ -828,7 +833,7 @@ def fill_diagonal(a, val, wrap=False): >>> # wide matrices >>> a = np.zeros((3, 5),int) - >>> fill_diagonal(a, 4, wrap=True) + >>> np.fill_diagonal(a, 4, wrap=True) >>> a array([[4, 0, 0, 0, 0], [0, 4, 0, 0, 0], @@ -856,6 +861,7 @@ def fill_diagonal(a, val, wrap=False): a.flat[:end:step] = val +@set_module('numpy') def diag_indices(n, ndim=2): """ Return the indices to access the main diagonal of an array. diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py index 0379ecb1a..52ad45b68 100644 --- a/numpy/lib/mixins.py +++ b/numpy/lib/mixins.py @@ -69,9 +69,6 @@ class NDArrayOperatorsMixin(object): deferring to the ``__array_ufunc__`` method, which subclasses must implement. - This class does not yet implement the special operators corresponding - to ``matmul`` (``@``), because ``np.matmul`` is not yet a NumPy ufunc. - It is useful for writing classes that do not inherit from `numpy.ndarray`, but that should support arithmetic and numpy universal functions like arrays as described in `A Mechanism for Overriding Ufuncs @@ -155,6 +152,8 @@ class NDArrayOperatorsMixin(object): __add__, __radd__, __iadd__ = _numeric_methods(um.add, 'add') __sub__, __rsub__, __isub__ = _numeric_methods(um.subtract, 'sub') __mul__, __rmul__, __imul__ = _numeric_methods(um.multiply, 'mul') + __matmul__, __rmatmul__, __imatmul__ = _numeric_methods( + um.matmul, 'matmul') if sys.version_info.major < 3: # Python 3 uses only __truediv__ and __floordiv__ __div__, __rdiv__, __idiv__ = _numeric_methods(um.divide, 'div') diff --git a/numpy/lib/nanfunctions.py b/numpy/lib/nanfunctions.py index d73d84467..b3bf1880b 100644 --- a/numpy/lib/nanfunctions.py +++ b/numpy/lib/nanfunctions.py @@ -271,9 +271,9 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): >>> np.nanmin(a) 1.0 >>> np.nanmin(a, axis=0) - array([ 1., 2.]) + array([1., 2.]) >>> np.nanmin(a, axis=1) - array([ 1., 3.]) + array([1., 3.]) When positive infinity and negative infinity are present: @@ -384,9 +384,9 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): >>> np.nanmax(a) 3.0 >>> np.nanmax(a, axis=0) - array([ 3., 2.]) + array([3., 2.]) >>> np.nanmax(a, axis=1) - array([ 2., 3.]) + array([2., 3.]) When positive infinity and negative infinity are present: @@ -601,12 +601,15 @@ def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): >>> np.nansum(a) 3.0 >>> np.nansum(a, axis=0) - array([ 2., 1.]) + array([2., 1.]) >>> np.nansum([1, np.nan, np.inf]) inf >>> np.nansum([1, np.nan, np.NINF]) -inf - >>> np.nansum([1, np.nan, np.inf, -np.inf]) # both +/- infinity present + >>> from numpy.testing import suppress_warnings + >>> with suppress_warnings() as sup: + ... sup.filter(RuntimeWarning) + ... np.nansum([1, np.nan, np.inf, -np.inf]) # both +/- infinity present nan """ @@ -677,7 +680,7 @@ def nanprod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): >>> np.nanprod(a) 6.0 >>> np.nanprod(a, axis=0) - array([ 3., 2.]) + array([3., 2.]) """ a, mask = _replace_nan(a, 1) @@ -738,16 +741,16 @@ def nancumsum(a, axis=None, dtype=None, out=None): >>> np.nancumsum([1]) array([1]) >>> np.nancumsum([1, np.nan]) - array([ 1., 1.]) + array([1., 1.]) >>> a = np.array([[1, 2], [3, np.nan]]) >>> np.nancumsum(a) - array([ 1., 3., 6., 6.]) + array([1., 3., 6., 6.]) >>> np.nancumsum(a, axis=0) - array([[ 1., 2.], - [ 4., 2.]]) + array([[1., 2.], + [4., 2.]]) >>> np.nancumsum(a, axis=1) - array([[ 1., 3.], - [ 3., 3.]]) + array([[1., 3.], + [3., 3.]]) """ a, mask = _replace_nan(a, 0) @@ -805,16 +808,16 @@ def nancumprod(a, axis=None, dtype=None, out=None): >>> np.nancumprod([1]) array([1]) >>> np.nancumprod([1, np.nan]) - array([ 1., 1.]) + array([1., 1.]) >>> a = np.array([[1, 2], [3, np.nan]]) >>> np.nancumprod(a) - array([ 1., 2., 6., 6.]) + array([1., 2., 6., 6.]) >>> np.nancumprod(a, axis=0) - array([[ 1., 2.], - [ 3., 2.]]) + array([[1., 2.], + [3., 2.]]) >>> np.nancumprod(a, axis=1) - array([[ 1., 2.], - [ 3., 3.]]) + array([[1., 2.], + [3., 3.]]) """ a, mask = _replace_nan(a, 1) @@ -895,9 +898,9 @@ def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): >>> np.nanmean(a) 2.6666666666666665 >>> np.nanmean(a, axis=0) - array([ 2., 4.]) + array([2., 4.]) >>> np.nanmean(a, axis=1) - array([ 1., 3.5]) + array([1., 3.5]) # may vary """ arr, mask = _replace_nan(a, 0) @@ -1049,19 +1052,19 @@ def nanmedian(a, axis=None, out=None, overwrite_input=False, keepdims=np._NoValu >>> a = np.array([[10.0, 7, 4], [3, 2, 1]]) >>> a[0, 1] = np.nan >>> a - array([[ 10., nan, 4.], - [ 3., 2., 1.]]) + array([[10., nan, 4.], + [ 3., 2., 1.]]) >>> np.median(a) nan >>> np.nanmedian(a) 3.0 >>> np.nanmedian(a, axis=0) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> np.median(a, axis=1) - array([ 7., 2.]) + array([nan, 2.]) >>> b = a.copy() >>> np.nanmedian(b, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a==b) >>> b = a.copy() >>> np.nanmedian(b, axis=None, overwrite_input=True) @@ -1177,27 +1180,27 @@ def nanpercentile(a, q, axis=None, out=None, overwrite_input=False, >>> a = np.array([[10., 7., 4.], [3., 2., 1.]]) >>> a[0][1] = np.nan >>> a - array([[ 10., nan, 4.], - [ 3., 2., 1.]]) + array([[10., nan, 4.], + [ 3., 2., 1.]]) >>> np.percentile(a, 50) nan >>> np.nanpercentile(a, 50) - 3.5 + 3.0 >>> np.nanpercentile(a, 50, axis=0) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> np.nanpercentile(a, 50, axis=1, keepdims=True) - array([[ 7.], - [ 2.]]) + array([[7.], + [2.]]) >>> m = np.nanpercentile(a, 50, axis=0) >>> out = np.zeros_like(m) >>> np.nanpercentile(a, 50, axis=0, out=out) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> m - array([ 6.5, 2. , 2.5]) + array([6.5, 2. , 2.5]) >>> b = a.copy() >>> np.nanpercentile(b, 50, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a==b) """ @@ -1291,26 +1294,26 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, >>> a = np.array([[10., 7., 4.], [3., 2., 1.]]) >>> a[0][1] = np.nan >>> a - array([[ 10., nan, 4.], - [ 3., 2., 1.]]) + array([[10., nan, 4.], + [ 3., 2., 1.]]) >>> np.quantile(a, 0.5) nan >>> np.nanquantile(a, 0.5) - 3.5 + 3.0 >>> np.nanquantile(a, 0.5, axis=0) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> np.nanquantile(a, 0.5, axis=1, keepdims=True) - array([[ 7.], - [ 2.]]) + array([[7.], + [2.]]) >>> m = np.nanquantile(a, 0.5, axis=0) >>> out = np.zeros_like(m) >>> np.nanquantile(a, 0.5, axis=0, out=out) - array([ 6.5, 2., 2.5]) + array([6.5, 2. , 2.5]) >>> m - array([ 6.5, 2. , 2.5]) + array([6.5, 2. , 2.5]) >>> b = a.copy() >>> np.nanquantile(b, 0.5, axis=1, overwrite_input=True) - array([ 7., 2.]) + array([7., 2.]) >>> assert not np.all(a==b) """ a = np.asanyarray(a) @@ -1465,12 +1468,12 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): Examples -------- >>> a = np.array([[1, np.nan], [3, 4]]) - >>> np.var(a) + >>> np.nanvar(a) 1.5555555555555554 >>> np.nanvar(a, axis=0) - array([ 1., 0.]) + array([1., 0.]) >>> np.nanvar(a, axis=1) - array([ 0., 0.25]) + array([0., 0.25]) # may vary """ arr, mask = _replace_nan(a, 0) @@ -1619,9 +1622,9 @@ def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): >>> np.nanstd(a) 1.247219128924647 >>> np.nanstd(a, axis=0) - array([ 1., 0.]) + array([1., 0.]) >>> np.nanstd(a, axis=1) - array([ 0., 0.5]) + array([0., 0.5]) # may vary """ var = nanvar(a, axis=axis, dtype=dtype, out=out, ddof=ddof, diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 6fbb7e805..704fea108 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -3,6 +3,7 @@ from __future__ import division, absolute_import, print_function import sys import os import re +import functools import itertools import warnings import weakref @@ -11,8 +12,9 @@ from operator import itemgetter, index as opindex import numpy as np from . import format from ._datasource import DataSource +from numpy.core import overrides from numpy.core.multiarray import packbits, unpackbits -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import set_module from numpy.core._internal import recursive from ._iotools import ( LineSplitter, NameValidator, StringConverter, ConverterError, @@ -33,6 +35,7 @@ else: from collections import Mapping +@set_module('numpy') def loads(*args, **kwargs): # NumPy 1.15.0, 2017-12-10 warnings.warn( @@ -48,6 +51,10 @@ __all__ = [ ] +array_function_dispatch = functools.partial( + overrides.array_function_dispatch, module='numpy') + + class BagObj(object): """ BagObj(obj) @@ -161,13 +168,13 @@ class NpzFile(Mapping): >>> x = np.arange(10) >>> y = np.sin(x) >>> np.savez(outfile, x=x, y=y) - >>> outfile.seek(0) + >>> _ = outfile.seek(0) >>> npz = np.load(outfile) >>> isinstance(npz, np.lib.io.NpzFile) True - >>> npz.files - ['y', 'x'] + >>> sorted(npz.files) + ['x', 'y'] >>> npz['x'] # getitem access array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) >>> npz.f.x # attribute lookup @@ -277,6 +284,7 @@ class NpzFile(Mapping): return self.keys() +@set_module('numpy') def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, encoding='ASCII'): """ @@ -433,8 +441,8 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, else: # Try a pickle if not allow_pickle: - raise ValueError("allow_pickle=False, but file does not contain " - "non-pickled data") + raise ValueError("Cannot load file containing pickled data " + "when allow_pickle=False") try: return pickle.load(fid, **pickle_kwargs) except Exception: @@ -494,7 +502,7 @@ def save(file, arr, allow_pickle=True, fix_imports=True): >>> x = np.arange(10) >>> np.save(outfile, x) - >>> outfile.seek(0) # Only needed here to simulate closing & reopening file + >>> _ = outfile.seek(0) # Only needed here to simulate closing & reopening file >>> np.load(outfile) array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -589,10 +597,10 @@ def savez(file, *args, **kwds): Using `savez` with \\*args, the arrays are saved with default names. >>> np.savez(outfile, x, y) - >>> outfile.seek(0) # Only needed here to simulate closing & reopening file + >>> _ = outfile.seek(0) # Only needed here to simulate closing & reopening file >>> npzfile = np.load(outfile) >>> npzfile.files - ['arr_1', 'arr_0'] + ['arr_0', 'arr_1'] >>> npzfile['arr_0'] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -600,10 +608,10 @@ def savez(file, *args, **kwds): >>> outfile = TemporaryFile() >>> np.savez(outfile, x=x, y=y) - >>> outfile.seek(0) + >>> _ = outfile.seek(0) >>> npzfile = np.load(outfile) - >>> npzfile.files - ['y', 'x'] + >>> sorted(npzfile.files) + ['x', 'y'] >>> npzfile['x'] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -784,6 +792,8 @@ def _getconv(dtype): # amount of lines loadtxt reads in one chunk, can be overridden for testing _loadtxt_chunksize = 50000 + +@set_module('numpy') def loadtxt(fname, dtype=float, comments='#', delimiter=None, converters=None, skiprows=0, usecols=None, unpack=False, ndmin=0, encoding='bytes', max_rows=None): @@ -819,7 +829,7 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, `genfromtxt`): ``converters = {3: lambda s: float(s.strip() or 0)}``. Default: None. skiprows : int, optional - Skip the first `skiprows` lines; default: 0. + Skip the first `skiprows` lines, including comments; default: 0. usecols : int or sequence, optional Which columns to read, with 0 being the first. For example, ``usecols = (1,4,5)`` will extract the 2nd, 5th and 6th columns. @@ -881,21 +891,21 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, >>> from io import StringIO # StringIO behaves like a file object >>> c = StringIO(u"0 1\\n2 3") >>> np.loadtxt(c) - array([[ 0., 1.], - [ 2., 3.]]) + array([[0., 1.], + [2., 3.]]) >>> d = StringIO(u"M 21 72\\nF 35 58") >>> np.loadtxt(d, dtype={'names': ('gender', 'age', 'weight'), ... 'formats': ('S1', 'i4', 'f4')}) - array([('M', 21, 72.0), ('F', 35, 58.0)], - dtype=[('gender', '|S1'), ('age', '<i4'), ('weight', '<f4')]) + array([(b'M', 21, 72.), (b'F', 35, 58.)], + dtype=[('gender', 'S1'), ('age', '<i4'), ('weight', '<f4')]) >>> c = StringIO(u"1,0,2\\n3,0,4") >>> x, y = np.loadtxt(c, delimiter=',', usecols=(0, 2), unpack=True) >>> x - array([ 1., 3.]) + array([1., 3.]) >>> y - array([ 2., 4.]) + array([2., 4.]) """ # Type conversions for Py3 convenience @@ -1424,6 +1434,7 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', fh.close() +@set_module('numpy') def fromregex(file, regexp, dtype, encoding=None): """ Construct an array from a text file, using regular expression parsing. @@ -1470,17 +1481,17 @@ def fromregex(file, regexp, dtype, encoding=None): Examples -------- >>> f = open('test.dat', 'w') - >>> f.write("1312 foo\\n1534 bar\\n444 qux") + >>> _ = f.write("1312 foo\\n1534 bar\\n444 qux") >>> f.close() >>> regexp = r"(\\d+)\\s+(...)" # match [digits, whitespace, anything] >>> output = np.fromregex('test.dat', regexp, ... [('num', np.int64), ('key', 'S3')]) >>> output - array([(1312L, 'foo'), (1534L, 'bar'), (444L, 'qux')], - dtype=[('num', '<i8'), ('key', '|S3')]) + array([(1312, b'foo'), (1534, b'bar'), ( 444, b'qux')], + dtype=[('num', '<i8'), ('key', 'S3')]) >>> output['num'] - array([1312, 1534, 444], dtype=int64) + array([1312, 1534, 444]) """ own_fh = False @@ -1522,6 +1533,7 @@ def fromregex(file, regexp, dtype, encoding=None): #####-------------------------------------------------------------------------- +@set_module('numpy') def genfromtxt(fname, dtype=float, comments='#', delimiter=None, skip_header=0, skip_footer=0, converters=None, missing_values=None, filling_values=None, usecols=None, @@ -1662,26 +1674,26 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, >>> data = np.genfromtxt(s, dtype=[('myint','i8'),('myfloat','f8'), ... ('mystring','S5')], delimiter=",") >>> data - array((1, 1.3, 'abcde'), - dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '|S5')]) + array((1, 1.3, b'abcde'), + dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')]) Using dtype = None - >>> s.seek(0) # needed for StringIO example only + >>> _ = s.seek(0) # needed for StringIO example only >>> data = np.genfromtxt(s, dtype=None, ... names = ['myint','myfloat','mystring'], delimiter=",") >>> data - array((1, 1.3, 'abcde'), - dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '|S5')]) + array((1, 1.3, b'abcde'), + dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')]) Specifying dtype and names - >>> s.seek(0) + >>> _ = s.seek(0) >>> data = np.genfromtxt(s, dtype="i8,f8,S5", ... names=['myint','myfloat','mystring'], delimiter=",") >>> data - array((1, 1.3, 'abcde'), - dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '|S5')]) + array((1, 1.3, b'abcde'), + dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')]) An example with fixed-width columns @@ -1689,8 +1701,8 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, >>> data = np.genfromtxt(s, dtype=None, names=['intvar','fltvar','strvar'], ... delimiter=[1,3,5]) >>> data - array((1, 1.3, 'abcde'), - dtype=[('intvar', '<i8'), ('fltvar', '<f8'), ('strvar', '|S5')]) + array((1, 1.3, b'abcde'), + dtype=[('intvar', '<i8'), ('fltvar', '<f8'), ('strvar', 'S5')]) """ if max_rows is not None: @@ -2114,10 +2126,10 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, if names is None: # If the dtype is uniform (before sizing strings) - base = set([ + base = { c_type for c, c_type in zip(converters, column_types) - if c._checked]) + if c._checked} if len(base) == 1: uniform_type, = base (ddtype, mdtype) = (uniform_type, bool) diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index c2702f0a7..7904092ed 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -16,6 +16,7 @@ import numpy.core.numeric as NX from numpy.core import (isscalar, abs, finfo, atleast_1d, hstack, dot, array, ones) from numpy.core import overrides +from numpy.core.overrides import set_module from numpy.lib.twodim_base import diag, vander from numpy.lib.function_base import trim_zeros from numpy.lib.type_check import iscomplex, real, imag, mintypecode @@ -26,6 +27,7 @@ array_function_dispatch = functools.partial( overrides.array_function_dispatch, module='numpy') +@set_module('numpy') class RankWarning(UserWarning): """ Issued by `polyfit` when the Vandermonde matrix is rank deficient. @@ -108,7 +110,7 @@ def poly(seq_of_zeros): Given a sequence of a polynomial's zeros: >>> np.poly((0, 0, 0)) # Multiple root example - array([1, 0, 0, 0]) + array([1., 0., 0., 0.]) The line above represents z**3 + 0*z**2 + 0*z + 0. @@ -117,14 +119,14 @@ def poly(seq_of_zeros): The line above represents z**3 - z/4 - >>> np.poly((np.random.random(1.)[0], 0, np.random.random(1.)[0])) - array([ 1. , -0.77086955, 0.08618131, 0. ]) #random + >>> np.poly((np.random.random(1)[0], 0, np.random.random(1)[0])) + array([ 1. , -0.77086955, 0.08618131, 0. ]) # random Given a square array object: >>> P = np.array([[0, 1./3], [-1./2, 0]]) >>> np.poly(P) - array([ 1. , 0. , 0.16666667]) + array([1. , 0. , 0.16666667]) Note how in all cases the leading coefficient is always 1. @@ -270,7 +272,7 @@ def polyint(p, m=1, k=None): Parameters ---------- p : array_like or poly1d - Polynomial to differentiate. + Polynomial to integrate. A sequence is interpreted as polynomial coefficients, see `poly1d`. m : int, optional Order of the antiderivative. (Default: 1) @@ -293,7 +295,7 @@ def polyint(p, m=1, k=None): >>> p = np.poly1d([1,1,1]) >>> P = np.polyint(p) >>> P - poly1d([ 0.33333333, 0.5 , 1. , 0. ]) + poly1d([ 0.33333333, 0.5 , 1. , 0. ]) # may vary >>> np.polyder(P) == p True @@ -308,7 +310,7 @@ def polyint(p, m=1, k=None): 0.0 >>> P = np.polyint(p, 3, k=[6,5,3]) >>> P - poly1d([ 0.01666667, 0.04166667, 0.16666667, 3. , 5. , 3. ]) + poly1d([ 0.01666667, 0.04166667, 0.16666667, 3. , 5. , 3. ]) # may vary Note that 3 = 6 / 2!, and that the constants are given in the order of integrations. Constant of the highest-order polynomial term comes first: @@ -402,7 +404,7 @@ def polyder(p, m=1): >>> np.polyder(p, 3) poly1d([6]) >>> np.polyder(p, 4) - poly1d([ 0.]) + poly1d([0.]) """ m = int(m) @@ -461,9 +463,14 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): w : array_like, shape (M,), optional Weights to apply to the y-coordinates of the sample points. For gaussian uncertainties, use 1/sigma (not 1/sigma**2). - cov : bool, optional - Return the estimate and the covariance matrix of the estimate - If full is True, then cov is not returned. + cov : bool or str, optional + If given and not `False`, return not just the estimate but also its + covariance matrix. By default, the covariance are scaled by + chi2/sqrt(N-dof), i.e., the weights are presumed to be unreliable + except in a relative sense and everything is scaled such that the + reduced chi2 is unity. This scaling is omitted if ``cov='unscaled'``, + as is relevant for the case that the weights are 1/sigma**2, with + sigma known to be a reliable estimate of the uncertainty. Returns ------- @@ -545,28 +552,29 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): >>> y = np.array([0.0, 0.8, 0.9, 0.1, -0.8, -1.0]) >>> z = np.polyfit(x, y, 3) >>> z - array([ 0.08703704, -0.81349206, 1.69312169, -0.03968254]) + array([ 0.08703704, -0.81349206, 1.69312169, -0.03968254]) # may vary It is convenient to use `poly1d` objects for dealing with polynomials: >>> p = np.poly1d(z) >>> p(0.5) - 0.6143849206349179 + 0.6143849206349179 # may vary >>> p(3.5) - -0.34732142857143039 + -0.34732142857143039 # may vary >>> p(10) - 22.579365079365115 + 22.579365079365115 # may vary High-order polynomials may oscillate wildly: >>> p30 = np.poly1d(np.polyfit(x, y, 30)) - /... RankWarning: Polyfit may be poorly conditioned... + ... + >>> # RankWarning: Polyfit may be poorly conditioned... >>> p30(4) - -0.80000000000000204 + -0.80000000000000204 # may vary >>> p30(5) - -0.99999999999999445 + -0.99999999999999445 # may vary >>> p30(4.5) - -0.10547061179440398 + -0.10547061179440398 # may vary Illustration: @@ -631,14 +639,17 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): elif cov: Vbase = inv(dot(lhs.T, lhs)) Vbase /= NX.outer(scale, scale) - # Some literature ignores the extra -2.0 factor in the denominator, but - # it is included here because the covariance of Multivariate Student-T - # (which is implied by a Bayesian uncertainty analysis) includes it. - # Plus, it gives a slightly more conservative estimate of uncertainty. - if len(x) <= order + 2: - raise ValueError("the number of data points must exceed order + 2 " - "for Bayesian estimate the covariance matrix") - fac = resids / (len(x) - order - 2.0) + if cov == "unscaled": + fac = 1 + else: + if len(x) <= order: + raise ValueError("the number of data points must exceed order " + "to scale the covariance matrix") + # note, this used to be: fac = resids / (len(x) - order - 2.0) + # it was deciced that the "- 2" (originally justified by "Bayesian + # uncertainty analysis") is not was the user expects + # (see gh-11196 and gh-11197) + fac = resids / (len(x) - order) if y.ndim == 1: return c, Vbase * fac else: @@ -704,11 +715,11 @@ def polyval(p, x): >>> np.polyval([3,0,1], 5) # 3 * 5**2 + 0 * 5**1 + 1 76 >>> np.polyval([3,0,1], np.poly1d(5)) - poly1d([ 76.]) + poly1d([76.]) >>> np.polyval(np.poly1d([3,0,1]), 5) 76 >>> np.polyval(np.poly1d([3,0,1]), np.poly1d(5)) - poly1d([ 76.]) + poly1d([76.]) """ p = NX.asarray(p) @@ -941,7 +952,7 @@ def polydiv(u, v): >>> x = np.array([3.0, 5.0, 2.0]) >>> y = np.array([2.0, 1.0]) >>> np.polydiv(x, y) - (array([ 1.5 , 1.75]), array([ 0.25])) + (array([1.5 , 1.75]), array([0.25])) """ truepoly = (isinstance(u, poly1d) or isinstance(u, poly1d)) @@ -992,6 +1003,7 @@ def _raise_power(astr, wrap=70): return output + astr[n:] +@set_module('numpy') class poly1d(object): """ A one-dimensional polynomial class. @@ -1035,7 +1047,7 @@ class poly1d(object): >>> p.r array([-1.+1.41421356j, -1.-1.41421356j]) >>> p(p.r) - array([ -4.44089210e-16+0.j, -4.44089210e-16+0.j]) + array([ -4.44089210e-16+0.j, -4.44089210e-16+0.j]) # may vary These numbers in the previous line represent (0, 0) to machine precision @@ -1062,7 +1074,7 @@ class poly1d(object): poly1d([ 1, 4, 10, 12, 9]) >>> (p**3 + 4) / p - (poly1d([ 1., 4., 10., 12., 9.]), poly1d([ 4.])) + (poly1d([ 1., 4., 10., 12., 9.]), poly1d([4.])) ``asarray(p)`` gives the coefficient array, so polynomials can be used in all functions that accept arrays: @@ -1084,7 +1096,7 @@ class poly1d(object): Construct a polynomial from its roots: >>> np.poly1d([1, 2], True) - poly1d([ 1, -3, 2]) + poly1d([ 1., -3., 2.]) This is the same polynomial as obtained by: diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index 53a586f56..5ff35f0bb 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -17,6 +17,7 @@ from numpy.ma.mrecords import MaskedRecords from numpy.core.overrides import array_function_dispatch from numpy.lib._iotools import _is_string_like from numpy.compat import basestring +from numpy.testing import suppress_warnings if sys.version_info[0] < 3: from future_builtins import zip @@ -56,11 +57,10 @@ def recursive_fill_fields(input, output): Examples -------- >>> from numpy.lib import recfunctions as rfn - >>> a = np.array([(1, 10.), (2, 20.)], dtype=[('A', int), ('B', float)]) + >>> a = np.array([(1, 10.), (2, 20.)], dtype=[('A', np.int64), ('B', np.float64)]) >>> b = np.zeros((3,), dtype=a.dtype) >>> rfn.recursive_fill_fields(a, b) - array([(1, 10.0), (2, 20.0), (0, 0.0)], - dtype=[('A', '<i4'), ('B', '<f8')]) + array([(1, 10.), (2, 20.), (0, 0.)], dtype=[('A', '<i8'), ('B', '<f8')]) """ newdtype = output.dtype @@ -88,11 +88,11 @@ def get_fieldspec(dtype): Examples -------- - >>> dt = np.dtype([(('a', 'A'), int), ('b', float, 3)]) + >>> dt = np.dtype([(('a', 'A'), np.int64), ('b', np.double, 3)]) >>> dt.descr - [(('a', 'A'), '<i4'), ('b', '<f8', (3,))] + [(('a', 'A'), '<i8'), ('b', '<f8', (3,))] >>> get_fieldspec(dt) - [(('a', 'A'), dtype('int32')), ('b', dtype(('<f8', (3,))))] + [(('a', 'A'), dtype('int64')), ('b', dtype(('<f8', (3,))))] """ if dtype.names is None: @@ -102,7 +102,7 @@ def get_fieldspec(dtype): fields = ((name, dtype.fields[name]) for name in dtype.names) # keep any titles, if present return [ - (name if len(f) == 2 else (f[2], name), f[0]) + (name if len(f) == 2 else (f[2], name), f[0]) for name, f in fields ] @@ -119,10 +119,15 @@ def get_names(adtype): Examples -------- >>> from numpy.lib import recfunctions as rfn - >>> rfn.get_names(np.empty((1,), dtype=int)) is None - True + >>> rfn.get_names(np.empty((1,), dtype=int)) + Traceback (most recent call last): + ... + AttributeError: 'numpy.ndarray' object has no attribute 'names' + >>> rfn.get_names(np.empty((1,), dtype=[('A',int), ('B', float)])) - ('A', 'B') + Traceback (most recent call last): + ... + AttributeError: 'numpy.ndarray' object has no attribute 'names' >>> adtype = np.dtype([('a', int), ('b', [('ba', int), ('bb', int)])]) >>> rfn.get_names(adtype) ('a', ('b', ('ba', 'bb'))) @@ -152,9 +157,13 @@ def get_names_flat(adtype): -------- >>> from numpy.lib import recfunctions as rfn >>> rfn.get_names_flat(np.empty((1,), dtype=int)) is None - True + Traceback (most recent call last): + ... + AttributeError: 'numpy.ndarray' object has no attribute 'names' >>> rfn.get_names_flat(np.empty((1,), dtype=[('A',int), ('B', float)])) - ('A', 'B') + Traceback (most recent call last): + ... + AttributeError: 'numpy.ndarray' object has no attribute 'names' >>> adtype = np.dtype([('a', int), ('b', [('ba', int), ('bb', int)])]) >>> rfn.get_names_flat(adtype) ('a', 'b', 'ba', 'bb') @@ -402,20 +411,18 @@ def merge_arrays(seqarrays, fill_value=-1, flatten=False, -------- >>> from numpy.lib import recfunctions as rfn >>> rfn.merge_arrays((np.array([1, 2]), np.array([10., 20., 30.]))) - masked_array(data = [(1, 10.0) (2, 20.0) (--, 30.0)], - mask = [(False, False) (False, False) (True, False)], - fill_value = (999999, 1e+20), - dtype = [('f0', '<i4'), ('f1', '<f8')]) - - >>> rfn.merge_arrays((np.array([1, 2]), np.array([10., 20., 30.])), - ... usemask=False) - array([(1, 10.0), (2, 20.0), (-1, 30.0)], - dtype=[('f0', '<i4'), ('f1', '<f8')]) - >>> rfn.merge_arrays((np.array([1, 2]).view([('a', int)]), + array([( 1, 10.), ( 2, 20.), (-1, 30.)], + dtype=[('f0', '<i8'), ('f1', '<f8')]) + + >>> rfn.merge_arrays((np.array([1, 2], dtype=np.int64), + ... np.array([10., 20., 30.])), usemask=False) + array([(1, 10.0), (2, 20.0), (-1, 30.0)], + dtype=[('f0', '<i8'), ('f1', '<f8')]) + >>> rfn.merge_arrays((np.array([1, 2]).view([('a', np.int64)]), ... np.array([10., 20., 30.])), ... usemask=False, asrecarray=True) - rec.array([(1, 10.0), (2, 20.0), (-1, 30.0)], - dtype=[('a', '<i4'), ('f1', '<f8')]) + rec.array([( 1, 10.), ( 2, 20.), (-1, 30.)], + dtype=[('a', '<i8'), ('f1', '<f8')]) Notes ----- @@ -546,16 +553,14 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): -------- >>> from numpy.lib import recfunctions as rfn >>> a = np.array([(1, (2, 3.0)), (4, (5, 6.0))], - ... dtype=[('a', int), ('b', [('ba', float), ('bb', int)])]) + ... dtype=[('a', np.int64), ('b', [('ba', np.double), ('bb', np.int64)])]) >>> rfn.drop_fields(a, 'a') - array([((2.0, 3),), ((5.0, 6),)], - dtype=[('b', [('ba', '<f8'), ('bb', '<i4')])]) + array([((2., 3),), ((5., 6),)], + dtype=[('b', [('ba', '<f8'), ('bb', '<i8')])]) >>> rfn.drop_fields(a, 'ba') - array([(1, (3,)), (4, (6,))], - dtype=[('a', '<i4'), ('b', [('bb', '<i4')])]) + array([(1, (3,)), (4, (6,))], dtype=[('a', '<i8'), ('b', [('bb', '<i8')])]) >>> rfn.drop_fields(a, ['ba', 'bb']) - array([(1,), (4,)], - dtype=[('a', '<i4')]) + array([(1,), (4,)], dtype=[('a', '<i8')]) """ if _is_string_like(drop_names): drop_names = [drop_names] @@ -647,8 +652,8 @@ def rename_fields(base, namemapper): >>> a = np.array([(1, (2, [3.0, 30.])), (4, (5, [6.0, 60.]))], ... dtype=[('a', int),('b', [('ba', float), ('bb', (float, 2))])]) >>> rfn.rename_fields(a, {'a':'A', 'bb':'BB'}) - array([(1, (2.0, [3.0, 30.0])), (4, (5.0, [6.0, 60.0]))], - dtype=[('A', '<i4'), ('b', [('ba', '<f8'), ('BB', '<f8', 2)])]) + array([(1, (2., [ 3., 30.])), (4, (5., [ 6., 60.]))], + dtype=[('A', '<i8'), ('b', [('ba', '<f8'), ('BB', '<f8', (2,))])]) """ def _recursive_rename_fields(ndtype, namemapper): @@ -833,18 +838,18 @@ def repack_fields(a, align=False, recurse=False): ... print("offsets:", [d.fields[name][1] for name in d.names]) ... print("itemsize:", d.itemsize) ... - >>> dt = np.dtype('u1,i4,f4', align=True) + >>> dt = np.dtype('u1,<i4,<f4', align=True) >>> dt - dtype({'names':['f0','f1','f2'], 'formats':['u1','<i4','<f8'], 'offsets':[0,4,8], 'itemsize':16}, align=True) + dtype({'names':['f0','f1','f2'], 'formats':['u1','<i8','<f8'], 'offsets':[0,8,16], 'itemsize':24}, align=True) >>> print_offsets(dt) - offsets: [0, 4, 8] - itemsize: 16 + offsets: [0, 8, 16] + itemsize: 24 >>> packed_dt = repack_fields(dt) >>> packed_dt - dtype([('f0', 'u1'), ('f1', '<i4'), ('f2', '<f8')]) + dtype([('f0', 'u1'), ('f1', '<i8'), ('f2', '<f8')]) >>> print_offsets(packed_dt) - offsets: [0, 1, 5] - itemsize: 13 + offsets: [0, 1, 9] + itemsize: 17 """ if not isinstance(a, np.dtype): @@ -870,6 +875,344 @@ def repack_fields(a, align=False, recurse=False): dt = np.dtype(fieldinfo, align=align) return np.dtype((a.type, dt)) +def _get_fields_and_offsets(dt, offset=0): + """ + Returns a flat list of (dtype, count, offset) tuples of all the + scalar fields in the dtype "dt", including nested fields, in left + to right order. + """ + fields = [] + for name in dt.names: + field = dt.fields[name] + if field[0].names is None: + count = 1 + for size in field[0].shape: + count *= size + fields.append((field[0], count, field[1] + offset)) + else: + fields.extend(_get_fields_and_offsets(field[0], field[1] + offset)) + return fields + + +def _structured_to_unstructured_dispatcher(arr, dtype=None, copy=None, + casting=None): + return (arr,) + +@array_function_dispatch(_structured_to_unstructured_dispatcher) +def structured_to_unstructured(arr, dtype=None, copy=False, casting='unsafe'): + """ + Converts and n-D structured array into an (n+1)-D unstructured array. + + The new array will have a new last dimension equal in size to the + number of field-elements of the input array. If not supplied, the output + datatype is determined from the numpy type promotion rules applied to all + the field datatypes. + + Nested fields, as well as each element of any subarray fields, all count + as a single field-elements. + + Parameters + ---------- + arr : ndarray + Structured array or dtype to convert. Cannot contain object datatype. + dtype : dtype, optional + The dtype of the output unstructured array. + copy : bool, optional + See copy argument to `ndarray.astype`. If true, always return a copy. + If false, and `dtype` requirements are satisfied, a view is returned. + casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional + See casting argument of `ndarray.astype`. Controls what kind of data + casting may occur. + + Returns + ------- + unstructured : ndarray + Unstructured array with one more dimension. + + Examples + -------- + + >>> a = np.zeros(4, dtype=[('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + >>> a + array([(0, (0., 0), [0., 0.]), (0, (0., 0), [0., 0.]), + (0, (0., 0), [0., 0.]), (0, (0., 0), [0., 0.])], + dtype=[('a', '<i4'), ('b', [('f0', '<f4'), ('f1', '<u2')]), ('c', '<f4', (2,))]) + >>> structured_to_unstructured(arr) + array([[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]]) + + >>> b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + ... dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + >>> np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) + array([ 3. , 5.5, 9. , 11. ]) + + """ + if arr.dtype.names is None: + raise ValueError('arr must be a structured array') + + fields = _get_fields_and_offsets(arr.dtype) + n_fields = len(fields) + dts, counts, offsets = zip(*fields) + names = ['f{}'.format(n) for n in range(n_fields)] + + if dtype is None: + out_dtype = np.result_type(*[dt.base for dt in dts]) + else: + out_dtype = dtype + + # Use a series of views and casts to convert to an unstructured array: + + # first view using flattened fields (doesn't work for object arrays) + # Note: dts may include a shape for subarrays + flattened_fields = np.dtype({'names': names, + 'formats': dts, + 'offsets': offsets, + 'itemsize': arr.dtype.itemsize}) + with suppress_warnings() as sup: # until 1.16 (gh-12447) + sup.filter(FutureWarning, "Numpy has detected") + arr = arr.view(flattened_fields) + + # next cast to a packed format with all fields converted to new dtype + packed_fields = np.dtype({'names': names, + 'formats': [(out_dtype, c) for c in counts]}) + arr = arr.astype(packed_fields, copy=copy, casting=casting) + + # finally is it safe to view the packed fields as the unstructured type + return arr.view((out_dtype, sum(counts))) + +def _unstructured_to_structured_dispatcher(arr, dtype=None, names=None, + align=None, copy=None, casting=None): + return (arr,) + +@array_function_dispatch(_unstructured_to_structured_dispatcher) +def unstructured_to_structured(arr, dtype=None, names=None, align=False, + copy=False, casting='unsafe'): + """ + Converts and n-D unstructured array into an (n-1)-D structured array. + + The last dimension of the input array is converted into a structure, with + number of field-elements equal to the size of the last dimension of the + input array. By default all output fields have the input array's dtype, but + an output structured dtype with an equal number of fields-elements can be + supplied instead. + + Nested fields, as well as each element of any subarray fields, all count + towards the number of field-elements. + + Parameters + ---------- + arr : ndarray + Unstructured array or dtype to convert. + dtype : dtype, optional + The structured dtype of the output array + names : list of strings, optional + If dtype is not supplied, this specifies the field names for the output + dtype, in order. The field dtypes will be the same as the input array. + align : boolean, optional + Whether to create an aligned memory layout. + copy : bool, optional + See copy argument to `ndarray.astype`. If true, always return a copy. + If false, and `dtype` requirements are satisfied, a view is returned. + casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional + See casting argument of `ndarray.astype`. Controls what kind of data + casting may occur. + + Returns + ------- + structured : ndarray + Structured array with fewer dimensions. + + Examples + -------- + + >>> dt = np.dtype([('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + >>> a = np.arange(20).reshape((4,5)) + >>> a + array([[ 0, 1, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, 13, 14], + [15, 16, 17, 18, 19]]) + >>> unstructured_to_structured(a, dt) + array([( 0, ( 1., 2), [ 3., 4.]), ( 5, ( 6., 7), [ 8., 9.]), + (10, (11., 12), [13., 14.]), (15, (16., 17), [18., 19.])], + dtype=[('a', '<i4'), ('b', [('f0', '<f4'), ('f1', '<u2')]), ('c', '<f4', (2,))]) + + """ + if arr.shape == (): + raise ValueError('arr must have at least one dimension') + n_elem = arr.shape[-1] + + if dtype is None: + if names is None: + names = ['f{}'.format(n) for n in range(n_elem)] + out_dtype = np.dtype([(n, arr.dtype) for n in names], align=align) + fields = _get_fields_and_offsets(out_dtype) + dts, counts, offsets = zip(*fields) + else: + if names is not None: + raise ValueError("don't supply both dtype and names") + # sanity check of the input dtype + fields = _get_fields_and_offsets(dtype) + dts, counts, offsets = zip(*fields) + if n_elem != sum(counts): + raise ValueError('The length of the last dimension of arr must ' + 'be equal to the number of fields in dtype') + out_dtype = dtype + if align and not out_dtype.isalignedstruct: + raise ValueError("align was True but dtype is not aligned") + + names = ['f{}'.format(n) for n in range(len(fields))] + + # Use a series of views and casts to convert to a structured array: + + # first view as a packed structured array of one dtype + packed_fields = np.dtype({'names': names, + 'formats': [(arr.dtype, c) for c in counts]}) + arr = np.ascontiguousarray(arr).view(packed_fields) + + # next cast to an unpacked but flattened format with varied dtypes + flattened_fields = np.dtype({'names': names, + 'formats': dts, + 'offsets': offsets, + 'itemsize': out_dtype.itemsize}) + arr = arr.astype(flattened_fields, copy=copy, casting=casting) + + # finally view as the final nested dtype and remove the last axis + return arr.view(out_dtype)[..., 0] + +def _apply_along_fields_dispatcher(func, arr): + return (arr,) + +@array_function_dispatch(_apply_along_fields_dispatcher) +def apply_along_fields(func, arr): + """ + Apply function 'func' as a reduction across fields of a structured array. + + This is similar to `apply_along_axis`, but treats the fields of a + structured array as an extra axis. The fields are all first cast to a + common type following the type-promotion rules from `numpy.result_type` + applied to the field's dtypes. + + Parameters + ---------- + func : function + Function to apply on the "field" dimension. This function must + support an `axis` argument, like np.mean, np.sum, etc. + arr : ndarray + Structured array for which to apply func. + + Returns + ------- + out : ndarray + Result of the recution operation + + Examples + -------- + + >>> b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + ... dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + >>> apply_along_fields(np.mean, b) + array([ 2.66666667, 5.33333333, 8.66666667, 11. ]) + >>> apply_along_fields(np.mean, b[['x', 'z']]) + array([ 3. , 5.5, 9. , 11. ]) + + """ + if arr.dtype.names is None: + raise ValueError('arr must be a structured array') + + uarr = structured_to_unstructured(arr) + return func(uarr, axis=-1) + # works and avoids axis requirement, but very, very slow: + #return np.apply_along_axis(func, -1, uarr) + +def _assign_fields_by_name_dispatcher(dst, src, zero_unassigned=None): + return dst, src + +@array_function_dispatch(_assign_fields_by_name_dispatcher) +def assign_fields_by_name(dst, src, zero_unassigned=True): + """ + Assigns values from one structured array to another by field name. + + Normally in numpy >= 1.14, assignment of one structured array to another + copies fields "by position", meaning that the first field from the src is + copied to the first field of the dst, and so on, regardless of field name. + + This function instead copies "by field name", such that fields in the dst + are assigned from the identically named field in the src. This applies + recursively for nested structures. This is how structure assignment worked + in numpy >= 1.6 to <= 1.13. + + Parameters + ---------- + dst : ndarray + src : ndarray + The source and destination arrays during assignment. + zero_unassigned : bool, optional + If True, fields in the dst for which there was no matching + field in the src are filled with the value 0 (zero). This + was the behavior of numpy <= 1.13. If False, those fields + are not modified. + """ + + if dst.dtype.names is None: + dst[...] = src + return + + for name in dst.dtype.names: + if name not in src.dtype.names: + if zero_unassigned: + dst[name] = 0 + else: + assign_fields_by_name(dst[name], src[name], + zero_unassigned) + +def _require_fields_dispatcher(array, required_dtype): + return (array,) + +@array_function_dispatch(_require_fields_dispatcher) +def require_fields(array, required_dtype): + """ + Casts a structured array to a new dtype using assignment by field-name. + + This function assigns from the old to the new array by name, so the + value of a field in the output array is the value of the field with the + same name in the source array. This has the effect of creating a new + ndarray containing only the fields "required" by the required_dtype. + + If a field name in the required_dtype does not exist in the + input array, that field is created and set to 0 in the output array. + + Parameters + ---------- + a : ndarray + array to cast + required_dtype : dtype + datatype for output array + + Returns + ------- + out : ndarray + array with the new dtype, with field values copied from the fields in + the input array with the same name + + Examples + -------- + + >>> a = np.ones(4, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) + >>> require_fields(a, [('b', 'f4'), ('c', 'u1')]) + array([(1., 1), (1., 1), (1., 1), (1., 1)], + dtype=[('b', '<f4'), ('c', 'u1')]) + >>> require_fields(a, [('b', 'f4'), ('newf', 'u1')]) + array([(1., 0), (1., 0), (1., 0), (1., 0)], + dtype=[('b', '<f4'), ('newf', 'u1')]) + + """ + out = np.empty(array.shape, dtype=required_dtype) + assign_fields_by_name(out, array) + return out + def _stack_arrays_dispatcher(arrays, defaults=None, usemask=None, asrecarray=None, autoconvert=None): @@ -905,15 +1248,16 @@ def stack_arrays(arrays, defaults=None, usemask=True, asrecarray=False, True >>> z = np.array([('A', 1), ('B', 2)], dtype=[('A', '|S3'), ('B', float)]) >>> zz = np.array([('a', 10., 100.), ('b', 20., 200.), ('c', 30., 300.)], - ... dtype=[('A', '|S3'), ('B', float), ('C', float)]) + ... dtype=[('A', '|S3'), ('B', np.double), ('C', np.double)]) >>> test = rfn.stack_arrays((z,zz)) >>> test - masked_array(data = [('A', 1.0, --) ('B', 2.0, --) ('a', 10.0, 100.0) ('b', 20.0, 200.0) - ('c', 30.0, 300.0)], - mask = [(False, False, True) (False, False, True) (False, False, False) - (False, False, False) (False, False, False)], - fill_value = ('N/A', 1e+20, 1e+20), - dtype = [('A', '|S3'), ('B', '<f8'), ('C', '<f8')]) + masked_array(data=[(b'A', 1.0, --), (b'B', 2.0, --), (b'a', 10.0, 100.0), + (b'b', 20.0, 200.0), (b'c', 30.0, 300.0)], + mask=[(False, False, True), (False, False, True), + (False, False, False), (False, False, False), + (False, False, False)], + fill_value=(b'N/A', 1.e+20, 1.e+20), + dtype=[('A', 'S3'), ('B', '<f8'), ('C', '<f8')]) """ if isinstance(arrays, ndarray): @@ -992,7 +1336,10 @@ def find_duplicates(a, key=None, ignoremask=True, return_index=False): >>> a = np.ma.array([1, 1, 1, 2, 2, 3, 3], ... mask=[0, 0, 1, 0, 0, 0, 1]).view(ndtype) >>> rfn.find_duplicates(a, ignoremask=True, return_index=True) - ... # XXX: judging by the output, the ignoremask flag has no effect + (masked_array(data=[(1,), (1,), (2,), (2,)], + mask=[(False,), (False,), (False,), (False,)], + fill_value=(999999,), + dtype=[('a', '<i8')]), array([0, 1, 3, 4])) """ a = np.asanyarray(a).ravel() # Get a dictionary of fields diff --git a/numpy/lib/scimath.py b/numpy/lib/scimath.py index 9ca006841..5ac790ce9 100644 --- a/numpy/lib/scimath.py +++ b/numpy/lib/scimath.py @@ -59,7 +59,7 @@ def _tocomplex(arr): >>> a = np.array([1,2,3],np.short) >>> ac = np.lib.scimath._tocomplex(a); ac - array([ 1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) + array([1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) >>> ac.dtype dtype('complex64') @@ -70,7 +70,7 @@ def _tocomplex(arr): >>> b = np.array([1,2,3],np.double) >>> bc = np.lib.scimath._tocomplex(b); bc - array([ 1.+0.j, 2.+0.j, 3.+0.j]) + array([1.+0.j, 2.+0.j, 3.+0.j]) >>> bc.dtype dtype('complex128') @@ -81,13 +81,13 @@ def _tocomplex(arr): >>> c = np.array([1,2,3],np.csingle) >>> cc = np.lib.scimath._tocomplex(c); cc - array([ 1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) + array([1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) >>> c *= 2; c - array([ 2.+0.j, 4.+0.j, 6.+0.j], dtype=complex64) + array([2.+0.j, 4.+0.j, 6.+0.j], dtype=complex64) >>> cc - array([ 1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) + array([1.+0.j, 2.+0.j, 3.+0.j], dtype=complex64) """ if issubclass(arr.dtype.type, (nt.single, nt.byte, nt.short, nt.ubyte, nt.ushort, nt.csingle)): @@ -170,7 +170,7 @@ def _fix_real_abs_gt_1(x): array([0, 1]) >>> np.lib.scimath._fix_real_abs_gt_1([0,2]) - array([ 0.+0.j, 2.+0.j]) + array([0.+0.j, 2.+0.j]) """ x = asarray(x) if any(isreal(x) & (abs(x) > 1)): @@ -212,14 +212,14 @@ def sqrt(x): >>> np.lib.scimath.sqrt(1) 1.0 >>> np.lib.scimath.sqrt([1, 4]) - array([ 1., 2.]) + array([1., 2.]) But it automatically handles negative inputs: >>> np.lib.scimath.sqrt(-1) - (0.0+1.0j) + 1j >>> np.lib.scimath.sqrt([-1,4]) - array([ 0.+1.j, 2.+0.j]) + array([0.+1.j, 2.+0.j]) """ x = _fix_real_lt_zero(x) @@ -317,7 +317,7 @@ def log10(x): 1.0 >>> np.emath.log10([-10**1, -10**2, 10**2]) - array([ 1.+1.3644j, 2.+1.3644j, 2.+0.j ]) + array([1.+1.3644j, 2.+1.3644j, 2.+0.j ]) """ x = _fix_real_lt_zero(x) @@ -354,9 +354,9 @@ def logn(n, x): >>> np.set_printoptions(precision=4) >>> np.lib.scimath.logn(2, [4, 8]) - array([ 2., 3.]) + array([2., 3.]) >>> np.lib.scimath.logn(2, [-4, -8, 8]) - array([ 2.+4.5324j, 3.+4.5324j, 3.+0.j ]) + array([2.+4.5324j, 3.+4.5324j, 3.+0.j ]) """ x = _fix_real_lt_zero(x) @@ -405,7 +405,7 @@ def log2(x): >>> np.emath.log2(8) 3.0 >>> np.emath.log2([-4, -8, 8]) - array([ 2.+4.5324j, 3.+4.5324j, 3.+0.j ]) + array([2.+4.5324j, 3.+4.5324j, 3.+0.j ]) """ x = _fix_real_lt_zero(x) @@ -451,9 +451,9 @@ def power(x, p): >>> np.lib.scimath.power([2, 4], 2) array([ 4, 16]) >>> np.lib.scimath.power([2, 4], -2) - array([ 0.25 , 0.0625]) + array([0.25 , 0.0625]) >>> np.lib.scimath.power([-2, 4], 2) - array([ 4.+0.j, 16.+0.j]) + array([ 4.-0.j, 16.+0.j]) """ x = _fix_real_lt_zero(x) @@ -499,7 +499,7 @@ def arccos(x): 0.0 >>> np.emath.arccos([1,2]) - array([ 0.-0.j , 0.+1.317j]) + array([0.-0.j , 0.-1.317j]) """ x = _fix_real_abs_gt_1(x) @@ -545,7 +545,7 @@ def arcsin(x): 0.0 >>> np.emath.arcsin([0,1]) - array([ 0. , 1.5708]) + array([0. , 1.5708]) """ x = _fix_real_abs_gt_1(x) @@ -589,11 +589,14 @@ def arctanh(x): -------- >>> np.set_printoptions(precision=4) - >>> np.emath.arctanh(np.eye(2)) - array([[ Inf, 0.], - [ 0., Inf]]) + >>> from numpy.testing import suppress_warnings + >>> with suppress_warnings() as sup: + ... sup.filter(RuntimeWarning) + ... np.emath.arctanh(np.eye(2)) + array([[inf, 0.], + [ 0., inf]]) >>> np.emath.arctanh([1j]) - array([ 0.+0.7854j]) + array([0.+0.7854j]) """ x = _fix_real_abs_gt_1(x) diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py index 6e7cab3fa..e088a6c4a 100644 --- a/numpy/lib/shape_base.py +++ b/numpy/lib/shape_base.py @@ -11,7 +11,8 @@ from numpy.core.fromnumeric import product, reshape, transpose from numpy.core.multiarray import normalize_axis_index from numpy.core import overrides from numpy.core import vstack, atleast_3d -from numpy.core.shape_base import _arrays_for_stack_dispatcher +from numpy.core.shape_base import ( + _arrays_for_stack_dispatcher, _warn_for_nonsequence) from numpy.lib.index_tricks import ndindex from numpy.matrixlib.defmatrix import matrix # this raises all the right alarm bells @@ -128,7 +129,7 @@ def take_along_axis(arr, indices, axis): [40, 50, 60]]) >>> ai = np.argsort(a, axis=1); ai array([[0, 2, 1], - [1, 2, 0]], dtype=int64) + [1, 2, 0]]) >>> np.take_along_axis(a, ai, axis=1) array([[10, 20, 30], [40, 50, 60]]) @@ -141,7 +142,7 @@ def take_along_axis(arr, indices, axis): >>> ai = np.expand_dims(np.argmax(a, axis=1), axis=1) >>> ai array([[1], - [0], dtype=int64) + [0]]) >>> np.take_along_axis(a, ai, axis=1) array([[30], [60]]) @@ -151,10 +152,10 @@ def take_along_axis(arr, indices, axis): >>> ai_min = np.expand_dims(np.argmin(a, axis=1), axis=1) >>> ai_max = np.expand_dims(np.argmax(a, axis=1), axis=1) - >>> ai = np.concatenate([ai_min, ai_max], axis=axis) - >> ai + >>> ai = np.concatenate([ai_min, ai_max], axis=1) + >>> ai array([[0, 1], - [1, 0]], dtype=int64) + [1, 0]]) >>> np.take_along_axis(a, ai, axis=1) array([[10, 30], [40, 60]]) @@ -242,7 +243,7 @@ def put_along_axis(arr, indices, values, axis): >>> ai = np.expand_dims(np.argmax(a, axis=1), axis=1) >>> ai array([[1], - [0]], dtype=int64) + [0]]) >>> np.put_along_axis(a, ai, 99, axis=1) >>> a array([[10, 99, 20], @@ -329,9 +330,9 @@ def apply_along_axis(func1d, axis, arr, *args, **kwargs): ... return (a[0] + a[-1]) * 0.5 >>> b = np.array([[1,2,3], [4,5,6], [7,8,9]]) >>> np.apply_along_axis(my_func, 0, b) - array([ 4., 5., 6.]) + array([4., 5., 6.]) >>> np.apply_along_axis(my_func, 1, b) - array([ 2., 5., 8.]) + array([2., 5., 8.]) For a function that returns a 1D array, the number of dimensions in `outarr` is the same as `arr`. @@ -629,6 +630,7 @@ def column_stack(tup): [3, 4]]) """ + _warn_for_nonsequence(tup) arrays = [] for v in tup: arr = array(v, copy=False, subok=True) @@ -693,6 +695,7 @@ def dstack(tup): [[3, 4]]]) """ + _warn_for_nonsequence(tup) return _nx.concatenate([atleast_3d(_m) for _m in tup], 2) @@ -729,11 +732,11 @@ def array_split(ary, indices_or_sections, axis=0): -------- >>> x = np.arange(8.0) >>> np.array_split(x, 3) - [array([ 0., 1., 2.]), array([ 3., 4., 5.]), array([ 6., 7.])] + [array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7.])] >>> x = np.arange(7.0) >>> np.array_split(x, 3) - [array([ 0., 1., 2.]), array([ 3., 4.]), array([ 5., 6.])] + [array([0., 1., 2.]), array([3., 4.]), array([5., 6.])] """ try: @@ -825,14 +828,14 @@ def split(ary, indices_or_sections, axis=0): -------- >>> x = np.arange(9.0) >>> np.split(x, 3) - [array([ 0., 1., 2.]), array([ 3., 4., 5.]), array([ 6., 7., 8.])] + [array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7., 8.])] >>> x = np.arange(8.0) >>> np.split(x, [3, 5, 6, 10]) - [array([ 0., 1., 2.]), - array([ 3., 4.]), - array([ 5.]), - array([ 6., 7.]), + [array([0., 1., 2.]), + array([3., 4.]), + array([5.]), + array([6., 7.]), array([], dtype=float64)] """ @@ -869,43 +872,43 @@ def hsplit(ary, indices_or_sections): -------- >>> x = np.arange(16.0).reshape(4, 4) >>> x - array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]]) + array([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) >>> np.hsplit(x, 2) [array([[ 0., 1.], [ 4., 5.], [ 8., 9.], - [ 12., 13.]]), + [12., 13.]]), array([[ 2., 3.], [ 6., 7.], - [ 10., 11.], - [ 14., 15.]])] + [10., 11.], + [14., 15.]])] >>> np.hsplit(x, np.array([3, 6])) - [array([[ 0., 1., 2.], - [ 4., 5., 6.], - [ 8., 9., 10.], - [ 12., 13., 14.]]), - array([[ 3.], - [ 7.], - [ 11.], - [ 15.]]), - array([], dtype=float64)] + [array([[ 0., 1., 2.], + [ 4., 5., 6.], + [ 8., 9., 10.], + [12., 13., 14.]]), + array([[ 3.], + [ 7.], + [11.], + [15.]]), + array([], shape=(4, 0), dtype=float64)] With a higher dimensional array the split is still along the second axis. >>> x = np.arange(8.0).reshape(2, 2, 2) >>> x - array([[[ 0., 1.], - [ 2., 3.]], - [[ 4., 5.], - [ 6., 7.]]]) + array([[[0., 1.], + [2., 3.]], + [[4., 5.], + [6., 7.]]]) >>> np.hsplit(x, 2) - [array([[[ 0., 1.]], - [[ 4., 5.]]]), - array([[[ 2., 3.]], - [[ 6., 7.]]])] + [array([[[0., 1.]], + [[4., 5.]]]), + array([[[2., 3.]], + [[6., 7.]]])] """ if _nx.ndim(ary) == 0: @@ -933,35 +936,31 @@ def vsplit(ary, indices_or_sections): -------- >>> x = np.arange(16.0).reshape(4, 4) >>> x - array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]]) + array([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) >>> np.vsplit(x, 2) - [array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.]]), - array([[ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]])] + [array([[0., 1., 2., 3.], + [4., 5., 6., 7.]]), array([[ 8., 9., 10., 11.], + [12., 13., 14., 15.]])] >>> np.vsplit(x, np.array([3, 6])) - [array([[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.], - [ 8., 9., 10., 11.]]), - array([[ 12., 13., 14., 15.]]), - array([], dtype=float64)] + [array([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.]]), array([[12., 13., 14., 15.]]), array([], shape=(0, 4), dtype=float64)] With a higher dimensional array the split is still along the first axis. >>> x = np.arange(8.0).reshape(2, 2, 2) >>> x - array([[[ 0., 1.], - [ 2., 3.]], - [[ 4., 5.], - [ 6., 7.]]]) + array([[[0., 1.], + [2., 3.]], + [[4., 5.], + [6., 7.]]]) >>> np.vsplit(x, 2) - [array([[[ 0., 1.], - [ 2., 3.]]]), - array([[[ 4., 5.], - [ 6., 7.]]])] + [array([[[0., 1.], + [2., 3.]]]), array([[[4., 5.], + [6., 7.]]])] """ if _nx.ndim(ary) < 2: @@ -986,30 +985,28 @@ def dsplit(ary, indices_or_sections): -------- >>> x = np.arange(16.0).reshape(2, 2, 4) >>> x - array([[[ 0., 1., 2., 3.], - [ 4., 5., 6., 7.]], - [[ 8., 9., 10., 11.], - [ 12., 13., 14., 15.]]]) + array([[[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.]], + [[ 8., 9., 10., 11.], + [12., 13., 14., 15.]]]) >>> np.dsplit(x, 2) - [array([[[ 0., 1.], - [ 4., 5.]], - [[ 8., 9.], - [ 12., 13.]]]), - array([[[ 2., 3.], - [ 6., 7.]], - [[ 10., 11.], - [ 14., 15.]]])] + [array([[[ 0., 1.], + [ 4., 5.]], + [[ 8., 9.], + [12., 13.]]]), array([[[ 2., 3.], + [ 6., 7.]], + [[10., 11.], + [14., 15.]]])] >>> np.dsplit(x, np.array([3, 6])) - [array([[[ 0., 1., 2.], - [ 4., 5., 6.]], - [[ 8., 9., 10.], - [ 12., 13., 14.]]]), - array([[[ 3.], - [ 7.]], - [[ 11.], - [ 15.]]]), - array([], dtype=float64)] - + [array([[[ 0., 1., 2.], + [ 4., 5., 6.]], + [[ 8., 9., 10.], + [12., 13., 14.]]]), + array([[[ 3.], + [ 7.]], + [[11.], + [15.]]]), + array([], shape=(2, 2, 0), dtype=float64)] """ if _nx.ndim(ary) < 3: raise ValueError('dsplit only works on arrays of 3 or more dimensions') @@ -1089,15 +1086,15 @@ def kron(a, b): Examples -------- >>> np.kron([1,10,100], [5,6,7]) - array([ 5, 6, 7, 50, 60, 70, 500, 600, 700]) + array([ 5, 6, 7, ..., 500, 600, 700]) >>> np.kron([5,6,7], [1,10,100]) - array([ 5, 50, 500, 6, 60, 600, 7, 70, 700]) + array([ 5, 50, 500, ..., 7, 70, 700]) >>> np.kron(np.eye(2), np.ones((2,2))) - array([[ 1., 1., 0., 0.], - [ 1., 1., 0., 0.], - [ 0., 0., 1., 1.], - [ 0., 0., 1., 1.]]) + array([[1., 1., 0., 0.], + [1., 1., 0., 0.], + [0., 0., 1., 1.], + [0., 0., 1., 1.]]) >>> a = np.arange(100).reshape((2,5,2,5)) >>> b = np.arange(24).reshape((2,3,4)) diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index 1df8bebf6..8eac16b58 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -361,3 +361,18 @@ class TestOpenFunc(object): fp = datasource.open(local_file) assert_(fp) fp.close() + +def test_del_attr_handling(): + # DataSource __del__ can be called + # even if __init__ fails when the + # Exception object is caught by the + # caller as happens in refguide_check + # is_deprecated() function + + ds = datasource.DataSource() + # simulate failed __init__ by removing key attribute + # produced within __init__ and expected by __del__ + del ds._istmpdest + # should not raise an AttributeError if __del__ + # gracefully handles failed __init__: + ds.__del__() diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index b4888f1bd..e04fdc808 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -1,6 +1,5 @@ from __future__ import division, absolute_import, print_function -import sys import time from datetime import date @@ -246,7 +245,7 @@ class TestStringConverter(object): converter = StringConverter(int, default=0, missing_values="N/A") assert_equal( - converter.missing_values, set(['', 'N/A'])) + converter.missing_values, {'', 'N/A'}) def test_int64_dtype(self): "Check that int64 integer types can be specified" diff --git a/numpy/lib/tests/test_arraypad.py b/numpy/lib/tests/test_arraypad.py index e62fccaa0..20f6e4a1b 100644 --- a/numpy/lib/tests/test_arraypad.py +++ b/numpy/lib/tests/test_arraypad.py @@ -9,6 +9,91 @@ import numpy as np from numpy.testing import (assert_array_equal, assert_raises, assert_allclose, assert_equal) from numpy.lib import pad +from numpy.lib.arraypad import _as_pairs + + +class TestAsPairs(object): + + def test_single_value(self): + """Test casting for a single value.""" + expected = np.array([[3, 3]] * 10) + for x in (3, [3], [[3]]): + result = _as_pairs(x, 10) + assert_equal(result, expected) + # Test with dtype=object + obj = object() + assert_equal( + _as_pairs(obj, 10), + np.array([[obj, obj]] * 10) + ) + + def test_two_values(self): + """Test proper casting for two different values.""" + # Broadcasting in the first dimension with numbers + expected = np.array([[3, 4]] * 10) + for x in ([3, 4], [[3, 4]]): + result = _as_pairs(x, 10) + assert_equal(result, expected) + # and with dtype=object + obj = object() + assert_equal( + _as_pairs(["a", obj], 10), + np.array([["a", obj]] * 10) + ) + + # Broadcasting in the second / last dimension with numbers + assert_equal( + _as_pairs([[3], [4]], 2), + np.array([[3, 3], [4, 4]]) + ) + # and with dtype=object + assert_equal( + _as_pairs([["a"], [obj]], 2), + np.array([["a", "a"], [obj, obj]]) + ) + + def test_with_none(self): + expected = ((None, None), (None, None), (None, None)) + assert_equal( + _as_pairs(None, 3, as_index=False), + expected + ) + assert_equal( + _as_pairs(None, 3, as_index=True), + expected + ) + + def test_pass_through(self): + """Test if `x` already matching desired output are passed through.""" + expected = np.arange(12).reshape((6, 2)) + assert_equal( + _as_pairs(expected, 6), + expected + ) + + def test_as_index(self): + """Test results if `as_index=True`.""" + assert_equal( + _as_pairs([2.6, 3.3], 10, as_index=True), + np.array([[3, 3]] * 10, dtype=np.intp) + ) + assert_equal( + _as_pairs([2.6, 4.49], 10, as_index=True), + np.array([[3, 4]] * 10, dtype=np.intp) + ) + for x in (-3, [-3], [[-3]], [-3, 4], [3, -4], [[-3, 4]], [[4, -3]], + [[1, 2]] * 9 + [[1, -2]]): + with pytest.raises(ValueError, match="negative values"): + _as_pairs(x, 10, as_index=True) + + def test_exceptions(self): + """Ensure faulty usage is discovered.""" + with pytest.raises(ValueError, match="more dimensions than allowed"): + _as_pairs([[[3]]], 10) + with pytest.raises(ValueError, match="could not be broadcast"): + _as_pairs([[1, 2], [3, 4]], 3) + with pytest.raises(ValueError, match="could not be broadcast"): + _as_pairs(np.ones((2, 3)), 3) class TestConditionalShortcuts(object): @@ -535,6 +620,7 @@ class TestConstant(object): assert_array_equal(arr, expected) + class TestLinearRamp(object): def test_check_simple(self): a = np.arange(100).astype('f') diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index fef06ba53..a17fc66e5 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -4,7 +4,6 @@ from __future__ import division, absolute_import, print_function import numpy as np -import sys from numpy.testing import (assert_array_equal, assert_equal, assert_raises, assert_raises_regex) diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 3185e32ac..077507082 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -287,7 +287,6 @@ from io import BytesIO import numpy as np from numpy.testing import ( assert_, assert_array_equal, assert_raises, assert_raises_regex, - raises ) from numpy.lib import format @@ -524,6 +523,30 @@ def test_compressed_roundtrip(): assert_array_equal(arr, arr1) +# aligned +dt1 = np.dtype('i1, i4, i1', align=True) +# non-aligned, explicit offsets +dt2 = np.dtype({'names': ['a', 'b'], 'formats': ['i4', 'i4'], + 'offsets': [1, 6]}) +# nested struct-in-struct +dt3 = np.dtype({'names': ['c', 'd'], 'formats': ['i4', dt2]}) +# field with '' name +dt4 = np.dtype({'names': ['a', '', 'b'], 'formats': ['i4']*3}) +# titles +dt5 = np.dtype({'names': ['a', 'b'], 'formats': ['i4', 'i4'], + 'offsets': [1, 6], 'titles': ['aa', 'bb']}) + +@pytest.mark.parametrize("dt", [dt1, dt2, dt3, dt4, dt5]) +def test_load_padded_dtype(dt): + arr = np.zeros(3, dt) + for i in range(3): + arr[i] = i + 5 + npz_file = os.path.join(tempdir, 'aligned.npz') + np.savez(npz_file, arr=arr) + arr1 = np.load(npz_file)['arr'] + assert_array_equal(arr, arr1) + + def test_python2_python3_interoperability(): if sys.version_info[0] >= 3: fname = 'win64python2.npy' @@ -533,7 +556,6 @@ def test_python2_python3_interoperability(): data = np.load(path) assert_array_equal(data, np.ones(2)) - def test_pickle_python2_python3(): # Test that loading object arrays saved on Python 2 works both on # Python 2 and Python 3 and vice versa diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 0c789e012..3d4b0e3b2 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -11,17 +11,15 @@ from numpy import ma from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_raises, assert_allclose, - assert_array_max_ulp, assert_warns, assert_raises_regex, suppress_warnings, - HAS_REFCOUNT, + assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT, ) import numpy.lib.function_base as nfb from numpy.random import rand from numpy.lib import ( add_newdoc_ufunc, angle, average, bartlett, blackman, corrcoef, cov, delete, diff, digitize, extract, flipud, gradient, hamming, hanning, - histogram, histogramdd, i0, insert, interp, kaiser, meshgrid, msort, - piecewise, place, rot90, select, setxor1d, sinc, split, trapz, trim_zeros, - unwrap, unique, vectorize + i0, insert, interp, kaiser, meshgrid, msort, piecewise, place, rot90, + select, setxor1d, sinc, trapz, trim_zeros, unwrap, unique, vectorize ) from numpy.compat import long diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py index 1b5a71d0e..c96b01d42 100644 --- a/numpy/lib/tests/test_histograms.py +++ b/numpy/lib/tests/test_histograms.py @@ -6,7 +6,7 @@ from numpy.lib.histograms import histogram, histogramdd, histogram_bin_edges from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_raises, assert_allclose, - assert_array_max_ulp, assert_warns, assert_raises_regex, suppress_warnings, + assert_array_max_ulp, assert_raises_regex, suppress_warnings, ) @@ -289,13 +289,13 @@ class TestHistogram(object): def test_object_array_of_0d(self): # gh-7864 assert_raises(ValueError, - histogram, [np.array([0.4]) for i in range(10)] + [-np.inf]) + histogram, [np.array(0.4) for i in range(10)] + [-np.inf]) assert_raises(ValueError, - histogram, [np.array([0.4]) for i in range(10)] + [np.inf]) + histogram, [np.array(0.4) for i in range(10)] + [np.inf]) # these should not crash - np.histogram([np.array([0.5]) for i in range(10)] + [.500000000000001]) - np.histogram([np.array([0.5]) for i in range(10)] + [.5]) + np.histogram([np.array(0.5) for i in range(10)] + [.500000000000001]) + np.histogram([np.array(0.5) for i in range(10)] + [.5]) def test_some_nan_values(self): # gh-7503 @@ -431,7 +431,7 @@ class TestHistogramOptimBinNums(object): def test_empty(self): estimator_list = ['fd', 'scott', 'rice', 'sturges', - 'doane', 'sqrt', 'auto'] + 'doane', 'sqrt', 'auto', 'stone'] # check it can deal with empty data for estimator in estimator_list: a, b = histogram([], bins=estimator) @@ -447,11 +447,11 @@ class TestHistogramOptimBinNums(object): # Some basic sanity checking, with some fixed data. # Checking for the correct number of bins basic_test = {50: {'fd': 4, 'scott': 4, 'rice': 8, 'sturges': 7, - 'doane': 8, 'sqrt': 8, 'auto': 7}, + 'doane': 8, 'sqrt': 8, 'auto': 7, 'stone': 2}, 500: {'fd': 8, 'scott': 8, 'rice': 16, 'sturges': 10, - 'doane': 12, 'sqrt': 23, 'auto': 10}, + 'doane': 12, 'sqrt': 23, 'auto': 10, 'stone': 9}, 5000: {'fd': 17, 'scott': 17, 'rice': 35, 'sturges': 14, - 'doane': 17, 'sqrt': 71, 'auto': 17}} + 'doane': 17, 'sqrt': 71, 'auto': 17, 'stone': 20}} for testlen, expectedResults in basic_test.items(): # Create some sort of non uniform data to test with @@ -471,11 +471,11 @@ class TestHistogramOptimBinNums(object): precalculated. """ small_dat = {1: {'fd': 1, 'scott': 1, 'rice': 1, 'sturges': 1, - 'doane': 1, 'sqrt': 1}, + 'doane': 1, 'sqrt': 1, 'stone': 1}, 2: {'fd': 2, 'scott': 1, 'rice': 3, 'sturges': 2, - 'doane': 1, 'sqrt': 2}, + 'doane': 1, 'sqrt': 2, 'stone': 1}, 3: {'fd': 2, 'scott': 2, 'rice': 3, 'sturges': 3, - 'doane': 3, 'sqrt': 2}} + 'doane': 3, 'sqrt': 2, 'stone': 1}} for testlen, expectedResults in small_dat.items(): testdat = np.arange(testlen) @@ -499,7 +499,7 @@ class TestHistogramOptimBinNums(object): """ novar_dataset = np.ones(100) novar_resultdict = {'fd': 1, 'scott': 1, 'rice': 1, 'sturges': 1, - 'doane': 1, 'sqrt': 1, 'auto': 1} + 'doane': 1, 'sqrt': 1, 'auto': 1, 'stone': 1} for estimator, numbins in novar_resultdict.items(): a, b = np.histogram(novar_dataset, estimator) @@ -538,12 +538,32 @@ class TestHistogramOptimBinNums(object): xcenter = np.linspace(-10, 10, 50) outlier_dataset = np.hstack((np.linspace(-110, -100, 5), xcenter)) - outlier_resultdict = {'fd': 21, 'scott': 5, 'doane': 11} + outlier_resultdict = {'fd': 21, 'scott': 5, 'doane': 11, 'stone': 6} for estimator, numbins in outlier_resultdict.items(): a, b = np.histogram(outlier_dataset, estimator) assert_equal(len(a), numbins) + def test_scott_vs_stone(self): + """Verify that Scott's rule and Stone's rule converges for normally distributed data""" + + def nbins_ratio(seed, size): + rng = np.random.RandomState(seed) + x = rng.normal(loc=0, scale=2, size=size) + a, b = len(np.histogram(x, 'stone')[0]), len(np.histogram(x, 'scott')[0]) + return a / (a + b) + + ll = [[nbins_ratio(seed, size) for size in np.geomspace(start=10, stop=100, num=4).round().astype(int)] + for seed in range(256)] + + # the average difference between the two methods decreases as the dataset size increases. + assert_almost_equal(abs(np.mean(ll, axis=0) - 0.5), + [0.1065248, + 0.0968844, + 0.0331818, + 0.0178057], + decimal=3) + def test_simple_range(self): """ Straightforward testing with a mixture of linspace data (for @@ -555,11 +575,11 @@ class TestHistogramOptimBinNums(object): # Checking for the correct number of bins basic_test = { 50: {'fd': 8, 'scott': 8, 'rice': 15, - 'sturges': 14, 'auto': 14}, + 'sturges': 14, 'auto': 14, 'stone': 8}, 500: {'fd': 15, 'scott': 16, 'rice': 32, - 'sturges': 20, 'auto': 20}, + 'sturges': 20, 'auto': 20, 'stone': 80}, 5000: {'fd': 33, 'scott': 33, 'rice': 69, - 'sturges': 27, 'auto': 33} + 'sturges': 27, 'auto': 33, 'stone': 80} } for testlen, expectedResults in basic_test.items(): @@ -794,3 +814,20 @@ class TestHistogramdd(object): hist_dd, edges_dd = histogramdd((v,), (bins,), density=True) assert_equal(hist, hist_dd) assert_equal(edges, edges_dd[0]) + + def test_density_via_normed(self): + # normed should simply alias to density argument + v = np.arange(10) + bins = np.array([0, 1, 3, 6, 10]) + hist, edges = histogram(v, bins, density=True) + hist_dd, edges_dd = histogramdd((v,), (bins,), normed=True) + assert_equal(hist, hist_dd) + assert_equal(edges, edges_dd[0]) + + def test_density_normed_redundancy(self): + v = np.arange(10) + bins = np.array([0, 1, 3, 6, 10]) + with assert_raises_regex(TypeError, "Cannot specify both"): + hist_dd, edges_dd = histogramdd((v,), (bins,), + density=True, + normed=True) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index b746937b9..7ef25538b 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -6,7 +6,6 @@ import os import threading import time import warnings -import gc import io import re import pytest @@ -18,7 +17,7 @@ import locale import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConversionWarning -from numpy.compat import asbytes, bytes, unicode, Path +from numpy.compat import asbytes, bytes, Path from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, @@ -355,6 +354,16 @@ class TestSaveTxt(object): c.seek(0) assert_equal(c.readlines(), [b'1 2\n', b'3 4\n']) + @pytest.mark.skipif(Path is None, reason="No pathlib.Path") + def test_multifield_view(self): + a = np.ones(1, dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'f4')]) + v = a[['x', 'z']] + with temppath(suffix='.npy') as path: + path = Path(path) + np.save(path, v) + data = np.load(path) + assert_array_equal(data, v) + def test_delimiter(self): a = np.array([[1., 2.], [3., 4.]]) c = BytesIO() @@ -2049,7 +2058,6 @@ M 33 21.99 def test_utf8_file(self): utf8 = b"\xcf\x96" - latin1 = b"\xf6\xfc\xf6" with temppath() as path: with open(path, "wb") as f: f.write((b"test1,testNonethe" + utf8 + b",test3\n") * 2) diff --git a/numpy/lib/tests/test_mixins.py b/numpy/lib/tests/test_mixins.py index f2d915502..3dd5346b6 100644 --- a/numpy/lib/tests/test_mixins.py +++ b/numpy/lib/tests/test_mixins.py @@ -199,6 +199,17 @@ class TestNDArrayOperatorsMixin(object): err_msg = 'failed for operator {}'.format(op) _assert_equal_type_and_value(expected, actual, err_msg=err_msg) + def test_matmul(self): + array = np.array([1, 2], dtype=np.float64) + array_like = ArrayLike(array) + expected = ArrayLike(np.float64(5)) + _assert_equal_type_and_value(expected, np.matmul(array_like, array)) + if not PY2: + _assert_equal_type_and_value( + expected, operator.matmul(array_like, array)) + _assert_equal_type_and_value( + expected, operator.matmul(array, array_like)) + def test_ufunc_at(self): array = ArrayLike(np.array([1, 2, 3, 4])) assert_(np.negative.at(array, np.array([0, 1])) is None) diff --git a/numpy/lib/tests/test_polynomial.py b/numpy/lib/tests/test_polynomial.py index 9f7c117a2..77414ba7c 100644 --- a/numpy/lib/tests/test_polynomial.py +++ b/numpy/lib/tests/test_polynomial.py @@ -3,7 +3,7 @@ from __future__ import division, absolute_import, print_function import numpy as np from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, - assert_array_almost_equal, assert_raises + assert_array_almost_equal, assert_raises, assert_allclose ) @@ -122,27 +122,34 @@ class TestPolynomial(object): weights = np.arange(8, 1, -1)**2/7.0 # Check exception when too few points for variance estimate. Note that - # the Bayesian estimate requires the number of data points to exceed - # degree + 3. + # the estimate requires the number of data points to exceed + # degree + 1 assert_raises(ValueError, np.polyfit, - [0, 1, 3], [0, 1, 3], deg=0, cov=True) + [1], [1], deg=0, cov=True) # check 1D case m, cov = np.polyfit(x, y+err, 2, cov=True) est = [3.8571, 0.2857, 1.619] assert_almost_equal(est, m, decimal=4) - val0 = [[2.9388, -5.8776, 1.6327], - [-5.8776, 12.7347, -4.2449], - [1.6327, -4.2449, 2.3220]] + val0 = [[ 1.4694, -2.9388, 0.8163], + [-2.9388, 6.3673, -2.1224], + [ 0.8163, -2.1224, 1.161 ]] assert_almost_equal(val0, cov, decimal=4) m2, cov2 = np.polyfit(x, y+err, 2, w=weights, cov=True) assert_almost_equal([4.8927, -1.0177, 1.7768], m2, decimal=4) - val = [[8.7929, -10.0103, 0.9756], - [-10.0103, 13.6134, -1.8178], - [0.9756, -1.8178, 0.6674]] + val = [[ 4.3964, -5.0052, 0.4878], + [-5.0052, 6.8067, -0.9089], + [ 0.4878, -0.9089, 0.3337]] assert_almost_equal(val, cov2, decimal=4) + m3, cov3 = np.polyfit(x, y+err, 2, w=weights, cov="unscaled") + assert_almost_equal([4.8927, -1.0177, 1.7768], m3, decimal=4) + val = [[ 0.1473, -0.1677, 0.0163], + [-0.1677, 0.228 , -0.0304], + [ 0.0163, -0.0304, 0.0112]] + assert_almost_equal(val, cov3, decimal=4) + # check 2D (n,1) case y = y[:, np.newaxis] c = c[:, np.newaxis] @@ -158,6 +165,29 @@ class TestPolynomial(object): assert_almost_equal(val0, cov[:, :, 0], decimal=4) assert_almost_equal(val0, cov[:, :, 1], decimal=4) + # check order 1 (deg=0) case, were the analytic results are simple + np.random.seed(123) + y = np.random.normal(size=(4, 10000)) + mean, cov = np.polyfit(np.zeros(y.shape[0]), y, deg=0, cov=True) + # Should get sigma_mean = sigma/sqrt(N) = 1./sqrt(4) = 0.5. + assert_allclose(mean.std(), 0.5, atol=0.01) + assert_allclose(np.sqrt(cov.mean()), 0.5, atol=0.01) + # Without scaling, since reduced chi2 is 1, the result should be the same. + mean, cov = np.polyfit(np.zeros(y.shape[0]), y, w=np.ones(y.shape[0]), + deg=0, cov="unscaled") + assert_allclose(mean.std(), 0.5, atol=0.01) + assert_almost_equal(np.sqrt(cov.mean()), 0.5) + # If we estimate our errors wrong, no change with scaling: + w = np.full(y.shape[0], 1./0.5) + mean, cov = np.polyfit(np.zeros(y.shape[0]), y, w=w, deg=0, cov=True) + assert_allclose(mean.std(), 0.5, atol=0.01) + assert_allclose(np.sqrt(cov.mean()), 0.5, atol=0.01) + # But if we do not scale, our estimate for the error in the mean will + # differ. + mean, cov = np.polyfit(np.zeros(y.shape[0]), y, w=w, deg=0, cov="unscaled") + assert_allclose(mean.std(), 0.5, atol=0.01) + assert_almost_equal(np.sqrt(cov.mean()), 0.25) + def test_objects(self): from decimal import Decimal p = np.poly1d([Decimal('4.0'), Decimal('3.0'), Decimal('2.0')]) diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 5585a95f9..11f8a5afa 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -10,7 +10,8 @@ from numpy.testing import assert_, assert_raises from numpy.lib.recfunctions import ( drop_fields, rename_fields, get_fieldstructure, recursive_fill_fields, find_duplicates, merge_arrays, append_fields, stack_arrays, join_by, - repack_fields) + repack_fields, unstructured_to_structured, structured_to_unstructured, + apply_along_fields, require_fields, assign_fields_by_name) get_names = np.lib.recfunctions.get_names get_names_flat = np.lib.recfunctions.get_names_flat zip_descr = np.lib.recfunctions.zip_descr @@ -204,6 +205,77 @@ class TestRecFunctions(object): dt = np.dtype((np.record, dt)) assert_(repack_fields(dt).type is np.record) + def test_structured_to_unstructured(self): + a = np.zeros(4, dtype=[('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + out = structured_to_unstructured(a) + assert_equal(out, np.zeros((4,5), dtype='f8')) + + b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + out = np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) + assert_equal(out, np.array([ 3. , 5.5, 9. , 11. ])) + + c = np.arange(20).reshape((4,5)) + out = unstructured_to_structured(c, a.dtype) + want = np.array([( 0, ( 1., 2), [ 3., 4.]), + ( 5, ( 6., 7), [ 8., 9.]), + (10, (11., 12), [13., 14.]), + (15, (16., 17), [18., 19.])], + dtype=[('a', '<i4'), + ('b', [('f0', '<f4'), ('f1', '<u2')]), + ('c', '<f4', (2,))]) + assert_equal(out, want) + + d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + assert_equal(apply_along_fields(np.mean, d), + np.array([ 8.0/3, 16.0/3, 26.0/3, 11. ])) + assert_equal(apply_along_fields(np.mean, d[['x', 'z']]), + np.array([ 3. , 5.5, 9. , 11. ])) + + # check that for uniform field dtypes we get a view, not a copy: + d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'i4')]) + dd = structured_to_unstructured(d) + ddd = unstructured_to_structured(dd, d.dtype) + assert_(dd.base is d) + assert_(ddd.base is d) + + # test that nested fields with identical names don't break anything + point = np.dtype([('x', int), ('y', int)]) + triangle = np.dtype([('a', point), ('b', point), ('c', point)]) + arr = np.zeros(10, triangle) + res = structured_to_unstructured(arr, dtype=int) + assert_equal(res, np.zeros((10, 6), dtype=int)) + + + def test_field_assignment_by_name(self): + a = np.ones(2, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) + newdt = [('b', 'f4'), ('c', 'u1')] + + assert_equal(require_fields(a, newdt), np.ones(2, newdt)) + + b = np.array([(1,2), (3,4)], dtype=newdt) + assign_fields_by_name(a, b, zero_unassigned=False) + assert_equal(a, np.array([(1,1,2),(1,3,4)], dtype=a.dtype)) + assign_fields_by_name(a, b) + assert_equal(a, np.array([(0,1,2),(0,3,4)], dtype=a.dtype)) + + # test nested fields + a = np.ones(2, dtype=[('a', [('b', 'f8'), ('c', 'u1')])]) + newdt = [('a', [('c', 'u1')])] + assert_equal(require_fields(a, newdt), np.ones(2, newdt)) + b = np.array([((2,),), ((3,),)], dtype=newdt) + assign_fields_by_name(a, b, zero_unassigned=False) + assert_equal(a, np.array([((1,2),), ((1,3),)], dtype=a.dtype)) + assign_fields_by_name(a, b) + assert_equal(a, np.array([((0,2),), ((0,3),)], dtype=a.dtype)) + + # test unstructured code path for 0d arrays + a, b = np.array(3), np.array(0) + assign_fields_by_name(b, a) + assert_equal(b[()], 3) + class TestRecursiveFillFields(object): # Test recursive_fill_fields. diff --git a/numpy/lib/tests/test_shape_base.py b/numpy/lib/tests/test_shape_base.py index e338467f9..01ea028bb 100644 --- a/numpy/lib/tests/test_shape_base.py +++ b/numpy/lib/tests/test_shape_base.py @@ -260,8 +260,8 @@ class TestApplyAlongAxis(object): def test_with_iterable_object(self): # from issue 5248 d = np.array([ - [set([1, 11]), set([2, 22]), set([3, 33])], - [set([4, 44]), set([5, 55]), set([6, 66])] + [{1, 11}, {2, 22}, {3, 33}], + [{4, 44}, {5, 55}, {6, 66}] ]) actual = np.apply_along_axis(lambda a: set.union(*a), 0, d) expected = np.array([{1, 11, 4, 44}, {2, 22, 5, 55}, {3, 33, 6, 66}]) diff --git a/numpy/lib/twodim_base.py b/numpy/lib/twodim_base.py index a05e68375..54d0240ef 100644 --- a/numpy/lib/twodim_base.py +++ b/numpy/lib/twodim_base.py @@ -10,6 +10,7 @@ from numpy.core.numeric import ( asarray, where, int8, int16, int32, int64, empty, promote_types, diagonal, nonzero ) +from numpy.core.overrides import set_module from numpy.core import overrides from numpy.core import iinfo, transpose @@ -76,13 +77,13 @@ def fliplr(m): -------- >>> A = np.diag([1.,2.,3.]) >>> A - array([[ 1., 0., 0.], - [ 0., 2., 0.], - [ 0., 0., 3.]]) + array([[1., 0., 0.], + [0., 2., 0.], + [0., 0., 3.]]) >>> np.fliplr(A) - array([[ 0., 0., 1.], - [ 0., 2., 0.], - [ 3., 0., 0.]]) + array([[0., 0., 1.], + [0., 2., 0.], + [3., 0., 0.]]) >>> A = np.random.randn(2,3,5) >>> np.all(np.fliplr(A) == A[:,::-1,...]) @@ -128,13 +129,13 @@ def flipud(m): -------- >>> A = np.diag([1.0, 2, 3]) >>> A - array([[ 1., 0., 0.], - [ 0., 2., 0.], - [ 0., 0., 3.]]) + array([[1., 0., 0.], + [0., 2., 0.], + [0., 0., 3.]]) >>> np.flipud(A) - array([[ 0., 0., 3.], - [ 0., 2., 0.], - [ 1., 0., 0.]]) + array([[0., 0., 3.], + [0., 2., 0.], + [1., 0., 0.]]) >>> A = np.random.randn(2,3,5) >>> np.all(np.flipud(A) == A[::-1,...]) @@ -150,6 +151,7 @@ def flipud(m): return m[::-1, ...] +@set_module('numpy') def eye(N, M=None, k=0, dtype=float, order='C'): """ Return a 2-D array with ones on the diagonal and zeros elsewhere. @@ -189,9 +191,9 @@ def eye(N, M=None, k=0, dtype=float, order='C'): array([[1, 0], [0, 1]]) >>> np.eye(3, k=1) - array([[ 0., 1., 0.], - [ 0., 0., 1.], - [ 0., 0., 0.]]) + array([[0., 1., 0.], + [0., 0., 1.], + [0., 0., 0.]]) """ if M is None: @@ -343,6 +345,7 @@ def diagflat(v, k=0): return wrap(res) +@set_module('numpy') def tri(N, M=None, k=0, dtype=float): """ An array with ones at and below the given diagonal and zeros elsewhere. @@ -375,9 +378,9 @@ def tri(N, M=None, k=0, dtype=float): [1, 1, 1, 1, 1]]) >>> np.tri(3, 5, -1) - array([[ 0., 0., 0., 0., 0.], - [ 1., 0., 0., 0., 0.], - [ 1., 1., 0., 0., 0.]]) + array([[0., 0., 0., 0., 0.], + [1., 0., 0., 0., 0.], + [1., 1., 0., 0., 0.]]) """ if M is None: @@ -537,7 +540,7 @@ def vander(x, N=None, increasing=False): of the differences between the values of the input vector: >>> np.linalg.det(np.vander(x)) - 48.000000000000043 + 48.000000000000043 # may vary >>> (5-3)*(5-2)*(5-1)*(3-2)*(3-1)*(2-1) 48 @@ -641,6 +644,9 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, Examples -------- + >>> import matplotlib + >>> import matplotlib.pyplot + >>> matplotlib.pyplot.switch_backend('agg') >>> import matplotlib as mpl >>> import matplotlib.pyplot as plt @@ -663,6 +669,7 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, >>> ax = fig.add_subplot(131, title='imshow: square bins') >>> plt.imshow(H, interpolation='nearest', origin='low', ... extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]]) + <matplotlib.image.AxesImage object at 0x...> :func:`pcolormesh <matplotlib.pyplot.pcolormesh>` can display actual edges: @@ -670,6 +677,7 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, ... aspect='equal') >>> X, Y = np.meshgrid(xedges, yedges) >>> ax.pcolormesh(X, Y, H) + <matplotlib.collections.QuadMesh object at 0x...> :class:`NonUniformImage <matplotlib.image.NonUniformImage>` can be used to display actual bin edges with interpolation: @@ -698,6 +706,7 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, return hist, edges[0], edges[1] +@set_module('numpy') def mask_indices(n, mask_func, k=0): """ Return the indices to access (n, n) arrays, given a masking function. @@ -768,6 +777,7 @@ def mask_indices(n, mask_func, k=0): return nonzero(a != 0) +@set_module('numpy') def tril_indices(n, k=0, m=None): """ Return the indices for the lower-triangle of an (n, m) array. @@ -824,7 +834,7 @@ def tril_indices(n, k=0, m=None): Both for indexing: >>> a[il1] - array([ 0, 4, 5, 8, 9, 10, 12, 13, 14, 15]) + array([ 0, 4, 5, ..., 13, 14, 15]) And for assigning values: @@ -881,6 +891,7 @@ def tril_indices_from(arr, k=0): return tril_indices(arr.shape[-2], k=k, m=arr.shape[-1]) +@set_module('numpy') def triu_indices(n, k=0, m=None): """ Return the indices for the upper-triangle of an (n, m) array. @@ -938,7 +949,7 @@ def triu_indices(n, k=0, m=None): Both for indexing: >>> a[iu1] - array([ 0, 1, 2, 3, 5, 6, 7, 10, 11, 15]) + array([ 0, 1, 2, ..., 10, 11, 15]) And for assigning values: diff --git a/numpy/lib/type_check.py b/numpy/lib/type_check.py index 9153e1692..f55517732 100644 --- a/numpy/lib/type_check.py +++ b/numpy/lib/type_check.py @@ -11,7 +11,8 @@ __all__ = ['iscomplexobj', 'isrealobj', 'imag', 'iscomplex', 'common_type'] import numpy.core.numeric as _nx -from numpy.core.numeric import asarray, asanyarray, array, isnan, zeros +from numpy.core.numeric import asarray, asanyarray, isnan, zeros +from numpy.core.overrides import set_module from numpy.core import overrides from .ufunclike import isneginf, isposinf @@ -23,7 +24,8 @@ array_function_dispatch = functools.partial( _typecodes_by_elsize = 'GDFgdfQqLlIiHhBb?' -def mintypecode(typechars,typeset='GDFgdf',default='d'): +@set_module('numpy') +def mintypecode(typechars, typeset='GDFgdf', default='d'): """ Return the character for the minimum-size type to which given types can be safely cast. @@ -73,13 +75,16 @@ def mintypecode(typechars,typeset='GDFgdf',default='d'): return default if 'F' in intersection and 'd' in intersection: return 'D' - l = [] - for t in intersection: - i = _typecodes_by_elsize.index(t) - l.append((i, t)) + l = [(_typecodes_by_elsize.index(t), t) for t in intersection] l.sort() return l[0][1] + +def _asfarray_dispatcher(a, dtype=None): + return (a,) + + +@array_function_dispatch(_asfarray_dispatcher) def asfarray(a, dtype=_nx.float_): """ Return an array converted to a float type. @@ -100,11 +105,11 @@ def asfarray(a, dtype=_nx.float_): Examples -------- >>> np.asfarray([2, 3]) - array([ 2., 3.]) + array([2., 3.]) >>> np.asfarray([2, 3], dtype='float') - array([ 2., 3.]) + array([2., 3.]) >>> np.asfarray([2, 3], dtype='int8') - array([ 2., 3.]) + array([2., 3.]) """ if not _nx.issubdtype(dtype, _nx.inexact): @@ -141,13 +146,13 @@ def real(val): -------- >>> a = np.array([1+2j, 3+4j, 5+6j]) >>> a.real - array([ 1., 3., 5.]) + array([1., 3., 5.]) >>> a.real = 9 >>> a - array([ 9.+2.j, 9.+4.j, 9.+6.j]) + array([9.+2.j, 9.+4.j, 9.+6.j]) >>> a.real = np.array([9, 8, 7]) >>> a - array([ 9.+2.j, 8.+4.j, 7.+6.j]) + array([9.+2.j, 8.+4.j, 7.+6.j]) >>> np.real(1 + 1j) 1.0 @@ -187,10 +192,10 @@ def imag(val): -------- >>> a = np.array([1+2j, 3+4j, 5+6j]) >>> a.imag - array([ 2., 4., 6.]) + array([2., 4., 6.]) >>> a.imag = np.array([8, 10, 12]) >>> a - array([ 1. +8.j, 3.+10.j, 5.+12.j]) + array([1. +8.j, 3.+10.j, 5.+12.j]) >>> np.imag(1 + 1j) 1.0 @@ -417,11 +422,13 @@ def nan_to_num(x, copy=True): 0.0 >>> x = np.array([np.inf, -np.inf, np.nan, -128, 128]) >>> np.nan_to_num(x) - array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, - -1.28000000e+002, 1.28000000e+002]) + array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary + -1.28000000e+002, 1.28000000e+002]) >>> y = np.array([complex(np.inf, np.nan), np.nan, complex(np.nan, np.inf)]) + array([ 1.79769313e+308, -1.79769313e+308, 0.00000000e+000, # may vary + -1.28000000e+002, 1.28000000e+002]) >>> np.nan_to_num(y) - array([ 1.79769313e+308 +0.00000000e+000j, + array([ 1.79769313e+308 +0.00000000e+000j, # may vary 0.00000000e+000 +0.00000000e+000j, 0.00000000e+000 +1.79769313e+308j]) """ @@ -485,12 +492,12 @@ def real_if_close(a, tol=100): Examples -------- >>> np.finfo(float).eps - 2.2204460492503131e-16 + 2.2204460492503131e-16 # may vary >>> np.real_if_close([2.1 + 4e-14j], tol=1000) - array([ 2.1]) + array([2.1]) >>> np.real_if_close([2.1 + 4e-13j], tol=1000) - array([ 2.1 +4.00000000e-13j]) + array([2.1+4.e-13j]) """ a = asanyarray(a) @@ -533,7 +540,6 @@ def asscalar(a): -------- >>> np.asscalar(np.array([24])) 24 - """ # 2018-10-10, 1.16 @@ -567,6 +573,7 @@ _namefromtype = {'S1': 'character', 'O': 'object' } +@set_module('numpy') def typename(char): """ Return a description for the given data type code. @@ -666,11 +673,11 @@ def common_type(*arrays): Examples -------- >>> np.common_type(np.arange(2, dtype=np.float32)) - <type 'numpy.float32'> + <class 'numpy.float32'> >>> np.common_type(np.arange(2, dtype=np.float32), np.arange(2)) - <type 'numpy.float64'> + <class 'numpy.float64'> >>> np.common_type(np.arange(4), np.array([45, 6.j]), np.array([45.0])) - <type 'numpy.complex128'> + <class 'numpy.complex128'> """ is_complex = False diff --git a/numpy/lib/ufunclike.py b/numpy/lib/ufunclike.py index ac0af0b37..5c411e8c8 100644 --- a/numpy/lib/ufunclike.py +++ b/numpy/lib/ufunclike.py @@ -8,7 +8,7 @@ from __future__ import division, absolute_import, print_function __all__ = ['fix', 'isneginf', 'isposinf'] import numpy.core.numeric as nx -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import array_function_dispatch, ENABLE_ARRAY_FUNCTION import warnings import functools @@ -55,6 +55,10 @@ def _fix_out_named_y(f): return func +if not ENABLE_ARRAY_FUNCTION: + _fix_out_named_y = _deprecate_out_named_y + + @_deprecate_out_named_y def _dispatcher(x, out=None): return (x, out) @@ -150,11 +154,11 @@ def isposinf(x, out=None): Examples -------- >>> np.isposinf(np.PINF) - array(True, dtype=bool) + True >>> np.isposinf(np.inf) - array(True, dtype=bool) + True >>> np.isposinf(np.NINF) - array(False, dtype=bool) + False >>> np.isposinf([-np.inf, 0., np.inf]) array([False, False, True]) @@ -220,11 +224,11 @@ def isneginf(x, out=None): Examples -------- >>> np.isneginf(np.NINF) - array(True, dtype=bool) + True >>> np.isneginf(np.inf) - array(False, dtype=bool) + False >>> np.isneginf(np.PINF) - array(False, dtype=bool) + False >>> np.isneginf([-np.inf, 0., np.inf]) array([ True, False, False]) diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index 249873654..5a4cae235 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -7,6 +7,7 @@ import re import warnings from numpy.core.numerictypes import issubclass_, issubsctype, issubdtype +from numpy.core.overrides import set_module from numpy.core import ndarray, ufunc, asarray import numpy as np @@ -149,10 +150,8 @@ def deprecate(*args, **kwargs): Warning: >>> olduint = np.deprecate(np.uint) + DeprecationWarning: `uint64` is deprecated! # may vary >>> olduint(6) - /usr/lib/python2.5/site-packages/numpy/lib/utils.py:114: - DeprecationWarning: uint32 is deprecated - warnings.warn(str1, DeprecationWarning, stacklevel=2) 6 """ @@ -164,13 +163,6 @@ def deprecate(*args, **kwargs): fn = args[0] args = args[1:] - # backward compatibility -- can be removed - # after next release - if 'newname' in kwargs: - kwargs['new_name'] = kwargs.pop('newname') - if 'oldname' in kwargs: - kwargs['old_name'] = kwargs.pop('oldname') - return _Deprecate(*args, **kwargs)(fn) else: return _Deprecate(*args, **kwargs) @@ -207,8 +199,8 @@ def byte_bounds(a): >>> low, high = np.byte_bounds(I) >>> high - low == I.size*I.itemsize True - >>> I = np.eye(2, dtype='G'); I.dtype - dtype('complex192') + >>> I = np.eye(2); I.dtype + dtype('float64') >>> low, high = np.byte_bounds(I) >>> high - low == I.size*I.itemsize True @@ -269,17 +261,17 @@ def who(vardict=None): >>> np.who() Name Shape Bytes Type =========================================================== - a 10 40 int32 + a 10 80 int64 b 20 160 float64 - Upper bound on total bytes = 200 + Upper bound on total bytes = 240 >>> d = {'x': np.arange(2.0), 'y': np.arange(3.0), 'txt': 'Some str', ... 'idx':5} >>> np.who(d) Name Shape Bytes Type =========================================================== - y 3 24 float64 x 2 16 float64 + y 3 24 float64 Upper bound on total bytes = 40 """ @@ -439,6 +431,7 @@ def _info(obj, output=sys.stdout): print("type: %s" % obj.dtype, file=output) +@set_module('numpy') def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): """ Get help information for a function, class, or module. @@ -644,6 +637,7 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): print(inspect.getdoc(object), file=output) +@set_module('numpy') def source(object, output=sys.stdout): """ Print or write to a file the source code for a NumPy object. @@ -701,6 +695,8 @@ _lookfor_caches = {} # signature _function_signature_re = re.compile(r"[a-z0-9_]+\(.*[,=].*\)", re.I) + +@set_module('numpy') def lookfor(what, module=None, import_modules=True, regenerate=False, output=None): """ @@ -735,7 +731,7 @@ def lookfor(what, module=None, import_modules=True, regenerate=False, Examples -------- - >>> np.lookfor('binary representation') + >>> np.lookfor('binary representation') # doctest: +SKIP Search results for 'binary representation' ------------------------------------------ numpy.binary_repr @@ -1106,7 +1102,7 @@ def safe_eval(source): >>> np.safe_eval('open("/home/user/.ssh/id_dsa").read()') Traceback (most recent call last): ... - SyntaxError: Unsupported source construct: compiler.ast.CallFunc + ValueError: malformed node or string: <_ast.Call object at 0x...> """ # Local import to speed up numpy's import time. diff --git a/numpy/linalg/linalg.py b/numpy/linalg/linalg.py index 771481e8e..92fa6cb73 100644 --- a/numpy/linalg/linalg.py +++ b/numpy/linalg/linalg.py @@ -29,6 +29,7 @@ from numpy.core import ( swapaxes, divide, count_nonzero, isnan ) from numpy.core.multiarray import normalize_axis_index +from numpy.core.overrides import set_module from numpy.core import overrides from numpy.lib.twodim_base import triu, eye from numpy.linalg import lapack_lite, _umath_linalg @@ -47,7 +48,8 @@ _L = b'L' fortran_int = intc -# Error object + +@set_module('numpy.linalg') class LinAlgError(Exception): """ Generic Python-exception-derived object raised by linalg functions. @@ -75,7 +77,6 @@ class LinAlgError(Exception): numpy.linalg.LinAlgError: Singular matrix """ - pass def _determine_error_states(): @@ -376,7 +377,7 @@ def solve(a, b): >>> b = np.array([9,8]) >>> x = np.linalg.solve(a, b) >>> x - array([ 2., 3.]) + array([2., 3.]) Check that the solution is correct: @@ -534,10 +535,10 @@ def inv(a): >>> a = np.array([[[1., 2.], [3., 4.]], [[1, 3], [3, 5]]]) >>> inv(a) - array([[[-2. , 1. ], - [ 1.5, -0.5]], - [[-5. , 2. ], - [ 3. , -1. ]]]) + array([[[-2. , 1. ], + [ 1.5 , -0.5 ]], + [[-1.25, 0.75], + [ 0.75, -0.25]]]) """ a, wrap = _makearray(a) @@ -729,21 +730,21 @@ def cholesky(a): -------- >>> A = np.array([[1,-2j],[2j,5]]) >>> A - array([[ 1.+0.j, 0.-2.j], + array([[ 1.+0.j, -0.-2.j], [ 0.+2.j, 5.+0.j]]) >>> L = np.linalg.cholesky(A) >>> L - array([[ 1.+0.j, 0.+0.j], - [ 0.+2.j, 1.+0.j]]) + array([[1.+0.j, 0.+0.j], + [0.+2.j, 1.+0.j]]) >>> np.dot(L, L.T.conj()) # verify that L * L.H = A - array([[ 1.+0.j, 0.-2.j], - [ 0.+2.j, 5.+0.j]]) + array([[1.+0.j, 0.-2.j], + [0.+2.j, 5.+0.j]]) >>> A = [[1,-2j],[2j,5]] # what happens if A is only array_like? >>> np.linalg.cholesky(A) # an ndarray object is returned - array([[ 1.+0.j, 0.+0.j], - [ 0.+2.j, 1.+0.j]]) + array([[1.+0.j, 0.+0.j], + [0.+2.j, 1.+0.j]]) >>> # But a matrix object is returned if A is a matrix object - >>> LA.cholesky(np.matrix(A)) + >>> np.linalg.cholesky(np.matrix(A)) matrix([[ 1.+0.j, 0.+0.j], [ 0.+2.j, 1.+0.j]]) @@ -877,9 +878,9 @@ def qr(a, mode='reduced'): [1, 1], [2, 1]]) >>> b = np.array([1, 0, 2, 1]) - >>> q, r = LA.qr(A) + >>> q, r = np.linalg.qr(A) >>> p = np.dot(q.T, b) - >>> np.dot(LA.inv(r), p) + >>> np.dot(np.linalg.inv(r), p) array([ 1.1e-16, 1.0e+00]) """ @@ -1048,7 +1049,7 @@ def eigvals(a): >>> A = np.dot(Q, D) >>> A = np.dot(A, Q.T) >>> LA.eigvals(A) - array([ 1., -1.]) + array([ 1., -1.]) # random """ a, wrap = _makearray(a) @@ -1130,24 +1131,24 @@ def eigvalsh(a, UPLO='L'): >>> from numpy import linalg as LA >>> a = np.array([[1, -2j], [2j, 5]]) >>> LA.eigvalsh(a) - array([ 0.17157288, 5.82842712]) + array([ 0.17157288, 5.82842712]) # may vary >>> # demonstrate the treatment of the imaginary part of the diagonal >>> a = np.array([[5+2j, 9-2j], [0+2j, 2-1j]]) >>> a - array([[ 5.+2.j, 9.-2.j], - [ 0.+2.j, 2.-1.j]]) + array([[5.+2.j, 9.-2.j], + [0.+2.j, 2.-1.j]]) >>> # with UPLO='L' this is numerically equivalent to using LA.eigvals() >>> # with: >>> b = np.array([[5.+0.j, 0.-2.j], [0.+2.j, 2.-0.j]]) >>> b - array([[ 5.+0.j, 0.-2.j], - [ 0.+2.j, 2.+0.j]]) + array([[5.+0.j, 0.-2.j], + [0.+2.j, 2.+0.j]]) >>> wa = LA.eigvalsh(a) >>> wb = LA.eigvals(b) >>> wa; wb - array([ 1., 6.]) - array([ 6.+0.j, 1.+0.j]) + array([1., 6.]) + array([6.+0.j, 1.+0.j]) """ UPLO = UPLO.upper() @@ -1263,19 +1264,19 @@ def eig(a): >>> w, v = LA.eig(np.diag((1, 2, 3))) >>> w; v - array([ 1., 2., 3.]) - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) + array([1., 2., 3.]) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) Real matrix possessing complex e-values and e-vectors; note that the e-values are complex conjugates of each other. >>> w, v = LA.eig(np.array([[1, -1], [1, 1]])) >>> w; v - array([ 1. + 1.j, 1. - 1.j]) - array([[ 0.70710678+0.j , 0.70710678+0.j ], - [ 0.00000000-0.70710678j, 0.00000000+0.70710678j]]) + array([1.+1.j, 1.-1.j]) + array([[0.70710678+0.j , 0.70710678-0.j ], + [0. -0.70710678j, 0. +0.70710678j]]) Complex-valued matrix with real e-values (but complex-valued e-vectors); note that a.conj().T = a, i.e., a is Hermitian. @@ -1283,9 +1284,9 @@ def eig(a): >>> a = np.array([[1, 1j], [-1j, 1]]) >>> w, v = LA.eig(a) >>> w; v - array([ 2.00000000e+00+0.j, 5.98651912e-36+0.j]) # i.e., {2, 0} - array([[ 0.00000000+0.70710678j, 0.70710678+0.j ], - [ 0.70710678+0.j , 0.00000000+0.70710678j]]) + array([2.+0.j, 0.+0.j]) + array([[ 0. +0.70710678j, 0.70710678+0.j ], # may vary + [ 0.70710678+0.j , -0. +0.70710678j]]) Be careful about round-off error! @@ -1293,9 +1294,9 @@ def eig(a): >>> # Theor. e-values are 1 +/- 1e-9 >>> w, v = LA.eig(a) >>> w; v - array([ 1., 1.]) - array([[ 1., 0.], - [ 0., 1.]]) + array([1., 1.]) + array([[1., 0.], + [0., 1.]]) """ a, wrap = _makearray(a) @@ -1391,49 +1392,49 @@ def eigh(a, UPLO='L'): >>> from numpy import linalg as LA >>> a = np.array([[1, -2j], [2j, 5]]) >>> a - array([[ 1.+0.j, 0.-2.j], + array([[ 1.+0.j, -0.-2.j], [ 0.+2.j, 5.+0.j]]) >>> w, v = LA.eigh(a) >>> w; v - array([ 0.17157288, 5.82842712]) - array([[-0.92387953+0.j , -0.38268343+0.j ], - [ 0.00000000+0.38268343j, 0.00000000-0.92387953j]]) + array([0.17157288, 5.82842712]) + array([[-0.92387953+0.j , -0.38268343+0.j ], # may vary + [ 0. +0.38268343j, 0. -0.92387953j]]) >>> np.dot(a, v[:, 0]) - w[0] * v[:, 0] # verify 1st e-val/vec pair - array([2.77555756e-17 + 0.j, 0. + 1.38777878e-16j]) + array([5.55111512e-17+0.0000000e+00j, 0.00000000e+00+1.2490009e-16j]) >>> np.dot(a, v[:, 1]) - w[1] * v[:, 1] # verify 2nd e-val/vec pair - array([ 0.+0.j, 0.+0.j]) + array([0.+0.j, 0.+0.j]) >>> A = np.matrix(a) # what happens if input is a matrix object >>> A - matrix([[ 1.+0.j, 0.-2.j], + matrix([[ 1.+0.j, -0.-2.j], [ 0.+2.j, 5.+0.j]]) >>> w, v = LA.eigh(A) >>> w; v - array([ 0.17157288, 5.82842712]) - matrix([[-0.92387953+0.j , -0.38268343+0.j ], - [ 0.00000000+0.38268343j, 0.00000000-0.92387953j]]) + array([0.17157288, 5.82842712]) + matrix([[-0.92387953+0.j , -0.38268343+0.j ], # may vary + [ 0. +0.38268343j, 0. -0.92387953j]]) >>> # demonstrate the treatment of the imaginary part of the diagonal >>> a = np.array([[5+2j, 9-2j], [0+2j, 2-1j]]) >>> a - array([[ 5.+2.j, 9.-2.j], - [ 0.+2.j, 2.-1.j]]) + array([[5.+2.j, 9.-2.j], + [0.+2.j, 2.-1.j]]) >>> # with UPLO='L' this is numerically equivalent to using LA.eig() with: >>> b = np.array([[5.+0.j, 0.-2.j], [0.+2.j, 2.-0.j]]) >>> b - array([[ 5.+0.j, 0.-2.j], - [ 0.+2.j, 2.+0.j]]) + array([[5.+0.j, 0.-2.j], + [0.+2.j, 2.+0.j]]) >>> wa, va = LA.eigh(a) >>> wb, vb = LA.eig(b) >>> wa; wb - array([ 1., 6.]) - array([ 6.+0.j, 1.+0.j]) + array([1., 6.]) + array([6.+0.j, 1.+0.j]) >>> va; vb - array([[-0.44721360-0.j , -0.89442719+0.j ], - [ 0.00000000+0.89442719j, 0.00000000-0.4472136j ]]) - array([[ 0.89442719+0.j , 0.00000000-0.4472136j], - [ 0.00000000-0.4472136j, 0.89442719+0.j ]]) + array([[-0.4472136 +0.j , -0.89442719+0.j ], # may vary + [ 0. +0.89442719j, 0. -0.4472136j ]]) + array([[ 0.89442719+0.j , -0. +0.4472136j], + [-0. +0.4472136j, 0.89442719+0.j ]]) """ UPLO = UPLO.upper() if UPLO not in ('L', 'U'): @@ -1704,9 +1705,9 @@ def cond(x, p=None): >>> LA.cond(a, 2) 1.4142135623730951 >>> LA.cond(a, -2) - 0.70710678118654746 + 0.70710678118654746 # may vary >>> min(LA.svd(a, compute_uv=0))*min(LA.svd(LA.inv(a), compute_uv=0)) - 0.70710678118654746 + 0.70710678118654746 # may vary """ x = asarray(x) # in case we have a matrix @@ -2001,7 +2002,7 @@ def slogdet(a): >>> a = np.array([[1, 2], [3, 4]]) >>> (sign, logdet) = np.linalg.slogdet(a) >>> (sign, logdet) - (-1, 0.69314718055994529) + (-1, 0.69314718055994529) # may vary >>> sign * np.exp(logdet) -2.0 @@ -2073,7 +2074,7 @@ def det(a): >>> a = np.array([[1, 2], [3, 4]]) >>> np.linalg.det(a) - -2.0 + -2.0 # may vary Computing determinants for a stack of matrices: @@ -2180,15 +2181,15 @@ def lstsq(a, b, rcond="warn"): [ 3., 1.]]) >>> m, c = np.linalg.lstsq(A, y, rcond=None)[0] - >>> print(m, c) - 1.0 -0.95 + >>> m, c + (1.0 -0.95) # may vary Plot the data along with the fitted line: >>> import matplotlib.pyplot as plt - >>> plt.plot(x, y, 'o', label='Original data', markersize=10) - >>> plt.plot(x, m*x + c, 'r', label='Fitted line') - >>> plt.legend() + >>> _ = plt.plot(x, y, 'o', label='Original data', markersize=10) + >>> _ = plt.plot(x, m*x + c, 'r', label='Fitted line') + >>> _ = plt.legend() >>> plt.show() """ @@ -2204,6 +2205,7 @@ def lstsq(a, b, rcond="warn"): raise LinAlgError('Incompatible dimensions') t, result_t = _commonType(a, b) + # FIXME: real_t is unused real_t = _linalgRealType(t) result_real_t = _realType(result_t) @@ -2365,7 +2367,7 @@ def norm(x, ord=None, axis=None, keepdims=False): >>> from numpy import linalg as LA >>> a = np.arange(9) - 4 >>> a - array([-4, -3, -2, -1, 0, 1, 2, 3, 4]) + array([-4, -3, -2, ..., 2, 3, 4]) >>> b = a.reshape((3, 3)) >>> b array([[-4, -3, -2], @@ -2401,13 +2403,13 @@ def norm(x, ord=None, axis=None, keepdims=False): 7.3484692283495345 >>> LA.norm(a, -2) - nan + 0.0 >>> LA.norm(b, -2) - 1.8570331885190563e-016 + 1.8570331885190563e-016 # may vary >>> LA.norm(a, 3) - 5.8480354764257312 + 5.8480354764257312 # may vary >>> LA.norm(a, -3) - nan + 0.0 Using the `axis` argument to compute vector norms: @@ -2582,18 +2584,20 @@ def multi_dot(arrays): >>> from numpy.linalg import multi_dot >>> # Prepare some data - >>> A = np.random.random(10000, 100) - >>> B = np.random.random(100, 1000) - >>> C = np.random.random(1000, 5) - >>> D = np.random.random(5, 333) + >>> A = np.random.random((10000, 100)) + >>> B = np.random.random((100, 1000)) + >>> C = np.random.random((1000, 5)) + >>> D = np.random.random((5, 333)) >>> # the actual dot multiplication - >>> multi_dot([A, B, C, D]) + >>> _ = multi_dot([A, B, C, D]) instead of:: - >>> np.dot(np.dot(np.dot(A, B), C), D) + >>> _ = np.dot(np.dot(np.dot(A, B), C), D) + ... >>> # or - >>> A.dot(B).dot(C).dot(D) + >>> _ = A.dot(B).dot(C).dot(D) + ... Notes ----- diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index 905f3a549..235488c6e 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -13,7 +13,7 @@ import pytest import numpy as np from numpy import array, single, double, csingle, cdouble, dot, identity, matmul -from numpy import multiply, atleast_2d, inf, asarray, matrix +from numpy import multiply, atleast_2d, inf, asarray from numpy import linalg from numpy.linalg import matrix_power, norm, matrix_rank, multi_dot, LinAlgError from numpy.linalg.linalg import _multi_dot_matrix_chain_order @@ -955,7 +955,6 @@ class TestMatrixPower(object): dtnoinv = [object, np.dtype('e'), np.dtype('g'), np.dtype('G')] def test_large_power(self, dt): - power = matrix_power rshft = self.rshft_1.astype(dt) assert_equal( matrix_power(rshft, 2**100 + 2**10 + 2**5 + 0), self.rshft_0) @@ -1619,8 +1618,6 @@ class TestQR(object): def test_qr_empty(self, m, n): k = min(m, n) a = np.empty((m, n)) - a_type = type(a) - a_dtype = a.dtype self.check_qr(a) diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 9ee44e9ff..63a61599c 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -449,17 +449,15 @@ def _check_fill_value(fill_value, ndtype): """ ndtype = np.dtype(ndtype) - fields = ndtype.fields if fill_value is None: fill_value = default_fill_value(ndtype) - elif fields: - fdtype = [(_[0], _[1]) for _ in ndtype.descr] + elif ndtype.names is not None: if isinstance(fill_value, (ndarray, np.void)): try: - fill_value = np.array(fill_value, copy=False, dtype=fdtype) + fill_value = np.array(fill_value, copy=False, dtype=ndtype) except ValueError: err_msg = "Unable to transform %s to dtype %s" - raise ValueError(err_msg % (fill_value, fdtype)) + raise ValueError(err_msg % (fill_value, ndtype)) else: fill_value = np.asarray(fill_value, dtype=object) fill_value = np.array(_recursive_set_fill_value(fill_value, ndtype), @@ -518,18 +516,18 @@ def set_fill_value(a, fill_value): array([0, 1, 2, 3, 4]) >>> a = ma.masked_where(a < 3, a) >>> a - masked_array(data = [-- -- -- 3 4], - mask = [ True True True False False], - fill_value=999999) + masked_array(data=[--, --, --, 3, 4], + mask=[ True, True, True, False, False], + fill_value=999999) >>> ma.set_fill_value(a, -999) >>> a - masked_array(data = [-- -- -- 3 4], - mask = [ True True True False False], - fill_value=-999) + masked_array(data=[--, --, --, 3, 4], + mask=[ True, True, True, False, False], + fill_value=-999) Nothing happens if `a` is not a masked array. - >>> a = range(5) + >>> a = list(range(5)) >>> a [0, 1, 2, 3, 4] >>> ma.set_fill_value(a, 100) @@ -691,13 +689,12 @@ def getdata(a, subok=True): >>> import numpy.ma as ma >>> a = ma.masked_equal([[1,2],[3,4]], 2) >>> a - masked_array(data = - [[1 --] - [3 4]], - mask = - [[False True] - [False False]], - fill_value=999999) + masked_array( + data=[[1, --], + [3, 4]], + mask=[[False, True], + [False, False]], + fill_value=2) >>> ma.getdata(a) array([[1, 2], [3, 4]]) @@ -754,20 +751,19 @@ def fix_invalid(a, mask=nomask, copy=True, fill_value=None): -------- >>> x = np.ma.array([1., -1, np.nan, np.inf], mask=[1] + [0]*3) >>> x - masked_array(data = [-- -1.0 nan inf], - mask = [ True False False False], - fill_value = 1e+20) + masked_array(data=[--, -1.0, nan, inf], + mask=[ True, False, False, False], + fill_value=1e+20) >>> np.ma.fix_invalid(x) - masked_array(data = [-- -1.0 -- --], - mask = [ True False True True], - fill_value = 1e+20) + masked_array(data=[--, -1.0, --, --], + mask=[ True, False, True, True], + fill_value=1e+20) >>> fixed = np.ma.fix_invalid(x) >>> fixed.data - array([ 1.00000000e+00, -1.00000000e+00, 1.00000000e+20, - 1.00000000e+20]) + array([ 1.e+00, -1.e+00, 1.e+20, 1.e+20]) >>> x.data - array([ 1., -1., NaN, Inf]) + array([ 1., -1., nan, inf]) """ a = masked_array(a, copy=copy, mask=mask, subok=True) @@ -780,6 +776,10 @@ def fix_invalid(a, mask=nomask, copy=True, fill_value=None): a._data[invalid] = fill_value return a +def is_string_or_list_of_strings(val): + return (isinstance(val, basestring) or + (isinstance(val, list) and val and + builtins.all(isinstance(s, basestring) for s in val))) ############################################################################### # Ufuncs # @@ -1344,9 +1344,9 @@ def make_mask_descr(ndtype): -------- >>> import numpy.ma as ma >>> dtype = np.dtype({'names':['foo', 'bar'], - 'formats':[np.float32, int]}) + ... 'formats':[np.float32, np.int64]}) >>> dtype - dtype([('foo', '<f4'), ('bar', '<i4')]) + dtype([('foo', '<f4'), ('bar', '<i8')]) >>> ma.make_mask_descr(dtype) dtype([('foo', '|b1'), ('bar', '|b1')]) >>> ma.make_mask_descr(np.float32) @@ -1379,13 +1379,12 @@ def getmask(a): >>> import numpy.ma as ma >>> a = ma.masked_equal([[1,2],[3,4]], 2) >>> a - masked_array(data = - [[1 --] - [3 4]], - mask = - [[False True] - [False False]], - fill_value=999999) + masked_array( + data=[[1, --], + [3, 4]], + mask=[[False, True], + [False, False]], + fill_value=2) >>> ma.getmask(a) array([[False, True], [False, False]]) @@ -1400,12 +1399,11 @@ def getmask(a): >>> b = ma.masked_array([[1,2],[3,4]]) >>> b - masked_array(data = - [[1 2] - [3 4]], - mask = - False, - fill_value=999999) + masked_array( + data=[[1, 2], + [3, 4]], + mask=False, + fill_value=999999) >>> ma.nomask False >>> ma.getmask(b) == ma.nomask @@ -1443,13 +1441,12 @@ def getmaskarray(arr): >>> import numpy.ma as ma >>> a = ma.masked_equal([[1,2],[3,4]], 2) >>> a - masked_array(data = - [[1 --] - [3 4]], - mask = - [[False True] - [False False]], - fill_value=999999) + masked_array( + data=[[1, --], + [3, 4]], + mask=[[False, True], + [False, False]], + fill_value=2) >>> ma.getmaskarray(a) array([[False, True], [False, False]]) @@ -1458,13 +1455,12 @@ def getmaskarray(arr): >>> b = ma.masked_array([[1,2],[3,4]]) >>> b - masked_array(data = - [[1 2] - [3 4]], - mask = - False, - fill_value=999999) - >>> >ma.getmaskarray(b) + masked_array( + data=[[1, 2], + [3, 4]], + mask=False, + fill_value=999999) + >>> ma.getmaskarray(b) array([[False, False], [False, False]]) @@ -1502,9 +1498,9 @@ def is_mask(m): >>> import numpy.ma as ma >>> m = ma.masked_equal([0, 1, 0, 2, 3], 0) >>> m - masked_array(data = [-- 1 -- 2 3], - mask = [ True False True False False], - fill_value=999999) + masked_array(data=[--, 1, --, 2, 3], + mask=[ True, False, True, False, False], + fill_value=0) >>> ma.is_mask(m) False >>> ma.is_mask(m.mask) @@ -1525,14 +1521,14 @@ def is_mask(m): Arrays with complex dtypes don't return True. >>> dtype = np.dtype({'names':['monty', 'pithon'], - 'formats':[bool, bool]}) + ... 'formats':[bool, bool]}) >>> dtype dtype([('monty', '|b1'), ('pithon', '|b1')]) >>> m = np.array([(True, False), (False, True), (True, False)], - dtype=dtype) + ... dtype=dtype) >>> m - array([(True, False), (False, True), (True, False)], - dtype=[('monty', '|b1'), ('pithon', '|b1')]) + array([( True, False), (False, True), ( True, False)], + dtype=[('monty', '?'), ('pithon', '?')]) >>> ma.is_mask(m) False @@ -1598,7 +1594,7 @@ def make_mask(m, copy=False, shrink=True, dtype=MaskType): >>> m = np.zeros(4) >>> m - array([ 0., 0., 0., 0.]) + array([0., 0., 0., 0.]) >>> ma.make_mask(m) False >>> ma.make_mask(m, shrink=False) @@ -1614,11 +1610,11 @@ def make_mask(m, copy=False, shrink=True, dtype=MaskType): >>> arr [(1, 0), (0, 1), (1, 0), (1, 0)] >>> dtype = np.dtype({'names':['man', 'mouse'], - 'formats':[int, int]}) + ... 'formats':[np.int64, np.int64]}) >>> arr = np.array(arr, dtype=dtype) >>> arr array([(1, 0), (0, 1), (1, 0), (1, 0)], - dtype=[('man', '<i4'), ('mouse', '<i4')]) + dtype=[('man', '<i8'), ('mouse', '<i8')]) >>> ma.make_mask(arr, dtype=dtype) array([(True, False), (False, True), (True, False), (True, False)], dtype=[('man', '|b1'), ('mouse', '|b1')]) @@ -1677,9 +1673,9 @@ def make_mask_none(newshape, dtype=None): Defining a more complex dtype. >>> dtype = np.dtype({'names':['foo', 'bar'], - 'formats':[np.float32, int]}) + ... 'formats':[np.float32, np.int64]}) >>> dtype - dtype([('foo', '<f4'), ('bar', '<i4')]) + dtype([('foo', '<f4'), ('bar', '<i8')]) >>> ma.make_mask_none((3,), dtype=dtype) array([(False, False), (False, False), (False, False)], dtype=[('foo', '|b1'), ('bar', '|b1')]) @@ -1777,16 +1773,16 @@ def flatten_mask(mask): Examples -------- >>> mask = np.array([0, 0, 1]) - >>> flatten_mask(mask) + >>> np.ma.flatten_mask(mask) array([False, False, True]) >>> mask = np.array([(0, 0), (0, 1)], dtype=[('a', bool), ('b', bool)]) - >>> flatten_mask(mask) + >>> np.ma.flatten_mask(mask) array([False, False, False, True]) >>> mdtype = [('a', bool), ('b', [('ba', bool), ('bb', bool)])] >>> mask = np.array([(0, (0, 0)), (0, (0, 1))], dtype=mdtype) - >>> flatten_mask(mask) + >>> np.ma.flatten_mask(mask) array([False, False, False, False, False, True]) """ @@ -1871,38 +1867,39 @@ def masked_where(condition, a, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_where(a <= 2, a) - masked_array(data = [-- -- -- 3], - mask = [ True True True False], - fill_value=999999) + masked_array(data=[--, --, --, 3], + mask=[ True, True, True, False], + fill_value=999999) Mask array `b` conditional on `a`. >>> b = ['a', 'b', 'c', 'd'] >>> ma.masked_where(a == 2, b) - masked_array(data = [a b -- d], - mask = [False False True False], - fill_value=N/A) + masked_array(data=['a', 'b', --, 'd'], + mask=[False, False, True, False], + fill_value='N/A', + dtype='<U1') Effect of the `copy` argument. >>> c = ma.masked_where(a <= 2, a) >>> c - masked_array(data = [-- -- -- 3], - mask = [ True True True False], - fill_value=999999) + masked_array(data=[--, --, --, 3], + mask=[ True, True, True, False], + fill_value=999999) >>> c[0] = 99 >>> c - masked_array(data = [99 -- -- 3], - mask = [False True True False], - fill_value=999999) + masked_array(data=[99, --, --, 3], + mask=[False, True, True, False], + fill_value=999999) >>> a array([0, 1, 2, 3]) >>> c = ma.masked_where(a <= 2, a, copy=False) >>> c[0] = 99 >>> c - masked_array(data = [99 -- -- 3], - mask = [False True True False], - fill_value=999999) + masked_array(data=[99, --, --, 3], + mask=[False, True, True, False], + fill_value=999999) >>> a array([99, 1, 2, 3]) @@ -1911,19 +1908,19 @@ def masked_where(condition, a, copy=True): >>> a = np.arange(4) >>> a = ma.masked_where(a == 2, a) >>> a - masked_array(data = [0 1 -- 3], - mask = [False False True False], - fill_value=999999) + masked_array(data=[0, 1, --, 3], + mask=[False, False, True, False], + fill_value=999999) >>> b = np.arange(4) >>> b = ma.masked_where(b == 0, b) >>> b - masked_array(data = [-- 1 2 3], - mask = [ True False False False], - fill_value=999999) + masked_array(data=[--, 1, 2, 3], + mask=[ True, False, False, False], + fill_value=999999) >>> ma.masked_where(a == 3, b) - masked_array(data = [-- 1 -- --], - mask = [ True False True True], - fill_value=999999) + masked_array(data=[--, 1, --, --], + mask=[ True, False, True, True], + fill_value=999999) """ # Make sure that condition is a valid standard-type mask. @@ -1963,9 +1960,9 @@ def masked_greater(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_greater(a, 2) - masked_array(data = [0 1 2 --], - mask = [False False False True], - fill_value=999999) + masked_array(data=[0, 1, 2, --], + mask=[False, False, False, True], + fill_value=999999) """ return masked_where(greater(x, value), x, copy=copy) @@ -1989,9 +1986,9 @@ def masked_greater_equal(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_greater_equal(a, 2) - masked_array(data = [0 1 -- --], - mask = [False False True True], - fill_value=999999) + masked_array(data=[0, 1, --, --], + mask=[False, False, True, True], + fill_value=999999) """ return masked_where(greater_equal(x, value), x, copy=copy) @@ -2015,9 +2012,9 @@ def masked_less(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_less(a, 2) - masked_array(data = [-- -- 2 3], - mask = [ True True False False], - fill_value=999999) + masked_array(data=[--, --, 2, 3], + mask=[ True, True, False, False], + fill_value=999999) """ return masked_where(less(x, value), x, copy=copy) @@ -2041,9 +2038,9 @@ def masked_less_equal(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_less_equal(a, 2) - masked_array(data = [-- -- -- 3], - mask = [ True True True False], - fill_value=999999) + masked_array(data=[--, --, --, 3], + mask=[ True, True, True, False], + fill_value=999999) """ return masked_where(less_equal(x, value), x, copy=copy) @@ -2067,9 +2064,9 @@ def masked_not_equal(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_not_equal(a, 2) - masked_array(data = [-- -- 2 --], - mask = [ True True False True], - fill_value=999999) + masked_array(data=[--, --, 2, --], + mask=[ True, True, False, True], + fill_value=999999) """ return masked_where(not_equal(x, value), x, copy=copy) @@ -2095,9 +2092,9 @@ def masked_equal(x, value, copy=True): >>> a array([0, 1, 2, 3]) >>> ma.masked_equal(a, 2) - masked_array(data = [0 1 -- 3], - mask = [False False True False], - fill_value=999999) + masked_array(data=[0, 1, --, 3], + mask=[False, False, True, False], + fill_value=2) """ output = masked_where(equal(x, value), x, copy=copy) @@ -2126,16 +2123,16 @@ def masked_inside(x, v1, v2, copy=True): >>> import numpy.ma as ma >>> x = [0.31, 1.2, 0.01, 0.2, -0.4, -1.1] >>> ma.masked_inside(x, -0.3, 0.3) - masked_array(data = [0.31 1.2 -- -- -0.4 -1.1], - mask = [False False True True False False], - fill_value=1e+20) + masked_array(data=[0.31, 1.2, --, --, -0.4, -1.1], + mask=[False, False, True, True, False, False], + fill_value=1e+20) The order of `v1` and `v2` doesn't matter. >>> ma.masked_inside(x, 0.3, -0.3) - masked_array(data = [0.31 1.2 -- -- -0.4 -1.1], - mask = [False False True True False False], - fill_value=1e+20) + masked_array(data=[0.31, 1.2, --, --, -0.4, -1.1], + mask=[False, False, True, True, False, False], + fill_value=1e+20) """ if v2 < v1: @@ -2166,16 +2163,16 @@ def masked_outside(x, v1, v2, copy=True): >>> import numpy.ma as ma >>> x = [0.31, 1.2, 0.01, 0.2, -0.4, -1.1] >>> ma.masked_outside(x, -0.3, 0.3) - masked_array(data = [-- -- 0.01 0.2 -- --], - mask = [ True True False False True True], - fill_value=1e+20) + masked_array(data=[--, --, 0.01, 0.2, --, --], + mask=[ True, True, False, False, True, True], + fill_value=1e+20) The order of `v1` and `v2` doesn't matter. >>> ma.masked_outside(x, 0.3, -0.3) - masked_array(data = [-- -- 0.01 0.2 -- --], - mask = [ True True False False True True], - fill_value=1e+20) + masked_array(data=[--, --, 0.01, 0.2, --, --], + mask=[ True, True, False, False, True, True], + fill_value=1e+20) """ if v2 < v1: @@ -2220,20 +2217,27 @@ def masked_object(x, value, copy=True, shrink=True): >>> food = np.array(['green_eggs', 'ham'], dtype=object) >>> # don't eat spoiled food >>> eat = ma.masked_object(food, 'green_eggs') - >>> print(eat) - [-- ham] + >>> eat + masked_array(data=[--, 'ham'], + mask=[ True, False], + fill_value='green_eggs', + dtype=object) >>> # plain ol` ham is boring >>> fresh_food = np.array(['cheese', 'ham', 'pineapple'], dtype=object) >>> eat = ma.masked_object(fresh_food, 'green_eggs') - >>> print(eat) - [cheese ham pineapple] + >>> eat + masked_array(data=['cheese', 'ham', 'pineapple'], + mask=False, + fill_value='green_eggs', + dtype=object) Note that `mask` is set to ``nomask`` if possible. >>> eat - masked_array(data = [cheese ham pineapple], - mask = False, - fill_value=?) + masked_array(data=['cheese', 'ham', 'pineapple'], + mask=False, + fill_value='green_eggs', + dtype=object) """ if isMaskedArray(x): @@ -2288,16 +2292,16 @@ def masked_values(x, value, rtol=1e-5, atol=1e-8, copy=True, shrink=True): >>> import numpy.ma as ma >>> x = np.array([1, 1.1, 2, 1.1, 3]) >>> ma.masked_values(x, 1.1) - masked_array(data = [1.0 -- 2.0 -- 3.0], - mask = [False True False True False], - fill_value=1.1) + masked_array(data=[1.0, --, 2.0, --, 3.0], + mask=[False, True, False, True, False], + fill_value=1.1) Note that `mask` is set to ``nomask`` if possible. >>> ma.masked_values(x, 1.5) - masked_array(data = [ 1. 1.1 2. 1.1 3. ], - mask = False, - fill_value=1.5) + masked_array(data=[1. , 1.1, 2. , 1.1, 3. ], + mask=False, + fill_value=1.5) For integers, the fill value will be different in general to the result of ``masked_equal``. @@ -2306,13 +2310,13 @@ def masked_values(x, value, rtol=1e-5, atol=1e-8, copy=True, shrink=True): >>> x array([0, 1, 2, 3, 4]) >>> ma.masked_values(x, 2) - masked_array(data = [0 1 -- 3 4], - mask = [False False True False False], - fill_value=2) + masked_array(data=[0, 1, --, 3, 4], + mask=[False, False, True, False, False], + fill_value=2) >>> ma.masked_equal(x, 2) - masked_array(data = [0 1 -- 3 4], - mask = [False False True False False], - fill_value=999999) + masked_array(data=[0, 1, --, 3, 4], + mask=[False, False, True, False, False], + fill_value=2) """ xnew = filled(x, value) @@ -2346,11 +2350,11 @@ def masked_invalid(a, copy=True): >>> a[2] = np.NaN >>> a[3] = np.PINF >>> a - array([ 0., 1., NaN, Inf, 4.]) + array([ 0., 1., nan, inf, 4.]) >>> ma.masked_invalid(a) - masked_array(data = [0.0 1.0 -- -- 4.0], - mask = [False False True True False], - fill_value=1e+20) + masked_array(data=[0.0, 1.0, --, --, 4.0], + mask=[False, False, True, True, False], + fill_value=1e+20) """ a = np.array(a, copy=copy, subok=True) @@ -2511,7 +2515,7 @@ def flatten_structured_array(a): -------- >>> ndtype = [('a', int), ('b', float)] >>> a = np.array([(1, 1), (2, 2)], dtype=ndtype) - >>> flatten_structured_array(a) + >>> np.ma.flatten_structured_array(a) array([[1., 1.], [2., 2.]]) @@ -2682,9 +2686,7 @@ class MaskedIterator(object): >>> fl.next() 3 >>> fl.next() - masked_array(data = --, - mask = True, - fill_value = 1e+20) + masked >>> fl.next() Traceback (most recent call last): File "<stdin>", line 1, in <module> @@ -3245,7 +3247,7 @@ class MaskedArray(ndarray): # Inherit attributes from self dout._update_from(self) # Check the fill_value - if isinstance(indx, basestring): + if is_string_or_list_of_strings(indx): if self._fill_value is not None: dout._fill_value = self._fill_value[indx] @@ -3549,6 +3551,11 @@ class MaskedArray(ndarray): array([[False, False], [False, False]]) >>> x.shrink_mask() + masked_array( + data=[[1, 2], + [3, 4]], + mask=False, + fill_value=999999) >>> x.mask False @@ -3637,7 +3644,7 @@ class MaskedArray(ndarray): -inf >>> x.set_fill_value(np.pi) >>> x.fill_value - 3.1415926535897931 + 3.1415926535897931 # may vary Reset to default: @@ -3686,9 +3693,9 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999) >>> x.filled() - array([1, 2, -999, 4, -999]) + array([ 1, 2, -999, 4, -999]) >>> type(x.filled()) - <type 'numpy.ndarray'> + <class 'numpy.ndarray'> Subclassing is preserved. This means that if, e.g., the data part of the masked array is a recarray, `filled` returns a recarray: @@ -3753,7 +3760,7 @@ class MaskedArray(ndarray): >>> x.compressed() array([0, 1]) >>> type(x.compressed()) - <type 'numpy.ndarray'> + <class 'numpy.ndarray'> """ data = ndarray.ravel(self._data) @@ -3795,25 +3802,29 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) >>> x.compress([1, 0, 1]) - masked_array(data = [1 3], - mask = [False False], - fill_value=999999) + masked_array(data=[1, 3], + mask=[False, False], + fill_value=999999) >>> x.compress([1, 0, 1], axis=1) - masked_array(data = - [[1 3] - [-- --] - [7 9]], - mask = - [[False False] - [ True True] - [False False]], - fill_value=999999) + masked_array( + data=[[1, 3], + [--, --], + [7, 9]], + mask=[[False, False], + [ True, True], + [False, False]], + fill_value=999999) """ # Get the basic components @@ -4346,9 +4357,9 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array([1+1.j, -2j, 3.45+1.6j], mask=[False, True, False]) >>> x.get_imag() - masked_array(data = [1.0 -- 1.6], - mask = [False True False], - fill_value = 1e+20) + masked_array(data=[1.0, --, 1.6], + mask=[False, True, False], + fill_value=1e+20) """ result = self._data.imag.view(type(self)) @@ -4381,9 +4392,9 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array([1+1.j, -2j, 3.45+1.6j], mask=[False, True, False]) >>> x.get_real() - masked_array(data = [1.0 -- 3.45], - mask = [False True False], - fill_value = 1e+20) + masked_array(data=[1.0, --, 3.45], + mask=[False, True, False], + fill_value=1e+20) """ result = self._data.real.view(type(self)) @@ -4429,13 +4440,12 @@ class MaskedArray(ndarray): >>> a = ma.arange(6).reshape((2, 3)) >>> a[1, :] = ma.masked >>> a - masked_array(data = - [[0 1 2] - [-- -- --]], - mask = - [[False False False] - [ True True True]], - fill_value = 999999) + masked_array( + data=[[0, 1, 2], + [--, --, --]], + mask=[[False, False, False], + [ True, True, True]], + fill_value=999999) >>> a.count() 3 @@ -4520,12 +4530,20 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] - >>> print(x.ravel()) - [1 -- 3 -- 5 -- 7 -- 9] + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) + >>> x.ravel() + masked_array(data=[1, --, 3, --, 5, --, 7, --, 9], + mask=[False, True, False, True, False, True, False, True, + False], + fill_value=999999) """ r = ndarray.ravel(self._data, order=order).view(type(self)) @@ -4574,15 +4592,25 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2],[3,4]], mask=[1,0,0,1]) - >>> print(x) - [[-- 2] - [3 --]] + >>> x + masked_array( + data=[[--, 2], + [3, --]], + mask=[[ True, False], + [False, True]], + fill_value=999999) >>> x = x.reshape((4,1)) - >>> print(x) - [[--] - [2] - [3] - [--]] + >>> x + masked_array( + data=[[--], + [2], + [3], + [--]], + mask=[[ True], + [False], + [False], + [ True]], + fill_value=999999) """ kwargs.update(order=kwargs.get('order', 'C')) @@ -4639,21 +4667,36 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) >>> x.put([0,4,8],[10,20,30]) - >>> print(x) - [[10 -- 3] - [-- 20 --] - [7 -- 30]] + >>> x + masked_array( + data=[[10, --, 3], + [--, 20, --], + [7, --, 30]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) >>> x.put(4,999) - >>> print(x) - [[10 -- 3] - [-- 999 --] - [7 -- 30]] + >>> x + masked_array( + data=[[10, --, 3], + [--, 999, --], + [7, --, 30]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) """ # Hard mask: Get rid of the values/indices that fall on masked data @@ -4693,14 +4736,14 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array([1, 2, 3], mask=[0, 1, 1]) >>> x.ids() - (166670640, 166659832) + (166670640, 166659832) # may vary If the array has no mask, the address of `nomask` is returned. This address is typically not close to the data in memory: >>> x = np.ma.array([1, 2, 3]) >>> x.ids() - (166691080, 3083169284L) + (166691080, 3083169284L) # may vary """ if self._mask is nomask: @@ -4849,13 +4892,12 @@ class MaskedArray(ndarray): >>> import numpy.ma as ma >>> x = ma.array(np.eye(3)) >>> x - masked_array(data = - [[ 1. 0. 0.] - [ 0. 1. 0.] - [ 0. 0. 1.]], - mask = - False, - fill_value=1e+20) + masked_array( + data=[[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]], + mask=False, + fill_value=1e+20) >>> x.nonzero() (array([0, 1, 2]), array([0, 1, 2])) @@ -4863,15 +4905,14 @@ class MaskedArray(ndarray): >>> x[1, 1] = ma.masked >>> x - masked_array(data = - [[1.0 0.0 0.0] - [0.0 -- 0.0] - [0.0 0.0 1.0]], - mask = - [[False False False] - [False True False] - [False False False]], - fill_value=1e+20) + masked_array( + data=[[1.0, 0.0, 0.0], + [0.0, --, 0.0], + [0.0, 0.0, 1.0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1e+20) >>> x.nonzero() (array([0, 2]), array([0, 2])) @@ -4888,13 +4929,12 @@ class MaskedArray(ndarray): >>> a = ma.array([[1,2,3],[4,5,6],[7,8,9]]) >>> a > 3 - masked_array(data = - [[False False False] - [ True True True] - [ True True True]], - mask = - False, - fill_value=999999) + masked_array( + data=[[False, False, False], + [ True, True, True], + [ True, True, True]], + mask=False, + fill_value=True) >>> ma.nonzero(a > 3) (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) @@ -4976,18 +5016,27 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] - >>> print(x.sum()) + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) + >>> x.sum() 25 - >>> print(x.sum(axis=1)) - [4 5 16] - >>> print(x.sum(axis=0)) - [8 5 12] + >>> x.sum(axis=1) + masked_array(data=[4, 5, 16], + mask=[False, False, False], + fill_value=999999) + >>> x.sum(axis=0) + masked_array(data=[8, 5, 12], + mask=[False, False, False], + fill_value=999999) >>> print(type(x.sum(axis=0, dtype=np.int64)[0])) - <type 'numpy.int64'> + <class 'numpy.int64'> """ kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims} @@ -5038,8 +5087,11 @@ class MaskedArray(ndarray): Examples -------- >>> marr = np.ma.array(np.arange(10), mask=[0,0,0,1,1,1,0,0,0,0]) - >>> print(marr.cumsum()) - [0 1 3 -- -- -- 9 16 24 33] + >>> marr.cumsum() + masked_array(data=[0, 1, 3, --, --, --, 9, 16, 24, 33], + mask=[False, False, False, True, True, True, False, False, + False, False], + fill_value=999999) """ result = self.filled(0).cumsum(axis=axis, dtype=dtype, out=out) @@ -5143,9 +5195,9 @@ class MaskedArray(ndarray): -------- >>> a = np.ma.array([1,2,3], mask=[False, False, True]) >>> a - masked_array(data = [1 2 --], - mask = [False False True], - fill_value = 999999) + masked_array(data=[1, 2, --], + mask=[False, False, True], + fill_value=999999) >>> a.mean() 1.5 @@ -5198,9 +5250,9 @@ class MaskedArray(ndarray): -------- >>> a = np.ma.array([1,2,3]) >>> a.anom() - masked_array(data = [-1. 0. 1.], - mask = False, - fill_value = 1e+20) + masked_array(data=[-1., 0., 1.], + mask=False, + fill_value=1e+20) """ m = self.mean(axis, dtype) @@ -5380,9 +5432,9 @@ class MaskedArray(ndarray): -------- >>> a = np.ma.array([3,2,1], mask=[False, False, True]) >>> a - masked_array(data = [3 2 --], - mask = [False False True], - fill_value = 999999) + masked_array(data=[3, 2, --], + mask=[False, False, True], + fill_value=999999) >>> a.argsort() array([1, 0, 2]) @@ -5430,15 +5482,19 @@ class MaskedArray(ndarray): Examples -------- - >>> x = np.ma.array(arange(4), mask=[1,1,0,0]) + >>> x = np.ma.array(np.arange(4), mask=[1,1,0,0]) >>> x.shape = (2,2) - >>> print(x) - [[-- --] - [2 3]] - >>> print(x.argmin(axis=0, fill_value=-1)) - [0 0] - >>> print(x.argmin(axis=0, fill_value=9)) - [1 1] + >>> x + masked_array( + data=[[--, --], + [2, 3]], + mask=[[ True, True], + [False, False]], + fill_value=999999) + >>> x.argmin(axis=0, fill_value=-1) + array([0, 0]) + >>> x.argmin(axis=0, fill_value=9) + array([1, 1]) """ if fill_value is None: @@ -5529,23 +5585,29 @@ class MaskedArray(ndarray): Examples -------- - >>> a = ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) + >>> a = np.ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) >>> # Default >>> a.sort() - >>> print(a) - [1 3 5 -- --] + >>> a + masked_array(data=[1, 3, 5, --, --], + mask=[False, False, False, True, True], + fill_value=999999) - >>> a = ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) + >>> a = np.ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) >>> # Put missing values in the front >>> a.sort(endwith=False) - >>> print(a) - [-- -- 1 3 5] + >>> a + masked_array(data=[--, --, 1, 3, 5], + mask=[ True, True, False, False, False], + fill_value=999999) - >>> a = ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) + >>> a = np.ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0]) >>> # fill_value takes over endwith >>> a.sort(endwith=False, fill_value=3) - >>> print(a) - [1 -- -- 3 5] + >>> a + masked_array(data=[1, --, --, 3, 5], + mask=[False, True, True, False, False], + fill_value=999999) """ if self._mask is nomask: @@ -5651,27 +5713,36 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array(np.arange(6), mask=[0 ,1, 0, 0, 0 ,1]).reshape(3, 2) - >>> print(x) - [[0 --] - [2 3] - [4 --]] + >>> x + masked_array( + data=[[0, --], + [2, 3], + [4, --]], + mask=[[False, True], + [False, False], + [False, True]], + fill_value=999999) >>> x.mini() - 0 + masked_array(data=0, + mask=False, + fill_value=999999) >>> x.mini(axis=0) - masked_array(data = [0 3], - mask = [False False], - fill_value = 999999) - >>> print(x.mini(axis=1)) - [0 2 4] + masked_array(data=[0, 3], + mask=[False, False], + fill_value=999999) + >>> x.mini(axis=1) + masked_array(data=[0, 2, 4], + mask=[False, False, False], + fill_value=999999) There is a small difference between `mini` and `min`: >>> x[:,1].mini(axis=0) - masked_array(data = --, - mask = True, - fill_value = 999999) + masked_array(data=3, + mask=False, + fill_value=999999) >>> x[:,1].min(axis=0) - masked + 3 """ # 2016-04-13, 1.13.0, gh-8764 @@ -5924,7 +5995,7 @@ class MaskedArray(ndarray): -------- >>> x = np.ma.array(np.array([[1, 2], [3, 4]]), mask=[[0, 1], [1, 0]]) >>> x.tobytes() - '\\x01\\x00\\x00\\x00?B\\x0f\\x00?B\\x0f\\x00\\x04\\x00\\x00\\x00' + b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00?B\\x0f\\x00\\x00\\x00\\x00\\x00?B\\x0f\\x00\\x00\\x00\\x00\\x00\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00' """ return self.filled(fill_value).tobytes(order=order) @@ -5972,14 +6043,20 @@ class MaskedArray(ndarray): Examples -------- >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4) - >>> print(x) - [[1 -- 3] - [-- 5 --] - [7 -- 9]] - >>> print(x.toflex()) - [[(1, False) (2, True) (3, False)] - [(4, True) (5, False) (6, True)] - [(7, False) (8, True) (9, False)]] + >>> x + masked_array( + data=[[1, --, 3], + [--, 5, --], + [7, --, 9]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) + >>> x.toflex() + array([[(1, False), (2, True), (3, False)], + [(4, True), (5, False), (6, True)], + [(7, False), (8, True), (9, False)]], + dtype=[('_data', '<i8'), ('_mask', '?')]) """ # Get the basic dtype. @@ -6226,15 +6303,14 @@ def isMaskedArray(x): [ 0., 0., 1.]]) >>> m = ma.masked_values(a, 0) >>> m - masked_array(data = - [[1.0 -- --] - [-- 1.0 --] - [-- -- 1.0]], - mask = - [[False True True] - [ True False True] - [ True True False]], - fill_value=0.0) + masked_array( + data=[[1.0, --, --], + [--, 1.0, --], + [--, --, 1.0]], + mask=[[False, True, True], + [ True, False, True], + [ True, True, False]], + fill_value=0.0) >>> ma.isMaskedArray(a) False >>> ma.isMaskedArray(m) @@ -6338,7 +6414,7 @@ class MaskedConstant(MaskedArray): def __copy__(self): return self - + def __deepcopy__(self, memo): return self @@ -6398,16 +6474,16 @@ def is_masked(x): >>> import numpy.ma as ma >>> x = ma.masked_equal([0, 1, 0, 2, 3], 0) >>> x - masked_array(data = [-- 1 -- 2 3], - mask = [ True False True False False], - fill_value=999999) + masked_array(data=[--, 1, --, 2, 3], + mask=[ True, False, True, False, False], + fill_value=0) >>> ma.is_masked(x) True >>> x = ma.masked_equal([0, 1, 0, 2, 3], 42) >>> x - masked_array(data = [0 1 0 2 3], - mask = False, - fill_value=999999) + masked_array(data=[0, 1, 0, 2, 3], + mask=False, + fill_value=42) >>> ma.is_masked(x) False @@ -6757,17 +6833,17 @@ def concatenate(arrays, axis=0): >>> a[1] = ma.masked >>> b = ma.arange(2, 5) >>> a - masked_array(data = [0 -- 2], - mask = [False True False], - fill_value = 999999) + masked_array(data=[0, --, 2], + mask=[False, True, False], + fill_value=999999) >>> b - masked_array(data = [2 3 4], - mask = False, - fill_value = 999999) + masked_array(data=[2, 3, 4], + mask=False, + fill_value=999999) >>> ma.concatenate([a, b]) - masked_array(data = [0 -- 2 2 3 4], - mask = [False True False False False False], - fill_value = 999999) + masked_array(data=[0, --, 2, 2, 3, 4], + mask=[False, True, False, False, False, False], + fill_value=999999) """ d = np.concatenate([getdata(a) for a in arrays], axis) @@ -6922,24 +6998,21 @@ def transpose(a, axes=None): >>> import numpy.ma as ma >>> x = ma.arange(4).reshape((2,2)) >>> x[1, 1] = ma.masked - >>>> x - masked_array(data = - [[0 1] - [2 --]], - mask = - [[False False] - [False True]], - fill_value = 999999) + >>> x + masked_array( + data=[[0, 1], + [2, --]], + mask=[[False, False], + [False, True]], + fill_value=999999) >>> ma.transpose(x) - masked_array(data = - [[0 2] - [1 --]], - mask = - [[False False] - [False True]], - fill_value = 999999) - + masked_array( + data=[[0, 2], + [1, --]], + mask=[[False, False], + [False, True]], + fill_value=999999) """ # We can't use 'frommethod', as 'transpose' doesn't take keywords try: @@ -6986,39 +7059,39 @@ def resize(x, new_shape): >>> a = ma.array([[1, 2] ,[3, 4]]) >>> a[0, 1] = ma.masked >>> a - masked_array(data = - [[1 --] - [3 4]], - mask = - [[False True] - [False False]], - fill_value = 999999) + masked_array( + data=[[1, --], + [3, 4]], + mask=[[False, True], + [False, False]], + fill_value=999999) >>> np.resize(a, (3, 3)) - array([[1, 2, 3], - [4, 1, 2], - [3, 4, 1]]) + masked_array( + data=[[1, 2, 3], + [4, 1, 2], + [3, 4, 1]], + mask=False, + fill_value=999999) >>> ma.resize(a, (3, 3)) - masked_array(data = - [[1 -- 3] - [4 1 --] - [3 4 1]], - mask = - [[False True False] - [False False True] - [False False False]], - fill_value = 999999) + masked_array( + data=[[1, --, 3], + [4, 1, --], + [3, 4, 1]], + mask=[[False, True, False], + [False, False, True], + [False, False, False]], + fill_value=999999) A MaskedArray is always returned, regardless of the input type. >>> a = np.array([[1, 2] ,[3, 4]]) >>> ma.resize(a, (3, 3)) - masked_array(data = - [[1 2 3] - [4 1 2] - [3 4 1]], - mask = - False, - fill_value = 999999) + masked_array( + data=[[1, 2, 3], + [4, 1, 2], + [3, 4, 1]], + mask=False, + fill_value=999999) """ # We can't use _frommethods here, as N.resize is notoriously whiny. @@ -7087,7 +7160,7 @@ def where(condition, x=_NoValue, y=_NoValue): Parameters ---------- condition : array_like, bool - Where True, yield `x`, otherwise yield `y`. + Where True, yield `x`, otherwise yield `y`. x, y : array_like, optional Values from which to choose. `x`, `y` and `condition` need to be broadcastable to some shape. @@ -7109,14 +7182,24 @@ def where(condition, x=_NoValue, y=_NoValue): >>> x = np.ma.array(np.arange(9.).reshape(3, 3), mask=[[0, 1, 0], ... [1, 0, 1], ... [0, 1, 0]]) - >>> print(x) - [[0.0 -- 2.0] - [-- 4.0 --] - [6.0 -- 8.0]] - >>> print(np.ma.where(x > 5, x, -3.1416)) - [[-3.1416 -- -3.1416] - [-- -3.1416 --] - [6.0 -- 8.0]] + >>> x + masked_array( + data=[[0.0, --, 2.0], + [--, 4.0, --], + [6.0, --, 8.0]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=1e+20) + >>> np.ma.where(x > 5, x, -3.1416) + masked_array( + data=[[-3.1416, --, -3.1416], + [--, -3.1416, --], + [6.0, --, 8.0]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=1e+20) """ @@ -7196,9 +7279,9 @@ def choose(indices, choices, out=None, mode='raise'): >>> choice = np.array([[1,1,1], [2,2,2], [3,3,3]]) >>> a = np.array([2, 1, 0]) >>> np.ma.choose(a, choice) - masked_array(data = [3 2 1], - mask = False, - fill_value=999999) + masked_array(data=[3, 2, 1], + mask=False, + fill_value=999999) """ def fmask(x): @@ -7321,25 +7404,23 @@ def mask_rowcols(a, axis=None): [0, 0, 0]]) >>> a = ma.masked_equal(a, 1) >>> a - masked_array(data = - [[0 0 0] - [0 -- 0] - [0 0 0]], - mask = - [[False False False] - [False True False] - [False False False]], - fill_value=999999) + masked_array( + data=[[0, 0, 0], + [0, --, 0], + [0, 0, 0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1) >>> ma.mask_rowcols(a) - masked_array(data = - [[0 -- 0] - [-- -- --] - [0 -- 0]], - mask = - [[False True False] - [ True True True] - [False True False]], - fill_value=999999) + masked_array( + data=[[0, --, 0], + [--, --, --], + [0, --, 0]], + mask=[[False, True, False], + [ True, True, True], + [False, True, False]], + fill_value=1) """ a = array(a, subok=False) @@ -7400,24 +7481,22 @@ def dot(a, b, strict=False, out=None): Examples -------- - >>> a = ma.array([[1, 2, 3], [4, 5, 6]], mask=[[1, 0, 0], [0, 0, 0]]) - >>> b = ma.array([[1, 2], [3, 4], [5, 6]], mask=[[1, 0], [0, 0], [0, 0]]) + >>> a = np.ma.array([[1, 2, 3], [4, 5, 6]], mask=[[1, 0, 0], [0, 0, 0]]) + >>> b = np.ma.array([[1, 2], [3, 4], [5, 6]], mask=[[1, 0], [0, 0], [0, 0]]) >>> np.ma.dot(a, b) - masked_array(data = - [[21 26] - [45 64]], - mask = - [[False False] - [False False]], - fill_value = 999999) + masked_array( + data=[[21, 26], + [45, 64]], + mask=[[False, False], + [False, False]], + fill_value=999999) >>> np.ma.dot(a, b, strict=True) - masked_array(data = - [[-- --] - [-- 64]], - mask = - [[ True True] - [ True False]], - fill_value = 999999) + masked_array( + data=[[--, --], + [--, 64]], + mask=[[ True, True], + [ True, False]], + fill_value=999999) """ # !!!: Works only with 2D arrays. There should be a way to get it to run @@ -7585,18 +7664,18 @@ def allequal(a, b, fill_value=True): Examples -------- - >>> a = ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1]) + >>> a = np.ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1]) >>> a - masked_array(data = [10000000000.0 1e-07 --], - mask = [False False True], - fill_value=1e+20) + masked_array(data=[10000000000.0, 1e-07, --], + mask=[False, False, True], + fill_value=1e+20) - >>> b = array([1e10, 1e-7, -42.0]) + >>> b = np.array([1e10, 1e-7, -42.0]) >>> b array([ 1.00000000e+10, 1.00000000e-07, -4.20000000e+01]) - >>> ma.allequal(a, b, fill_value=False) + >>> np.ma.allequal(a, b, fill_value=False) False - >>> ma.allequal(a, b) + >>> np.ma.allequal(a, b) True """ @@ -7662,29 +7741,29 @@ def allclose(a, b, masked_equal=True, rtol=1e-5, atol=1e-8): Examples -------- - >>> a = ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1]) + >>> a = np.ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1]) >>> a - masked_array(data = [10000000000.0 1e-07 --], - mask = [False False True], - fill_value = 1e+20) - >>> b = ma.array([1e10, 1e-8, -42.0], mask=[0, 0, 1]) - >>> ma.allclose(a, b) + masked_array(data=[10000000000.0, 1e-07, --], + mask=[False, False, True], + fill_value=1e+20) + >>> b = np.ma.array([1e10, 1e-8, -42.0], mask=[0, 0, 1]) + >>> np.ma.allclose(a, b) False - >>> a = ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1]) - >>> b = ma.array([1.00001e10, 1e-9, -42.0], mask=[0, 0, 1]) - >>> ma.allclose(a, b) + >>> a = np.ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1]) + >>> b = np.ma.array([1.00001e10, 1e-9, -42.0], mask=[0, 0, 1]) + >>> np.ma.allclose(a, b) True - >>> ma.allclose(a, b, masked_equal=False) + >>> np.ma.allclose(a, b, masked_equal=False) False Masked values are not compared directly. - >>> a = ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1]) - >>> b = ma.array([1.00001e10, 1e-9, 42.0], mask=[0, 0, 1]) - >>> ma.allclose(a, b) + >>> a = np.ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1]) + >>> b = np.ma.array([1.00001e10, 1e-9, 42.0], mask=[0, 0, 1]) + >>> np.ma.allclose(a, b) True - >>> ma.allclose(a, b, masked_equal=False) + >>> np.ma.allclose(a, b, masked_equal=False) False """ @@ -7751,15 +7830,14 @@ def asarray(a, dtype=None, order=None): -------- >>> x = np.arange(10.).reshape(2, 5) >>> x - array([[ 0., 1., 2., 3., 4.], - [ 5., 6., 7., 8., 9.]]) + array([[0., 1., 2., 3., 4.], + [5., 6., 7., 8., 9.]]) >>> np.ma.asarray(x) - masked_array(data = - [[ 0. 1. 2. 3. 4.] - [ 5. 6. 7. 8. 9.]], - mask = - False, - fill_value = 1e+20) + masked_array( + data=[[0., 1., 2., 3., 4.], + [5., 6., 7., 8., 9.]], + mask=False, + fill_value=1e+20) >>> type(np.ma.asarray(x)) <class 'numpy.ma.core.MaskedArray'> @@ -7799,15 +7877,14 @@ def asanyarray(a, dtype=None): -------- >>> x = np.arange(10.).reshape(2, 5) >>> x - array([[ 0., 1., 2., 3., 4.], - [ 5., 6., 7., 8., 9.]]) + array([[0., 1., 2., 3., 4.], + [5., 6., 7., 8., 9.]]) >>> np.ma.asanyarray(x) - masked_array(data = - [[ 0. 1. 2. 3. 4.] - [ 5. 6. 7. 8. 9.]], - mask = - False, - fill_value = 1e+20) + masked_array( + data=[[0., 1., 2., 3., 4.], + [5., 6., 7., 8., 9.]], + mask=False, + fill_value=1e+20) >>> type(np.ma.asanyarray(x)) <class 'numpy.ma.core.MaskedArray'> @@ -7951,39 +8028,38 @@ def fromflex(fxarray): >>> x = np.ma.array(np.arange(9).reshape(3, 3), mask=[0] + [1, 0] * 4) >>> rec = x.toflex() >>> rec - array([[(0, False), (1, True), (2, False)], - [(3, True), (4, False), (5, True)], - [(6, False), (7, True), (8, False)]], - dtype=[('_data', '<i4'), ('_mask', '|b1')]) + array([[(0, False), (1, True), (2, False)], + [(3, True), (4, False), (5, True)], + [(6, False), (7, True), (8, False)]], + dtype=[('_data', '<i8'), ('_mask', '?')]) >>> x2 = np.ma.fromflex(rec) >>> x2 - masked_array(data = - [[0 -- 2] - [-- 4 --] - [6 -- 8]], - mask = - [[False True False] - [ True False True] - [False True False]], - fill_value = 999999) + masked_array( + data=[[0, --, 2], + [--, 4, --], + [6, --, 8]], + mask=[[False, True, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) Extra fields can be present in the structured array but are discarded: >>> dt = [('_data', '<i4'), ('_mask', '|b1'), ('field3', '<f4')] >>> rec2 = np.zeros((2, 2), dtype=dt) >>> rec2 - array([[(0, False, 0.0), (0, False, 0.0)], - [(0, False, 0.0), (0, False, 0.0)]], - dtype=[('_data', '<i4'), ('_mask', '|b1'), ('field3', '<f4')]) + array([[(0, False, 0.), (0, False, 0.)], + [(0, False, 0.), (0, False, 0.)]], + dtype=[('_data', '<i4'), ('_mask', '?'), ('field3', '<f4')]) >>> y = np.ma.fromflex(rec2) >>> y - masked_array(data = - [[0 0] - [0 0]], - mask = - [[False False] - [False False]], - fill_value = 999999) + masked_array( + data=[[0, 0], + [0, 0]], + mask=[[False, False], + [False, False]], + fill_value=999999, + dtype=int32) """ return masked_array(fxarray['_data'], mask=fxarray['_mask']) @@ -8084,7 +8160,10 @@ def append(a, b, axis=None): >>> import numpy.ma as ma >>> a = ma.masked_values([1, 2, 3], 2) >>> b = ma.masked_values([[4, 5, 6], [7, 8, 9]], 7) - >>> print(ma.append(a, b)) - [1 -- 3 4 5 6 -- 8 9] + >>> ma.append(a, b) + masked_array(data=[1, --, 3, 4, 5, 6, --, 8, 9], + mask=[False, True, False, False, False, False, True, False, + False], + fill_value=999999) """ return concatenate([a, b], axis) diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py index 3be4d3625..2e3b84e1c 100644 --- a/numpy/ma/extras.py +++ b/numpy/ma/extras.py @@ -81,15 +81,14 @@ def count_masked(arr, axis=None): >>> a[1, 2] = ma.masked >>> a[2, 1] = ma.masked >>> a - masked_array(data = - [[0 1 2] - [-- 4 --] - [6 -- 8]], - mask = - [[False False False] - [ True False True] - [False True False]], - fill_value=999999) + masked_array( + data=[[0, 1, 2], + [--, 4, --], + [6, --, 8]], + mask=[[False, False, False], + [ True, False, True], + [False, True, False]], + fill_value=999999) >>> ma.count_masked(a) 3 @@ -132,15 +131,15 @@ def masked_all(shape, dtype=float): -------- >>> import numpy.ma as ma >>> ma.masked_all((3, 3)) - masked_array(data = - [[-- -- --] - [-- -- --] - [-- -- --]], - mask = - [[ True True True] - [ True True True] - [ True True True]], - fill_value=1e+20) + masked_array( + data=[[--, --, --], + [--, --, --], + [--, --, --]], + mask=[[ True, True, True], + [ True, True, True], + [ True, True, True]], + fill_value=1e+20, + dtype=float64) The `dtype` parameter defines the underlying data type. @@ -188,16 +187,16 @@ def masked_all_like(arr): >>> import numpy.ma as ma >>> arr = np.zeros((2, 3), dtype=np.float32) >>> arr - array([[ 0., 0., 0.], - [ 0., 0., 0.]], dtype=float32) + array([[0., 0., 0.], + [0., 0., 0.]], dtype=float32) >>> ma.masked_all_like(arr) - masked_array(data = - [[-- -- --] - [-- -- --]], - mask = - [[ True True True] - [ True True True]], - fill_value=1e+20) + masked_array( + data=[[--, --, --], + [--, --, --]], + mask=[[ True, True, True], + [ True, True, True]], + fill_value=1e+20, + dtype=float32) The dtype of the masked array matches the dtype of `arr`. @@ -492,28 +491,45 @@ if apply_over_axes.__doc__ is not None: Examples -------- - >>> a = ma.arange(24).reshape(2,3,4) - >>> a[:,0,1] = ma.masked - >>> a[:,1,:] = ma.masked - >>> print(a) - [[[0 -- 2 3] - [-- -- -- --] - [8 9 10 11]] - - [[12 -- 14 15] - [-- -- -- --] - [20 21 22 23]]] - >>> print(ma.apply_over_axes(ma.sum, a, [0,2])) - [[[46] - [--] - [124]]] + >>> a = np.ma.arange(24).reshape(2,3,4) + >>> a[:,0,1] = np.ma.masked + >>> a[:,1,:] = np.ma.masked + >>> a + masked_array( + data=[[[0, --, 2, 3], + [--, --, --, --], + [8, 9, 10, 11]], + [[12, --, 14, 15], + [--, --, --, --], + [20, 21, 22, 23]]], + mask=[[[False, True, False, False], + [ True, True, True, True], + [False, False, False, False]], + [[False, True, False, False], + [ True, True, True, True], + [False, False, False, False]]], + fill_value=999999) + >>> np.ma.apply_over_axes(np.ma.sum, a, [0,2]) + masked_array( + data=[[[46], + [--], + [124]]], + mask=[[[False], + [ True], + [False]]], + fill_value=999999) Tuple axis arguments to ufuncs are equivalent: - >>> print(ma.sum(a, axis=(0,2)).reshape((1,-1,1))) - [[[46] - [--] - [124]]] + >>> np.ma.sum(a, axis=(0,2)).reshape((1,-1,1)) + masked_array( + data=[[[46], + [--], + [124]]], + mask=[[[False], + [ True], + [False]]], + fill_value=999999) """ @@ -558,14 +574,19 @@ def average(a, axis=None, weights=None, returned=False): 1.25 >>> x = np.ma.arange(6.).reshape(3, 2) - >>> print(x) - [[ 0. 1.] - [ 2. 3.] - [ 4. 5.]] + >>> x + masked_array( + data=[[0., 1.], + [2., 3.], + [4., 5.]], + mask=False, + fill_value=1e+20) >>> avg, sumweights = np.ma.average(x, axis=0, weights=[1, 2, 3], ... returned=True) - >>> print(avg) - [2.66666666667 3.66666666667] + >>> avg + masked_array(data=[2.6666666666666665, 3.6666666666666665], + mask=[False, False], + fill_value=1e+20) """ a = asarray(a) @@ -676,9 +697,9 @@ def median(a, axis=None, out=None, overwrite_input=False, keepdims=False): >>> np.ma.median(x) 2.5 >>> np.ma.median(x, axis=-1, overwrite_input=True) - masked_array(data = [ 2. 5.], - mask = False, - fill_value = 1e+20) + masked_array(data=[2.0, 5.0], + mask=[False, False], + fill_value=1e+20) """ if not hasattr(a, 'mask'): @@ -856,15 +877,14 @@ def compress_rowcols(x, axis=None): ... [1, 0, 0], ... [0, 0, 0]]) >>> x - masked_array(data = - [[-- 1 2] - [-- 4 5] - [6 7 8]], - mask = - [[ True False False] - [ True False False] - [False False False]], - fill_value = 999999) + masked_array( + data=[[--, 1, 2], + [--, 4, 5], + [6, 7, 8]], + mask=[[ True, False, False], + [ True, False, False], + [False, False, False]], + fill_value=999999) >>> np.ma.compress_rowcols(x) array([[7, 8]]) @@ -937,25 +957,24 @@ def mask_rows(a, axis=None): [0, 0, 0]]) >>> a = ma.masked_equal(a, 1) >>> a - masked_array(data = - [[0 0 0] - [0 -- 0] - [0 0 0]], - mask = - [[False False False] - [False True False] - [False False False]], - fill_value=999999) + masked_array( + data=[[0, 0, 0], + [0, --, 0], + [0, 0, 0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1) + >>> ma.mask_rows(a) - masked_array(data = - [[0 0 0] - [-- -- --] - [0 0 0]], - mask = - [[False False False] - [ True True True] - [False False False]], - fill_value=999999) + masked_array( + data=[[0, 0, 0], + [--, --, --], + [0, 0, 0]], + mask=[[False, False, False], + [ True, True, True], + [False, False, False]], + fill_value=1) """ return mask_rowcols(a, 0) @@ -982,25 +1001,23 @@ def mask_cols(a, axis=None): [0, 0, 0]]) >>> a = ma.masked_equal(a, 1) >>> a - masked_array(data = - [[0 0 0] - [0 -- 0] - [0 0 0]], - mask = - [[False False False] - [False True False] - [False False False]], - fill_value=999999) + masked_array( + data=[[0, 0, 0], + [0, --, 0], + [0, 0, 0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1) >>> ma.mask_cols(a) - masked_array(data = - [[0 -- 0] - [0 -- 0] - [0 -- 0]], - mask = - [[False True False] - [False True False] - [False True False]], - fill_value=999999) + masked_array( + data=[[0, --, 0], + [0, --, 0], + [0, --, 0]], + mask=[[False, True, False], + [False, True, False], + [False, True, False]], + fill_value=1) """ return mask_rowcols(a, 1) @@ -1078,12 +1095,12 @@ def intersect1d(ar1, ar2, assume_unique=False): Examples -------- - >>> x = array([1, 3, 3, 3], mask=[0, 0, 0, 1]) - >>> y = array([3, 1, 1, 1], mask=[0, 0, 0, 1]) - >>> intersect1d(x, y) - masked_array(data = [1 3 --], - mask = [False False True], - fill_value = 999999) + >>> x = np.ma.array([1, 3, 3, 3], mask=[0, 0, 0, 1]) + >>> y = np.ma.array([3, 1, 1, 1], mask=[0, 0, 0, 1]) + >>> np.ma.intersect1d(x, y) + masked_array(data=[1, 3, --], + mask=[False, False, True], + fill_value=999999) """ if assume_unique: @@ -1216,9 +1233,9 @@ def setdiff1d(ar1, ar2, assume_unique=False): -------- >>> x = np.ma.array([1, 2, 3, 4], mask=[0, 1, 0, 1]) >>> np.ma.setdiff1d(x, [1, 2]) - masked_array(data = [3 --], - mask = [False True], - fill_value = 999999) + masked_array(data=[3, --], + mask=[False, True], + fill_value=999999) """ if assume_unique: @@ -1483,7 +1500,9 @@ class mr_class(MAxisConcatenator): Examples -------- >>> np.ma.mr_[np.ma.array([1,2,3]), 0, 0, np.ma.array([4,5,6])] - array([1, 2, 3, 0, 0, 4, 5, 6]) + masked_array(data=[1, 2, 3, ..., 4, 5, 6], + mask=False, + fill_value=999999) """ def __init__(self): @@ -1524,19 +1543,19 @@ def flatnotmasked_edges(a): Examples -------- >>> a = np.ma.arange(10) - >>> flatnotmasked_edges(a) - [0,-1] + >>> np.ma.flatnotmasked_edges(a) + array([0, 9]) >>> mask = (a < 3) | (a > 8) | (a == 5) >>> a[mask] = np.ma.masked >>> np.array(a[~a.mask]) array([3, 4, 6, 7, 8]) - >>> flatnotmasked_edges(a) + >>> np.ma.flatnotmasked_edges(a) array([3, 8]) >>> a[:] = np.ma.masked - >>> print(flatnotmasked_edges(ma)) + >>> print(np.ma.flatnotmasked_edges(a)) None """ @@ -1588,7 +1607,7 @@ def notmasked_edges(a, axis=None): >>> np.array(am[~am.mask]) array([0, 1, 2, 3, 6]) - >>> np.ma.notmasked_edges(ma) + >>> np.ma.notmasked_edges(am) array([0, 6]) """ @@ -1709,15 +1728,11 @@ def notmasked_contiguous(a, axis=None): [slice(0, 1, None), slice(2, 4, None), slice(7, 9, None), slice(11, 12, None)] >>> np.ma.notmasked_contiguous(ma, axis=0) - [[slice(0, 1, None), slice(2, 3, None)], # column broken into two segments - [], # fully masked column - [slice(0, 1, None)], - [slice(0, 3, None)]] + [[slice(0, 1, None), slice(2, 3, None)], [], [slice(0, 1, None)], [slice(0, 3, None)]] >>> np.ma.notmasked_contiguous(ma, axis=1) - [[slice(0, 1, None), slice(2, 4, None)], # row broken into two segments - [slice(3, 4, None)], - [slice(0, 1, None), slice(3, 4, None)]] + [[slice(0, 1, None), slice(2, 4, None)], [slice(3, 4, None)], [slice(0, 1, None), slice(3, 4, None)]] + """ a = asarray(a) nd = a.ndim diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 8a015e609..e0dbf1b1a 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -26,7 +26,7 @@ from numpy.testing import ( assert_raises, assert_warns, suppress_warnings ) from numpy import ndarray -from numpy.compat import asbytes, asbytes_nested +from numpy.compat import asbytes from numpy.ma.testutils import ( assert_, assert_array_equal, assert_equal, assert_almost_equal, assert_equal_records, fail_if_equal, assert_not_equal, @@ -233,7 +233,7 @@ class TestMaskedArray(object): x = np.array([('A', 0)], dtype={'names':['f0','f1'], 'formats':['S4','i8'], 'offsets':[0,8]}) - data = array(x) # used to fail due to 'V' padding field in x.dtype.descr + array(x) # used to fail due to 'V' padding field in x.dtype.descr def test_asarray(self): (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d @@ -2029,6 +2029,17 @@ class TestFillingValues(object): assert_equal(x.fill_value, 999.) assert_equal(x._fill_value, np.array(999.)) + def test_subarray_fillvalue(self): + # gh-10483 test multi-field index fill value + fields = array([(1, 1, 1)], + dtype=[('i', int), ('s', '|S8'), ('f', float)]) + with suppress_warnings() as sup: + sup.filter(FutureWarning, "Numpy has detected") + subfields = fields[['i', 'f']] + assert_equal(tuple(subfields.fill_value), (999999, 1.e+20)) + # test comparison does not raise: + subfields[1:] == subfields[:-1] + def test_fillvalue_exotic_dtype(self): # Tests yet more exotic flexible dtypes _check_fill_value = np.ma.core._check_fill_value @@ -2390,9 +2401,9 @@ class TestMaskedArrayInPlaceArithmetics(object): assert_equal(xm, y + 1) (x, _, xm) = self.floatdata - id1 = x.data.ctypes._data + id1 = x.data.ctypes.data x += 1. - assert_(id1 == x.data.ctypes._data) + assert_(id1 == x.data.ctypes.data) assert_equal(x, y + 1.) def test_inplace_addition_array(self): diff --git a/numpy/ma/tests/test_extras.py b/numpy/ma/tests/test_extras.py index c29bec2bd..5243cf714 100644 --- a/numpy/ma/tests/test_extras.py +++ b/numpy/ma/tests/test_extras.py @@ -14,7 +14,7 @@ import itertools import numpy as np from numpy.testing import ( - assert_warns, suppress_warnings, assert_raises, + assert_warns, suppress_warnings ) from numpy.ma.testutils import ( assert_, assert_array_equal, assert_equal, assert_almost_equal @@ -29,9 +29,8 @@ from numpy.ma.extras import ( ediff1d, apply_over_axes, apply_along_axis, compress_nd, compress_rowcols, mask_rowcols, clump_masked, clump_unmasked, flatnotmasked_contiguous, notmasked_contiguous, notmasked_edges, masked_all, masked_all_like, isin, - diagflat, stack, vstack, hstack + diagflat, stack, vstack ) -import numpy.ma.extras as mae class TestGeneric(object): diff --git a/numpy/ma/tests/test_mrecords.py b/numpy/ma/tests/test_mrecords.py index 8b9e3fbc9..dbbf1c8a1 100644 --- a/numpy/ma/tests/test_mrecords.py +++ b/numpy/ma/tests/test_mrecords.py @@ -7,8 +7,6 @@ """ from __future__ import division, absolute_import, print_function -import warnings - import numpy as np import numpy.ma as ma from numpy import recarray diff --git a/numpy/ma/tests/test_old_ma.py b/numpy/ma/tests/test_old_ma.py index 807121184..2978be22c 100644 --- a/numpy/ma/tests/test_old_ma.py +++ b/numpy/ma/tests/test_old_ma.py @@ -8,7 +8,6 @@ import numpy.core.fromnumeric as fromnumeric from numpy.testing import ( assert_, assert_raises, assert_equal, ) -from numpy.ma.testutils import assert_array_equal from numpy.ma import ( MaskType, MaskedArray, absolute, add, all, allclose, allequal, alltrue, arange, arccos, arcsin, arctan, arctan2, array, average, choose, diff --git a/numpy/ma/tests/test_regression.py b/numpy/ma/tests/test_regression.py index 96c418a51..54f1bda7d 100644 --- a/numpy/ma/tests/test_regression.py +++ b/numpy/ma/tests/test_regression.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import warnings - import numpy as np from numpy.testing import ( assert_, assert_array_equal, assert_allclose, suppress_warnings @@ -84,3 +82,8 @@ class TestRegression(object): assert_(a.mask.shape == (2,)) assert_(b.shape == (2, 2)) assert_(b.mask.shape == (2, 2)) + + def test_empty_list_on_structured(self): + # See gh-12464. Indexing with empty list should give empty result. + ma = np.ma.MaskedArray([(1, 1.), (2, 2.), (3, 3.)], dtype='i4,f4') + assert_array_equal(ma[[]], ma[:0]) diff --git a/numpy/matlib.py b/numpy/matlib.py index 004e5f0c8..9e115943a 100644 --- a/numpy/matlib.py +++ b/numpy/matlib.py @@ -39,11 +39,11 @@ def empty(shape, dtype=None, order='C'): -------- >>> import numpy.matlib >>> np.matlib.empty((2, 2)) # filled with random data - matrix([[ 6.76425276e-320, 9.79033856e-307], - [ 7.39337286e-309, 3.22135945e-309]]) #random + matrix([[ 6.76425276e-320, 9.79033856e-307], # random + [ 7.39337286e-309, 3.22135945e-309]]) >>> np.matlib.empty((2, 2), dtype=int) - matrix([[ 6600475, 0], - [ 6586976, 22740995]]) #random + matrix([[ 6600475, 0], # random + [ 6586976, 22740995]]) """ return ndarray.__new__(matrix, shape, dtype, order=order) @@ -82,11 +82,11 @@ def ones(shape, dtype=None, order='C'): Examples -------- >>> np.matlib.ones((2,3)) - matrix([[ 1., 1., 1.], - [ 1., 1., 1.]]) + matrix([[1., 1., 1.], + [1., 1., 1.]]) >>> np.matlib.ones(2) - matrix([[ 1., 1.]]) + matrix([[1., 1.]]) """ a = ndarray.__new__(matrix, shape, dtype, order=order) @@ -126,11 +126,11 @@ def zeros(shape, dtype=None, order='C'): -------- >>> import numpy.matlib >>> np.matlib.zeros((2, 3)) - matrix([[ 0., 0., 0.], - [ 0., 0., 0.]]) + matrix([[0., 0., 0.], + [0., 0., 0.]]) >>> np.matlib.zeros(2) - matrix([[ 0., 0.]]) + matrix([[0., 0.]]) """ a = ndarray.__new__(matrix, shape, dtype, order=order) @@ -210,9 +210,9 @@ def eye(n,M=None, k=0, dtype=float, order='C'): -------- >>> import numpy.matlib >>> np.matlib.eye(3, k=1, dtype=float) - matrix([[ 0., 1., 0.], - [ 0., 0., 1.], - [ 0., 0., 0.]]) + matrix([[0., 1., 0.], + [0., 0., 1.], + [0., 0., 0.]]) """ return asmatrix(np.eye(n, M=M, k=k, dtype=dtype, order=order)) @@ -243,19 +243,20 @@ def rand(*args): Examples -------- + >>> np.random.seed(123) >>> import numpy.matlib >>> np.matlib.rand(2, 3) - matrix([[ 0.68340382, 0.67926887, 0.83271405], - [ 0.00793551, 0.20468222, 0.95253525]]) #random + matrix([[0.69646919, 0.28613933, 0.22685145], + [0.55131477, 0.71946897, 0.42310646]]) >>> np.matlib.rand((2, 3)) - matrix([[ 0.84682055, 0.73626594, 0.11308016], - [ 0.85429008, 0.3294825 , 0.89139555]]) #random + matrix([[0.9807642 , 0.68482974, 0.4809319 ], + [0.39211752, 0.34317802, 0.72904971]]) If the first argument is a tuple, other arguments are ignored: >>> np.matlib.rand((2, 3), 4) - matrix([[ 0.46898646, 0.15163588, 0.95188261], - [ 0.59208621, 0.09561818, 0.00583606]]) #random + matrix([[0.43857224, 0.0596779 , 0.39804426], + [0.73799541, 0.18249173, 0.17545176]]) """ if isinstance(args[0], tuple): @@ -294,18 +295,19 @@ def randn(*args): Examples -------- + >>> np.random.seed(123) >>> import numpy.matlib >>> np.matlib.randn(1) - matrix([[-0.09542833]]) #random + matrix([[-1.0856306]]) >>> np.matlib.randn(1, 2, 3) - matrix([[ 0.16198284, 0.0194571 , 0.18312985], - [-0.7509172 , 1.61055 , 0.45298599]]) #random + matrix([[ 0.99734545, 0.2829785 , -1.50629471], + [-0.57860025, 1.65143654, -2.42667924]]) Two-by-four matrix of samples from :math:`N(3, 6.25)`: >>> 2.5 * np.matlib.randn((2, 4)) + 3 - matrix([[ 4.74085004, 8.89381862, 4.09042411, 4.83721922], - [ 7.52373709, 5.07933944, -2.64043543, 0.45610557]]) #random + matrix([[1.92771843, 6.16484065, 0.83314899, 1.30278462], + [2.76322758, 6.72847407, 1.40274501, 1.8900451 ]]) """ if isinstance(args[0], tuple): diff --git a/numpy/matrixlib/defmatrix.py b/numpy/matrixlib/defmatrix.py index 7baa401a8..6f8eadf86 100644 --- a/numpy/matrixlib/defmatrix.py +++ b/numpy/matrixlib/defmatrix.py @@ -7,6 +7,7 @@ import warnings import ast import numpy.core.numeric as N from numpy.core.numeric import concatenate, isscalar +from numpy.core.overrides import set_module # While not in __all__, matrix_power used to be defined here, so we import # it for backward compatibility. from numpy.linalg import matrix_power @@ -33,6 +34,8 @@ def _convert_from_string(data): newdata.append(newrow) return newdata + +@set_module('numpy') def asmatrix(data, dtype=None): """ Interpret the input as a matrix. @@ -67,6 +70,8 @@ def asmatrix(data, dtype=None): """ return matrix(data, dtype=dtype, copy=False) + +@set_module('numpy') class matrix(N.ndarray): """ matrix(data, dtype=None, copy=True) @@ -99,9 +104,9 @@ class matrix(N.ndarray): Examples -------- >>> a = np.matrix('1 2; 3 4') - >>> print(a) - [[1 2] - [3 4]] + >>> a + matrix([[1, 2], + [3, 4]]) >>> np.matrix([[1, 2], [3, 4]]) matrix([[1, 2], @@ -305,12 +310,12 @@ class matrix(N.ndarray): matrix([[3], [7]]) >>> x.sum(axis=1, dtype='float') - matrix([[ 3.], - [ 7.]]) - >>> out = np.zeros((1, 2), dtype='float') - >>> x.sum(axis=1, dtype='float', out=out) - matrix([[ 3.], - [ 7.]]) + matrix([[3.], + [7.]]) + >>> out = np.zeros((2, 1), dtype='float') + >>> x.sum(axis=1, dtype='float', out=np.asmatrix(out)) + matrix([[3.], + [7.]]) """ return N.ndarray.sum(self, axis, dtype, out, keepdims=True)._collapse(axis) @@ -432,7 +437,7 @@ class matrix(N.ndarray): >>> x.mean() 5.5 >>> x.mean(0) - matrix([[ 4., 5., 6., 7.]]) + matrix([[4., 5., 6., 7.]]) >>> x.mean(1) matrix([[ 1.5], [ 5.5], @@ -464,9 +469,9 @@ class matrix(N.ndarray): [ 4, 5, 6, 7], [ 8, 9, 10, 11]]) >>> x.std() - 3.4520525295346629 + 3.4520525295346629 # may vary >>> x.std(0) - matrix([[ 3.26598632, 3.26598632, 3.26598632, 3.26598632]]) + matrix([[ 3.26598632, 3.26598632, 3.26598632, 3.26598632]]) # may vary >>> x.std(1) matrix([[ 1.11803399], [ 1.11803399], @@ -500,11 +505,11 @@ class matrix(N.ndarray): >>> x.var() 11.916666666666666 >>> x.var(0) - matrix([[ 10.66666667, 10.66666667, 10.66666667, 10.66666667]]) + matrix([[ 10.66666667, 10.66666667, 10.66666667, 10.66666667]]) # may vary >>> x.var(1) - matrix([[ 1.25], - [ 1.25], - [ 1.25]]) + matrix([[1.25], + [1.25], + [1.25]]) """ return N.ndarray.var(self, axis, dtype, out, ddof, keepdims=True)._collapse(axis) @@ -819,7 +824,7 @@ class matrix(N.ndarray): matrix([[-2. , 1. ], [ 1.5, -0.5]]) >>> m.getI() * m - matrix([[ 1., 0.], + matrix([[ 1., 0.], # may vary [ 0., 1.]]) """ @@ -881,7 +886,8 @@ class matrix(N.ndarray): [ 4, 5, 6, 7], [ 8, 9, 10, 11]]) >>> x.getA1() - array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + array([ 0, 1, 2, ..., 9, 10, 11]) + """ return self.__array__().ravel() @@ -981,10 +987,10 @@ class matrix(N.ndarray): [ 4. -4.j, 5. -5.j, 6. -6.j, 7. -7.j], [ 8. -8.j, 9. -9.j, 10.-10.j, 11.-11.j]]) >>> z.getH() - matrix([[ 0. +0.j, 4. +4.j, 8. +8.j], - [ 1. +1.j, 5. +5.j, 9. +9.j], - [ 2. +2.j, 6. +6.j, 10.+10.j], - [ 3. +3.j, 7. +7.j, 11.+11.j]]) + matrix([[ 0. -0.j, 4. +4.j, 8. +8.j], + [ 1. +1.j, 5. +5.j, 9. +9.j], + [ 2. +2.j, 6. +6.j, 10.+10.j], + [ 3. +3.j, 7. +7.j, 11.+11.j]]) """ if issubclass(self.dtype.type, N.complexfloating): @@ -1023,6 +1029,7 @@ def _from_string(str, gdict, ldict): return concatenate(rowtup, axis=0) +@set_module('numpy') def bmat(obj, ldict=None, gdict=None): """ Build a matrix object from a string, nested sequence, or array. diff --git a/numpy/matrixlib/tests/test_defmatrix.py b/numpy/matrixlib/tests/test_defmatrix.py index f8a8ad511..aa6e08d64 100644 --- a/numpy/matrixlib/tests/test_defmatrix.py +++ b/numpy/matrixlib/tests/test_defmatrix.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - try: # Accessing collections abstract classes from collections # has been deprecated since Python 3.3 diff --git a/numpy/matrixlib/tests/test_masked_matrix.py b/numpy/matrixlib/tests/test_masked_matrix.py index 1751020db..52fd18577 100644 --- a/numpy/matrixlib/tests/test_masked_matrix.py +++ b/numpy/matrixlib/tests/test_masked_matrix.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.ma.testutils import (assert_, assert_equal, assert_raises, assert_array_equal) diff --git a/numpy/matrixlib/tests/test_matrix_linalg.py b/numpy/matrixlib/tests/test_matrix_linalg.py index 8d31ec5b0..6fc733c2e 100644 --- a/numpy/matrixlib/tests/test_matrix_linalg.py +++ b/numpy/matrixlib/tests/test_matrix_linalg.py @@ -1,8 +1,6 @@ """ Test functions for linalg module using the matrix class.""" from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.linalg.tests.test_linalg import ( diff --git a/numpy/matrixlib/tests/test_multiarray.py b/numpy/matrixlib/tests/test_multiarray.py index 8de0a7c6a..6d84bd477 100644 --- a/numpy/matrixlib/tests/test_multiarray.py +++ b/numpy/matrixlib/tests/test_multiarray.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.testing import assert_, assert_equal, assert_array_equal diff --git a/numpy/matrixlib/tests/test_numeric.py b/numpy/matrixlib/tests/test_numeric.py index e9f44e747..95e1c8001 100644 --- a/numpy/matrixlib/tests/test_numeric.py +++ b/numpy/matrixlib/tests/test_numeric.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.testing import assert_equal diff --git a/numpy/matrixlib/tests/test_regression.py b/numpy/matrixlib/tests/test_regression.py index 88654c76a..70e147279 100644 --- a/numpy/matrixlib/tests/test_regression.py +++ b/numpy/matrixlib/tests/test_regression.py @@ -1,7 +1,5 @@ from __future__ import division, absolute_import, print_function -import pytest - import numpy as np from numpy.testing import assert_, assert_equal, assert_raises diff --git a/numpy/polynomial/chebyshev.py b/numpy/polynomial/chebyshev.py index f1ddc9b06..e0734e1b8 100644 --- a/numpy/polynomial/chebyshev.py +++ b/numpy/polynomial/chebyshev.py @@ -361,12 +361,12 @@ def poly2cheb(pol): >>> from numpy import polynomial as P >>> p = P.Polynomial(range(4)) >>> p - Polynomial([ 0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) + Polynomial([0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) >>> c = p.convert(kind=P.Chebyshev) >>> c - Chebyshev([ 1. , 3.25, 1. , 0.75], domain=[-1, 1], window=[-1, 1]) + Chebyshev([1. , 3.25, 1. , 0.75], domain=[-1., 1.], window=[-1., 1.]) >>> P.chebyshev.poly2cheb(range(4)) - array([ 1. , 3.25, 1. , 0.75]) + array([1. , 3.25, 1. , 0.75]) """ [pol] = pu.as_series([pol]) @@ -413,12 +413,12 @@ def cheb2poly(c): >>> from numpy import polynomial as P >>> c = P.Chebyshev(range(4)) >>> c - Chebyshev([ 0., 1., 2., 3.], [-1., 1.]) + Chebyshev([0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) >>> p = c.convert(kind=P.Polynomial) >>> p - Polynomial([ -2., -8., 4., 12.], [-1., 1.]) + Polynomial([-2., -8., 4., 12.], domain=[-1., 1.], window=[-1., 1.]) >>> P.chebyshev.cheb2poly(range(4)) - array([ -2., -8., 4., 12.]) + array([-2., -8., 4., 12.]) """ from .polynomial import polyadd, polysub, polymulx @@ -538,7 +538,7 @@ def chebfromroots(roots): array([ 0. , -0.25, 0. , 0.25]) >>> j = complex(0,1) >>> C.chebfromroots((-j,j)) # x^2 + 1 relative to the standard basis - array([ 1.5+0.j, 0.0+0.j, 0.5+0.j]) + array([1.5+0.j, 0. +0.j, 0.5+0.j]) """ if len(roots) == 0: @@ -594,7 +594,7 @@ def chebadd(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> C.chebadd(c1,c2) - array([ 4., 4., 4.]) + array([4., 4., 4.]) """ # c1, c2 are trimmed copies @@ -688,7 +688,7 @@ def chebmulx(c): -------- >>> from numpy.polynomial import chebyshev as C >>> C.chebmulx([1,2,3]) - array([ 1., 2.5, 3., 1.5, 2.]) + array([1. , 2.5, 1. , 1.5]) """ # c is a trimmed copy @@ -796,10 +796,10 @@ def chebdiv(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> C.chebdiv(c1,c2) # quotient "intuitive," remainder not - (array([ 3.]), array([-8., -4.])) + (array([3.]), array([-8., -4.])) >>> c2 = (0,1,2,3) >>> C.chebdiv(c2,c1) # neither "intuitive" - (array([ 0., 2.]), array([-2., -4.])) + (array([0., 2.]), array([-2., -4.])) """ # c1, c2 are trimmed copies @@ -853,7 +853,7 @@ def chebpow(c, pow, maxpower=16): -------- >>> from numpy.polynomial import chebyshev as C >>> C.chebpow([1, 2, 3, 4], 2) - array([15.5, 22. , 16. , 14. , 12.5, 12. , 8. ]) + array([15.5, 22. , 16. , ..., 12.5, 12. , 8. ]) """ # c is a trimmed copy @@ -928,13 +928,13 @@ def chebder(c, m=1, scl=1, axis=0): >>> from numpy.polynomial import chebyshev as C >>> c = (1,2,3,4) >>> C.chebder(c) - array([ 14., 12., 24.]) + array([14., 12., 24.]) >>> C.chebder(c,3) - array([ 96.]) + array([96.]) >>> C.chebder(c,scl=-1) array([-14., -12., -24.]) >>> C.chebder(c,2,-1) - array([ 12., 96.]) + array([12., 96.]) """ c = np.array(c, ndmin=1, copy=1) @@ -1048,8 +1048,8 @@ def chebint(c, m=1, k=[], lbnd=0, scl=1, axis=0): >>> C.chebint(c) array([ 0.5, -0.5, 0.5, 0.5]) >>> C.chebint(c,3) - array([ 0.03125 , -0.1875 , 0.04166667, -0.05208333, 0.01041667, - 0.00625 ]) + array([ 0.03125 , -0.1875 , 0.04166667, -0.05208333, 0.01041667, # may vary + 0.00625 ]) >>> C.chebint(c, k=3) array([ 3.5, -0.5, 0.5, 0.5]) >>> C.chebint(c,lbnd=-2) @@ -1096,7 +1096,7 @@ def chebint(c, m=1, k=[], lbnd=0, scl=1, axis=0): if n > 1: tmp[2] = c[1]/4 for j in range(2, n): - t = c[j]/(2*j + 1) + t = c[j]/(2*j + 1) # FIXME: t never used tmp[j + 1] = c[j]/(2*(j + 1)) tmp[j - 1] -= c[j]/(2*(j - 1)) tmp[0] += k[i] - chebval(lbnd, tmp) @@ -1674,7 +1674,7 @@ def chebfit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1885,7 +1885,7 @@ def chebroots(c): -------- >>> import numpy.polynomial.chebyshev as cheb >>> cheb.chebroots((-1, 1,-1, 1)) # T3 - T2 + T1 - T0 has real roots - array([ -5.00000000e-01, 2.60860684e-17, 1.00000000e+00]) + array([ -5.00000000e-01, 2.60860684e-17, 1.00000000e+00]) # may vary """ # c is a trimmed copy diff --git a/numpy/polynomial/hermite.py b/numpy/polynomial/hermite.py index 4905f366f..93c9fc564 100644 --- a/numpy/polynomial/hermite.py +++ b/numpy/polynomial/hermite.py @@ -114,7 +114,7 @@ def poly2herm(pol): -------- >>> from numpy.polynomial.hermite import poly2herm >>> poly2herm(np.arange(4)) - array([ 1. , 2.75 , 0.5 , 0.375]) + array([1. , 2.75 , 0.5 , 0.375]) """ [pol] = pu.as_series([pol]) @@ -160,7 +160,7 @@ def herm2poly(c): -------- >>> from numpy.polynomial.hermite import herm2poly >>> herm2poly([ 1. , 2.75 , 0.5 , 0.375]) - array([ 0., 1., 2., 3.]) + array([0., 1., 2., 3.]) """ from .polynomial import polyadd, polysub, polymulx @@ -280,10 +280,10 @@ def hermfromroots(roots): >>> from numpy.polynomial.hermite import hermfromroots, hermval >>> coef = hermfromroots((-1, 0, 1)) >>> hermval((-1, 0, 1), coef) - array([ 0., 0., 0.]) + array([0., 0., 0.]) >>> coef = hermfromroots((-1j, 1j)) >>> hermval((-1j, 1j), coef) - array([ 0.+0.j, 0.+0.j]) + array([0.+0.j, 0.+0.j]) """ if len(roots) == 0: @@ -337,7 +337,7 @@ def hermadd(c1, c2): -------- >>> from numpy.polynomial.hermite import hermadd >>> hermadd([1, 2, 3], [1, 2, 3, 4]) - array([ 2., 4., 6., 4.]) + array([2., 4., 6., 4.]) """ # c1, c2 are trimmed copies @@ -385,7 +385,7 @@ def hermsub(c1, c2): -------- >>> from numpy.polynomial.hermite import hermsub >>> hermsub([1, 2, 3, 4], [1, 2, 3]) - array([ 0., 0., 0., 4.]) + array([0., 0., 0., 4.]) """ # c1, c2 are trimmed copies @@ -435,7 +435,7 @@ def hermmulx(c): -------- >>> from numpy.polynomial.hermite import hermmulx >>> hermmulx([1, 2, 3]) - array([ 2. , 6.5, 1. , 1.5]) + array([2. , 6.5, 1. , 1.5]) """ # c is a trimmed copy @@ -488,7 +488,7 @@ def hermmul(c1, c2): -------- >>> from numpy.polynomial.hermite import hermmul >>> hermmul([1, 2, 3], [0, 1, 2]) - array([ 52., 29., 52., 7., 6.]) + array([52., 29., 52., 7., 6.]) """ # s1, s2 are trimmed copies @@ -557,11 +557,11 @@ def hermdiv(c1, c2): -------- >>> from numpy.polynomial.hermite import hermdiv >>> hermdiv([ 52., 29., 52., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 0.])) + (array([1., 2., 3.]), array([0.])) >>> hermdiv([ 54., 31., 52., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 2., 2.])) + (array([1., 2., 3.]), array([2., 2.])) >>> hermdiv([ 53., 30., 52., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 1., 1.])) + (array([1., 2., 3.]), array([1., 1.])) """ # c1, c2 are trimmed copies @@ -617,7 +617,7 @@ def hermpow(c, pow, maxpower=16): -------- >>> from numpy.polynomial.hermite import hermpow >>> hermpow([1, 2, 3], 2) - array([ 81., 52., 82., 12., 9.]) + array([81., 52., 82., 12., 9.]) """ # c is a trimmed copy @@ -690,9 +690,9 @@ def hermder(c, m=1, scl=1, axis=0): -------- >>> from numpy.polynomial.hermite import hermder >>> hermder([ 1. , 0.5, 0.5, 0.5]) - array([ 1., 2., 3.]) + array([1., 2., 3.]) >>> hermder([-0.5, 1./2., 1./8., 1./12., 1./16.], m=2) - array([ 1., 2., 3.]) + array([1., 2., 3.]) """ c = np.array(c, ndmin=1, copy=1) @@ -799,15 +799,15 @@ def hermint(c, m=1, k=[], lbnd=0, scl=1, axis=0): -------- >>> from numpy.polynomial.hermite import hermint >>> hermint([1,2,3]) # integrate once, value 0 at 0. - array([ 1. , 0.5, 0.5, 0.5]) + array([1. , 0.5, 0.5, 0.5]) >>> hermint([1,2,3], m=2) # integrate twice, value & deriv 0 at 0 - array([-0.5 , 0.5 , 0.125 , 0.08333333, 0.0625 ]) + array([-0.5 , 0.5 , 0.125 , 0.08333333, 0.0625 ]) # may vary >>> hermint([1,2,3], k=1) # integrate once, value 1 at 0. - array([ 2. , 0.5, 0.5, 0.5]) + array([2. , 0.5, 0.5, 0.5]) >>> hermint([1,2,3], lbnd=-1) # integrate once, value 0 at -1 array([-2. , 0.5, 0.5, 0.5]) >>> hermint([1,2,3], m=2, k=[1,2], lbnd=-1) - array([ 1.66666667, -0.5 , 0.125 , 0.08333333, 0.0625 ]) + array([ 1.66666667, -0.5 , 0.125 , 0.08333333, 0.0625 ]) # may vary """ c = np.array(c, ndmin=1, copy=1) @@ -918,8 +918,8 @@ def hermval(x, c, tensor=True): >>> hermval(1, coef) 11.0 >>> hermval([[1,2],[3,4]], coef) - array([[ 11., 51.], - [ 115., 203.]]) + array([[ 11., 51.], + [115., 203.]]) """ c = np.array(c, ndmin=1, copy=0) @@ -1437,7 +1437,7 @@ def hermfit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1490,7 +1490,7 @@ def hermfit(x, y, deg, rcond=None, full=False, w=None): >>> err = np.random.randn(len(x))/10 >>> y = hermval(x, [1, 2, 3]) + err >>> hermfit(x, y, 2) - array([ 0.97902637, 1.99849131, 3.00006 ]) + array([1.0218, 1.9986, 2.9999]) # may vary """ x = np.asarray(x) + 0.0 @@ -1656,9 +1656,9 @@ def hermroots(c): >>> from numpy.polynomial.hermite import hermroots, hermfromroots >>> coef = hermfromroots([-1, 0, 1]) >>> coef - array([ 0. , 0.25 , 0. , 0.125]) + array([0. , 0.25 , 0. , 0.125]) >>> hermroots(coef) - array([ -1.00000000e+00, -1.38777878e-17, 1.00000000e+00]) + array([-1.00000000e+00, -1.38777878e-17, 1.00000000e+00]) """ # c is a trimmed copy diff --git a/numpy/polynomial/hermite_e.py b/numpy/polynomial/hermite_e.py index 6cb044a55..bafb4b997 100644 --- a/numpy/polynomial/hermite_e.py +++ b/numpy/polynomial/hermite_e.py @@ -161,7 +161,7 @@ def herme2poly(c): -------- >>> from numpy.polynomial.hermite_e import herme2poly >>> herme2poly([ 2., 10., 2., 3.]) - array([ 0., 1., 2., 3.]) + array([0., 1., 2., 3.]) """ from .polynomial import polyadd, polysub, polymulx @@ -281,10 +281,10 @@ def hermefromroots(roots): >>> from numpy.polynomial.hermite_e import hermefromroots, hermeval >>> coef = hermefromroots((-1, 0, 1)) >>> hermeval((-1, 0, 1), coef) - array([ 0., 0., 0.]) + array([0., 0., 0.]) >>> coef = hermefromroots((-1j, 1j)) >>> hermeval((-1j, 1j), coef) - array([ 0.+0.j, 0.+0.j]) + array([0.+0.j, 0.+0.j]) """ if len(roots) == 0: @@ -338,7 +338,7 @@ def hermeadd(c1, c2): -------- >>> from numpy.polynomial.hermite_e import hermeadd >>> hermeadd([1, 2, 3], [1, 2, 3, 4]) - array([ 2., 4., 6., 4.]) + array([2., 4., 6., 4.]) """ # c1, c2 are trimmed copies @@ -386,7 +386,7 @@ def hermesub(c1, c2): -------- >>> from numpy.polynomial.hermite_e import hermesub >>> hermesub([1, 2, 3, 4], [1, 2, 3]) - array([ 0., 0., 0., 4.]) + array([0., 0., 0., 4.]) """ # c1, c2 are trimmed copies @@ -432,7 +432,7 @@ def hermemulx(c): -------- >>> from numpy.polynomial.hermite_e import hermemulx >>> hermemulx([1, 2, 3]) - array([ 2., 7., 2., 3.]) + array([2., 7., 2., 3.]) """ # c is a trimmed copy @@ -485,7 +485,7 @@ def hermemul(c1, c2): -------- >>> from numpy.polynomial.hermite_e import hermemul >>> hermemul([1, 2, 3], [0, 1, 2]) - array([ 14., 15., 28., 7., 6.]) + array([14., 15., 28., 7., 6.]) """ # s1, s2 are trimmed copies @@ -554,9 +554,9 @@ def hermediv(c1, c2): -------- >>> from numpy.polynomial.hermite_e import hermediv >>> hermediv([ 14., 15., 28., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 0.])) + (array([1., 2., 3.]), array([0.])) >>> hermediv([ 15., 17., 28., 7., 6.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 1., 2.])) + (array([1., 2., 3.]), array([1., 2.])) """ # c1, c2 are trimmed copies @@ -612,7 +612,7 @@ def hermepow(c, pow, maxpower=16): -------- >>> from numpy.polynomial.hermite_e import hermepow >>> hermepow([1, 2, 3], 2) - array([ 23., 28., 46., 12., 9.]) + array([23., 28., 46., 12., 9.]) """ # c is a trimmed copy @@ -685,9 +685,9 @@ def hermeder(c, m=1, scl=1, axis=0): -------- >>> from numpy.polynomial.hermite_e import hermeder >>> hermeder([ 1., 1., 1., 1.]) - array([ 1., 2., 3.]) + array([1., 2., 3.]) >>> hermeder([-0.25, 1., 1./2., 1./3., 1./4 ], m=2) - array([ 1., 2., 3.]) + array([1., 2., 3.]) """ c = np.array(c, ndmin=1, copy=1) @@ -794,15 +794,15 @@ def hermeint(c, m=1, k=[], lbnd=0, scl=1, axis=0): -------- >>> from numpy.polynomial.hermite_e import hermeint >>> hermeint([1, 2, 3]) # integrate once, value 0 at 0. - array([ 1., 1., 1., 1.]) + array([1., 1., 1., 1.]) >>> hermeint([1, 2, 3], m=2) # integrate twice, value & deriv 0 at 0 - array([-0.25 , 1. , 0.5 , 0.33333333, 0.25 ]) + array([-0.25 , 1. , 0.5 , 0.33333333, 0.25 ]) # may vary >>> hermeint([1, 2, 3], k=1) # integrate once, value 1 at 0. - array([ 2., 1., 1., 1.]) + array([2., 1., 1., 1.]) >>> hermeint([1, 2, 3], lbnd=-1) # integrate once, value 0 at -1 array([-1., 1., 1., 1.]) >>> hermeint([1, 2, 3], m=2, k=[1, 2], lbnd=-1) - array([ 1.83333333, 0. , 0.5 , 0.33333333, 0.25 ]) + array([ 1.83333333, 0. , 0.5 , 0.33333333, 0.25 ]) # may vary """ c = np.array(c, ndmin=1, copy=1) @@ -913,8 +913,8 @@ def hermeval(x, c, tensor=True): >>> hermeval(1, coef) 3.0 >>> hermeval([[1,2],[3,4]], coef) - array([[ 3., 14.], - [ 31., 54.]]) + array([[ 3., 14.], + [31., 54.]]) """ c = np.array(c, ndmin=1, copy=0) @@ -1430,7 +1430,7 @@ def hermefit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1480,10 +1480,11 @@ def hermefit(x, y, deg, rcond=None, full=False, w=None): -------- >>> from numpy.polynomial.hermite_e import hermefit, hermeval >>> x = np.linspace(-10, 10) + >>> np.random.seed(123) >>> err = np.random.randn(len(x))/10 >>> y = hermeval(x, [1, 2, 3]) + err >>> hermefit(x, y, 2) - array([ 1.01690445, 1.99951418, 2.99948696]) + array([ 1.01690445, 1.99951418, 2.99948696]) # may vary """ x = np.asarray(x) + 0.0 @@ -1650,9 +1651,9 @@ def hermeroots(c): >>> from numpy.polynomial.hermite_e import hermeroots, hermefromroots >>> coef = hermefromroots([-1, 0, 1]) >>> coef - array([ 0., 2., 0., 1.]) + array([0., 2., 0., 1.]) >>> hermeroots(coef) - array([-1., 0., 1.]) + array([-1., 0., 1.]) # may vary """ # c is a trimmed copy diff --git a/numpy/polynomial/laguerre.py b/numpy/polynomial/laguerre.py index a116d20a7..9207c9afe 100644 --- a/numpy/polynomial/laguerre.py +++ b/numpy/polynomial/laguerre.py @@ -160,7 +160,7 @@ def lag2poly(c): -------- >>> from numpy.polynomial.laguerre import lag2poly >>> lag2poly([ 23., -63., 58., -18.]) - array([ 0., 1., 2., 3.]) + array([0., 1., 2., 3.]) """ from .polynomial import polyadd, polysub, polymulx @@ -277,10 +277,10 @@ def lagfromroots(roots): >>> from numpy.polynomial.laguerre import lagfromroots, lagval >>> coef = lagfromroots((-1, 0, 1)) >>> lagval((-1, 0, 1), coef) - array([ 0., 0., 0.]) + array([0., 0., 0.]) >>> coef = lagfromroots((-1j, 1j)) >>> lagval((-1j, 1j), coef) - array([ 0.+0.j, 0.+0.j]) + array([0.+0.j, 0.+0.j]) """ if len(roots) == 0: @@ -334,7 +334,7 @@ def lagadd(c1, c2): -------- >>> from numpy.polynomial.laguerre import lagadd >>> lagadd([1, 2, 3], [1, 2, 3, 4]) - array([ 2., 4., 6., 4.]) + array([2., 4., 6., 4.]) """ @@ -383,7 +383,7 @@ def lagsub(c1, c2): -------- >>> from numpy.polynomial.laguerre import lagsub >>> lagsub([1, 2, 3, 4], [1, 2, 3]) - array([ 0., 0., 0., 4.]) + array([0., 0., 0., 4.]) """ # c1, c2 are trimmed copies @@ -433,7 +433,7 @@ def lagmulx(c): -------- >>> from numpy.polynomial.laguerre import lagmulx >>> lagmulx([1, 2, 3]) - array([ -1., -1., 11., -9.]) + array([-1., -1., 11., -9.]) """ # c is a trimmed copy @@ -556,9 +556,9 @@ def lagdiv(c1, c2): -------- >>> from numpy.polynomial.laguerre import lagdiv >>> lagdiv([ 8., -13., 38., -51., 36.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 0.])) + (array([1., 2., 3.]), array([0.])) >>> lagdiv([ 9., -12., 38., -51., 36.], [0, 1, 2]) - (array([ 1., 2., 3.]), array([ 1., 1.])) + (array([1., 2., 3.]), array([1., 1.])) """ # c1, c2 are trimmed copies @@ -687,9 +687,9 @@ def lagder(c, m=1, scl=1, axis=0): -------- >>> from numpy.polynomial.laguerre import lagder >>> lagder([ 1., 1., 1., -3.]) - array([ 1., 2., 3.]) + array([1., 2., 3.]) >>> lagder([ 1., 0., 0., -4., 3.], m=2) - array([ 1., 2., 3.]) + array([1., 2., 3.]) """ c = np.array(c, ndmin=1, copy=1) @@ -805,9 +805,9 @@ def lagint(c, m=1, k=[], lbnd=0, scl=1, axis=0): >>> lagint([1,2,3], k=1) array([ 2., 1., 1., -3.]) >>> lagint([1,2,3], lbnd=-1) - array([ 11.5, 1. , 1. , -3. ]) + array([11.5, 1. , 1. , -3. ]) >>> lagint([1,2], m=2, k=[1,2], lbnd=-1) - array([ 11.16666667, -5. , -3. , 2. ]) + array([ 11.16666667, -5. , -3. , 2. ]) # may vary """ c = np.array(c, ndmin=1, copy=1) @@ -1436,7 +1436,7 @@ def lagfit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1489,7 +1489,7 @@ def lagfit(x, y, deg, rcond=None, full=False, w=None): >>> err = np.random.randn(len(x))/10 >>> y = lagval(x, [1, 2, 3]) + err >>> lagfit(x, y, 2) - array([ 0.96971004, 2.00193749, 3.00288744]) + array([ 0.96971004, 2.00193749, 3.00288744]) # may vary """ x = np.asarray(x) + 0.0 @@ -1656,7 +1656,7 @@ def lagroots(c): >>> coef array([ 2., -8., 12., -6.]) >>> lagroots(coef) - array([ -4.44089210e-16, 1.00000000e+00, 2.00000000e+00]) + array([-4.4408921e-16, 1.0000000e+00, 2.0000000e+00]) """ # c is a trimmed copy diff --git a/numpy/polynomial/legendre.py b/numpy/polynomial/legendre.py index e9c24594b..f81bc002c 100644 --- a/numpy/polynomial/legendre.py +++ b/numpy/polynomial/legendre.py @@ -136,10 +136,10 @@ def poly2leg(pol): >>> from numpy import polynomial as P >>> p = P.Polynomial(np.arange(4)) >>> p - Polynomial([ 0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) + Polynomial([0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) >>> c = P.Legendre(P.legendre.poly2leg(p.coef)) >>> c - Legendre([ 1. , 3.25, 1. , 0.75], domain=[-1, 1], window=[-1, 1]) + Legendre([ 1. , 3.25, 1. , 0.75], domain=[-1, 1], window=[-1, 1]) # may vary """ [pol] = pu.as_series([pol]) @@ -183,12 +183,13 @@ def leg2poly(c): Examples -------- + >>> from numpy import polynomial as P >>> c = P.Legendre(range(4)) >>> c - Legendre([ 0., 1., 2., 3.], [-1., 1.]) + Legendre([0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) >>> p = c.convert(kind=P.Polynomial) >>> p - Polynomial([-1. , -3.5, 3. , 7.5], [-1., 1.]) + Polynomial([-1. , -3.5, 3. , 7.5], domain=[-1., 1.], window=[-1., 1.]) >>> P.leg2poly(range(4)) array([-1. , -3.5, 3. , 7.5]) @@ -310,7 +311,7 @@ def legfromroots(roots): array([ 0. , -0.4, 0. , 0.4]) >>> j = complex(0,1) >>> L.legfromroots((-j,j)) # x^2 + 1 relative to the standard basis - array([ 1.33333333+0.j, 0.00000000+0.j, 0.66666667+0.j]) + array([ 1.33333333+0.j, 0.00000000+0.j, 0.66666667+0.j]) # may vary """ if len(roots) == 0: @@ -366,7 +367,7 @@ def legadd(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> L.legadd(c1,c2) - array([ 4., 4., 4.]) + array([4., 4., 4.]) """ # c1, c2 are trimmed copies @@ -468,7 +469,7 @@ def legmulx(c): -------- >>> from numpy.polynomial import legendre as L >>> L.legmulx([1,2,3]) - array([ 0.66666667, 2.2, 1.33333333, 1.8]) + array([ 0.66666667, 2.2, 1.33333333, 1.8]) # may vary """ # c is a trimmed copy @@ -525,8 +526,8 @@ def legmul(c1, c2): >>> from numpy.polynomial import legendre as L >>> c1 = (1,2,3) >>> c2 = (3,2) - >>> P.legmul(c1,c2) # multiplication requires "reprojection" - array([ 4.33333333, 10.4 , 11.66666667, 3.6 ]) + >>> L.legmul(c1,c2) # multiplication requires "reprojection" + array([ 4.33333333, 10.4 , 11.66666667, 3.6 ]) # may vary """ # s1, s2 are trimmed copies @@ -597,10 +598,10 @@ def legdiv(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> L.legdiv(c1,c2) # quotient "intuitive," remainder not - (array([ 3.]), array([-8., -4.])) + (array([3.]), array([-8., -4.])) >>> c2 = (0,1,2,3) >>> L.legdiv(c2,c1) # neither "intuitive" - (array([-0.07407407, 1.66666667]), array([-1.03703704, -2.51851852])) + (array([-0.07407407, 1.66666667]), array([-1.03703704, -2.51851852])) # may vary """ # c1, c2 are trimmed copies @@ -729,7 +730,7 @@ def legder(c, m=1, scl=1, axis=0): >>> L.legder(c) array([ 6., 9., 20.]) >>> L.legder(c, 3) - array([ 60.]) + array([60.]) >>> L.legder(c, scl=-1) array([ -6., -9., -20.]) >>> L.legder(c, 2,-1) @@ -845,16 +846,16 @@ def legint(c, m=1, k=[], lbnd=0, scl=1, axis=0): >>> from numpy.polynomial import legendre as L >>> c = (1,2,3) >>> L.legint(c) - array([ 0.33333333, 0.4 , 0.66666667, 0.6 ]) + array([ 0.33333333, 0.4 , 0.66666667, 0.6 ]) # may vary >>> L.legint(c, 3) - array([ 1.66666667e-02, -1.78571429e-02, 4.76190476e-02, - -1.73472348e-18, 1.90476190e-02, 9.52380952e-03]) + array([ 1.66666667e-02, -1.78571429e-02, 4.76190476e-02, # may vary + -1.73472348e-18, 1.90476190e-02, 9.52380952e-03]) >>> L.legint(c, k=3) - array([ 3.33333333, 0.4 , 0.66666667, 0.6 ]) + array([ 3.33333333, 0.4 , 0.66666667, 0.6 ]) # may vary >>> L.legint(c, lbnd=-2) - array([ 7.33333333, 0.4 , 0.66666667, 0.6 ]) + array([ 7.33333333, 0.4 , 0.66666667, 0.6 ]) # may vary >>> L.legint(c, scl=2) - array([ 0.66666667, 0.8 , 1.33333333, 1.2 ]) + array([ 0.66666667, 0.8 , 1.33333333, 1.2 ]) # may vary """ c = np.array(c, ndmin=1, copy=1) @@ -1476,7 +1477,7 @@ def legfit(x, y, deg, rcond=None, full=False, w=None): warnings can be turned off by >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1686,7 +1687,7 @@ def legroots(c): -------- >>> import numpy.polynomial.legendre as leg >>> leg.legroots((1, 2, 3, 4)) # 4L_3 + 3L_2 + 2L_1 + 1L_0, all real roots - array([-0.85099543, -0.11407192, 0.51506735]) + array([-0.85099543, -0.11407192, 0.51506735]) # may vary """ # c is a trimmed copy diff --git a/numpy/polynomial/polynomial.py b/numpy/polynomial/polynomial.py index 259cd31f5..69599e3fd 100644 --- a/numpy/polynomial/polynomial.py +++ b/numpy/polynomial/polynomial.py @@ -185,7 +185,7 @@ def polyfromroots(roots): array([ 0., -1., 0., 1.]) >>> j = complex(0,1) >>> P.polyfromroots((-j,j)) # complex returned, though values are real - array([ 1.+0.j, 0.+0.j, 1.+0.j]) + array([1.+0.j, 0.+0.j, 1.+0.j]) """ if len(roots) == 0: @@ -233,7 +233,7 @@ def polyadd(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> sum = P.polyadd(c1,c2); sum - array([ 4., 4., 4.]) + array([4., 4., 4.]) >>> P.polyval(2, sum) # 4 + 4(2) + 4(2**2) 28.0 @@ -401,9 +401,9 @@ def polydiv(c1, c2): >>> c1 = (1,2,3) >>> c2 = (3,2,1) >>> P.polydiv(c1,c2) - (array([ 3.]), array([-8., -4.])) + (array([3.]), array([-8., -4.])) >>> P.polydiv(c2,c1) - (array([ 0.33333333]), array([ 2.66666667, 1.33333333])) + (array([ 0.33333333]), array([ 2.66666667, 1.33333333])) # may vary """ # c1, c2 are trimmed copies @@ -529,7 +529,7 @@ def polyder(c, m=1, scl=1, axis=0): >>> P.polyder(c) # (d/dx)(c) = 2 + 6x + 12x**2 array([ 2., 6., 12.]) >>> P.polyder(c,3) # (d**3/dx**3)(c) = 24 - array([ 24.]) + array([24.]) >>> P.polyder(c,scl=-1) # (d/d(-x))(c) = -2 - 6x - 12x**2 array([ -2., -6., -12.]) >>> P.polyder(c,2,-1) # (d**2/d(-x)**2)(c) = 6 + 24x @@ -636,14 +636,14 @@ def polyint(c, m=1, k=[], lbnd=0, scl=1, axis=0): >>> from numpy.polynomial import polynomial as P >>> c = (1,2,3) >>> P.polyint(c) # should return array([0, 1, 1, 1]) - array([ 0., 1., 1., 1.]) + array([0., 1., 1., 1.]) >>> P.polyint(c,3) # should return array([0, 0, 0, 1/6, 1/12, 1/20]) - array([ 0. , 0. , 0. , 0.16666667, 0.08333333, - 0.05 ]) + array([ 0. , 0. , 0. , 0.16666667, 0.08333333, # may vary + 0.05 ]) >>> P.polyint(c,k=3) # should return array([3, 1, 1, 1]) - array([ 3., 1., 1., 1.]) + array([3., 1., 1., 1.]) >>> P.polyint(c,lbnd=-2) # should return array([6, 1, 1, 1]) - array([ 6., 1., 1., 1.]) + array([6., 1., 1., 1.]) >>> P.polyint(c,scl=-2) # should return array([0, -2, -2, -2]) array([ 0., -2., -2., -2.]) @@ -761,17 +761,17 @@ def polyval(x, c, tensor=True): array([[0, 1], [2, 3]]) >>> polyval(a, [1,2,3]) - array([[ 1., 6.], - [ 17., 34.]]) + array([[ 1., 6.], + [17., 34.]]) >>> coef = np.arange(4).reshape(2,2) # multidimensional coefficients >>> coef array([[0, 1], [2, 3]]) >>> polyval([1,2], coef, tensor=True) - array([[ 2., 4.], - [ 4., 7.]]) + array([[2., 4.], + [4., 7.]]) >>> polyval([1,2], coef, tensor=False) - array([ 2., 7.]) + array([2., 7.]) """ c = np.array(c, ndmin=1, copy=0) @@ -851,8 +851,8 @@ def polyvalfromroots(x, r, tensor=True): array([[0, 1], [2, 3]]) >>> polyvalfromroots(a, [-1, 0, 1]) - array([[ -0., 0.], - [ 6., 24.]]) + array([[-0., 0.], + [ 6., 24.]]) >>> r = np.arange(-2, 2).reshape(2,2) # multidimensional coefficients >>> r # each column of r defines one polynomial array([[-2, -1], @@ -1363,7 +1363,7 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None): be turned off by: >>> import warnings - >>> warnings.simplefilter('ignore', RankWarning) + >>> warnings.simplefilter('ignore', np.RankWarning) See Also -------- @@ -1410,26 +1410,27 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None): Examples -------- + >>> np.random.seed(123) >>> from numpy.polynomial import polynomial as P >>> x = np.linspace(-1,1,51) # x "data": [-1, -0.96, ..., 0.96, 1] >>> y = x**3 - x + np.random.randn(len(x)) # x^3 - x + N(0,1) "noise" >>> c, stats = P.polyfit(x,y,3,full=True) + >>> np.random.seed(123) >>> c # c[0], c[2] should be approx. 0, c[1] approx. -1, c[3] approx. 1 - array([ 0.01909725, -1.30598256, -0.00577963, 1.02644286]) + array([ 0.01909725, -1.30598256, -0.00577963, 1.02644286]) # may vary >>> stats # note the large SSR, explaining the rather poor results - [array([ 38.06116253]), 4, array([ 1.38446749, 1.32119158, 0.50443316, - 0.28853036]), 1.1324274851176597e-014] + [array([ 38.06116253]), 4, array([ 1.38446749, 1.32119158, 0.50443316, # may vary + 0.28853036]), 1.1324274851176597e-014] Same thing without the added noise >>> y = x**3 - x >>> c, stats = P.polyfit(x,y,3,full=True) >>> c # c[0], c[2] should be "very close to 0", c[1] ~= -1, c[3] ~= 1 - array([ -1.73362882e-17, -1.00000000e+00, -2.67471909e-16, - 1.00000000e+00]) + array([-6.36925336e-18, -1.00000000e+00, -4.08053781e-16, 1.00000000e+00]) >>> stats # note the minuscule SSR - [array([ 7.46346754e-31]), 4, array([ 1.38446749, 1.32119158, - 0.50443316, 0.28853036]), 1.1324274851176597e-014] + [array([ 7.46346754e-31]), 4, array([ 1.38446749, 1.32119158, # may vary + 0.50443316, 0.28853036]), 1.1324274851176597e-014] """ x = np.asarray(x) + 0.0 @@ -1591,7 +1592,7 @@ def polyroots(c): dtype('float64') >>> j = complex(0,1) >>> poly.polyroots(poly.polyfromroots((-j,0,j))) - array([ 0.00000000e+00+0.j, 0.00000000e+00+1.j, 2.77555756e-17-1.j]) + array([ 0.00000000e+00+0.j, 0.00000000e+00+1.j, 2.77555756e-17-1.j]) # may vary """ # c is a trimmed copy diff --git a/numpy/polynomial/polyutils.py b/numpy/polynomial/polyutils.py index c1ed0c9b3..eff4a8ee1 100644 --- a/numpy/polynomial/polyutils.py +++ b/numpy/polynomial/polyutils.py @@ -156,19 +156,19 @@ def as_series(alist, trim=True): >>> from numpy.polynomial import polyutils as pu >>> a = np.arange(4) >>> pu.as_series(a) - [array([ 0.]), array([ 1.]), array([ 2.]), array([ 3.])] + [array([0.]), array([1.]), array([2.]), array([3.])] >>> b = np.arange(6).reshape((2,3)) >>> pu.as_series(b) - [array([ 0., 1., 2.]), array([ 3., 4., 5.])] + [array([0., 1., 2.]), array([3., 4., 5.])] >>> pu.as_series((1, np.arange(3), np.arange(2, dtype=np.float16))) - [array([ 1.]), array([ 0., 1., 2.]), array([ 0., 1.])] + [array([1.]), array([0., 1., 2.]), array([0., 1.])] >>> pu.as_series([2, [1.1, 0.]]) - [array([ 2.]), array([ 1.1])] + [array([2.]), array([1.1])] >>> pu.as_series([2, [1.1, 0.]], trim=False) - [array([ 2.]), array([ 1.1, 0. ])] + [array([2.]), array([1.1, 0. ])] """ arrays = [np.array(a, ndmin=1, copy=0) for a in alist] @@ -233,12 +233,12 @@ def trimcoef(c, tol=0): -------- >>> from numpy.polynomial import polyutils as pu >>> pu.trimcoef((0,0,3,0,5,0,0)) - array([ 0., 0., 3., 0., 5.]) + array([0., 0., 3., 0., 5.]) >>> pu.trimcoef((0,0,1e-3,0,1e-5,0,0),1e-3) # item == tol is trimmed - array([ 0.]) + array([0.]) >>> i = complex(0,1) # works for complex >>> pu.trimcoef((3e-4,1e-3*(1-i),5e-4,2e-5*(1+i)), 1e-3) - array([ 0.0003+0.j , 0.0010-0.001j]) + array([0.0003+0.j , 0.001 -0.001j]) """ if tol < 0: @@ -332,10 +332,10 @@ def mapparms(old, new): >>> pu.mapparms((-1,1),(-1,1)) (0.0, 1.0) >>> pu.mapparms((1,-1),(-1,1)) - (0.0, -1.0) + (-0.0, -1.0) >>> i = complex(0,1) >>> pu.mapparms((-i,-1),(1,i)) - ((1+1j), (1+0j)) + ((1+1j), (1-0j)) """ oldlen = old[1] - old[0] @@ -390,10 +390,10 @@ def mapdomain(x, old, new): >>> x = np.linspace(-1,1,6); x array([-1. , -0.6, -0.2, 0.2, 0.6, 1. ]) >>> x_out = pu.mapdomain(x, old_domain, new_domain); x_out - array([ 0. , 1.25663706, 2.51327412, 3.76991118, 5.02654825, + array([ 0. , 1.25663706, 2.51327412, 3.76991118, 5.02654825, # may vary 6.28318531]) >>> x - pu.mapdomain(x_out, new_domain, old_domain) - array([ 0., 0., 0., 0., 0., 0.]) + array([0., 0., 0., 0., 0., 0.]) Also works for complex numbers (and thus can be used to map any line in the complex plane to any other line therein). @@ -402,9 +402,9 @@ def mapdomain(x, old, new): >>> old = (-1 - i, 1 + i) >>> new = (-1 + i, 1 - i) >>> z = np.linspace(old[0], old[1], 6); z - array([-1.0-1.j , -0.6-0.6j, -0.2-0.2j, 0.2+0.2j, 0.6+0.6j, 1.0+1.j ]) - >>> new_z = P.mapdomain(z, old, new); new_z - array([-1.0+1.j , -0.6+0.6j, -0.2+0.2j, 0.2-0.2j, 0.6-0.6j, 1.0-1.j ]) + array([-1. -1.j , -0.6-0.6j, -0.2-0.2j, 0.2+0.2j, 0.6+0.6j, 1. +1.j ]) + >>> new_z = pu.mapdomain(z, old, new); new_z + array([-1.0+1.j , -0.6+0.6j, -0.2+0.2j, 0.2-0.2j, 0.6-0.6j, 1.0-1.j ]) # may vary """ x = np.asanyarray(x) diff --git a/numpy/random/mtrand/distributions.c b/numpy/random/mtrand/distributions.c index b7e157915..2548a646e 100644 --- a/numpy/random/mtrand/distributions.c +++ b/numpy/random/mtrand/distributions.c @@ -650,6 +650,9 @@ double rk_pareto(rk_state *state, double a) double rk_weibull(rk_state *state, double a) { + if (a == 0.0) { + return 0.0; + } return pow(rk_standard_exponential(state), 1./a); } diff --git a/numpy/random/mtrand/mtrand.pyx b/numpy/random/mtrand/mtrand.pyx index efd364605..2a6daa88c 100644 --- a/numpy/random/mtrand/mtrand.pyx +++ b/numpy/random/mtrand/mtrand.pyx @@ -1,3 +1,5 @@ +# cython: language_level=3 + # mtrand.pyx -- A Pyrex wrapper of Jean-Sebastien Roy's RandomKit # # Copyright 2005 Robert Kern (robert.kern@gmail.com) @@ -844,16 +846,16 @@ cdef class RandomState: Examples -------- >>> np.random.random_sample() - 0.47108547995356098 + 0.47108547995356098 # random >>> type(np.random.random_sample()) - <type 'float'> + <class 'float'> >>> np.random.random_sample((5,)) - array([ 0.30220482, 0.86820401, 0.1654503 , 0.11659149, 0.54323428]) + array([ 0.30220482, 0.86820401, 0.1654503 , 0.11659149, 0.54323428]) # random Three-by-two array of random numbers from [-5, 0): >>> 5 * np.random.random_sample((3, 2)) - 5 - array([[-3.99149989, -0.52338984], + array([[-3.99149989, -0.52338984], # random [-2.99091858, -0.79479508], [-1.23204345, -1.75224494]]) @@ -954,14 +956,14 @@ cdef class RandomState: Examples -------- >>> np.random.randint(2, size=10) - array([1, 0, 0, 0, 1, 1, 0, 0, 1, 0]) + array([1, 0, 0, 0, 1, 1, 0, 0, 1, 0]) # random >>> np.random.randint(1, size=10) array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) Generate a 2 x 4 array of ints between 0 and 4, inclusive: >>> np.random.randint(5, size=(2, 4)) - array([[4, 0, 2, 1], + array([[4, 0, 2, 1], # random [3, 2, 2, 0]]) """ @@ -1076,34 +1078,34 @@ cdef class RandomState: Generate a uniform random sample from np.arange(5) of size 3: >>> np.random.choice(5, 3) - array([0, 3, 4]) + array([0, 3, 4]) # random >>> #This is equivalent to np.random.randint(0,5,3) Generate a non-uniform random sample from np.arange(5) of size 3: >>> np.random.choice(5, 3, p=[0.1, 0, 0.3, 0.6, 0]) - array([3, 3, 0]) + array([3, 3, 0]) # random Generate a uniform random sample from np.arange(5) of size 3 without replacement: >>> np.random.choice(5, 3, replace=False) - array([3,1,0]) + array([3,1,0]) # random >>> #This is equivalent to np.random.permutation(np.arange(5))[:3] Generate a non-uniform random sample from np.arange(5) of size 3 without replacement: >>> np.random.choice(5, 3, replace=False, p=[0.1, 0, 0.3, 0.6, 0]) - array([2, 3, 0]) + array([2, 3, 0]) # random Any of the above can be repeated with an arbitrary array-like instead of just integers. For instance: >>> aa_milne_arr = ['pooh', 'rabbit', 'piglet', 'Christopher'] >>> np.random.choice(aa_milne_arr, 5, p=[0.5, 0.1, 0.1, 0.3]) - array(['pooh', 'pooh', 'pooh', 'Christopher', 'piglet'], - dtype='|S11') + array(['pooh', 'pooh', 'pooh', 'Christopher', 'piglet'], # random + dtype='<U11') """ @@ -1470,11 +1472,11 @@ cdef class RandomState: Examples -------- >>> np.random.random_integers(5) - 4 + 4 # random >>> type(np.random.random_integers(5)) - <type 'int'> + <class 'numpy.int64'> >>> np.random.random_integers(5, size=(3,2)) - array([[5, 4], + array([[5, 4], # random [3, 3], [4, 5]]) @@ -1483,7 +1485,7 @@ cdef class RandomState: :math:`{0, 5/8, 10/8, 15/8, 20/8}`): >>> 2.5 * (np.random.random_integers(5, size=(5,)) - 1) / 4. - array([ 0.625, 1.25 , 0.625, 0.625, 2.5 ]) + array([ 0.625, 1.25 , 0.625, 0.625, 2.5 ]) # random Roll two six sided dice 1000 times and sum the results: @@ -1679,9 +1681,9 @@ cdef class RandomState: Parameters ---------- a : float or array_like of floats - Alpha, non-negative. + Alpha, positive (>0). b : float or array_like of floats - Beta, non-negative. + Beta, positive (>0). size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. If size is ``None`` (default), @@ -2068,8 +2070,8 @@ cdef class RandomState: The lower bound for the top 1% of the samples is : - >>> sort(s)[-10] - 7.61988120985 + >>> np.sort(s)[-10] + 7.61988120985 # random So there is about a 1% chance that the F statistic will exceed 7.62, the measured value is 36, so the null hypothesis is rejected at the 1% @@ -2166,6 +2168,8 @@ cdef class RandomState: >>> NF = np.histogram(nc_vals, bins=50, density=True) >>> c_vals = np.random.f(dfnum, dfden, 1000000) >>> F = np.histogram(c_vals, bins=50, density=True) + >>> import matplotlib + >>> import matplotlib.pyplot as plt >>> plt.plot(F[1][1:], F[0]) >>> plt.plot(NF[1][1:], NF[0]) >>> plt.show() @@ -2261,7 +2265,7 @@ cdef class RandomState: Examples -------- >>> np.random.chisquare(2,4) - array([ 1.89920014, 9.00867716, 3.13710533, 5.62318272]) + array([ 1.89920014, 9.00867716, 3.13710533, 5.62318272]) # random """ cdef ndarray odf @@ -2443,6 +2447,8 @@ cdef class RandomState: -------- Draw samples and plot the distribution: + >>> import matplotlib + >>> import matplotlib.pyplot as plt >>> s = np.random.standard_cauchy(1000000) >>> s = s[(s>-25) & (s<25)] # truncate distribution so it plots well >>> plt.hist(s, bins=100) @@ -2785,7 +2791,7 @@ cdef class RandomState: Parameters ---------- a : float or array_like of floats - Shape of the distribution. Should be greater than zero. + Shape parameter of the distribution. Must be nonnegative. size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. If size is ``None`` (default), @@ -3279,12 +3285,14 @@ cdef class RandomState: >>> loc, scale = 10, 1 >>> s = np.random.logistic(loc, scale, 10000) + >>> import matplotlib + >>> import matplotlib.pyplot as plt >>> count, bins, ignored = plt.hist(s, bins=50) # plot against distribution >>> def logist(x, loc, scale): - ... return exp((loc-x)/scale)/(scale*(1+exp((loc-x)/scale))**2) + ... return np.exp((loc-x)/scale)/(scale*(1+np.exp((loc-x)/scale))**2) >>> plt.plot(bins, logist(bins, loc, scale)*count.max()/\\ ... logist(bins, loc, scale).max()) >>> plt.show() @@ -3479,6 +3487,8 @@ cdef class RandomState: -------- Draw values from the distribution and plot the histogram + >>> import matplotlib + >>> from matplotlib.pyplot import hist >>> values = hist(np.random.rayleigh(3, 100000), bins=200, density=True) Wave heights tend to follow a Rayleigh distribution. If the mean wave @@ -3492,7 +3502,7 @@ cdef class RandomState: The percentage of waves larger than 3 meters is: >>> 100.*sum(s>3)/1000000. - 0.087300000000000003 + 0.087300000000000003 # random """ cdef ndarray oscale @@ -3873,9 +3883,9 @@ cdef class RandomState: single success after drilling 5 wells, after 6 wells, etc.? >>> s = np.random.negative_binomial(1, 0.1, 100000) - >>> for i in range(1, 11): + >>> for i in range(1, 11): # doctest: +SKIP ... probability = sum(s<i) / 100000. - ... print i, "wells drilled, probability of one success =", probability + ... print(i, "wells drilled, probability of one success =", probability) """ cdef ndarray on @@ -4233,6 +4243,8 @@ cdef class RandomState: >>> ngood, nbad, nsamp = 100, 2, 10 # number of good, number of bad, and number of samples >>> s = np.random.hypergeometric(ngood, nbad, nsamp, 1000) + >>> import matplotlib + >>> from matplotlib.pyplot import hist >>> hist(s) # note that it is very unlikely to grab both bad items @@ -4342,14 +4354,16 @@ cdef class RandomState: >>> a = .6 >>> s = np.random.logseries(a, 10000) + >>> import matplotlib + >>> import matplotlib.pyplot as plt >>> count, bins, ignored = plt.hist(s) # plot against distribution >>> def logseries(k, p): - ... return -p**k/(k*log(1-p)) + ... return -p**k/(k*np.log(1-p)) >>> plt.plot(bins, logseries(bins, a)*count.max()/ - logseries(bins, a).max(), 'r') + ... logseries(bins, a).max(), 'r') >>> plt.show() """ @@ -4474,7 +4488,7 @@ cdef class RandomState: standard deviation: >>> list((x[0,0,:] - mean) < 0.6) - [True, True] + [True, True] # random """ from numpy.dual import svd @@ -4580,14 +4594,14 @@ cdef class RandomState: Throw a dice 20 times: >>> np.random.multinomial(20, [1/6.]*6, size=1) - array([[4, 1, 7, 5, 2, 1]]) + array([[4, 1, 7, 5, 2, 1]]) # random It landed 4 times on 1, once on 2, etc. Now, throw the dice 20 times, and 20 times again: >>> np.random.multinomial(20, [1/6.]*6, size=2) - array([[3, 4, 3, 3, 4, 3], + array([[3, 4, 3, 3, 4, 3], # random [2, 4, 3, 4, 0, 7]]) For the first run, we threw 3 times 1, 4 times 2, etc. For the second, @@ -4596,7 +4610,7 @@ cdef class RandomState: A loaded die is more likely to land on number 6: >>> np.random.multinomial(100, [1/7.]*5 + [2/7.]) - array([11, 16, 14, 17, 16, 26]) + array([11, 16, 14, 17, 16, 26]) # random The probability inputs should be normalized. As an implementation detail, the value of the last entry is ignored and assumed to take @@ -4605,7 +4619,7 @@ cdef class RandomState: other should be sampled like so: >>> np.random.multinomial(100, [1.0 / 3, 2.0 / 3]) # RIGHT - array([38, 62]) + array([38, 62]) # random not like: @@ -4722,6 +4736,8 @@ cdef class RandomState: >>> s = np.random.dirichlet((10, 5, 3), 20).transpose() + >>> import matplotlib + >>> import matplotlib.pyplot as plt >>> plt.barh(range(20), s[0]) >>> plt.barh(range(20), s[1], left=s[0], color='g') >>> plt.barh(range(20), s[2], left=s[0]+s[1], color='r') @@ -4810,14 +4826,14 @@ cdef class RandomState: >>> arr = np.arange(10) >>> np.random.shuffle(arr) >>> arr - [1 7 5 2 9 4 3 6 0 8] + [1 7 5 2 9 4 3 6 0 8] # random Multi-dimensional arrays are only shuffled along the first axis: >>> arr = np.arange(9).reshape((3, 3)) >>> np.random.shuffle(arr) >>> arr - array([[3, 4, 5], + array([[3, 4, 5], # random [6, 7, 8], [0, 1, 2]]) @@ -4897,14 +4913,14 @@ cdef class RandomState: Examples -------- >>> np.random.permutation(10) - array([1, 7, 4, 3, 0, 9, 2, 5, 8, 6]) + array([1, 7, 4, 3, 0, 9, 2, 5, 8, 6]) # random >>> np.random.permutation([1, 4, 9, 12, 15]) - array([15, 1, 9, 4, 12]) + array([15, 1, 9, 4, 12]) # random >>> arr = np.arange(9).reshape((3, 3)) >>> np.random.permutation(arr) - array([[6, 7, 8], + array([[6, 7, 8], # random [0, 1, 2], [3, 4, 5]]) diff --git a/numpy/random/mtrand/numpy.pxd b/numpy/random/mtrand/numpy.pxd index 9092fa113..1b4fe6c10 100644 --- a/numpy/random/mtrand/numpy.pxd +++ b/numpy/random/mtrand/numpy.pxd @@ -1,3 +1,5 @@ +# cython: language_level=3 + # :Author: Travis Oliphant from cpython.exc cimport PyErr_Print diff --git a/numpy/random/tests/test_random.py b/numpy/random/tests/test_random.py index 276517363..d0bb92a73 100644 --- a/numpy/random/tests/test_random.py +++ b/numpy/random/tests/test_random.py @@ -942,7 +942,8 @@ class TestRandomDist(object): assert_array_almost_equal(actual, desired, decimal=15) def test_weibull_0(self): - assert_equal(np.random.weibull(a=0), 0) + np.random.seed(self.seed) + assert_equal(np.random.weibull(a=0, size=12), np.zeros(12)) assert_raises(ValueError, np.random.weibull, a=-0.) def test_zipf(self): diff --git a/numpy/testing/_private/parameterized.py b/numpy/testing/_private/parameterized.py index 53e67517d..a5fa4fb5e 100644 --- a/numpy/testing/_private/parameterized.py +++ b/numpy/testing/_private/parameterized.py @@ -190,7 +190,7 @@ def parameterized_argument_value_pairs(func, p): in zip(named_args, argspec.defaults or []) ]) - seen_arg_names = set([ n for (n, _) in result ]) + seen_arg_names = {n for (n, _) in result} keywords = QuietOrderedDict(sorted([ (name, p.kwargs[name]) for name in p.kwargs diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 20a7dfd0b..4059f6ee6 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -19,7 +19,7 @@ from warnings import WarningMessage import pprint from numpy.core import( - bool_, float32, empty, arange, array_repr, ndarray, isnat, array) + float32, empty, arange, array_repr, ndarray, isnat, array) from numpy.lib.utils import deprecate if sys.version_info[0] >= 3: @@ -521,7 +521,6 @@ def assert_almost_equal(actual,desired,decimal=7,err_msg='',verbose=True): ... <type 'exceptions.AssertionError'>: Arrays are not almost equal - <BLANKLINE> (mismatch 50.0%) x: array([ 1. , 2.33333333]) y: array([ 1. , 2.33333334]) @@ -687,7 +686,6 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, equal_inf=True): __tracebackhide__ = True # Hide traceback for py.test from numpy.core import array, isnan, inf, bool_ - from numpy.core.fromnumeric import all as npall x = array(x, copy=False, subok=True) y = array(y, copy=False, subok=True) @@ -855,7 +853,6 @@ def assert_array_equal(x, y, err_msg='', verbose=True): <type 'exceptions.ValueError'>: AssertionError: Arrays are not equal - <BLANKLINE> (mismatch 50.0%) x: array([ 1. , 3.14159265, NaN]) y: array([ 1. , 3.14159265, NaN]) @@ -931,7 +928,6 @@ def assert_array_almost_equal(x, y, decimal=6, err_msg='', verbose=True): <type 'exceptions.AssertionError'>: AssertionError: Arrays are not almost equal - <BLANKLINE> (mismatch 50.0%) x: array([ 1. , 2.33333, NaN]) y: array([ 1. , 2.33339, NaN]) @@ -946,7 +942,7 @@ def assert_array_almost_equal(x, y, decimal=6, err_msg='', verbose=True): """ __tracebackhide__ = True # Hide traceback for py.test - from numpy.core import around, number, float_, result_type, array + from numpy.core import number, float_, result_type, array from numpy.core.numerictypes import issubdtype from numpy.core.fromnumeric import any as npany diff --git a/numpy/testing/tests/test_decorators.py b/numpy/testing/tests/test_decorators.py index b8283d9de..bb3ea1acb 100644 --- a/numpy/testing/tests/test_decorators.py +++ b/numpy/testing/tests/test_decorators.py @@ -13,7 +13,7 @@ from numpy.testing import ( try: - import nose + import nose # noqa: F401 except ImportError: HAVE_NOSE = False else: diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py index e54fbc390..43afafaa8 100644 --- a/numpy/testing/tests/test_utils.py +++ b/numpy/testing/tests/test_utils.py @@ -180,18 +180,15 @@ class TestArrayEqual(_GenericTest): self._test_not_equal(b, a) def test_subclass_that_does_not_implement_npall(self): - # While we cannot guarantee testing functions will always work for - # subclasses, the tests should ideally rely only on subclasses having - # comparison operators, not on them being able to store booleans - # (which, e.g., astropy Quantity cannot usefully do). See gh-8452. class MyArray(np.ndarray): def __array_function__(self, *args, **kwargs): return NotImplemented a = np.array([1., 2.]).view(MyArray) b = np.array([2., 3.]).view(MyArray) - with assert_raises(TypeError): - np.all(a) + if np.core.overrides.ENABLE_ARRAY_FUNCTION: + with assert_raises(TypeError): + np.all(a) self._test_equal(a, a) self._test_not_equal(a, b) self._test_not_equal(b, a) @@ -1435,7 +1432,7 @@ def test_tempdir(): def test_temppath(): with temppath() as fpath: - with open(fpath, 'w') as f: + with open(fpath, 'w'): pass assert_(not os.path.isfile(fpath)) diff --git a/numpy/tests/test_ctypeslib.py b/numpy/tests/test_ctypeslib.py index 675f8d242..d389b37a8 100644 --- a/numpy/tests/test_ctypeslib.py +++ b/numpy/tests/test_ctypeslib.py @@ -2,6 +2,7 @@ from __future__ import division, absolute_import, print_function import sys import pytest +import weakref import numpy as np from numpy.ctypeslib import ndpointer, load_library, as_array @@ -9,20 +10,30 @@ from numpy.distutils.misc_util import get_shared_lib_extension from numpy.testing import assert_, assert_array_equal, assert_raises, assert_equal try: + import ctypes +except ImportError: + ctypes = None +else: cdll = None + test_cdll = None if hasattr(sys, 'gettotalrefcount'): try: cdll = load_library('_multiarray_umath_d', np.core._multiarray_umath.__file__) except OSError: pass + try: + test_cdll = load_library('_multiarray_tests', np.core._multiarray_tests.__file__) + except OSError: + pass if cdll is None: cdll = load_library('_multiarray_umath', np.core._multiarray_umath.__file__) - _HAS_CTYPE = True -except ImportError: - _HAS_CTYPE = False + if test_cdll is None: + test_cdll = load_library('_multiarray_tests', np.core._multiarray_tests.__file__) + + c_forward_pointer = test_cdll.forward_pointer -@pytest.mark.skipif(not _HAS_CTYPE, +@pytest.mark.skipif(ctypes is None, reason="ctypes not available in this python") @pytest.mark.skipif(sys.platform == 'cygwin', reason="Known to fail on cygwin") @@ -108,12 +119,72 @@ class TestNdpointer(object): assert_raises(TypeError, p.from_param, np.array([[1, 2], [3, 4]])) def test_cache(self): - a1 = ndpointer(dtype=np.float64) - a2 = ndpointer(dtype=np.float64) - assert_(a1 == a2) + assert_(ndpointer(dtype=np.float64) is ndpointer(dtype=np.float64)) + # shapes are normalized + assert_(ndpointer(shape=2) is ndpointer(shape=(2,))) -@pytest.mark.skipif(not _HAS_CTYPE, + # 1.12 <= v < 1.16 had a bug that made these fail + assert_(ndpointer(shape=2) is not ndpointer(ndim=2)) + assert_(ndpointer(ndim=2) is not ndpointer(shape=2)) + +@pytest.mark.skipif(ctypes is None, + reason="ctypes not available on this python installation") +class TestNdpointerCFunc(object): + def test_arguments(self): + """ Test that arguments are coerced from arrays """ + c_forward_pointer.restype = ctypes.c_void_p + c_forward_pointer.argtypes = (ndpointer(ndim=2),) + + c_forward_pointer(np.zeros((2, 3))) + # too many dimensions + assert_raises( + ctypes.ArgumentError, c_forward_pointer, np.zeros((2, 3, 4))) + + @pytest.mark.parametrize( + 'dt', [ + float, + np.dtype(dict( + formats=['<i4', '<i4'], + names=['a', 'b'], + offsets=[0, 2], + itemsize=6 + )) + ], ids=[ + 'float', + 'overlapping-fields' + ] + ) + def test_return(self, dt): + """ Test that return values are coerced to arrays """ + arr = np.zeros((2, 3), dt) + ptr_type = ndpointer(shape=arr.shape, dtype=arr.dtype) + + c_forward_pointer.restype = ptr_type + c_forward_pointer.argtypes = (ptr_type,) + + # check that the arrays are equivalent views on the same data + arr2 = c_forward_pointer(arr) + assert_equal(arr2.dtype, arr.dtype) + assert_equal(arr2.shape, arr.shape) + assert_equal( + arr2.__array_interface__['data'], + arr.__array_interface__['data'] + ) + + def test_vague_return_value(self): + """ Test that vague ndpointer return values do not promote to arrays """ + arr = np.zeros((2, 3)) + ptr_type = ndpointer(dtype=arr.dtype) + + c_forward_pointer.restype = ptr_type + c_forward_pointer.argtypes = (ptr_type,) + + ret = c_forward_pointer(arr) + assert_(isinstance(ret, ptr_type)) + + +@pytest.mark.skipif(ctypes is None, reason="ctypes not available on this python installation") class TestAsArray(object): def test_array(self): @@ -190,3 +261,15 @@ class TestAsArray(object): b = np.ctypeslib.as_array(newpnt, (N,)) # now delete both, which should cleanup both objects del newpnt, b + + def test_segmentation_fault(self): + arr = np.zeros((224, 224, 3)) + c_arr = np.ctypeslib.as_ctypes(arr) + arr_ref = weakref.ref(arr) + del arr + + # check the reference wasn't cleaned up + assert_(arr_ref() is not None) + + # check we avoid the segfault + c_arr[0][0][0] diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py new file mode 100644 index 000000000..194f8ecbb --- /dev/null +++ b/numpy/tests/test_public_api.py @@ -0,0 +1,89 @@ +from __future__ import division, absolute_import, print_function + +import sys + +import numpy as np +import pytest +try: + import ctypes +except ImportError: + ctypes = None + +def check_dir(module, module_name=None): + """Returns a mapping of all objects with the wrong __module__ attribute.""" + if module_name is None: + module_name = module.__name__ + results = {} + for name in dir(module): + item = getattr(module, name) + if (hasattr(item, '__module__') and hasattr(item, '__name__') + and item.__module__ != module_name): + results[name] = item.__module__ + '.' + item.__name__ + return results + + +@pytest.mark.skipif( + sys.version_info[0] < 3, + reason="NumPy exposes slightly different functions on Python 2") +def test_numpy_namespace(): + # None of these objects are publicly documented. + undocumented = { + 'Tester': 'numpy.testing._private.nosetester.NoseTester', + '_add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc', + 'add_docstring': 'numpy.core._multiarray_umath.add_docstring', + 'add_newdoc': 'numpy.core.function_base.add_newdoc', + 'add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc', + 'byte_bounds': 'numpy.lib.utils.byte_bounds', + 'compare_chararrays': 'numpy.core._multiarray_umath.compare_chararrays', + 'deprecate': 'numpy.lib.utils.deprecate', + 'deprecate_with_doc': 'numpy.lib.utils.<lambda>', + 'disp': 'numpy.lib.function_base.disp', + 'fastCopyAndTranspose': 'numpy.core._multiarray_umath._fastCopyAndTranspose', + 'get_array_wrap': 'numpy.lib.shape_base.get_array_wrap', + 'get_include': 'numpy.lib.utils.get_include', + 'int_asbuffer': 'numpy.core._multiarray_umath.int_asbuffer', + 'mafromtxt': 'numpy.lib.npyio.mafromtxt', + 'maximum_sctype': 'numpy.core.numerictypes.maximum_sctype', + 'ndfromtxt': 'numpy.lib.npyio.ndfromtxt', + 'recfromcsv': 'numpy.lib.npyio.recfromcsv', + 'recfromtxt': 'numpy.lib.npyio.recfromtxt', + 'safe_eval': 'numpy.lib.utils.safe_eval', + 'set_string_function': 'numpy.core.arrayprint.set_string_function', + 'show_config': 'numpy.__config__.show', + 'who': 'numpy.lib.utils.who', + } + # These built-in types are re-exported by numpy. + builtins = { + 'bool': 'builtins.bool', + 'complex': 'builtins.complex', + 'float': 'builtins.float', + 'int': 'builtins.int', + 'long': 'builtins.int', + 'object': 'builtins.object', + 'str': 'builtins.str', + 'unicode': 'builtins.str', + } + whitelist = dict(undocumented, **builtins) + bad_results = check_dir(np) + # pytest gives better error messages with the builtin assert than with + # assert_equal + assert bad_results == whitelist + + +def test_numpy_linalg(): + bad_results = check_dir(np.linalg) + assert bad_results == {} + + +def test_numpy_fft(): + bad_results = check_dir(np.fft) + assert bad_results == {} + +@pytest.mark.skipif(ctypes is None, + reason="ctypes not available in this python") +def test_NPY_NO_EXPORT(): + cdll = ctypes.CDLL(np.core._multiarray_tests.__file__) + # Make sure an arbitrary NPY_NO_EXPORT function is actually hidden + f = getattr(cdll, 'test_not_exported', None) + assert f is None, ("'test_not_exported' is mistakenly exported, " + "NPY_NO_EXPORT does not work") diff --git a/numpy/tests/test_scripts.py b/numpy/tests/test_scripts.py index 33210cc42..e42dc25f9 100644 --- a/numpy/tests/test_scripts.py +++ b/numpy/tests/test_scripts.py @@ -7,8 +7,8 @@ from __future__ import division, print_function, absolute_import import sys import os import pytest -from os.path import join as pathjoin, isfile, dirname, basename -from subprocess import Popen, PIPE +from os.path import join as pathjoin, isfile, dirname +import subprocess import numpy as np from numpy.compat.py3k import basestring @@ -17,74 +17,13 @@ from numpy.testing import assert_, assert_equal is_inplace = isfile(pathjoin(dirname(np.__file__), '..', 'setup.py')) -def run_command(cmd, check_code=True): - """ Run command sequence `cmd` returning exit code, stdout, stderr - - Parameters - ---------- - cmd : str or sequence - string with command name or sequence of strings defining command - check_code : {True, False}, optional - If True, raise error for non-zero return code - - Returns - ------- - returncode : int - return code from execution of `cmd` - stdout : bytes (python 3) or str (python 2) - stdout from `cmd` - stderr : bytes (python 3) or str (python 2) - stderr from `cmd` - - Raises - ------ - RuntimeError - If `check_code` is True, and return code !=0 - """ - cmd = [cmd] if isinstance(cmd, basestring) else list(cmd) - if os.name == 'nt': - # Quote any arguments with spaces. The quotes delimit the arguments - # on Windows, and the arguments might be file paths with spaces. - # On Unix the list elements are each separate arguments. - cmd = ['"{0}"'.format(c) if ' ' in c else c for c in cmd] - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - stdout, stderr = proc.communicate() - if proc.poll() is None: - proc.terminate() - if check_code and proc.returncode != 0: - raise RuntimeError('\n'.join( - ['Command "{0}" failed with', - 'stdout', '------', '{1}', '', - 'stderr', '------', '{2}']).format(cmd, stdout, stderr)) - return proc.returncode, stdout, stderr - - -@pytest.mark.skipif(is_inplace, reason="Cannot test f2py command inplace") -@pytest.mark.xfail(reason="Test is unreliable") -def test_f2py(): - # test that we can run f2py script - - def try_f2py_commands(cmds): - success = 0 - for f2py_cmd in cmds: - try: - code, stdout, stderr = run_command([f2py_cmd, '-v']) - assert_equal(stdout.strip(), b'2') - success += 1 - except Exception: - pass - return success - +def find_f2py_commands(): if sys.platform == 'win32': - # Only the single 'f2py' script is installed in windows. exe_dir = dirname(sys.executable) if exe_dir.endswith('Scripts'): # virtualenv - f2py_cmds = [os.path.join(exe_dir, 'f2py')] + return [os.path.join(exe_dir, 'f2py')] else: - f2py_cmds = [os.path.join(exe_dir, "Scripts", 'f2py')] - success = try_f2py_commands(f2py_cmds) - msg = "Warning: f2py not found in path" - assert_(success == 1, msg) + return [os.path.join(exe_dir, "Scripts", 'f2py')] else: # Three scripts are installed in Unix-like systems: # 'f2py', 'f2py{major}', and 'f2py{major.minor}'. For example, @@ -93,7 +32,18 @@ def test_f2py(): version = sys.version_info major = str(version.major) minor = str(version.minor) - f2py_cmds = ('f2py', 'f2py' + major, 'f2py' + major + '.' + minor) - success = try_f2py_commands(f2py_cmds) - msg = "Warning: not all of %s, %s, and %s are found in path" % f2py_cmds - assert_(success == 3, msg) + return ['f2py', 'f2py' + major, 'f2py' + major + '.' + minor] + + +@pytest.mark.skipif(is_inplace, reason="Cannot test f2py command inplace") +@pytest.mark.xfail(reason="Test is unreliable") +@pytest.mark.parametrize('f2py_cmd', find_f2py_commands()) +def test_f2py(f2py_cmd): + # test that we can run f2py script + stdout = subprocess.check_output([f2py_cmd, '-v']) + assert_equal(stdout.strip(), b'2') + + +def test_pep338(): + stdout = subprocess.check_output([sys.executable, '-mnumpy.f2py', '-v']) + assert_equal(stdout.strip(), b'2') |