diff options
Diffstat (limited to 'numpy')
274 files changed, 7188 insertions, 4838 deletions
diff --git a/numpy/__init__.py b/numpy/__init__.py index 73e979a18..575e8ea3d 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -141,7 +141,8 @@ else: from .core import * from . import compat from . import lib - # FIXME: why have numpy.lib if everything is imported here?? + # NOTE: to be revisited following future namespace cleanup. + # See gh-14454 and gh-15672 for discussion. from .lib import * from . import linalg @@ -152,16 +153,12 @@ else: from . import ma from . import matrixlib as _mat from .matrixlib import * - from .compat import long # Make these accessible from numpy name-space # but not imported in from numpy import * # TODO[gh-6103]: Deprecate these - if sys.version_info[0] >= 3: - from builtins import bool, int, float, complex, object, str - unicode = str - else: - from __builtin__ import bool, int, float, complex, object, unicode, str + from builtins import bool, int, float, complex, object, str + from .compat import long, unicode from .core import round, abs, max, min # now that numpy modules are imported, can initialize limits @@ -218,7 +215,7 @@ else: "{!r}".format(__name__, attr)) def __dir__(): - return list(globals().keys()) + ['Tester', 'testing'] + return list(globals().keys() | {'Tester', 'testing'}) else: # We don't actually use this ourselves anymore, but I'm not 100% sure that @@ -256,3 +253,56 @@ else: _sanity_check() del _sanity_check + + def _mac_os_check(): + """ + Quick Sanity check for Mac OS look for accelerate build bugs. + Testing numpy polyfit calls init_dgelsd(LAPACK) + """ + try: + c = array([3., 2., 1.]) + x = linspace(0, 2, 5) + y = polyval(c, x) + _ = polyfit(x, y, 2, cov=True) + except ValueError: + pass + + import sys + if sys.platform == "darwin": + with warnings.catch_warnings(record=True) as w: + _mac_os_check() + # Throw runtime error, if the test failed Check for warning and error_message + error_message = "" + if len(w) > 0: + error_message = "{}: {}".format(w[-1].category.__name__, str(w[-1].message)) + msg = ( + "Polyfit sanity test emitted a warning, most likely due " + "to using a buggy Accelerate backend. " + "If you compiled yourself, " + "see site.cfg.example for information. " + "Otherwise report this to the vendor " + "that provided NumPy.\n{}\n".format( + error_message)) + raise RuntimeError(msg) + del _mac_os_check + + # We usually use madvise hugepages support, but on some old kernels it + # is slow and thus better avoided. + # Specifically kernel version 4.6 had a bug fix which probably fixed this: + # https://github.com/torvalds/linux/commit/7cf91a98e607c2f935dbcc177d70011e95b8faff + import os + use_hugepage = os.environ.get("NUMPY_MADVISE_HUGEPAGE", None) + if sys.platform == "linux" and use_hugepage is None: + use_hugepage = 1 + kernel_version = os.uname().release.split(".")[:2] + kernel_version = tuple(int(v) for v in kernel_version) + if kernel_version < (4, 6): + use_hugepage = 0 + elif use_hugepage is None: + # This is not Linux, so it should not matter, just enable anyway + use_hugepage = 1 + else: + use_hugepage = int(use_hugepage) + + # Note that this will currently only make a difference on Linux + core.multiarray._set_madvise_hugepage(use_hugepage) diff --git a/numpy/_pytesttester.py b/numpy/_pytesttester.py index 8b6e3217e..ca86aeb22 100644 --- a/numpy/_pytesttester.py +++ b/numpy/_pytesttester.py @@ -125,13 +125,6 @@ class PytestTester: import pytest import warnings - #FIXME This is no longer needed? Assume it was for use in tests. - # cap verbosity at 3, which is equivalent to the pytest '-vv' option - #from . import utils - #verbose = min(int(verbose), 3) - #utils.verbose = verbose - # - module = sys.modules[self.module_name] module_path = os.path.abspath(module.__path__[0]) @@ -160,18 +153,9 @@ class PytestTester: # When testing matrices, ignore their PendingDeprecationWarnings pytest_args += [ "-W ignore:the matrix subclass is not", + "-W ignore:Importing from numpy.matlib is", ] - # Ignore python2.7 -3 warnings - pytest_args += [ - r"-W ignore:buffer\(\) not supported in 3\.x:DeprecationWarning", - r"-W ignore:CObject type is not supported in 3\.x:DeprecationWarning", - r"-W ignore:comparing unequal types not supported in 3\.x:DeprecationWarning", - r"-W ignore:the commands module has been removed in Python 3\.0:DeprecationWarning", - r"-W ignore:The 'new' module has been removed in Python 3\.0:DeprecationWarning", - ] - - if doctests: raise ValueError("Doctests not supported") @@ -185,7 +169,13 @@ class PytestTester: pytest_args += ["--cov=" + module_path] if label == "fast": - pytest_args += ["-m", "not slow"] + # not importing at the top level to avoid circular import of module + from numpy.testing import IS_PYPY + if IS_PYPY: + pytest_args += ["-m", "not slow and not slow_pypy"] + else: + pytest_args += ["-m", "not slow"] + elif label != "full": pytest_args += ["-m", label] diff --git a/numpy/conftest.py b/numpy/conftest.py index a843f725f..20c8a449e 100644 --- a/numpy/conftest.py +++ b/numpy/conftest.py @@ -3,6 +3,7 @@ Pytest configuration and fixtures for the Numpy test suite. """ import os +import hypothesis import pytest import numpy @@ -12,6 +13,12 @@ from numpy.core._multiarray_tests import get_fpu_mode _old_fpu_mode = None _collect_results = {} +# See https://hypothesis.readthedocs.io/en/latest/settings.html +hypothesis.settings.register_profile( + name="numpy-profile", deadline=None, print_blob=True, +) +hypothesis.settings.load_profile("numpy-profile") + def pytest_configure(config): config.addinivalue_line("markers", @@ -20,6 +27,8 @@ def pytest_configure(config): "leaks_references: Tests that are known to leak references.") config.addinivalue_line("markers", "slow: Tests that are very slow.") + config.addinivalue_line("markers", + "slow_pypy: Tests that are very slow on pypy.") def pytest_addoption(parser): diff --git a/numpy/core/__init__.py b/numpy/core/__init__.py index c2d53fe3e..c77885954 100644 --- a/numpy/core/__init__.py +++ b/numpy/core/__init__.py @@ -26,25 +26,21 @@ except ImportError as exc: IMPORTANT: PLEASE READ THIS FOR ADVICE ON HOW TO SOLVE THIS ISSUE! -Importing the numpy c-extensions failed. -- Try uninstalling and reinstalling numpy. -- If you have already done that, then: - 1. Check that you expected to use Python%d.%d from "%s", - and that you have no directories in your PATH or PYTHONPATH that can - interfere with the Python and numpy version "%s" you're trying to use. - 2. If (1) looks fine, you can open a new issue at - https://github.com/numpy/numpy/issues. Please include details on: - - how you installed Python - - how you installed numpy - - your operating system - - whether or not you have multiple versions of Python installed - - if you built from source, your compiler versions and ideally a build log - -- If you're working with a numpy git repository, try `git clean -xdf` - (removes all files not under version control) and rebuild numpy. - -Note: this error has many possible causes, so please don't comment on -an existing issue about this - open a new one instead. +Importing the numpy C-extensions failed. This error can happen for +many reasons, often due to issues with your setup or how NumPy was +installed. + +We have compiled some common reasons and troubleshooting tips at: + + https://numpy.org/devdocs/user/troubleshooting-importerror.html + +Please note and check the following: + + * The Python version is: Python%d.%d from "%s" + * The NumPy version is: "%s" + +and make sure that they are the versions you expect. +Please carefully study the documentation linked above for further help. Original error was: %s """ % (sys.version_info[0], sys.version_info[1], sys.executable, diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index cb68b8360..e54103634 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -8,7 +8,6 @@ NOTE: Many of the methods of ndarray have corresponding functions. core/fromnumeric.py, core/defmatrix.py up-to-date. """ -import sys from numpy.core import numerictypes as _numerictypes from numpy.core import dtype @@ -153,6 +152,8 @@ add_newdoc('numpy.core', 'flatiter', ('copy', add_newdoc('numpy.core', 'nditer', """ + nditer(op, flags=None, op_flags=None, op_dtypes=None, order='K', casting='safe', op_axes=None, itershape=None, buffersize=0) + Efficient multi-dimensional iterator object to iterate over arrays. To get started using this object, see the :ref:`introductory guide to array iteration <arrays.nditer>`. @@ -785,7 +786,7 @@ add_newdoc('numpy.core', 'broadcast', ('reset', add_newdoc('numpy.core.multiarray', 'array', """ - array(object, dtype=None, copy=True, order='K', subok=False, ndmin=0) + array(object, dtype=None, *, copy=True, order='K', subok=False, ndmin=0) Create an array. @@ -2026,25 +2027,22 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('ctypes', Examples -------- >>> import ctypes + >>> x = np.array([[0, 1], [2, 3]], dtype=np.int32) >>> x array([[0, 1], - [2, 3]]) + [2, 3]], dtype=int32) >>> x.ctypes.data - 30439712 - >>> x.ctypes.data_as(ctypes.POINTER(ctypes.c_long)) - <ctypes.LP_c_long object at 0x01F01300> - >>> x.ctypes.data_as(ctypes.POINTER(ctypes.c_long)).contents - c_long(0) - >>> x.ctypes.data_as(ctypes.POINTER(ctypes.c_longlong)).contents - c_longlong(4294967296L) + 31962608 # may vary + >>> x.ctypes.data_as(ctypes.POINTER(ctypes.c_uint32)) + <__main__.LP_c_uint object at 0x7ff2fc1fc200> # may vary + >>> x.ctypes.data_as(ctypes.POINTER(ctypes.c_uint32)).contents + c_uint(0) + >>> x.ctypes.data_as(ctypes.POINTER(ctypes.c_uint64)).contents + c_ulong(4294967296) >>> x.ctypes.shape - <numpy.core._internal.c_long_Array_2 object at 0x01FFD580> - >>> x.ctypes.shape_as(ctypes.c_long) - <numpy.core._internal.c_long_Array_2 object at 0x01FCE620> + <numpy.core._internal.c_long_Array_2 object at 0x7ff2fc1fce60> # may vary >>> x.ctypes.strides - <numpy.core._internal.c_long_Array_2 object at 0x01FCE620> - >>> x.ctypes.strides_as(ctypes.c_longlong) - <numpy.core._internal.c_longlong_Array_2 object at 0x01F01300> + <numpy.core._internal.c_long_Array_2 object at 0x7ff2fc1ff320> # may vary """)) @@ -2318,7 +2316,8 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('shape', >>> np.zeros((4,2))[::2].shape = (-1,) Traceback (most recent call last): File "<stdin>", line 1, in <module> - AttributeError: incompatible shape for a non-contiguous array + AttributeError: Incompatible shape for in-place modification. Use + `.reshape()` to make a copy with the desired shape. See Also -------- @@ -3933,8 +3932,8 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('tolist', """)) -tobytesdoc = """ - a.{name}(order='C') +add_newdoc('numpy.core.multiarray', 'ndarray', ('tobytes', """ + a.tobytes(order='C') Construct Python bytes containing the raw data bytes in the array. @@ -3944,11 +3943,11 @@ tobytesdoc = """ unless the F_CONTIGUOUS flag in the array is set, in which case it means 'Fortran' order. - {deprecated} + .. versionadded:: 1.9.0 Parameters ---------- - order : {{'C', 'F', None}}, optional + order : {'C', 'F', None}, optional Order of the data for multidimensional arrays: C, Fortran, or the same as for the original array. @@ -3967,18 +3966,19 @@ tobytesdoc = """ >>> x.tobytes('F') b'\\x00\\x00\\x02\\x00\\x01\\x00\\x03\\x00' - """ + """)) + + +add_newdoc('numpy.core.multiarray', 'ndarray', ('tostring', r""" + a.tostring(order='C') + + A compatibility alias for `tobytes`, with exactly the same behavior. + + Despite its name, it returns `bytes` not `str`\ s. + + .. deprecated:: 1.19.0 + """)) -add_newdoc('numpy.core.multiarray', 'ndarray', - ('tostring', tobytesdoc.format(name='tostring', - deprecated= - 'This function is a compatibility ' - 'alias for tobytes. Despite its ' - 'name it returns bytes not ' - 'strings.'))) -add_newdoc('numpy.core.multiarray', 'ndarray', - ('tobytes', tobytesdoc.format(name='tobytes', - deprecated='.. versionadded:: 1.9.0'))) add_newdoc('numpy.core.multiarray', 'ndarray', ('trace', """ @@ -4394,6 +4394,14 @@ add_newdoc('numpy.core.umath', '_add_newdoc_ufunc', and then throwing away the ufunc. """) +add_newdoc('numpy.core.multiarray', '_set_madvise_hugepage', + """ + _set_madvise_hugepage(enabled: bool) -> bool + + Set or unset use of ``madvise (2)`` MADV_HUGEPAGE support when + allocating the array data. Returns the previously set value. + See `global_state` for more information. + """) add_newdoc('numpy.core._multiarray_tests', 'format_float_OSprintf_g', """ diff --git a/numpy/core/_dtype.py b/numpy/core/_dtype.py index fa39dfcd4..6b0ec5903 100644 --- a/numpy/core/_dtype.py +++ b/numpy/core/_dtype.py @@ -3,8 +3,6 @@ A place for code to be called from the implementation of np.dtype String handling is much easier to do correctly in python. """ -import sys - import numpy as np @@ -17,18 +15,10 @@ _kind_to_stem = { 'V': 'void', 'O': 'object', 'M': 'datetime', - 'm': 'timedelta' + 'm': 'timedelta', + 'S': 'bytes', + 'U': 'str', } -if sys.version_info[0] >= 3: - _kind_to_stem.update({ - 'S': 'bytes', - 'U': 'str' - }) -else: - _kind_to_stem.update({ - 'S': 'string', - 'U': 'unicode' - }) def _kind_name(dtype): diff --git a/numpy/core/_exceptions.py b/numpy/core/_exceptions.py index 88a45561f..99172e23d 100644 --- a/numpy/core/_exceptions.py +++ b/numpy/core/_exceptions.py @@ -157,7 +157,6 @@ class _ArrayMemoryError(MemoryError): @staticmethod def _size_to_string(num_bytes): """ Convert a number of bytes into a binary size string """ - import math # https://en.wikipedia.org/wiki/Binary_prefix LOG2_STEP = 10 diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index f21774cb6..1378497bb 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -9,7 +9,6 @@ import re import sys import platform -from numpy.compat import unicode from .multiarray import dtype, array, ndarray try: import ctypes @@ -365,7 +364,7 @@ def _newnames(datatype, order): """ oldnames = datatype.names nameslist = list(oldnames) - if isinstance(order, (str, unicode)): + if isinstance(order, str): order = [order] seen = set() if isinstance(order, (list, tuple)): diff --git a/numpy/core/_methods.py b/numpy/core/_methods.py index 694523b20..86ddf4d17 100644 --- a/numpy/core/_methods.py +++ b/numpy/core/_methods.py @@ -21,6 +21,17 @@ umr_prod = um.multiply.reduce umr_any = um.logical_or.reduce umr_all = um.logical_and.reduce +# Complex types to -> (2,)float view for fast-path computation in _var() +_complex_to_float = { + nt.dtype(nt.csingle) : nt.dtype(nt.single), + nt.dtype(nt.cdouble) : nt.dtype(nt.double), +} +# Special case for windows: ensure double takes precedence +if nt.dtype(nt.longdouble) != nt.dtype(nt.double): + _complex_to_float.update({ + nt.dtype(nt.clongdouble) : nt.dtype(nt.longdouble), + }) + # avoid keyword arguments to speed up parsing, saves about 15%-20% for very # small reductions def _amax(a, axis=None, out=None, keepdims=False, @@ -52,7 +63,7 @@ def _count_reduce_items(arr, axis): axis = (axis,) items = 1 for ax in axis: - items *= arr.shape[ax] + items *= arr.shape[mu.normalize_axis_index(ax, arr.ndim)] return items # Numpy 1.17.0, 2019-02-24 @@ -189,8 +200,16 @@ def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): # Note that x may not be inexact and that we need it to be an array, # not a scalar. x = asanyarray(arr - arrmean) + if issubclass(arr.dtype.type, (nt.floating, nt.integer)): x = um.multiply(x, x, out=x) + # Fast-paths for built-in complex types + elif x.dtype in _complex_to_float: + xv = x.view(dtype=(_complex_to_float[x.dtype], (2,))) + um.multiply(xv, xv, out=xv) + x = um.add(xv[..., 0], xv[..., 1], out=x.real).real + # Most general case; includes handling object arrays containing imaginary + # numbers and complex types with non-native byteorder else: x = um.multiply(x, um.conjugate(x), out=x).real diff --git a/numpy/core/_type_aliases.py b/numpy/core/_type_aliases.py index d6e1a1fb7..c26431443 100644 --- a/numpy/core/_type_aliases.py +++ b/numpy/core/_type_aliases.py @@ -23,7 +23,6 @@ and sometimes other mappings too. """ import warnings -import sys from numpy.compat import unicode from numpy._globals import VisibleDeprecationWarning @@ -203,22 +202,16 @@ def _set_up_aliases(): ('bool_', 'bool'), ('bytes_', 'string'), ('string_', 'string'), + ('str_', 'unicode'), ('unicode_', 'unicode'), ('object_', 'object')] - if sys.version_info[0] >= 3: - type_pairs.extend([('str_', 'unicode')]) - else: - type_pairs.extend([('str_', 'string')]) for alias, t in type_pairs: allTypes[alias] = allTypes[t] sctypeDict[alias] = sctypeDict[t] # Remove aliases overriding python types and modules to_remove = ['ulong', 'object', 'int', 'float', - 'complex', 'bool', 'string', 'datetime', 'timedelta'] - if sys.version_info[0] >= 3: - to_remove.extend(['bytes', 'str']) - else: - to_remove.extend(['unicode', 'long']) + 'complex', 'bool', 'string', 'datetime', 'timedelta', + 'bytes', 'str'] for t in to_remove: try: @@ -267,11 +260,8 @@ _set_array_types() # Add additional strings to the sctypeDict -_toadd = ['int', 'float', 'complex', 'bool', 'object'] -if sys.version_info[0] >= 3: - _toadd.extend(['str', 'bytes', ('a', 'bytes_')]) -else: - _toadd.extend(['string', ('str', 'string_'), 'unicode', ('a', 'string_')]) +_toadd = ['int', 'float', 'complex', 'bool', 'object', + 'str', 'bytes', ('a', 'bytes_')] for name in _toadd: if isinstance(name, tuple): diff --git a/numpy/core/_ufunc_config.py b/numpy/core/_ufunc_config.py index 4872a5385..454d911cf 100644 --- a/numpy/core/_ufunc_config.py +++ b/numpy/core/_ufunc_config.py @@ -3,12 +3,7 @@ Functions for changing global ufunc configuration This provides helpers which wrap `umath.geterrobj` and `umath.seterrobj` """ -try: - # Accessing collections abstract classes from collections - # has been deprecated since Python 3.3 - import collections.abc as collections_abc -except ImportError: - import collections as collections_abc +import collections.abc import contextlib from .overrides import set_module @@ -307,8 +302,9 @@ def seterrcall(func): OrderedDict([('divide', 'log'), ('invalid', 'log'), ('over', 'log'), ('under', 'log')]) """ - if func is not None and not isinstance(func, collections_abc.Callable): - if not hasattr(func, 'write') or not isinstance(func.write, collections_abc.Callable): + if func is not None and not isinstance(func, collections.abc.Callable): + if (not hasattr(func, 'write') or + not isinstance(func.write, collections.abc.Callable)): raise ValueError("Only callable can be used as callback") pyvals = umath.geterrobj() old = geterrcall() diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index ec7e4261f..456ef76f0 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -31,12 +31,12 @@ except ImportError: import numpy as np from . import numerictypes as _nt -from .umath import absolute, not_equal, isnan, isinf, isfinite, isnat +from .umath import absolute, isinf, isfinite, isnat from . import multiarray from .multiarray import (array, dragon4_positional, dragon4_scientific, datetime_as_string, datetime_data, ndarray, set_legacy_print_mode) -from .fromnumeric import ravel, any +from .fromnumeric import any from .numeric import concatenate, asarray, errstate from .numerictypes import (longlong, intc, int_, float_, complex_, bool_, flexible) diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 4fa68a1f0..88dc2d90a 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -8,10 +8,11 @@ specified. """ from numpy.distutils.conv_template import process_file as process_c_file -import sys, os, re import hashlib import io - +import os +import re +import sys import textwrap from os.path import join diff --git a/numpy/core/code_generators/generate_ufunc_api.py b/numpy/core/code_generators/generate_ufunc_api.py index c32cf3e44..04c023675 100644 --- a/numpy/core/code_generators/generate_ufunc_api.py +++ b/numpy/core/code_generators/generate_ufunc_api.py @@ -3,8 +3,7 @@ import genapi import numpy_api -from genapi import \ - TypeApi, GlobalVarApi, FunctionApi, BoolValuesApi +from genapi import TypeApi, FunctionApi h_template = r""" #ifdef _UMATHMODULE diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index 7599360f5..52ae3cdd7 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -233,6 +233,7 @@ flts = 'efdg' fltsO = flts + O fltsP = flts + P cmplx = 'FDG' +cmplxvec = 'FD' cmplxO = cmplx + O cmplxP = cmplx + P inexact = flts + cmplx @@ -268,7 +269,7 @@ defdict = { Ufunc(2, 1, Zero, docstrings.get('numpy.core.umath.add'), 'PyUFunc_AdditionTypeResolver', - TD(notimes_or_obj, simd=[('avx2', ints)]), + TD(notimes_or_obj, simd=[('avx512f', cmplxvec),('avx2', ints)]), [TypeDescription('M', FullTypeDescr, 'Mm', 'M'), TypeDescription('m', FullTypeDescr, 'mm', 'm'), TypeDescription('M', FullTypeDescr, 'mM', 'M'), @@ -279,7 +280,7 @@ defdict = { Ufunc(2, 1, None, # Zero is only a unit to the right, not the left docstrings.get('numpy.core.umath.subtract'), 'PyUFunc_SubtractionTypeResolver', - TD(ints + inexact, simd=[('avx2', ints)]), + TD(ints + inexact, simd=[('avx512f', cmplxvec),('avx2', ints)]), [TypeDescription('M', FullTypeDescr, 'Mm', 'M'), TypeDescription('m', FullTypeDescr, 'mm', 'm'), TypeDescription('M', FullTypeDescr, 'MM', 'm'), @@ -290,7 +291,7 @@ defdict = { Ufunc(2, 1, One, docstrings.get('numpy.core.umath.multiply'), 'PyUFunc_MultiplicationTypeResolver', - TD(notimes_or_obj, simd=[('avx2', ints)]), + TD(notimes_or_obj, simd=[('avx512f', cmplxvec),('avx2', ints)]), [TypeDescription('m', FullTypeDescr, 'mq', 'm'), TypeDescription('m', FullTypeDescr, 'qm', 'm'), TypeDescription('m', FullTypeDescr, 'md', 'm'), @@ -298,6 +299,7 @@ defdict = { ], TD(O, f='PyNumber_Multiply'), ), +#'divide' : aliased to true_divide in umathmodule.c:initumath 'floor_divide': Ufunc(2, 1, None, # One is only a unit to the right, not the left docstrings.get('numpy.core.umath.floor_divide'), @@ -324,7 +326,7 @@ defdict = { Ufunc(1, 1, None, docstrings.get('numpy.core.umath.conjugate'), None, - TD(ints+flts+cmplx, simd=[('avx2', ints)]), + TD(ints+flts+cmplx, simd=[('avx2', ints), ('avx512f', cmplxvec)]), TD(P, f='conjugate'), ), 'fmod': @@ -339,7 +341,7 @@ defdict = { Ufunc(1, 1, None, docstrings.get('numpy.core.umath.square'), None, - TD(ints+inexact, simd=[('avx2', ints), ('fma', 'fd'), ('avx512f', 'fd')]), + TD(ints+inexact, simd=[('avx2', ints), ('fma', 'fd'), ('avx512f', 'FDfd')]), TD(O, f='Py_square'), ), 'reciprocal': @@ -377,7 +379,7 @@ defdict = { docstrings.get('numpy.core.umath.absolute'), 'PyUFunc_AbsoluteTypeResolver', TD(bints+flts+timedeltaonly, simd=[('fma', 'fd'), ('avx512f', 'fd')]), - TD(cmplx, out=('f', 'd', 'g')), + TD(cmplx, simd=[('avx512f', cmplxvec)], out=('f', 'd', 'g')), TD(O, f='PyNumber_Absolute'), ), '_arg': @@ -491,14 +493,14 @@ defdict = { Ufunc(2, 1, ReorderableNone, docstrings.get('numpy.core.umath.maximum'), 'PyUFunc_SimpleUniformOperationTypeResolver', - TD(noobj), + TD(noobj, simd=[('avx512f', 'fd')]), TD(O, f='npy_ObjectMax') ), 'minimum': Ufunc(2, 1, ReorderableNone, docstrings.get('numpy.core.umath.minimum'), 'PyUFunc_SimpleUniformOperationTypeResolver', - TD(noobj), + TD(noobj, simd=[('avx512f', 'fd')]), TD(O, f='npy_ObjectMin') ), 'clip': @@ -700,6 +702,7 @@ defdict = { None, TD('e', f='exp', astype={'e':'f'}), TD('f', simd=[('fma', 'f'), ('avx512f', 'f')]), + TD('d', simd=[('avx512f', 'd')]), TD('fdg' + cmplx, f='exp'), TD(P, f='exp'), ), @@ -1013,7 +1016,7 @@ def make_arrays(funcdict): for vt in t.simd: code2list.append(textwrap.dedent("""\ #ifdef HAVE_ATTRIBUTE_TARGET_{ISA} - if (npy_cpu_supports("{isa}")) {{ + if (NPY_CPU_HAVE({ISA})) {{ {fname}_functions[{idx}] = {type}_{fname}_{isa}; }} #endif @@ -1137,7 +1140,6 @@ def make_code(funcdict, filename): Please make changes to the code generator program (%s) **/ - #include "cpuid.h" #include "ufunc_object.h" #include "ufunc_type_resolution.h" #include "loops.h" diff --git a/numpy/core/code_generators/ufunc_docstrings.py b/numpy/core/code_generators/ufunc_docstrings.py index 33ad1502d..82cd6fb27 100644 --- a/numpy/core/code_generators/ufunc_docstrings.py +++ b/numpy/core/code_generators/ufunc_docstrings.py @@ -1835,6 +1835,17 @@ add_newdoc('numpy.core.umath', 'left_shift', >>> np.left_shift(5, [1,2,3]) array([10, 20, 40]) + Note that the dtype of the second argument may change the dtype of the + result and can lead to unexpected results in some cases (see + :ref:`Casting Rules <ufuncs.casting>`): + + >>> a = np.left_shift(np.uint8(255), 1) # Expect 254 + >>> print(a, type(a)) # Unexpected result due to upcasting + 510 <class 'numpy.int64'> + >>> b = np.left_shift(np.uint8(255), np.uint8(1)) + >>> print(b, type(b)) + 254 <class 'numpy.uint8'> + """) add_newdoc('numpy.core.umath', 'less', @@ -3835,14 +3846,11 @@ add_newdoc('numpy.core.umath', 'true_divide', >>> np.true_divide(x, 4) array([ 0. , 0.25, 0.5 , 0.75, 1. ]) - >>> x//4 - array([0, 0, 0, 0, 1]) - >>> x/4 array([ 0. , 0.25, 0.5 , 0.75, 1. ]) + >>> x//4 array([0, 0, 0, 0, 1]) - """) add_newdoc('numpy.core.umath', 'frexp', diff --git a/numpy/core/defchararray.py b/numpy/core/defchararray.py index 1cfdc55c0..cd01c0e77 100644 --- a/numpy/core/defchararray.py +++ b/numpy/core/defchararray.py @@ -17,13 +17,14 @@ The preferred alias for `defchararray` is `numpy.char`. """ import functools import sys -from .numerictypes import string_, unicode_, integer, object_, bool_, character +from .numerictypes import ( + string_, unicode_, integer, int_, object_, bool_, character) from .numeric import ndarray, compare_chararrays from .numeric import array as narray from numpy.core.multiarray import _vec_string from numpy.core.overrides import set_module from numpy.core import overrides -from numpy.compat import asbytes, long +from numpy.compat import asbytes import numpy __all__ = [ @@ -274,9 +275,12 @@ def str_len(a): See also -------- - __builtin__.len + builtins.len """ - return _vec_string(a, integer, '__len__') + # Note: __len__, etc. currently return ints, which are not C-integers. + # Generally intp would be expected for lengths, although int is sufficient + # due to the dtype itemsize limitation. + return _vec_string(a, int_, '__len__') @array_function_dispatch(_binary_op_dispatcher) @@ -336,7 +340,7 @@ def multiply(a, i): i_arr = numpy.asarray(i) if not issubclass(i_arr.dtype.type, integer): raise ValueError("Can only multiply by integers") - out_size = _get_num_chars(a_arr) * max(long(i_arr.max()), 0) + out_size = _get_num_chars(a_arr) * max(int(i_arr.max()), 0) return _vec_string( a_arr, (a_arr.dtype.type, out_size), '__mul__', (i_arr,)) @@ -446,7 +450,7 @@ def center(a, width, fillchar=' '): """ a_arr = numpy.asarray(a) width_arr = numpy.asarray(width) - size = long(numpy.max(width_arr.flat)) + size = int(numpy.max(width_arr.flat)) if numpy.issubdtype(a_arr.dtype, numpy.string_): fillchar = asbytes(fillchar) return _vec_string( @@ -500,7 +504,7 @@ def count(a, sub, start=0, end=None): array([1, 0, 0]) """ - return _vec_string(a, integer, 'count', [sub, start] + _clean_args(end)) + return _vec_string(a, int_, 'count', [sub, start] + _clean_args(end)) def _code_dispatcher(a, encoding=None, errors=None): @@ -710,7 +714,7 @@ def find(a, sub, start=0, end=None): """ return _vec_string( - a, integer, 'find', [sub, start] + _clean_args(end)) + a, int_, 'find', [sub, start] + _clean_args(end)) @array_function_dispatch(_count_dispatcher) @@ -739,7 +743,7 @@ def index(a, sub, start=0, end=None): """ return _vec_string( - a, integer, 'index', [sub, start] + _clean_args(end)) + a, int_, 'index', [sub, start] + _clean_args(end)) @array_function_dispatch(_unary_op_dispatcher) @@ -991,7 +995,7 @@ def ljust(a, width, fillchar=' '): """ a_arr = numpy.asarray(a) width_arr = numpy.asarray(width) - size = long(numpy.max(width_arr.flat)) + size = int(numpy.max(width_arr.flat)) if numpy.issubdtype(a_arr.dtype, numpy.string_): fillchar = asbytes(fillchar) return _vec_string( @@ -1199,7 +1203,7 @@ def rfind(a, sub, start=0, end=None): """ return _vec_string( - a, integer, 'rfind', [sub, start] + _clean_args(end)) + a, int_, 'rfind', [sub, start] + _clean_args(end)) @array_function_dispatch(_count_dispatcher) @@ -1229,7 +1233,7 @@ def rindex(a, sub, start=0, end=None): """ return _vec_string( - a, integer, 'rindex', [sub, start] + _clean_args(end)) + a, int_, 'rindex', [sub, start] + _clean_args(end)) @array_function_dispatch(_just_dispatcher) @@ -1261,7 +1265,7 @@ def rjust(a, width, fillchar=' '): """ a_arr = numpy.asarray(a) width_arr = numpy.asarray(width) - size = long(numpy.max(width_arr.flat)) + size = int(numpy.max(width_arr.flat)) if numpy.issubdtype(a_arr.dtype, numpy.string_): fillchar = asbytes(fillchar) return _vec_string( @@ -1729,7 +1733,7 @@ def zfill(a, width): """ a_arr = numpy.asarray(a) width_arr = numpy.asarray(width) - size = long(numpy.max(width_arr.flat)) + size = int(numpy.max(width_arr.flat)) return _vec_string( a_arr, (a_arr.dtype.type, size), 'zfill', (width_arr,)) @@ -1908,7 +1912,7 @@ class chararray(ndarray): unicode : bool, optional Are the array elements of type unicode (True) or string (False). Default is False. - buffer : int, optional + buffer : object exposing the buffer interface or str, optional Memory address of the start of the array data. Default is None, in which case a new array is created. offset : int, optional @@ -1948,10 +1952,10 @@ class chararray(ndarray): else: dtype = string_ - # force itemsize to be a Python long, since using NumPy integer + # force itemsize to be a Python int, since using NumPy integer # types results in itemsize.itemsize being used as the size of # strings in the new array. - itemsize = long(itemsize) + itemsize = int(itemsize) if isinstance(buffer, str): # unicode objects do not have the buffer interface @@ -2679,25 +2683,6 @@ def array(obj, itemsize=None, copy=True, unicode=None, order=None): itemsize = len(obj) shape = len(obj) // itemsize - if unicode: - if sys.maxunicode == 0xffff: - # On a narrow Python build, the buffer for Unicode - # strings is UCS2, which doesn't match the buffer for - # NumPy Unicode types, which is ALWAYS UCS4. - # Therefore, we need to convert the buffer. On Python - # 2.6 and later, we can use the utf_32 codec. Earlier - # versions don't have that codec, so we convert to a - # numerical array that matches the input buffer, and - # then use NumPy to convert it to UCS4. All of this - # should happen in native endianness. - obj = obj.encode('utf_32') - else: - obj = str(obj) - else: - # Let the default Unicode -> string encoding (if any) take - # precedence. - obj = bytes(obj) - return chararray(shape, itemsize=itemsize, unicode=unicode, buffer=obj, order=order) @@ -2735,7 +2720,7 @@ def array(obj, itemsize=None, copy=True, unicode=None, order=None): (itemsize != obj.itemsize) or (not unicode and isinstance(obj, unicode_)) or (unicode and isinstance(obj, string_))): - obj = obj.astype((dtype, long(itemsize))) + obj = obj.astype((dtype, int(itemsize))) return obj if isinstance(obj, ndarray) and issubclass(obj.dtype.type, object): diff --git a/numpy/core/einsumfunc.py b/numpy/core/einsumfunc.py index ec3eb19d2..a1e2efdb4 100644 --- a/numpy/core/einsumfunc.py +++ b/numpy/core/einsumfunc.py @@ -4,7 +4,6 @@ Implementation of optimized einsum. """ import itertools -from numpy.compat import basestring from numpy.core.multiarray import c_einsum from numpy.core.numeric import asanyarray, tensordot from numpy.core.overrides import array_function_dispatch @@ -550,7 +549,7 @@ def _parse_einsum_input(operands): if len(operands) == 0: raise ValueError("No input operands") - if isinstance(operands[0], basestring): + if isinstance(operands[0], str): subscripts = operands[0].replace(" ", "") operands = [asanyarray(v) for v in operands[1:]] @@ -820,7 +819,7 @@ def einsum_path(*operands, optimize='greedy', einsum_call=False): memory_limit = None # No optimization or a named path algorithm - if (path_type is False) or isinstance(path_type, basestring): + if (path_type is False) or isinstance(path_type, str): pass # Given an explicit path @@ -828,7 +827,7 @@ def einsum_path(*operands, optimize='greedy', einsum_call=False): pass # Path tuple with memory limit - elif ((len(path_type) == 2) and isinstance(path_type[0], basestring) and + elif ((len(path_type) == 2) and isinstance(path_type[0], str) and isinstance(path_type[1], (int, float))): memory_limit = int(path_type[1]) path_type = path_type[0] @@ -985,8 +984,7 @@ def einsum_path(*operands, optimize='greedy', einsum_call=False): def _einsum_dispatcher(*operands, out=None, optimize=None, **kwargs): # Arguably we dispatch on more arguments that we really should; see note in # _einsum_path_dispatcher for why. - for op in operands: - yield op + yield from operands yield out diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 213a4438f..b32ad8d35 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -6,7 +6,6 @@ import types import warnings import numpy as np -from .. import VisibleDeprecationWarning from . import multiarray as mu from . import overrides from . import umath as um @@ -253,7 +252,8 @@ def reshape(a, newshape, order='C'): >>> c.shape = (20) Traceback (most recent call last): ... - AttributeError: incompatible shape for a non-contiguous array + AttributeError: Incompatible shape for in-place modification. Use + `.reshape()` to make a copy with the desired shape. The `order` keyword gives the index ordering both for *fetching* the values from `a`, and then *placing* the values into the output array. @@ -301,8 +301,7 @@ def reshape(a, newshape, order='C'): def _choose_dispatcher(a, choices, out=None, mode=None): yield a - for c in choices: - yield c + yield from choices yield out @@ -1447,7 +1446,8 @@ def squeeze(a, axis=None): squeezed : ndarray The input array, but with all or a subset of the dimensions of length 1 removed. This is always `a` itself - or a view into `a`. + or a view into `a`. Note that if all axes are squeezed, + the result is a 0d array and not a scalar. Raises ------ @@ -1474,6 +1474,15 @@ def squeeze(a, axis=None): ValueError: cannot select an axis to squeeze out which has size not equal to one >>> np.squeeze(x, axis=2).shape (1, 3) + >>> x = np.array([[1234]]) + >>> x.shape + (1, 1) + >>> np.squeeze(x) + array(1234) # 0d array + >>> np.squeeze(x).shape + () + >>> np.squeeze(x)[()] + 1234 """ try: @@ -2031,8 +2040,8 @@ def clip(a, a_min, a_max, out=None, **kwargs): is specified, values smaller than 0 become 0, and values larger than 1 become 1. - Equivalent to but faster than ``np.maximum(a_min, np.minimum(a, a_max))`` - assuming ``a_min < a_max``. + Equivalent to but faster than ``np.minimum(a_max, np.maximum(a, a_min))``. + No check is performed to ensure ``a_min < a_max``. Parameters diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 8f92a4f71..9e46f0ea5 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -4,8 +4,7 @@ import operator import types from . import numeric as _nx -from .numeric import (result_type, NaN, shares_memory, MAY_SHARE_BOUNDS, - TooHardError, asanyarray, ndim) +from .numeric import result_type, NaN, asanyarray, ndim from numpy.core.multiarray import add_docstring from numpy.core import overrides @@ -111,13 +110,7 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, >>> plt.show() """ - try: - num = operator.index(num) - except TypeError: - raise TypeError( - "object of type {} cannot be safely interpreted as an integer." - .format(type(num))) - + num = operator.index(num) if num < 0: raise ValueError("Number of samples, %s, must be non-negative." % num) div = (num - 1) if endpoint else num diff --git a/numpy/core/getlimits.py b/numpy/core/getlimits.py index b00ef64bd..e2ff49393 100644 --- a/numpy/core/getlimits.py +++ b/numpy/core/getlimits.py @@ -300,12 +300,13 @@ class finfo: bits : int The number of bits occupied by the type. eps : float - The smallest representable positive number such that - ``1.0 + eps != 1.0``. Type of `eps` is an appropriate floating - point type. - epsneg : floating point number of the appropriate type - The smallest representable positive number such that - ``1.0 - epsneg != 1.0``. + The difference between 1.0 and the next smallest representable float + larger than 1.0. For example, for 64-bit binary floats in the IEEE-754 + standard, ``eps = 2**-52``, approximately 2.22e-16. + epsneg : float + The difference between 1.0 and the next smallest representable float + less than 1.0. For example, for 64-bit binary floats in the IEEE-754 + standard, ``epsneg = 2**-53``, approximately 1.11e-16. iexp : int The number of bits in the exponent portion of the floating point representation. @@ -348,6 +349,8 @@ class finfo: -------- MachAr : The implementation of the tests that produce this information. iinfo : The equivalent for integer data types. + spacing : The distance between a value and the nearest adjacent number + nextafter : The next floating point value after x1 towards x2 Notes ----- diff --git a/numpy/core/include/numpy/arrayscalars.h b/numpy/core/include/numpy/arrayscalars.h index 64450e713..42a0df76a 100644 --- a/numpy/core/include/numpy/arrayscalars.h +++ b/numpy/core/include/numpy/arrayscalars.h @@ -135,7 +135,13 @@ typedef struct { } PyScalarObject; #define PyStringScalarObject PyStringObject -#define PyUnicodeScalarObject PyUnicodeObject +#define PyStringScalarObject PyStringObject +typedef struct { + /* note that the PyObject_HEAD macro lives right here */ + PyUnicodeObject base; + Py_UCS4 *obval; +} PyUnicodeScalarObject; + typedef struct { PyObject_VAR_HEAD diff --git a/numpy/core/include/numpy/ndarrayobject.h b/numpy/core/include/numpy/ndarrayobject.h index b18d75f35..5ef1f10aa 100644 --- a/numpy/core/include/numpy/ndarrayobject.h +++ b/numpy/core/include/numpy/ndarrayobject.h @@ -214,7 +214,7 @@ PyArray_DiscardWritebackIfCopy(PyArrayObject *arr) /* Check to see if this key in the dictionary is the "title" entry of the tuple (i.e. a duplicate dictionary entry in the fields - dict. + dict). */ static NPY_INLINE int diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index bec6fcf30..5b7e8952e 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1448,9 +1448,8 @@ PyArrayNeighborhoodIter_Next2D(PyArrayNeighborhoodIterObject* iter); * checking of correctness when working with these objects in C. */ -#define PyArray_ISONESEGMENT(m) (PyArray_NDIM(m) == 0 || \ - PyArray_CHKFLAGS(m, NPY_ARRAY_C_CONTIGUOUS) || \ - PyArray_CHKFLAGS(m, NPY_ARRAY_F_CONTIGUOUS)) +#define PyArray_ISONESEGMENT(m) (PyArray_CHKFLAGS(m, NPY_ARRAY_C_CONTIGUOUS) || \ + PyArray_CHKFLAGS(m, NPY_ARRAY_F_CONTIGUOUS)) #define PyArray_ISFORTRAN(m) (PyArray_CHKFLAGS(m, NPY_ARRAY_F_CONTIGUOUS) && \ (!PyArray_CHKFLAGS(m, NPY_ARRAY_C_CONTIGUOUS))) diff --git a/numpy/core/include/numpy/npy_math.h b/numpy/core/include/numpy/npy_math.h index 69e690f28..a07f49501 100644 --- a/numpy/core/include/numpy/npy_math.h +++ b/numpy/core/include/numpy/npy_math.h @@ -114,53 +114,6 @@ NPY_INLINE static float __npy_nzerof(void) #define NPY_SQRT1_2l 0.707106781186547524400844362104849039L /* 1/sqrt(2) */ /* - * Constants used in vector implementation of exp(x) - */ -#define NPY_RINT_CVT_MAGICf 0x1.800000p+23f -#define NPY_CODY_WAITE_LOGE_2_HIGHf -6.93145752e-1f -#define NPY_CODY_WAITE_LOGE_2_LOWf -1.42860677e-6f -#define NPY_COEFF_P0_EXPf 9.999999999980870924916e-01f -#define NPY_COEFF_P1_EXPf 7.257664613233124478488e-01f -#define NPY_COEFF_P2_EXPf 2.473615434895520810817e-01f -#define NPY_COEFF_P3_EXPf 5.114512081637298353406e-02f -#define NPY_COEFF_P4_EXPf 6.757896990527504603057e-03f -#define NPY_COEFF_P5_EXPf 5.082762527590693718096e-04f -#define NPY_COEFF_Q0_EXPf 1.000000000000000000000e+00f -#define NPY_COEFF_Q1_EXPf -2.742335390411667452936e-01f -#define NPY_COEFF_Q2_EXPf 2.159509375685829852307e-02f - -/* - * Constants used in vector implementation of log(x) - */ -#define NPY_COEFF_P0_LOGf 0.000000000000000000000e+00f -#define NPY_COEFF_P1_LOGf 9.999999999999998702752e-01f -#define NPY_COEFF_P2_LOGf 2.112677543073053063722e+00f -#define NPY_COEFF_P3_LOGf 1.480000633576506585156e+00f -#define NPY_COEFF_P4_LOGf 3.808837741388407920751e-01f -#define NPY_COEFF_P5_LOGf 2.589979117907922693523e-02f -#define NPY_COEFF_Q0_LOGf 1.000000000000000000000e+00f -#define NPY_COEFF_Q1_LOGf 2.612677543073109236779e+00f -#define NPY_COEFF_Q2_LOGf 2.453006071784736363091e+00f -#define NPY_COEFF_Q3_LOGf 9.864942958519418960339e-01f -#define NPY_COEFF_Q4_LOGf 1.546476374983906719538e-01f -#define NPY_COEFF_Q5_LOGf 5.875095403124574342950e-03f -/* - * Constants used in vector implementation of sinf/cosf(x) - */ -#define NPY_TWO_O_PIf 0x1.45f306p-1f -#define NPY_CODY_WAITE_PI_O_2_HIGHf -0x1.921fb0p+00f -#define NPY_CODY_WAITE_PI_O_2_MEDf -0x1.5110b4p-22f -#define NPY_CODY_WAITE_PI_O_2_LOWf -0x1.846988p-48f -#define NPY_COEFF_INVF0_COSINEf 0x1.000000p+00f -#define NPY_COEFF_INVF2_COSINEf -0x1.000000p-01f -#define NPY_COEFF_INVF4_COSINEf 0x1.55553cp-05f -#define NPY_COEFF_INVF6_COSINEf -0x1.6c06dcp-10f -#define NPY_COEFF_INVF8_COSINEf 0x1.98e616p-16f -#define NPY_COEFF_INVF3_SINEf -0x1.555556p-03f -#define NPY_COEFF_INVF5_SINEf 0x1.11119ap-07f -#define NPY_COEFF_INVF7_SINEf -0x1.a06bbap-13f -#define NPY_COEFF_INVF9_SINEf 0x1.7d3bbcp-19f -/* * Integer functions. */ NPY_INPLACE npy_uint npy_gcdu(npy_uint a, npy_uint b); diff --git a/numpy/core/memmap.py b/numpy/core/memmap.py index ad0d7ad79..cb025736e 100644 --- a/numpy/core/memmap.py +++ b/numpy/core/memmap.py @@ -1,7 +1,7 @@ import numpy as np from .numeric import uint8, ndarray, dtype from numpy.compat import ( - long, basestring, os_fspath, contextlib_nullcontext, is_pathlib_path + os_fspath, contextlib_nullcontext, is_pathlib_path ) from numpy.core.overrides import set_module @@ -209,10 +209,12 @@ class memmap(ndarray): import os.path try: mode = mode_equivalents[mode] - except KeyError: + except KeyError as e: if mode not in valid_filemodes: - raise ValueError("mode must be one of %s" % - (valid_filemodes + list(mode_equivalents.keys()))) + raise ValueError( + "mode must be one of {!r} (got {!r})" + .format(valid_filemodes + list(mode_equivalents.keys()), mode) + ) from None if mode == 'w+' and shape is None: raise ValueError("shape must be given") @@ -242,7 +244,7 @@ class memmap(ndarray): for k in shape: size *= k - bytes = long(offset + size*_dbytes) + bytes = int(offset + size*_dbytes) if mode in ('w+', 'r+') and flen < bytes: fid.seek(bytes - 1, 0) @@ -271,7 +273,7 @@ class memmap(ndarray): # special case - if we were constructed with a pathlib.path, # then filename is a path object, not a string self.filename = filename.resolve() - elif hasattr(fid, "name") and isinstance(fid.name, basestring): + elif hasattr(fid, "name") and isinstance(fid.name, str): # py3 returns int for TemporaryFile().name self.filename = os.path.abspath(fid.name) # same as memmap copies (e.g. memmap + 1) diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 5749afdcc..ec36f4f7e 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -7,17 +7,17 @@ by importing from the extension module. """ import functools -import sys import warnings -import sys from . import overrides from . import _multiarray_umath -import numpy as np -from numpy.core._multiarray_umath import * -from numpy.core._multiarray_umath import ( +from ._multiarray_umath import * # noqa: F403 +# These imports are needed for backward compatibility, +# do not change them. issue gh-15518 +# _get_ndarray_c_version is semi-public, on purpose not added to __all__ +from ._multiarray_umath import ( _fastCopyAndTranspose, _flagdict, _insert, _reconstruct, _vec_string, - _ARRAY_API, _monotonicity, _get_ndarray_c_version + _ARRAY_API, _monotonicity, _get_ndarray_c_version, _set_madvise_hugepage, ) __all__ = [ @@ -909,8 +909,9 @@ def bincount(x, weights=None, minlength=None): >>> np.bincount(np.arange(5, dtype=float)) Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: array cannot be safely cast to required type + ... + TypeError: Cannot cast array data from dtype('float64') to dtype('int64') + according to the rule 'safe' A possible use of ``bincount`` is to perform sums over variable-size chunks of an array, using the ``weights`` keyword. diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 72c6089b8..83d985a7c 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -4,10 +4,8 @@ import operator import sys import warnings import numbers -import contextlib import numpy as np -from numpy.compat import pickle, basestring from . import multiarray from .multiarray import ( _fastCopyAndTranspose as fastCopyAndTranspose, ALLOW_THREADS, @@ -386,12 +384,12 @@ def full_like(a, fill_value, dtype=None, order='K', subok=True, shape=None): return res -def _count_nonzero_dispatcher(a, axis=None): +def _count_nonzero_dispatcher(a, axis=None, *, keepdims=None): return (a,) @array_function_dispatch(_count_nonzero_dispatcher) -def count_nonzero(a, axis=None): +def count_nonzero(a, axis=None, *, keepdims=False): """ Counts the number of non-zero values in the array ``a``. @@ -416,6 +414,13 @@ def count_nonzero(a, axis=None): .. versionadded:: 1.12.0 + keepdims : bool, optional + If this is set to True, the axes that are counted are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the input array. + + .. versionadded:: 1.19.0 + Returns ------- count : int or array of int @@ -431,15 +436,19 @@ def count_nonzero(a, axis=None): -------- >>> np.count_nonzero(np.eye(4)) 4 - >>> np.count_nonzero([[0,1,7,0,0],[3,0,0,2,19]]) + >>> a = np.array([[0, 1, 7, 0], + ... [3, 0, 2, 19]]) + >>> np.count_nonzero(a) 5 - >>> np.count_nonzero([[0,1,7,0,0],[3,0,0,2,19]], axis=0) - array([1, 1, 1, 1, 1]) - >>> np.count_nonzero([[0,1,7,0,0],[3,0,0,2,19]], axis=1) + >>> np.count_nonzero(a, axis=0) + array([1, 1, 2, 1]) + >>> np.count_nonzero(a, axis=1) array([2, 3]) - + >>> np.count_nonzero(a, axis=1, keepdims=True) + array([[2], + [3]]) """ - if axis is None: + if axis is None and not keepdims: return multiarray.count_nonzero(a) a = asanyarray(a) @@ -450,7 +459,7 @@ def count_nonzero(a, axis=None): else: a_bool = a.astype(np.bool_, copy=False) - return a_bool.sum(axis=axis, dtype=np.intp) + return a_bool.sum(axis=axis, dtype=np.intp, keepdims=keepdims) @set_module('numpy') @@ -626,7 +635,7 @@ _mode_from_name_dict = {'v': 0, def _mode_from_name(mode): - if isinstance(mode, basestring): + if isinstance(mode, str): return _mode_from_name_dict[mode.lower()[0]] return mode diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index c06552c4e..aac741612 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -83,7 +83,6 @@ import types as _types import numbers import warnings -from numpy.compat import bytes, long from numpy.core.multiarray import ( typeinfo, ndarray, array, empty, dtype, datetime_data, datetime_as_string, busday_offset, busday_count, is_busday, @@ -119,8 +118,8 @@ from ._dtype import _kind_name # we don't export these for import *, but we do want them accessible # as numerictypes.bool, etc. -from builtins import bool, int, float, complex, object, str -unicode = str +from builtins import bool, int, float, complex, object, str, bytes +from numpy.compat import long, unicode # We use this later @@ -313,9 +312,11 @@ def issubclass_(arg1, arg2): Examples -------- >>> np.issubclass_(np.int32, int) - False # True on Python 2.7 + False >>> np.issubclass_(np.int32, float) False + >>> np.issubclass_(np.float64, float) + True """ try: @@ -386,35 +387,7 @@ def issubdtype(arg1, arg2): if not issubclass_(arg1, generic): arg1 = dtype(arg1).type if not issubclass_(arg2, generic): - arg2_orig = arg2 arg2 = dtype(arg2).type - if not isinstance(arg2_orig, dtype): - # weird deprecated behaviour, that tried to infer np.floating from - # float, and similar less obvious things, such as np.generic from - # basestring - mro = arg2.mro() - arg2 = mro[1] if len(mro) > 1 else mro[0] - - def type_repr(x): - """ Helper to produce clear error messages """ - if not isinstance(x, type): - return repr(x) - elif issubclass(x, generic): - return "np.{}".format(x.__name__) - else: - return x.__name__ - - # 1.14, 2017-08-01 - warnings.warn( - "Conversion of the second argument of issubdtype from `{raw}` " - "to `{abstract}` is deprecated. In future, it will be treated " - "as `{concrete} == np.dtype({raw}).type`.".format( - raw=type_repr(arg2_orig), - abstract=type_repr(arg2), - concrete=type_repr(dtype(arg2_orig).type) - ), - FutureWarning, stacklevel=2 - ) return issubclass(arg1, arg2) diff --git a/numpy/core/records.py b/numpy/core/records.py index d4aa2feb9..04c970cf4 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -33,7 +33,6 @@ Record arrays allow us to access fields as properties:: array([2., 2.]) """ -import sys import os import warnings from collections import Counter, OrderedDict @@ -41,7 +40,7 @@ from collections import Counter, OrderedDict from . import numeric as sb from . import numerictypes as nt from numpy.compat import ( - isfileobj, bytes, long, unicode, os_fspath, contextlib_nullcontext + isfileobj, os_fspath, contextlib_nullcontext ) from numpy.core.overrides import set_module from .arrayprint import get_printoptions @@ -158,8 +157,7 @@ class format_parser: def __init__(self, formats, names, titles, aligned=False, byteorder=None): self._parseFormats(formats, aligned) self._setfieldnames(names, titles) - self._createdescr(byteorder) - self.dtype = self._descr + self._createdtype(byteorder) def _parseFormats(self, formats, aligned=False): """ Parse the field formats """ @@ -186,10 +184,10 @@ class format_parser: """convert input field names into a list and assign to the _names attribute """ - if (names): - if (type(names) in [list, tuple]): + if names: + if type(names) in [list, tuple]: pass - elif isinstance(names, (str, unicode)): + elif isinstance(names, str): names = names.split(',') else: raise NameError("illegal input names %s" % repr(names)) @@ -209,25 +207,28 @@ class format_parser: if _dup: raise ValueError("Duplicate field names: %s" % _dup) - if (titles): + if titles: self._titles = [n.strip() for n in titles[:self._nfields]] else: self._titles = [] titles = [] - if (self._nfields > len(titles)): + if self._nfields > len(titles): self._titles += [None] * (self._nfields - len(titles)) - def _createdescr(self, byteorder): - descr = sb.dtype({'names':self._names, - 'formats':self._f_formats, - 'offsets':self._offsets, - 'titles':self._titles}) - if (byteorder is not None): + def _createdtype(self, byteorder): + dtype = sb.dtype({ + 'names': self._names, + 'formats': self._f_formats, + 'offsets': self._offsets, + 'titles': self._titles, + }) + if byteorder is not None: byteorder = _byteorderconv[byteorder[0]] - descr = descr.newbyteorder(byteorder) + dtype = dtype.newbyteorder(byteorder) + + self.dtype = dtype - self._descr = descr class record(nt.void): """A data-type scalar that allows field access as attribute lookup. @@ -249,7 +250,7 @@ class record(nt.void): return super(record, self).__str__() def __getattribute__(self, attr): - if attr in ['setfield', 'getfield', 'dtype']: + if attr in ('setfield', 'getfield', 'dtype'): return nt.void.__getattribute__(self, attr) try: return nt.void.__getattribute__(self, attr) @@ -274,7 +275,7 @@ class record(nt.void): "attribute '%s'" % attr) def __setattr__(self, attr, val): - if attr in ['setfield', 'getfield', 'dtype']: + if attr in ('setfield', 'getfield', 'dtype'): raise AttributeError("Cannot set '%s' attribute" % attr) fielddict = nt.void.__getattribute__(self, 'dtype').fields res = fielddict.get(attr, None) @@ -431,7 +432,7 @@ class recarray(ndarray): if dtype is not None: descr = sb.dtype(dtype) else: - descr = format_parser(formats, names, titles, aligned, byteorder)._descr + descr = format_parser(formats, names, titles, aligned, byteorder).dtype if buf is None: self = ndarray.__new__(subtype, shape, (record, descr), order=order) @@ -532,8 +533,7 @@ class recarray(ndarray): def __repr__(self): repr_dtype = self.dtype - if (self.dtype.type is record - or (not issubclass(self.dtype.type, nt.void))): + if self.dtype.type is record or not issubclass(self.dtype.type, nt.void): # If this is a full record array (has numpy.record dtype), # or if it has a scalar (non-void) dtype with no records, # represent it using the rec.array function. Since rec.array @@ -595,8 +595,30 @@ def _deprecate_shape_0_as_None(shape): def fromarrays(arrayList, dtype=None, shape=None, formats=None, names=None, titles=None, aligned=False, byteorder=None): - """ create a record array from a (flat) list of arrays + """Create a record array from a (flat) list of arrays + + Parameters + ---------- + arrayList : list or tuple + List of array-like objects (such as lists, tuples, + and ndarrays). + dtype : data-type, optional + valid dtype for all arrays + shape : int or tuple of ints, optional + Shape of the resulting array. If not provided, inferred from + ``arrayList[0]``. + formats, names, titles, aligned, byteorder : + If `dtype` is ``None``, these arguments are passed to + `numpy.format_parser` to construct a dtype. See that function for + detailed documentation. + + Returns + ------- + np.recarray + Record array consisting of given arrayList columns. + Examples + -------- >>> x1=np.array([1,2,3,4]) >>> x2=np.array(['a','dd','xyz','12']) >>> x3=np.array([1.1,2,3,4]) @@ -606,6 +628,17 @@ def fromarrays(arrayList, dtype=None, shape=None, formats=None, >>> x1[1]=34 >>> r.a array([1, 2, 3, 4]) + + >>> x1 = np.array([1, 2, 3, 4]) + >>> x2 = np.array(['a', 'dd', 'xyz', '12']) + >>> x3 = np.array([1.1, 2, 3,4]) + >>> r = np.core.records.fromarrays( + ... [x1, x2, x3], + ... dtype=np.dtype([('a', np.int32), ('b', 'S3'), ('c', np.float32)])) + >>> r + rec.array([(1, b'a', 1.1), (2, b'dd', 2. ), (3, b'xyz', 3. ), + (4, b'12', 4. )], + dtype=[('a', '<i4'), ('b', 'S3'), ('c', '<f4')]) """ arrayList = [sb.asarray(x) for x in arrayList] @@ -621,17 +654,13 @@ def fromarrays(arrayList, dtype=None, shape=None, formats=None, if formats is None and dtype is None: # go through each object in the list to see if it is an ndarray # and determine the formats. - formats = [] - for obj in arrayList: - formats.append(obj.dtype) + formats = [obj.dtype for obj in arrayList] if dtype is not None: descr = sb.dtype(dtype) - _names = descr.names else: - parsed = format_parser(formats, names, titles, aligned, byteorder) - _names = parsed._names - descr = parsed._descr + descr = format_parser(formats, names, titles, aligned, byteorder).dtype + _names = descr.names # Determine shape from data-type. if len(descr) != len(arrayList): @@ -659,20 +688,33 @@ def fromarrays(arrayList, dtype=None, shape=None, formats=None, def fromrecords(recList, dtype=None, shape=None, formats=None, names=None, titles=None, aligned=False, byteorder=None): - """ create a recarray from a list of records in text form - - The data in the same field can be heterogeneous, they will be promoted - to the highest data type. This method is intended for creating - smaller record arrays. If used to create large array without formats - defined - - r=fromrecords([(2,3.,'abc')]*100000) + """Create a recarray from a list of records in text form. - it can be slow. + Parameters + ---------- + recList : sequence + data in the same field may be heterogeneous - they will be promoted + to the highest data type. + dtype : data-type, optional + valid dtype for all arrays + shape : int or tuple of ints, optional + shape of each array. + formats, names, titles, aligned, byteorder : + If `dtype` is ``None``, these arguments are passed to + `numpy.format_parser` to construct a dtype. See that function for + detailed documentation. + + If both `formats` and `dtype` are None, then this will auto-detect + formats. Use list of tuples rather than list of lists for faster + processing. - If formats is None, then this will auto-detect formats. Use list of - tuples rather than list of lists for faster processing. + Returns + ------- + np.recarray + record array consisting of given recList rows. + Examples + -------- >>> r=np.core.records.fromrecords([(456,'dbe',1.2),(2,'de',1.3)], ... names='col1,col2,col3') >>> print(r[0]) @@ -696,7 +738,7 @@ def fromrecords(recList, dtype=None, shape=None, formats=None, names=None, if dtype is not None: descr = sb.dtype((record, dtype)) else: - descr = format_parser(formats, names, titles, aligned, byteorder)._descr + descr = format_parser(formats, names, titles, aligned, byteorder).dtype try: retval = sb.array(recList, dtype=descr) @@ -705,7 +747,7 @@ def fromrecords(recList, dtype=None, shape=None, formats=None, names=None, shape = _deprecate_shape_0_as_None(shape) if shape is None: shape = len(recList) - if isinstance(shape, (int, long)): + if isinstance(shape, int): shape = (shape,) if len(shape) > 1: raise ValueError("Can only deal with 1-d array.") @@ -730,7 +772,7 @@ def fromrecords(recList, dtype=None, shape=None, formats=None, names=None, def fromstring(datastring, dtype=None, shape=None, offset=0, formats=None, names=None, titles=None, aligned=False, byteorder=None): - """ create a (read-only) record array from binary data contained in + """Create a (read-only) record array from binary data contained in a string""" if dtype is None and formats is None: @@ -739,14 +781,14 @@ def fromstring(datastring, dtype=None, shape=None, offset=0, formats=None, if dtype is not None: descr = sb.dtype(dtype) else: - descr = format_parser(formats, names, titles, aligned, byteorder)._descr + descr = format_parser(formats, names, titles, aligned, byteorder).dtype itemsize = descr.itemsize # NumPy 1.19.0, 2020-01-01 shape = _deprecate_shape_0_as_None(shape) - if shape is None or shape == -1: + if shape in (None, -1): shape = (len(datastring) - offset) // itemsize _array = recarray(shape, descr, buf=datastring, offset=offset) @@ -765,10 +807,30 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, names=None, titles=None, aligned=False, byteorder=None): """Create an array from binary file data - If file is a string or a path-like object then that file is opened, - else it is assumed to be a file object. The file object must - support random access (i.e. it must have tell and seek methods). + Parameters + ---------- + fd : str or file type + If file is a string or a path-like object then that file is opened, + else it is assumed to be a file object. The file object must + support random access (i.e. it must have tell and seek methods). + dtype : data-type, optional + valid dtype for all arrays + shape : int or tuple of ints, optional + shape of each array. + offset : int, optional + Position in the file to start reading from. + formats, names, titles, aligned, byteorder : + If `dtype` is ``None``, these arguments are passed to + `numpy.format_parser` to construct a dtype. See that function for + detailed documentation + + Returns + ------- + np.recarray + record array consisting of data enclosed in file. + Examples + -------- >>> from tempfile import TemporaryFile >>> a = np.empty(10,dtype='f8,i4,a5') >>> a[5] = (0.5,10,'abcde') @@ -794,7 +856,7 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, if shape is None: shape = (-1,) - elif isinstance(shape, (int, long)): + elif isinstance(shape, int): shape = (shape,) if isfileobj(fd): @@ -805,14 +867,14 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, ctx = open(os_fspath(fd), 'rb') with ctx as fd: - if (offset > 0): + if offset > 0: fd.seek(offset, 1) size = get_remaining_size(fd) if dtype is not None: descr = sb.dtype(dtype) else: - descr = format_parser(formats, names, titles, aligned, byteorder)._descr + descr = format_parser(formats, names, titles, aligned, byteorder).dtype itemsize = descr.itemsize @@ -844,7 +906,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None, """ if ((isinstance(obj, (type(None), str)) or isfileobj(obj)) and - (formats is None) and (dtype is None)): + formats is None and dtype is None): raise ValueError("Must define formats (or dtype) if object is " "None, string, or an open file") @@ -853,7 +915,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None, dtype = sb.dtype(dtype) elif formats is not None: dtype = format_parser(formats, names, titles, - aligned, byteorder)._descr + aligned, byteorder).dtype else: kwds = {'formats': formats, 'names': names, diff --git a/numpy/core/setup.py b/numpy/core/setup.py index e376d5a23..15e732614 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -14,7 +14,7 @@ from numpy._build_utils.apple_accelerate import ( uses_accelerate_framework, get_sgemv_fix ) from numpy.compat import npy_load_module -from setup_common import * +from setup_common import * # noqa: F403 # Set to True to enable relaxed strides checking. This (mostly) means # that `strides[dim]` is ignored if `shape[dim] == 1` when setting flags. @@ -392,7 +392,7 @@ def visibility_define(config): def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration, dot_join - from numpy.distutils.system_info import get_info, dict_append + from numpy.distutils.system_info import get_info config = Configuration('core', parent_package, top_path) local_dir = config.local_path @@ -745,6 +745,7 @@ def configuration(parent_package='',top_path=None): join('src', 'common', 'ucsnarrow.c'), join('src', 'common', 'ufunc_override.c'), join('src', 'common', 'numpyos.c'), + join('src', 'common', 'npy_cpu_features.c.src'), ] if os.environ.get('NPY_USE_BLAS_ILP64', "0") != "0": @@ -898,7 +899,6 @@ def configuration(parent_package='',top_path=None): join('src', 'umath', 'clip.c.src'), join('src', 'umath', 'ufunc_object.c'), join('src', 'umath', 'extobj.c'), - join('src', 'umath', 'cpuid.c'), join('src', 'umath', 'scalarmath.c.src'), join('src', 'umath', 'ufunc_type_resolution.c'), join('src', 'umath', 'override.c'), diff --git a/numpy/core/setup_common.py b/numpy/core/setup_common.py index 6d8e603a7..63c4a76a9 100644 --- a/numpy/core/setup_common.py +++ b/numpy/core/setup_common.py @@ -2,7 +2,6 @@ import sys import warnings import copy -import binascii import textwrap from numpy.distutils.misc_util import mingw32 @@ -133,11 +132,6 @@ OPTIONAL_INTRINSICS = [("__builtin_isnan", '5.'), ("__builtin_bswap64", '5u'), ("__builtin_expect", '5, 0'), ("__builtin_mul_overflow", '5, 5, (int*)5'), - # broken on OSX 10.11, make sure its not optimized away - ("volatile int r = __builtin_cpu_supports", '"sse"', - "stdio.h", "__BUILTIN_CPU_SUPPORTS"), - ("volatile int r = __builtin_cpu_supports", '"avx512f"', - "stdio.h", "__BUILTIN_CPU_SUPPORTS_AVX512F"), # MMX only needed for icc, but some clangs don't have it ("_m_from_int64", '0', "emmintrin.h"), ("_mm_load_ps", '(float*)0', "xmmintrin.h"), # SSE @@ -311,32 +305,15 @@ def pyod(filename): We only implement enough to get the necessary information for long double representation, this is not intended as a compatible replacement for od. """ - def _pyod2(): - out = [] - - with open(filename, 'rb') as fid: - yo = [int(oct(int(binascii.b2a_hex(o), 16))) for o in fid.read()] - for i in range(0, len(yo), 16): - line = ['%07d' % int(oct(i))] - line.extend(['%03d' % c for c in yo[i:i+16]]) - out.append(" ".join(line)) - return out - - def _pyod3(): - out = [] - - with open(filename, 'rb') as fid: - yo2 = [oct(o)[2:] for o in fid.read()] - for i in range(0, len(yo2), 16): - line = ['%07d' % int(oct(i)[2:])] - line.extend(['%03d' % int(c) for c in yo2[i:i+16]]) - out.append(" ".join(line)) - return out - - if sys.version_info[0] < 3: - return _pyod2() - else: - return _pyod3() + out = [] + with open(filename, 'rb') as fid: + yo2 = [oct(o)[2:] for o in fid.read()] + for i in range(0, len(yo2), 16): + line = ['%07d' % int(oct(i)[2:])] + line.extend(['%03d' % int(c) for c in yo2[i:i+16]]) + out.append(" ".join(line)) + return out + _BEFORE_SEQ = ['000', '000', '000', '000', '000', '000', '000', '000', '001', '043', '105', '147', '211', '253', '315', '357'] diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index d2f26149b..ee56dbe43 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -2,6 +2,7 @@ __all__ = ['atleast_1d', 'atleast_2d', 'atleast_3d', 'block', 'hstack', 'stack', 'vstack'] import functools +import itertools import operator import warnings @@ -530,14 +531,7 @@ def _atleast_nd(a, ndim): def _accumulate(values): - # Helper function because Python 2.7 doesn't have - # itertools.accumulate - value = 0 - accumulated = [] - for v in values: - value += v - accumulated.append(value) - return accumulated + return list(itertools.accumulate(values)) def _concatenate_shapes(shapes, axis): @@ -676,8 +670,7 @@ def _block_dispatcher(arrays): # tuple. Also, we know that list.__array_function__ will never exist. if type(arrays) is list: for subarrays in arrays: - for subarray in _block_dispatcher(subarrays): - yield subarray + yield from _block_dispatcher(subarrays) else: yield arrays diff --git a/numpy/core/src/common/lowlevel_strided_loops.h b/numpy/core/src/common/lowlevel_strided_loops.h index 9208d5499..f2f12a55b 100644 --- a/numpy/core/src/common/lowlevel_strided_loops.h +++ b/numpy/core/src/common/lowlevel_strided_loops.h @@ -638,7 +638,7 @@ npy_bswap8_unaligned(char * x) * * Here is example code for a single array: * - * if (PyArray_TRIVIALLY_ITERABLE(self) { + * if (PyArray_TRIVIALLY_ITERABLE(self)) { * char *data; * npy_intp count, stride; * @@ -656,7 +656,7 @@ npy_bswap8_unaligned(char * x) * * Here is example code for a pair of arrays: * - * if (PyArray_TRIVIALLY_ITERABLE_PAIR(a1, a2) { + * if (PyArray_TRIVIALLY_ITERABLE_PAIR(a1, a2)) { * char *data1, *data2; * npy_intp count, stride1, stride2; * diff --git a/numpy/core/src/common/npy_config.h b/numpy/core/src/common/npy_config.h index eedfbe364..aebe241a5 100644 --- a/numpy/core/src/common/npy_config.h +++ b/numpy/core/src/common/npy_config.h @@ -2,6 +2,7 @@ #define _NPY_NPY_CONFIG_H_ #include "config.h" +#include "npy_cpu_features.h" #include "numpy/numpyconfig.h" #include "numpy/npy_cpu.h" #include "numpy/npy_os.h" diff --git a/numpy/core/src/common/npy_cpu_features.c.src b/numpy/core/src/common/npy_cpu_features.c.src new file mode 100644 index 000000000..4f193a471 --- /dev/null +++ b/numpy/core/src/common/npy_cpu_features.c.src @@ -0,0 +1,410 @@ +#include "npy_cpu_features.h" +#include "numpy/npy_common.h" // for NPY_INLINE +#include "numpy/npy_cpu.h" // To guarantee of having CPU definitions in scope. + +/******************** Private Definitions *********************/ + +// Hold all CPU features boolean values +static unsigned char npy__cpu_have[NPY_CPU_FEATURE_MAX]; + +/******************** Private Declarations *********************/ + +// Almost detect all CPU features in runtime +static void +npy__cpu_init_features(void); + +/******************** Public Definitions *********************/ + +NPY_VISIBILITY_HIDDEN int +npy_cpu_have(int feature_id) +{ + if (feature_id <= NPY_CPU_FEATURE_NONE || feature_id >= NPY_CPU_FEATURE_MAX) + return 0; + return npy__cpu_have[feature_id]; +} + +NPY_VISIBILITY_HIDDEN int +npy_cpu_init(void) +{ + npy__cpu_init_features(); + return 0; +} + +NPY_VISIBILITY_HIDDEN PyObject * +npy_cpu_features_dict(void) +{ + PyObject *dict = PyDict_New(); + if (dict) { + /**begin repeat + * #feature = MMX, SSE, SSE2, SSE3, SSSE3, SSE41, POPCNT, SSE42, + * AVX, F16C, XOP, FMA4, FMA3, AVX2, AVX512F, + * AVX512CD, AVX512ER, AVX512PF, AVX5124FMAPS, AVX5124VNNIW, + * AVX512VPOPCNTDQ, AVX512VL, AVX512BW, AVX512DQ, AVX512VNNI, + * AVX512IFMA, AVX512VBMI, AVX512VBMI2, AVX512BITALG, + * AVX512_KNL, AVX512_KNM, AVX512_SKX, AVX512_CLX, AVX512_CNL, AVX512_ICL, + * VSX, VSX2, VSX3, + * NEON, NEON_FP16, NEON_VFPV4, ASIMD, FPHP, ASIMDHP, ASIMDDP, ASIMDFHM# + */ + if (PyDict_SetItemString(dict, "@feature@", + npy__cpu_have[NPY_CPU_FEATURE_@feature@] ? Py_True : Py_False) < 0) { + Py_DECREF(dict); + return NULL; + } + /**end repeat**/ + } + return dict; +} + +/**************************************************************** + * This section is reserved to defining @npy__cpu_init_features + * for each CPU architecture, please try to keep it clean. Ty + ****************************************************************/ + +/***************** X86 ******************/ + +#if defined(NPY_CPU_AMD64) || defined(NPY_CPU_X86) + +#ifdef _MSC_VER + #include <intrin.h> +#elif defined(__INTEL_COMPILER) + #include <immintrin.h> +#endif + +static int +npy__cpu_getxcr0(void) +{ +#if defined(_MSC_VER) || defined (__INTEL_COMPILER) + return _xgetbv(0); +#elif defined(__GNUC__) || defined(__clang__) + unsigned int eax, edx; + __asm__("xgetbv" : "=a" (eax), "=d" (edx) : "c" (0)); + return (eax | (unsigned long long)edx << 32); +#else + // TODO: handle other x86 compilers + return 0; +#endif +} + +static void +npy__cpu_cpuid(int reg[4], int func_id) +{ +#if defined(_MSC_VER) + __cpuidex(reg, func_id, 0); +#elif defined(__INTEL_COMPILER) + __cpuid(reg, func_id); +#elif defined(__GNUC__) || defined(__clang__) + #if defined(NPY_CPU_X86) && defined(__PIC__) + // %ebx may be the PIC register + __asm__("xchg{l}\t{%%}ebx, %1\n\t" + "cpuid\n\t" + "xchg{l}\t{%%}ebx, %1\n\t" + : "=a" (reg[0]), "=r" (reg[1]), "=c" (reg[2]), + "=d" (reg[3]) + : "a" (func_id), "c" (0) + ); + #else + __asm__("cpuid\n\t" + : "=a" (reg[0]), "=b" (reg[1]), "=c" (reg[2]), + "=d" (reg[3]) + : "a" (func_id), "c" (0) + ); + #endif +#else + // TODO: handle other x86 compilers + reg[0] = 0; +#endif +} + +static void +npy__cpu_init_features(void) +{ + memset(npy__cpu_have, 0, sizeof(npy__cpu_have[0]) * NPY_CPU_FEATURE_MAX); + + // validate platform support + int reg[] = {0, 0, 0, 0}; + npy__cpu_cpuid(reg, 0); + if (reg[0] == 0) + return; + + npy__cpu_cpuid(reg, 1); + npy__cpu_have[NPY_CPU_FEATURE_MMX] = (reg[3] & (1 << 23)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_SSE] = (reg[3] & (1 << 25)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_SSE2] = (reg[3] & (1 << 26)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_SSE3] = (reg[2] & (1 << 0)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_SSSE3] = (reg[2] & (1 << 9)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_SSE41] = (reg[2] & (1 << 19)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_POPCNT] = (reg[2] & (1 << 23)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_SSE42] = (reg[2] & (1 << 20)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_F16C] = (reg[2] & (1 << 29)) != 0; + + // check OSXSAVE + if ((reg[2] & (1 << 27)) == 0) + return; + // check AVX OS support + int xcr = npy__cpu_getxcr0(); + if ((xcr & 6) != 6) + return; + npy__cpu_have[NPY_CPU_FEATURE_AVX] = (reg[2] & (1 << 28)) != 0; + if (!npy__cpu_have[NPY_CPU_FEATURE_AVX]) + return; + npy__cpu_have[NPY_CPU_FEATURE_FMA3] = (reg[2] & (1 << 12)) != 0; + + // second call to the cpuid to get extended AMD feature bits + npy__cpu_cpuid(reg, 0x80000001); + npy__cpu_have[NPY_CPU_FEATURE_XOP] = (reg[2] & (1 << 11)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_FMA4] = (reg[2] & (1 << 16)) != 0; + + // third call to the cpuid to get extended AVX2 & AVX512 feature bits + npy__cpu_cpuid(reg, 7); + npy__cpu_have[NPY_CPU_FEATURE_AVX2] = (reg[1] & (1 << 5)) != 0; + if (!npy__cpu_have[NPY_CPU_FEATURE_AVX2]) + return; + // detect AVX2 & FMA3 + npy__cpu_have[NPY_CPU_FEATURE_FMA] = npy__cpu_have[NPY_CPU_FEATURE_FMA3]; + + // check AVX512 OS support + if ((xcr & 0xe6) != 0xe6) + return; + npy__cpu_have[NPY_CPU_FEATURE_AVX512F] = (reg[1] & (1 << 16)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512CD] = (reg[1] & (1 << 28)) != 0; + if (npy__cpu_have[NPY_CPU_FEATURE_AVX512F] && npy__cpu_have[NPY_CPU_FEATURE_AVX512CD]) { + // Knights Landing + npy__cpu_have[NPY_CPU_FEATURE_AVX512PF] = (reg[1] & (1 << 26)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512ER] = (reg[1] & (1 << 27)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512_KNL] = npy__cpu_have[NPY_CPU_FEATURE_AVX512ER] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512PF]; + // Knights Mill + npy__cpu_have[NPY_CPU_FEATURE_AVX512VPOPCNTDQ] = (reg[2] & (1 << 14)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX5124VNNIW] = (reg[3] & (1 << 2)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX5124FMAPS] = (reg[3] & (1 << 3)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512_KNM] = npy__cpu_have[NPY_CPU_FEATURE_AVX512_KNL] && + npy__cpu_have[NPY_CPU_FEATURE_AVX5124FMAPS] && + npy__cpu_have[NPY_CPU_FEATURE_AVX5124VNNIW] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512VPOPCNTDQ]; + + // Skylake-X + npy__cpu_have[NPY_CPU_FEATURE_AVX512DQ] = (reg[1] & (1 << 17)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512BW] = (reg[1] & (1 << 30)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512VL] = (reg[1] & (1 << 31)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512_SKX] = npy__cpu_have[NPY_CPU_FEATURE_AVX512BW] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512DQ] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512VL]; + // Cascade Lake + npy__cpu_have[NPY_CPU_FEATURE_AVX512VNNI] = (reg[2] & (1 << 11)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512_CLX] = npy__cpu_have[NPY_CPU_FEATURE_AVX512_SKX] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512VNNI]; + + // Cannon Lake + npy__cpu_have[NPY_CPU_FEATURE_AVX512IFMA] = (reg[1] & (1 << 21)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512VBMI] = (reg[2] & (1 << 1)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512_CNL] = npy__cpu_have[NPY_CPU_FEATURE_AVX512_SKX] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512IFMA] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512VBMI]; + // Ice Lake + npy__cpu_have[NPY_CPU_FEATURE_AVX512VBMI2] = (reg[2] & (1 << 6)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512BITALG] = (reg[2] & (1 << 12)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_AVX512_ICL] = npy__cpu_have[NPY_CPU_FEATURE_AVX512_CLX] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512_CNL] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512VBMI2] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512BITALG] && + npy__cpu_have[NPY_CPU_FEATURE_AVX512VPOPCNTDQ]; + } +} + +/***************** POWER ******************/ + +#elif defined(NPY_CPU_PPC64) || defined(NPY_CPU_PPC64LE) + +#ifdef __linux__ + #include <sys/auxv.h> + #ifndef AT_HWCAP2 + #define AT_HWCAP2 26 + #endif + #ifndef PPC_FEATURE2_ARCH_3_00 + #define PPC_FEATURE2_ARCH_3_00 0x00800000 + #endif +#endif + +static void +npy__cpu_init_features(void) +{ + memset(npy__cpu_have, 0, sizeof(npy__cpu_have[0]) * NPY_CPU_FEATURE_MAX); +#ifdef __linux__ + unsigned int hwcap = getauxval(AT_HWCAP); + if ((hwcap & PPC_FEATURE_HAS_VSX) == 0) + return; + + hwcap = getauxval(AT_HWCAP2); + if (hwcap & PPC_FEATURE2_ARCH_3_00) + { + npy__cpu_have[NPY_CPU_FEATURE_VSX] = + npy__cpu_have[NPY_CPU_FEATURE_VSX2] = + npy__cpu_have[NPY_CPU_FEATURE_VSX3] = 1; + return; + } + npy__cpu_have[NPY_CPU_FEATURE_VSX2] = (hwcap & PPC_FEATURE2_ARCH_2_07) != 0; + npy__cpu_have[NPY_CPU_FEATURE_VSX] = 1; +// TODO: AIX, FreeBSD +#else + npy__cpu_have[NPY_CPU_FEATURE_VSX] = 1; + #if defined(NPY_CPU_PPC64LE) || defined(NPY_HAVE_VSX2) + npy__cpu_have[NPY_CPU_FEATURE_VSX2] = 1; + #endif + #ifdef NPY_HAVE_VSX3 + npy__cpu_have[NPY_CPU_FEATURE_VSX3] = 1; + #endif +#endif +} + +/***************** ARM ******************/ + +#elif defined(__arm__) || defined(__aarch64__) + +static NPY_INLINE void +npy__cpu_init_features_arm8(void) +{ + npy__cpu_have[NPY_CPU_FEATURE_NEON] = + npy__cpu_have[NPY_CPU_FEATURE_NEON_FP16] = + npy__cpu_have[NPY_CPU_FEATURE_NEON_VFPV4] = + npy__cpu_have[NPY_CPU_FEATURE_ASIMD] = 1; +} + +#ifdef __linux__ +/* + * we aren't sure of what kind kernel or clib we deal with + * so we play it safe +*/ +#include <stdio.h> +#include <fcntl.h> + +#define NPY__HWCAP 16 +#define NPY__HWCAP2 26 + +// arch/arm/include/uapi/asm/hwcap.h +#define NPY__HWCAP_HALF (1 << 1) +#define NPY__HWCAP_NEON (1 << 12) +#define NPY__HWCAP_VFPv3 (1 << 13) +#define NPY__HWCAP_VFPv4 (1 << 16) +#define NPY__HWCAP2_AES (1 << 0) +#define NPY__HWCAP2_PMULL (1 << 1) +#define NPY__HWCAP2_SHA1 (1 << 2) +#define NPY__HWCAP2_SHA2 (1 << 3) +#define NPY__HWCAP2_CRC32 (1 << 4) +// arch/arm64/include/uapi/asm/hwcap.h +#define NPY__HWCAP_FP (1 << 0) +#define NPY__HWCAP_ASIMD (1 << 1) +#define NPY__HWCAP_FPHP (1 << 9) +#define NPY__HWCAP_ASIMDHP (1 << 10) +#define NPY__HWCAP_ASIMDDP (1 << 20) +#define NPY__HWCAP_ASIMDFHM (1 << 23) + +__attribute__((weak)) unsigned long getauxval(unsigned long); // linker should handle it +static int +npy__cpu_init_features_linux(void) +{ + unsigned long hwcap = 0, hwcap2 = 0; + if (getauxval != 0) { + hwcap = getauxval(NPY__HWCAP); + #ifdef __arm__ + hwcap2 = getauxval(NPY__HWCAP2); + #endif + } else { + unsigned long auxv[2]; + int fd = open("/proc/self/auxv", O_RDONLY); + if (fd >= 0) { + while (read(fd, &auxv, sizeof(auxv)) == sizeof(auxv)) { + if (auxv[0] == NPY__HWCAP) { + hwcap = auxv[1]; + } + #ifdef __arm__ + else if (auxv[0] == NPY__HWCAP2) { + hwcap2 = auxv[1]; + } + #endif + // detect the end + else if (auxv[0] == 0 && auxv[1] == 0) { + break; + } + } + close(fd); + } + } + if (hwcap == 0 && hwcap2 == 0) { + /* + * FIXME: failback to compiler definitions, + * BTW we can parse /proc/cpuinfo for badly patched kernels + */ + return 0; + } +#ifdef __arm__ + // Detect Arm8 (aarch32 state) + if ((hwcap2 & NPY__HWCAP2_AES) || (hwcap2 & NPY__HWCAP2_SHA1) || + (hwcap2 & NPY__HWCAP2_SHA2) || (hwcap2 & NPY__HWCAP2_PMULL) || + (hwcap2 & NPY__HWCAP2_CRC32)) +#else + if (1) +#endif + { + if (!(hwcap & (NPY__HWCAP_FP | NPY__HWCAP_ASIMD))) { + // Is this could happen? maybe disabled by kernel + // BTW this will break the baseline of AARCH64 + return 1; + } + npy__cpu_have[NPY_CPU_FEATURE_FPHP] = (hwcap & NPY__HWCAP_FPHP) != 0; + npy__cpu_have[NPY_CPU_FEATURE_ASIMDHP] = (hwcap & NPY__HWCAP_ASIMDHP) != 0; + npy__cpu_have[NPY_CPU_FEATURE_ASIMDDP] = (hwcap & NPY__HWCAP_ASIMDDP) != 0; + npy__cpu_have[NPY_CPU_FEATURE_ASIMDFHM] = (hwcap & NPY__HWCAP_ASIMDFHM) != 0; + npy__cpu_init_features_arm8(); + } else { + npy__cpu_have[NPY_CPU_FEATURE_NEON] = (hwcap & NPY__HWCAP_NEON) != 0; + npy__cpu_have[NPY_CPU_FEATURE_NEON_FP16] = (hwcap & (NPY__HWCAP_NEON | NPY__HWCAP_VFPv3 | + NPY__HWCAP_HALF)) != 0; + npy__cpu_have[NPY_CPU_FEATURE_NEON_VFPV4] = (hwcap & (NPY__HWCAP_NEON | NPY__HWCAP_VFPv4)) != 0; + } + return 1; +} +#endif + +static void +npy__cpu_init_features(void) +{ + memset(npy__cpu_have, 0, sizeof(npy__cpu_have[0]) * NPY_CPU_FEATURE_MAX); +#ifdef __linux__ + if (npy__cpu_init_features_linux()) + return; +#endif + // We have nothing else todo +#if defined(NPY_HAVE_NEON_ARM8) || defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH >= 8) + #if defined(NPY_HAVE_FPHP) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + npy__cpu_have[NPY_CPU_FEATURE_FPHP] = 1; + #endif + #if defined(NPY_HAVE_ASIMDHP) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + npy__cpu_have[NPY_CPU_FEATURE_ASIMDHP] = 1; + #endif + #if defined(NPY_HAVE_ASIMDDP) || defined(__ARM_FEATURE_DOTPROD) + npy__cpu_have[NPY_CPU_FEATURE_ASIMDDP] = 1; + #endif + #if defined(NPY_HAVE_ASIMDFHM) || defined(__ARM_FEATURE_FP16FML) + npy__cpu_have[NPY_CPU_FEATURE_ASIMDFHM] = 1; + #endif + npy__cpu_init_features_arm8(); +#else + #if defined(NPY_HAVE_NEON) || defined(__ARM_NEON__) + npy__cpu_have[NPY_CPU_FEATURE_NEON] = 1; + #endif + #if defined(NPY_HAVE_NEON_FP16) || defined(__ARM_FP16_FORMAT_IEEE) || (defined(__ARM_FP) && (__ARM_FP & 2)) + npy__cpu_have[NPY_CPU_FEATURE_NEON_FP16] = npy__cpu_have[NPY_CPU_FEATURE_NEON]; + #endif + #if defined(NPY_HAVE_NEON_VFPV4) || defined(__ARM_FEATURE_FMA) + npy__cpu_have[NPY_CPU_FEATURE_NEON_VFPV4] = npy__cpu_have[NPY_CPU_FEATURE_NEON]; + #endif +#endif +} + +/*********** Unsupported ARCH ***********/ +#else +static void +npy__cpu_init_features(void) +{ +} +#endif diff --git a/numpy/core/src/common/npy_cpu_features.h b/numpy/core/src/common/npy_cpu_features.h new file mode 100644 index 000000000..0e8901328 --- /dev/null +++ b/numpy/core/src/common/npy_cpu_features.h @@ -0,0 +1,117 @@ +#ifndef _NPY_CPU_FEATURES_H_ +#define _NPY_CPU_FEATURES_H_ + +#include "numpy/numpyconfig.h" // for NPY_VISIBILITY_HIDDEN +#include <Python.h> // for PyObject + +#ifdef __cplusplus +extern "C" { +#endif + +enum npy_cpu_features +{ + NPY_CPU_FEATURE_NONE = 0, + // X86 + NPY_CPU_FEATURE_MMX = 1, + NPY_CPU_FEATURE_SSE = 2, + NPY_CPU_FEATURE_SSE2 = 3, + NPY_CPU_FEATURE_SSE3 = 4, + NPY_CPU_FEATURE_SSSE3 = 5, + NPY_CPU_FEATURE_SSE41 = 6, + NPY_CPU_FEATURE_POPCNT = 7, + NPY_CPU_FEATURE_SSE42 = 8, + NPY_CPU_FEATURE_AVX = 9, + NPY_CPU_FEATURE_F16C = 10, + NPY_CPU_FEATURE_XOP = 11, + NPY_CPU_FEATURE_FMA4 = 12, + NPY_CPU_FEATURE_FMA3 = 13, + NPY_CPU_FEATURE_AVX2 = 14, + NPY_CPU_FEATURE_FMA = 15, // AVX2 & FMA3, provides backward compatibility + + NPY_CPU_FEATURE_AVX512F = 30, + NPY_CPU_FEATURE_AVX512CD = 31, + NPY_CPU_FEATURE_AVX512ER = 32, + NPY_CPU_FEATURE_AVX512PF = 33, + NPY_CPU_FEATURE_AVX5124FMAPS = 34, + NPY_CPU_FEATURE_AVX5124VNNIW = 35, + NPY_CPU_FEATURE_AVX512VPOPCNTDQ = 36, + NPY_CPU_FEATURE_AVX512BW = 37, + NPY_CPU_FEATURE_AVX512DQ = 38, + NPY_CPU_FEATURE_AVX512VL = 39, + NPY_CPU_FEATURE_AVX512IFMA = 40, + NPY_CPU_FEATURE_AVX512VBMI = 41, + NPY_CPU_FEATURE_AVX512VNNI = 42, + NPY_CPU_FEATURE_AVX512VBMI2 = 43, + NPY_CPU_FEATURE_AVX512BITALG = 44, + + // X86 CPU Groups + // Knights Landing (F,CD,ER,PF) + NPY_CPU_FEATURE_AVX512_KNL = 101, + // Knights Mill (F,CD,ER,PF,4FMAPS,4VNNIW,VPOPCNTDQ) + NPY_CPU_FEATURE_AVX512_KNM = 102, + // Skylake-X (F,CD,BW,DQ,VL) + NPY_CPU_FEATURE_AVX512_SKX = 103, + // Cascade Lake (F,CD,BW,DQ,VL,VNNI) + NPY_CPU_FEATURE_AVX512_CLX = 104, + // Cannon Lake (F,CD,BW,DQ,VL,IFMA,VBMI) + NPY_CPU_FEATURE_AVX512_CNL = 105, + // Ice Lake (F,CD,BW,DQ,VL,IFMA,VBMI,VNNI,VBMI2,BITALG,VPOPCNTDQ) + NPY_CPU_FEATURE_AVX512_ICL = 106, + + // IBM/POWER VSX + // POWER7 + NPY_CPU_FEATURE_VSX = 200, + // POWER8 + NPY_CPU_FEATURE_VSX2 = 201, + // POWER9 + NPY_CPU_FEATURE_VSX3 = 202, + + // ARM + NPY_CPU_FEATURE_NEON = 300, + NPY_CPU_FEATURE_NEON_FP16 = 301, + // FMA + NPY_CPU_FEATURE_NEON_VFPV4 = 302, + // Advanced SIMD + NPY_CPU_FEATURE_ASIMD = 303, + // ARMv8.2 half-precision + NPY_CPU_FEATURE_FPHP = 304, + // ARMv8.2 half-precision vector arithm + NPY_CPU_FEATURE_ASIMDHP = 305, + // ARMv8.2 dot product + NPY_CPU_FEATURE_ASIMDDP = 306, + // ARMv8.2 single&half-precision multiply + NPY_CPU_FEATURE_ASIMDFHM = 307, + + NPY_CPU_FEATURE_MAX +}; + +/* + * Initialize CPU features + * return 0 on success otherwise return -1 +*/ +NPY_VISIBILITY_HIDDEN int +npy_cpu_init(void); + +/* + * return 0 if CPU feature isn't available + * note: `npy_cpu_init` must be called first otherwise it will always return 0 +*/ +NPY_VISIBILITY_HIDDEN int +npy_cpu_have(int feature_id); + +#define NPY_CPU_HAVE(FEATURE_NAME) \ +npy_cpu_have(NPY_CPU_FEATURE_##FEATURE_NAME) + +/* + * return a new dictionary contains CPU feature names + * with runtime availability. + * same as npy_cpu_have, `npy_cpu_init` must be called first. + */ +NPY_VISIBILITY_HIDDEN PyObject * +npy_cpu_features_dict(void); + +#ifdef __cplusplus +} +#endif + +#endif // _NPY_CPU_FEATURES_H_ diff --git a/numpy/core/src/common/numpyos.c b/numpy/core/src/common/numpyos.c index 7a629f46f..42a71777b 100644 --- a/numpy/core/src/common/numpyos.c +++ b/numpy/core/src/common/numpyos.c @@ -248,7 +248,7 @@ check_ascii_format(const char *format) * Fix the generated string: make sure the decimal is ., that exponent has a * minimal number of digits, and that it has a decimal + one digit after that * decimal if decimal argument != 0 (Same effect that 'Z' format in - * PyOS_ascii_formatd + * PyOS_ascii_formatd) */ static char* fix_ascii_format(char* buf, size_t buflen, int decimal) diff --git a/numpy/core/src/common/ucsnarrow.c b/numpy/core/src/common/ucsnarrow.c index 946a72257..3ef5d6878 100644 --- a/numpy/core/src/common/ucsnarrow.c +++ b/numpy/core/src/common/ucsnarrow.c @@ -16,76 +16,12 @@ #include "ctors.h" /* - * Functions only needed on narrow builds of Python for converting back and - * forth between the NumPy Unicode data-type (always 4-bytes) and the - * Python Unicode scalar (2-bytes on a narrow build). - */ - -/* - * The ucs2 buffer must be large enough to hold 2*ucs4length characters - * due to the use of surrogate pairs. + * This file originally contained functions only needed on narrow builds of + * Python for converting back and forth between the NumPy Unicode data-type + * (always 4-bytes) and the Python Unicode scalar (2-bytes on a narrow build). * - * The return value is the number of ucs2 bytes used-up which - * is ucs4length + number of surrogate pairs found. - * - * Values above 0xffff are converted to surrogate pairs. + * This "narrow" interface is now deprecated in python and unused in NumPy. */ -NPY_NO_EXPORT int -PyUCS2Buffer_FromUCS4(Py_UNICODE *ucs2, npy_ucs4 const *ucs4, int ucs4length) -{ - int i; - int numucs2 = 0; - npy_ucs4 chr; - for (i = 0; i < ucs4length; i++) { - chr = *ucs4++; - if (chr > 0xffff) { - numucs2++; - chr -= 0x10000L; - *ucs2++ = 0xD800 + (Py_UNICODE) (chr >> 10); - *ucs2++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF); - } - else { - *ucs2++ = (Py_UNICODE) chr; - } - numucs2++; - } - return numucs2; -} - - -/* - * This converts a UCS2 buffer of the given length to UCS4 buffer. - * It converts up to ucs4len characters of UCS2 - * - * It returns the number of characters converted which can - * be less than ucs2len if there are surrogate pairs in ucs2. - * - * The return value is the actual size of the used part of the ucs4 buffer. - */ -NPY_NO_EXPORT int -PyUCS2Buffer_AsUCS4(Py_UNICODE const *ucs2, npy_ucs4 *ucs4, int ucs2len, int ucs4len) -{ - int i; - npy_ucs4 chr; - Py_UNICODE ch; - int numchars=0; - - for (i = 0; (i < ucs2len) && (numchars < ucs4len); i++) { - ch = *ucs2++; - if (ch >= 0xd800 && ch <= 0xdfff) { - /* surrogate pair */ - chr = ((npy_ucs4)(ch-0xd800)) << 10; - chr += *ucs2++ + 0x2400; /* -0xdc00 + 0x10000 */ - i++; - } - else { - chr = (npy_ucs4) ch; - } - *ucs4++ = chr; - numchars++; - } - return numchars; -} /* * Returns a PyUnicodeObject initialized from a buffer containing @@ -112,14 +48,13 @@ PyUnicode_FromUCS4(char const *src_char, Py_ssize_t size, int swap, int align) Py_ssize_t ucs4len = size / sizeof(npy_ucs4); npy_ucs4 const *src = (npy_ucs4 const *)src_char; npy_ucs4 *buf = NULL; - PyUnicodeObject *ret; /* swap and align if needed */ if (swap || align) { buf = (npy_ucs4 *)malloc(size); if (buf == NULL) { PyErr_NoMemory(); - goto fail; + return NULL; } memcpy(buf, src, size); if (swap) { @@ -132,43 +67,8 @@ PyUnicode_FromUCS4(char const *src_char, Py_ssize_t size, int swap, int align) while (ucs4len > 0 && src[ucs4len - 1] == 0) { ucs4len--; } - - /* produce PyUnicode object */ -#ifdef Py_UNICODE_WIDE - { - ret = (PyUnicodeObject *)PyUnicode_FromUnicode((Py_UNICODE const*)src, - (Py_ssize_t) ucs4len); - if (ret == NULL) { - goto fail; - } - } -#else - { - Py_ssize_t tmpsiz = 2 * sizeof(Py_UNICODE) * ucs4len; - Py_ssize_t ucs2len; - Py_UNICODE *tmp; - - if ((tmp = (Py_UNICODE *)malloc(tmpsiz)) == NULL) { - PyErr_NoMemory(); - goto fail; - } - ucs2len = PyUCS2Buffer_FromUCS4(tmp, src, ucs4len); - ret = (PyUnicodeObject *)PyUnicode_FromUnicode(tmp, (Py_ssize_t) ucs2len); - free(tmp); - if (ret == NULL) { - goto fail; - } - } -#endif - - if (buf) { - free(buf); - } + PyUnicodeObject *ret = (PyUnicodeObject *)PyUnicode_FromKindAndData( + PyUnicode_4BYTE_KIND, src, ucs4len); + free(buf); return ret; - -fail: - if (buf) { - free(buf); - } - return NULL; } diff --git a/numpy/core/src/common/ucsnarrow.h b/numpy/core/src/common/ucsnarrow.h index fe31a5e25..c811e1f2c 100644 --- a/numpy/core/src/common/ucsnarrow.h +++ b/numpy/core/src/common/ucsnarrow.h @@ -1,12 +1,6 @@ #ifndef _NPY_UCSNARROW_H_ #define _NPY_UCSNARROW_H_ -NPY_NO_EXPORT int -PyUCS2Buffer_FromUCS4(Py_UNICODE *ucs2, npy_ucs4 *ucs4, int ucs4length); - -NPY_NO_EXPORT int -PyUCS2Buffer_AsUCS4(Py_UNICODE *ucs2, npy_ucs4 *ucs4, int ucs2len, int ucs4len); - NPY_NO_EXPORT PyUnicodeObject * PyUnicode_FromUCS4(char *src, Py_ssize_t size, int swap, int align); diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index 7007dd204..318559885 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -1877,21 +1877,21 @@ PrintFloat_Printf_g(PyObject *obj, int precision) char str[1024]; if (PyArray_IsScalar(obj, Half)) { - npy_half x = ((PyHalfScalarObject *)obj)->obval; + npy_half x = PyArrayScalar_VAL(obj, Half); PyOS_snprintf(str, sizeof(str), "%.*g", precision, npy_half_to_double(x)); } else if (PyArray_IsScalar(obj, Float)) { - npy_float x = ((PyFloatScalarObject *)obj)->obval; + npy_float x = PyArrayScalar_VAL(obj, Float); PyOS_snprintf(str, sizeof(str), "%.*g", precision, x); } else if (PyArray_IsScalar(obj, Double)) { - npy_double x = ((PyDoubleScalarObject *)obj)->obval; + npy_double x = PyArrayScalar_VAL(obj, Double); PyOS_snprintf(str, sizeof(str), "%.*g", precision, x); /* would be better to use lg, but not available in C90 */ } else if (PyArray_IsScalar(obj, LongDouble)) { - npy_longdouble x = ((PyLongDoubleScalarObject *)obj)->obval; + npy_longdouble x = PyArrayScalar_VAL(obj, LongDouble); PyOS_snprintf(str, sizeof(str), "%.*Lg", precision, x); } else{ @@ -1938,6 +1938,114 @@ getset_numericops(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) return ret; } +static PyObject * +run_byteorder_converter(PyObject* NPY_UNUSED(self), PyObject *args) +{ + char byteorder; + if (!PyArg_ParseTuple(args, "O&", PyArray_ByteorderConverter, &byteorder)) { + return NULL; + } + switch (byteorder) { + case NPY_BIG: return PyUnicode_FromString("NPY_BIG"); + case NPY_LITTLE: return PyUnicode_FromString("NPY_LITTLE"); + case NPY_NATIVE: return PyUnicode_FromString("NPY_NATIVE"); + case NPY_SWAP: return PyUnicode_FromString("NPY_SWAP"); + case NPY_IGNORE: return PyUnicode_FromString("NPY_IGNORE"); + } + return PyInt_FromLong(byteorder); +} + +static PyObject * +run_sortkind_converter(PyObject* NPY_UNUSED(self), PyObject *args) +{ + NPY_SORTKIND kind; + if (!PyArg_ParseTuple(args, "O&", PyArray_SortkindConverter, &kind)) { + return NULL; + } + switch (kind) { + case NPY_QUICKSORT: return PyUnicode_FromString("NPY_QUICKSORT"); + case NPY_HEAPSORT: return PyUnicode_FromString("NPY_HEAPSORT"); + case NPY_STABLESORT: return PyUnicode_FromString("NPY_STABLESORT"); + } + return PyInt_FromLong(kind); +} + +static PyObject * +run_selectkind_converter(PyObject* NPY_UNUSED(self), PyObject *args) +{ + NPY_SELECTKIND kind; + if (!PyArg_ParseTuple(args, "O&", PyArray_SelectkindConverter, &kind)) { + return NULL; + } + switch (kind) { + case NPY_INTROSELECT: return PyUnicode_FromString("NPY_INTROSELECT"); + } + return PyInt_FromLong(kind); +} + +static PyObject * +run_searchside_converter(PyObject* NPY_UNUSED(self), PyObject *args) +{ + NPY_SEARCHSIDE side; + if (!PyArg_ParseTuple(args, "O&", PyArray_SearchsideConverter, &side)) { + return NULL; + } + switch (side) { + case NPY_SEARCHLEFT: return PyUnicode_FromString("NPY_SEARCHLEFT"); + case NPY_SEARCHRIGHT: return PyUnicode_FromString("NPY_SEARCHRIGHT"); + } + return PyInt_FromLong(side); +} + +static PyObject * +run_order_converter(PyObject* NPY_UNUSED(self), PyObject *args) +{ + NPY_ORDER order; + if (!PyArg_ParseTuple(args, "O&", PyArray_OrderConverter, &order)) { + return NULL; + } + switch (order) { + case NPY_ANYORDER: return PyUnicode_FromString("NPY_ANYORDER"); + case NPY_CORDER: return PyUnicode_FromString("NPY_CORDER"); + case NPY_FORTRANORDER: return PyUnicode_FromString("NPY_FORTRANORDER"); + case NPY_KEEPORDER: return PyUnicode_FromString("NPY_KEEPORDER"); + } + return PyInt_FromLong(order); +} + +static PyObject * +run_clipmode_converter(PyObject* NPY_UNUSED(self), PyObject *args) +{ + NPY_CLIPMODE mode; + if (!PyArg_ParseTuple(args, "O&", PyArray_ClipmodeConverter, &mode)) { + return NULL; + } + switch (mode) { + case NPY_CLIP: return PyUnicode_FromString("NPY_CLIP"); + case NPY_WRAP: return PyUnicode_FromString("NPY_WRAP"); + case NPY_RAISE: return PyUnicode_FromString("NPY_RAISE"); + } + return PyInt_FromLong(mode); +} + +static PyObject * +run_casting_converter(PyObject* NPY_UNUSED(self), PyObject *args) +{ + NPY_CASTING casting; + if (!PyArg_ParseTuple(args, "O&", PyArray_CastingConverter, &casting)) { + return NULL; + } + switch (casting) { + case NPY_NO_CASTING: return PyUnicode_FromString("NPY_NO_CASTING"); + case NPY_EQUIV_CASTING: return PyUnicode_FromString("NPY_EQUIV_CASTING"); + case NPY_SAFE_CASTING: return PyUnicode_FromString("NPY_SAFE_CASTING"); + case NPY_SAME_KIND_CASTING: return PyUnicode_FromString("NPY_SAME_KIND_CASTING"); + case NPY_UNSAFE_CASTING: return PyUnicode_FromString("NPY_UNSAFE_CASTING"); + } + return PyInt_FromLong(casting); +} + + static PyMethodDef Multiarray_TestsMethods[] = { {"IsPythonScalar", IsPythonScalar, @@ -2089,6 +2197,27 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"get_struct_alignments", get_struct_alignments, METH_VARARGS, NULL}, + {"run_byteorder_converter", + run_byteorder_converter, + METH_VARARGS, NULL}, + {"run_sortkind_converter", + run_sortkind_converter, + METH_VARARGS, NULL}, + {"run_selectkind_converter", + run_selectkind_converter, + METH_VARARGS, NULL}, + {"run_searchside_converter", + run_searchside_converter, + METH_VARARGS, NULL}, + {"run_order_converter", + run_order_converter, + METH_VARARGS, NULL}, + {"run_clipmode_converter", + run_clipmode_converter, + METH_VARARGS, NULL}, + {"run_casting_converter", + run_casting_converter, + METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/numpy/core/src/multiarray/alloc.c b/numpy/core/src/multiarray/alloc.c index c2b7e9ca7..795fc7315 100644 --- a/numpy/core/src/multiarray/alloc.c +++ b/numpy/core/src/multiarray/alloc.c @@ -47,6 +47,32 @@ typedef struct { static cache_bucket datacache[NBUCKETS]; static cache_bucket dimcache[NBUCKETS_DIM]; +static int _madvise_hugepage = 1; + + +/* + * This function enables or disables the use of `MADV_HUGEPAGE` on Linux + * by modifying the global static `_madvise_hugepage`. + * It returns the previous value of `_madvise_hugepage`. + * + * It is exposed to Python as `np.core.multiarray._set_madvise_hugepage`. + */ +NPY_NO_EXPORT PyObject * +_set_madvise_hugepage(PyObject *NPY_UNUSED(self), PyObject *enabled_obj) +{ + int was_enabled = _madvise_hugepage; + int enabled = PyObject_IsTrue(enabled_obj); + if (enabled < 0) { + return NULL; + } + _madvise_hugepage = enabled; + if (was_enabled) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + + /* as the cache is managed in global variables verify the GIL is held */ /* @@ -75,7 +101,7 @@ _npy_alloc_cache(npy_uintp nelem, npy_uintp esz, npy_uint msz, #endif #ifdef NPY_OS_LINUX /* allow kernel allocating huge pages for large arrays */ - if (NPY_UNLIKELY(nelem * esz >= ((1u<<22u)))) { + if (NPY_UNLIKELY(nelem * esz >= ((1u<<22u))) && _madvise_hugepage) { npy_uintp offset = 4096u - (npy_uintp)p % (4096u); npy_uintp length = nelem * esz - offset; /** diff --git a/numpy/core/src/multiarray/alloc.h b/numpy/core/src/multiarray/alloc.h index 2b69efc35..15e31ebb5 100644 --- a/numpy/core/src/multiarray/alloc.h +++ b/numpy/core/src/multiarray/alloc.h @@ -6,6 +6,9 @@ #define NPY_TRACE_DOMAIN 389047 +NPY_NO_EXPORT PyObject * +_set_madvise_hugepage(PyObject *NPY_UNUSED(self), PyObject *enabled_obj); + NPY_NO_EXPORT void * npy_alloc_cache(npy_uintp sz); diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index aef0db36c..dedaf38eb 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -41,6 +41,7 @@ maintainer email: oliphant.travis@ieee.org #include "arraytypes.h" #include "scalartypes.h" #include "arrayobject.h" +#include "conversion_utils.h" #include "ctors.h" #include "methods.h" #include "descriptor.h" @@ -278,8 +279,8 @@ PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object) * Get either an array object we can copy from, or its parameters * if there isn't a convenient array available. */ - if (PyArray_GetArrayParamsFromObject(src_object, PyArray_DESCR(dest), - 0, &dtype, &ndim, dims, &src, NULL) < 0) { + if (PyArray_GetArrayParamsFromObject_int(src_object, + PyArray_DESCR(dest), 0, &dtype, &ndim, dims, &src) < 0) { Py_DECREF(src_object); return -1; } @@ -1123,7 +1124,7 @@ _void_compare(PyArrayObject *self, PyArrayObject *other, int cmp_op) op = (cmp_op == Py_EQ ? n_ops.logical_and : n_ops.logical_or); while (PyDict_Next(PyArray_DESCR(self)->fields, &pos, &key, &value)) { - if NPY_TITLE_KEY(key, value) { + if (NPY_TITLE_KEY(key, value)) { continue; } a = array_subscript_asarray(self, key); @@ -1329,9 +1330,8 @@ _failed_comparison_workaround(PyArrayObject *self, PyObject *other, int cmp_op) /* * For LE, LT, GT, GE and a flexible self or other, we return * NotImplemented, which is the correct answer since the ufuncs do - * not in fact implement loops for those. On python 3 this will - * get us the desired TypeError, but on python 2, one gets strange - * ordering, so we emit a warning. + * not in fact implement loops for those. This will get us the + * desired TypeError. */ Py_XDECREF(exc); Py_XDECREF(val); @@ -1539,8 +1539,7 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) * (MHvK, 2018-06-18: not sure about this, but it's what we have). * * However, for backwards compatibility, we cannot yet return arrays, - * so we raise warnings instead. Furthermore, we warn on python2 - * for LT, LE, GE, GT, since fall-back behaviour is poorly defined. + * so we raise warnings instead. */ result = _failed_comparison_workaround(self, other, cmp_op); } @@ -1626,7 +1625,7 @@ array_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) PyArray_Descr *descr = NULL; int itemsize; PyArray_Dims dims = {NULL, 0}; - PyArray_Dims strides = {NULL, 0}; + PyArray_Dims strides = {NULL, -1}; PyArray_Chunk buffer; npy_longlong offset = 0; NPY_ORDER order = NPY_CORDER; @@ -1647,7 +1646,7 @@ array_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) PyArray_BufferConverter, &buffer, &offset, - &PyArray_IntpConverter, + &PyArray_OptionalIntpConverter, &strides, &PyArray_OrderConverter, &order)) { @@ -1662,7 +1661,7 @@ array_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) itemsize = descr->elsize; - if (strides.ptr != NULL) { + if (strides.len != -1) { npy_intp nb, off; if (strides.len != dims.len) { PyErr_SetString(PyExc_ValueError, @@ -1783,56 +1782,28 @@ array_free(PyObject * v) NPY_NO_EXPORT PyTypeObject PyArray_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.ndarray", /* tp_name */ - NPY_SIZEOF_PYARRAYOBJECT, /* tp_basicsize */ - 0, /* tp_itemsize */ + .tp_name = "numpy.ndarray", + .tp_basicsize = NPY_SIZEOF_PYARRAYOBJECT, /* methods */ - (destructor)array_dealloc, /* tp_dealloc */ - (printfunc)NULL, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - (reprfunc)array_repr, /* tp_repr */ - &array_as_number, /* tp_as_number */ - &array_as_sequence, /* tp_as_sequence */ - &array_as_mapping, /* tp_as_mapping */ + .tp_dealloc = (destructor)array_dealloc, + .tp_repr = (reprfunc)array_repr, + .tp_as_number = &array_as_number, + .tp_as_sequence = &array_as_sequence, + .tp_as_mapping = &array_as_mapping, /* * The tp_hash slot will be set PyObject_HashNotImplemented when the * module is loaded. */ - (hashfunc)0, /* tp_hash */ - (ternaryfunc)0, /* tp_call */ - (reprfunc)array_str, /* tp_str */ - (getattrofunc)0, /* tp_getattro */ - (setattrofunc)0, /* tp_setattro */ - &array_as_buffer, /* tp_as_buffer */ - (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE), /* tp_flags */ - 0, /* tp_doc */ - - (traverseproc)0, /* tp_traverse */ - (inquiry)0, /* tp_clear */ - (richcmpfunc)array_richcompare, /* tp_richcompare */ - offsetof(PyArrayObject_fields, weakreflist), /* tp_weaklistoffset */ - (getiterfunc)array_iter, /* tp_iter */ - (iternextfunc)0, /* tp_iternext */ - array_methods, /* tp_methods */ - 0, /* tp_members */ - array_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)0, /* tp_init */ - (allocfunc)array_alloc, /* tp_alloc */ - (newfunc)array_new, /* tp_new */ - (freefunc)array_free, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_str = (reprfunc)array_str, + .tp_as_buffer = &array_as_buffer, + .tp_flags =(Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE), + + .tp_richcompare = (richcmpfunc)array_richcompare, + .tp_weaklistoffset = offsetof(PyArrayObject_fields, weakreflist), + .tp_iter = (getiterfunc)array_iter, + .tp_methods = array_methods, + .tp_getset = array_getsetlist, + .tp_alloc = (allocfunc)array_alloc, + .tp_new = (newfunc)array_new, + .tp_free = (freefunc)array_free, }; diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 808cfaa14..5e07f0df4 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -210,7 +210,7 @@ static int @type@ temp; /* ensures alignment */ if (PyArray_IsScalar(op, @kind@)) { - temp = ((Py@kind@ScalarObject *)op)->obval; + temp = PyArrayScalar_VAL(op, @kind@); } else { temp = (@type@)@func2@(op); @@ -291,7 +291,7 @@ static int } if (PyArray_IsScalar(op, @kind@)){ - temp = ((Py@kind@ScalarObject *)op)->obval; + temp = PyArrayScalar_VAL(op, @kind@); } else { if (op == Py_None) { @@ -406,7 +406,7 @@ LONGDOUBLE_setitem(PyObject *op, void *ov, void *vap) } if (PyArray_IsScalar(op, LongDouble)) { - temp = ((PyLongDoubleScalarObject *)op)->obval; + temp = PyArrayScalar_VAL(op, LongDouble); } else { /* In case something funny happened in PyArray_IsScalar */ @@ -450,12 +450,6 @@ static int UNICODE_setitem(PyObject *op, void *ov, void *vap) { PyArrayObject *ap = vap; - PyObject *temp; - Py_UNICODE *ptr; - int datalen; -#ifndef Py_UNICODE_WIDE - char *buffer; -#endif if (PyArray_IsZeroDim(op)) { return convert_to_scalar_and_retry(op, ov, vap, UNICODE_setitem); @@ -466,6 +460,8 @@ UNICODE_setitem(PyObject *op, void *ov, void *vap) "setting an array element with a sequence"); return -1; } + + PyObject *temp; if (PyBytes_Check(op)) { /* Try to decode from ASCII */ temp = PyUnicode_FromEncodedObject(op, "ASCII", "strict"); @@ -476,18 +472,27 @@ UNICODE_setitem(PyObject *op, void *ov, void *vap) else if ((temp=PyObject_Str(op)) == NULL) { return -1; } - ptr = PyUnicode_AS_UNICODE(temp); - if ((ptr == NULL) || (PyErr_Occurred())) { + + /* truncate if needed */ + Py_ssize_t max_len = PyArray_DESCR(ap)->elsize >> 2; + Py_ssize_t actual_len = PyUnicode_GetLength(temp); + if (actual_len < 0) { Py_DECREF(temp); return -1; } - datalen = PyUnicode_GET_DATA_SIZE(temp); + if (actual_len > max_len) { + Py_SETREF(temp, PyUnicode_Substring(temp, 0, max_len)); + if (temp == NULL) { + return -1; + } + actual_len = max_len; + } -#ifdef Py_UNICODE_WIDE - memcpy(ov, ptr, PyArray_MIN(PyArray_DESCR(ap)->elsize, datalen)); -#else + Py_ssize_t num_bytes = actual_len * 4; + + char *buffer; if (!PyArray_ISALIGNED(ap)) { - buffer = PyArray_malloc(PyArray_DESCR(ap)->elsize); + buffer = PyArray_malloc(num_bytes); if (buffer == NULL) { Py_DECREF(temp); PyErr_NoMemory(); @@ -497,20 +502,23 @@ UNICODE_setitem(PyObject *op, void *ov, void *vap) else { buffer = ov; } - datalen = PyUCS2Buffer_AsUCS4(ptr, (npy_ucs4 *)buffer, - datalen >> 1, PyArray_DESCR(ap)->elsize >> 2); - datalen <<= 2; + if (PyUnicode_AsUCS4(temp, (Py_UCS4 *)buffer, actual_len, 0) == NULL) { + PyArray_free(buffer); + Py_DECREF(temp); + return -1; + } + if (!PyArray_ISALIGNED(ap)) { - memcpy(ov, buffer, datalen); + memcpy(ov, buffer, num_bytes); PyArray_free(buffer); } -#endif + /* Fill in the rest of the space with 0 */ - if (PyArray_DESCR(ap)->elsize > datalen) { - memset((char*)ov + datalen, 0, (PyArray_DESCR(ap)->elsize - datalen)); + if (PyArray_DESCR(ap)->elsize > num_bytes) { + memset((char*)ov + num_bytes, 0, (PyArray_DESCR(ap)->elsize - num_bytes)); } if (PyArray_ISBYTESWAPPED(ap)) { - byte_swap_vector(ov, PyArray_DESCR(ap)->elsize >> 2, 4); + byte_swap_vector(ov, actual_len, 4); } Py_DECREF(temp); return 0; @@ -1498,70 +1506,8 @@ OBJECT_to_@TOTYPE@(void *input, void *output, npy_intp n, * #oskip = 1*18,(PyArray_DESCR(aop)->elsize)*3,1*2, * 1*18,(PyArray_DESCR(aop)->elsize)*3,1*2, * 1*18,(PyArray_DESCR(aop)->elsize)*3,1*2# - * #convert = 1*18, 0*3, 1*2, - * 1*18, 0*3, 1*2, - * 0*23# - * #convstr = (Long*9, Long*2, Float*4, Complex*3, Tuple*3, Long*2)*3# */ -#if @convert@ - -#define IS_@from@ - -static void -@from@_to_@to@(void *input, void *output, npy_intp n, - void *vaip, void *aop) -{ - @fromtyp@ *ip = input; - @totyp@ *op = output; - PyArrayObject *aip = vaip; - - npy_intp i; - int skip = PyArray_DESCR(aip)->elsize; - int oskip = @oskip@; - - for (i = 0; i < n; i++, ip+=skip, op+=oskip) { - PyObject *new; - PyObject *temp = PyArray_Scalar(ip, PyArray_DESCR(aip), (PyObject *)aip); - if (temp == NULL) { - return; - } - -#if defined(IS_STRING) - /* Work around some Python 3K */ - new = PyUnicode_FromEncodedObject(temp, "ascii", "strict"); - Py_DECREF(temp); - temp = new; - if (temp == NULL) { - return; - } -#endif - /* convert from Python object to needed one */ - { - PyObject *args; - - /* call out to the Python builtin given by convstr */ - args = Py_BuildValue("(N)", temp); - new = Py@convstr@_Type.tp_new(&Py@convstr@_Type, args, NULL); - Py_DECREF(args); - temp = new; - if (temp == NULL) { - return; - } - } - - if (@to@_setitem(temp, op, aop)) { - Py_DECREF(temp); - return; - } - Py_DECREF(temp); - } -} - -#undef IS_@from@ - -#else - static void @from@_to_@to@(void *input, void *output, npy_intp n, void *vaip, void *aop) @@ -1587,7 +1533,6 @@ static void } } -#endif /**end repeat**/ @@ -2290,6 +2235,7 @@ static void STRING_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, npy_intp n, int NPY_UNUSED(swap), PyArrayObject *arr) { + assert(arr != NULL); if (arr == NULL) { return; } @@ -2304,6 +2250,7 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, { PyArray_Descr *descr; + assert(arr != NULL); if (arr == NULL) { return; } @@ -2394,6 +2341,7 @@ VOID_copyswap (char *dst, char *src, int swap, PyArrayObject *arr) { PyArray_Descr *descr; + assert(arr != NULL); if (arr == NULL) { return; } @@ -2475,6 +2423,7 @@ UNICODE_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, { int itemsize; + assert(arr != NULL); if (arr == NULL) { return; } @@ -2502,6 +2451,7 @@ UNICODE_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, static void STRING_copyswap(char *dst, char *src, int NPY_UNUSED(swap), PyArrayObject *arr) { + assert(arr != NULL); if (arr == NULL) { return; } @@ -2514,6 +2464,7 @@ UNICODE_copyswap (char *dst, char *src, int swap, PyArrayObject *arr) { int itemsize; + assert(arr != NULL); if (arr == NULL) { return; } @@ -2644,12 +2595,6 @@ STRING_nonzero (char *ip, PyArrayObject *ap) return nonz; } -#ifdef Py_UNICODE_WIDE -#define PyArray_UCS4_ISSPACE Py_UNICODE_ISSPACE -#else -#define PyArray_UCS4_ISSPACE(ch) Py_STRING_ISSPACE((char)ch) -#endif - static npy_bool UNICODE_nonzero (npy_ucs4 *ip, PyArrayObject *ap) { @@ -2675,7 +2620,7 @@ UNICODE_nonzero (npy_ucs4 *ip, PyArrayObject *ap) if (*ip == '\0') { seen_null = NPY_TRUE; } - else if (seen_null || !PyArray_UCS4_ISSPACE(*ip)) { + else if (seen_null || !Py_UNICODE_ISSPACE(*ip)) { nonz = NPY_TRUE; break; } diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 576186362..9a1f7b230 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -832,11 +832,6 @@ gentype_getbuffer(PyObject *self, Py_buffer *view, int flags) descr = PyArray_DescrFromScalar(self); view->buf = (void *)scalar_value(self, descr); elsize = descr->elsize; -#ifndef Py_UNICODE_WIDE - if (descr->type_num == NPY_UNICODE) { - elsize >>= 1; - } -#endif view->len = elsize; if (PyArray_IsScalar(self, Datetime) || PyArray_IsScalar(self, Timedelta)) { elsize = 1; /* descr->elsize,char is 8,'M', but we return 1,'B' */ diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index c9ec32268..0150ae10e 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -121,6 +121,57 @@ PyArray_DTypeFromObject(PyObject *obj, int maxdims, PyArray_Descr **out_dtype) return res; } +/* + * Get a suitable string dtype by calling `__str__`. + * For `np.bytes_`, this assumes an ASCII encoding. + */ +static PyArray_Descr * +PyArray_DTypeFromObjectStringDiscovery( + PyObject *obj, PyArray_Descr *last_dtype, int string_type) +{ + int itemsize; + + if (string_type == NPY_STRING) { + PyObject *temp = PyObject_Str(obj); + if (temp == NULL) { + return NULL; + } + /* assume that when we do the encoding elsewhere we'll use ASCII */ + itemsize = PyUnicode_GetLength(temp); + Py_DECREF(temp); + if (itemsize < 0) { + return NULL; + } + } + else if (string_type == NPY_UNICODE) { + PyObject *temp = PyObject_Str(obj); + if (temp == NULL) { + return NULL; + } + itemsize = PyUnicode_GetLength(temp); + Py_DECREF(temp); + if (itemsize < 0) { + return NULL; + } + itemsize *= 4; /* convert UCS4 codepoints to bytes */ + } + else { + return NULL; + } + if (last_dtype != NULL && + last_dtype->type_num == string_type && + last_dtype->elsize >= itemsize) { + Py_INCREF(last_dtype); + return last_dtype; + } + PyArray_Descr *dtype = PyArray_DescrNewFromType(string_type); + if (dtype == NULL) { + return NULL; + } + dtype->elsize = itemsize; + return dtype; +} + NPY_NO_EXPORT int PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, PyArray_Descr **out_dtype, int string_type) @@ -158,38 +209,17 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, } } else { - int itemsize; - PyObject *temp; - - if (string_type == NPY_STRING) { - if ((temp = PyObject_Str(obj)) == NULL) { - goto fail; - } - itemsize = PyUnicode_GetLength(temp); - } - else if (string_type == NPY_UNICODE) { - if ((temp = PyObject_Str(obj)) == NULL) { - goto fail; - } - itemsize = PyUnicode_GET_DATA_SIZE(temp); -#ifndef Py_UNICODE_WIDE - itemsize <<= 1; -#endif - } - else { + dtype = PyArray_DTypeFromObjectStringDiscovery( + obj, *out_dtype, string_type); + if (dtype == NULL) { goto fail; } - Py_DECREF(temp); - if (*out_dtype != NULL && - (*out_dtype)->type_num == string_type && - (*out_dtype)->elsize >= itemsize) { + + /* nothing to do, dtype is already correct */ + if (dtype == *out_dtype){ + Py_DECREF(dtype); return 0; } - dtype = PyArray_DescrNewFromType(string_type); - if (dtype == NULL) { - goto fail; - } - dtype->elsize = itemsize; } goto promote_types; } @@ -198,42 +228,19 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, dtype = _array_find_python_scalar_type(obj); if (dtype != NULL) { if (string_type) { - int itemsize; - PyObject *temp; - /* dtype is not used in this (string discovery) branch */ Py_DECREF(dtype); - dtype = NULL; - - if (string_type == NPY_STRING) { - if ((temp = PyObject_Str(obj)) == NULL) { - goto fail; - } - itemsize = PyUnicode_GetLength(temp); - } - else if (string_type == NPY_UNICODE) { - if ((temp = PyObject_Str(obj)) == NULL) { - goto fail; - } - itemsize = PyUnicode_GET_DATA_SIZE(temp); -#ifndef Py_UNICODE_WIDE - itemsize <<= 1; -#endif - } - else { + dtype = PyArray_DTypeFromObjectStringDiscovery( + obj, *out_dtype, string_type); + if (dtype == NULL) { goto fail; } - Py_DECREF(temp); - if (*out_dtype != NULL && - (*out_dtype)->type_num == string_type && - (*out_dtype)->elsize >= itemsize) { + + /* nothing to do, dtype is already correct */ + if (dtype == *out_dtype){ + Py_DECREF(dtype); return 0; } - dtype = PyArray_DescrNewFromType(string_type); - if (dtype == NULL) { - goto fail; - } - dtype->elsize = itemsize; } goto promote_types; } @@ -258,10 +265,11 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, /* Check if it's a Unicode string */ if (PyUnicode_Check(obj)) { - int itemsize = PyUnicode_GET_DATA_SIZE(obj); -#ifndef Py_UNICODE_WIDE - itemsize <<= 1; -#endif + int itemsize = PyUnicode_GetLength(obj); + if (itemsize < 0) { + goto fail; + } + itemsize *= 4; /* * If it's already a big enough unicode object, diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 484a13134..14d546867 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -137,6 +137,20 @@ PyArray_IntpConverter(PyObject *obj, PyArray_Dims *seq) return NPY_SUCCEED; } +/* + * Like PyArray_IntpConverter, but leaves `seq` untouched if `None` is passed + * rather than treating `None` as `()`. + */ +NPY_NO_EXPORT int +PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq) +{ + if (obj == Py_None) { + return NPY_SUCCEED; + } + + return PyArray_IntpConverter(obj, seq); +} + /*NUMPY_API * Get buffer chunk from object * @@ -308,100 +322,112 @@ PyArray_BoolConverter(PyObject *object, npy_bool *val) return NPY_SUCCEED; } -/*NUMPY_API - * Convert object to endian - */ -NPY_NO_EXPORT int -PyArray_ByteorderConverter(PyObject *obj, char *endian) +static int +string_converter_helper( + PyObject *object, + void *out, + int (*str_func)(char const*, Py_ssize_t, void*), + char const *name, + char const *message) { - char *str; - PyObject *tmp = NULL; - - if (PyUnicode_Check(obj)) { - obj = tmp = PyUnicode_AsASCIIString(obj); + /* allow bytes for compatibility */ + PyObject *str_object = NULL; + if (PyBytes_Check(object)) { + str_object = PyUnicode_FromEncodedObject(object, NULL, NULL); + if (str_object == NULL) { + PyErr_Format(PyExc_ValueError, + "%s %s (got %R)", name, message, object); + return NPY_FAIL; + } } - - *endian = NPY_SWAP; - str = PyBytes_AsString(obj); - if (!str) { - Py_XDECREF(tmp); + else if (PyUnicode_Check(object)) { + str_object = object; + Py_INCREF(str_object); + } + else { + PyErr_Format(PyExc_TypeError, + "%s must be str, not %s", name, Py_TYPE(object)->tp_name); return NPY_FAIL; } - if (strlen(str) < 1) { - PyErr_SetString(PyExc_ValueError, - "Byteorder string must be at least length 1"); - Py_XDECREF(tmp); + + Py_ssize_t length; + char const *str = PyUnicode_AsUTF8AndSize(str_object, &length); + if (str == NULL) { + Py_DECREF(str_object); return NPY_FAIL; } - *endian = str[0]; - if (str[0] != NPY_BIG && str[0] != NPY_LITTLE - && str[0] != NPY_NATIVE && str[0] != NPY_IGNORE) { - if (str[0] == 'b' || str[0] == 'B') { - *endian = NPY_BIG; - } - else if (str[0] == 'l' || str[0] == 'L') { - *endian = NPY_LITTLE; - } - else if (str[0] == 'n' || str[0] == 'N') { - *endian = NPY_NATIVE; - } - else if (str[0] == 'i' || str[0] == 'I') { - *endian = NPY_IGNORE; - } - else if (str[0] == 's' || str[0] == 'S') { - *endian = NPY_SWAP; - } - else { + + int ret = str_func(str, length, out); + Py_DECREF(str_object); + if (ret < 0) { PyErr_Format(PyExc_ValueError, - "%s is an unrecognized byteorder", - str); - Py_XDECREF(tmp); - return NPY_FAIL; - } + "%s %s (got %R)", name, message, object); + return NPY_FAIL; } - Py_XDECREF(tmp); return NPY_SUCCEED; } -/*NUMPY_API - * Convert object to sort kind - */ -NPY_NO_EXPORT int -PyArray_SortkindConverter(PyObject *obj, NPY_SORTKIND *sortkind) +static int byteorder_parser(char const *str, Py_ssize_t length, void *data) { - char *str; - PyObject *tmp = NULL; + char *endian = (char *)data; - if (obj == Py_None) { - *sortkind = NPY_QUICKSORT; - return NPY_SUCCEED; + if (length < 1) { + return -1; } - - if (PyUnicode_Check(obj)) { - obj = tmp = PyUnicode_AsASCIIString(obj); - if (obj == NULL) { - return NPY_FAIL; - } + else if (str[0] == NPY_BIG || str[0] == NPY_LITTLE + || str[0] == NPY_NATIVE || str[0] == NPY_IGNORE) { + *endian = str[0]; + return 0; + } + else if (str[0] == 'b' || str[0] == 'B') { + *endian = NPY_BIG; + return 0; + } + else if (str[0] == 'l' || str[0] == 'L') { + *endian = NPY_LITTLE; + return 0; + } + else if (str[0] == 'n' || str[0] == 'N') { + *endian = NPY_NATIVE; + return 0; } + else if (str[0] == 'i' || str[0] == 'I') { + *endian = NPY_IGNORE; + return 0; + } + else if (str[0] == 's' || str[0] == 'S') { + *endian = NPY_SWAP; + return 0; + } + else { + return -1; + } +} - *sortkind = NPY_QUICKSORT; +/*NUMPY_API + * Convert object to endian + */ +NPY_NO_EXPORT int +PyArray_ByteorderConverter(PyObject *obj, char *endian) +{ + return string_converter_helper( + obj, (void *)endian, byteorder_parser, "byteorder", "not recognized"); +} - str = PyBytes_AsString(obj); - if (!str) { - Py_XDECREF(tmp); - return NPY_FAIL; - } - if (strlen(str) < 1) { - PyErr_SetString(PyExc_ValueError, - "Sort kind string must be at least length 1"); - Py_XDECREF(tmp); - return NPY_FAIL; +static int sortkind_parser(char const *str, Py_ssize_t length, void *data) +{ + NPY_SORTKIND *sortkind = (NPY_SORTKIND *)data; + + if (length < 1) { + return -1; } if (str[0] == 'q' || str[0] == 'Q') { *sortkind = NPY_QUICKSORT; + return 0; } else if (str[0] == 'h' || str[0] == 'H') { *sortkind = NPY_HEAPSORT; + return 0; } else if (str[0] == 'm' || str[0] == 'M') { /* @@ -410,6 +436,7 @@ PyArray_SortkindConverter(PyObject *obj, NPY_SORTKIND *sortkind) * allowing other types of stable sorts to be used. */ *sortkind = NPY_MERGESORT; + return 0; } else if (str[0] == 's' || str[0] == 'S') { /* @@ -421,16 +448,39 @@ PyArray_SortkindConverter(PyObject *obj, NPY_SORTKIND *sortkind) * Which one is used depends on the data type. */ *sortkind = NPY_STABLESORT; + return 0; } else { - PyErr_Format(PyExc_ValueError, - "%s is an unrecognized kind of sort", - str); - Py_XDECREF(tmp); - return NPY_FAIL; + return -1; + } +} + +/*NUMPY_API + * Convert object to sort kind + */ +NPY_NO_EXPORT int +PyArray_SortkindConverter(PyObject *obj, NPY_SORTKIND *sortkind) +{ + /* Leave the desired default from the caller for Py_None */ + if (obj == Py_None) { + return NPY_SUCCEED; + } + return string_converter_helper( + obj, (void *)sortkind, sortkind_parser, "sort kind", + "must be one of 'quick', 'heap', or 'stable'"); +} + +static int selectkind_parser(char const *str, Py_ssize_t length, void *data) +{ + NPY_SELECTKIND *selectkind = (NPY_SELECTKIND *)data; + + if (length == 11 && strcmp(str, "introselect") == 0) { + *selectkind = NPY_INTROSELECT; + return 0; + } + else { + return -1; } - Py_XDECREF(tmp); - return NPY_SUCCEED; } /*NUMPY_API @@ -439,40 +489,29 @@ PyArray_SortkindConverter(PyObject *obj, NPY_SORTKIND *sortkind) NPY_NO_EXPORT int PyArray_SelectkindConverter(PyObject *obj, NPY_SELECTKIND *selectkind) { - char *str; - PyObject *tmp = NULL; + return string_converter_helper( + obj, (void *)selectkind, selectkind_parser, "select kind", + "must be 'introselect'"); +} - if (PyUnicode_Check(obj)) { - obj = tmp = PyUnicode_AsASCIIString(obj); - if (obj == NULL) { - return NPY_FAIL; - } - } +static int searchside_parser(char const *str, Py_ssize_t length, void *data) +{ + NPY_SEARCHSIDE *side = (NPY_SEARCHSIDE *)data; - *selectkind = NPY_INTROSELECT; - str = PyBytes_AsString(obj); - if (!str) { - Py_XDECREF(tmp); - return NPY_FAIL; + if (length < 1) { + return -1; } - if (strlen(str) < 1) { - PyErr_SetString(PyExc_ValueError, - "Select kind string must be at least length 1"); - Py_XDECREF(tmp); - return NPY_FAIL; + else if (str[0] == 'l' || str[0] == 'L') { + *side = NPY_SEARCHLEFT; + return 0; } - if (strcmp(str, "introselect") == 0) { - *selectkind = NPY_INTROSELECT; + else if (str[0] == 'r' || str[0] == 'R') { + *side = NPY_SEARCHRIGHT; + return 0; } else { - PyErr_Format(PyExc_ValueError, - "%s is an unrecognized kind of select", - str); - Py_XDECREF(tmp); - return NPY_FAIL; + return -1; } - Py_XDECREF(tmp); - return NPY_SUCCEED; } /*NUMPY_API @@ -481,36 +520,36 @@ PyArray_SelectkindConverter(PyObject *obj, NPY_SELECTKIND *selectkind) NPY_NO_EXPORT int PyArray_SearchsideConverter(PyObject *obj, void *addr) { - NPY_SEARCHSIDE *side = (NPY_SEARCHSIDE *)addr; - char *str; - PyObject *tmp = NULL; + return string_converter_helper( + obj, addr, searchside_parser, "search side", + "must be 'left' or 'right'"); +} - if (PyUnicode_Check(obj)) { - obj = tmp = PyUnicode_AsASCIIString(obj); +static int order_parser(char const *str, Py_ssize_t length, void *data) +{ + NPY_ORDER *val = (NPY_ORDER *)data; + if (length != 1) { + return -1; } - - str = PyBytes_AsString(obj); - if (!str || strlen(str) < 1) { - PyErr_SetString(PyExc_ValueError, - "expected nonempty string for keyword 'side'"); - Py_XDECREF(tmp); - return NPY_FAIL; + if (str[0] == 'C' || str[0] == 'c') { + *val = NPY_CORDER; + return 0; } - - if (str[0] == 'l' || str[0] == 'L') { - *side = NPY_SEARCHLEFT; + else if (str[0] == 'F' || str[0] == 'f') { + *val = NPY_FORTRANORDER; + return 0; } - else if (str[0] == 'r' || str[0] == 'R') { - *side = NPY_SEARCHRIGHT; + else if (str[0] == 'A' || str[0] == 'a') { + *val = NPY_ANYORDER; + return 0; + } + else if (str[0] == 'K' || str[0] == 'k') { + *val = NPY_KEEPORDER; + return 0; } else { - PyErr_Format(PyExc_ValueError, - "'%s' is an invalid value for keyword 'side'", str); - Py_XDECREF(tmp); - return NPY_FAIL; + return -1; } - Py_XDECREF(tmp); - return NPY_SUCCEED; } /*NUMPY_API @@ -519,59 +558,36 @@ PyArray_SearchsideConverter(PyObject *obj, void *addr) NPY_NO_EXPORT int PyArray_OrderConverter(PyObject *object, NPY_ORDER *val) { - char *str; - /* Leave the desired default from the caller for NULL/Py_None */ - if (object == NULL || object == Py_None) { + /* Leave the desired default from the caller for Py_None */ + if (object == Py_None) { return NPY_SUCCEED; } - else if (PyUnicode_Check(object)) { - PyObject *tmp; - int ret; - tmp = PyUnicode_AsASCIIString(object); - if (tmp == NULL) { - PyErr_SetString(PyExc_ValueError, - "Invalid unicode string passed in for the array ordering. " - "Please pass in 'C', 'F', 'A' or 'K' instead"); - return NPY_FAIL; - } - ret = PyArray_OrderConverter(tmp, val); - Py_DECREF(tmp); - return ret; - } - else if (!PyBytes_Check(object) || PyBytes_GET_SIZE(object) < 1) { - PyErr_SetString(PyExc_ValueError, - "Non-string object detected for the array ordering. " - "Please pass in 'C', 'F', 'A', or 'K' instead"); - return NPY_FAIL; + return string_converter_helper( + object, (void *)val, order_parser, "order", + "must be one of 'C', 'F', 'A', or 'K'"); +} + +static int clipmode_parser(char const *str, Py_ssize_t length, void *data) +{ + NPY_CLIPMODE *val = (NPY_CLIPMODE *)data; + if (length < 1) { + return -1; + } + if (str[0] == 'C' || str[0] == 'c') { + *val = NPY_CLIP; + return 0; + } + else if (str[0] == 'W' || str[0] == 'w') { + *val = NPY_WRAP; + return 0; + } + else if (str[0] == 'R' || str[0] == 'r') { + *val = NPY_RAISE; + return 0; } else { - str = PyBytes_AS_STRING(object); - if (strlen(str) != 1) { - PyErr_SetString(PyExc_ValueError, - "Non-string object detected for the array ordering. " - "Please pass in 'C', 'F', 'A', or 'K' instead"); - return NPY_FAIL; - } - - if (str[0] == 'C' || str[0] == 'c') { - *val = NPY_CORDER; - } - else if (str[0] == 'F' || str[0] == 'f') { - *val = NPY_FORTRANORDER; - } - else if (str[0] == 'A' || str[0] == 'a') { - *val = NPY_ANYORDER; - } - else if (str[0] == 'K' || str[0] == 'k') { - *val = NPY_KEEPORDER; - } - else { - PyErr_SetString(PyExc_TypeError, - "order not understood"); - return NPY_FAIL; - } + return -1; } - return NPY_SUCCEED; } /*NUMPY_API @@ -583,36 +599,14 @@ PyArray_ClipmodeConverter(PyObject *object, NPY_CLIPMODE *val) if (object == NULL || object == Py_None) { *val = NPY_RAISE; } - else if (PyBytes_Check(object)) { - char *str; - str = PyBytes_AS_STRING(object); - if (str[0] == 'C' || str[0] == 'c') { - *val = NPY_CLIP; - } - else if (str[0] == 'W' || str[0] == 'w') { - *val = NPY_WRAP; - } - else if (str[0] == 'R' || str[0] == 'r') { - *val = NPY_RAISE; - } - else { - PyErr_SetString(PyExc_TypeError, - "clipmode not understood"); - return NPY_FAIL; - } - } - else if (PyUnicode_Check(object)) { - PyObject *tmp; - int ret; - tmp = PyUnicode_AsASCIIString(object); - if (tmp == NULL) { - return NPY_FAIL; - } - ret = PyArray_ClipmodeConverter(tmp, val); - Py_DECREF(tmp); - return ret; + + else if (PyBytes_Check(object) || PyUnicode_Check(object)) { + return string_converter_helper( + object, (void *)val, clipmode_parser, "clipmode", + "must be one of 'clip', 'raise', or 'wrap'"); } else { + /* For users passing `np.RAISE`, `np.WRAP`, `np.CLIP` */ int number = PyArray_PyIntAsInt(object); if (error_converting(number)) { goto fail; @@ -622,7 +616,8 @@ PyArray_ClipmodeConverter(PyObject *object, NPY_CLIPMODE *val) *val = (NPY_CLIPMODE) number; } else { - goto fail; + PyErr_Format(PyExc_ValueError, + "integer clipmode must be np.RAISE, np.WRAP, or np.CLIP"); } } return NPY_SUCCEED; @@ -676,66 +671,56 @@ PyArray_ConvertClipmodeSequence(PyObject *object, NPY_CLIPMODE *modes, int n) return NPY_SUCCEED; } +static int casting_parser(char const *str, Py_ssize_t length, void *data) +{ + NPY_CASTING *casting = (NPY_CASTING *)data; + if (length < 2) { + return -1; + } + switch (str[2]) { + case 0: + if (length == 2 && strcmp(str, "no") == 0) { + *casting = NPY_NO_CASTING; + return 0; + } + break; + case 'u': + if (length == 5 && strcmp(str, "equiv") == 0) { + *casting = NPY_EQUIV_CASTING; + return 0; + } + break; + case 'f': + if (length == 4 && strcmp(str, "safe") == 0) { + *casting = NPY_SAFE_CASTING; + return 0; + } + break; + case 'm': + if (length == 9 && strcmp(str, "same_kind") == 0) { + *casting = NPY_SAME_KIND_CASTING; + return 0; + } + break; + case 's': + if (length == 6 && strcmp(str, "unsafe") == 0) { + *casting = NPY_UNSAFE_CASTING; + return 0; + } + break; + } + return -1; +} + /*NUMPY_API * Convert any Python object, *obj*, to an NPY_CASTING enum. */ NPY_NO_EXPORT int PyArray_CastingConverter(PyObject *obj, NPY_CASTING *casting) { - char *str = NULL; - Py_ssize_t length = 0; - - if (PyUnicode_Check(obj)) { - PyObject *str_obj; - int ret; - str_obj = PyUnicode_AsASCIIString(obj); - if (str_obj == NULL) { - return 0; - } - ret = PyArray_CastingConverter(str_obj, casting); - Py_DECREF(str_obj); - return ret; - } - - if (PyBytes_AsStringAndSize(obj, &str, &length) < 0) { - return 0; - } - - if (length >= 2) switch (str[2]) { - case 0: - if (strcmp(str, "no") == 0) { - *casting = NPY_NO_CASTING; - return 1; - } - break; - case 'u': - if (strcmp(str, "equiv") == 0) { - *casting = NPY_EQUIV_CASTING; - return 1; - } - break; - case 'f': - if (strcmp(str, "safe") == 0) { - *casting = NPY_SAFE_CASTING; - return 1; - } - break; - case 'm': - if (strcmp(str, "same_kind") == 0) { - *casting = NPY_SAME_KIND_CASTING; - return 1; - } - break; - case 's': - if (strcmp(str, "unsafe") == 0) { - *casting = NPY_UNSAFE_CASTING; - return 1; - } - break; - } - - PyErr_SetString(PyExc_ValueError, - "casting must be one of 'no', 'equiv', 'safe', " + return string_converter_helper( + obj, (void *)casting, casting_parser, "casting", + "must be one of 'no', 'equiv', 'safe', " "'same_kind', or 'unsafe'"); return 0; } diff --git a/numpy/core/src/multiarray/conversion_utils.h b/numpy/core/src/multiarray/conversion_utils.h index 9bf712c3b..bee0c6064 100644 --- a/numpy/core/src/multiarray/conversion_utils.h +++ b/numpy/core/src/multiarray/conversion_utils.h @@ -7,6 +7,9 @@ NPY_NO_EXPORT int PyArray_IntpConverter(PyObject *obj, PyArray_Dims *seq); NPY_NO_EXPORT int +PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq); + +NPY_NO_EXPORT int PyArray_BufferConverter(PyObject *obj, PyArray_Chunk *buf); NPY_NO_EXPORT int diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index d4774c2b2..d59a62ed8 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -13,6 +13,7 @@ #include "numpy/npy_math.h" #include "common.h" +#include "ctors.h" #include "scalartypes.h" #include "mapping.h" @@ -212,13 +213,17 @@ PyArray_AdaptFlexibleDType(PyObject *data_obj, PyArray_Descr *data_dtype, case NPY_HALF: case NPY_FLOAT: case NPY_DOUBLE: - case NPY_LONGDOUBLE: size = 32; break; + case NPY_LONGDOUBLE: + size = 48; + break; case NPY_CFLOAT: case NPY_CDOUBLE: + size = 2 * 32; + break; case NPY_CLONGDOUBLE: - size = 64; + size = 2 * 48; break; case NPY_OBJECT: size = 64; @@ -255,11 +260,11 @@ PyArray_AdaptFlexibleDType(PyObject *data_obj, PyArray_Descr *data_dtype, int ndim = 0; npy_intp dims[NPY_MAXDIMS]; list = PyArray_ToList((PyArrayObject *)data_obj); - result = PyArray_GetArrayParamsFromObject( + result = PyArray_GetArrayParamsFromObject_int( list, retval, 0, &dtype, - &ndim, dims, &arr, NULL); + &ndim, dims, &arr); Py_DECREF(list); Py_XDECREF(arr); if (result < 0) { diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 0616bed65..12bf9eace 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -53,6 +53,9 @@ typedef int (*next_element)(void **, void *, PyArray_Descr *, void *); typedef int (*skip_separator)(void **, const char *, void *); +static PyObject * +_array_from_array_like(PyObject *op, + PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context); static npy_bool string_is_fully_read(char const* start, char const* end) { @@ -509,10 +512,29 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, goto fail; } + /* Try __array__ before using s as a sequence */ + PyObject *tmp = _array_from_array_like(s, NULL, 0, NULL); + if (tmp == NULL) { + goto fail; + } + else if (tmp == Py_NotImplemented) { + Py_DECREF(tmp); + } + else { + int r = PyArray_CopyInto(dst, (PyArrayObject *)tmp); + Py_DECREF(tmp); + if (r < 0) { + goto fail; + } + Py_DECREF(s); + return 0; + } + slen = PySequence_Length(s); if (slen < 0) { goto fail; } + /* * Either the dimensions match, or the sequence has length 1 and can * be broadcast to the destination. @@ -678,12 +700,25 @@ discover_itemsize(PyObject *s, int nd, int *itemsize, int string_type) return 0; } + typedef enum { DISCOVERED_OK = 0, DISCOVERED_RAGGED = 1, DISCOVERED_OBJECT = 2 } discovered_t; + +static void +_discover_dimensions_array(PyArrayObject *arr, int *maxndim, npy_intp *d) { + if (PyArray_NDIM(arr) < *maxndim) { + *maxndim = PyArray_NDIM(arr); + } + for (int i = 0; i < *maxndim; i++) { + d[i] = PyArray_DIM(arr, i); + } +} + + /* * Take an arbitrary object and discover how many dimensions it * has, filling in the dimensions as we go. @@ -695,7 +730,6 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, { PyObject *e; npy_intp n, i; - Py_buffer buffer_view; PyObject * seq; if (*maxndim == 0) { @@ -704,15 +738,7 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, /* obj is an Array */ if (PyArray_Check(obj)) { - PyArrayObject *arr = (PyArrayObject *)obj; - - if (PyArray_NDIM(arr) < *maxndim) { - *maxndim = PyArray_NDIM(arr); - } - - for (i=0; i<*maxndim; i++) { - d[i] = PyArray_DIM(arr,i); - } + _discover_dimensions_array((PyArrayObject *)obj, maxndim, d); return 0; } @@ -749,115 +775,24 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, return 0; } - /* obj is a PEP 3118 buffer */ - /* PEP 3118 buffer interface */ - if (PyObject_CheckBuffer(obj) == 1) { - memset(&buffer_view, 0, sizeof(Py_buffer)); - if (PyObject_GetBuffer(obj, &buffer_view, - PyBUF_STRIDES|PyBUF_SIMPLE) == 0 || - PyObject_GetBuffer(obj, &buffer_view, - PyBUF_ND|PyBUF_SIMPLE) == 0) { - int nd = buffer_view.ndim; - - if (nd < *maxndim) { - *maxndim = nd; - } - for (i = 0; i < *maxndim; i++) { - d[i] = buffer_view.shape[i]; - } - PyBuffer_Release(&buffer_view); - _dealloc_cached_buffer_info(obj); - return 0; - } - else if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_BufferError) || - PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Clear(); - } else { - return -1; - } - } - else if (PyObject_GetBuffer(obj, &buffer_view, PyBUF_SIMPLE) == 0) { - d[0] = buffer_view.len; - *maxndim = 1; - PyBuffer_Release(&buffer_view); - _dealloc_cached_buffer_info(obj); - return 0; - } - else if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_BufferError) || - PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Clear(); - } else { - return -1; - } - } - } - - /* obj has the __array_struct__ interface */ - e = PyArray_LookupSpecial_OnInstance(obj, "__array_struct__"); - if (e != NULL) { - int nd = -1; - - if (NpyCapsule_Check(e)) { - PyArrayInterface *inter; - inter = (PyArrayInterface *)NpyCapsule_AsVoidPtr(e); - if (inter->two == 2) { - nd = inter->nd; - if (nd >= 0) { - if (nd < *maxndim) { - *maxndim = nd; - } - for (i=0; i<*maxndim; i++) { - d[i] = inter->shape[i]; - } - } - } - } + /* + * In the future, the result of `_array_from_array_like` should possibly + * be cached. This may require passing the correct dtype/writable + * information already in the dimension discovery step (if they are + * distinct steps). + */ + e = _array_from_array_like(obj, NULL, NPY_FALSE, NULL); + if (e == Py_NotImplemented) { Py_DECREF(e); - if (nd >= 0) { - return 0; - } - } - else if (PyErr_Occurred()) { - PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ } - - - /* obj has the __array_interface__ interface */ - e = PyArray_LookupSpecial_OnInstance(obj, "__array_interface__"); - if (e != NULL) { - int nd = -1; - if (PyDict_Check(e)) { - PyObject *new; - new = _PyDict_GetItemStringWithError(e, "shape"); - if (new == NULL && PyErr_Occurred()) { - Py_DECREF(e); - return -1; - } - if (new && PyTuple_Check(new)) { - nd = PyTuple_GET_SIZE(new); - if (nd < *maxndim) { - *maxndim = nd; - } - for (i=0; i<*maxndim; i++) { - d[i] = PyInt_AsSsize_t(PyTuple_GET_ITEM(new, i)); - if (d[i] < 0) { - PyErr_SetString(PyExc_RuntimeError, - "Invalid shape in __array_interface__"); - Py_DECREF(e); - return -1; - } - } - } - } + else if (e != NULL) { + _discover_dimensions_array((PyArrayObject *)e, maxndim, d); Py_DECREF(e); - if (nd >= 0) { - return 0; - } + return 0; } else if (PyErr_Occurred()) { - PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + /* TODO[gh-14801]: propagate crashes during attribute access? */ + PyErr_Clear(); } seq = PySequence_Fast(obj, "Could not convert object to sequence"); @@ -1276,8 +1211,8 @@ PyArray_NewFromDescrAndBase( * NPY_ANYORDER - Fortran if prototype is Fortran, C otherwise. * NPY_KEEPORDER - Keeps the axis ordering of prototype. * dtype - If not NULL, overrides the data type of the result. - * ndim - If not 0 and dims not NULL, overrides the shape of the result. - * dims - If not NULL and ndim not 0, overrides the shape of the result. + * ndim - If not -1, overrides the shape of the result. + * dims - If ndim is not -1, overrides the shape of the result. * subok - If 1, use the prototype's array subtype, otherwise * always create a base-class array. * @@ -1290,7 +1225,7 @@ PyArray_NewLikeArrayWithShape(PyArrayObject *prototype, NPY_ORDER order, { PyObject *ret = NULL; - if (dims == NULL) { + if (ndim == -1) { ndim = PyArray_NDIM(prototype); dims = PyArray_DIMS(prototype); } @@ -1387,7 +1322,7 @@ NPY_NO_EXPORT PyObject * PyArray_NewLikeArray(PyArrayObject *prototype, NPY_ORDER order, PyArray_Descr *dtype, int subok) { - return PyArray_NewLikeArrayWithShape(prototype, order, dtype, 0, NULL, subok); + return PyArray_NewLikeArrayWithShape(prototype, order, dtype, -1, NULL, subok); } /*NUMPY_API @@ -1574,7 +1509,107 @@ fail: } -/*NUMPY_API + +/** + * Attempts to extract an array from an array-like object. + * + * array-like is defined as either + * + * * an object implementing the PEP 3118 buffer interface; + * * an object with __array_struct__ or __array_interface__ attributes; + * * an object with an __array__ function. + * + * @param op The object to convert to an array + * @param requested_type a requested dtype instance, may be NULL; The result + * DType may be used, but is not enforced. + * @param writeable whether the result must be writeable. + * @param context Unused parameter, must be NULL (should be removed later). + * + * @returns The array object, Py_NotImplemented if op is not array-like, + * or NULL with an error set. (A new reference to Py_NotImplemented + * is returned.) + */ +static PyObject * +_array_from_array_like(PyObject *op, + PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context) { + PyObject* tmp; + + /* + * If op supports the PEP 3118 buffer interface. + * We skip bytes and unicode since they are considered scalars. Unicode + * would fail but bytes would be incorrectly converted to a uint8 array. + */ + if (!PyBytes_Check(op) && !PyUnicode_Check(op)) { + PyObject *memoryview = PyMemoryView_FromObject(op); + if (memoryview == NULL) { + PyErr_Clear(); + } + else { + tmp = _array_from_buffer_3118(memoryview); + Py_DECREF(memoryview); + if (tmp == NULL) { + return NULL; + } + + if (writeable + && PyArray_FailUnlessWriteable( + (PyArrayObject *)tmp, "PEP 3118 buffer") < 0) { + Py_DECREF(tmp); + return NULL; + } + + return tmp; + } + } + + /* + * If op supports the __array_struct__ or __array_interface__ interface. + */ + tmp = PyArray_FromStructInterface(op); + if (tmp == NULL) { + return NULL; + } + if (tmp == Py_NotImplemented) { + /* Until the return, NotImplemented is always a borrowed reference*/ + tmp = PyArray_FromInterface(op); + if (tmp == NULL) { + return NULL; + } + } + + /* + * If op supplies the __array__ function. + * The documentation says this should produce a copy, so + * we skip this method if writeable is true, because the intent + * of writeable is to modify the operand. + * XXX: If the implementation is wrong, and/or if actual + * usage requires this behave differently, + * this should be changed! + */ + if (!writeable && tmp == Py_NotImplemented) { + tmp = PyArray_FromArrayAttr(op, requested_dtype, context); + if (tmp == NULL) { + return NULL; + } + } + + if (tmp != Py_NotImplemented) { + if (writeable && + PyArray_FailUnlessWriteable((PyArrayObject *)tmp, + "array interface object") < 0) { + Py_DECREF(tmp); + return NULL; + } + return tmp; + } + + /* Until here Py_NotImplemented was borrowed */ + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + +/* * Retrieves the array parameters for viewing/converting an arbitrary * PyObject* to a NumPy array. This allows the "innate type and shape" * of Python list-of-lists to be discovered without @@ -1629,16 +1664,14 @@ fail: * validate and possibly copy arr itself ... * } * ... use arr ... - * context is passed to PyArray_FromArrayAttr, which ignores it. Since this is - * a NUMPY_API function, we cannot remove it. */ NPY_NO_EXPORT int -PyArray_GetArrayParamsFromObject(PyObject *op, +PyArray_GetArrayParamsFromObject_int(PyObject *op, PyArray_Descr *requested_dtype, npy_bool writeable, PyArray_Descr **out_dtype, int *out_ndim, npy_intp *out_dims, - PyArrayObject **out_arr, PyObject *context) + PyArrayObject **out_arr) { PyObject *tmp; @@ -1683,66 +1716,17 @@ PyArray_GetArrayParamsFromObject(PyObject *op, return 0; } - /* If op supports the PEP 3118 buffer interface */ - if (!PyBytes_Check(op) && !PyUnicode_Check(op)) { - - PyObject *memoryview = PyMemoryView_FromObject(op); - if (memoryview == NULL) { - PyErr_Clear(); - } - else { - PyObject *arr = _array_from_buffer_3118(memoryview); - Py_DECREF(memoryview); - if (arr == NULL) { - return -1; - } - if (writeable - && PyArray_FailUnlessWriteable((PyArrayObject *)arr, "PEP 3118 buffer") < 0) { - Py_DECREF(arr); - return -1; - } - *out_arr = (PyArrayObject *)arr; - return 0; - } - } - - /* If op supports the __array_struct__ or __array_interface__ interface */ - tmp = PyArray_FromStructInterface(op); + /* If op is an array-like */ + tmp = _array_from_array_like(op, requested_dtype, writeable, NULL); if (tmp == NULL) { return -1; } - if (tmp == Py_NotImplemented) { - tmp = PyArray_FromInterface(op); - if (tmp == NULL) { - return -1; - } - } - if (tmp != Py_NotImplemented) { - if (writeable - && PyArray_FailUnlessWriteable((PyArrayObject *)tmp, - "array interface object") < 0) { - Py_DECREF(tmp); - return -1; - } - *out_arr = (PyArrayObject *)tmp; - return (*out_arr) == NULL ? -1 : 0; + else if (tmp != Py_NotImplemented) { + *out_arr = (PyArrayObject*) tmp; + return 0; } - - /* - * If op supplies the __array__ function. - * The documentation says this should produce a copy, so - * we skip this method if writeable is true, because the intent - * of writeable is to modify the operand. - * XXX: If the implementation is wrong, and/or if actual - * usage requires this behave differently, - * this should be changed! - */ - if (!writeable) { - tmp = PyArray_FromArrayAttr(op, requested_dtype, context); - if (tmp != Py_NotImplemented) { - *out_arr = (PyArrayObject *)tmp; - return (*out_arr) == NULL ? -1 : 0; - } + else { + Py_DECREF(Py_NotImplemented); } /* Try to treat op as a list of lists */ @@ -1900,13 +1884,41 @@ PyArray_GetArrayParamsFromObject(PyObject *op, return -1; } + +/*NUMPY_API*/ +NPY_NO_EXPORT int +PyArray_GetArrayParamsFromObject(PyObject *op, + PyArray_Descr *requested_dtype, + npy_bool writeable, + PyArray_Descr **out_dtype, + int *out_ndim, npy_intp *out_dims, + PyArrayObject **out_arr, PyObject *context) +{ + /* NumPy 1.19, 2020-01-24 */ + if (DEPRECATE( + "PyArray_GetArrayParamsFromObject() C-API function is deprecated " + "and expected to be removed rapidly. If you are using it (i.e. see " + "this warning/error), please notify the NumPy developers. " + "As of now it is expected that any use case is served similarly " + "well by `PyArray_FromAny()` and this function is unused outside " + "of NumPy itself.") < 0) { + return -1; + } + + if (context != NULL) { + PyErr_SetString(PyExc_RuntimeError, "'context' must be NULL"); + return -1; + } + + return PyArray_GetArrayParamsFromObject_int(op, + requested_dtype, writeable, out_dtype, out_ndim, out_dims, + out_arr); +} + + /*NUMPY_API * Does not check for NPY_ARRAY_ENSURECOPY and NPY_ARRAY_NOTSWAPPED in flags * Steals a reference to newtype --- which can be NULL - * - * context is passed to PyArray_GetArrayParamsFromObject, which passes it to - * PyArray_FromArrayAttr, which raises if it is not NULL. Since this is a - * NUMPY_API function, we cannot remove it. */ NPY_NO_EXPORT PyObject * PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, @@ -1921,10 +1933,14 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, int ndim = 0; npy_intp dims[NPY_MAXDIMS]; + if (context != NULL) { + PyErr_SetString(PyExc_RuntimeError, "'context' must be NULL"); + return NULL; + } + /* Get either the array or its parameters if it isn't an array */ - if (PyArray_GetArrayParamsFromObject(op, newtype, - 0, &dtype, - &ndim, dims, &arr, context) < 0) { + if (PyArray_GetArrayParamsFromObject_int(op, + newtype, 0, &dtype, &ndim, dims, &arr) < 0) { Py_XDECREF(newtype); return NULL; } diff --git a/numpy/core/src/multiarray/ctors.h b/numpy/core/src/multiarray/ctors.h index 4768e4efd..9e63cd7d2 100644 --- a/numpy/core/src/multiarray/ctors.h +++ b/numpy/core/src/multiarray/ctors.h @@ -30,6 +30,14 @@ PyArray_New( PyTypeObject *, int nd, npy_intp const *, int, npy_intp const*, void *, int, int, PyObject *); +NPY_NO_EXPORT int +PyArray_GetArrayParamsFromObject_int(PyObject *op, + PyArray_Descr *requested_dtype, + npy_bool writeable, + PyArray_Descr **out_dtype, + int *out_ndim, npy_intp *out_dims, + PyArrayObject **out_arr); + NPY_NO_EXPORT PyObject * PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, int max_depth, int flags, PyObject *context); diff --git a/numpy/core/src/multiarray/datetime_busdaycal.c b/numpy/core/src/multiarray/datetime_busdaycal.c index 1aa5f6ab1..6936a803f 100644 --- a/numpy/core/src/multiarray/datetime_busdaycal.c +++ b/numpy/core/src/multiarray/datetime_busdaycal.c @@ -494,51 +494,11 @@ static PyGetSetDef busdaycalendar_getsets[] = { NPY_NO_EXPORT PyTypeObject NpyBusDayCalendar_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.busdaycalendar", /* tp_name */ - sizeof(NpyBusDayCalendar), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)busdaycalendar_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - busdaycalendar_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)busdaycalendar_init, /* tp_init */ - 0, /* tp_alloc */ - busdaycalendar_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.busdaycalendar", + .tp_basicsize = sizeof(NpyBusDayCalendar), + .tp_dealloc = (destructor)busdaycalendar_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_getset = busdaycalendar_getsets, + .tp_init = (initproc)busdaycalendar_init, + .tp_new = busdaycalendar_new, }; diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 0f35e867c..b26a26abf 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -39,15 +39,6 @@ static PyObject *typeDict = NULL; /* Must be explicitly loaded */ -/* - * Generate a vague error message when a function returned NULL but forgot - * to set an exception. We should aim to remove this eventually. - */ -static void -_report_generic_error(void) { - PyErr_SetString(PyExc_TypeError, "data type not understood"); -} - static PyArray_Descr * _try_convert_from_inherit_tuple(PyArray_Descr *type, PyObject *newobj); @@ -251,7 +242,9 @@ static PyArray_Descr * _convert_from_tuple(PyObject *obj, int align) { if (PyTuple_GET_SIZE(obj) != 2) { - _report_generic_error(); + PyErr_Format(PyExc_TypeError, + "Tuple must have size 2, but has size %zd", + PyTuple_GET_SIZE(obj)); return NULL; } PyArray_Descr *type = _convert_from_any(PyTuple_GET_ITEM(obj, 0), align); @@ -441,7 +434,9 @@ _convert_from_array_descr(PyObject *obj, int align) for (int i = 0; i < n; i++) { PyObject *item = PyList_GET_ITEM(obj, i); if (!PyTuple_Check(item) || (PyTuple_GET_SIZE(item) < 2)) { - _report_generic_error(); + PyErr_Format(PyExc_TypeError, + "Field elements must be 2- or 3-tuples, got '%R'", + item); goto fail; } PyObject *name = PyTuple_GET_ITEM(item, 0); @@ -451,18 +446,23 @@ _convert_from_array_descr(PyObject *obj, int align) } else if (PyTuple_Check(name)) { if (PyTuple_GET_SIZE(name) != 2) { - _report_generic_error(); + PyErr_Format(PyExc_TypeError, + "If a tuple, the first element of a field tuple must have " + "two elements, not %zd", + PyTuple_GET_SIZE(name)); goto fail; } title = PyTuple_GET_ITEM(name, 0); name = PyTuple_GET_ITEM(name, 1); if (!PyBaseString_Check(name)) { - _report_generic_error(); + PyErr_SetString(PyExc_TypeError, "Field name must be a str"); goto fail; } } else { - _report_generic_error(); + PyErr_SetString(PyExc_TypeError, + "First element of field tuple is " + "neither a tuple nor str"); goto fail; } @@ -483,7 +483,7 @@ _convert_from_array_descr(PyObject *obj, int align) Py_INCREF(name); } else { - _report_generic_error(); + PyErr_SetString(PyExc_TypeError, "Field titles must be non-empty strings"); goto fail; } } @@ -506,7 +506,8 @@ _convert_from_array_descr(PyObject *obj, int align) } } else { - _report_generic_error(); + PyErr_Format(PyExc_TypeError, + "Field elements must be tuples with at most 3 elements, got '%R'", item); goto fail; } if ((PyDict_GetItemWithError(fields, name) != NULL) @@ -725,11 +726,7 @@ _convert_from_commastring(PyObject *obj, int align) PyObject *listobj; PyArray_Descr *res; PyObject *_numpy_internal; - - if (!PyUnicode_Check(obj)) { - _report_generic_error(); - return NULL; - } + assert(PyUnicode_Check(obj)); _numpy_internal = PyImport_ImportModule("numpy.core._internal"); if (_numpy_internal == NULL) { return NULL; @@ -1402,14 +1399,25 @@ _convert_from_type(PyObject *obj) { return PyArray_DescrFromType(NPY_BOOL); } else if (typ == &PyBytes_Type) { + /* + * TODO: This should be deprecated, and have special handling for + * dtype=bytes/"S" in coercion: It should not rely on "S0". + */ return PyArray_DescrFromType(NPY_STRING); } else if (typ == &PyUnicode_Type) { + /* + * TODO: This should be deprecated, and have special handling for + * dtype=str/"U" in coercion: It should not rely on "U0". + */ return PyArray_DescrFromType(NPY_UNICODE); } else if (typ == &PyMemoryView_Type) { return PyArray_DescrFromType(NPY_VOID); } + else if (typ == &PyBaseObject_Type) { + return PyArray_DescrFromType(NPY_OBJECT); + } else { PyArray_Descr *ret = _try_convert_from_dtype_attr(obj); if ((PyObject *)ret != Py_NotImplemented) { @@ -1428,7 +1436,13 @@ _convert_from_type(PyObject *obj) { } Py_DECREF(ret); - /* All other classes are treated as object */ + /* + * All other classes are treated as object. This can be convenient + * to convey an intention of using it for a specific python type + * and possibly allow converting to a new type-specific dtype in the future. It may make sense to + * only allow this only within `dtype=...` keyword argument context + * in the future. + */ return PyArray_DescrFromType(NPY_OBJECT); } } @@ -1484,7 +1498,7 @@ _convert_from_any(PyObject *obj, int align) return _convert_from_dict(obj, align); } else if (PyArray_Check(obj)) { - _report_generic_error(); + PyErr_SetString(PyExc_TypeError, "Cannot construct a dtype from an array"); return NULL; } else { @@ -1503,7 +1517,7 @@ _convert_from_any(PyObject *obj, int align) return ret; } Py_DECREF(ret); - _report_generic_error(); + PyErr_Format(PyExc_TypeError, "Cannot interpret '%R' as a data type", obj); return NULL; } } @@ -2001,7 +2015,7 @@ _arraydescr_isnative(PyArray_Descr *self) int offset; Py_ssize_t pos = 0; while (PyDict_Next(self->fields, &pos, &key, &value)) { - if NPY_TITLE_KEY(key, value) { + if (NPY_TITLE_KEY(key, value)) { continue; } if (!PyArg_ParseTuple(value, "Oi|O", &new, &offset, &title)) { @@ -2420,7 +2434,7 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args)) } obj = PyUString_FromFormat("%c%d",self->kind, elsize); } - PyTuple_SET_ITEM(ret, 1, Py_BuildValue("(Nii)", obj, 0, 1)); + PyTuple_SET_ITEM(ret, 1, Py_BuildValue("(NOO)", obj, Py_False, Py_True)); /* * Now return the state which is at least byteorder, @@ -2511,7 +2525,7 @@ _descr_find_object(PyArray_Descr *self) Py_ssize_t pos = 0; while (PyDict_Next(self->fields, &pos, &key, &value)) { - if NPY_TITLE_KEY(key, value) { + if (NPY_TITLE_KEY(key, value)) { continue; } if (!PyArg_ParseTuple(value, "Oi|O", &new, &offset, &title)) { @@ -2985,7 +2999,7 @@ PyArray_DescrNewByteorder(PyArray_Descr *self, char newendian) newfields = PyDict_New(); /* make new dictionary with replaced PyArray_Descr Objects */ while (PyDict_Next(self->fields, &pos, &key, &value)) { - if NPY_TITLE_KEY(key, value) { + if (NPY_TITLE_KEY(key, value)) { continue; } if (!PyUString_Check(key) || !PyTuple_Check(value) || @@ -3152,72 +3166,41 @@ arraydescr_str(PyArray_Descr *dtype) static PyObject * arraydescr_richcompare(PyArray_Descr *self, PyObject *other, int cmp_op) { - PyArray_Descr *new = NULL; - PyObject *result = Py_NotImplemented; - if (!PyArray_DescrCheck(other)) { - new = _convert_from_any(other, 0); - if (new == NULL) { - return NULL; - } - } - else { - new = (PyArray_Descr *)other; - Py_INCREF(new); + PyArray_Descr *new = _convert_from_any(other, 0); + if (new == NULL) { + return NULL; } + + npy_bool ret; switch (cmp_op) { case Py_LT: - if (!PyArray_EquivTypes(self, new) && PyArray_CanCastTo(self, new)) { - result = Py_True; - } - else { - result = Py_False; - } - break; + ret = !PyArray_EquivTypes(self, new) && PyArray_CanCastTo(self, new); + Py_DECREF(new); + return PyBool_FromLong(ret); case Py_LE: - if (PyArray_CanCastTo(self, new)) { - result = Py_True; - } - else { - result = Py_False; - } - break; + ret = PyArray_CanCastTo(self, new); + Py_DECREF(new); + return PyBool_FromLong(ret); case Py_EQ: - if (PyArray_EquivTypes(self, new)) { - result = Py_True; - } - else { - result = Py_False; - } - break; + ret = PyArray_EquivTypes(self, new); + Py_DECREF(new); + return PyBool_FromLong(ret); case Py_NE: - if (PyArray_EquivTypes(self, new)) - result = Py_False; - else - result = Py_True; - break; + ret = !PyArray_EquivTypes(self, new); + Py_DECREF(new); + return PyBool_FromLong(ret); case Py_GT: - if (!PyArray_EquivTypes(self, new) && PyArray_CanCastTo(new, self)) { - result = Py_True; - } - else { - result = Py_False; - } - break; + ret = !PyArray_EquivTypes(self, new) && PyArray_CanCastTo(new, self); + Py_DECREF(new); + return PyBool_FromLong(ret); case Py_GE: - if (PyArray_CanCastTo(new, self)) { - result = Py_True; - } - else { - result = Py_False; - } - break; + ret = PyArray_CanCastTo(new, self); + Py_DECREF(new); + return PyBool_FromLong(ret); default: - result = Py_NotImplemented; + Py_DECREF(new); + Py_RETURN_NOTIMPLEMENTED; } - - Py_XDECREF(new); - Py_INCREF(result); - return result; } static int @@ -3229,16 +3212,7 @@ descr_nonzero(PyObject *NPY_UNUSED(self)) } static PyNumberMethods descr_as_number = { - (binaryfunc)0, /* nb_add */ - (binaryfunc)0, /* nb_subtract */ - (binaryfunc)0, /* nb_multiply */ - (binaryfunc)0, /* nb_remainder */ - (binaryfunc)0, /* nb_divmod */ - (ternaryfunc)0, /* nb_power */ - (unaryfunc)0, /* nb_negative */ - (unaryfunc)0, /* nb_positive */ - (unaryfunc)0, /* nb_absolute */ - (inquiry)descr_nonzero, /* nb_nonzero */ + .nb_bool = (inquiry)descr_nonzero, }; /************************************************************************* @@ -3484,51 +3458,19 @@ static PyMappingMethods descr_as_mapping = { NPY_NO_EXPORT PyTypeObject PyArrayDescr_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.dtype", /* tp_name */ - sizeof(PyArray_Descr), /* tp_basicsize */ - 0, /* tp_itemsize */ + .tp_name = "numpy.dtype", + .tp_basicsize = sizeof(PyArray_Descr), /* methods */ - (destructor)arraydescr_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - (void *)0, /* tp_reserved */ - (reprfunc)arraydescr_repr, /* tp_repr */ - &descr_as_number, /* tp_as_number */ - &descr_as_sequence, /* tp_as_sequence */ - &descr_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)arraydescr_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - (richcmpfunc)arraydescr_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - arraydescr_methods, /* tp_methods */ - arraydescr_members, /* tp_members */ - arraydescr_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - arraydescr_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_dealloc = (destructor)arraydescr_dealloc, + .tp_repr = (reprfunc)arraydescr_repr, + .tp_as_number = &descr_as_number, + .tp_as_sequence = &descr_as_sequence, + .tp_as_mapping = &descr_as_mapping, + .tp_str = (reprfunc)arraydescr_str, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_richcompare = (richcmpfunc)arraydescr_richcompare, + .tp_methods = arraydescr_methods, + .tp_members = arraydescr_members, + .tp_getset = arraydescr_getsets, + .tp_new = arraydescr_new, }; diff --git a/numpy/core/src/multiarray/dragon4.c b/numpy/core/src/multiarray/dragon4.c index d14b8e638..282cdad28 100644 --- a/numpy/core/src/multiarray/dragon4.c +++ b/numpy/core/src/multiarray/dragon4.c @@ -3183,19 +3183,19 @@ Dragon4_Positional(PyObject *obj, DigitMode digit_mode, CutoffMode cutoff_mode, opt.exp_digits = -1; if (PyArray_IsScalar(obj, Half)) { - npy_half x = ((PyHalfScalarObject *)obj)->obval; + npy_half x = PyArrayScalar_VAL(obj, Half); return Dragon4_Positional_Half_opt(&x, &opt); } else if (PyArray_IsScalar(obj, Float)) { - npy_float x = ((PyFloatScalarObject *)obj)->obval; + npy_float x = PyArrayScalar_VAL(obj, Float); return Dragon4_Positional_Float_opt(&x, &opt); } else if (PyArray_IsScalar(obj, Double)) { - npy_double x = ((PyDoubleScalarObject *)obj)->obval; + npy_double x = PyArrayScalar_VAL(obj, Double); return Dragon4_Positional_Double_opt(&x, &opt); } else if (PyArray_IsScalar(obj, LongDouble)) { - npy_longdouble x = ((PyLongDoubleScalarObject *)obj)->obval; + npy_longdouble x = PyArrayScalar_VAL(obj, LongDouble); return Dragon4_Positional_LongDouble_opt(&x, &opt); } @@ -3224,19 +3224,19 @@ Dragon4_Scientific(PyObject *obj, DigitMode digit_mode, int precision, opt.exp_digits = exp_digits; if (PyArray_IsScalar(obj, Half)) { - npy_half x = ((PyHalfScalarObject *)obj)->obval; + npy_half x = PyArrayScalar_VAL(obj, Half); return Dragon4_Scientific_Half_opt(&x, &opt); } else if (PyArray_IsScalar(obj, Float)) { - npy_float x = ((PyFloatScalarObject *)obj)->obval; + npy_float x = PyArrayScalar_VAL(obj, Float); return Dragon4_Scientific_Float_opt(&x, &opt); } else if (PyArray_IsScalar(obj, Double)) { - npy_double x = ((PyDoubleScalarObject *)obj)->obval; + npy_double x = PyArrayScalar_VAL(obj, Double); return Dragon4_Scientific_Double_opt(&x, &opt); } else if (PyArray_IsScalar(obj, LongDouble)) { - npy_longdouble x = ((PyLongDoubleScalarObject *)obj)->obval; + npy_longdouble x = PyArrayScalar_VAL(obj, LongDouble); return Dragon4_Scientific_LongDouble_opt(&x, &opt); } diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index b26d5ac89..ecaa680ec 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -312,6 +312,7 @@ typedef struct { NpyAuxData *wrappeddata, *todata, *fromdata; npy_intp src_itemsize, dst_itemsize; char *bufferin, *bufferout; + npy_bool init_dest, out_needs_api; } _align_wrap_data; /* transfer data free function */ @@ -372,6 +373,9 @@ static NpyAuxData *_align_wrap_data_clone(NpyAuxData *data) } } + newdata->init_dest = d->init_dest; + newdata->out_needs_api = d->out_needs_api; + return (NpyAuxData *)newdata; } @@ -391,57 +395,26 @@ _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride, *todata = d->todata, *fromdata = d->fromdata; char *bufferin = d->bufferin, *bufferout = d->bufferout; + npy_bool init_dest = d->init_dest, out_needs_api = d->out_needs_api; for(;;) { - if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) { - tobuffer(bufferin, inner_src_itemsize, src, src_stride, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - src_itemsize, todata); - wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - inner_src_itemsize, wrappeddata); - frombuffer(dst, dst_stride, bufferout, dst_itemsize, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - dst_itemsize, fromdata); - N -= NPY_LOWLEVEL_BUFFER_BLOCKSIZE; - src += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_stride; - dst += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_stride; - } - else { - tobuffer(bufferin, inner_src_itemsize, src, src_stride, N, - src_itemsize, todata); - wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, N, - inner_src_itemsize, wrappeddata); - frombuffer(dst, dst_stride, bufferout, dst_itemsize, N, - dst_itemsize, fromdata); + /* + * The caller does not know if a previous call resulted in a Python + * exception. Much of the Python API is unsafe while an exception is in + * flight, so just skip all the work. Someone higher in the call stack + * will check for errors and propagate them. + */ + if (out_needs_api && PyErr_Occurred()) { return; } - } -} - -static void -_strided_to_strided_contig_align_wrap_init_dest(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) -{ - _align_wrap_data *d = (_align_wrap_data *)data; - PyArray_StridedUnaryOp *wrapped = d->wrapped, - *tobuffer = d->tobuffer, - *frombuffer = d->frombuffer; - npy_intp inner_src_itemsize = d->src_itemsize, - dst_itemsize = d->dst_itemsize; - NpyAuxData *wrappeddata = d->wrappeddata, - *todata = d->todata, - *fromdata = d->fromdata; - char *bufferin = d->bufferin, *bufferout = d->bufferout; - - for(;;) { if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) { tobuffer(bufferin, inner_src_itemsize, src, src_stride, NPY_LOWLEVEL_BUFFER_BLOCKSIZE, src_itemsize, todata); - memset(bufferout, 0, dst_itemsize*NPY_LOWLEVEL_BUFFER_BLOCKSIZE); + if (init_dest) { + memset(bufferout, 0, + dst_itemsize*NPY_LOWLEVEL_BUFFER_BLOCKSIZE); + } wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, NPY_LOWLEVEL_BUFFER_BLOCKSIZE, inner_src_itemsize, wrappeddata); @@ -455,7 +428,9 @@ _strided_to_strided_contig_align_wrap_init_dest(char *dst, npy_intp dst_stride, else { tobuffer(bufferin, inner_src_itemsize, src, src_stride, N, src_itemsize, todata); - memset(bufferout, 0, dst_itemsize*N); + if (init_dest) { + memset(bufferout, 0, dst_itemsize*N); + } wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, N, inner_src_itemsize, wrappeddata); frombuffer(dst, dst_stride, bufferout, dst_itemsize, N, @@ -477,6 +452,7 @@ _strided_to_strided_contig_align_wrap_init_dest(char *dst, npy_intp dst_stride, * wrapped - contig to contig transfer function being wrapped * wrappeddata - data for wrapped * init_dest - 1 means to memset the dest buffer to 0 before calling wrapped. + * out_needs_api - if NPY_TRUE, check for (and break on) Python API errors. * * Returns NPY_SUCCEED or NPY_FAIL. */ @@ -487,6 +463,7 @@ wrap_aligned_contig_transfer_function( PyArray_StridedUnaryOp *frombuffer, NpyAuxData *fromdata, PyArray_StridedUnaryOp *wrapped, NpyAuxData *wrappeddata, int init_dest, + int out_needs_api, PyArray_StridedUnaryOp **out_stransfer, NpyAuxData **out_transferdata) { @@ -519,14 +496,11 @@ wrap_aligned_contig_transfer_function( data->bufferin = (char *)data + basedatasize; data->bufferout = data->bufferin + NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_itemsize; + data->init_dest = (npy_bool) init_dest; + data->out_needs_api = (npy_bool) out_needs_api; /* Set the function and data */ - if (init_dest) { - *out_stransfer = &_strided_to_strided_contig_align_wrap_init_dest; - } - else { - *out_stransfer = &_strided_to_strided_contig_align_wrap; - } + *out_stransfer = &_strided_to_strided_contig_align_wrap; *out_transferdata = (NpyAuxData *)data; return NPY_SUCCEED; @@ -1171,6 +1145,7 @@ get_datetime_to_unicode_transfer_function(int aligned, frombuffer, fromdata, caststransfer, castdata, PyDataType_FLAGCHK(str_dtype, NPY_NEEDS_INIT), + *out_needs_api, out_stransfer, out_transferdata) != NPY_SUCCEED) { NPY_AUXDATA_FREE(castdata); NPY_AUXDATA_FREE(todata); @@ -1293,6 +1268,7 @@ get_unicode_to_datetime_transfer_function(int aligned, frombuffer, fromdata, caststransfer, castdata, PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT), + *out_needs_api, out_stransfer, out_transferdata) != NPY_SUCCEED) { Py_DECREF(str_dtype); NPY_AUXDATA_FREE(castdata); @@ -1613,6 +1589,7 @@ get_cast_transfer_function(int aligned, frombuffer, fromdata, caststransfer, castdata, PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT), + *out_needs_api, out_stransfer, out_transferdata) != NPY_SUCCEED) { NPY_AUXDATA_FREE(castdata); NPY_AUXDATA_FREE(todata); diff --git a/numpy/core/src/multiarray/flagsobject.c b/numpy/core/src/multiarray/flagsobject.c index 6fe0eff4a..d5f24e75a 100644 --- a/numpy/core/src/multiarray/flagsobject.c +++ b/numpy/core/src/multiarray/flagsobject.c @@ -772,51 +772,14 @@ arrayflags_new(PyTypeObject *NPY_UNUSED(self), PyObject *args, PyObject *NPY_UNU NPY_NO_EXPORT PyTypeObject PyArrayFlags_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.flagsobj", - sizeof(PyArrayFlagsObject), - 0, /* tp_itemsize */ - /* methods */ - (destructor)arrayflags_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - (reprfunc)arrayflags_print, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - &arrayflags_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)arrayflags_print, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - arrayflags_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - arrayflags_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - arrayflags_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.flagsobj", + .tp_basicsize = sizeof(PyArrayFlagsObject), + .tp_dealloc = (destructor)arrayflags_dealloc, + .tp_repr = (reprfunc)arrayflags_print, + .tp_as_mapping = &arrayflags_as_mapping, + .tp_str = (reprfunc)arrayflags_print, + .tp_flags =Py_TPFLAGS_DEFAULT, + .tp_richcompare = arrayflags_richcompare, + .tp_getset = arrayflags_getsets, + .tp_new = arrayflags_new, }; diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index 9a9c51fee..80a1cd4a1 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -13,6 +13,7 @@ #include "npy_import.h" #include "common.h" +#include "conversion_utils.h" #include "ctors.h" #include "scalartypes.h" #include "descriptor.h" @@ -62,33 +63,39 @@ array_shape_set(PyArrayObject *self, PyObject *val) if (PyArray_DATA(ret) != PyArray_DATA(self)) { Py_DECREF(ret); PyErr_SetString(PyExc_AttributeError, - "incompatible shape for a non-contiguous "\ - "array"); + "Incompatible shape for in-place modification. Use " + "`.reshape()` to make a copy with the desired shape."); return -1; } - /* Free old dimensions and strides */ - npy_free_cache_dim_array(self); nd = PyArray_NDIM(ret); - ((PyArrayObject_fields *)self)->nd = nd; if (nd > 0) { /* create new dimensions and strides */ - ((PyArrayObject_fields *)self)->dimensions = npy_alloc_cache_dim(2 * nd); - if (PyArray_DIMS(self) == NULL) { + npy_intp *_dimensions = npy_alloc_cache_dim(2 * nd); + if (_dimensions == NULL) { Py_DECREF(ret); - PyErr_SetString(PyExc_MemoryError,""); + PyErr_NoMemory(); return -1; } - ((PyArrayObject_fields *)self)->strides = PyArray_DIMS(self) + nd; + /* Free old dimensions and strides */ + npy_free_cache_dim_array(self); + ((PyArrayObject_fields *)self)->nd = nd; + ((PyArrayObject_fields *)self)->dimensions = _dimensions; + ((PyArrayObject_fields *)self)->strides = _dimensions + nd; + if (nd) { memcpy(PyArray_DIMS(self), PyArray_DIMS(ret), nd*sizeof(npy_intp)); memcpy(PyArray_STRIDES(self), PyArray_STRIDES(ret), nd*sizeof(npy_intp)); } } else { + /* Free old dimensions and strides */ + npy_free_cache_dim_array(self); + ((PyArrayObject_fields *)self)->nd = 0; ((PyArrayObject_fields *)self)->dimensions = NULL; ((PyArrayObject_fields *)self)->strides = NULL; } + Py_DECREF(ret); PyArray_UpdateFlags(self, NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS); return 0; @@ -104,7 +111,7 @@ array_strides_get(PyArrayObject *self) static int array_strides_set(PyArrayObject *self, PyObject *obj) { - PyArray_Dims newstrides = {NULL, 0}; + PyArray_Dims newstrides = {NULL, -1}; PyArrayObject *new; npy_intp numbytes = 0; npy_intp offset = 0; @@ -117,8 +124,8 @@ array_strides_set(PyArrayObject *self, PyObject *obj) "Cannot delete array strides"); return -1; } - if (!PyArray_IntpConverter(obj, &newstrides) || - newstrides.ptr == NULL) { + if (!PyArray_OptionalIntpConverter(obj, &newstrides) || + newstrides.len == -1) { PyErr_SetString(PyExc_TypeError, "invalid strides"); return -1; } diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c index f0ef8ba3b..45c019f49 100644 --- a/numpy/core/src/multiarray/item_selection.c +++ b/numpy/core/src/multiarray/item_selection.c @@ -1564,6 +1564,16 @@ PyArray_LexSort(PyObject *sort_keys, int axis) /* Now we can check the axis */ nd = PyArray_NDIM(mps[0]); + /* + * Special case letting axis={-1,0} slip through for scalars, + * for backwards compatibility reasons. + */ + if (nd == 0 && (axis == 0 || axis == -1)) { + /* TODO: can we deprecate this? */ + } + else if (check_and_adjust_axis(&axis, nd) < 0) { + goto fail; + } if ((nd == 0) || (PyArray_SIZE(mps[0]) <= 1)) { /* empty/single element case */ ret = (PyArrayObject *)PyArray_NewFromDescr( @@ -1579,9 +1589,6 @@ PyArray_LexSort(PyObject *sort_keys, int axis) } goto finish; } - if (check_and_adjust_axis(&axis, nd) < 0) { - goto fail; - } for (i = 0; i < n; i++) { its[i] = (PyArrayIterObject *)PyArray_IterAllButAxis( diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 14e70fe02..c71b7b770 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -1103,53 +1103,16 @@ static PyGetSetDef iter_getsets[] = { NPY_NO_EXPORT PyTypeObject PyArrayIter_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.flatiter", /* tp_name */ - sizeof(PyArrayIterObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)arrayiter_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - &iter_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - (richcmpfunc)iter_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - (iternextfunc)arrayiter_next, /* tp_iternext */ - iter_methods, /* tp_methods */ - iter_members, /* tp_members */ - iter_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.flatiter", + .tp_basicsize = sizeof(PyArrayIterObject), + .tp_dealloc = (destructor)arrayiter_dealloc, + .tp_as_mapping = &iter_as_mapping, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_richcompare = (richcmpfunc)iter_richcompare, + .tp_iternext = (iternextfunc)arrayiter_next, + .tp_methods = iter_methods, + .tp_members = iter_members, + .tp_getset = iter_getsets, }; /** END of Array Iterator **/ @@ -1552,53 +1515,15 @@ static PyMethodDef arraymultiter_methods[] = { NPY_NO_EXPORT PyTypeObject PyArrayMultiIter_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.broadcast", /* tp_name */ - sizeof(PyArrayMultiIterObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)arraymultiter_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - (iternextfunc)arraymultiter_next, /* tp_iternext */ - arraymultiter_methods, /* tp_methods */ - arraymultiter_members, /* tp_members */ - arraymultiter_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)0, /* tp_init */ - 0, /* tp_alloc */ - arraymultiter_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.broadcast", + .tp_basicsize = sizeof(PyArrayMultiIterObject), + .tp_dealloc = (destructor)arraymultiter_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_iternext = (iternextfunc)arraymultiter_next, + .tp_methods = arraymultiter_methods, + .tp_members = arraymultiter_members, + .tp_getset = arraymultiter_getsetlist, + .tp_new = arraymultiter_new, }; /*========================= Neighborhood iterator ======================*/ @@ -1873,50 +1798,8 @@ static void neighiter_dealloc(PyArrayNeighborhoodIterObject* iter) NPY_NO_EXPORT PyTypeObject PyArrayNeighborhoodIter_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.neigh_internal_iter", /* tp_name*/ - sizeof(PyArrayNeighborhoodIterObject), /* tp_basicsize*/ - 0, /* tp_itemsize*/ - (destructor)neighiter_dealloc, /* tp_dealloc*/ - 0, /* tp_print*/ - 0, /* tp_getattr*/ - 0, /* tp_setattr*/ - 0, /* tp_reserved */ - 0, /* tp_repr*/ - 0, /* tp_as_number*/ - 0, /* tp_as_sequence*/ - 0, /* tp_as_mapping*/ - 0, /* tp_hash */ - 0, /* tp_call*/ - 0, /* tp_str*/ - 0, /* tp_getattro*/ - 0, /* tp_setattro*/ - 0, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /* tp_flags*/ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - (iternextfunc)0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.neigh_internal_iter", + .tp_basicsize = sizeof(PyArrayNeighborhoodIterObject), + .tp_dealloc = (destructor)neighiter_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, }; diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 4122d27ad..7047304eb 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -727,8 +727,11 @@ prepare_index(PyArrayObject *self, PyObject *index, } } else if (used_ndim > PyArray_NDIM(self)) { - PyErr_SetString(PyExc_IndexError, - "too many indices for array"); + PyErr_Format(PyExc_IndexError, + "too many indices for array: " + "array is %d-dimensional, but %d were indexed", + PyArray_NDIM(self), + used_ndim); goto failed_building_indices; } else if (index_ndim == 0) { @@ -3336,51 +3339,8 @@ arraymapiter_dealloc(PyArrayMapIterObject *mit) */ NPY_NO_EXPORT PyTypeObject PyArrayMapIter_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.mapiter", /* tp_name */ - sizeof(PyArrayMapIterObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)arraymapiter_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.mapiter", + .tp_basicsize = sizeof(PyArrayMapIterObject), + .tp_dealloc = (destructor)arraymapiter_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, }; diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 7b9aa4794..7bfbeca15 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -566,6 +566,23 @@ array_tobytes(PyArrayObject *self, PyObject *args, PyObject *kwds) return PyArray_ToString(self, order); } +static PyObject * +array_tostring(PyArrayObject *self, PyObject *args, PyObject *kwds) +{ + NPY_ORDER order = NPY_CORDER; + static char *kwlist[] = {"order", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&:tostring", kwlist, + PyArray_OrderConverter, &order)) { + return NULL; + } + /* 2020-03-30, NumPy 1.19 */ + if (DEPRECATE("tostring() is deprecated. Use tobytes() instead.") < 0) { + return NULL; + } + return PyArray_ToString(self, order); +} + /* This should grow an order= keyword to be consistent */ @@ -1503,7 +1520,7 @@ _deepcopy_call(char *iptr, char *optr, PyArray_Descr *dtype, int offset; Py_ssize_t pos = 0; while (PyDict_Next(dtype->fields, &pos, &key, &value)) { - if NPY_TITLE_KEY(key, value) { + if (NPY_TITLE_KEY(key, value)) { continue; } if (!PyArg_ParseTuple(value, "Oi|O", &new, &offset, @@ -2844,7 +2861,7 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { (PyCFunction)array_tolist, METH_VARARGS, NULL}, {"tostring", - (PyCFunction)array_tobytes, + (PyCFunction)array_tostring, METH_VARARGS | METH_KEYWORDS, NULL}, {"trace", (PyCFunction)array_trace, diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 11e0bc44d..4c316052d 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -34,6 +34,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; /* Internal APIs */ +#include "alloc.h" #include "arrayfunction_override.h" #include "arraytypes.h" #include "arrayobject.h" @@ -1582,8 +1583,9 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) "ndmin", NULL}; if (PyTuple_GET_SIZE(args) > 2) { - PyErr_SetString(PyExc_ValueError, - "only 2 non-keyword arguments accepted"); + PyErr_Format(PyExc_TypeError, + "array() takes from 1 to 2 positional arguments but " + "%zd were given", PyTuple_GET_SIZE(args)); return NULL; } @@ -1856,14 +1858,15 @@ array_empty_like(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) NPY_ORDER order = NPY_KEEPORDER; PyArrayObject *ret = NULL; int subok = 1; - PyArray_Dims shape = {NULL, 0}; + /* -1 is a special value meaning "not specified" */ + PyArray_Dims shape = {NULL, -1}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&iO&:empty_like", kwlist, &PyArray_Converter, &prototype, &PyArray_DescrConverter2, &dtype, &PyArray_OrderConverter, &order, &subok, - &PyArray_IntpConverter, &shape)) { + &PyArray_OptionalIntpConverter, &shape)) { goto fail; } /* steals the reference to dtype if it's not NULL */ @@ -3287,12 +3290,14 @@ array_datetime_data(PyObject *NPY_UNUSED(dummy), PyObject *args) } meta = get_datetime_metadata_from_dtype(dtype); - Py_DECREF(dtype); if (meta == NULL) { + Py_DECREF(dtype); return NULL; } - return convert_datetime_metadata_to_tuple(meta); + PyObject *res = convert_datetime_metadata_to_tuple(meta); + Py_DECREF(dtype); + return res; } /* @@ -4157,6 +4162,8 @@ static struct PyMethodDef array_module_methods[] = { METH_VARARGS, NULL}, {"_add_newdoc_ufunc", (PyCFunction)add_newdoc_ufunc, METH_VARARGS, NULL}, + {"_set_madvise_hugepage", (PyCFunction)_set_madvise_hugepage, + METH_O, NULL}, {NULL, NULL, 0, NULL} /* sentinel */ }; @@ -4386,6 +4393,11 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { PyObject *m, *d, *s; PyObject *c_api; + /* Initialize CPU features */ + if (npy_cpu_init() < 0) { + goto err; + } + /* Create the module and add the functions */ m = PyModule_Create(&moduledef); if (!m) { @@ -4511,6 +4523,16 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { PyDict_SetItemString(d, "__version__", s); Py_DECREF(s); + s = npy_cpu_features_dict(); + if (s == NULL) { + goto err; + } + if (PyDict_SetItemString(d, "__cpu_features__", s) < 0) { + Py_DECREF(s); + goto err; + } + Py_DECREF(s); + s = NpyCapsule_FromVoidPtr((void *)_datetime_strings, NULL); if (s == NULL) { goto err; diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index b4b3c6704..7f31a5096 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -17,6 +17,7 @@ #include "npy_pycompat.h" #include "alloc.h" #include "common.h" +#include "conversion_utils.h" #include "ctors.h" /* Functions not part of the public NumPy C API */ @@ -231,50 +232,6 @@ NpyIter_GlobalFlagsConverter(PyObject *flags_in, npy_uint32 *flags) return 1; } -/* TODO: Use PyArray_OrderConverter once 'K' is added there */ -static int -npyiter_order_converter(PyObject *order_in, NPY_ORDER *order) -{ - char *str = NULL; - Py_ssize_t length = 0; - - if (PyUnicode_Check(order_in)) { - /* accept unicode input */ - PyObject *str_obj; - int ret; - str_obj = PyUnicode_AsASCIIString(order_in); - if (str_obj == NULL) { - return 0; - } - ret = npyiter_order_converter(str_obj, order); - Py_DECREF(str_obj); - return ret; - } - - if (PyBytes_AsStringAndSize(order_in, &str, &length) < 0) { - return 0; - } - - if (length == 1) switch (str[0]) { - case 'C': - *order = NPY_CORDER; - return 1; - case 'F': - *order = NPY_FORTRANORDER; - return 1; - case 'A': - *order = NPY_ANYORDER; - return 1; - case 'K': - *order = NPY_KEEPORDER; - return 1; - } - - PyErr_SetString(PyExc_ValueError, - "order must be one of 'C', 'F', 'A', or 'K'"); - return 0; -} - static int NpyIter_OpFlagsConverter(PyObject *op_flags_in, npy_uint32 *op_flags) @@ -748,7 +705,7 @@ npyiter_init(NewNpyArrayIterObject *self, PyObject *args, PyObject *kwds) int oa_ndim = -1; int op_axes_arrays[NPY_MAXARGS][NPY_MAXDIMS]; int *op_axes[NPY_MAXARGS]; - PyArray_Dims itershape = {NULL, 0}; + PyArray_Dims itershape = {NULL, -1}; int buffersize = 0; if (self->iter != NULL) { @@ -762,10 +719,10 @@ npyiter_init(NewNpyArrayIterObject *self, PyObject *args, PyObject *kwds) NpyIter_GlobalFlagsConverter, &flags, &op_flags_in, &op_dtypes_in, - npyiter_order_converter, &order, + PyArray_OrderConverter, &order, PyArray_CastingConverter, &casting, &op_axes_in, - PyArray_IntpConverter, &itershape, + PyArray_OptionalIntpConverter, &itershape, &buffersize)) { npy_free_cache_dim_obj(itershape); return -1; @@ -800,7 +757,7 @@ npyiter_init(NewNpyArrayIterObject *self, PyObject *args, PyObject *kwds) } } - if (itershape.len > 0) { + if (itershape.len != -1) { if (oa_ndim == -1) { oa_ndim = itershape.len; memset(op_axes, 0, sizeof(op_axes[0]) * nop); @@ -812,10 +769,6 @@ npyiter_init(NewNpyArrayIterObject *self, PyObject *args, PyObject *kwds) goto fail; } } - else if (itershape.ptr != NULL) { - npy_free_cache_dim_obj(itershape); - itershape.ptr = NULL; - } self->iter = NpyIter_AdvancedNew(nop, op, flags, order, casting, op_flags, op_request_dtypes, @@ -895,7 +848,7 @@ NpyIter_NestedIters(PyObject *NPY_UNUSED(self), NpyIter_GlobalFlagsConverter, &flags, &op_flags_in, &op_dtypes_in, - npyiter_order_converter, &order, + PyArray_OrderConverter, &order, PyArray_CastingConverter, &casting, &buffersize)) { return NULL; @@ -2491,51 +2444,16 @@ NPY_NO_EXPORT PyMappingMethods npyiter_as_mapping = { NPY_NO_EXPORT PyTypeObject NpyIter_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.nditer", /* tp_name */ - sizeof(NewNpyArrayIterObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)npyiter_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &npyiter_as_sequence, /* tp_as_sequence */ - &npyiter_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - (iternextfunc)npyiter_next, /* tp_iternext */ - npyiter_methods, /* tp_methods */ - npyiter_members, /* tp_members */ - npyiter_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)npyiter_init, /* tp_init */ - 0, /* tp_alloc */ - npyiter_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.nditer", + .tp_basicsize = sizeof(NewNpyArrayIterObject), + .tp_dealloc = (destructor)npyiter_dealloc, + .tp_as_sequence = &npyiter_as_sequence, + .tp_as_mapping = &npyiter_as_mapping, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_iternext = (iternextfunc)npyiter_next, + .tp_methods = npyiter_methods, + .tp_members = npyiter_members, + .tp_getset = npyiter_getsets, + .tp_init = (initproc)npyiter_init, + .tp_new = npyiter_new, }; diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index 21471a80a..19ac7d7f9 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -923,47 +923,43 @@ array_index(PyArrayObject *v) NPY_NO_EXPORT PyNumberMethods array_as_number = { - (binaryfunc)array_add, /*nb_add*/ - (binaryfunc)array_subtract, /*nb_subtract*/ - (binaryfunc)array_multiply, /*nb_multiply*/ - (binaryfunc)array_remainder, /*nb_remainder*/ - (binaryfunc)array_divmod, /*nb_divmod*/ - (ternaryfunc)array_power, /*nb_power*/ - (unaryfunc)array_negative, /*nb_neg*/ - (unaryfunc)array_positive, /*nb_pos*/ - (unaryfunc)array_absolute, /*(unaryfunc)array_abs,*/ - (inquiry)_array_nonzero, /*nb_nonzero*/ - (unaryfunc)array_invert, /*nb_invert*/ - (binaryfunc)array_left_shift, /*nb_lshift*/ - (binaryfunc)array_right_shift, /*nb_rshift*/ - (binaryfunc)array_bitwise_and, /*nb_and*/ - (binaryfunc)array_bitwise_xor, /*nb_xor*/ - (binaryfunc)array_bitwise_or, /*nb_or*/ - (unaryfunc)array_int, /*nb_int*/ - 0, /*nb_reserved*/ - (unaryfunc)array_float, /*nb_float*/ - - /* - * This code adds augmented assignment functionality - * that was made available in Python 2.0 - */ - (binaryfunc)array_inplace_add, /*nb_inplace_add*/ - (binaryfunc)array_inplace_subtract, /*nb_inplace_subtract*/ - (binaryfunc)array_inplace_multiply, /*nb_inplace_multiply*/ - (binaryfunc)array_inplace_remainder, /*nb_inplace_remainder*/ - (ternaryfunc)array_inplace_power, /*nb_inplace_power*/ - (binaryfunc)array_inplace_left_shift, /*nb_inplace_lshift*/ - (binaryfunc)array_inplace_right_shift, /*nb_inplace_rshift*/ - (binaryfunc)array_inplace_bitwise_and, /*nb_inplace_and*/ - (binaryfunc)array_inplace_bitwise_xor, /*nb_inplace_xor*/ - (binaryfunc)array_inplace_bitwise_or, /*nb_inplace_or*/ - - (binaryfunc)array_floor_divide, /*nb_floor_divide*/ - (binaryfunc)array_true_divide, /*nb_true_divide*/ - (binaryfunc)array_inplace_floor_divide, /*nb_inplace_floor_divide*/ - (binaryfunc)array_inplace_true_divide, /*nb_inplace_true_divide*/ - (unaryfunc)array_index, /*nb_index */ - - (binaryfunc)array_matrix_multiply, /*nb_matrix_multiply*/ - (binaryfunc)array_inplace_matrix_multiply, /*nb_inplace_matrix_multiply*/ + .nb_add = (binaryfunc)array_add, + .nb_subtract = (binaryfunc)array_subtract, + .nb_multiply = (binaryfunc)array_multiply, + .nb_remainder = (binaryfunc)array_remainder, + .nb_divmod = (binaryfunc)array_divmod, + .nb_power = (ternaryfunc)array_power, + .nb_negative = (unaryfunc)array_negative, + .nb_positive = (unaryfunc)array_positive, + .nb_absolute = (unaryfunc)array_absolute, + .nb_bool = (inquiry)_array_nonzero, + .nb_invert = (unaryfunc)array_invert, + .nb_lshift = (binaryfunc)array_left_shift, + .nb_rshift = (binaryfunc)array_right_shift, + .nb_and = (binaryfunc)array_bitwise_and, + .nb_xor = (binaryfunc)array_bitwise_xor, + .nb_or = (binaryfunc)array_bitwise_or, + + .nb_int = (unaryfunc)array_int, + .nb_float = (unaryfunc)array_float, + .nb_index = (unaryfunc)array_index, + + .nb_inplace_add = (binaryfunc)array_inplace_add, + .nb_inplace_subtract = (binaryfunc)array_inplace_subtract, + .nb_inplace_multiply = (binaryfunc)array_inplace_multiply, + .nb_inplace_remainder = (binaryfunc)array_inplace_remainder, + .nb_inplace_power = (ternaryfunc)array_inplace_power, + .nb_inplace_lshift = (binaryfunc)array_inplace_left_shift, + .nb_inplace_rshift = (binaryfunc)array_inplace_right_shift, + .nb_inplace_and = (binaryfunc)array_inplace_bitwise_and, + .nb_inplace_xor = (binaryfunc)array_inplace_bitwise_xor, + .nb_inplace_or = (binaryfunc)array_inplace_bitwise_or, + + .nb_floor_divide = (binaryfunc)array_floor_divide, + .nb_true_divide = (binaryfunc)array_true_divide, + .nb_inplace_floor_divide = (binaryfunc)array_inplace_floor_divide, + .nb_inplace_true_divide = (binaryfunc)array_inplace_true_divide, + + .nb_matrix_multiply = (binaryfunc)array_matrix_multiply, + .nb_inplace_matrix_multiply = (binaryfunc)array_inplace_matrix_multiply, }; diff --git a/numpy/core/src/multiarray/refcount.c b/numpy/core/src/multiarray/refcount.c index 6033929d9..c869b5eea 100644 --- a/numpy/core/src/multiarray/refcount.c +++ b/numpy/core/src/multiarray/refcount.c @@ -46,7 +46,7 @@ PyArray_Item_INCREF(char *data, PyArray_Descr *descr) Py_ssize_t pos = 0; while (PyDict_Next(descr->fields, &pos, &key, &value)) { - if NPY_TITLE_KEY(key, value) { + if (NPY_TITLE_KEY(key, value)) { continue; } if (!PyArg_ParseTuple(value, "Oi|O", &new, &offset, @@ -108,7 +108,7 @@ PyArray_Item_XDECREF(char *data, PyArray_Descr *descr) Py_ssize_t pos = 0; while (PyDict_Next(descr->fields, &pos, &key, &value)) { - if NPY_TITLE_KEY(key, value) { + if (NPY_TITLE_KEY(key, value)) { continue; } if (!PyArg_ParseTuple(value, "Oi|O", &new, &offset, @@ -318,7 +318,7 @@ _fillobject(char *optr, PyObject *obj, PyArray_Descr *dtype) Py_ssize_t pos = 0; while (PyDict_Next(dtype->fields, &pos, &key, &value)) { - if NPY_TITLE_KEY(key, value) { + if (NPY_TITLE_KEY(key, value)) { continue; } if (!PyArg_ParseTuple(value, "Oi|O", &new, &offset, &title)) { diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index 3a66df454..8a7139fb2 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -45,7 +45,7 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) type_num = descr->type_num; } switch (type_num) { -#define CASE(ut,lt) case NPY_##ut: return &(((Py##lt##ScalarObject *)scalar)->obval) +#define CASE(ut,lt) case NPY_##ut: return &PyArrayScalar_VAL(scalar, lt) CASE(BOOL, Bool); CASE(BYTE, Byte); CASE(UBYTE, UByte); @@ -71,9 +71,19 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) case NPY_STRING: return (void *)PyString_AS_STRING(scalar); case NPY_UNICODE: - return (void *)PyUnicode_AS_DATA(scalar); + /* lazy initialization, to reduce the memory used by string scalars */ + if (PyArrayScalar_VAL(scalar, Unicode) == NULL) { + Py_UCS4 *raw_data = PyUnicode_AsUCS4Copy(scalar); + if (raw_data == NULL) { + return NULL; + } + PyArrayScalar_VAL(scalar, Unicode) = raw_data; + return (void *)raw_data; + } + return PyArrayScalar_VAL(scalar, Unicode); case NPY_VOID: - return ((PyVoidScalarObject *)scalar)->obval; + /* Note: no & needed here, so can't use CASE */ + return PyArrayScalar_VAL(scalar, Void); } /* @@ -81,14 +91,13 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) * scalar it inherits from. */ -#define _CHK(cls) (PyObject_IsInstance(scalar, \ - (PyObject *)&Py##cls##ArrType_Type)) -#define _OBJ(lt) &(((Py##lt##ScalarObject *)scalar)->obval) -#define _IFCASE(cls) if _CHK(cls) return _OBJ(cls) +#define _CHK(cls) PyObject_IsInstance(scalar, \ + (PyObject *)&Py##cls##ArrType_Type) +#define _IFCASE(cls) if (_CHK(cls)) return &PyArrayScalar_VAL(scalar, cls) - if _CHK(Number) { - if _CHK(Integer) { - if _CHK(SignedInteger) { + if (_CHK(Number)) { + if (_CHK(Integer)) { + if (_CHK(SignedInteger)) { _IFCASE(Byte); _IFCASE(Short); _IFCASE(Int); @@ -107,7 +116,7 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) } else { /* Inexact */ - if _CHK(Floating) { + if (_CHK(Floating)) { _IFCASE(Half); _IFCASE(Float); _IFCASE(Double); @@ -122,10 +131,10 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) } } else if (_CHK(Bool)) { - return _OBJ(Bool); + return &PyArrayScalar_VAL(scalar, Bool); } else if (_CHK(Datetime)) { - return _OBJ(Datetime); + return &PyArrayScalar_VAL(scalar, Datetime); } else if (_CHK(Flexible)) { if (_CHK(String)) { @@ -135,7 +144,8 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) return (void *)PyUnicode_AS_DATA(scalar); } if (_CHK(Void)) { - return ((PyVoidScalarObject *)scalar)->obval; + /* Note: no & needed here, so can't use _IFCASE */ + return PyArrayScalar_VAL(scalar, Void); } } else { @@ -156,7 +166,6 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) } return (void *)memloc; #undef _IFCASE -#undef _OBJ #undef _CHK } @@ -319,21 +328,10 @@ PyArray_FromScalar(PyObject *scalar, PyArray_Descr *outcode) memptr = scalar_value(scalar, typecode); -#ifndef Py_UNICODE_WIDE - if (typecode->type_num == NPY_UNICODE) { - PyUCS2Buffer_AsUCS4((Py_UNICODE *)memptr, - (npy_ucs4 *)PyArray_DATA(r), - PyUnicode_GET_SIZE(scalar), - PyArray_ITEMSIZE(r) >> 2); - } - else -#endif - { - memcpy(PyArray_DATA(r), memptr, PyArray_ITEMSIZE(r)); - if (PyDataType_FLAGCHK(typecode, NPY_ITEM_HASOBJECT)) { - /* Need to INCREF just the PyObject portion */ - PyArray_Item_INCREF(memptr, typecode); - } + memcpy(PyArray_DATA(r), memptr, PyArray_ITEMSIZE(r)); + if (PyDataType_FLAGCHK(typecode, NPY_ITEM_HASOBJECT)) { + /* Need to INCREF just the PyObject portion */ + PyArray_Item_INCREF(memptr, typecode); } finish: @@ -435,23 +433,59 @@ PyArray_DescrFromTypeObject(PyObject *type) if ((type == (PyObject *) &PyNumberArrType_Type) || (type == (PyObject *) &PyInexactArrType_Type) || (type == (PyObject *) &PyFloatingArrType_Type)) { + if (DEPRECATE("Converting `np.inexact` or `np.floating` to " + "a dtype is deprecated. The current result is `float64` " + "which is not strictly correct.") < 0) { + return NULL; + } typenum = NPY_DOUBLE; } else if (type == (PyObject *)&PyComplexFloatingArrType_Type) { + if (DEPRECATE("Converting `np.complex` to a dtype is deprecated. " + "The current result is `complex128` which is not " + "strictly correct.") < 0) { + return NULL; + } typenum = NPY_CDOUBLE; } else if ((type == (PyObject *)&PyIntegerArrType_Type) || (type == (PyObject *)&PySignedIntegerArrType_Type)) { + if (DEPRECATE("Converting `np.integer` or `np.signedinteger` to " + "a dtype is deprecated. The current result is " + "`np.dtype(np.int_)` which is not strictly correct. " + "Note that the result depends on the system. To ensure " + "stable results use may want to use `np.int64` or " + "`np.int32`.") < 0) { + return NULL; + } typenum = NPY_LONG; } else if (type == (PyObject *) &PyUnsignedIntegerArrType_Type) { + if (DEPRECATE("Converting `np.unsignedinteger` to a dtype is " + "deprecated. The current result is `np.dtype(np.uint)` " + "which is not strictly correct. Note that the result " + "depends on the system. To ensure stable results you may " + "want to use `np.uint64` or `np.uint32`.") < 0) { + return NULL; + } typenum = NPY_ULONG; } else if (type == (PyObject *) &PyCharacterArrType_Type) { + if (DEPRECATE("Converting `np.character` to a dtype is deprecated. " + "The current result is `np.dtype(np.str_)` " + "which is not strictly correct. Note that `np.character` " + "is generally deprecated and 'S1' should be used.") < 0) { + return NULL; + } typenum = NPY_STRING; } else if ((type == (PyObject *) &PyGenericArrType_Type) || (type == (PyObject *) &PyFlexibleArrType_Type)) { + if (DEPRECATE("Converting `np.generic` to a dtype is " + "deprecated. The current result is `np.dtype(np.void)` " + "which is not strictly correct.") < 0) { + return NULL; + } typenum = NPY_VOID; } @@ -561,6 +595,9 @@ PyArray_DescrFromScalar(PyObject *sc) } descr = PyArray_DescrFromTypeObject((PyObject *)Py_TYPE(sc)); + if (descr == NULL) { + return NULL; + } if (PyDataType_ISUNSIZED(descr)) { PyArray_DESCR_REPLACE(descr); type_num = descr->type_num; @@ -568,10 +605,7 @@ PyArray_DescrFromScalar(PyObject *sc) descr->elsize = PyString_GET_SIZE(sc); } else if (type_num == NPY_UNICODE) { - descr->elsize = PyUnicode_GET_DATA_SIZE(sc); -#ifndef Py_UNICODE_WIDE - descr->elsize <<= 1; -#endif + descr->elsize = PyUnicode_GET_LENGTH(sc) * 4; } else { PyArray_Descr *dtype; @@ -654,23 +688,30 @@ PyArray_Scalar(void *data, PyArray_Descr *descr, PyObject *base) } } if (type_num == NPY_UNICODE) { - PyObject *u, *args; - int byteorder; - -#if NPY_BYTE_ORDER == NPY_LITTLE_ENDIAN - byteorder = -1; -#elif NPY_BYTE_ORDER == NPY_BIG_ENDIAN - byteorder = +1; -#else - #error Endianness undefined ? -#endif - if (swap) byteorder *= -1; - - u = PyUnicode_DecodeUTF32(data, itemsize, NULL, &byteorder); + /* we need the full string length here, else copyswap will write too + many bytes */ + void *buff = PyArray_malloc(descr->elsize); + if (buff == NULL) { + return PyErr_NoMemory(); + } + /* copyswap needs an array object, but only actually cares about the + * dtype + */ + PyArrayObject_fields dummy_arr; + if (base == NULL) { + dummy_arr.descr = descr; + base = (PyObject *)&dummy_arr; + } + copyswap(buff, data, swap, base); + + /* truncation occurs here */ + PyObject *u = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buff, itemsize / 4); + PyArray_free(buff); if (u == NULL) { return NULL; } - args = Py_BuildValue("(O)", u); + + PyObject *args = Py_BuildValue("(O)", u); if (args == NULL) { Py_DECREF(u); return NULL; diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index cd26d20fa..2f1767391 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -55,53 +55,8 @@ NPY_NO_EXPORT PyTypeObject PyTimeIntegerArrType_Type; */ NPY_NO_EXPORT PyTypeObject Py@NAME@ArrType_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.@name@", /* tp_name*/ - sizeof(PyObject), /* tp_basicsize*/ - 0, /* tp_itemsize */ - /* methods */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - 0, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.@name@", + .tp_basicsize = sizeof(PyObject), }; /**end repeat**/ @@ -318,7 +273,7 @@ gentype_format(PyObject *self, PyObject *args) * because it throws away precision. */ if (Py_TYPE(self) == &PyBoolArrType_Type) { - obj = PyBool_FromLong(((PyBoolScalarObject *)self)->obval); + obj = PyBool_FromLong(PyArrayScalar_VAL(self, Bool)); } else if (PyArray_IsScalar(self, Integer)) { obj = Py_TYPE(self)->tp_as_number->nb_int(self); @@ -390,6 +345,10 @@ format_@name@(@type@ val, npy_bool scientific, * over-ride repr and str of array-scalar strings and unicode to * remove NULL bytes and then call the corresponding functions * of string and unicode. + * + * FIXME: + * is this really a good idea? + * stop using Py_UNICODE here. */ /**begin repeat @@ -903,7 +862,7 @@ static PyObject * static PyObject * @name@type_@kind@(PyObject *self) { - return @name@type_@kind@_either(((Py@Name@ScalarObject *)self)->obval, + return @name@type_@kind@_either(PyArrayScalar_VAL(self, @Name@), TrimMode_LeaveOneZero, TrimMode_DptZeros, 0); } @@ -911,7 +870,7 @@ static PyObject * c@name@type_@kind@(PyObject *self) { PyObject *rstr, *istr, *ret; - npy_c@name@ val = ((PyC@Name@ScalarObject *)self)->obval; + npy_c@name@ val = PyArrayScalar_VAL(self, C@Name@); TrimMode trim = TrimMode_DptZeros; if (npy_legacy_print_mode == 113) { @@ -975,7 +934,7 @@ c@name@type_@kind@(PyObject *self) static PyObject * halftype_@kind@(PyObject *self) { - npy_half val = ((PyHalfScalarObject *)self)->obval; + npy_half val = PyArrayScalar_VAL(self, Half); float floatval = npy_half_to_float(val); float absval; @@ -1016,42 +975,26 @@ static PyObject * /**end repeat**/ static PyNumberMethods gentype_as_number = { - (binaryfunc)gentype_add, /*nb_add*/ - (binaryfunc)gentype_subtract, /*nb_subtract*/ - (binaryfunc)gentype_multiply, /*nb_multiply*/ - (binaryfunc)gentype_remainder, /*nb_remainder*/ - (binaryfunc)gentype_divmod, /*nb_divmod*/ - (ternaryfunc)gentype_power, /*nb_power*/ - (unaryfunc)gentype_negative, - (unaryfunc)gentype_positive, /*nb_pos*/ - (unaryfunc)gentype_absolute, /*(unaryfunc)gentype_abs,*/ - (inquiry)gentype_nonzero_number, /*nb_nonzero*/ - (unaryfunc)gentype_invert, /*nb_invert*/ - (binaryfunc)gentype_lshift, /*nb_lshift*/ - (binaryfunc)gentype_rshift, /*nb_rshift*/ - (binaryfunc)gentype_and, /*nb_and*/ - (binaryfunc)gentype_xor, /*nb_xor*/ - (binaryfunc)gentype_or, /*nb_or*/ - (unaryfunc)gentype_int, /*nb_int*/ - 0, /*nb_reserved*/ - (unaryfunc)gentype_float, /*nb_float*/ - 0, /*inplace_add*/ - 0, /*inplace_subtract*/ - 0, /*inplace_multiply*/ - 0, /*inplace_remainder*/ - 0, /*inplace_power*/ - 0, /*inplace_lshift*/ - 0, /*inplace_rshift*/ - 0, /*inplace_and*/ - 0, /*inplace_xor*/ - 0, /*inplace_or*/ - (binaryfunc)gentype_floor_divide, /*nb_floor_divide*/ - (binaryfunc)gentype_true_divide, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - (unaryfunc)NULL, /*nb_index*/ - 0, /*np_matmul*/ - 0, /*np_inplace_matmul*/ + .nb_add = (binaryfunc)gentype_add, + .nb_subtract = (binaryfunc)gentype_subtract, + .nb_multiply = (binaryfunc)gentype_multiply, + .nb_remainder = (binaryfunc)gentype_remainder, + .nb_divmod = (binaryfunc)gentype_divmod, + .nb_power = (ternaryfunc)gentype_power, + .nb_negative = (unaryfunc)gentype_negative, + .nb_positive = (unaryfunc)gentype_positive, + .nb_absolute = (unaryfunc)gentype_absolute, + .nb_bool = (inquiry)gentype_nonzero_number, + .nb_invert = (unaryfunc)gentype_invert, + .nb_lshift = (binaryfunc)gentype_lshift, + .nb_rshift = (binaryfunc)gentype_rshift, + .nb_and = (binaryfunc)gentype_and, + .nb_xor = (binaryfunc)gentype_xor, + .nb_or = (binaryfunc)gentype_or, + .nb_int = (unaryfunc)gentype_int, + .nb_float = (unaryfunc)gentype_float, + .nb_floor_divide = (binaryfunc)gentype_floor_divide, + .nb_true_divide = (binaryfunc)gentype_true_divide, }; @@ -1155,11 +1098,6 @@ gentype_itemsize_get(PyObject *self) typecode = PyArray_DescrFromScalar(self); elsize = typecode->elsize; -#ifndef Py_UNICODE_WIDE - if (typecode->type_num == NPY_UNICODE) { - elsize >>= 1; - } -#endif ret = PyInt_FromLong((long) elsize); Py_DECREF(typecode); return ret; @@ -1318,7 +1256,7 @@ gentype_real_get(PyObject *self) return ret; } else if (PyArray_IsScalar(self, Object)) { - PyObject *obj = ((PyObjectScalarObject *)self)->obval; + PyObject *obj = PyArrayScalar_VAL(self, Object); ret = PyObject_GetAttrString(obj, "real"); if (ret != NULL) { return ret; @@ -1343,7 +1281,7 @@ gentype_imag_get(PyObject *self) ret = PyArray_Scalar(ptr + typecode->elsize, typecode, NULL); } else if (PyArray_IsScalar(self, Object)) { - PyObject *obj = ((PyObjectScalarObject *)self)->obval; + PyObject *obj = PyArrayScalar_VAL(self, Object); PyArray_Descr *newtype; ret = PyObject_GetAttrString(obj, "imag"); if (ret == NULL) { @@ -1551,14 +1489,6 @@ gentype_itemset(PyObject *NPY_UNUSED(self), PyObject *NPY_UNUSED(args)) return NULL; } -/* This function matches the Python 2.7 PyBufferProcs.bf_getreadbuffer - * interface, but no longer needs to. In the future we could consider - * rewriting callers to use `gentype_getbuffer`, or inline the function body - * at the caller. - */ -static Py_ssize_t -gentype_getreadbuf(PyObject *, Py_ssize_t, void **); - static PyObject * gentype_byteswap(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1581,8 +1511,9 @@ gentype_byteswap(PyObject *self, PyObject *args, PyObject *kwds) PyObject *new; char *newmem; - gentype_getreadbuf(self, 0, (void **)&data); descr = PyArray_DescrFromScalar(self); + data = (void *)scalar_value(self, descr); + newmem = PyObject_Malloc(descr->elsize); if (newmem == NULL) { Py_DECREF(descr); @@ -1618,6 +1549,58 @@ gentype_@name@(PyObject *self, PyObject *args, PyObject *kwds) } /**end repeat**/ + +/**begin repeat + * #name = integer, floating, complexfloating# + * #complex = 0, 0, 1# + */ +static PyObject * +@name@type_dunder_round(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"ndigits", NULL}; + PyObject *ndigits = Py_None; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:__round__", kwlist, &ndigits)) { + return NULL; + } + +#if @complex@ + if (DEPRECATE("The Python built-in `round` is deprecated for complex " + "scalars, and will raise a `TypeError` in a future release. " + "Use `np.round` or `scalar.round` instead.") < 0) { + return NULL; + } +#endif + + PyObject *tup; + if (ndigits == Py_None) { + tup = PyTuple_Pack(0); + } + else { + tup = PyTuple_Pack(1, ndigits); + } + + if (tup == NULL) { + return NULL; + } + + PyObject *obj = gentype_round(self, tup, NULL); + Py_DECREF(tup); + if (obj == NULL) { + return NULL; + } + +#if !@complex@ + if (ndigits == Py_None) { + PyObject *ret = PyNumber_Long(obj); + Py_DECREF(obj); + return ret; + } +#endif + + return obj; +} +/**end repeat**/ + static PyObject * voidtype_getfield(PyVoidScalarObject *self, PyObject *args, PyObject *kwds) { @@ -1726,12 +1709,7 @@ gentype_reduce(PyObject *self, PyObject *NPY_UNUSED(args)) return NULL; } - if (PyArray_IsScalar(self, Unicode)) { - /* Unicode on Python 3 does not expose the buffer interface */ - buffer = PyUnicode_AS_DATA(self); - buflen = PyUnicode_GET_DATA_SIZE(self); - } - else if (PyObject_GetBuffer(self, &view, PyBUF_SIMPLE) >= 0) { + if (PyObject_GetBuffer(self, &view, PyBUF_SIMPLE) >= 0) { buffer = view.buf; buflen = view.len; /* @@ -1760,7 +1738,7 @@ gentype_reduce(PyObject *self, PyObject *NPY_UNUSED(args)) PyTuple_SET_ITEM(ret, 0, obj); obj = PyObject_GetAttrString((PyObject *)self, "dtype"); if (PyArray_IsScalar(self, Object)) { - PyObject *val = ((PyObjectScalarObject *)self)->obval; + PyObject *val = PyArrayScalar_VAL(self, Object); PyObject *tup = Py_BuildValue("NO", obj, val); if (tup == NULL) { return NULL; @@ -1786,48 +1764,13 @@ gentype_reduce(PyObject *self, PyObject *NPY_UNUSED(args)) PyTuple_SET_ITEM(ret, 1, tup); } else { -#ifndef Py_UNICODE_WIDE - /* - * We need to expand the buffer so that we always write - * UCS4 to disk for pickle of unicode scalars. - * - * This could be in a unicode_reduce function, but - * that would require re-factoring. - */ - int alloc = 0; - char *tmp; - int newlen; - - if (PyArray_IsScalar(self, Unicode)) { - tmp = PyArray_malloc(buflen*2); - if (tmp == NULL) { - Py_DECREF(ret); - return PyErr_NoMemory(); - } - alloc = 1; - newlen = PyUCS2Buffer_AsUCS4((Py_UNICODE *)buffer, - (npy_ucs4 *)tmp, - buflen / 2, buflen / 2); - buflen = newlen*4; - buffer = tmp; - } -#endif mod = PyBytes_FromStringAndSize(buffer, buflen); if (mod == NULL) { Py_DECREF(ret); -#ifndef Py_UNICODE_WIDE - ret = NULL; - goto fail; -#else return NULL; -#endif } PyTuple_SET_ITEM(ret, 1, Py_BuildValue("NN", obj, mod)); -#ifndef Py_UNICODE_WIDE -fail: - if (alloc) PyArray_free((char *)buffer); -#endif } return ret; } @@ -2165,10 +2108,6 @@ static PyMethodDef gentype_methods[] = { {"round", (PyCFunction)gentype_round, METH_VARARGS | METH_KEYWORDS, NULL}, - /* Hook for the round() builtin */ - {"__round__", - (PyCFunction)gentype_round, - METH_VARARGS | METH_KEYWORDS, NULL}, /* For the format function */ {"__format__", gentype_format, @@ -2239,6 +2178,18 @@ static PyMethodDef @name@type_methods[] = { /**end repeat**/ /**begin repeat + * #name = integer,floating, complexfloating# + */ +static PyMethodDef @name@type_methods[] = { + /* Hook for the round() builtin */ + {"__round__", + (PyCFunction)@name@type_dunder_round, + METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL, 0, NULL} /* sentinel */ +}; +/**end repeat**/ + +/**begin repeat * #name = half,float,double,longdouble# */ static PyMethodDef @name@type_methods[] = { @@ -2429,54 +2380,22 @@ fail: } static PyMappingMethods voidtype_as_mapping = { - (lenfunc)voidtype_length, /*mp_length*/ - (binaryfunc)voidtype_subscript, /*mp_subscript*/ - (objobjargproc)voidtype_ass_subscript, /*mp_ass_subscript*/ + .mp_length = (lenfunc)voidtype_length, + .mp_subscript = (binaryfunc)voidtype_subscript, + .mp_ass_subscript = (objobjargproc)voidtype_ass_subscript, }; static PySequenceMethods voidtype_as_sequence = { - (lenfunc)voidtype_length, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - (ssizeargfunc)voidtype_item, /*sq_item*/ - 0, /*sq_slice*/ - (ssizeobjargproc)voidtype_ass_item, /*sq_ass_item*/ - 0, /* ssq_ass_slice */ - 0, /* sq_contains */ - 0, /* sq_inplace_concat */ - 0, /* sq_inplace_repeat */ + .sq_length = (lenfunc)voidtype_length, + .sq_item = (ssizeargfunc)voidtype_item, + .sq_ass_item = (ssizeobjargproc)voidtype_ass_item, }; -static Py_ssize_t -gentype_getreadbuf(PyObject *self, Py_ssize_t segment, void **ptrptr) -{ - int numbytes; - PyArray_Descr *outcode; - - if (segment != 0) { - PyErr_SetString(PyExc_SystemError, - "Accessing non-existent array segment"); - return -1; - } - - outcode = PyArray_DescrFromScalar(self); - numbytes = outcode->elsize; - *ptrptr = (void *)scalar_value(self, outcode); - -#ifndef Py_UNICODE_WIDE - if (outcode->type_num == NPY_UNICODE) { - numbytes >>= 1; - } -#endif - Py_DECREF(outcode); - return numbytes; -} - static PyBufferProcs gentype_as_buffer = { - gentype_getbuffer, /* bf_getbuffer */ - NULL, /* bf_releasebuffer */ + .bf_getbuffer = gentype_getbuffer, + /* release buffer not defined (see buffer.c) */ }; @@ -2485,53 +2404,8 @@ static PyBufferProcs gentype_as_buffer = { NPY_NO_EXPORT PyTypeObject PyGenericArrType_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.generic", /* tp_name*/ - sizeof(PyObject), /* tp_basicsize*/ - 0, /* tp_itemsize */ - /* methods */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - 0, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.generic", + .tp_basicsize = sizeof(PyObject), }; static void @@ -2550,179 +2424,163 @@ void_dealloc(PyVoidScalarObject *v) static void object_arrtype_dealloc(PyObject *v) { - Py_XDECREF(((PyObjectScalarObject *)v)->obval); + Py_XDECREF(PyArrayScalar_VAL(v, Object)); Py_TYPE(v)->tp_free(v); } -/* - * string and unicode inherit from Python Type first and so GET_ITEM - * is different to get to the Python Type. - * - * ok is a work-around for a bug in complex_new that doesn't allocate - * memory from the sub-types memory allocator. - */ - -#define _WORK(num) \ - if (type->tp_bases && (PyTuple_GET_SIZE(type->tp_bases)==2)) { \ - PyTypeObject *sup; \ - /* We are inheriting from a Python type as well so \ - give it first dibs on conversion */ \ - sup = (PyTypeObject *)PyTuple_GET_ITEM(type->tp_bases, num); \ - /* Prevent recursion */ \ - if (thisfunc != sup->tp_new) { \ - robj = sup->tp_new(type, args, kwds); \ - if (robj != NULL) goto finish; \ - if (PyTuple_GET_SIZE(args)!=1) return NULL; \ - PyErr_Clear(); \ - } \ - /* now do default conversion */ \ - } - -#define _WORK1 _WORK(1) -#define _WORKz _WORK(0) -#define _WORK0 +static void +unicode_arrtype_dealloc(PyObject *v) +{ + /* note: may be null if it was never requested */ + PyMem_Free(PyArrayScalar_VAL(v, Unicode)); + /* delegate to the base class */ + PyUnicode_Type.tp_dealloc(v); +} /**begin repeat * #name = byte, short, int, long, longlong, ubyte, ushort, uint, ulong, * ulonglong, half, float, double, longdouble, cfloat, cdouble, - * clongdouble, string, unicode, object# + * clongdouble, string, unicode# * #Name = Byte, Short, Int, Long, LongLong, UByte, UShort, UInt, ULong, * ULongLong, Half, Float, Double, LongDouble, CFloat, CDouble, - * CLongDouble, String, Unicode, Object# + * CLongDouble, String, Unicode# * #TYPE = BYTE, SHORT, INT, LONG, LONGLONG, UBYTE, USHORT, UINT, ULONG, * ULONGLONG, HALF, FLOAT, DOUBLE, LONGDOUBLE, CFLOAT, CDOUBLE, - * CLONGDOUBLE, STRING, UNICODE, OBJECT# - * #work = 0,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,z,z,0# - * #default = 0*17,1*2,2# + * CLONGDOUBLE, STRING, UNICODE# */ -#define _NPY_UNUSED2_1 -#define _NPY_UNUSED2_z -#define _NPY_UNUSED2_0 NPY_UNUSED -#define _NPY_UNUSED1_0 -#define _NPY_UNUSED1_1 -#define _NPY_UNUSED1_2 NPY_UNUSED +/* used as a pattern for testing token equality */ +#define _@TYPE@_IS_@TYPE@ static PyObject * -@name@_arrtype_new(PyTypeObject *_NPY_UNUSED1_@default@(type), PyObject *args, PyObject *_NPY_UNUSED2_@work@(kwds)) +@name@_arrtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *obj = NULL; - PyObject *robj; - PyArrayObject *arr; - PyArray_Descr *typecode = NULL; -#if (@work@ != 0) || (@default@ == 1) - void *thisfunc = (void *)@name@_arrtype_new; + /* allow base-class (if any) to do conversion */ +#if defined(_@TYPE@_IS_UNICODE) + PyObject *from_superclass = PyUnicode_Type.tp_new(type, args, kwds); +#elif defined(_@TYPE@_IS_STRING) + PyObject *from_superclass = PyBytes_Type.tp_new(type, args, kwds); +#elif defined(_@TYPE@_IS_DOUBLE) + PyObject *from_superclass = PyFloat_Type.tp_new(type, args, kwds); #endif -#if !(@default@ == 2) - int itemsize; - void *dest, *src; +#if defined(_@TYPE@_IS_UNICODE) || defined(_@TYPE@_IS_STRING) || defined(_@TYPE@_IS_DOUBLE) + if (from_superclass == NULL) { + /* don't clear the exception unless numpy can handle the arguments */ + if (PyTuple_GET_SIZE(args) != 1 || (kwds && PyDict_Size(kwds) != 0)) { + return NULL; + } + PyErr_Clear(); + } + else { +#if defined(_@TYPE@_IS_UNICODE) + PyArrayScalar_VAL(from_superclass, Unicode) = NULL; +#endif + return from_superclass; + } #endif - - /* - * allow base-class (if any) to do conversion - * If successful, this will jump to finish: - */ - _WORK@work@ /* TODO: include type name in error message, which is not @name@ */ - if (!PyArg_ParseTuple(args, "|O", &obj)) { + PyObject *obj = NULL; + char *kwnames[] = {"", NULL}; /* positional-only */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwnames, &obj)) { return NULL; } - typecode = PyArray_DescrFromType(NPY_@TYPE@); + PyArray_Descr *typecode = PyArray_DescrFromType(NPY_@TYPE@); if (typecode == NULL) { return NULL; } - /* - * typecode is new reference and stolen by - * PyArray_FromAny but not PyArray_Scalar - */ if (obj == NULL) { -#if @default@ == 0 - robj = PyArray_Scalar(NULL, typecode, NULL); + PyObject *robj = PyArray_Scalar(NULL, typecode, NULL); + Py_DECREF(typecode); if (robj == NULL) { - Py_DECREF(typecode); return NULL; } - memset(&((Py@Name@ScalarObject *)robj)->obval, 0, sizeof(npy_@name@)); -#elif @default@ == 1 - robj = PyArray_Scalar(NULL, typecode, NULL); -#elif @default@ == 2 - Py_INCREF(Py_None); - robj = Py_None; +#if !defined(_@TYPE@_IS_STRING) && !defined(_@TYPE@_IS_UNICODE) + memset(&PyArrayScalar_VAL(robj, @Name@), 0, sizeof(npy_@name@)); #endif - Py_DECREF(typecode); - goto finish; + return robj; } - /* - * It is expected at this point that robj is a PyArrayScalar - * (even for Object Data Type) - */ - arr = (PyArrayObject *)PyArray_FromAny(obj, typecode, - 0, 0, NPY_ARRAY_FORCECAST, NULL); - if ((arr == NULL) || (PyArray_NDIM(arr) > 0)) { + /* PyArray_FromAny steals a reference, reclaim it before it's gone */ + Py_INCREF(typecode); + PyArrayObject *arr = (PyArrayObject *)PyArray_FromAny( + obj, typecode, 0, 0, NPY_ARRAY_FORCECAST, NULL); + if (arr == NULL) { + Py_DECREF(typecode); + return NULL; + } + if (PyArray_NDIM(arr) > 0) { + Py_DECREF(typecode); return (PyObject *)arr; } - /* 0-d array */ - robj = PyArray_ToScalar(PyArray_DATA(arr), arr); + + /* Convert the 0-d array to a scalar*/ + PyObject *robj = PyArray_ToScalar(PyArray_DATA(arr), arr); Py_DECREF(arr); -finish: - /* - * In OBJECT case, robj is no longer a - * PyArrayScalar at this point but the - * remaining code assumes it is - */ -#if @default@ == 2 - return robj; -#else - /* Normal return */ - if ((robj == NULL) || (Py_TYPE(robj) == type)) { + if (robj == NULL || Py_TYPE(robj) == type) { + Py_DECREF(typecode); return robj; } /* - * This return path occurs when the requested type is not created - * but another scalar object is created instead (i.e. when - * the base-class does the conversion in _WORK macro) + * `typecode` does not contain any subclass information, as it was thrown + * out by the call to `PyArray_DescrFromType` - we need to add this back. + * + * FIXME[gh-15467]: This branch is also hit for the "shadowed" builtin + * types like `longdouble` (which on platforms where they are the same size + * is shadowed by `double`), because `PyArray_FromAny` returns the + * shadowing type rather than the requested one. */ /* Need to allocate new type and copy data-area over */ + int itemsize; if (type->tp_itemsize) { itemsize = PyBytes_GET_SIZE(robj); } else { itemsize = 0; } - obj = type->tp_alloc(type, itemsize); - if (obj == NULL) { + PyObject *new_obj = type->tp_alloc(type, itemsize); + if (new_obj == NULL) { Py_DECREF(robj); + Py_DECREF(typecode); return NULL; } - /* typecode will be NULL */ - typecode = PyArray_DescrFromType(NPY_@TYPE@); - dest = scalar_value(obj, typecode); - src = scalar_value(robj, typecode); + void *dest = scalar_value(new_obj, typecode); + void *src = scalar_value(robj, typecode); Py_DECREF(typecode); -#if @default@ == 0 - *((npy_@name@ *)dest) = *((npy_@name@ *)src); -#elif @default@ == 1 /* unicode and strings */ +#if defined(_@TYPE@_IS_STRING) || defined(_@TYPE@_IS_UNICODE) if (itemsize == 0) { /* unicode */ itemsize = PyUnicode_GetLength(robj) * PyUnicode_KIND(robj); } memcpy(dest, src, itemsize); - /* @default@ == 2 won't get here */ +#else + *((npy_@name@ *)dest) = *((npy_@name@ *)src); #endif Py_DECREF(robj); - return obj; -#endif + return new_obj; } +#undef _@TYPE@_IS_@TYPE@ + /**end repeat**/ -#undef _WORK1 -#undef _WORKz -#undef _WORK0 -#undef _WORK +static PyObject * +object_arrtype_new(PyTypeObject *NPY_UNUSED(type), PyObject *args, PyObject *kwds) +{ + PyObject *obj = Py_None; + char *kwnames[] = {"", NULL}; /* positional-only */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:object_", kwnames, &obj)) { + return NULL; + } + PyArray_Descr *typecode = PyArray_DescrFromType(NPY_OBJECT); + if (typecode == NULL) { + return NULL; + } + PyArrayObject *arr = (PyArrayObject *)PyArray_FromAny(obj, typecode, + 0, 0, NPY_ARRAY_FORCECAST, NULL); + return PyArray_Return(arr); +} /**begin repeat * #name = datetime, timedelta# @@ -2737,8 +2595,8 @@ static PyObject * PyObject *obj = NULL, *meta_obj = NULL; Py@Name@ScalarObject *ret; - /* TODO: include type name in error message, which is not @name@ */ - if (!PyArg_ParseTuple(args, "|OO", &obj, &meta_obj)) { + char *kwnames[] = {"", "", NULL}; /* positional-only */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwnames, &obj, &meta_obj)) { return NULL; } @@ -2791,12 +2649,13 @@ static PyObject * /* bool->tp_new only returns Py_True or Py_False */ static PyObject * -bool_arrtype_new(PyTypeObject *NPY_UNUSED(type), PyObject *args, PyObject *NPY_UNUSED(kwds)) +bool_arrtype_new(PyTypeObject *NPY_UNUSED(type), PyObject *args, PyObject *kwds) { PyObject *obj = NULL; PyArrayObject *arr; - if (!PyArg_ParseTuple(args, "|O:bool_", &obj)) { + char *kwnames[] = {"", NULL}; /* positional-only */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:bool_", kwnames, &obj)) { return NULL; } if (obj == NULL) { @@ -2894,53 +2753,20 @@ bool_index(PyObject *a) /* Arithmetic methods -- only so we can override &, |, ^. */ NPY_NO_EXPORT PyNumberMethods bool_arrtype_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)bool_arrtype_nonzero, /* nb_nonzero / nb_bool */ - 0, /* nb_invert */ - 0, /* nb_lshift */ - 0, /* nb_rshift */ - (binaryfunc)bool_arrtype_and, /* nb_and */ - (binaryfunc)bool_arrtype_xor, /* nb_xor */ - (binaryfunc)bool_arrtype_or, /* nb_or */ - 0, /* nb_int */ - 0, /* nb_reserved */ - 0, /* nb_float */ - /* Added in release 2.0 */ - 0, /* nb_inplace_add */ - 0, /* nb_inplace_subtract */ - 0, /* nb_inplace_multiply */ - 0, /* nb_inplace_remainder */ - 0, /* nb_inplace_power */ - 0, /* nb_inplace_lshift */ - 0, /* nb_inplace_rshift */ - 0, /* nb_inplace_and */ - 0, /* nb_inplace_xor */ - 0, /* nb_inplace_or */ - /* Added in release 2.2 */ - /* The following require the Py_TPFLAGS_HAVE_CLASS flag */ - 0, /* nb_floor_divide */ - 0, /* nb_true_divide */ - 0, /* nb_inplace_floor_divide */ - 0, /* nb_inplace_true_divide */ - /* Added in release 2.5 */ - 0, /* nb_index */ + .nb_bool = (inquiry)bool_arrtype_nonzero, + .nb_and = (binaryfunc)bool_arrtype_and, + .nb_xor = (binaryfunc)bool_arrtype_xor, + .nb_or = (binaryfunc)bool_arrtype_or, }; static PyObject * -void_arrtype_new(PyTypeObject *type, PyObject *args, PyObject *NPY_UNUSED(kwds)) +void_arrtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyObject *obj, *arr; PyObject *new = NULL; - if (!PyArg_ParseTuple(args, "O:void", &obj)) { + char *kwnames[] = {"", NULL}; /* positional-only */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:void", kwnames, &obj)) { return NULL; } /* @@ -3000,7 +2826,7 @@ void_arrtype_new(PyTypeObject *type, PyObject *args, PyObject *NPY_UNUSED(kwds)) static npy_hash_t @lname@_arrtype_hash(PyObject *obj) { - return (npy_hash_t)(((Py@name@ScalarObject *)obj)->obval); + return (npy_hash_t)(PyArrayScalar_VAL(obj, @name@)); } /**end repeat**/ @@ -3011,7 +2837,7 @@ static npy_hash_t static npy_hash_t @lname@_arrtype_hash(PyObject *obj) { - npy_hash_t x = (npy_hash_t)(((Py@name@ScalarObject *)obj)->obval); + npy_hash_t x = (npy_hash_t)(PyArrayScalar_VAL(obj, @name@)); if (x == -1) { x = -2; } @@ -3022,7 +2848,7 @@ static npy_hash_t static npy_hash_t ulong_arrtype_hash(PyObject *obj) { - PyObject * l = PyLong_FromUnsignedLong(((PyULongScalarObject*)obj)->obval); + PyObject * l = PyLong_FromUnsignedLong(PyArrayScalar_VAL(obj, ULong)); npy_hash_t x = PyObject_Hash(l); Py_DECREF(l); return x; @@ -3031,7 +2857,7 @@ ulong_arrtype_hash(PyObject *obj) static npy_hash_t int_arrtype_hash(PyObject *obj) { - npy_hash_t x = (npy_hash_t)(((PyIntScalarObject *)obj)->obval); + npy_hash_t x = (npy_hash_t)(PyArrayScalar_VAL(obj, Int)); if (x == -1) { x = -2; } @@ -3041,7 +2867,7 @@ int_arrtype_hash(PyObject *obj) static npy_hash_t long_arrtype_hash(PyObject *obj) { - PyObject * l = PyLong_FromLong(((PyLongScalarObject*)obj)->obval); + PyObject * l = PyLong_FromLong(PyArrayScalar_VAL(obj, Long)); npy_hash_t x = PyObject_Hash(l); Py_DECREF(l); return x; @@ -3056,7 +2882,7 @@ static NPY_INLINE npy_hash_t @char@longlong_arrtype_hash(PyObject *obj) { PyObject * l = PyLong_From@Word@LongLong( - ((Py@Char@LongLongScalarObject*)obj)->obval); + PyArrayScalar_VAL(obj, @Char@LongLong)); npy_hash_t x = PyObject_Hash(l); Py_DECREF(l); return x; @@ -3072,7 +2898,7 @@ static NPY_INLINE npy_hash_t static npy_hash_t @lname@_arrtype_hash(PyObject *obj) { - npy_hash_t x = (npy_hash_t)(((Py@name@ScalarObject *)obj)->obval); + npy_hash_t x = (npy_hash_t)(PyArrayScalar_VAL(obj, @name@)); if (x == -1) { x = -2; } @@ -3083,7 +2909,7 @@ static npy_hash_t @lname@_arrtype_hash(PyObject *obj) { npy_hash_t y; - npy_longlong x = (((Py@name@ScalarObject *)obj)->obval); + npy_longlong x = (PyArrayScalar_VAL(obj, @name@)); if ((x <= LONG_MAX)) { y = (npy_hash_t) x; @@ -3116,7 +2942,7 @@ static npy_hash_t static npy_hash_t @lname@_arrtype_hash(PyObject *obj) { - return _Py_HashDouble((double) ((Py@name@ScalarObject *)obj)->obval); + return _Py_HashDouble((double) PyArrayScalar_VAL(obj, @name@)); } /* borrowed from complex_hash */ @@ -3125,13 +2951,13 @@ c@lname@_arrtype_hash(PyObject *obj) { npy_hash_t hashreal, hashimag, combined; hashreal = _Py_HashDouble((double) - (((PyC@name@ScalarObject *)obj)->obval).real); + PyArrayScalar_VAL(obj, C@name@).real); if (hashreal == -1) { return -1; } hashimag = _Py_HashDouble((double) - (((PyC@name@ScalarObject *)obj)->obval).imag); + PyArrayScalar_VAL(obj, C@name@).imag); if (hashimag == -1) { return -1; } @@ -3146,13 +2972,13 @@ c@lname@_arrtype_hash(PyObject *obj) static npy_hash_t half_arrtype_hash(PyObject *obj) { - return _Py_HashDouble(npy_half_to_double(((PyHalfScalarObject *)obj)->obval)); + return _Py_HashDouble(npy_half_to_double(PyArrayScalar_VAL(obj, Half))); } static npy_hash_t object_arrtype_hash(PyObject *obj) { - return PyObject_Hash(((PyObjectScalarObject *)obj)->obval); + return PyObject_Hash(PyArrayScalar_VAL(obj, Object)); } /* we used to just hash the pointer */ @@ -3267,22 +3093,18 @@ object_arrtype_inplace_repeat(PyObjectScalarObject *self, Py_ssize_t count) } static PySequenceMethods object_arrtype_as_sequence = { - (lenfunc)object_arrtype_length, /*sq_length*/ - (binaryfunc)object_arrtype_concat, /*sq_concat*/ - (ssizeargfunc)object_arrtype_repeat, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc)object_arrtype_contains, /* sq_contains */ - (binaryfunc)object_arrtype_inplace_concat, /* sq_inplace_concat */ - (ssizeargfunc)object_arrtype_inplace_repeat, /* sq_inplace_repeat */ + .sq_length = (lenfunc)object_arrtype_length, + .sq_concat = (binaryfunc)object_arrtype_concat, + .sq_repeat = (ssizeargfunc)object_arrtype_repeat, + .sq_contains = (objobjproc)object_arrtype_contains, + .sq_inplace_concat = (binaryfunc)object_arrtype_inplace_concat, + .sq_inplace_repeat = (ssizeargfunc)object_arrtype_inplace_repeat, }; static PyMappingMethods object_arrtype_as_mapping = { - (lenfunc)object_arrtype_length, - (binaryfunc)object_arrtype_subscript, - (objobjargproc)object_arrtype_ass_subscript, + .mp_length = (lenfunc)object_arrtype_length, + .mp_subscript = (binaryfunc)object_arrtype_subscript, + .mp_ass_subscript = (objobjargproc)object_arrtype_ass_subscript, }; static int @@ -3312,8 +3134,8 @@ object_arrtype_releasebuffer(PyObjectScalarObject *self, Py_buffer *view) } static PyBufferProcs object_arrtype_as_buffer = { - (getbufferproc)object_arrtype_getbuffer, - (releasebufferproc)object_arrtype_releasebuffer, + .bf_getbuffer = (getbufferproc)object_arrtype_getbuffer, + .bf_releasebuffer = (releasebufferproc)object_arrtype_releasebuffer, }; static PyObject * @@ -3324,52 +3146,15 @@ object_arrtype_call(PyObjectScalarObject *obj, PyObject *args, PyObject *kwds) NPY_NO_EXPORT PyTypeObject PyObjectArrType_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.object_", /* tp_name*/ - sizeof(PyObjectScalarObject), /* tp_basicsize*/ - 0, /* tp_itemsize */ - (destructor)object_arrtype_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &object_arrtype_as_sequence, /* tp_as_sequence */ - &object_arrtype_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc)object_arrtype_call, /* tp_call */ - 0, /* tp_str */ - (getattrofunc)object_arrtype_getattro, /* tp_getattro */ - (setattrofunc)object_arrtype_setattro, /* tp_setattro */ - &object_arrtype_as_buffer, /* tp_as_buffer */ - 0, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.object_", + .tp_basicsize = sizeof(PyObjectScalarObject), + .tp_dealloc = (destructor)object_arrtype_dealloc, + .tp_as_sequence = &object_arrtype_as_sequence, + .tp_as_mapping = &object_arrtype_as_mapping, + .tp_call = (ternaryfunc)object_arrtype_call, + .tp_getattro = (getattrofunc)object_arrtype_getattro, + .tp_setattro = (setattrofunc)object_arrtype_setattro, + .tp_as_buffer = &object_arrtype_as_buffer, }; static PyObject * @@ -3408,52 +3193,8 @@ gen_arrtype_subscript(PyObject *self, PyObject *key) */ NPY_NO_EXPORT PyTypeObject Py@NAME@ArrType_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy." NAME_@name@ "@ex@", /* tp_name*/ - sizeof(Py@NAME@ScalarObject), /* tp_basicsize*/ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - 0, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy." NAME_@name@ "@ex@", + .tp_basicsize = sizeof(Py@NAME@ScalarObject), }; /**end repeat**/ @@ -3488,62 +3229,17 @@ NPY_NO_EXPORT PyTypeObject Py@NAME@ArrType_Type = { #endif NPY_NO_EXPORT PyTypeObject Py@NAME@ArrType_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.@name@" _THIS_SIZE, /* tp_name*/ - sizeof(Py@NAME@ScalarObject), /* tp_basicsize*/ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - 0, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.@name@" _THIS_SIZE, + .tp_basicsize = sizeof(Py@NAME@ScalarObject), }; + #undef _THIS_SIZE /**end repeat**/ static PyMappingMethods gentype_as_mapping = { - NULL, - (binaryfunc)gen_arrtype_subscript, - NULL + .mp_subscript = (binaryfunc)gen_arrtype_subscript, }; @@ -3570,52 +3266,9 @@ static PyMappingMethods gentype_as_mapping = { NPY_NO_EXPORT PyTypeObject Py@NAME@ArrType_Type = { PyVarObject_HEAD_INIT(0, 0) - "numpy.@name@" _THIS_SIZE, /* tp_name*/ - sizeof(Py@NAME@ScalarObject), /* tp_basicsize*/ - 0, /* tp_itemsize*/ - 0, /* tp_dealloc*/ - 0, /* tp_print*/ - 0, /* tp_getattr*/ - 0, /* tp_setattr*/ - 0, /* tp_reserved */ - 0, /* tp_repr*/ - 0, /* tp_as_number*/ - 0, /* tp_as_sequence*/ - 0, /* tp_as_mapping*/ - 0, /* tp_hash */ - 0, /* tp_call*/ - 0, /* tp_str*/ - 0, /* tp_getattro*/ - 0, /* tp_setattro*/ - 0, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /* tp_flags*/ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.@name@" _THIS_SIZE, + .tp_basicsize = sizeof(Py@NAME@ScalarObject), + .tp_flags = Py_TPFLAGS_DEFAULT, }; #undef _THIS_SIZE @@ -4045,6 +3698,9 @@ initialize_numeric_types(void) /**end repeat**/ + PyUnicodeArrType_Type.tp_dealloc = unicode_arrtype_dealloc; + PyUnicodeArrType_Type.tp_as_buffer = &gentype_as_buffer; + /**begin repeat * #name = bool, byte, short, ubyte, ushort, uint, ulong, ulonglong, * half, float, longdouble, cfloat, clongdouble, void, object, @@ -4059,8 +3715,8 @@ initialize_numeric_types(void) /**end repeat**/ /**begin repeat - * #name = cfloat, clongdouble# - * #NAME = CFloat, CLongDouble# + * #name = cfloat, clongdouble, floating, integer, complexfloating# + * #NAME = CFloat, CLongDouble, Floating, Integer, ComplexFloating# */ Py@NAME@ArrType_Type.tp_methods = @name@type_methods; diff --git a/numpy/core/src/multiarray/sequence.c b/numpy/core/src/multiarray/sequence.c index 4769bdad9..1efdd204f 100644 --- a/numpy/core/src/multiarray/sequence.c +++ b/numpy/core/src/multiarray/sequence.c @@ -38,8 +38,13 @@ array_contains(PyArrayObject *self, PyObject *el) if (res == NULL) { return -1; } + any = PyArray_Any((PyArrayObject *)res, NPY_MAXDIMS, NULL); Py_DECREF(res); + if (any == NULL) { + return -1; + } + ret = PyObject_IsTrue(any); Py_DECREF(any); return ret; diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 127ac5134..30507112d 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -317,7 +317,7 @@ _putzero(char *optr, PyObject *zero, PyArray_Descr *dtype) int offset; Py_ssize_t pos = 0; while (PyDict_Next(dtype->fields, &pos, &key, &value)) { - if NPY_TITLE_KEY(key, value) { + if (NPY_TITLE_KEY(key, value)) { continue; } if (!PyArg_ParseTuple(value, "Oi|O", &new, &offset, &title)) { diff --git a/numpy/core/src/multiarray/typeinfo.c b/numpy/core/src/multiarray/typeinfo.c index 30053887b..b0563b3c0 100644 --- a/numpy/core/src/multiarray/typeinfo.c +++ b/numpy/core/src/multiarray/typeinfo.c @@ -5,8 +5,10 @@ */ #include "typeinfo.h" -/* In python 2, this is not exported from Python.h */ +#if (defined(PYPY_VERSION_NUM) && (PYPY_VERSION_NUM <= 0x07030000)) +/* PyPy issue 3160 */ #include <structseq.h> +#endif #define NPY_NO_DEPRECATED_API NPY_API_VERSION #define _MULTIARRAYMODULE diff --git a/numpy/core/src/npymath/halffloat.c b/numpy/core/src/npymath/halffloat.c index 84af86009..cbaa11e43 100644 --- a/numpy/core/src/npymath/halffloat.c +++ b/numpy/core/src/npymath/halffloat.c @@ -115,10 +115,7 @@ npy_half npy_half_nextafter(npy_half x, npy_half y) { npy_half ret; - if (!npy_half_isfinite(x) || npy_half_isnan(y)) { -#if NPY_HALF_GENERATE_INVALID - npy_set_floatstatus_invalid(); -#endif + if (npy_half_isnan(x) || npy_half_isnan(y)) { ret = NPY_HALF_NAN; } else if (npy_half_eq_nonan(x, y)) { ret = x; @@ -138,7 +135,7 @@ npy_half npy_half_nextafter(npy_half x, npy_half y) } } #if NPY_HALF_GENERATE_OVERFLOW - if (npy_half_isinf(ret)) { + if (npy_half_isinf(ret) && npy_half_isfinite(x)) { npy_set_floatstatus_overflow(); } #endif diff --git a/numpy/core/src/umath/cpuid.c b/numpy/core/src/umath/cpuid.c deleted file mode 100644 index 72c6493e8..000000000 --- a/numpy/core/src/umath/cpuid.c +++ /dev/null @@ -1,97 +0,0 @@ -#define _UMATHMODULE -#define _MULTIARRAYMODULE -#define NPY_NO_DEPRECATED_API NPY_API_VERSION - -#include <Python.h> - -#include "npy_config.h" - -#include "cpuid.h" - -#define XCR_XFEATURE_ENABLED_MASK 0x0 -#define XSTATE_SSE 0x2 -#define XSTATE_YMM 0x4 -#define XSTATE_ZMM 0x70 - -/* - * verify the OS supports avx instructions - * it can be disabled in some OS, e.g. with the nosavex boot option of linux - */ -static NPY_INLINE -int os_avx_support(void) -{ -#if HAVE_XGETBV - /* - * use bytes for xgetbv to avoid issues with compiler not knowing the - * instruction - */ - unsigned int eax, edx; - unsigned int ecx = XCR_XFEATURE_ENABLED_MASK; - __asm__("xgetbv" : "=a" (eax), "=d" (edx) : "c" (ecx)); - return (eax & (XSTATE_SSE | XSTATE_YMM)) == (XSTATE_SSE | XSTATE_YMM); -#else - return 0; -#endif -} - -static NPY_INLINE -int os_avx512_support(void) -{ -#if HAVE_XGETBV - unsigned int eax, edx; - unsigned int ecx = XCR_XFEATURE_ENABLED_MASK; - unsigned int xcr0 = XSTATE_ZMM | XSTATE_YMM | XSTATE_SSE; - __asm__("xgetbv" : "=a" (eax), "=d" (edx) : "c" (ecx)); - return (eax & xcr0) == xcr0; -#else - return 0; -#endif -} - -static NPY_INLINE -int cpu_supports_fma(void) -{ -#ifdef __x86_64__ - unsigned int feature = 0x01; - unsigned int a, b, c, d; - __asm__ volatile ( - "cpuid" "\n\t" - : "=a" (a), "=b" (b), "=c" (c), "=d" (d) - : "a" (feature)); - /* - * FMA is the 12th bit of ECX - */ - return (c >> 12) & 1; -#else - return 0; -#endif -} - -/* - * Primitive cpu feature detect function - * Currently only supports checking for avx on gcc compatible compilers. - */ -NPY_NO_EXPORT int -npy_cpu_supports(const char * feature) -{ -#ifdef HAVE___BUILTIN_CPU_SUPPORTS - if (strcmp(feature, "avx512f") == 0) { -#ifdef HAVE___BUILTIN_CPU_SUPPORTS_AVX512F - return __builtin_cpu_supports("avx512f") && os_avx512_support(); -#else - return 0; -#endif - } - else if (strcmp(feature, "fma") == 0) { - return cpu_supports_fma() && __builtin_cpu_supports("avx2") && os_avx_support(); - } - else if (strcmp(feature, "avx2") == 0) { - return __builtin_cpu_supports("avx2") && os_avx_support(); - } - else if (strcmp(feature, "avx") == 0) { - return __builtin_cpu_supports("avx") && os_avx_support(); - } -#endif - - return 0; -} diff --git a/numpy/core/src/umath/cpuid.h b/numpy/core/src/umath/cpuid.h deleted file mode 100644 index 33702ed41..000000000 --- a/numpy/core/src/umath/cpuid.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef _NPY_PRIVATE__CPUID_H_ -#define _NPY_PRIVATE__CPUID_H_ - -#include <numpy/ndarraytypes.h> /* for NPY_NO_EXPORT */ - -NPY_NO_EXPORT int -npy_cpu_supports(const char * feature); - -#endif diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index 3b180ce59..eea82309c 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -1558,6 +1558,15 @@ FLOAT_@func@(char **args, npy_intp const *dimensions, npy_intp const *steps, voi /**end repeat**/ +NPY_NO_EXPORT NPY_GCC_OPT_3 void +DOUBLE_exp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) +{ + UNARY_LOOP { + const npy_double in1 = *(npy_double *)ip1; + *(npy_double *)op1 = npy_exp(in1); + } +} + /**begin repeat * #isa = avx512f, fma# * #ISA = AVX512F, FMA# @@ -1688,6 +1697,17 @@ FLOAT_@func@_@isa@(char **args, npy_intp const *dimensions, npy_intp const *step /**end repeat1**/ /**end repeat**/ +NPY_NO_EXPORT NPY_GCC_OPT_3 void +DOUBLE_exp_avx512f(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) +{ + if (!run_unary_avx512f_exp_DOUBLE(args, dimensions, steps)) { + UNARY_LOOP { + const npy_double in1 = *(npy_double *)ip1; + *(npy_double *)op1 = npy_exp(in1); + } + } +} + /**begin repeat * Float types @@ -1898,6 +1918,34 @@ NPY_NO_EXPORT void * #OP = >=, <=# **/ NPY_NO_EXPORT void +@TYPE@_@kind@_avx512f(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ + /* */ + if (IS_BINARY_REDUCE) { + if (!run_unary_reduce_simd_@kind@_@TYPE@(args, dimensions, steps)) { + BINARY_REDUCE_LOOP(@type@) { + const @type@ in2 = *(@type@ *)ip2; + /* Order of operations important for MSVC 2015 */ + io1 = (io1 @OP@ in2 || npy_isnan(io1)) ? io1 : in2; + } + *((@type@ *)iop1) = io1; + } + } + else { + if (!run_binary_avx512f_@kind@_@TYPE@(args, dimensions, steps)) { + BINARY_LOOP { + @type@ in1 = *(@type@ *)ip1; + const @type@ in2 = *(@type@ *)ip2; + /* Order of operations important for MSVC 2015 */ + in1 = (in1 @OP@ in2 || npy_isnan(in1)) ? in1 : in2; + *((@type@ *)op1) = in1; + } + } + } + npy_clear_floatstatus_barrier((char*)dimensions); +} + +NPY_NO_EXPORT void @TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { /* */ @@ -2481,6 +2529,7 @@ HALF_ldexp_long(char **args, npy_intp const *dimensions, npy_intp const *steps, * #ftype = npy_float, npy_double, npy_longdouble# * #c = f, , l# * #C = F, , L# + * #SIMD = 1, 1, 0# */ /* similar to pairwise sum of real floats */ @@ -2556,6 +2605,7 @@ pairwise_sum_@TYPE@(@ftype@ *rr, @ftype@ * ri, char * a, npy_intp n, } } + /**begin repeat1 * arithmetic * #kind = add, subtract# @@ -2634,6 +2684,32 @@ NPY_NO_EXPORT void } } +#if @SIMD@ +NPY_NO_EXPORT void +@TYPE@_add_avx512f(char **args, const npy_intp *dimensions, const npy_intp *steps, void *func) +{ + if (IS_BINARY_REDUCE) { + @TYPE@_add(args, dimensions, steps, func); + } + else if (!run_binary_avx512f_add_@TYPE@(args, dimensions, steps)) { + @TYPE@_add(args, dimensions, steps, func); + } +} + +/**begin repeat1 + * arithmetic + * #kind = subtract, multiply# + */ +NPY_NO_EXPORT void +@TYPE@_@kind@_avx512f(char **args, const npy_intp *dimensions, const npy_intp *steps, void *func) +{ + if (!run_binary_avx512f_@kind@_@TYPE@(args, dimensions, steps)) { + @TYPE@_@kind@(args, dimensions, steps, func); + } +} +/**end repeat1**/ +#endif + NPY_NO_EXPORT void @TYPE@_floor_divide(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2791,6 +2867,21 @@ NPY_NO_EXPORT void } } +#if @SIMD@ +/**begin repeat1 + * arithmetic + * #kind = conjugate, square, absolute# + */ +NPY_NO_EXPORT void +@TYPE@_@kind@_avx512f(char **args, const npy_intp *dimensions, const npy_intp *steps, void *func) +{ + if (!run_unary_avx512f_@kind@_@TYPE@(args, dimensions, steps)) { + @TYPE@_@kind@(args, dimensions, steps, func); + } +} +/**end repeat1**/ +#endif + NPY_NO_EXPORT void @TYPE@__arg(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { diff --git a/numpy/core/src/umath/loops.h.src b/numpy/core/src/umath/loops.h.src index 8ddf201d7..50a7ccfee 100644 --- a/numpy/core/src/umath/loops.h.src +++ b/numpy/core/src/umath/loops.h.src @@ -175,6 +175,14 @@ NPY_NO_EXPORT void @TYPE@_sqrt(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); /**begin repeat1 + * #func = maximum, minimum# + */ +NPY_NO_EXPORT void +@TYPE@_@func@_avx512f(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); + +/**end repeat1**/ + +/**begin repeat1 * #isa = avx512f, fma# */ @@ -188,6 +196,12 @@ NPY_NO_EXPORT void /**end repeat1**/ /**end repeat**/ +NPY_NO_EXPORT void +DOUBLE_exp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); + +NPY_NO_EXPORT void +DOUBLE_exp_avx512f(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); + /**begin repeat * #func = sin, cos, exp, log# */ @@ -348,20 +362,27 @@ NPY_NO_EXPORT void * #TYPE = FLOAT, DOUBLE, LONGDOUBLE# * #c = f, , l# * #C = F, , L# + * #IFSIMD = 1, 1, 0# */ /**begin repeat1 + * #isa = , _avx512f# + */ + +/**begin repeat2 * arithmetic * #kind = add, subtract# * #OP = +, -# */ + NPY_NO_EXPORT void -C@TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); +C@TYPE@_@kind@@isa@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); -/**end repeat1**/ +/**end repeat2**/ NPY_NO_EXPORT void -C@TYPE@_multiply(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); +C@TYPE@_multiply@isa@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); +/**end repeat1**/ NPY_NO_EXPORT void C@TYPE@_divide(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); @@ -401,19 +422,24 @@ C@TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, v /**end repeat1**/ NPY_NO_EXPORT void -C@TYPE@_square(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)); - -NPY_NO_EXPORT void C@TYPE@_reciprocal(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)); NPY_NO_EXPORT void C@TYPE@__ones_like(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)); +/**begin repeat1 + * #isa = , _avx512f# + */ + +NPY_NO_EXPORT void +C@TYPE@_conjugate@isa@(char **args, const npy_intp *dimensions, const npy_intp *steps, void *NPY_UNUSED(func)); + NPY_NO_EXPORT void -C@TYPE@_conjugate(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); +C@TYPE@_absolute@isa@(char **args, const npy_intp *dimensions, const npy_intp *steps, void *NPY_UNUSED(func)); NPY_NO_EXPORT void -C@TYPE@_absolute(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); +C@TYPE@_square@isa@(char **args, const npy_intp *dimensions, const npy_intp *steps, void *NPY_UNUSED(data)); +/**end repeat1**/ NPY_NO_EXPORT void C@TYPE@__arg(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); @@ -436,7 +462,6 @@ C@TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, v NPY_NO_EXPORT void C@TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); /**end repeat1**/ - #define C@TYPE@_true_divide C@TYPE@_divide /**end repeat**/ diff --git a/numpy/core/src/umath/npy_simd_data.h b/numpy/core/src/umath/npy_simd_data.h new file mode 100644 index 000000000..36c8b6c03 --- /dev/null +++ b/numpy/core/src/umath/npy_simd_data.h @@ -0,0 +1,137 @@ +#ifndef __NPY_SIMD_DATA_H_ +#define __NPY_SIMD_DATA_H_ +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +/* + * Constants used in vector implementation of float64 exp(x) + */ +#define NPY_RINT_CVT_MAGIC 0x1.8p52 +#define NPY_INV_LN2_MUL_32 0x1.71547652b82fep+5 +#define NPY_TANG_NEG_L1 -0x1.62e42fefp-6 +#define NPY_TANG_NEG_L2 -0x1.473de6af278edp-39 +#define NPY_TANG_A1 0x1p-1 +#define NPY_TANG_A2 0x1.5555555548f7cp-3 +#define NPY_TANG_A3 0x1.5555555545d4ep-5 +#define NPY_TANG_A4 0x1.11115b7aa905ep-7 +#define NPY_TANG_A5 0x1.6c1728d739765p-10 + +/* Lookup table for 2^(j/32) */ +static npy_uint64 EXP_Table_top[32] = { + 0x3FF0000000000000, + 0x3FF059B0D3158540, + 0x3FF0B5586CF98900, + 0x3FF11301D0125B40, + 0x3FF172B83C7D5140, + 0x3FF1D4873168B980, + 0x3FF2387A6E756200, + 0x3FF29E9DF51FDEC0, + 0x3FF306FE0A31B700, + 0x3FF371A7373AA9C0, + 0x3FF3DEA64C123400, + 0x3FF44E0860618900, + 0x3FF4BFDAD5362A00, + 0x3FF5342B569D4F80, + 0x3FF5AB07DD485400, + 0x3FF6247EB03A5580, + 0x3FF6A09E667F3BC0, + 0x3FF71F75E8EC5F40, + 0x3FF7A11473EB0180, + 0x3FF82589994CCE00, + 0x3FF8ACE5422AA0C0, + 0x3FF93737B0CDC5C0, + 0x3FF9C49182A3F080, + 0x3FFA5503B23E2540, + 0x3FFAE89F995AD380, + 0x3FFB7F76F2FB5E40, + 0x3FFC199BDD855280, + 0x3FFCB720DCEF9040, + 0x3FFD5818DCFBA480, + 0x3FFDFC97337B9B40, + 0x3FFEA4AFA2A490C0, + 0x3FFF50765B6E4540, +}; + +static npy_uint64 EXP_Table_tail[32] = { + 0x0000000000000000, + 0x3D0A1D73E2A475B4, + 0x3CEEC5317256E308, + 0x3CF0A4EBBF1AED93, + 0x3D0D6E6FBE462876, + 0x3D053C02DC0144C8, + 0x3D0C3360FD6D8E0B, + 0x3D009612E8AFAD12, + 0x3CF52DE8D5A46306, + 0x3CE54E28AA05E8A9, + 0x3D011ADA0911F09F, + 0x3D068189B7A04EF8, + 0x3D038EA1CBD7F621, + 0x3CBDF0A83C49D86A, + 0x3D04AC64980A8C8F, + 0x3CD2C7C3E81BF4B7, + 0x3CE921165F626CDD, + 0x3D09EE91B8797785, + 0x3CDB5F54408FDB37, + 0x3CF28ACF88AFAB35, + 0x3CFB5BA7C55A192D, + 0x3D027A280E1F92A0, + 0x3CF01C7C46B071F3, + 0x3CFC8B424491CAF8, + 0x3D06AF439A68BB99, + 0x3CDBAA9EC206AD4F, + 0x3CFC2220CB12A092, + 0x3D048A81E5E8F4A5, + 0x3CDC976816BAD9B8, + 0x3CFEB968CAC39ED3, + 0x3CF9858F73A18F5E, + 0x3C99D3E12DD8A18B, +}; +#endif + +/* + * Constants used in vector implementation of exp(x) + */ +#define NPY_RINT_CVT_MAGICf 0x1.800000p+23f +#define NPY_CODY_WAITE_LOGE_2_HIGHf -6.93145752e-1f +#define NPY_CODY_WAITE_LOGE_2_LOWf -1.42860677e-6f +#define NPY_COEFF_P0_EXPf 9.999999999980870924916e-01f +#define NPY_COEFF_P1_EXPf 7.257664613233124478488e-01f +#define NPY_COEFF_P2_EXPf 2.473615434895520810817e-01f +#define NPY_COEFF_P3_EXPf 5.114512081637298353406e-02f +#define NPY_COEFF_P4_EXPf 6.757896990527504603057e-03f +#define NPY_COEFF_P5_EXPf 5.082762527590693718096e-04f +#define NPY_COEFF_Q0_EXPf 1.000000000000000000000e+00f +#define NPY_COEFF_Q1_EXPf -2.742335390411667452936e-01f +#define NPY_COEFF_Q2_EXPf 2.159509375685829852307e-02f + +/* + * Constants used in vector implementation of log(x) + */ +#define NPY_COEFF_P0_LOGf 0.000000000000000000000e+00f +#define NPY_COEFF_P1_LOGf 9.999999999999998702752e-01f +#define NPY_COEFF_P2_LOGf 2.112677543073053063722e+00f +#define NPY_COEFF_P3_LOGf 1.480000633576506585156e+00f +#define NPY_COEFF_P4_LOGf 3.808837741388407920751e-01f +#define NPY_COEFF_P5_LOGf 2.589979117907922693523e-02f +#define NPY_COEFF_Q0_LOGf 1.000000000000000000000e+00f +#define NPY_COEFF_Q1_LOGf 2.612677543073109236779e+00f +#define NPY_COEFF_Q2_LOGf 2.453006071784736363091e+00f +#define NPY_COEFF_Q3_LOGf 9.864942958519418960339e-01f +#define NPY_COEFF_Q4_LOGf 1.546476374983906719538e-01f +#define NPY_COEFF_Q5_LOGf 5.875095403124574342950e-03f +/* + * Constants used in vector implementation of sinf/cosf(x) + */ +#define NPY_TWO_O_PIf 0x1.45f306p-1f +#define NPY_CODY_WAITE_PI_O_2_HIGHf -0x1.921fb0p+00f +#define NPY_CODY_WAITE_PI_O_2_MEDf -0x1.5110b4p-22f +#define NPY_CODY_WAITE_PI_O_2_LOWf -0x1.846988p-48f +#define NPY_COEFF_INVF0_COSINEf 0x1.000000p+00f +#define NPY_COEFF_INVF2_COSINEf -0x1.000000p-01f +#define NPY_COEFF_INVF4_COSINEf 0x1.55553cp-05f +#define NPY_COEFF_INVF6_COSINEf -0x1.6c06dcp-10f +#define NPY_COEFF_INVF8_COSINEf 0x1.98e616p-16f +#define NPY_COEFF_INVF3_SINEf -0x1.555556p-03f +#define NPY_COEFF_INVF5_SINEf 0x1.11119ap-07f +#define NPY_COEFF_INVF7_SINEf -0x1.a06bbap-13f +#define NPY_COEFF_INVF9_SINEf 0x1.7d3bbcp-19f + +#endif diff --git a/numpy/core/src/umath/reduction.c b/numpy/core/src/umath/reduction.c index 79c302755..79adb0051 100644 --- a/numpy/core/src/umath/reduction.c +++ b/numpy/core/src/umath/reduction.c @@ -18,6 +18,7 @@ #include "npy_config.h" #include "npy_pycompat.h" +#include "ctors.h" #include "numpy/ufuncobject.h" #include "lowlevel_strided_loops.h" @@ -88,14 +89,13 @@ conform_reduce_result(PyArrayObject *in, const npy_bool *axis_flags, PyArrayObject *out, int keepdims, const char *funcname, int need_copy) { - int ndim = PyArray_NDIM(in); - npy_intp *shape_in = PyArray_DIMS(in); - npy_intp strides[NPY_MAXDIMS], shape[NPY_MAXDIMS]; - npy_intp *strides_out = PyArray_STRIDES(out); - npy_intp *shape_out = PyArray_DIMS(out); - int idim, idim_out, ndim_out = PyArray_NDIM(out); - PyArray_Descr *dtype; - PyArrayObject_fields *ret; + /* unpack shape information */ + int const ndim = PyArray_NDIM(in); + npy_intp const *shape_in = PyArray_DIMS(in); + + int const ndim_out = PyArray_NDIM(out); + npy_intp const *strides_out = PyArray_STRIDES(out); + npy_intp const *shape_out = PyArray_DIMS(out); /* * If the 'keepdims' parameter is true, do a simpler validation and @@ -110,7 +110,7 @@ conform_reduce_result(PyArrayObject *in, const npy_bool *axis_flags, return NULL; } - for (idim = 0; idim < ndim; ++idim) { + for (int idim = 0; idim < ndim; ++idim) { if (axis_flags[idim]) { if (shape_out[idim] != 1) { PyErr_Format(PyExc_ValueError, @@ -137,8 +137,9 @@ conform_reduce_result(PyArrayObject *in, const npy_bool *axis_flags, } /* Construct the strides and shape */ - idim_out = 0; - for (idim = 0; idim < ndim; ++idim) { + npy_intp strides[NPY_MAXDIMS], shape[NPY_MAXDIMS]; + int idim_out = 0; + for (int idim = 0; idim < ndim; ++idim) { if (axis_flags[idim]) { strides[idim] = 0; shape[idim] = 1; @@ -171,31 +172,20 @@ conform_reduce_result(PyArrayObject *in, const npy_bool *axis_flags, } /* Allocate the view */ - dtype = PyArray_DESCR(out); + PyArray_Descr *dtype = PyArray_DESCR(out); Py_INCREF(dtype); - /* TODO: use PyArray_NewFromDescrAndBase here once multiarray and umath - * are merged - */ - ret = (PyArrayObject_fields *)PyArray_NewFromDescr( + PyArrayObject_fields *ret = (PyArrayObject_fields *)PyArray_NewFromDescrAndBase( &PyArray_Type, dtype, ndim, shape, strides, PyArray_DATA(out), - PyArray_FLAGS(out), NULL); + PyArray_FLAGS(out), NULL, (PyObject *)out); if (ret == NULL) { return NULL; } - Py_INCREF(out); - if (PyArray_SetBaseObject((PyArrayObject *)ret, (PyObject *)out) < 0) { - Py_DECREF(ret); - return NULL; - } - if (need_copy) { - PyArrayObject *ret_copy; - - ret_copy = (PyArrayObject *)PyArray_NewLikeArray( - (PyArrayObject *)ret, NPY_ANYORDER, NULL, 0); + PyArrayObject *ret_copy = (PyArrayObject *)PyArray_NewLikeArray( + (PyArrayObject *)ret, NPY_ANYORDER, NULL, 0); if (ret_copy == NULL) { Py_DECREF(ret); return NULL; @@ -208,7 +198,6 @@ conform_reduce_result(PyArrayObject *in, const npy_bool *axis_flags, } if (PyArray_SetWritebackIfCopyBase(ret_copy, (PyArrayObject *)ret) < 0) { - Py_DECREF(ret); Py_DECREF(ret_copy); return NULL; } diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index b3826eef4..bb2915e09 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -1513,44 +1513,28 @@ static PyObject* * cfloat, cdouble, clongdouble# **/ static PyNumberMethods @name@_as_number = { - (binaryfunc)@name@_add, /*nb_add*/ - (binaryfunc)@name@_subtract, /*nb_subtract*/ - (binaryfunc)@name@_multiply, /*nb_multiply*/ - (binaryfunc)@name@_remainder, /*nb_remainder*/ - (binaryfunc)@name@_divmod, /*nb_divmod*/ - (ternaryfunc)@name@_power, /*nb_power*/ - (unaryfunc)@name@_negative, - (unaryfunc)@name@_positive, /*nb_pos*/ - (unaryfunc)@name@_absolute, /*nb_abs*/ - (inquiry)@name@_bool, /*nb_bool*/ - (unaryfunc)@name@_invert, /*nb_invert*/ - (binaryfunc)@name@_lshift, /*nb_lshift*/ - (binaryfunc)@name@_rshift, /*nb_rshift*/ - (binaryfunc)@name@_and, /*nb_and*/ - (binaryfunc)@name@_xor, /*nb_xor*/ - (binaryfunc)@name@_or, /*nb_or*/ - (unaryfunc)@name@_int, /*nb_int*/ - (unaryfunc)0, /*nb_reserved*/ - (unaryfunc)@name@_float, /*nb_float*/ - 0, /*inplace_add*/ - 0, /*inplace_subtract*/ - 0, /*inplace_multiply*/ - 0, /*inplace_remainder*/ - 0, /*inplace_power*/ - 0, /*inplace_lshift*/ - 0, /*inplace_rshift*/ - 0, /*inplace_and*/ - 0, /*inplace_xor*/ - 0, /*inplace_or*/ - (binaryfunc)@name@_floor_divide, /*nb_floor_divide*/ - (binaryfunc)@name@_true_divide, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - (unaryfunc)NULL, /*nb_index*/ -#if PY_VERSION_HEX >= 0x03050000 - 0, /*nb_matrix_multiply*/ - 0, /*nb_inplace_matrix_multiply*/ -#endif + .nb_add = (binaryfunc)@name@_add, + .nb_subtract = (binaryfunc)@name@_subtract, + .nb_multiply = (binaryfunc)@name@_multiply, + .nb_remainder = (binaryfunc)@name@_remainder, + .nb_divmod = (binaryfunc)@name@_divmod, + .nb_power = (ternaryfunc)@name@_power, + .nb_negative = (unaryfunc)@name@_negative, + .nb_positive = (unaryfunc)@name@_positive, + .nb_absolute = (unaryfunc)@name@_absolute, + .nb_bool = (inquiry)@name@_bool, + .nb_invert = (unaryfunc)@name@_invert, + .nb_lshift = (binaryfunc)@name@_lshift, + .nb_rshift = (binaryfunc)@name@_rshift, + .nb_and = (binaryfunc)@name@_and, + .nb_xor = (binaryfunc)@name@_xor, + .nb_or = (binaryfunc)@name@_or, + .nb_int = (unaryfunc)@name@_int, + .nb_float = (unaryfunc)@name@_float, + .nb_floor_divide = (binaryfunc)@name@_floor_divide, + .nb_true_divide = (binaryfunc)@name@_true_divide, + /* TODO: This struct/initialization should not be split between files */ + .nb_index = (unaryfunc)NULL, /* set in add_scalarmath below */ }; /**end repeat**/ diff --git a/numpy/core/src/umath/simd.inc.src b/numpy/core/src/umath/simd.inc.src index 5473b58f1..106c7e7c9 100644 --- a/numpy/core/src/umath/simd.inc.src +++ b/numpy/core/src/umath/simd.inc.src @@ -18,6 +18,7 @@ #include "lowlevel_strided_loops.h" #include "numpy/npy_common.h" #include "numpy/npy_math.h" +#include "npy_simd_data.h" #ifdef NPY_HAVE_SSE2_INTRINSICS #include <emmintrin.h> #if !defined(_MSC_VER) || _MSC_VER >= 1600 @@ -34,6 +35,21 @@ #define VECTOR_SIZE_BYTES 16 +/* + * MAX_STEP_SIZE is used to determine if we need to use SIMD version of the ufunc. + * Very large step size can be as slow as processing it using scalar. The + * value of 2097152 ( = 2MB) was chosen using 2 considerations: + * 1) Typical linux kernel page size is 4Kb, but sometimes it could also be 2MB + * which is == 2097152 Bytes. For a step size as large as this, surely all + * the loads/stores of gather/scatter instructions falls on 16 different pages + * which one would think would slow down gather/scatter instructions. + * 2) It additionally satisfies MAX_STEP_SIZE*16/esize < NPY_MAX_INT32 which + * allows us to use i32 version of gather/scatter (as opposed to the i64 version) + * without problems (step larger than NPY_MAX_INT32*esize/16 would require use of + * i64gather/scatter). esize = element size = 4/8 bytes for float/double. + */ +#define MAX_STEP_SIZE 2097152 + static NPY_INLINE npy_uintp abs_ptrdiff(char *a, char *b) { @@ -41,6 +57,44 @@ abs_ptrdiff(char *a, char *b) } /* + * nomemoverlap - returns true if two strided arrays have an overlapping + * region in memory. ip_size/op_size = size of the arrays which can be negative + * indicating negative steps. + */ +static NPY_INLINE npy_bool +nomemoverlap(char *ip, + npy_intp ip_size, + char *op, + npy_intp op_size) +{ + char *ip_start, *ip_end, *op_start, *op_end; + if (ip_size < 0) { + ip_start = ip + ip_size; + ip_end = ip; + } + else { + ip_start = ip; + ip_end = ip + ip_size; + } + if (op_size < 0) { + op_start = op + op_size; + op_end = op; + } + else { + op_start = op; + op_end = op + op_size; + } + return (ip_start > op_end) | (op_start > ip_end); +} + +#define IS_BINARY_STRIDE_ONE(esize, vsize) \ + ((steps[0] == esize) && \ + (steps[1] == esize) && \ + (steps[2] == esize) && \ + (abs_ptrdiff(args[2], args[0]) >= vsize) && \ + (abs_ptrdiff(args[2], args[1]) >= vsize)) + +/* * stride is equal to element size and input and destination are equal or * don't overlap within one register. The check of the steps against * esize also quarantees that steps are >= 0. @@ -52,13 +106,34 @@ abs_ptrdiff(char *a, char *b) ((abs_ptrdiff(args[1], args[0]) == 0)))) /* - * output should be contiguous, can handle strided input data + * Avoid using SIMD for very large step sizes for several reasons: + * 1) Supporting large step sizes requires use of i64gather/scatter_ps instructions, + * in which case we need two i64gather instructions and an additional vinsertf32x8 + * instruction to load a single zmm register (since one i64gather instruction + * loads into a ymm register). This is not ideal for performance. + * 2) Gather and scatter instructions can be slow when the loads/stores + * cross page boundaries. + * + * We instead rely on i32gather/scatter_ps instructions which use a 32-bit index + * element. The index needs to be < INT_MAX to avoid overflow. MAX_STEP_SIZE + * ensures this. The condition also requires that the input and output arrays + * should have no overlap in memory. + */ +#define IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP \ + ((abs(steps[0]) < MAX_STEP_SIZE) && \ + (abs(steps[1]) < MAX_STEP_SIZE) && \ + (abs(steps[2]) < MAX_STEP_SIZE) && \ + (nomemoverlap(args[0], steps[0] * dimensions[0], args[2], steps[2] * dimensions[0])) && \ + (nomemoverlap(args[1], steps[1] * dimensions[0], args[2], steps[2] * dimensions[0]))) + +/* + * 1) Output should be contiguous, can handle strided input data + * 2) Input step should be smaller than MAX_STEP_SIZE for performance + * 3) Input and output arrays should have no overlap in memory */ #define IS_OUTPUT_BLOCKABLE_UNARY(esize, vsize) \ - (steps[1] == (esize) && \ - (npy_is_aligned(args[0], esize) && npy_is_aligned(args[1], esize)) && \ - ((abs_ptrdiff(args[1], args[0]) >= (vsize)) || \ - ((abs_ptrdiff(args[1], args[0]) == 0)))) + (steps[1] == (esize) && abs(steps[0]) < MAX_STEP_SIZE && \ + (nomemoverlap(args[1], steps[1] * dimensions[0], args[0], steps[0] * dimensions[0]))) #define IS_BLOCKABLE_REDUCE(esize, vsize) \ (steps[1] == (esize) && abs_ptrdiff(args[1], args[0]) >= (vsize) && \ @@ -125,11 +200,109 @@ abs_ptrdiff(char *a, char *b) /* ***************************************************************************** + ** CMPLX DISPATCHERS + ***************************************************************************** + */ + +/**begin repeat + * #TYPE = CFLOAT, CDOUBLE# + * #type= npy_float, npy_double# + * #esize = 8, 16# + */ + +/**begin repeat1 + * #func = add, subtract, multiply# + */ + +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_INLINE NPY_GCC_TARGET_AVX512F void +AVX512F_@func@_@TYPE@(char **args, const npy_intp *dimensions, const npy_intp *steps); +#endif + +static NPY_INLINE int +run_binary_avx512f_@func@_@TYPE@(char **args, const npy_intp *dimensions, const npy_intp *steps) +{ +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS + if (IS_BINARY_STRIDE_ONE(@esize@, 64)) { + AVX512F_@func@_@TYPE@(args, dimensions, steps); + return 1; + } + else + return 0; +#endif + return 0; +} + +/**end repeat1**/ + +/**begin repeat1 + * #func = square, absolute, conjugate# + * #outsize = 1, 2, 1# + * #max_stride = 2, 8, 8# + */ + +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_INLINE NPY_GCC_TARGET_AVX512F void +AVX512F_@func@_@TYPE@(@type@*, @type@*, const npy_intp n, const npy_intp stride); +#endif + +static NPY_INLINE int +run_unary_avx512f_@func@_@TYPE@(char **args, const npy_intp *dimensions, const npy_intp *steps) +{ +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS + if ((IS_OUTPUT_BLOCKABLE_UNARY((npy_uint)(@esize@/@outsize@), 64)) && (labs(steps[0]) < 2*@max_stride@*@esize@)) { + AVX512F_@func@_@TYPE@((@type@*)args[1], (@type@*)args[0], dimensions[0], steps[0]); + return 1; + } + else + return 0; +#endif + return 0; +} + +/**end repeat1**/ +/**end repeat**/ + +/* + ***************************************************************************** ** FLOAT DISPATCHERS ***************************************************************************** */ /**begin repeat + * #type = npy_float, npy_double, npy_longdouble# + * #TYPE = FLOAT, DOUBLE, LONGDOUBLE# + * #EXISTS = 1, 1, 0# + */ + +/**begin repeat1 + * #func = maximum, minimum# + */ + +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS && @EXISTS@ +static NPY_INLINE NPY_GCC_TARGET_AVX512F void +AVX512F_@func@_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps); +#endif + +static NPY_INLINE int +run_binary_avx512f_@func@_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) +{ +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS && @EXISTS@ + if (IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP) { + AVX512F_@func@_@TYPE@(args, dimensions, steps); + return 1; + } + else + return 0; +#endif + return 0; +} + + +/**end repeat1**/ +/**end repeat**/ + +/**begin repeat * #ISA = FMA, AVX512F# * #isa = fma, avx512f# * #CHK = HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS# @@ -215,6 +388,25 @@ run_unary_@isa@_sincos_FLOAT(char **args, npy_intp const *dimensions, npy_intp c /**end repeat**/ +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_INLINE void +AVX512F_exp_DOUBLE(npy_double *, npy_double *, const npy_intp n, const npy_intp stride); +#endif +static NPY_INLINE int +run_unary_avx512f_exp_DOUBLE(char **args, npy_intp const *dimensions, npy_intp const *steps) +{ +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +#if !(defined(__clang__) && (__clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 1))) + if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(npy_double), 64)) { + AVX512F_exp_DOUBLE((npy_double*)args[1], (npy_double*)args[0], dimensions[0], steps[0]); + return 1; + } + else + return 0; +#endif +#endif + return 0; +} /**begin repeat * Float types @@ -1523,11 +1715,35 @@ avx512_scalef_ps(__m512 poly, __m512 quadrant) { return _mm512_scalef_ps(poly, quadrant); } + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512d +avx512_permute_x4var_pd(__m512d t0, + __m512d t1, + __m512d t2, + __m512d t3, + __m512i index) +{ + + __mmask8 lut_mask = _mm512_cmp_epi64_mask(index, _mm512_set1_epi64(15), + _MM_CMPINT_GT); + __m512d res1 = _mm512_permutex2var_pd(t0, index, t1); + __m512d res2 = _mm512_permutex2var_pd(t2, index, t3); + return _mm512_mask_blend_pd(lut_mask, res1, res2); +} + /**begin repeat * #vsub = ps, pd# + * #type= npy_float, npy_double# * #epi_vsub = epi32, epi64# * #vtype = __m512, __m512d# + * #mask = __mmask16, __mmask8# * #and_const = 0x7fffffff, 0x7fffffffffffffffLL# + * #neg_mask = 0x80000000, 0x8000000000000000# + * #perm_ = 0xb1, 0x55# + * #cmpx_img_mask = 0xAAAA, 0xAA# + * #cmpx_re_mask = 0x5555, 0x55# + * #INF = NPY_INFINITYF, NPY_INFINITY# + * #NAN = NPY_NANF, NPY_NAN# */ static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ avx512_abs_@vsub@(@vtype@ x) @@ -1565,6 +1781,96 @@ avx512_trunc_@vsub@(@vtype@ x) { return _mm512_roundscale_@vsub@(x, 0x0B); } + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_hadd_@vsub@(const @vtype@ x) +{ + return _mm512_add_@vsub@(x, _mm512_permute_@vsub@(x, @perm_@)); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_hsub_@vsub@(const @vtype@ x) +{ + return _mm512_sub_@vsub@(x, _mm512_permute_@vsub@(x, @perm_@)); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_cabsolute_@vsub@(const @vtype@ x1, + const @vtype@ x2, + const __m512i re_indices, + const __m512i im_indices) +{ + @vtype@ inf = _mm512_set1_@vsub@(@INF@); + @vtype@ nan = _mm512_set1_@vsub@(@NAN@); + @vtype@ x1_abs = avx512_abs_@vsub@(x1); + @vtype@ x2_abs = avx512_abs_@vsub@(x2); + @vtype@ re = _mm512_permutex2var_@vsub@(x1_abs, re_indices, x2_abs); + @vtype@ im = _mm512_permutex2var_@vsub@(x1_abs, im_indices , x2_abs); + /* + * If real or imag = INF, then convert it to inf + j*inf + * Handles: inf + j*nan, nan + j*inf + */ + @mask@ re_infmask = _mm512_cmp_@vsub@_mask(re, inf, _CMP_EQ_OQ); + @mask@ im_infmask = _mm512_cmp_@vsub@_mask(im, inf, _CMP_EQ_OQ); + im = _mm512_mask_mov_@vsub@(im, re_infmask, inf); + re = _mm512_mask_mov_@vsub@(re, im_infmask, inf); + + /* + * If real or imag = NAN, then convert it to nan + j*nan + * Handles: x + j*nan, nan + j*x + */ + @mask@ re_nanmask = _mm512_cmp_@vsub@_mask(re, re, _CMP_NEQ_UQ); + @mask@ im_nanmask = _mm512_cmp_@vsub@_mask(im, im, _CMP_NEQ_UQ); + im = _mm512_mask_mov_@vsub@(im, re_nanmask, nan); + re = _mm512_mask_mov_@vsub@(re, im_nanmask, nan); + + @vtype@ larger = _mm512_max_@vsub@(re, im); + @vtype@ smaller = _mm512_min_@vsub@(im, re); + + /* + * Calculate div_mask to prevent 0./0. and inf/inf operations in div + */ + @mask@ zeromask = _mm512_cmp_@vsub@_mask(larger, _mm512_setzero_@vsub@(), _CMP_EQ_OQ); + @mask@ infmask = _mm512_cmp_@vsub@_mask(smaller, inf, _CMP_EQ_OQ); + @mask@ div_mask = _mm512_knot(_mm512_kor(zeromask, infmask)); + @vtype@ ratio = _mm512_maskz_div_@vsub@(div_mask, smaller, larger); + @vtype@ hypot = _mm512_sqrt_@vsub@(_mm512_fmadd_@vsub@( + ratio, ratio, _mm512_set1_@vsub@(1.0f))); + return _mm512_mul_@vsub@(hypot, larger); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_conjugate_@vsub@(const @vtype@ x) +{ + /* + * __mm512_mask_xor_ps/pd requires AVX512DQ. We cast it to __m512i and + * use the xor_epi32/64 uinstruction instead. Cast is a zero latency instruction + */ + __m512i cast_x = _mm512_cast@vsub@_si512(x); + __m512i res = _mm512_mask_xor_@epi_vsub@(cast_x, @cmpx_img_mask@, + cast_x, _mm512_set1_@epi_vsub@(@neg_mask@)); + return _mm512_castsi512_@vsub@(res); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_cmul_@vsub@(@vtype@ x1, @vtype@ x2) +{ + // x1 = r1, i1 + // x2 = r2, i2 + @vtype@ x3 = _mm512_permute_@vsub@(x2, @perm_@); // i2, r2 + @vtype@ x12 = _mm512_mul_@vsub@(x1, x2); // r1*r2, i1*i2 + @vtype@ x13 = _mm512_mul_@vsub@(x1, x3); // r1*i2, r2*i1 + @vtype@ outreal = avx512_hsub_@vsub@(x12); // r1*r2 - i1*i2, r1*r2 - i1*i2 + @vtype@ outimg = avx512_hadd_@vsub@(x13); // r1*i2 + i1*r2, r1*i2 + i1*r2 + return _mm512_mask_blend_@vsub@(@cmpx_img_mask@, outreal, outimg); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_csquare_@vsub@(@vtype@ x) +{ + return avx512_cmul_@vsub@(x, x); +} + /**end repeat**/ #endif @@ -1671,6 +1977,101 @@ static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@d #endif /**end repeat**/ +/**begin repeat + * #type = npy_float, npy_double# + * #TYPE = FLOAT, DOUBLE# + * #num_lanes = 16, 8# + * #vsuffix = ps, pd# + * #mask = __mmask16, __mmask8# + * #vtype = __m512, __m512d# + * #scale = 4, 8# + * #vindextype = __m512i, __m256i# + * #vindexsize = 512, 256# + * #vindexload = _mm512_loadu_si512, _mm256_loadu_si256# + */ + +/**begin repeat1 + * #func = maximum, minimum# + * #vectorf = max, min# + */ + +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_INLINE NPY_GCC_TARGET_AVX512F void +AVX512F_@func@_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) +{ + const npy_intp stride_ip1 = steps[0]/(npy_intp)sizeof(@type@); + const npy_intp stride_ip2 = steps[1]/(npy_intp)sizeof(@type@); + const npy_intp stride_op = steps[2]/(npy_intp)sizeof(@type@); + const npy_intp array_size = dimensions[0]; + npy_intp num_remaining_elements = array_size; + @type@* ip1 = (@type@*) args[0]; + @type@* ip2 = (@type@*) args[1]; + @type@* op = (@type@*) args[2]; + + @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP + */ + + npy_int32 index_ip1[@num_lanes@], index_ip2[@num_lanes@], index_op[@num_lanes@]; + for (npy_int32 ii = 0; ii < @num_lanes@; ii++) { + index_ip1[ii] = ii*stride_ip1; + index_ip2[ii] = ii*stride_ip2; + index_op[ii] = ii*stride_op; + } + @vindextype@ vindex_ip1 = @vindexload@((@vindextype@*)&index_ip1[0]); + @vindextype@ vindex_ip2 = @vindexload@((@vindextype@*)&index_ip2[0]); + @vindextype@ vindex_op = @vindexload@((@vindextype@*)&index_op[0]); + @vtype@ zeros_f = _mm512_setzero_@vsuffix@(); + + while (num_remaining_elements > 0) { + if (num_remaining_elements < @num_lanes@) { + load_mask = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements, @num_lanes@); + } + @vtype@ x1, x2; + if (stride_ip1 == 1) { + x1 = avx512_masked_load_@vsuffix@(load_mask, ip1); + } + else { + x1 = avx512_masked_gather_@vsuffix@(zeros_f, ip1, vindex_ip1, load_mask); + } + if (stride_ip2 == 1) { + x2 = avx512_masked_load_@vsuffix@(load_mask, ip2); + } + else { + x2 = avx512_masked_gather_@vsuffix@(zeros_f, ip2, vindex_ip2, load_mask); + } + + /* + * when only one of the argument is a nan, the maxps/maxpd instruction + * returns the second argument. The additional blend instruction fixes + * this issue to conform with NumPy behaviour. + */ + @mask@ nan_mask = _mm512_cmp_@vsuffix@_mask(x1, x1, _CMP_NEQ_UQ); + @vtype@ out = _mm512_@vectorf@_@vsuffix@(x1, x2); + out = _mm512_mask_blend_@vsuffix@(nan_mask, out, x1); + + if (stride_op == 1) { + _mm512_mask_storeu_@vsuffix@(op, load_mask, out); + } + else { + /* scatter! */ + _mm512_mask_i32scatter_@vsuffix@(op, load_mask, vindex_op, out, @scale@); + } + + ip1 += @num_lanes@*stride_ip1; + ip2 += @num_lanes@*stride_ip2; + op += @num_lanes@*stride_op; + num_remaining_elements -= @num_lanes@; + } +} +#endif +/**end repeat**/ +/**end repeat1**/ /**begin repeat * #ISA = FMA, AVX512F# @@ -1699,16 +2100,23 @@ static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void const npy_intp array_size, const npy_intp steps) { - const npy_intp stride = steps/sizeof(npy_float); - const npy_int num_lanes = @BYTES@/sizeof(npy_float); + const npy_intp stride = steps/(npy_intp)sizeof(npy_float); + const npy_int num_lanes = @BYTES@/(npy_intp)sizeof(npy_float); npy_intp num_remaining_elements = array_size; @vtype@ ones_f = _mm@vsize@_set1_ps(1.0f); @mask@ load_mask = @isa@_get_full_load_mask_ps(); #if @replace_0_with_1@ @mask@ inv_load_mask = @isa@_invert_mask_ps(load_mask); #endif - npy_int indexarr[16]; - for (npy_int ii = 0; ii < 16; ii++) { + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_OUTPUT_BLOCKABLE_UNARY + */ + + npy_int32 indexarr[16]; + for (npy_int32 ii = 0; ii < 16; ii++) { indexarr[ii] = ii*stride; } @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)&indexarr[0]); @@ -1778,16 +2186,22 @@ static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void const npy_intp array_size, const npy_intp steps) { - const npy_intp stride = steps/sizeof(npy_double); - const npy_int num_lanes = @BYTES@/sizeof(npy_double); + const npy_intp stride = steps/(npy_intp)sizeof(npy_double); + const npy_int num_lanes = @BYTES@/(npy_intp)sizeof(npy_double); npy_intp num_remaining_elements = array_size; @mask@ load_mask = @isa@_get_full_load_mask_pd(); #if @replace_0_with_1@ @mask@ inv_load_mask = @isa@_invert_mask_pd(load_mask); #endif @vtype@ ones_d = _mm@vsize@_set1_pd(1.0f); - npy_int indexarr[8]; - for (npy_int ii = 0; ii < 8; ii++) { + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_OUTPUT_BLOCKABLE_UNARY + */ + npy_int32 indexarr[8]; + for (npy_int32 ii = 0; ii < 8; ii++) { indexarr[ii] = ii*stride; } @vindextype@ vindex = @vindexload@((@vindextype@*)&indexarr[0]); @@ -1874,8 +2288,8 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void const npy_intp steps, NPY_TRIG_OP my_trig_op) { - const npy_intp stride = steps/sizeof(npy_float); - const npy_int num_lanes = @BYTES@/sizeof(npy_float); + const npy_intp stride = steps/(npy_intp)sizeof(npy_float); + const npy_int num_lanes = @BYTES@/(npy_intp)sizeof(npy_float); npy_float large_number = 71476.0625f; if (my_trig_op == npy_compute_sin) { large_number = 117435.992f; @@ -1905,8 +2319,14 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void @mask@ nan_mask, glibc_mask, sine_mask, negate_mask; @mask@ load_mask = @isa@_get_full_load_mask_ps(); npy_intp num_remaining_elements = array_size; - npy_int indexarr[16]; - for (npy_int ii = 0; ii < 16; ii++) { + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_OUTPUT_BLOCKABLE_UNARY + */ + npy_int32 indexarr[16]; + for (npy_int32 ii = 0; ii < 16; ii++) { indexarr[ii] = ii*stride; } @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)&indexarr[0]); @@ -2017,12 +2437,18 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void const npy_intp array_size, const npy_intp steps) { - const npy_intp stride = steps/sizeof(npy_float); - const npy_int num_lanes = @BYTES@/sizeof(npy_float); + const npy_intp stride = steps/(npy_intp)sizeof(npy_float); + const npy_int num_lanes = @BYTES@/(npy_intp)sizeof(npy_float); npy_float xmax = 88.72283935546875f; npy_float xmin = -103.97208404541015625f; - npy_int indexarr[16]; - for (npy_int ii = 0; ii < 16; ii++) { + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_OUTPUT_BLOCKABLE_UNARY + */ + npy_int32 indexarr[16]; + for (npy_int32 ii = 0; ii < 16; ii++) { indexarr[ii] = ii*stride; } @@ -2143,10 +2569,16 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void const npy_intp array_size, const npy_intp steps) { - const npy_intp stride = steps/sizeof(npy_float); - const npy_int num_lanes = @BYTES@/sizeof(npy_float); - npy_int indexarr[16]; - for (npy_int ii = 0; ii < 16; ii++) { + const npy_intp stride = steps/(npy_intp)sizeof(npy_float); + const npy_int num_lanes = @BYTES@/(npy_intp)sizeof(npy_float); + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_OUTPUT_BLOCKABLE_UNARY + */ + npy_int32 indexarr[16]; + for (npy_int32 ii = 0; ii < 16; ii++) { indexarr[ii] = ii*stride; } @@ -2259,6 +2691,335 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void /**end repeat**/ /* + * Vectorized implementation of exp double using AVX512 + * Reference: Tang, P.T.P., "Table-driven implementation of the + * exponential function in IEEE floating-point + * arithmetic," ACM Transactions on Mathematical + * Software, vol. 15, pp. 144-157, 1989. + * 1) if x > mTH_max or x is INF; return INF (overflow) + * 2) if x < mTH_min; return 0.0f (underflow) + * 3) if abs(x) < mTH_nearzero; return 1.0f + x + * 4) if x is Nan; return Nan + * 5) Range reduction: + * x = (32m + j)ln2 / 32 + r; r in [-ln2/64, ln2/64] + * 6) exp(r) - 1 is approximated by a polynomial function p(r) + * exp(x) = 2^m(2^(j/32) + 2^(j/32)p(r)); + */ +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS +#if !(defined(__clang__) && (__clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 1))) +static NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F void +AVX512F_exp_DOUBLE(npy_double * op, + npy_double * ip, + const npy_intp array_size, + const npy_intp steps) +{ + npy_intp num_remaining_elements = array_size; + const npy_intp stride = steps / (npy_intp)sizeof(npy_double); + const npy_int num_lanes = 64 / (npy_intp)sizeof(npy_double); + npy_int32 indexarr[8]; + for (npy_int32 ii = 0; ii < 8; ii++) { + indexarr[ii] = ii*stride; + } + + __m512d InvLn2N = _mm512_set1_pd(NPY_INV_LN2_MUL_32); + __m512d mShift = _mm512_set1_pd(NPY_RINT_CVT_MAGIC); + __m512d mNegL1 = _mm512_set1_pd(NPY_TANG_NEG_L1); + __m512d mNegL2 = _mm512_set1_pd(NPY_TANG_NEG_L2); + __m512i mMod = _mm512_set1_epi64(0x1f); + __m512d mA1 = _mm512_set1_pd(NPY_TANG_A1); + __m512d mA2 = _mm512_set1_pd(NPY_TANG_A2); + __m512d mA3 = _mm512_set1_pd(NPY_TANG_A3); + __m512d mA4 = _mm512_set1_pd(NPY_TANG_A4); + __m512d mA5 = _mm512_set1_pd(NPY_TANG_A5); + __m512d mTH_nearzero = _mm512_set1_pd(0x1p-54); + __m512d mTH_max = _mm512_set1_pd(0x1.62e42fefa39efp+9); + __m512d mTH_min = _mm512_set1_pd(-0x1.74910d52d3053p+9); + __m512d mTH_inf = _mm512_set1_pd(NPY_INFINITY); + __m512d zeros_d = _mm512_set1_pd(0.0f); + __m512d ones_d = _mm512_set1_pd(1.0f); + __m256i vindex = _mm256_loadu_si256((__m256i*)&indexarr[0]); + + __m512d mTable_top_0 = _mm512_loadu_pd(&(EXP_Table_top[8*0])); + __m512d mTable_top_1 = _mm512_loadu_pd(&(EXP_Table_top[8*1])); + __m512d mTable_top_2 = _mm512_loadu_pd(&(EXP_Table_top[8*2])); + __m512d mTable_top_3 = _mm512_loadu_pd(&(EXP_Table_top[8*3])); + __m512d mTable_tail_0 = _mm512_loadu_pd(&(EXP_Table_tail[8*0])); + __m512d mTable_tail_1 = _mm512_loadu_pd(&(EXP_Table_tail[8*1])); + __m512d mTable_tail_2 = _mm512_loadu_pd(&(EXP_Table_tail[8*2])); + __m512d mTable_tail_3 = _mm512_loadu_pd(&(EXP_Table_tail[8*3])); + + __mmask8 overflow_mask = avx512_get_partial_load_mask_pd(0, num_lanes); + __mmask8 load_mask = avx512_get_full_load_mask_pd(); + __mmask8 xmin_mask, xmax_mask, inf_mask, nan_mask, nearzero_mask; + + while (num_remaining_elements > 0) { + if (num_remaining_elements < num_lanes) { + load_mask = avx512_get_partial_load_mask_pd(num_remaining_elements, + num_lanes); + } + + __m512d x; + if (1 == stride) { + x = avx512_masked_load_pd(load_mask, ip); + } + else { + x = avx512_masked_gather_pd(zeros_d, ip, vindex, load_mask); + } + + nan_mask = _mm512_cmp_pd_mask(x, x, _CMP_NEQ_UQ); + x = avx512_set_masked_lanes_pd(x, zeros_d, nan_mask); + xmax_mask = _mm512_cmp_pd_mask(x, mTH_max, _CMP_GT_OQ); + xmin_mask = _mm512_cmp_pd_mask(x, mTH_min, _CMP_LT_OQ); + inf_mask = _mm512_cmp_pd_mask(x, mTH_inf, _CMP_EQ_OQ); + __m512i x_abs = _mm512_and_epi64(_mm512_castpd_si512(x), + _mm512_set1_epi64(0x7FFFFFFFFFFFFFFF)); + nearzero_mask = _mm512_cmp_pd_mask(_mm512_castsi512_pd(x_abs), + mTH_nearzero, _CMP_LT_OQ); + nearzero_mask = _mm512_kxor(nearzero_mask, nan_mask); + overflow_mask = _mm512_kor(overflow_mask, + _mm512_kxor(xmax_mask, inf_mask)); + x = avx512_set_masked_lanes_pd(x, zeros_d, + _mm512_kor(_mm512_kor(nan_mask, xmin_mask), + _mm512_kor(xmax_mask, nearzero_mask))); + + /* z = x * 32/ln2 */ + __m512d z = _mm512_mul_pd(x, InvLn2N); + + /* round to nearest */ + __m512d kd = _mm512_add_pd(z, mShift); + __m512i ki = _mm512_castpd_si512(kd); + kd = _mm512_sub_pd(kd, mShift); + + /* r = (x + kd*mNegL1) + kd*mNegL2 */ + __m512d r1 = _mm512_fmadd_pd(kd, mNegL1, x); + __m512d r2 = _mm512_mul_pd(kd, mNegL2); + __m512d r = _mm512_add_pd(r1,r2); + + /* Polynomial approximation for exp(r) - 1 */ + __m512d q = _mm512_fmadd_pd(mA5, r, mA4); + q = _mm512_fmadd_pd(q, r, mA3); + q = _mm512_fmadd_pd(q, r, mA2); + q = _mm512_fmadd_pd(q, r, mA1); + q = _mm512_mul_pd(q, r); + __m512d p = _mm512_fmadd_pd(r, q, r2);; + p = _mm512_add_pd(r1, p); + + /* Get 2^(j/32) from lookup table */ + __m512i j = _mm512_and_epi64(ki, mMod); + __m512d top = avx512_permute_x4var_pd(mTable_top_0, mTable_top_1, + mTable_top_2, mTable_top_3, j); + __m512d tail = avx512_permute_x4var_pd(mTable_tail_0, mTable_tail_1, + mTable_tail_2, mTable_tail_3, j); + + /* + * s = top + tail; + * exp(x) = 2^m * (top + (tail + s * p)); + */ + __m512d s = _mm512_add_pd(top, tail); + __m512d res = _mm512_fmadd_pd(s, p, tail); + res = _mm512_add_pd(res, top); + res= _mm512_scalef_pd(res, _mm512_div_pd(kd, _mm512_set1_pd(32))); + + /* return special cases */ + res = avx512_set_masked_lanes_pd(res, _mm512_add_pd(x, ones_d), + nearzero_mask); + res = avx512_set_masked_lanes_pd(res, _mm512_set1_pd(NPY_NAN), + nan_mask); + res = avx512_set_masked_lanes_pd(res, mTH_inf, xmax_mask); + res = avx512_set_masked_lanes_pd(res, zeros_d, xmin_mask); + + _mm512_mask_storeu_pd(op, load_mask, res); + + ip += num_lanes * stride; + op += num_lanes; + num_remaining_elements -= num_lanes; + } + if (overflow_mask) { + npy_set_floatstatus_overflow(); + } +} +#endif +#endif + +/**begin repeat + * #TYPE = CFLOAT, CDOUBLE# + * #type = npy_float, npy_double# + * #num_lanes = 16, 8# + * #vsuffix = ps, pd# + * #epi_vsub = epi32, epi64# + * #mask = __mmask16, __mmask8# + * #vtype = __m512, __m512d# + * #scale = 4, 8# + * #vindextype = __m512i, __m256i# + * #vindexload = _mm512_loadu_si512, _mm256_loadu_si256# + * #storemask = 0xFF, 0xF# + * #IS_FLOAT = 1, 0# + */ + +/**begin repeat1 + * #func = add, subtract, multiply# + * #vectorf = _mm512_add, _mm512_sub, avx512_cmul# + */ + +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_GCC_OPT_3 NPY_INLINE NPY_GCC_TARGET_AVX512F void +AVX512F_@func@_@TYPE@(char **args, const npy_intp *dimensions, const npy_intp *steps) +{ + const npy_intp array_size = dimensions[0]; + npy_intp num_remaining_elements = 2*array_size; + @type@* ip1 = (@type@*) args[0]; + @type@* ip2 = (@type@*) args[1]; + @type@* op = (@type@*) args[2]; + + @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); + + while (num_remaining_elements > 0) { + if (num_remaining_elements < @num_lanes@) { + load_mask = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements, @num_lanes@); + } + @vtype@ x1, x2; + x1 = avx512_masked_load_@vsuffix@(load_mask, ip1); + x2 = avx512_masked_load_@vsuffix@(load_mask, ip2); + + @vtype@ out = @vectorf@_@vsuffix@(x1, x2); + + _mm512_mask_storeu_@vsuffix@(op, load_mask, out); + + ip1 += @num_lanes@; + ip2 += @num_lanes@; + op += @num_lanes@; + num_remaining_elements -= @num_lanes@; + } +} +#endif +/**end repeat1**/ + +/**begin repeat1 + * #func = square, conjugate# + * #vectorf = avx512_csquare, avx512_conjugate# + */ + +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_GCC_OPT_3 NPY_INLINE NPY_GCC_TARGET_AVX512F void +AVX512F_@func@_@TYPE@(@type@ * op, + @type@ * ip, + const npy_intp array_size, + const npy_intp steps) +{ + npy_intp num_remaining_elements = 2*array_size; + const npy_intp stride_ip1 = steps/(npy_intp)sizeof(@type@)/2; + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via max_stride + */ + npy_int32 index_ip1[16]; + for (npy_int32 ii = 0; ii < @num_lanes@; ii=ii+2) { + index_ip1[ii] = ii*stride_ip1; + index_ip1[ii+1] = ii*stride_ip1 + 1; + } + @vindextype@ vindex = @vindexload@((@vindextype@*)index_ip1); + @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); + @vtype@ zeros = _mm512_setzero_@vsuffix@(); + + while (num_remaining_elements > 0) { + if (num_remaining_elements < @num_lanes@) { + load_mask = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements, @num_lanes@); + } + @vtype@ x1; + if (stride_ip1 == 1) { + x1 = avx512_masked_load_@vsuffix@(load_mask, ip); + } + else { + x1 = avx512_masked_gather_@vsuffix@(zeros, ip, vindex, load_mask); + } + + @vtype@ out = @vectorf@_@vsuffix@(x1); + + _mm512_mask_storeu_@vsuffix@(op, load_mask, out); + op += @num_lanes@; + ip += @num_lanes@*stride_ip1; + num_remaining_elements -= @num_lanes@; + } +} +#endif +/**end repeat1**/ + +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_GCC_OPT_3 NPY_INLINE NPY_GCC_TARGET_AVX512F void +AVX512F_absolute_@TYPE@(@type@ * op, + @type@ * ip, + const npy_intp array_size, + const npy_intp steps) +{ + npy_intp num_remaining_elements = 2*array_size; + const npy_intp stride_ip1 = steps/(npy_intp)sizeof(@type@)/2; + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via max_stride + */ + npy_int32 index_ip[32]; + for (npy_int32 ii = 0; ii < 2*@num_lanes@; ii=ii+2) { + index_ip[ii] = ii*stride_ip1; + index_ip[ii+1] = ii*stride_ip1 + 1; + } + @vindextype@ vindex1 = @vindexload@((@vindextype@*)index_ip); + @vindextype@ vindex2 = @vindexload@((@vindextype@*)(index_ip+@num_lanes@)); + + @mask@ load_mask1 = avx512_get_full_load_mask_@vsuffix@(); + @mask@ load_mask2 = avx512_get_full_load_mask_@vsuffix@(); + @mask@ store_mask = avx512_get_full_load_mask_@vsuffix@(); + @vtype@ zeros = _mm512_setzero_@vsuffix@(); + +#if @IS_FLOAT@ + __m512i re_index = _mm512_set_epi32(30,28,26,24,22,20,18,16,14,12,10,8,6,4,2,0); + __m512i im_index = _mm512_set_epi32(31,29,27,25,23,21,19,17,15,13,11,9,7,5,3,1); +#else + __m512i re_index = _mm512_set_epi64(14,12,10,8,6,4,2,0); + __m512i im_index = _mm512_set_epi64(15,13,11,9,7,5,3,1); +#endif + + while (num_remaining_elements > 0) { + if (num_remaining_elements < @num_lanes@) { + load_mask1 = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements, @num_lanes@); + load_mask2 = 0x0000; + store_mask = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements/2, @num_lanes@); + } else if (num_remaining_elements < 2*@num_lanes@) { + load_mask1 = avx512_get_full_load_mask_@vsuffix@(); + load_mask2 = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements - @num_lanes@, @num_lanes@); + store_mask = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements/2, @num_lanes@); + } + @vtype@ x1, x2; + if (stride_ip1 == 1) { + x1 = avx512_masked_load_@vsuffix@(load_mask1, ip); + x2 = avx512_masked_load_@vsuffix@(load_mask2, ip+@num_lanes@); + } + else { + x1 = avx512_masked_gather_@vsuffix@(zeros, ip, vindex1, load_mask1); + x2 = avx512_masked_gather_@vsuffix@(zeros, ip, vindex2, load_mask2); + } + + @vtype@ out = avx512_cabsolute_@vsuffix@(x1, x2, re_index, im_index); + + _mm512_mask_storeu_@vsuffix@(op, store_mask, out); + op += @num_lanes@; + ip += 2*@num_lanes@*stride_ip1; + num_remaining_elements -= 2*@num_lanes@; + } + npy_clear_floatstatus_barrier((char*)op); +} + +#endif +/**end repeat**/ + +/* ***************************************************************************** ** BOOL LOOPS ***************************************************************************** diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 5f9f4655b..c57199c79 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -2926,7 +2926,7 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, /* * The first nop strides are for the inner loop (but only can - * copy them after removing the core axes + * copy them after removing the core axes) */ memcpy(inner_strides, NpyIter_GetInnerStrideArray(iter), NPY_SIZEOF_INTP * nop); @@ -3052,17 +3052,15 @@ fail: return retval; } -/*UFUNC_API - * +/* * This generic function is called with the ufunc object, the arguments to it, * and an array of (pointers to) PyArrayObjects which are NULL. * * 'op' is an array of at least NPY_MAXARGS PyArrayObject *. */ -NPY_NO_EXPORT int -PyUFunc_GenericFunction(PyUFuncObject *ufunc, - PyObject *args, PyObject *kwds, - PyArrayObject **op) +static int +PyUFunc_GenericFunction_int(PyUFuncObject *ufunc, + PyObject *args, PyObject *kwds, PyArrayObject **op) { int nin, nout; int i, nop; @@ -3268,6 +3266,27 @@ fail: return retval; } + +/*UFUNC_API*/ +NPY_NO_EXPORT int +PyUFunc_GenericFunction(PyUFuncObject *ufunc, + PyObject *args, PyObject *kwds, PyArrayObject **op) +{ + /* NumPy 1.19, 2020-01-24 */ + if (DEPRECATE( + "PyUFunc_GenericFunction() C-API function is deprecated " + "and expected to be removed rapidly. If you are using it (i.e. see " + "this warning/error), please notify the NumPy developers. " + "As of now it is expected that any use case is served better by " + "the direct use of `PyObject_Call(ufunc, args, kwargs)`. " + "PyUFunc_GenericFunction function has slightly different " + "untested behaviour.") < 0) { + return -1; + } + return PyUFunc_GenericFunction_int(ufunc, args, kwds, op); +} + + /* * Given the output type, finds the specified binary op. The * ufunc must have nin==2 and nout==1. The function may modify @@ -4679,7 +4698,7 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) return override; } - errval = PyUFunc_GenericFunction(ufunc, args, kwds, mps); + errval = PyUFunc_GenericFunction_int(ufunc, args, kwds, mps); if (errval < 0) { return NULL; } @@ -4973,6 +4992,16 @@ PyUFunc_FromFuncAndDataAndSignatureAndIdentity(PyUFuncGenericFunction *func, voi NPY_NO_EXPORT int PyUFunc_SetUsesArraysAsData(void **data, size_t i) { + /* NumPy 1.19, 2020-01-24 */ + if (DEPRECATE( + "PyUFunc_SetUsesArraysAsData() C-API function is deprecated " + "and expected to be removed rapidly. If you are using it (i.e. see " + "this warning/error), please notify the NumPy developers. " + "It is currently assumed that this function is simply unused and " + "its removal will facilitate the implementation of better " + "approaches.") < 0) { + return -1; + } data[i] = (void*)PyUFunc_SetUsesArraysAsData; return 0; } @@ -6040,53 +6069,16 @@ static PyGetSetDef ufunc_getset[] = { NPY_NO_EXPORT PyTypeObject PyUFunc_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "numpy.ufunc", /* tp_name */ - sizeof(PyUFuncObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)ufunc_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - (reprfunc)ufunc_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc)ufunc_generic_call, /* tp_call */ - (reprfunc)ufunc_repr, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - 0, /* tp_doc */ - (traverseproc)ufunc_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - ufunc_methods, /* tp_methods */ - 0, /* tp_members */ - ufunc_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ + .tp_name = "numpy.ufunc", + .tp_basicsize = sizeof(PyUFuncObject), + .tp_dealloc = (destructor)ufunc_dealloc, + .tp_repr = (reprfunc)ufunc_repr, + .tp_call = (ternaryfunc)ufunc_generic_call, + .tp_str = (reprfunc)ufunc_repr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = (traverseproc)ufunc_traverse, + .tp_methods = ufunc_methods, + .tp_getset = ufunc_getset, }; /* End of code for ufunc objects */ diff --git a/numpy/core/tests/data/umath-validation-set-cos b/numpy/core/tests/data/umath-validation-set-cos index 360ebcd6a..2e75f044c 100644 --- a/numpy/core/tests/data/umath-validation-set-cos +++ b/numpy/core/tests/data/umath-validation-set-cos @@ -19,9 +19,7 @@ np.float32,0x80000001,0x3f800000,2 np.float32,0x00000000,0x3f800000,2 np.float32,0x80000000,0x3f800000,2 np.float32,0x00800000,0x3f800000,2 -np.float32,0x7f7fffff,0x3f5a5f96,2 np.float32,0x80800000,0x3f800000,2 -np.float32,0xff7fffff,0x3f5a5f96,2 ## 1.00f + 0x00000001 ## np.float32,0x3f800000,0x3f0a5140,2 np.float32,0x3f800001,0x3f0a513f,2 @@ -36,26 +34,6 @@ np.float32,0x41d92388,0xbed987c7,2 np.float32,0x422dd66c,0x3f5dcab3,2 np.float32,0xc28f5be6,0xbf5688d8,2 np.float32,0x41ab2674,0xbf53aa3b,2 -np.float32,0xd0102756,0x3f45d12d,2 -np.float32,0xcf99405e,0xbe9cf281,2 -np.float32,0xcfd83a12,0x3eaae4ca,2 -np.float32,0x4fb54db0,0xbf7b2894,2 -np.float32,0xcfcca29d,0x3f752e4e,2 -np.float32,0xceec2ac0,0xbf745303,2 -np.float32,0xcfdca97f,0x3ef554a7,2 -np.float32,0xcfe92b0a,0x3f4618f2,2 -np.float32,0x5014b0eb,0x3ee933e6,2 -np.float32,0xcfa7ee96,0xbeedeeb2,2 -np.float32,0x754c09a0,0xbef298de,2 -np.float32,0x77a731fb,0x3f24599f,2 -np.float32,0x76de2494,0x3f79576c,2 -np.float32,0xf74920dc,0xbf4d196e,2 -np.float32,0x7707a312,0xbeb5cb8e,2 -np.float32,0x75bf9790,0xbf7fd7fe,2 -np.float32,0xf4ca7c40,0xbe15107d,2 -np.float32,0x77e91899,0xbe8a968b,2 -np.float32,0xf74c9820,0xbf7f9677,2 -np.float32,0x7785ca29,0xbe6ef93b,2 np.float32,0x3f490fdb,0x3f3504f3,2 np.float32,0xbf490fdb,0x3f3504f3,2 np.float32,0x3fc90fdb,0xb33bbd2e,2 @@ -660,26 +638,6 @@ np.float32,0x4350ea79,0x3631dadb,2 np.float32,0x42dbe957,0xbf800000,2 np.float32,0x425be957,0xb505522a,2 np.float32,0x435be957,0x3f800000,2 -np.float32,0x487fe5ab,0xba140185,2 -np.float32,0x497fe5ab,0x3f7fffd5,2 -np.float32,0x49ffe5ab,0x3f7fff55,2 -np.float32,0x49ffeb37,0x3b9382f5,2 -np.float32,0x497ff0c3,0x3b13049f,2 -np.float32,0x49fff0c3,0xbf7fff57,2 -np.float32,0x49fff64f,0xbb928618,2 -np.float32,0x497ffbdb,0xbf7fffd6,2 -np.float32,0x49fffbdb,0x3f7fff59,2 -np.float32,0x48fffbdb,0xba9207c6,2 -np.float32,0x4e736e56,0xbf800000,2 -np.float32,0x4d4da377,0xbf800000,2 -np.float32,0x4ece58c3,0xbf800000,2 -np.float32,0x4ee0db9c,0xbf800000,2 -np.float32,0x4dee7002,0x3f800000,2 -np.float32,0x4ee86afc,0x38857a23,2 -np.float32,0x4dca4f3f,0xbf800000,2 -np.float32,0x4ecb48af,0xb95d1e10,2 -np.float32,0x4e51e33f,0xbf800000,2 -np.float32,0x4ef5f421,0xbf800000,2 np.float32,0x46027eb2,0x3e7d94c9,2 np.float32,0x4477baed,0xbe7f1824,2 np.float32,0x454b8024,0x3e7f5268,2 diff --git a/numpy/core/tests/data/umath-validation-set-exp b/numpy/core/tests/data/umath-validation-set-exp index 1b2cc9ce4..7c5ef3b33 100644 --- a/numpy/core/tests/data/umath-validation-set-exp +++ b/numpy/core/tests/data/umath-validation-set-exp @@ -133,3 +133,280 @@ np.float32,0xc29b43d5,0x077ffffc,3 np.float32,0xc1e61ff7,0x2ab504f5,3 np.float32,0xc2867878,0x0effff15,3 np.float32,0xc2a2324a,0x04fffff4,3 +#float64 +## near zero ## +np.float64,0x8000000000000000,0x3ff0000000000000,1 +np.float64,0x8010000000000000,0x3ff0000000000000,1 +np.float64,0x8000000000000001,0x3ff0000000000000,1 +np.float64,0x8360000000000000,0x3ff0000000000000,1 +np.float64,0x9a70000000000000,0x3ff0000000000000,1 +np.float64,0xb9b0000000000000,0x3ff0000000000000,1 +np.float64,0xb810000000000000,0x3ff0000000000000,1 +np.float64,0xbc30000000000000,0x3ff0000000000000,1 +np.float64,0xb6a0000000000000,0x3ff0000000000000,1 +np.float64,0x0000000000000000,0x3ff0000000000000,1 +np.float64,0x0010000000000000,0x3ff0000000000000,1 +np.float64,0x0000000000000001,0x3ff0000000000000,1 +np.float64,0x0360000000000000,0x3ff0000000000000,1 +np.float64,0x1a70000000000000,0x3ff0000000000000,1 +np.float64,0x3c30000000000000,0x3ff0000000000000,1 +np.float64,0x36a0000000000000,0x3ff0000000000000,1 +np.float64,0x39b0000000000000,0x3ff0000000000000,1 +np.float64,0x3810000000000000,0x3ff0000000000000,1 +## underflow ## +np.float64,0xc0c6276800000000,0x0000000000000000,1 +np.float64,0xc0c62d918ce2421d,0x0000000000000000,1 +np.float64,0xc0c62d918ce2421e,0x0000000000000000,1 +np.float64,0xc0c62d91a0000000,0x0000000000000000,1 +np.float64,0xc0c62d9180000000,0x0000000000000000,1 +np.float64,0xc0c62dea45ee3e06,0x0000000000000000,1 +np.float64,0xc0c62dea45ee3e07,0x0000000000000000,1 +np.float64,0xc0c62dea40000000,0x0000000000000000,1 +np.float64,0xc0c62dea60000000,0x0000000000000000,1 +np.float64,0xc0875f1120000000,0x0000000000000000,1 +np.float64,0xc0875f113c30b1c8,0x0000000000000000,1 +np.float64,0xc0875f1140000000,0x0000000000000000,1 +np.float64,0xc093480000000000,0x0000000000000000,1 +np.float64,0xffefffffffffffff,0x0000000000000000,1 +np.float64,0xc7efffffe0000000,0x0000000000000000,1 +## overflow ## +np.float64,0x40862e52fefa39ef,0x7ff0000000000000,1 +np.float64,0x40872e42fefa39ef,0x7ff0000000000000,1 +## +/- INF, +/- NAN ## +np.float64,0x7ff0000000000000,0x7ff0000000000000,1 +np.float64,0xfff0000000000000,0x0000000000000000,1 +np.float64,0x7ff8000000000000,0x7ff8000000000000,1 +np.float64,0xfff8000000000000,0xfff8000000000000,1 +## output denormal ## +np.float64,0xc087438520000000,0x0000000000000001,1 +np.float64,0xc08743853f2f4461,0x0000000000000001,1 +np.float64,0xc08743853f2f4460,0x0000000000000001,1 +np.float64,0xc087438540000000,0x0000000000000001,1 +## between -745.13321910 and 709.78271289 ## +np.float64,0xbff760cd14774bd9,0x3fcdb14ced00ceb6,1 +np.float64,0xbff760cd20000000,0x3fcdb14cd7993879,1 +np.float64,0xbff760cd00000000,0x3fcdb14d12fbd264,1 +np.float64,0xc07f1cf360000000,0x130c1b369af14fda,1 +np.float64,0xbeb0000000000000,0x3feffffe00001000,1 +np.float64,0xbd70000000000000,0x3fefffffffffe000,1 +np.float64,0xc084fd46e5c84952,0x0360000000000139,1 +np.float64,0xc084fd46e5c84953,0x035ffffffffffe71,1 +np.float64,0xc084fd46e0000000,0x0360000b9096d32c,1 +np.float64,0xc084fd4700000000,0x035fff9721d12104,1 +np.float64,0xc086232bc0000000,0x0010003af5e64635,1 +np.float64,0xc086232bdd7abcd2,0x001000000000007c,1 +np.float64,0xc086232bdd7abcd3,0x000ffffffffffe7c,1 +np.float64,0xc086232be0000000,0x000ffffaf57a6fc9,1 +np.float64,0xc086233920000000,0x000fe590e3b45eb0,1 +np.float64,0xc086233938000000,0x000fe56133493c57,1 +np.float64,0xc086233940000000,0x000fe5514deffbbc,1 +np.float64,0xc086234c98000000,0x000fbf1024c32ccb,1 +np.float64,0xc086234ca0000000,0x000fbf0065bae78d,1 +np.float64,0xc086234c80000000,0x000fbf3f623a7724,1 +np.float64,0xc086234ec0000000,0x000fbad237c846f9,1 +np.float64,0xc086234ec8000000,0x000fbac27cfdec97,1 +np.float64,0xc086234ee0000000,0x000fba934cfd3dc2,1 +np.float64,0xc086234ef0000000,0x000fba73d7f618d9,1 +np.float64,0xc086234f00000000,0x000fba54632dddc0,1 +np.float64,0xc0862356e0000000,0x000faae0945b761a,1 +np.float64,0xc0862356f0000000,0x000faac13eb9a310,1 +np.float64,0xc086235700000000,0x000faaa1e9567b0a,1 +np.float64,0xc086236020000000,0x000f98cd75c11ed7,1 +np.float64,0xc086236ca0000000,0x000f8081b4d93f89,1 +np.float64,0xc086236cb0000000,0x000f8062b3f4d6c5,1 +np.float64,0xc086236cc0000000,0x000f8043b34e6f8c,1 +np.float64,0xc086238d98000000,0x000f41220d9b0d2c,1 +np.float64,0xc086238da0000000,0x000f4112cc80a01f,1 +np.float64,0xc086238d80000000,0x000f414fd145db5b,1 +np.float64,0xc08624fd00000000,0x000cbfce8ea1e6c4,1 +np.float64,0xc086256080000000,0x000c250747fcd46e,1 +np.float64,0xc08626c480000000,0x000a34f4bd975193,1 +np.float64,0xbf50000000000000,0x3feff800ffeaac00,1 +np.float64,0xbe10000000000000,0x3fefffffff800000,1 +np.float64,0xbcd0000000000000,0x3feffffffffffff8,1 +np.float64,0xc055d589e0000000,0x38100004bf94f63e,1 +np.float64,0xc055d58a00000000,0x380ffff97f292ce8,1 +np.float64,0xbfd962d900000000,0x3fe585a4b00110e1,1 +np.float64,0x3ff4bed280000000,0x400d411e7a58a303,1 +np.float64,0x3fff0b3620000000,0x401bd7737ffffcf3,1 +np.float64,0x3ff0000000000000,0x4005bf0a8b145769,1 +np.float64,0x3eb0000000000000,0x3ff0000100000800,1 +np.float64,0x3d70000000000000,0x3ff0000000001000,1 +np.float64,0x40862e42e0000000,0x7fefff841808287f,1 +np.float64,0x40862e42fefa39ef,0x7fefffffffffff2a,1 +np.float64,0x40862e0000000000,0x7feef85a11e73f2d,1 +np.float64,0x4000000000000000,0x401d8e64b8d4ddae,1 +np.float64,0x4009242920000000,0x40372a52c383a488,1 +np.float64,0x4049000000000000,0x44719103e4080b45,1 +np.float64,0x4008000000000000,0x403415e5bf6fb106,1 +np.float64,0x3f50000000000000,0x3ff00400800aab55,1 +np.float64,0x3e10000000000000,0x3ff0000000400000,1 +np.float64,0x3cd0000000000000,0x3ff0000000000004,1 +np.float64,0x40562e40a0000000,0x47effed088821c3f,1 +np.float64,0x40562e42e0000000,0x47effff082e6c7ff,1 +np.float64,0x40562e4300000000,0x47f00000417184b8,1 +np.float64,0x3fe8000000000000,0x4000ef9db467dcf8,1 +np.float64,0x402b12e8d4f33589,0x412718f68c71a6fe,1 +np.float64,0x402b12e8d4f3358a,0x412718f68c71a70a,1 +np.float64,0x402b12e8c0000000,0x412718f59a7f472e,1 +np.float64,0x402b12e8e0000000,0x412718f70c0eac62,1 +##use 1th entry +np.float64,0x40631659AE147CB4,0x4db3a95025a4890f,1 +np.float64,0xC061B87D2E85A4E2,0x332640c8e2de2c51,1 +np.float64,0x405A4A50BE243AF4,0x496a45e4b7f0339a,1 +np.float64,0xC0839898B98EC5C6,0x0764027828830df4,1 +#use 2th entry +np.float64,0xC072428C44B6537C,0x2596ade838b96f3e,1 +np.float64,0xC053057C5E1AE9BF,0x3912c8fad18fdadf,1 +np.float64,0x407E89C78328BAA3,0x6bfe35d5b9a1a194,1 +np.float64,0x4083501B6DD87112,0x77a855503a38924e,1 +#use 3th entry +np.float64,0x40832C6195F24540,0x7741e73c80e5eb2f,1 +np.float64,0xC083D4CD557C2EC9,0x06b61727c2d2508e,1 +np.float64,0x400C48F5F67C99BD,0x404128820f02b92e,1 +np.float64,0x4056E36D9B2DF26A,0x4830f52ff34a8242,1 +#use 4th entry +np.float64,0x4080FF700D8CBD06,0x70fa70df9bc30f20,1 +np.float64,0x406C276D39E53328,0x543eb8e20a8f4741,1 +np.float64,0xC070D6159BBD8716,0x27a4a0548c904a75,1 +np.float64,0xC052EBCF8ED61F83,0x391c0e92368d15e4,1 +#use 5th entry +np.float64,0xC061F892A8AC5FBE,0x32f807a89efd3869,1 +np.float64,0x4021D885D2DBA085,0x40bd4dc86d3e3270,1 +np.float64,0x40767AEEEE7D4FCF,0x605e22851ee2afb7,1 +np.float64,0xC0757C5D75D08C80,0x20f0751599b992a2,1 +#use 6th entry +np.float64,0x405ACF7A284C4CE3,0x499a4e0b7a27027c,1 +np.float64,0xC085A6C9E80D7AF5,0x0175914009d62ec2,1 +np.float64,0xC07E4C02F86F1DAE,0x1439269b29a9231e,1 +np.float64,0x4080D80F9691CC87,0x7088a6cdafb041de,1 +#use 7th entry +np.float64,0x407FDFD84FBA0AC1,0x6deb1ae6f9bc4767,1 +np.float64,0x40630C06A1A2213D,0x4dac7a9d51a838b7,1 +np.float64,0x40685FDB30BB8B4F,0x5183f5cc2cac9e79,1 +np.float64,0x408045A2208F77F4,0x6ee299e08e2aa2f0,1 +#use 8th entry +np.float64,0xC08104E391F5078B,0x0ed397b7cbfbd230,1 +np.float64,0xC031501CAEFAE395,0x3e6040fd1ea35085,1 +np.float64,0xC079229124F6247C,0x1babf4f923306b1e,1 +np.float64,0x407FB65F44600435,0x6db03beaf2512b8a,1 +#use 9th entry +np.float64,0xC07EDEE8E8E8A5AC,0x136536cec9cbef48,1 +np.float64,0x4072BB4086099A14,0x5af4d3c3008b56cc,1 +np.float64,0x4050442A2EC42CB4,0x45cd393bd8fad357,1 +np.float64,0xC06AC28FB3D419B4,0x2ca1b9d3437df85f,1 +#use 10th entry +np.float64,0x40567FC6F0A68076,0x480c977fd5f3122e,1 +np.float64,0x40620A2F7EDA59BB,0x4cf278e96f4ce4d7,1 +np.float64,0xC085044707CD557C,0x034aad6c968a045a,1 +np.float64,0xC07374EA5AC516AA,0x23dd6afdc03e83d5,1 +#use 11th entry +np.float64,0x4073CC95332619C1,0x5c804b1498bbaa54,1 +np.float64,0xC0799FEBBE257F31,0x1af6a954c43b87d2,1 +np.float64,0x408159F19EA424F6,0x7200858efcbfc84d,1 +np.float64,0x404A81F6F24C0792,0x44b664a07ce5bbfa,1 +#use 12th entry +np.float64,0x40295FF1EFB9A741,0x4113c0e74c52d7b0,1 +np.float64,0x4073975F4CC411DA,0x5c32be40b4fec2c1,1 +np.float64,0x406E9DE52E82A77E,0x56049c9a3f1ae089,1 +np.float64,0x40748C2F52560ED9,0x5d93bc14fd4cd23b,1 +#use 13th entry +np.float64,0x4062A553CDC4D04C,0x4d6266bfde301318,1 +np.float64,0xC079EC1D63598AB7,0x1a88cb184dab224c,1 +np.float64,0xC0725C1CB3167427,0x25725b46f8a081f6,1 +np.float64,0x407888771D9B45F9,0x6353b1ec6bd7ce80,1 +#use 14th entry +np.float64,0xC082CBA03AA89807,0x09b383723831ce56,1 +np.float64,0xC083A8961BB67DD7,0x0735b118d5275552,1 +np.float64,0xC076BC6ECA12E7E3,0x1f2222679eaef615,1 +np.float64,0xC072752503AA1A5B,0x254eb832242c77e1,1 +#use 15th entry +np.float64,0xC058800792125DEC,0x371882372a0b48d4,1 +np.float64,0x4082909FD863E81C,0x7580d5f386920142,1 +np.float64,0xC071616F8FB534F9,0x26dbe20ef64a412b,1 +np.float64,0x406D1AB571CAA747,0x54ee0d55cb38ac20,1 +#use 16th entry +np.float64,0x406956428B7DAD09,0x52358682c271237f,1 +np.float64,0xC07EFC2D9D17B621,0x133b3e77c27a4d45,1 +np.float64,0xC08469BAC5BA3CCA,0x050863e5f42cc52f,1 +np.float64,0x407189D9626386A5,0x593cb1c0b3b5c1d3,1 +#use 17th entry +np.float64,0x4077E652E3DEB8C6,0x6269a10dcbd3c752,1 +np.float64,0x407674C97DB06878,0x605485dcc2426ec2,1 +np.float64,0xC07CE9969CF4268D,0x16386cf8996669f2,1 +np.float64,0x40780EE32D5847C4,0x62a436bd1abe108d,1 +#use 18th entry +np.float64,0x4076C3AA5E1E8DA1,0x60c62f56a5e72e24,1 +np.float64,0xC0730AFC7239B9BE,0x24758ead095cec1e,1 +np.float64,0xC085CC2B9C420DDB,0x0109cdaa2e5694c1,1 +np.float64,0x406D0765CB6D7AA4,0x54e06f8dd91bd945,1 +#use 19th entry +np.float64,0xC082D011F3B495E7,0x09a6647661d279c2,1 +np.float64,0xC072826AF8F6AFBC,0x253acd3cd224507e,1 +np.float64,0x404EB9C4810CEA09,0x457933dbf07e8133,1 +np.float64,0x408284FBC97C58CE,0x755f6eb234aa4b98,1 +#use 20th entry +np.float64,0x40856008CF6EDC63,0x7d9c0b3c03f4f73c,1 +np.float64,0xC077CB2E9F013B17,0x1d9b3d3a166a55db,1 +np.float64,0xC0479CA3C20AD057,0x3bad40e081555b99,1 +np.float64,0x40844CD31107332A,0x7a821d70aea478e2,1 +#use 21th entry +np.float64,0xC07C8FCC0BFCC844,0x16ba1cc8c539d19b,1 +np.float64,0xC085C4E9A3ABA488,0x011ff675ba1a2217,1 +np.float64,0x4074D538B32966E5,0x5dfd9d78043c6ad9,1 +np.float64,0xC0630CA16902AD46,0x3231a446074cede6,1 +#use 22th entry +np.float64,0xC06C826733D7D0B7,0x2b5f1078314d41e1,1 +np.float64,0xC0520DF55B2B907F,0x396c13a6ce8e833e,1 +np.float64,0xC080712072B0F437,0x107eae02d11d98ea,1 +np.float64,0x40528A6150E19EFB,0x469fdabda02228c5,1 +#use 23th entry +np.float64,0xC07B1D74B6586451,0x18d1253883ae3b48,1 +np.float64,0x4045AFD7867DAEC0,0x43d7d634fc4c5d98,1 +np.float64,0xC07A08B91F9ED3E2,0x1a60973e6397fc37,1 +np.float64,0x407B3ECF0AE21C8C,0x673e03e9d98d7235,1 +#use 24th entry +np.float64,0xC078AEB6F30CEABF,0x1c530b93ab54a1b3,1 +np.float64,0x4084495006A41672,0x7a775b6dc7e63064,1 +np.float64,0x40830B1C0EBF95DD,0x76e1e6eed77cfb89,1 +np.float64,0x407D93E8F33D8470,0x6a9adbc9e1e4f1e5,1 +#use 25th entry +np.float64,0x4066B11A09EFD9E8,0x504dd528065c28a7,1 +np.float64,0x408545823723AEEB,0x7d504a9b1844f594,1 +np.float64,0xC068C711F2CA3362,0x2e104f3496ea118e,1 +np.float64,0x407F317FCC3CA873,0x6cf0732c9948ebf4,1 +#use 26th entry +np.float64,0x407AFB3EBA2ED50F,0x66dc28a129c868d5,1 +np.float64,0xC075377037708ADE,0x21531a329f3d793e,1 +np.float64,0xC07C30066A1F3246,0x174448baa16ded2b,1 +np.float64,0xC06689A75DE2ABD3,0x2fad70662fae230b,1 +#use 27th entry +np.float64,0x4081514E9FCCF1E0,0x71e673b9efd15f44,1 +np.float64,0xC0762C710AF68460,0x1ff1ed7d8947fe43,1 +np.float64,0xC0468102FF70D9C4,0x3be0c3a8ff3419a3,1 +np.float64,0xC07EA4CEEF02A83E,0x13b908f085102c61,1 +#use 28th entry +np.float64,0xC06290B04AE823C4,0x328a83da3c2e3351,1 +np.float64,0xC0770EB1D1C395FB,0x1eab281c1f1db5fe,1 +np.float64,0xC06F5D4D838A5BAE,0x29500ea32fb474ea,1 +np.float64,0x40723B3133B54C5D,0x5a3c82c7c3a2b848,1 +#use 29th entry +np.float64,0x4085E6454CE3B4AA,0x7f20319b9638d06a,1 +np.float64,0x408389F2A0585D4B,0x7850667c58aab3d0,1 +np.float64,0xC0382798F9C8AE69,0x3dc1c79fe8739d6d,1 +np.float64,0xC08299D827608418,0x0a4335f76cdbaeb5,1 +#use 30th entry +np.float64,0xC06F3DED43301BF1,0x2965670ae46750a8,1 +np.float64,0xC070CAF6BDD577D9,0x27b4aa4ffdd29981,1 +np.float64,0x4078529AD4B2D9F2,0x6305c12755d5e0a6,1 +np.float64,0xC055B14E75A31B96,0x381c2eda6d111e5d,1 +#use 31th entry +np.float64,0x407B13EE414FA931,0x6700772c7544564d,1 +np.float64,0x407EAFDE9DE3EC54,0x6c346a0e49724a3c,1 +np.float64,0xC08362F398B9530D,0x07ffeddbadf980cb,1 +np.float64,0x407E865CDD9EEB86,0x6bf866cac5e0d126,1 +#use 32th entry +np.float64,0x407FB62DBC794C86,0x6db009f708ac62cb,1 +np.float64,0xC063D0BAA68CDDDE,0x31a3b2a51ce50430,1 +np.float64,0xC05E7706A2231394,0x34f24bead6fab5c9,1 +np.float64,0x4083E3A06FDE444E,0x79527b7a386d1937,1 diff --git a/numpy/core/tests/data/umath-validation-set-sin b/numpy/core/tests/data/umath-validation-set-sin index a56273195..64e78ae15 100644 --- a/numpy/core/tests/data/umath-validation-set-sin +++ b/numpy/core/tests/data/umath-validation-set-sin @@ -19,9 +19,7 @@ np.float32,0x80000001,0x80000001,2 np.float32,0x00000000,0x00000000,2 np.float32,0x80000000,0x80000000,2 np.float32,0x00800000,0x00800000,2 -np.float32,0x7f7fffff,0xbf0599b3,2 np.float32,0x80800000,0x80800000,2 -np.float32,0xff7fffff,0x3f0599b3,2 ## 1.00f ## np.float32,0x3f800000,0x3f576aa4,2 np.float32,0x3f800001,0x3f576aa6,2 @@ -36,26 +34,6 @@ np.float32,0x41d92388,0x3f67beef,2 np.float32,0x422dd66c,0xbeffb0c1,2 np.float32,0xc28f5be6,0xbf0bae79,2 np.float32,0x41ab2674,0x3f0ffe2b,2 -np.float32,0xd0102756,0x3f227e8a,2 -np.float32,0xcf99405e,0x3f73ad00,2 -np.float32,0xcfd83a12,0xbf7151a7,2 -np.float32,0x4fb54db0,0xbe46354b,2 -np.float32,0xcfcca29d,0xbe9345e6,2 -np.float32,0xceec2ac0,0x3e98dc89,2 -np.float32,0xcfdca97f,0xbf60b2b4,2 -np.float32,0xcfe92b0a,0xbf222705,2 -np.float32,0x5014b0eb,0x3f63e75c,2 -np.float32,0xcfa7ee96,0x3f62ada4,2 -np.float32,0x754c09a0,0xbf617056,2 -np.float32,0x77a731fb,0x3f44472b,2 -np.float32,0x76de2494,0xbe680739,2 -np.float32,0xf74920dc,0xbf193338,2 -np.float32,0x7707a312,0xbf6f51b1,2 -np.float32,0x75bf9790,0xbd0f1a47,2 -np.float32,0xf4ca7c40,0xbf7d45e7,2 -np.float32,0x77e91899,0x3f767181,2 -np.float32,0xf74c9820,0xbd685b75,2 -np.float32,0x7785ca29,0x3f78ee61,2 np.float32,0x3f490fdb,0x3f3504f3,2 np.float32,0xbf490fdb,0xbf3504f3,2 np.float32,0x3fc90fdb,0x3f800000,2 @@ -660,46 +638,21 @@ np.float32,0x4350ea79,0x3f800000,2 np.float32,0x42dbe957,0x3585522a,2 np.float32,0x425be957,0xbf800000,2 np.float32,0x435be957,0xb605522a,2 -np.float32,0x487fe5ab,0xbf7ffffd,2 -np.float32,0x497fe5ab,0xbb14017d,2 -np.float32,0x49ffe5ab,0xbb940164,2 -np.float32,0x49ffeb37,0x3f7fff56,2 -np.float32,0x497ff0c3,0x3f7fffd6,2 -np.float32,0x49fff0c3,0x3b930487,2 -np.float32,0x49fff64f,0xbf7fff58,2 -np.float32,0x497ffbdb,0x3b1207c0,2 -np.float32,0x49fffbdb,0xbb9207a9,2 -np.float32,0x48fffbdb,0xbf7ffff6,2 -np.float32,0x4e736e56,0x397fa7f2,2 -np.float32,0x4d4da377,0xb57c64bc,2 -np.float32,0x4ece58c3,0xb80846c8,2 -np.float32,0x4ee0db9c,0x394c4786,2 -np.float32,0x4dee7002,0x381bce96,2 -np.float32,0x4ee86afc,0x3f800000,2 -np.float32,0x4dca4f3f,0xb8e25111,2 -np.float32,0x4ecb48af,0xbf800000,2 -np.float32,0x4e51e33f,0xb8a4fa6f,2 -np.float32,0x4ef5f421,0x387ca7df,2 np.float32,0x476362a2,0xbd7ff911,2 np.float32,0x464c99a4,0x3e7f4d41,2 np.float32,0x4471f73d,0x3e7fe1b0,2 np.float32,0x445a6752,0x3e7ef367,2 np.float32,0x474fa400,0x3e7f9fcd,2 -np.float32,0x47c9e70e,0xbb4bba09,2 np.float32,0x45c1e72f,0xbe7fc7af,2 np.float32,0x4558c91d,0x3e7e9f31,2 np.float32,0x43784f94,0xbdff6654,2 np.float32,0x466e8500,0xbe7ea0a3,2 np.float32,0x468e1c25,0x3e7e22fb,2 -np.float32,0x47d28adc,0xbe7d5e6b,2 np.float32,0x44ea6cfc,0x3dff70c3,2 np.float32,0x4605126c,0x3e7f89ef,2 np.float32,0x4788b3c6,0xbb87d853,2 np.float32,0x4531b042,0x3dffd163,2 -np.float32,0x47e46c29,0xbe7def2b,2 -np.float32,0x47c10e07,0xbdff63d4,2 np.float32,0x43f1f71d,0x3dfff387,2 -np.float32,0x47c3e38c,0x3e7f0b2f,2 np.float32,0x462c3fa5,0xbd7fe13d,2 np.float32,0x441c5354,0xbdff76b4,2 np.float32,0x44908b69,0x3e7dcf0d,2 diff --git a/numpy/core/tests/test_arrayprint.py b/numpy/core/tests/test_arrayprint.py index 96240be0f..e29217461 100644 --- a/numpy/core/tests/test_arrayprint.py +++ b/numpy/core/tests/test_arrayprint.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import sys import gc +from hypothesis import given +from hypothesis.extra import numpy as hynp import pytest import numpy as np @@ -393,6 +395,18 @@ class TestArray2String: "[ 'xxxxx']" ) + @given(hynp.from_dtype(np.dtype("U"))) + def test_any_text(self, text): + # This test checks that, given any value that can be represented in an + # array of dtype("U") (i.e. unicode string), ... + a = np.array([text, text, text]) + # casting a list of them to an array does not e.g. truncate the value + assert_equal(a[0], text) + # and that np.array2string puts a newline in the expected location + expected_repr = "[{0!r} {0!r}\n {0!r}]".format(text) + result = np.array2string(a, max_line_width=len(repr(text)) * 2 + 3) + assert_equal(result, expected_repr) + @pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") def test_refcount(self): # make sure we do not hold references to the array due to a recursive @@ -461,9 +475,7 @@ class TestPrintOptions: assert_equal(repr(x), "array([0., 1., 2.])") def test_0d_arrays(self): - unicode = type(u'') - - assert_equal(unicode(np.array(u'café', '<U4')), u'café') + assert_equal(str(np.array(u'café', '<U4')), u'café') assert_equal(repr(np.array('café', '<U4')), "array('café', dtype='<U4')") @@ -837,7 +849,6 @@ class TestPrintOptions: assert_raises(TypeError, np.set_printoptions, threshold=b'1') def test_unicode_object_array(): - import sys expected = "array(['é'], dtype=object)" x = np.array([u'\xe9'], dtype=object) assert_equal(repr(x), expected) diff --git a/numpy/core/tests/test_conversion_utils.py b/numpy/core/tests/test_conversion_utils.py new file mode 100644 index 000000000..3c3f9e6e1 --- /dev/null +++ b/numpy/core/tests/test_conversion_utils.py @@ -0,0 +1,156 @@ +""" +Tests for numpy/core/src/multiarray/conversion_utils.c +""" +import re + +import pytest + +import numpy as np +import numpy.core._multiarray_tests as mt + + +class StringConverterTestCase: + allow_bytes = True + case_insensitive = True + exact_match = False + + def _check_value_error(self, val): + pattern = r'\(got {}\)'.format(re.escape(repr(val))) + with pytest.raises(ValueError, match=pattern) as exc: + self.conv(val) + + def _check(self, val, expected): + assert self.conv(val) == expected + + if self.allow_bytes: + assert self.conv(val.encode('ascii')) == expected + else: + with pytest.raises(TypeError): + self.conv(val.encode('ascii')) + + if len(val) != 1: + if self.exact_match: + self._check_value_error(val[:1]) + self._check_value_error(val + '\0') + else: + assert self.conv(val[:1]) == expected + + if self.case_insensitive: + if val != val.lower(): + assert self.conv(val.lower()) == expected + if val != val.upper(): + assert self.conv(val.upper()) == expected + else: + if val != val.lower(): + self._check_value_error(val.lower()) + if val != val.upper(): + self._check_value_error(val.upper()) + + def test_wrong_type(self): + # common cases which apply to all the below + with pytest.raises(TypeError): + self.conv({}) + with pytest.raises(TypeError): + self.conv([]) + + def test_wrong_value(self): + # nonsense strings + self._check_value_error('') + self._check_value_error('\N{greek small letter pi}') + + if self.allow_bytes: + self._check_value_error(b'') + # bytes which can't be converted to strings via utf8 + self._check_value_error(b"\xFF") + if self.exact_match: + self._check_value_error("there's no way this is supported") + + +class TestByteorderConverter(StringConverterTestCase): + """ Tests of PyArray_ByteorderConverter """ + conv = mt.run_byteorder_converter + def test_valid(self): + for s in ['big', '>']: + self._check(s, 'NPY_BIG') + for s in ['little', '<']: + self._check(s, 'NPY_LITTLE') + for s in ['native', '=']: + self._check(s, 'NPY_NATIVE') + for s in ['ignore', '|']: + self._check(s, 'NPY_IGNORE') + for s in ['swap']: + self._check(s, 'NPY_SWAP') + + +class TestSortkindConverter(StringConverterTestCase): + """ Tests of PyArray_SortkindConverter """ + conv = mt.run_sortkind_converter + def test_valid(self): + self._check('quick', 'NPY_QUICKSORT') + self._check('heap', 'NPY_HEAPSORT') + self._check('merge', 'NPY_STABLESORT') # alias + self._check('stable', 'NPY_STABLESORT') + + +class TestSelectkindConverter(StringConverterTestCase): + """ Tests of PyArray_SelectkindConverter """ + conv = mt.run_selectkind_converter + case_insensitive = False + exact_match = True + + def test_valid(self): + self._check('introselect', 'NPY_INTROSELECT') + + +class TestSearchsideConverter(StringConverterTestCase): + """ Tests of PyArray_SearchsideConverter """ + conv = mt.run_searchside_converter + def test_valid(self): + self._check('left', 'NPY_SEARCHLEFT') + self._check('right', 'NPY_SEARCHRIGHT') + + +class TestOrderConverter(StringConverterTestCase): + """ Tests of PyArray_OrderConverter """ + conv = mt.run_order_converter + def test_valid(self): + self._check('c', 'NPY_CORDER') + self._check('f', 'NPY_FORTRANORDER') + self._check('a', 'NPY_ANYORDER') + self._check('k', 'NPY_KEEPORDER') + + def test_flatten_invalid_order(self): + # invalid after gh-14596 + with pytest.raises(ValueError): + self.conv('Z') + for order in [False, True, 0, 8]: + with pytest.raises(TypeError): + self.conv(order) + + +class TestClipmodeConverter(StringConverterTestCase): + """ Tests of PyArray_ClipmodeConverter """ + conv = mt.run_clipmode_converter + def test_valid(self): + self._check('clip', 'NPY_CLIP') + self._check('wrap', 'NPY_WRAP') + self._check('raise', 'NPY_RAISE') + + # integer values allowed here + assert self.conv(np.CLIP) == 'NPY_CLIP' + assert self.conv(np.WRAP) == 'NPY_WRAP' + assert self.conv(np.RAISE) == 'NPY_RAISE' + + +class TestCastingConverter(StringConverterTestCase): + """ Tests of PyArray_CastingConverter """ + conv = mt.run_casting_converter + case_insensitive = False + exact_match = True + + def test_valid(self): + self._check("no", "NPY_NO_CASTING") + self._check("equiv", "NPY_EQUIV_CASTING") + self._check("safe", "NPY_SAFE_CASTING") + self._check("same_kind", "NPY_SAME_KIND_CASTING") + self._check("unsafe", "NPY_UNSAFE_CASTING") diff --git a/numpy/core/tests/test_cpu_features.py b/numpy/core/tests/test_cpu_features.py new file mode 100644 index 000000000..3b5cb3157 --- /dev/null +++ b/numpy/core/tests/test_cpu_features.py @@ -0,0 +1,104 @@ +import sys, platform, re, pytest + +from numpy.testing import assert_equal +from numpy.core._multiarray_umath import __cpu_features__ + +class AbstractTest(object): + features = [] + features_groups = {} + features_map = {} + features_flags = set() + + def load_flags(self): + # a hook + pass + + def test_features(self): + self.load_flags() + for gname, features in self.features_groups.items(): + test_features = [self.features_map.get(f, f) in self.features_flags for f in features] + assert_equal(__cpu_features__.get(gname), all(test_features)) + + for feature_name in self.features: + map_name = self.features_map.get(feature_name, feature_name) + cpu_have = map_name in self.features_flags + npy_have = __cpu_features__.get(feature_name) + assert_equal(npy_have, cpu_have) + + def load_flags_proc(self, magic_key): + with open('/proc/cpuinfo') as fd: + for line in fd: + if not line.startswith(magic_key): + continue + flags_value = [s.strip() for s in line.split(':', 1)] + if len(flags_value) == 2: + self.features_flags = self.features_flags.union(flags_value[1].upper().split()) + + def load_flags_auxv(self): + import subprocess + auxv = subprocess.check_output(['/bin/true'], env=dict(LD_SHOW_AUXV="1")) + for at in auxv.split(b'\n'): + if not at.startswith(b"AT_HWCAP"): + continue + hwcap_value = [s.strip() for s in at.split(b':', 1)] + if len(hwcap_value) == 2: + self.features_flags = self.features_flags.union( + hwcap_value[1].upper().decode().split() + ) + +is_linux = sys.platform.startswith('linux') +machine = platform.machine() +is_x86 = re.match("^(amd64|x86|i386|i686)", machine, re.IGNORECASE) +@pytest.mark.skipif(not is_linux or not is_x86, reason="Only for Linux and x86") +class Test_X86_Features(AbstractTest): + features = [ + "MMX", "SSE", "SSE2", "SSE3", "SSSE3", "SSE41", "POPCNT", "SSE42", + "AVX", "F16C", "XOP", "FMA4", "FMA3", "AVX2", "AVX512F", "AVX512CD", + "AVX512ER", "AVX512PF", "AVX5124FMAPS", "AVX5124VNNIW", "AVX512VPOPCNTDQ", + "AVX512VL", "AVX512BW", "AVX512DQ", "AVX512VNNI", "AVX512IFMA", + "AVX512VBMI", "AVX512VBMI2", "AVX512BITALG", + ] + features_groups = dict( + AVX512_KNL = ["AVX512F", "AVX512CD", "AVX512ER", "AVX512PF"], + AVX512_KNM = ["AVX512F", "AVX512CD", "AVX512ER", "AVX512PF", "AVX5124FMAPS", + "AVX5124VNNIW", "AVX512VPOPCNTDQ"], + AVX512_SKX = ["AVX512F", "AVX512CD", "AVX512BW", "AVX512DQ", "AVX512VL"], + AVX512_CLX = ["AVX512F", "AVX512CD", "AVX512BW", "AVX512DQ", "AVX512VL", "AVX512VNNI"], + AVX512_CNL = ["AVX512F", "AVX512CD", "AVX512BW", "AVX512DQ", "AVX512VL", "AVX512IFMA", + "AVX512VBMI"], + AVX512_ICL = ["AVX512F", "AVX512CD", "AVX512BW", "AVX512DQ", "AVX512VL", "AVX512IFMA", + "AVX512VBMI", "AVX512VNNI", "AVX512VBMI2", "AVX512BITALG", "AVX512VPOPCNTDQ"], + ) + features_map = dict( + SSE3="PNI", SSE41="SSE4_1", SSE42="SSE4_2", FMA3="FMA", + AVX512VNNI="AVX512_VNNI", AVX512BITALG="AVX512_BITALG", AVX512VBMI2="AVX512_VBMI2", + AVX5124FMAPS="AVX512_4FMAPS", AVX5124VNNIW="AVX512_4VNNIW", AVX512VPOPCNTDQ="AVX512_VPOPCNTDQ", + ) + def load_flags(self): + self.load_flags_proc("flags") + +is_power = re.match("^(powerpc|ppc)64", machine, re.IGNORECASE) +@pytest.mark.skipif(not is_linux or not is_power, reason="Only for Linux and Power") +class Test_POWER_Features(AbstractTest): + features = ["VSX", "VSX2", "VSX3"] + features_map = dict(VSX2="ARCH_2_07", VSX3="ARCH_3_00") + + def load_flags(self): + self.load_flags_auxv() + +is_arm = re.match("^(arm|aarch64)", machine, re.IGNORECASE) +@pytest.mark.skipif(not is_linux or not is_arm, reason="Only for Linux and ARM") +class Test_ARM_Features(AbstractTest): + features = [ + "NEON", "ASIMD", "FPHP", "ASIMDHP", "ASIMDDP", "ASIMDFHM" + ] + features_groups = dict( + NEON_FP16 = ["NEON", "HALF"], + NEON_VFPV4 = ["NEON", "VFPV4"], + ) + def load_flags(self): + self.load_flags_proc("Features") + if re.match("^(aarch64|AARCH64)", platform.machine()): + self.features_map = dict( + NEON="ASIMD", HALF="ASIMD", VFPV4="ASIMD" + ) diff --git a/numpy/core/tests/test_defchararray.py b/numpy/core/tests/test_defchararray.py index 4c9016c3e..bbb94f7d3 100644 --- a/numpy/core/tests/test_defchararray.py +++ b/numpy/core/tests/test_defchararray.py @@ -1,10 +1,9 @@ -import sys import numpy as np from numpy.core.multiarray import _vec_string from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_raises, - assert_raises_regex, suppress_warnings, + assert_raises_regex ) kw_unicode_true = {'unicode': True} # make 2to3 work properly @@ -120,14 +119,14 @@ class TestVecString: def test_invalid_result_type(self): def fail(): - _vec_string(['a'], np.integer, 'strip') + _vec_string(['a'], np.int_, 'strip') assert_raises(TypeError, fail) def test_broadcast_error(self): def fail(): - _vec_string([['abc', 'def']], np.integer, 'find', (['a', 'd', 'j'],)) + _vec_string([['abc', 'def']], np.int_, 'find', (['a', 'd', 'j'],)) assert_raises(ValueError, fail) diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 252133d7b..82d24e0f7 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -4,12 +4,11 @@ to document how deprecations should eventually be turned into errors. """ import datetime -import sys import operator import warnings import pytest -import shutil import tempfile +import re import numpy as np from numpy.testing import ( @@ -549,3 +548,73 @@ def test_deprecate_ragged_arrays(): with assert_warns(np.VisibleDeprecationWarning): np.array(arg) + +class TestToString(_DeprecationTestCase): + # 2020-03-06 1.19.0 + message = re.escape("tostring() is deprecated. Use tobytes() instead.") + + def test_tostring(self): + arr = np.array(list(b"test\xFF"), dtype=np.uint8) + self.assert_deprecated(arr.tostring) + + def test_tostring_matches_tobytes(self): + arr = np.array(list(b"test\xFF"), dtype=np.uint8) + b = arr.tobytes() + with assert_warns(DeprecationWarning): + s = arr.tostring() + assert s == b + + +class TestDTypeCoercion(_DeprecationTestCase): + # 2020-02-06 1.19.0 + message = "Converting .* to a dtype .*is deprecated" + deprecated_types = [ + # The builtin scalar super types: + np.generic, np.flexible, np.number, + np.inexact, np.floating, np.complexfloating, + np.integer, np.unsignedinteger, np.signedinteger, + # character is a deprecated S1 special case: + np.character, + ] + + def test_dtype_coercion(self): + for scalar_type in self.deprecated_types: + self.assert_deprecated(np.dtype, args=(scalar_type,)) + + def test_array_construction(self): + for scalar_type in self.deprecated_types: + self.assert_deprecated(np.array, args=([], scalar_type,)) + + def test_not_deprecated(self): + # All specific types are not deprecated: + for group in np.sctypes.values(): + for scalar_type in group: + self.assert_not_deprecated(np.dtype, args=(scalar_type,)) + + for scalar_type in [type, dict, list, tuple]: + # Typical python types are coerced to object currently: + self.assert_not_deprecated(np.dtype, args=(scalar_type,)) + + +class BuiltInRoundComplexDType(_DeprecationTestCase): + # 2020-03-31 1.19.0 + deprecated_types = [np.csingle, np.cdouble, np.clongdouble] + not_deprecated_types = [ + np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64, + np.float16, np.float32, np.float64, + ] + + def test_deprecated(self): + for scalar_type in self.deprecated_types: + scalar = scalar_type(0) + self.assert_deprecated(round, args=(scalar,)) + self.assert_deprecated(round, args=(scalar, 0)) + self.assert_deprecated(round, args=(scalar,), kwargs={'ndigits': 0}) + + def test_not_deprecated(self): + for scalar_type in self.not_deprecated_types: + scalar = scalar_type(0) + self.assert_not_deprecated(round, args=(scalar,)) + self.assert_not_deprecated(round, args=(scalar, 0)) + self.assert_not_deprecated(round, args=(scalar,), kwargs={'ndigits': 0}) diff --git a/numpy/core/tests/test_einsum.py b/numpy/core/tests/test_einsum.py index 6ec61fb1d..68491681a 100644 --- a/numpy/core/tests/test_einsum.py +++ b/numpy/core/tests/test_einsum.py @@ -27,7 +27,7 @@ class TestEinsum: optimize=do_opt) # order parameter must be a valid order - assert_raises(TypeError, np.einsum, "", 0, order='W', + assert_raises(ValueError, np.einsum, "", 0, order='W', optimize=do_opt) # casting parameter must be a valid casting @@ -605,6 +605,10 @@ class TestEinsum: [[[1, 3], [3, 9], [5, 15], [7, 21]], [[8, 16], [16, 32], [24, 48], [32, 64]]]) + # Ensure explicitly setting out=None does not cause an error + # see issue gh-15776 and issue gh-15256 + assert_equal(np.einsum('i,j', [1], [2], out=None), [[2]]) + def test_subscript_range(self): # Issue #7741, make sure that all letters of Latin alphabet (both uppercase & lowercase) can be used # when creating a subscript from arrays diff --git a/numpy/core/tests/test_errstate.py b/numpy/core/tests/test_errstate.py index 7c1780607..184a37300 100644 --- a/numpy/core/tests/test_errstate.py +++ b/numpy/core/tests/test_errstate.py @@ -1,12 +1,19 @@ -import platform import pytest +import sysconfig import numpy as np from numpy.testing import assert_, assert_raises +# The floating point emulation on ARM EABI systems lacking a hardware FPU is +# known to be buggy. This is an attempt to identify these hosts. It may not +# catch all possible cases, but it catches the known cases of gh-413 and +# gh-15562. +hosttype = sysconfig.get_config_var('HOST_GNU_TYPE') +arm_softfloat = False if hosttype is None else hosttype.endswith('gnueabi') class TestErrstate: - @pytest.mark.skipif(platform.machine() == "armv5tel", reason="See gh-413.") + @pytest.mark.skipif(arm_softfloat, + reason='platform/cpu issue with FPU (gh-413,-15562)') def test_invalid(self): with np.errstate(all='raise', under='ignore'): a = -np.arange(3) @@ -17,6 +24,8 @@ class TestErrstate: with assert_raises(FloatingPointError): np.sqrt(a) + @pytest.mark.skipif(arm_softfloat, + reason='platform/cpu issue with FPU (gh-15562)') def test_divide(self): with np.errstate(all='raise', under='ignore'): a = -np.arange(3) @@ -26,6 +35,9 @@ class TestErrstate: # While this should fail! with assert_raises(FloatingPointError): a // 0 + # As should this, see gh-15562 + with assert_raises(FloatingPointError): + a // a def test_errcall(self): def foo(*args): diff --git a/numpy/core/tests/test_function_base.py b/numpy/core/tests/test_function_base.py index 7e5ea1cc6..2197ef0cd 100644 --- a/numpy/core/tests/test_function_base.py +++ b/numpy/core/tests/test_function_base.py @@ -4,7 +4,6 @@ from numpy import ( ) from numpy.testing import ( assert_, assert_equal, assert_raises, assert_array_equal, assert_allclose, - suppress_warnings ) diff --git a/numpy/core/tests/test_half.py b/numpy/core/tests/test_half.py index c6d046be1..ae9827bc7 100644 --- a/numpy/core/tests/test_half.py +++ b/numpy/core/tests/test_half.py @@ -329,6 +329,7 @@ class TestHalf: # All non-negative finite #'s a = np.arange(0x7c00, dtype=uint16) hinf = np.array((np.inf,), dtype=float16) + hnan = np.array((np.nan,), dtype=float16) a_f16 = a.view(dtype=float16) assert_equal(np.spacing(a_f16[:-1]), a_f16[1:]-a_f16[:-1]) @@ -337,6 +338,21 @@ class TestHalf: assert_equal(np.nextafter(a_f16[0], -hinf), -a_f16[1]) assert_equal(np.nextafter(a_f16[1:], -hinf), a_f16[:-1]) + assert_equal(np.nextafter(hinf, a_f16), a_f16[-1]) + assert_equal(np.nextafter(-hinf, a_f16), -a_f16[-1]) + + assert_equal(np.nextafter(hinf, hinf), hinf) + assert_equal(np.nextafter(hinf, -hinf), a_f16[-1]) + assert_equal(np.nextafter(-hinf, hinf), -a_f16[-1]) + assert_equal(np.nextafter(-hinf, -hinf), -hinf) + + assert_equal(np.nextafter(a_f16, hnan), hnan[0]) + assert_equal(np.nextafter(hnan, a_f16), hnan[0]) + + assert_equal(np.nextafter(hnan, hnan), hnan) + assert_equal(np.nextafter(hinf, hnan), hnan) + assert_equal(np.nextafter(hnan, hinf), hnan) + # switch to negatives a |= 0x8000 @@ -347,6 +363,12 @@ class TestHalf: assert_equal(np.nextafter(a_f16[1:], hinf), a_f16[:-1]) assert_equal(np.nextafter(a_f16[:-1], -hinf), a_f16[1:]) + assert_equal(np.nextafter(hinf, a_f16), -a_f16[-1]) + assert_equal(np.nextafter(-hinf, a_f16), a_f16[-1]) + + assert_equal(np.nextafter(a_f16, hnan), hnan[0]) + assert_equal(np.nextafter(hnan, a_f16), hnan[0]) + def test_half_ufuncs(self): """Test the various ufuncs""" @@ -487,9 +509,6 @@ class TestHalf: assert_raises_fpe('invalid', np.divide, float16(np.inf), float16(np.inf)) assert_raises_fpe('invalid', np.spacing, float16(np.inf)) assert_raises_fpe('invalid', np.spacing, float16(np.nan)) - assert_raises_fpe('invalid', np.nextafter, float16(np.inf), float16(0)) - assert_raises_fpe('invalid', np.nextafter, float16(-np.inf), float16(0)) - assert_raises_fpe('invalid', np.nextafter, float16(0), float16(np.nan)) # These should not raise float16(65472)+float16(32) @@ -498,6 +517,10 @@ class TestHalf: np.spacing(float16(-65504)) np.nextafter(float16(65504), float16(-np.inf)) np.nextafter(float16(-65504), float16(np.inf)) + np.nextafter(float16(np.inf), float16(0)) + np.nextafter(float16(-np.inf), float16(0)) + np.nextafter(float16(0), float16(np.nan)) + np.nextafter(float16(np.nan), float16(0)) float16(2**-14)/float16(2**10) float16(-2**-14)/float16(2**10) float16(2**-14+2**-23)/float16(2) diff --git a/numpy/core/tests/test_indexerrors.py b/numpy/core/tests/test_indexerrors.py index 9d2433fc5..a0e9a8c55 100644 --- a/numpy/core/tests/test_indexerrors.py +++ b/numpy/core/tests/test_indexerrors.py @@ -1,5 +1,8 @@ import numpy as np -from numpy.testing import assert_raises +from numpy.testing import ( + assert_raises, assert_raises_regex, + ) + class TestIndexErrors: '''Tests to exercise indexerrors not covered by other tests.''' @@ -110,6 +113,15 @@ class TestIndexErrors: assert_raises(IndexError, lambda: a[(1, [0, 1])]) assert_raises(IndexError, lambda: assign(a, (1, [0, 1]), 1)) + def test_mapping_error_message(self): + a = np.zeros((3, 5)) + index = (1, 2, 3, 4, 5) + assert_raises_regex( + IndexError, + "too many indices for array: " + "array is 2-dimensional, but 5 were indexed", + lambda: a[index]) + def test_methods(self): "cases from methods.c" diff --git a/numpy/core/tests/test_indexing.py b/numpy/core/tests/test_indexing.py index 237e381a7..4bb5cb11a 100644 --- a/numpy/core/tests/test_indexing.py +++ b/numpy/core/tests/test_indexing.py @@ -2,7 +2,6 @@ import sys import warnings import functools import operator -import pytest import numpy as np from numpy.core._multiarray_tests import array_indexing diff --git a/numpy/core/tests/test_longdouble.py b/numpy/core/tests/test_longdouble.py index bf12f0e1b..acef995f3 100644 --- a/numpy/core/tests/test_longdouble.py +++ b/numpy/core/tests/test_longdouble.py @@ -37,22 +37,36 @@ def test_repr_roundtrip(): assert_equal(np.longdouble(repr(o)), o, "repr was %s" % repr(o)) -def test_unicode(): - np.longdouble(u"1.2") +@pytest.mark.skipif(string_to_longdouble_inaccurate, reason="Need strtold_l") +def test_repr_roundtrip_bytes(): + o = 1 + LD_INFO.eps + assert_equal(np.longdouble(repr(o).encode("ascii")), o) -def test_string(): - np.longdouble("1.2") +@pytest.mark.skipif(string_to_longdouble_inaccurate, reason="Need strtold_l") +@pytest.mark.parametrize("strtype", (np.str_, np.bytes_, str, bytes)) +def test_array_and_stringlike_roundtrip(strtype): + """ + Test that string representations of long-double roundtrip both + for array casting and scalar coercion, see also gh-15608. + """ + o = 1 + LD_INFO.eps + if strtype in (np.bytes_, bytes): + o_str = strtype(repr(o).encode("ascii")) + else: + o_str = strtype(repr(o)) -def test_bytes(): - np.longdouble(b"1.2") + # Test that `o` is correctly coerced from the string-like + assert o == np.longdouble(o_str) + # Test that arrays also roundtrip correctly: + o_strarr = np.asarray([o] * 3, dtype=strtype) + assert (o == o_strarr.astype(np.longdouble)).all() -@pytest.mark.skipif(string_to_longdouble_inaccurate, reason="Need strtold_l") -def test_repr_roundtrip_bytes(): - o = 1 + LD_INFO.eps - assert_equal(np.longdouble(repr(o).encode("ascii")), o) + # And array coercion and casting to string give the same as scalar repr: + assert (o_strarr == o_str).all() + assert (np.asarray([o] * 3).astype(strtype) == o_str).all() def test_bogus_string(): diff --git a/numpy/core/tests/test_mem_overlap.py b/numpy/core/tests/test_mem_overlap.py index 7c1cff9b7..675613de4 100644 --- a/numpy/core/tests/test_mem_overlap.py +++ b/numpy/core/tests/test_mem_overlap.py @@ -1,4 +1,3 @@ -import sys import itertools import pytest @@ -6,7 +5,6 @@ import numpy as np from numpy.core._multiarray_tests import solve_diophantine, internal_overlap from numpy.core import _umath_tests from numpy.lib.stride_tricks import as_strided -from numpy.compat import long from numpy.testing import ( assert_, assert_raises, assert_equal, assert_array_equal ) @@ -42,9 +40,7 @@ def _indices_for_axis(): res = [] for nelems in (0, 2, 3): ind = _indices_for_nelems(nelems) - - # no itertools.product available in Py2.4 - res.extend([(a, b) for a in ind for b in ind]) # all assignments of size "nelems" + res.extend(itertools.product(ind, ind)) # all assignments of size "nelems" return res @@ -53,18 +49,7 @@ def _indices(ndims): """Returns ((axis0_src, axis0_dst), (axis1_src, axis1_dst), ... ) index pairs.""" ind = _indices_for_axis() - - # no itertools.product available in Py2.4 - - res = [[]] - for i in range(ndims): - newres = [] - for elem in ind: - for others in res: - newres.append([elem] + others) - res = newres - - return res + return itertools.product(ind, repeat=ndims) def _check_assignment(srcidx, dstidx): @@ -401,7 +386,6 @@ def test_shares_memory_api(): assert_equal(np.shares_memory(a, b), True) assert_equal(np.shares_memory(a, b, max_work=None), True) assert_raises(np.TooHardError, np.shares_memory, a, b, max_work=1) - assert_raises(np.TooHardError, np.shares_memory, a, b, max_work=long(1)) def test_may_share_memory_bad_max_work(): @@ -739,6 +723,7 @@ class TestUFunc: a = np.arange(10000, dtype=np.int16) check(np.add, a, a[::-1], a) + @pytest.mark.slow def test_unary_gufunc_fuzz(self): shapes = [7, 13, 8, 21, 29, 32] gufunc = _umath_tests.euclidean_pdist diff --git a/numpy/core/tests/test_memmap.py b/numpy/core/tests/test_memmap.py index bae7a318a..feef80ce8 100644 --- a/numpy/core/tests/test_memmap.py +++ b/numpy/core/tests/test_memmap.py @@ -3,11 +3,11 @@ import os import shutil import mmap import pytest +from pathlib import Path from tempfile import NamedTemporaryFile, TemporaryFile, mktemp, mkdtemp from numpy import ( memmap, sum, average, product, ndarray, isscalar, add, subtract, multiply) -from numpy.compat import Path from numpy import arange, allclose, asarray from numpy.testing import ( @@ -74,7 +74,6 @@ class TestMemmap: del b del fp - @pytest.mark.skipif(Path is None, reason="No pathlib.Path") def test_path(self): tmpname = mktemp('', 'mmap', dir=self.tempdir) fp = memmap(Path(tmpname), dtype=self.dtype, mode='w+', diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 270daad1e..f36c27c6c 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1,9 +1,4 @@ -try: - # Accessing collections abstract classes from collections - # has been deprecated since Python 3.3 - import collections.abc as collections_abc -except ImportError: - import collections as collections_abc +import collections.abc import tempfile import sys import shutil @@ -21,19 +16,12 @@ from contextlib import contextmanager from numpy.compat import pickle -try: - import pathlib -except ImportError: - try: - import pathlib2 as pathlib - except ImportError: - pathlib = None - +import pathlib import builtins from decimal import Decimal import numpy as np -from numpy.compat import strchar, unicode +from numpy.compat import strchar import numpy.core._multiarray_tests as _multiarray_tests from numpy.testing import ( assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal, @@ -368,6 +356,11 @@ class TestAttributes: a.strides = 1 a[::2].strides = 2 + # test 0d + arr_0d = np.array(0) + arr_0d.strides = () + assert_raises(TypeError, set_strides, arr_0d, None) + def test_fill(self): for t in "?bhilqpBHILQPfdgFDGO": x = np.empty((3, 2, 1), t) @@ -691,6 +684,12 @@ class TestZeroRank: y[()] = 6 assert_equal(x[()], 6) + # strides and shape must be the same length + with pytest.raises(ValueError): + np.ndarray((2,), strides=()) + with pytest.raises(ValueError): + np.ndarray((), strides=(2,)) + def test_output(self): x = np.array(2) assert_raises(ValueError, np.add, x, [1], x) @@ -1469,12 +1468,12 @@ class TestZeroSizeFlexible: assert_equal(zs.itemsize, 0) zs = self._zeros(10, np.void) assert_equal(zs.itemsize, 0) - zs = self._zeros(10, unicode) + zs = self._zeros(10, str) assert_equal(zs.itemsize, 0) def _test_sort_partition(self, name, kinds, **kwargs): # Previously, these would all hang - for dt in [bytes, np.void, unicode]: + for dt in [bytes, np.void, str]: zs = self._zeros(10, dt) sort_method = getattr(zs, name) sort_func = getattr(np, name) @@ -1496,13 +1495,13 @@ class TestZeroSizeFlexible: def test_resize(self): # previously an error - for dt in [bytes, np.void, unicode]: + for dt in [bytes, np.void, str]: zs = self._zeros(10, dt) zs.resize(25) zs.resize((10, 10)) def test_view(self): - for dt in [bytes, np.void, unicode]: + for dt in [bytes, np.void, str]: zs = self._zeros(10, dt) # viewing as itself should be allowed @@ -1517,7 +1516,7 @@ class TestZeroSizeFlexible: def test_pickle(self): for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): - for dt in [bytes, np.void, unicode]: + for dt in [bytes, np.void, str]: zs = self._zeros(10, dt) p = pickle.dumps(zs, protocol=proto) zs2 = pickle.loads(p) @@ -1702,53 +1701,60 @@ class TestMethods: b = np.sort(a) assert_equal(b, a[::-1], msg) - # all c scalar sorts use the same code with different types - # so it suffices to run a quick check with one type. The number - # of sorted items must be greater than ~50 to check the actual - # algorithm because quick and merge sort fall over to insertion - # sort for small arrays. - # Test unsigned dtypes and nonnegative numbers - for dtype in [np.uint8, np.uint16, np.uint32, np.uint64, np.float16, np.float32, np.float64, np.longdouble]: - a = np.arange(101, dtype=dtype) - b = a[::-1].copy() - for kind in self.sort_kinds: - msg = "scalar sort, kind=%s, dtype=%s" % (kind, dtype) - c = a.copy() - c.sort(kind=kind) - assert_equal(c, a, msg) - c = b.copy() - c.sort(kind=kind) - assert_equal(c, a, msg) - - # Test signed dtypes and negative numbers as well - for dtype in [np.int8, np.int16, np.int32, np.int64, np.float16, np.float32, np.float64, np.longdouble]: - a = np.arange(-50, 51, dtype=dtype) - b = a[::-1].copy() - for kind in self.sort_kinds: - msg = "scalar sort, kind=%s, dtype=%s" % (kind, dtype) - c = a.copy() - c.sort(kind=kind) - assert_equal(c, a, msg) - c = b.copy() - c.sort(kind=kind) - assert_equal(c, a, msg) + # all c scalar sorts use the same code with different types + # so it suffices to run a quick check with one type. The number + # of sorted items must be greater than ~50 to check the actual + # algorithm because quick and merge sort fall over to insertion + # sort for small arrays. + + @pytest.mark.parametrize('dtype', [np.uint8, np.uint16, np.uint32, np.uint64, + np.float16, np.float32, np.float64, + np.longdouble]) + def test_sort_unsigned(self, dtype): + a = np.arange(101, dtype=dtype) + b = a[::-1].copy() + for kind in self.sort_kinds: + msg = "scalar sort, kind=%s" % kind + c = a.copy() + c.sort(kind=kind) + assert_equal(c, a, msg) + c = b.copy() + c.sort(kind=kind) + assert_equal(c, a, msg) - # test complex sorts. These use the same code as the scalars - # but the compare function differs. - ai = a*1j + 1 - bi = b*1j + 1 + @pytest.mark.parametrize('dtype', + [np.int8, np.int16, np.int32, np.int64, np.float16, + np.float32, np.float64, np.longdouble]) + def test_sort_signed(self, dtype): + a = np.arange(-50, 51, dtype=dtype) + b = a[::-1].copy() for kind in self.sort_kinds: - msg = "complex sort, real part == 1, kind=%s" % kind - c = ai.copy() + msg = "scalar sort, kind=%s" % (kind) + c = a.copy() c.sort(kind=kind) - assert_equal(c, ai, msg) - c = bi.copy() + assert_equal(c, a, msg) + c = b.copy() c.sort(kind=kind) - assert_equal(c, ai, msg) - ai = a + 1j - bi = b + 1j + assert_equal(c, a, msg) + + @pytest.mark.parametrize('dtype', [np.float32, np.float64, np.longdouble]) + @pytest.mark.parametrize('part', ['real', 'imag']) + def test_sort_complex(self, part, dtype): + # test complex sorts. These use the same code as the scalars + # but the compare function differs. + cdtype = { + np.single: np.csingle, + np.double: np.cdouble, + np.longdouble: np.clongdouble, + }[dtype] + a = np.arange(-50, 51, dtype=dtype) + b = a[::-1].copy() + ai = (a * (1+1j)).astype(cdtype) + bi = (b * (1+1j)).astype(cdtype) + setattr(ai, part, 1) + setattr(bi, part, 1) for kind in self.sort_kinds: - msg = "complex sort, imag part == 1, kind=%s" % kind + msg = "complex sort, %s part == 1, kind=%s" % (part, kind) c = ai.copy() c.sort(kind=kind) assert_equal(c, ai, msg) @@ -1756,6 +1762,7 @@ class TestMethods: c.sort(kind=kind) assert_equal(c, ai, msg) + def test_sort_complex_byte_swapping(self): # test sorting of complex arrays requiring byte-swapping, gh-5441 for endianness in '<>': for dt in np.typecodes['Complex']: @@ -1765,25 +1772,13 @@ class TestMethods: msg = 'byte-swapped complex sort, dtype={0}'.format(dt) assert_equal(c, arr, msg) - # test string sorts. - s = 'aaaaaaaa' - a = np.array([s + chr(i) for i in range(101)]) + @pytest.mark.parametrize('dtype', [np.bytes_, np.unicode_]) + def test_sort_string(self, dtype): + # np.array will perform the encoding to bytes for us in the bytes test + a = np.array(['aaaaaaaa' + chr(i) for i in range(101)], dtype=dtype) b = a[::-1].copy() for kind in self.sort_kinds: - msg = "string sort, kind=%s" % kind - c = a.copy() - c.sort(kind=kind) - assert_equal(c, a, msg) - c = b.copy() - c.sort(kind=kind) - assert_equal(c, a, msg) - - # test unicode sorts. - s = 'aaaaaaaa' - a = np.array([s + chr(i) for i in range(101)], dtype=np.unicode_) - b = a[::-1].copy() - for kind in self.sort_kinds: - msg = "unicode sort, kind=%s" % kind + msg = "kind=%s" % kind c = a.copy() c.sort(kind=kind) assert_equal(c, a, msg) @@ -1791,37 +1786,27 @@ class TestMethods: c.sort(kind=kind) assert_equal(c, a, msg) + def test_sort_object(self): # test object array sorts. a = np.empty((101,), dtype=object) a[:] = list(range(101)) b = a[::-1] for kind in ['q', 'h', 'm']: - msg = "object sort, kind=%s" % kind + msg = "kind=%s" % kind c = a.copy() c.sort(kind=kind) assert_equal(c, a, msg) c = b.copy() c.sort(kind=kind) assert_equal(c, a, msg) - + + def test_sort_structured(self): # test record array sorts. dt = np.dtype([('f', float), ('i', int)]) a = np.array([(i, i) for i in range(101)], dtype=dt) b = a[::-1] for kind in ['q', 'h', 'm']: - msg = "object sort, kind=%s" % kind - c = a.copy() - c.sort(kind=kind) - assert_equal(c, a, msg) - c = b.copy() - c.sort(kind=kind) - assert_equal(c, a, msg) - - # test datetime64 sorts. - a = np.arange(0, 101, dtype='datetime64[D]') - b = a[::-1] - for kind in ['q', 'h', 'm']: - msg = "datetime64 sort, kind=%s" % kind + msg = "kind=%s" % kind c = a.copy() c.sort(kind=kind) assert_equal(c, a, msg) @@ -1829,11 +1814,13 @@ class TestMethods: c.sort(kind=kind) assert_equal(c, a, msg) - # test timedelta64 sorts. - a = np.arange(0, 101, dtype='timedelta64[D]') + @pytest.mark.parametrize('dtype', ['datetime64[D]', 'timedelta64[D]']) + def test_sort_time(self, dtype): + # test datetime64 and timedelta64 sorts. + a = np.arange(0, 101, dtype=dtype) b = a[::-1] for kind in ['q', 'h', 'm']: - msg = "timedelta64 sort, kind=%s" % kind + msg = "kind=%s" % kind c = a.copy() c.sort(kind=kind) assert_equal(c, a, msg) @@ -1841,6 +1828,7 @@ class TestMethods: c.sort(kind=kind) assert_equal(c, a, msg) + def test_sort_axis(self): # check axis handling. This should be the same for all type # specific sorts, so we only check it for one type and one kind a = np.array([[3, 2], [1, 0]]) @@ -1856,6 +1844,7 @@ class TestMethods: d.sort() assert_equal(d, c, "test sort with default axis") + def test_sort_size_0(self): # check axis handling for multidimensional empty arrays a = np.array([]) a.shape = (3, 2, 1, 0) @@ -1865,16 +1854,19 @@ class TestMethods: msg = 'test empty array sort with axis=None' assert_equal(np.sort(a, axis=None), a.ravel(), msg) + def test_sort_bad_ordering(self): # test generic class with bogus ordering, # should not segfault. class Boom: def __lt__(self, other): return True - a = np.array([Boom()]*100, dtype=object) + a = np.array([Boom()] * 100, dtype=object) for kind in self.sort_kinds: - msg = "bogus comparison object sort, kind=%s" % kind + msg = "kind=%s" % kind + c = a.copy() c.sort(kind=kind) + assert_equal(c, a, msg) def test_void_sort(self): # gh-8210 - previously segfaulted @@ -2766,11 +2758,6 @@ class TestMethods: assert_equal(x1.flatten('F'), y1f) assert_equal(x1.flatten('F'), x1.T.flatten()) - def test_flatten_invalid_order(self): - # invalid after gh-14596 - for order in ['Z', 'c', False, True, 0, 8]: - x = np.array([[1, 2, 3], [4, 5, 6]], np.int32) - assert_raises(ValueError, x.flatten, {"order": order}) @pytest.mark.parametrize('func', (np.dot, np.matmul)) def test_arr_mult(self, func): @@ -4461,7 +4448,7 @@ class TestPutmask: assert_equal(x[mask], np.array(val, T)) def test_ip_types(self): - unchecked_types = [bytes, unicode, np.void] + unchecked_types = [bytes, str, np.void] x = np.random.random(1000)*100 mask = x < 40 @@ -4515,7 +4502,7 @@ class TestTake: assert_array_equal(x.take(ind, axis=0), x) def test_ip_types(self): - unchecked_types = [bytes, unicode, np.void] + unchecked_types = [bytes, str, np.void] x = np.random.random(24)*100 x.shape = 2, 3, 4 @@ -4685,14 +4672,12 @@ class TestIO: y = np.fromfile(self.filename, dtype=self.dtype) assert_array_equal(y, self.x.flat) - @pytest.mark.skipif(pathlib is None, reason="pathlib not found") def test_roundtrip_pathlib(self): p = pathlib.Path(self.filename) self.x.tofile(p) y = np.fromfile(p, dtype=self.dtype) assert_array_equal(y, self.x.flat) - @pytest.mark.skipif(pathlib is None, reason="pathlib not found") def test_roundtrip_dump_pathlib(self): p = pathlib.Path(self.filename) self.x.dump(p) @@ -5503,6 +5488,12 @@ class TestStats: # of float32. assert_(_mean(np.ones(100000, dtype='float16')) == 1) + def test_mean_axis_error(self): + # Ensure that AxisError is raised instead of IndexError when axis is + # out of bounds, see gh-15817. + with assert_raises(np.core._exceptions.AxisError): + np.arange(10).mean(axis=2) + def test_var_values(self): for mat in [self.rmat, self.cmat, self.omat]: for axis in [0, 1, None]: @@ -5512,6 +5503,45 @@ class TestStats: res = _var(mat, axis=axis) assert_almost_equal(res, tgt) + @pytest.mark.parametrize(('complex_dtype', 'ndec'), ( + ('complex64', 6), + ('complex128', 7), + ('clongdouble', 7), + )) + def test_var_complex_values(self, complex_dtype, ndec): + # Test fast-paths for every builtin complex type + for axis in [0, 1, None]: + mat = self.cmat.copy().astype(complex_dtype) + msqr = _mean(mat * mat.conj(), axis=axis) + mean = _mean(mat, axis=axis) + tgt = msqr - mean * mean.conjugate() + res = _var(mat, axis=axis) + assert_almost_equal(res, tgt, decimal=ndec) + + def test_var_dimensions(self): + # _var paths for complex number introduce additions on views that + # increase dimensions. Ensure this generalizes to higher dims + mat = np.stack([self.cmat]*3) + for axis in [0, 1, 2, -1, None]: + msqr = _mean(mat * mat.conj(), axis=axis) + mean = _mean(mat, axis=axis) + tgt = msqr - mean * mean.conjugate() + res = _var(mat, axis=axis) + assert_almost_equal(res, tgt) + + def test_var_complex_byteorder(self): + # Test that var fast-path does not cause failures for complex arrays + # with non-native byteorder + cmat = self.cmat.copy().astype('complex128') + cmat_swapped = cmat.astype(cmat.dtype.newbyteorder()) + assert_almost_equal(cmat.var(), cmat_swapped.var()) + + def test_var_axis_error(self): + # Ensure that AxisError is raised instead of IndexError when axis is + # out of bounds, see gh-15817. + with assert_raises(np.core._exceptions.AxisError): + np.arange(10).var(axis=2) + def test_std_values(self): for mat in [self.rmat, self.cmat, self.omat]: for axis in [0, 1, None]: @@ -6278,10 +6308,7 @@ def test_matmul_inplace(): assert_raises(TypeError, a.__imatmul__, b) import operator assert_raises(TypeError, operator.imatmul, a, b) - # we avoid writing the token `exec` so as not to crash python 2's - # parser - exec_ = getattr(builtins, "exec") - assert_raises(TypeError, exec_, "a @= b", globals(), locals()) + assert_raises(TypeError, exec, "a @= b", globals(), locals()) def test_matmul_axes(): a = np.arange(3*4*5).reshape(3, 4, 5) @@ -7169,6 +7196,7 @@ class TestNewBufferProtocol: a = np.empty((1,) * 32) self._check_roundtrip(a) + @pytest.mark.slow def test_error_too_many_dims(self): def make_ctype(shape, scalar_type): t = scalar_type @@ -7756,7 +7784,7 @@ class TestHashing: def test_collections_hashable(self): x = np.array([]) - assert_(not isinstance(x, collections_abc.Hashable)) + assert_(not isinstance(x, collections.abc.Hashable)) class TestArrayPriority: @@ -7861,6 +7889,34 @@ class TestBytestringArrayNonzero: assert_(a) +class TestUnicodeEncoding: + """ + Tests for encoding related bugs, such as UCS2 vs UCS4, round-tripping + issues, etc + """ + def test_round_trip(self): + """ Tests that GETITEM, SETITEM, and PyArray_Scalar roundtrip """ + # gh-15363 + arr = np.zeros(shape=(), dtype="U1") + for i in range(1, sys.maxunicode + 1): + expected = chr(i) + arr[()] = expected + assert arr[()] == expected + assert arr.item() == expected + + def test_assign_scalar(self): + # gh-3258 + l = np.array(['aa', 'bb']) + l[:] = np.unicode_('cc') + assert_equal(l, ['cc', 'cc']) + + def test_fill_scalar(self): + # gh-7227 + l = np.array(['aa', 'bb']) + l.fill(np.unicode_('cc')) + assert_equal(l, ['cc', 'cc']) + + class TestUnicodeArrayNonzero: def test_empty_ustring_array_is_falsey(self): diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index 24272bb0d..c106c528d 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -2688,7 +2688,15 @@ def test_0d_iter(): i = nditer(np.arange(5), ['multi_index'], [['readonly']], op_axes=[()]) assert_equal(i.ndim, 0) assert_equal(len(i), 1) - # note that itershape=(), still behaves like None due to the conversions + + i = nditer(np.arange(5), ['multi_index'], [['readonly']], + op_axes=[()], itershape=()) + assert_equal(i.ndim, 0) + assert_equal(len(i), 1) + + # passing an itershape alone is not enough, the op_axes are also needed + with assert_raises(ValueError): + nditer(np.arange(5), ['multi_index'], [['readonly']], itershape=()) # Test a more complex buffered casting case (same as another test above) sdt = [('a', 'f4'), ('b', 'i8'), ('c', 'c8', (2, 3)), ('d', 'O')] diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index c4956c298..bcc6a0c4e 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -3,6 +3,7 @@ import warnings import itertools import platform import pytest +import math from decimal import Decimal import numpy as np @@ -11,9 +12,12 @@ from numpy.random import rand, randint, randn from numpy.testing import ( assert_, assert_equal, assert_raises, assert_raises_regex, assert_array_equal, assert_almost_equal, assert_array_almost_equal, - assert_warns, HAS_REFCOUNT + assert_warns, assert_array_max_ulp, HAS_REFCOUNT ) +from hypothesis import assume, given, strategies as st +from hypothesis.extra import numpy as hynp + class TestResize: def test_copies(self): @@ -136,6 +140,51 @@ class TestNonarrayArgs: arr = [1.56, 72.54, 6.35, 3.25] tgt = [1.6, 72.5, 6.4, 3.2] assert_equal(np.around(arr, decimals=1), tgt) + s = np.float64(1.) + assert_(isinstance(s.round(), np.float64)) + assert_equal(s.round(), 1.) + + @pytest.mark.parametrize('dtype', [ + np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64, + np.float16, np.float32, np.float64, + ]) + def test_dunder_round(self, dtype): + s = dtype(1) + assert_(isinstance(round(s), int)) + assert_(isinstance(round(s, None), int)) + assert_(isinstance(round(s, ndigits=None), int)) + assert_equal(round(s), 1) + assert_equal(round(s, None), 1) + assert_equal(round(s, ndigits=None), 1) + + @pytest.mark.parametrize('val, ndigits', [ + pytest.param(2**31 - 1, -1, + marks=pytest.mark.xfail(reason="Out of range of int32") + ), + (2**31 - 1, 1-math.ceil(math.log10(2**31 - 1))), + (2**31 - 1, -math.ceil(math.log10(2**31 - 1))) + ]) + def test_dunder_round_edgecases(self, val, ndigits): + assert_equal(round(val, ndigits), round(np.int32(val), ndigits)) + + def test_dunder_round_accuracy(self): + f = np.float64(5.1 * 10**73) + assert_(isinstance(round(f, -73), np.float64)) + assert_array_max_ulp(round(f, -73), 5.0 * 10**73) + assert_(isinstance(round(f, ndigits=-73), np.float64)) + assert_array_max_ulp(round(f, ndigits=-73), 5.0 * 10**73) + + i = np.int64(501) + assert_(isinstance(round(i, -2), np.int64)) + assert_array_max_ulp(round(i, -2), 500) + assert_(isinstance(round(i, ndigits=-2), np.int64)) + assert_array_max_ulp(round(i, ndigits=-2), 500) + + @pytest.mark.xfail(raises=AssertionError, reason="gh-15896") + def test_round_py_consistency(self): + f = 5.1 * 10**73 + assert_equal(round(np.float64(f), -73), round(f, -73)) def test_searchsorted(self): arr = [-8, -5, -1, 3, 6, 10] @@ -954,8 +1003,7 @@ class NIterError(Exception): class TestFromiter: def makegen(self): - for x in range(24): - yield x**2 + return (x**2 for x in range(24)) def test_types(self): ai32 = np.fromiter(self.makegen(), np.int32) @@ -1199,6 +1247,17 @@ class TestNonzero: a = np.array([[0, 0, 1], [1, 0, 1]]) assert_equal(np.count_nonzero(a, axis=()), a.astype(bool)) + def test_countnonzero_keepdims(self): + a = np.array([[0, 0, 1, 0], + [0, 3, 5, 0], + [7, 9, 2, 0]]) + assert_equal(np.count_nonzero(a, axis=0, keepdims=True), + [[1, 2, 3, 0]]) + assert_equal(np.count_nonzero(a, axis=1, keepdims=True), + [[1], [2], [3]]) + assert_equal(np.count_nonzero(a, keepdims=True), + [[6]]) + def test_array_method(self): # Tests that the array method # call to nonzero works @@ -1998,12 +2057,12 @@ class TestClip: np.array(np.nan), np.zeros(10, dtype=np.int32)), ]) + @pytest.mark.filterwarnings("ignore::DeprecationWarning") def test_clip_scalar_nan_propagation(self, arr, amin, amax): # enforcement of scalar nan propagation for comparisons # called through clip() expected = np.minimum(np.maximum(arr, amin), amax) - with assert_warns(DeprecationWarning): - actual = np.clip(arr, amin, amax) + actual = np.clip(arr, amin, amax) assert_equal(actual, expected) @pytest.mark.xfail(reason="propagation doesn't match spec") @@ -2012,13 +2071,76 @@ class TestClip: np.timedelta64('NaT'), np.zeros(10, dtype=np.int32)), ]) + @pytest.mark.filterwarnings("ignore::DeprecationWarning") def test_NaT_propagation(self, arr, amin, amax): # NOTE: the expected function spec doesn't # propagate NaT, but clip() now does - expected = np.minimum(np.maximum(a, amin), amax) + expected = np.minimum(np.maximum(arr, amin), amax) actual = np.clip(arr, amin, amax) assert_equal(actual, expected) + @given(data=st.data(), shape=hynp.array_shapes()) + def test_clip_property(self, data, shape): + """A property-based test using Hypothesis. + + This aims for maximum generality: it could in principle generate *any* + valid inputs to np.clip, and in practice generates much more varied + inputs than human testers come up with. + + Because many of the inputs have tricky dependencies - compatible dtypes + and mutually-broadcastable shapes - we use `st.data()` strategy draw + values *inside* the test function, from strategies we construct based + on previous values. An alternative would be to define a custom strategy + with `@st.composite`, but until we have duplicated code inline is fine. + + That accounts for most of the function; the actual test is just three + lines to calculate and compare actual vs expected results! + """ + # Our base array and bounds should not need to be of the same type as + # long as they are all compatible - so we allow any int or float type. + dtype_strategy = hynp.integer_dtypes() | hynp.floating_dtypes() + + # The following line is a total hack to disable the varied-dtypes + # component of this test, because result != expected if dtypes can vary. + dtype_strategy = st.just(data.draw(dtype_strategy)) + + # Generate an arbitrary array of the chosen shape and dtype + # This is the value that we clip. + arr = data.draw(hynp.arrays(dtype=dtype_strategy, shape=shape)) + + # Generate shapes for the bounds which can be broadcast with each other + # and with the base shape. Below, we might decide to use scalar bounds, + # but it's clearer to generate these shapes unconditionally in advance. + in_shapes, result_shape = data.draw( + hynp.mutually_broadcastable_shapes( + num_shapes=2, + base_shape=shape, + # Commenting out the min_dims line allows zero-dimensional arrays, + # and zero-dimensional arrays containing NaN make the test fail. + min_dims=1 + + ) + ) + amin = data.draw( + dtype_strategy.flatmap(hynp.from_dtype) + | hynp.arrays(dtype=dtype_strategy, shape=in_shapes[0]) + ) + amax = data.draw( + dtype_strategy.flatmap(hynp.from_dtype) + | hynp.arrays(dtype=dtype_strategy, shape=in_shapes[1]) + ) + # If we allow either bound to be a scalar `nan`, the test will fail - + # so we just "assume" that away (if it is, this raises a special + # exception and Hypothesis will try again with different inputs) + assume(not np.isscalar(amin) or not np.isnan(amin)) + assume(not np.isscalar(amax) or not np.isnan(amax)) + + # Then calculate our result and expected result and check that they're + # equal! See gh-12519 for discussion deciding on this property. + result = np.clip(arr, amin, amax) + expected = np.minimum(amax, np.maximum(arr, amin)) + assert_array_equal(result, expected) + class TestAllclose: rtol = 1e-5 @@ -2392,7 +2514,7 @@ class TestLikeFuncs: (np.arange(24).reshape(2, 3, 4).swapaxes(0, 1), None), (np.arange(24).reshape(4, 3, 2).swapaxes(0, 1), '?'), ] - self.shapes = [(5,), (5,6,), (5,6,7,)] + self.shapes = [(), (5,), (5,6,), (5,6,7,)] def compare_array_value(self, dz, value, fill_value): if value is not None: diff --git a/numpy/core/tests/test_numerictypes.py b/numpy/core/tests/test_numerictypes.py index c72d13947..9cb00342d 100644 --- a/numpy/core/tests/test_numerictypes.py +++ b/numpy/core/tests/test_numerictypes.py @@ -401,6 +401,35 @@ class TestIsSubDType: assert_(not np.issubdtype(w1(np.float32), w2(np.float64))) assert_(not np.issubdtype(w1(np.float64), w2(np.float32))) + def test_nondtype_nonscalartype(self): + # See gh-14619 and gh-9505 which introduced the deprecation to fix + # this. These tests are directly taken from gh-9505 + assert not np.issubdtype(np.float32, 'float64') + assert not np.issubdtype(np.float32, 'f8') + assert not np.issubdtype(np.int32, str) + assert not np.issubdtype(np.int32, 'int64') + assert not np.issubdtype(np.str_, 'void') + # for the following the correct spellings are + # np.integer, np.floating, or np.complexfloating respectively: + assert not np.issubdtype(np.int8, int) # np.int8 is never np.int_ + assert not np.issubdtype(np.float32, float) + assert not np.issubdtype(np.complex64, complex) + assert not np.issubdtype(np.float32, "float") + assert not np.issubdtype(np.float64, "f") + + # Test the same for the correct first datatype and abstract one + # in the case of int, float, complex: + assert np.issubdtype(np.float64, 'float64') + assert np.issubdtype(np.float64, 'f8') + assert np.issubdtype(np.str_, str) + assert np.issubdtype(np.int64, 'int64') + assert np.issubdtype(np.void, 'void') + assert np.issubdtype(np.int8, np.integer) + assert np.issubdtype(np.float32, np.floating) + assert np.issubdtype(np.complex64, np.complexfloating) + assert np.issubdtype(np.float64, "float") + assert np.issubdtype(np.float32, "f") + class TestSctypeDict: def test_longdouble(self): @@ -486,7 +515,8 @@ def test_issctype(rep, expected): @pytest.mark.skipif(sys.flags.optimize > 1, reason="no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1") -@pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") +@pytest.mark.xfail(IS_PYPY, + reason="PyPy cannot modify tp_doc after PyType_Ready") class TestDocStrings: def test_platform_dependent_aliases(self): if np.int64 is np.int_: diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py index bf43fedcc..4350a3407 100644 --- a/numpy/core/tests/test_records.py +++ b/numpy/core/tests/test_records.py @@ -1,16 +1,10 @@ -import sys -try: - # Accessing collections abstract classes from collections - # has been deprecated since Python 3.3 - import collections.abc as collections_abc -except ImportError: - import collections as collections_abc +import collections.abc import textwrap from os import path +from pathlib import Path import pytest import numpy as np -from numpy.compat import Path from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_array_almost_equal, assert_raises, temppath, @@ -252,7 +246,7 @@ class TestFromrecords: assert_array_equal(ra['shape'], [['A', 'B', 'C']]) ra.field = 5 assert_array_equal(ra['field'], [[5, 5, 5]]) - assert_(isinstance(ra.field, collections_abc.Callable)) + assert_(isinstance(ra.field, collections.abc.Callable)) def test_fromrecords_with_explicit_dtype(self): a = np.rec.fromrecords([(1, 'a'), (2, 'bbb')], @@ -319,7 +313,6 @@ class TestFromrecords: assert_equal(rec['f1'], [b'', b'', b'']) -@pytest.mark.skipif(Path is None, reason="No pathlib.Path") class TestPathUsage: # Test that pathlib.Path can be used def test_tofile_fromfile(self): diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 321723b9b..96a6d810f 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -15,7 +15,7 @@ from numpy.testing import ( _assert_valid_refcount, HAS_REFCOUNT, ) from numpy.testing._private.utils import _no_tracing -from numpy.compat import asbytes, asunicode, long, pickle +from numpy.compat import asbytes, asunicode, pickle try: RecursionError @@ -452,6 +452,11 @@ class TestRegression: xs.strides = (16, 16) assert np.lexsort((xs,), axis=0).shape[0] == 2 + def test_lexsort_invalid_axis(self): + assert_raises(np.AxisError, np.lexsort, (np.arange(1),), axis=2) + assert_raises(np.AxisError, np.lexsort, (np.array([]),), axis=1) + assert_raises(np.AxisError, np.lexsort, (np.array(1),), axis=10) + def test_lexsort_zerolen_element(self): dt = np.dtype([]) # a void dtype with no fields xs = np.empty(4, dt) @@ -1408,6 +1413,13 @@ class TestRegression: dtype='U') assert_raises(UnicodeEncodeError, np.array, a, 'S4') + def test_unicode_to_string_cast_error(self): + # gh-15790 + a = np.array([u'\x80'] * 129, dtype='U3') + assert_raises(UnicodeEncodeError, np.array, a, 'S') + b = a.reshape(3, 43)[:-1, :-1] + assert_raises(UnicodeEncodeError, np.array, b, 'S') + def test_mixed_string_unicode_array_creation(self): a = np.array(['1234', u'123']) assert_(a.itemsize == 16) @@ -1497,14 +1509,11 @@ class TestRegression: min //= -1 with np.errstate(divide="ignore"): - for t in (np.int8, np.int16, np.int32, np.int64, int, np.compat.long): + for t in (np.int8, np.int16, np.int32, np.int64, int): test_type(t) def test_buffer_hashlib(self): - try: - from hashlib import md5 - except ImportError: - from md5 import new as md5 + from hashlib import md5 x = np.array([1, 2, 3], dtype=np.dtype('<i4')) assert_equal(md5(x).hexdigest(), '2a1dd1e1e59d0a384c26951e316cd7e6') @@ -1799,7 +1808,6 @@ class TestRegression: a = np.array(0, dtype=object) a[()] = a assert_raises(RecursionError, int, a) - assert_raises(RecursionError, long, a) assert_raises(RecursionError, float, a) a[()] = None @@ -1825,7 +1833,6 @@ class TestRegression: b = np.array(0, dtype=object) a[()] = b assert_equal(int(a), int(0)) - assert_equal(long(a), long(0)) assert_equal(float(a), float(0)) def test_object_array_self_copy(self): @@ -2028,6 +2035,7 @@ class TestRegression: a[...] = [[1, 2]] assert_equal(a, [[1, 2], [1, 2]]) + @pytest.mark.slow_pypy def test_memoryleak(self): # Ticket #1917 - ensure that array data doesn't leak for i in range(1000): @@ -2451,6 +2459,27 @@ class TestRegression: np.array([T()]) + def test_2d__array__shape(self): + class T(object): + def __array__(self): + return np.ndarray(shape=(0,0)) + + # Make sure __array__ is used instead of Sequence methods. + def __iter__(self): + return iter([]) + + def __getitem__(self, idx): + raise AssertionError("__getitem__ was called") + + def __len__(self): + return 0 + + + t = T() + # gh-13659, would raise in broadcasting [x=t for x in result] + arr = np.array([t]) + assert arr.shape == (1, 0, 0) + @pytest.mark.skipif(sys.maxsize < 2 ** 31 + 1, reason='overflows 32-bit python') @pytest.mark.skipif(sys.platform == 'win32' and sys.version_info[:2] < (3, 8), reason='overflows on windows, fixed in bpo-16865') @@ -2460,4 +2489,3 @@ class TestRegression: assert arr.size * arr.itemsize > 2 ** 31 c_arr = np.ctypeslib.as_ctypes(arr) assert_equal(c_arr._length_, arr.size) - diff --git a/numpy/core/tests/test_scalar_ctors.py b/numpy/core/tests/test_scalar_ctors.py index d3592a5fc..7645a0853 100644 --- a/numpy/core/tests/test_scalar_ctors.py +++ b/numpy/core/tests/test_scalar_ctors.py @@ -1,13 +1,11 @@ """ Test the scalar constructors, which also do type-coercion """ -import sys -import platform import pytest import numpy as np from numpy.testing import ( - assert_equal, assert_almost_equal, assert_raises, assert_warns, + assert_equal, assert_almost_equal, assert_warns, ) class TestFromString: @@ -41,6 +39,39 @@ class TestFromString: assert_equal(flongdouble, -np.inf) +class TestExtraArgs: + def test_superclass(self): + # try both positional and keyword arguments + s = np.str_(b'\\x61', encoding='unicode-escape') + assert s == 'a' + s = np.str_(b'\\x61', 'unicode-escape') + assert s == 'a' + + # previously this would return '\\xx' + with pytest.raises(UnicodeDecodeError): + np.str_(b'\\xx', encoding='unicode-escape') + with pytest.raises(UnicodeDecodeError): + np.str_(b'\\xx', 'unicode-escape') + + # superclass fails, but numpy succeeds + assert np.bytes_(-2) == b'-2' + + def test_datetime(self): + dt = np.datetime64('2000-01', ('M', 2)) + assert np.datetime_data(dt) == ('M', 2) + + with pytest.raises(TypeError): + np.datetime64('2000', garbage=True) + + def test_bool(self): + with pytest.raises(TypeError): + np.bool(False, garbage=True) + + def test_void(self): + with pytest.raises(TypeError): + np.void(b'test', garbage=True) + + class TestFromInt: def test_intp(self): # Ticket #99 diff --git a/numpy/core/tests/test_scalar_methods.py b/numpy/core/tests/test_scalar_methods.py index c9de3e402..4f5fd2988 100644 --- a/numpy/core/tests/test_scalar_methods.py +++ b/numpy/core/tests/test_scalar_methods.py @@ -1,18 +1,14 @@ """ Test the scalar constructors, which also do type-coercion """ -import os import fractions import platform import pytest import numpy as np -from numpy.testing import ( - run_module_suite, - assert_equal, assert_almost_equal, assert_raises, assert_warns, - dec -) +from numpy.testing import assert_equal, assert_raises + class TestAsIntegerRatio: # derived in part from the cpython test "test_floatasratio" diff --git a/numpy/core/tests/test_scalarbuffer.py b/numpy/core/tests/test_scalarbuffer.py index bda1c5333..b1c1bbbb1 100644 --- a/numpy/core/tests/test_scalarbuffer.py +++ b/numpy/core/tests/test_scalarbuffer.py @@ -1,7 +1,6 @@ """ Test scalar buffer interface adheres to PEP 3118 """ -import sys import numpy as np import pytest @@ -77,27 +76,44 @@ class TestScalarPEP3118: assert_equal(mv_x.itemsize, mv_a.itemsize) assert_equal(mv_x.format, mv_a.format) + def _as_dict(self, m): + return dict(strides=m.strides, shape=m.shape, itemsize=m.itemsize, + ndim=m.ndim, format=m.format) + def test_datetime_memoryview(self): # gh-11656 # Values verified with v1.13.3, shape is not () as in test_scalar_dim - def as_dict(m): - return dict(strides=m.strides, shape=m.shape, itemsize=m.itemsize, - ndim=m.ndim, format=m.format) dt1 = np.datetime64('2016-01-01') dt2 = np.datetime64('2017-01-01') - expected = {'strides': (1,), 'itemsize': 1, 'ndim': 1, - 'shape': (8,), 'format': 'B'} + expected = dict(strides=(1,), itemsize=1, ndim=1, shape=(8,), + format='B') v = memoryview(dt1) - res = as_dict(v) - assert_equal(res, expected) + assert self._as_dict(v) == expected v = memoryview(dt2 - dt1) - res = as_dict(v) - assert_equal(res, expected) + assert self._as_dict(v) == expected dt = np.dtype([('a', 'uint16'), ('b', 'M8[s]')]) a = np.empty(1, dt) # Fails to create a PEP 3118 valid buffer assert_raises((ValueError, BufferError), memoryview, a[0]) + @pytest.mark.parametrize('s', [ + pytest.param("\x32\x32", id="ascii"), + pytest.param("\uFE0F\uFE0F", id="basic multilingual"), + pytest.param("\U0001f4bb\U0001f4bb", id="non-BMP"), + ]) + def test_str_ucs4(self, s): + s = np.str_(s) # only our subclass implements the buffer protocol + + # all the same, characters always encode as ucs4 + expected = dict(strides=(), itemsize=8, ndim=0, shape=(), format='2w') + + v = memoryview(s) + assert self._as_dict(v) == expected + + # integers of the paltform-appropriate endianness + code_points = np.frombuffer(v, dtype='i4') + + assert_equal(code_points, [ord(c) for c in s]) diff --git a/numpy/core/tests/test_scalarinherit.py b/numpy/core/tests/test_scalarinherit.py index af3669d73..74829986c 100644 --- a/numpy/core/tests/test_scalarinherit.py +++ b/numpy/core/tests/test_scalarinherit.py @@ -2,6 +2,8 @@ """ Test printing of scalar types. """ +import pytest + import numpy as np from numpy.testing import assert_ @@ -21,6 +23,14 @@ class B0(np.float64, A): class C0(B0): pass +class HasNew: + def __new__(cls, *args, **kwargs): + return cls, args, kwargs + +class B1(np.float64, HasNew): + pass + + class TestInherit: def test_init(self): x = B(1.0) @@ -36,6 +46,15 @@ class TestInherit: y = C0(2.0) assert_(str(y) == '2.0') + def test_gh_15395(self): + # HasNew is the second base, so `np.float64` should have priority + x = B1(1.0) + assert_(str(x) == '1.0') + + # previously caused RecursionError!? + with pytest.raises(TypeError): + B1(1.0, 2.0) + class TestCharacter: def test_char_radd(self): diff --git a/numpy/core/tests/test_scalarprint.py b/numpy/core/tests/test_scalarprint.py index 225b8295f..6502ec4c1 100644 --- a/numpy/core/tests/test_scalarprint.py +++ b/numpy/core/tests/test_scalarprint.py @@ -2,13 +2,14 @@ """ Test printing of scalar types. """ -import code, sys +import code import platform import pytest +import sys from tempfile import TemporaryFile import numpy as np -from numpy.testing import assert_, assert_equal, suppress_warnings +from numpy.testing import assert_, assert_equal class TestRealScalars: def test_str(self): @@ -30,12 +31,12 @@ class TestRealScalars: def test_scalar_cutoffs(self): # test that both the str and repr of np.float64 behaves - # like python floats in python3. Note that in python2 - # the str has truncated digits, but we do not do this + # like python floats in python3. def check(v): - # we compare str to repr, to avoid python2 truncation behavior + assert_equal(str(np.float64(v)), str(v)) assert_equal(str(np.float64(v)), repr(v)) assert_equal(repr(np.float64(v)), repr(v)) + assert_equal(repr(np.float64(v)), str(v)) # check we use the same number of significant digits check(1.12345678901234567890) @@ -82,10 +83,7 @@ class TestRealScalars: orig_stdout, orig_stderr = sys.stdout, sys.stderr sys.stdout, sys.stderr = fo, fe - # py2 code.interact sends irrelevant internal DeprecationWarnings - with suppress_warnings() as sup: - sup.filter(DeprecationWarning) - code.interact(local={'np': np}, readfunc=input_func, banner='') + code.interact(local={'np': np}, readfunc=input_func, banner='') sys.stdout, sys.stderr = orig_stdout, orig_stderr diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py index 7c23c7128..546ecf001 100644 --- a/numpy/core/tests/test_shape_base.py +++ b/numpy/core/tests/test_shape_base.py @@ -1,5 +1,4 @@ import pytest -import sys import numpy as np from numpy.core import ( array, arange, atleast_1d, atleast_2d, atleast_3d, block, vstack, hstack, @@ -12,7 +11,6 @@ from numpy.testing import ( assert_raises_regex, assert_warns ) -from numpy.compat import long class TestAtleast1d: def test_0D_array(self): @@ -50,7 +48,6 @@ class TestAtleast1d: """ assert_(atleast_1d(3).shape == (1,)) assert_(atleast_1d(3j).shape == (1,)) - assert_(atleast_1d(long(3)).shape == (1,)) assert_(atleast_1d(3.0).shape == (1,)) assert_(atleast_1d([[2, 3], [4, 5]]).shape == (2, 2)) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 10a1c0803..60c9fe437 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -640,6 +640,16 @@ class TestExp: yf = np.array(y, dtype=dt)*log2_ assert_almost_equal(np.exp(yf), xf) + def test_exp_strides(self): + np.random.seed(42) + strides = np.array([-4,-3,-2,-1,1,2,3,4]) + sizes = np.arange(2,100) + for ii in sizes: + x_f64 = np.float64(np.random.uniform(low=0.01, high=709.1,size=ii)) + y_true = np.exp(x_f64) + for jj in strides: + assert_array_almost_equal_nulp(np.exp(x_f64[::jj]), y_true[::jj], nulp=2) + class TestSpecialFloats: def test_exp_values(self): x = [np.nan, np.nan, np.inf, 0.] @@ -652,6 +662,8 @@ class TestSpecialFloats: with np.errstate(over='raise'): assert_raises(FloatingPointError, np.exp, np.float32(100.)) assert_raises(FloatingPointError, np.exp, np.float32(1E19)) + assert_raises(FloatingPointError, np.exp, np.float64(800.)) + assert_raises(FloatingPointError, np.exp, np.float64(1E19)) def test_log_values(self): with np.errstate(all='ignore'): @@ -1108,6 +1120,19 @@ class TestMaximum(_FilterInvalids): arg2 = arg1 + 1 assert_equal(np.maximum(arg1, arg2), arg2) + def test_strided_array(self): + arr1 = np.array([-4.0, 1.0, 10.0, 0.0, np.nan, -np.nan, np.inf, -np.inf]) + arr2 = np.array([-2.0,-1.0, np.nan, 1.0, 0.0, np.nan, 1.0, -3.0]) + maxtrue = np.array([-2.0, 1.0, np.nan, 1.0, np.nan, np.nan, np.inf, -3.0]) + out = np.ones(8) + out_maxtrue = np.array([-2.0, 1.0, 1.0, 10.0, 1.0, 1.0, np.nan, 1.0]) + assert_equal(np.maximum(arr1,arr2), maxtrue) + assert_equal(np.maximum(arr1[::2],arr2[::2]), maxtrue[::2]) + assert_equal(np.maximum(arr1[:4:], arr2[::2]), np.array([-2.0, np.nan, 10.0, 1.0])) + assert_equal(np.maximum(arr1[::3], arr2[:3:]), np.array([-2.0, 0.0, np.nan])) + assert_equal(np.maximum(arr1[:6:2], arr2[::3], out=out[::3]), np.array([-2.0, 10., np.nan])) + assert_equal(out, out_maxtrue) + class TestMinimum(_FilterInvalids): def test_reduce(self): @@ -1166,6 +1191,18 @@ class TestMinimum(_FilterInvalids): arg2 = arg1 + 1 assert_equal(np.minimum(arg1, arg2), arg1) + def test_strided_array(self): + arr1 = np.array([-4.0, 1.0, 10.0, 0.0, np.nan, -np.nan, np.inf, -np.inf]) + arr2 = np.array([-2.0,-1.0, np.nan, 1.0, 0.0, np.nan, 1.0, -3.0]) + mintrue = np.array([-4.0, -1.0, np.nan, 0.0, np.nan, np.nan, 1.0, -np.inf]) + out = np.ones(8) + out_mintrue = np.array([-4.0, 1.0, 1.0, 1.0, 1.0, 1.0, np.nan, 1.0]) + assert_equal(np.minimum(arr1,arr2), mintrue) + assert_equal(np.minimum(arr1[::2],arr2[::2]), mintrue[::2]) + assert_equal(np.minimum(arr1[:4:], arr2[::2]), np.array([-4.0, np.nan, 0.0, 0.0])) + assert_equal(np.minimum(arr1[::3], arr2[:3:]), np.array([-4.0, -1.0, np.nan])) + assert_equal(np.minimum(arr1[:6:2], arr2[::3], out=out[::3]), np.array([-4.0, 1.0, np.nan])) + assert_equal(out, out_mintrue) class TestFmax(_FilterInvalids): def test_reduce(self): @@ -3132,6 +3169,14 @@ def test_rint_big_int(): # Rint should not change the value assert_equal(val, np.rint(val)) +@pytest.mark.parametrize('ftype', [np.float32, np.float64]) +def test_memoverlap_accumulate(ftype): + # Reproduces bug https://github.com/numpy/numpy/issues/15597 + arr = np.array([0.61, 0.60, 0.77, 0.41, 0.19], dtype=ftype) + out_max = np.array([0.61, 0.61, 0.77, 0.77, 0.77], dtype=ftype) + out_min = np.array([0.61, 0.60, 0.60, 0.41, 0.19], dtype=ftype) + assert_equal(np.maximum.accumulate(arr), out_max) + assert_equal(np.minimum.accumulate(arr), out_min) def test_signaling_nan_exceptions(): with assert_no_warnings(): diff --git a/numpy/core/tests/test_umath_accuracy.py b/numpy/core/tests/test_umath_accuracy.py index 677d9af60..e3c2eb025 100644 --- a/numpy/core/tests/test_umath_accuracy.py +++ b/numpy/core/tests/test_umath_accuracy.py @@ -3,23 +3,28 @@ import platform from os import path import sys import pytest -from ctypes import * +from ctypes import c_longlong, c_double, c_float, c_int, cast, pointer, POINTER from numpy.testing import assert_array_max_ulp +from numpy.core._multiarray_umath import __cpu_features__ -runtest = sys.platform.startswith('linux') and (platform.machine() == 'x86_64') +IS_AVX = __cpu_features__.get('AVX512F', False) or \ + (__cpu_features__.get('FMA3', False) and __cpu_features__.get('AVX2', False)) +runtest = sys.platform.startswith('linux') and IS_AVX platform_skip = pytest.mark.skipif(not runtest, - reason=""" - stick to x86_64 and linux platforms. - test seems to fail on some of ARM and power - architectures. - """) + reason="avoid testing inconsistent platform " + "library implementations") # convert string to hex function taken from: # https://stackoverflow.com/questions/1592158/convert-hex-to-float # -def convert(s): +def convert(s, datatype="np.float32"): i = int(s, 16) # convert from hex to a Python int - cp = pointer(c_int(i)) # make this into a c integer - fp = cast(cp, POINTER(c_float)) # cast the int pointer to a float pointer + if (datatype == "np.float64"): + cp = pointer(c_longlong(i)) # make this into a c long long integer + fp = cast(cp, POINTER(c_double)) # cast the int pointer to a double pointer + else: + cp = pointer(c_int(i)) # make this into a c integer + fp = cast(cp, POINTER(c_float)) # cast the int pointer to a float pointer + return fp.contents.value # dereference the pointer, get the float str_to_float = np.vectorize(convert) @@ -29,7 +34,7 @@ files = ['umath-validation-set-exp', 'umath-validation-set-cos'] class TestAccuracy: - @pytest.mark.xfail(reason="Fails for MacPython/numpy-wheels builds") + @platform_skip def test_validate_transcendentals(self): with np.errstate(all='ignore'): for filename in files: @@ -37,18 +42,24 @@ class TestAccuracy: filepath = path.join(data_dir, filename) with open(filepath) as fid: file_without_comments = (r for r in fid if not r[0] in ('$', '#')) - data = np.genfromtxt(file_without_comments, - dtype=('|S39','|S39','|S39',int), - names=('type','input','output','ulperr'), - delimiter=',', - skip_header=1) - npfunc = getattr(np, filename.split('-')[3]) - for datatype in np.unique(data['type']): - data_subset = data[data['type'] == datatype] - inval = np.array(str_to_float(data_subset['input'].astype(str)), dtype=eval(datatype)) - outval = np.array(str_to_float(data_subset['output'].astype(str)), dtype=eval(datatype)) - perm = np.random.permutation(len(inval)) - inval = inval[perm] - outval = outval[perm] - maxulperr = data_subset['ulperr'].max() - assert_array_max_ulp(npfunc(inval), outval, maxulperr) + data = np.genfromtxt(file_without_comments, + dtype=('|S39','|S39','|S39',int), + names=('type','input','output','ulperr'), + delimiter=',', + skip_header=1) + npfunc = getattr(np, filename.split('-')[3]) + for datatype in np.unique(data['type']): + data_subset = data[data['type'] == datatype] + inval = np.array(str_to_float(data_subset['input'].astype(str), data_subset['type'].astype(str)), dtype=eval(datatype)) + outval = np.array(str_to_float(data_subset['output'].astype(str), data_subset['type'].astype(str)), dtype=eval(datatype)) + perm = np.random.permutation(len(inval)) + inval = inval[perm] + outval = outval[perm] + maxulperr = data_subset['ulperr'].max() + assert_array_max_ulp(npfunc(inval), outval, maxulperr) + + def test_ignore_nan_ulperror(self): + # Ignore ULP differences between various NAN's + nan1_f32 = np.array(str_to_float('0xffffffff'), dtype=np.float32) + nan2_f32 = np.array(str_to_float('0x7fddbfbf'), dtype=np.float32) + assert_array_max_ulp(nan1_f32, nan2_f32, 0) diff --git a/numpy/core/tests/test_umath_complex.py b/numpy/core/tests/test_umath_complex.py index 5e5ced85c..a21158420 100644 --- a/numpy/core/tests/test_umath_complex.py +++ b/numpy/core/tests/test_umath_complex.py @@ -6,7 +6,7 @@ import numpy as np # import the c-extension module directly since _arg is not exported via umath import numpy.core._multiarray_umath as ncu from numpy.testing import ( - assert_raises, assert_equal, assert_array_equal, assert_almost_equal + assert_raises, assert_equal, assert_array_equal, assert_almost_equal, assert_array_max_ulp ) # TODO: branch cuts (use Pauli code) @@ -540,3 +540,40 @@ def check_complex_value(f, x1, y1, x2, y2, exact=True): assert_equal(f(z1), z2) else: assert_almost_equal(f(z1), z2) + +class TestSpecialComplexAVX(object): + @pytest.mark.parametrize("stride", [-4,-2,-1,1,2,4]) + @pytest.mark.parametrize("astype", [np.complex64, np.complex128]) + def test_array(self, stride, astype): + arr = np.array([np.complex(np.nan , np.nan), + np.complex(np.nan , np.inf), + np.complex(np.inf , np.nan), + np.complex(np.inf , np.inf), + np.complex(0. , np.inf), + np.complex(np.inf , 0.), + np.complex(0. , 0.), + np.complex(0. , np.nan), + np.complex(np.nan , 0.)], dtype=astype) + abs_true = np.array([np.nan, np.inf, np.inf, np.inf, np.inf, np.inf, 0., np.nan, np.nan], dtype=arr.real.dtype) + sq_true = np.array([np.complex(np.nan, np.nan), + np.complex(np.nan, np.nan), + np.complex(np.nan, np.nan), + np.complex(np.nan, np.inf), + np.complex(-np.inf, np.nan), + np.complex(np.inf, np.nan), + np.complex(0., 0.), + np.complex(np.nan, np.nan), + np.complex(np.nan, np.nan)], dtype=astype) + assert_equal(np.abs(arr[::stride]), abs_true[::stride]) + with np.errstate(invalid='ignore'): + assert_equal(np.square(arr[::stride]), sq_true[::stride]) + +class TestComplexAbsoluteAVX(object): + @pytest.mark.parametrize("arraysize", [1,2,3,4,5,6,7,8,9,10,11,13,15,17,18,19]) + @pytest.mark.parametrize("stride", [-4,-3,-2,-1,1,2,3,4]) + @pytest.mark.parametrize("astype", [np.complex64, np.complex128]) + # test to ensure masking and strides work as intended in the AVX implementation + def test_array(self, arraysize, stride, astype): + arr = np.ones(arraysize, dtype=astype) + abs_true = np.ones(arraysize, dtype=arr.real.dtype) + assert_equal(np.abs(arr[::stride]), abs_true[::stride]) diff --git a/numpy/core/tests/test_unicode.py b/numpy/core/tests/test_unicode.py index f16789148..8e0dd47cb 100644 --- a/numpy/core/tests/test_unicode.py +++ b/numpy/core/tests/test_unicode.py @@ -1,12 +1,8 @@ -import sys - import numpy as np -from numpy.compat import unicode from numpy.testing import assert_, assert_equal, assert_array_equal def buffer_length(arr): - if isinstance(arr, unicode): - arr = str(arr) + if isinstance(arr, str): if not arr: charmax = 0 else: diff --git a/numpy/core/umath.py b/numpy/core/umath.py index f3b26ab72..6a5474ffe 100644 --- a/numpy/core/umath.py +++ b/numpy/core/umath.py @@ -7,10 +7,11 @@ by importing from the extension module. """ from . import _multiarray_umath -from numpy.core._multiarray_umath import * -from numpy.core._multiarray_umath import ( - _UFUNC_API, _add_newdoc_ufunc, _ones_like - ) +from ._multiarray_umath import * # noqa: F403 +# These imports are needed for backward compatibility, +# do not change them. issue gh-11862 +# _ones_like is semi-public, on purpose not added to __all__ +from ._multiarray_umath import _UFUNC_API, _add_newdoc_ufunc, _ones_like __all__ = [ '_UFUNC_API', 'ERR_CALL', 'ERR_DEFAULT', 'ERR_IGNORE', 'ERR_LOG', diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py index ea7912feb..9ea083774 100644 --- a/numpy/distutils/ccompiler.py +++ b/numpy/distutils/ccompiler.py @@ -1,15 +1,19 @@ import os import re import sys -import types import shlex import time import subprocess from copy import copy from distutils import ccompiler -from distutils.ccompiler import * -from distutils.errors import DistutilsExecError, DistutilsModuleError, \ - DistutilsPlatformError, CompileError +from distutils.ccompiler import ( + compiler_class, gen_lib_options, get_default_compiler, new_compiler, + CCompiler +) +from distutils.errors import ( + DistutilsExecError, DistutilsModuleError, DistutilsPlatformError, + CompileError, UnknownFileError +) from distutils.sysconfig import customize_compiler from distutils.version import LooseVersion diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py index 1b4666888..f6a84e351 100644 --- a/numpy/distutils/command/build_clib.py +++ b/numpy/distutils/command/build_clib.py @@ -9,9 +9,10 @@ from distutils.errors import DistutilsSetupError, DistutilsError, \ from numpy.distutils import log from distutils.dep_util import newer_group -from numpy.distutils.misc_util import filter_sources, has_f_sources,\ - has_cxx_sources, all_strings, get_lib_source_files, is_sequence, \ - get_numpy_include_dirs +from numpy.distutils.misc_util import ( + filter_sources, get_lib_source_files, get_numpy_include_dirs, + has_cxx_sources, has_f_sources, is_sequence +) # Fix Python distutils bug sf #1718574: _l = old_build_clib.user_options @@ -48,8 +49,8 @@ class build_clib(old_build_clib): if self.parallel: try: self.parallel = int(self.parallel) - except ValueError: - raise ValueError("--parallel/-j argument must be an integer") + except ValueError as e: + raise ValueError("--parallel/-j argument must be an integer") from e old_build_clib.finalize_options(self) self.set_undefined_options('build', ('parallel', 'parallel'), diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index a0da83ff4..078b8fb59 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -13,11 +13,11 @@ from distutils.file_util import copy_file from numpy.distutils import log from numpy.distutils.exec_command import filepath_from_subprocess_output -from numpy.distutils.system_info import combine_paths, system_info -from numpy.distutils.misc_util import filter_sources, has_f_sources, \ - has_cxx_sources, get_ext_source_files, \ - get_numpy_include_dirs, is_sequence, get_build_architecture, \ - msvc_version +from numpy.distutils.system_info import combine_paths +from numpy.distutils.misc_util import ( + filter_sources, get_ext_source_files, get_numpy_include_dirs, + has_cxx_sources, has_f_sources, is_sequence +) from numpy.distutils.command.config_compiler import show_fortran_compilers @@ -52,8 +52,8 @@ class build_ext (old_build_ext): if self.parallel: try: self.parallel = int(self.parallel) - except ValueError: - raise ValueError("--parallel/-j argument must be an integer") + except ValueError as e: + raise ValueError("--parallel/-j argument must be an integer") from e # Ensure that self.include_dirs and self.distribution.include_dirs # refer to the same list object. finalize_options will modify diff --git a/numpy/distutils/command/config.py b/numpy/distutils/command/config.py index 2c833aad7..e54a54449 100644 --- a/numpy/distutils/command/config.py +++ b/numpy/distutils/command/config.py @@ -2,11 +2,12 @@ # try_compile call. try_run works but is untested for most of Fortran # compilers (they must define linker_exe first). # Pearu Peterson -import os, signal -import warnings -import sys +import os +import signal import subprocess +import sys import textwrap +import warnings from distutils.command.config import config as old_config from distutils.command.config import LANG_EXT diff --git a/numpy/distutils/conv_template.py b/numpy/distutils/conv_template.py index ec5a84a68..d08015fdf 100644 --- a/numpy/distutils/conv_template.py +++ b/numpy/distutils/conv_template.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ takes templated file .xxx.src and produces .xxx file where .xxx is .i or .c or .h, using the following template rules diff --git a/numpy/distutils/core.py b/numpy/distutils/core.py index a78bbf484..d5551f349 100644 --- a/numpy/distutils/core.py +++ b/numpy/distutils/core.py @@ -1,5 +1,5 @@ import sys -from distutils.core import * +from distutils.core import Distribution if 'setuptools' in sys.modules: have_setuptools = True @@ -25,7 +25,7 @@ from numpy.distutils.command import config, config_compiler, \ build, build_py, build_ext, build_clib, build_src, build_scripts, \ sdist, install_data, install_headers, install, bdist_rpm, \ install_clib -from numpy.distutils.misc_util import get_data_files, is_sequence, is_string +from numpy.distutils.misc_util import is_sequence, is_string numpy_cmdclass = {'build': build.build, 'build_src': build_src.build_src, diff --git a/numpy/distutils/cpuinfo.py b/numpy/distutils/cpuinfo.py index e066f9888..51ce3c129 100644 --- a/numpy/distutils/cpuinfo.py +++ b/numpy/distutils/cpuinfo.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ cpuinfo @@ -14,13 +14,15 @@ Pearu Peterson """ __all__ = ['cpu'] -import sys, re, types import os +import platform +import re +import sys +import types +import warnings from subprocess import getstatusoutput -import warnings -import platform def getoutput(cmd, successful_status=(0,), stacklevel=1): try: diff --git a/numpy/distutils/extension.py b/numpy/distutils/extension.py index 704f1e7aa..67114ef2e 100644 --- a/numpy/distutils/extension.py +++ b/numpy/distutils/extension.py @@ -6,7 +6,6 @@ modules in setup scripts. Overridden to support f2py. """ -import sys import re from distutils.extension import Extension as old_Extension diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py index a88b0d713..1c3069363 100644 --- a/numpy/distutils/fcompiler/__init__.py +++ b/numpy/distutils/fcompiler/__init__.py @@ -19,7 +19,6 @@ __all__ = ['FCompiler', 'new_fcompiler', 'show_fcompilers', import os import sys import re -import types from numpy.compat import open_latin1 diff --git a/numpy/distutils/fcompiler/environment.py b/numpy/distutils/fcompiler/environment.py index ae5fc404a..21a5be003 100644 --- a/numpy/distutils/fcompiler/environment.py +++ b/numpy/distutils/fcompiler/environment.py @@ -1,5 +1,4 @@ import os -import warnings from distutils.dist import Distribution __metaclass__ = type diff --git a/numpy/distutils/fcompiler/gnu.py b/numpy/distutils/fcompiler/gnu.py index 128e54db3..796dff351 100644 --- a/numpy/distutils/fcompiler/gnu.py +++ b/numpy/distutils/fcompiler/gnu.py @@ -10,7 +10,6 @@ import subprocess from subprocess import Popen, PIPE, STDOUT from numpy.distutils.exec_command import filepath_from_subprocess_output from numpy.distutils.fcompiler import FCompiler -from numpy.distutils.system_info import system_info compilers = ['GnuFCompiler', 'Gnu95FCompiler'] @@ -123,26 +122,17 @@ class GnuFCompiler(FCompiler): # error checking. if not target: # If MACOSX_DEPLOYMENT_TARGET is not set in the environment, - # we try to get it first from the Python Makefile and then we - # fall back to setting it to 10.3 to maximize the set of - # versions we can work with. This is a reasonable default + # we try to get it first from sysconfig and then + # fall back to setting it to 10.9 This is a reasonable default # even when using the official Python dist and those derived # from it. - import distutils.sysconfig as sc - g = {} - try: - get_makefile_filename = sc.get_makefile_filename - except AttributeError: - pass # i.e. PyPy - else: - filename = get_makefile_filename() - sc.parse_makefile(filename, g) - target = g.get('MACOSX_DEPLOYMENT_TARGET', '10.3') - os.environ['MACOSX_DEPLOYMENT_TARGET'] = target - if target == '10.3': - s = 'Env. variable MACOSX_DEPLOYMENT_TARGET set to 10.3' + import sysconfig + target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if not target: + target = '10.9' + s = f'Env. variable MACOSX_DEPLOYMENT_TARGET set to {target}' warnings.warn(s, stacklevel=2) - + os.environ['MACOSX_DEPLOYMENT_TARGET'] = target opt.extend(['-undefined', 'dynamic_lookup', '-bundle']) else: opt.append("-shared") diff --git a/numpy/distutils/fcompiler/pg.py b/numpy/distutils/fcompiler/pg.py index 77bc4f08e..eb628cb63 100644 --- a/numpy/distutils/fcompiler/pg.py +++ b/numpy/distutils/fcompiler/pg.py @@ -1,7 +1,7 @@ # http://www.pgroup.com import sys -from numpy.distutils.fcompiler import FCompiler, dummy_fortran_file +from numpy.distutils.fcompiler import FCompiler from sys import platform from os.path import join, dirname, normpath @@ -62,72 +62,60 @@ class PGroupFCompiler(FCompiler): return '-R%s' % dir -if sys.version_info >= (3, 5): - import functools +import functools - class PGroupFlangCompiler(FCompiler): - compiler_type = 'flang' - description = 'Portland Group Fortran LLVM Compiler' - version_pattern = r'\s*(flang|clang) version (?P<version>[\d.-]+).*' +class PGroupFlangCompiler(FCompiler): + compiler_type = 'flang' + description = 'Portland Group Fortran LLVM Compiler' + version_pattern = r'\s*(flang|clang) version (?P<version>[\d.-]+).*' - ar_exe = 'lib.exe' - possible_executables = ['flang'] + ar_exe = 'lib.exe' + possible_executables = ['flang'] - executables = { - 'version_cmd': ["<F77>", "--version"], - 'compiler_f77': ["flang"], - 'compiler_fix': ["flang"], - 'compiler_f90': ["flang"], - 'linker_so': [None], - 'archiver': [ar_exe, "/verbose", "/OUT:"], - 'ranlib': None - } - - library_switch = '/OUT:' # No space after /OUT:! - module_dir_switch = '-module ' # Don't remove ending space! - - def get_libraries(self): - opt = FCompiler.get_libraries(self) - opt.extend(['flang', 'flangrti', 'ompstub']) - return opt - - @functools.lru_cache(maxsize=128) - def get_library_dirs(self): - """List of compiler library directories.""" - opt = FCompiler.get_library_dirs(self) - flang_dir = dirname(self.executables['compiler_f77'][0]) - opt.append(normpath(join(flang_dir, '..', 'lib'))) + executables = { + 'version_cmd': ["<F77>", "--version"], + 'compiler_f77': ["flang"], + 'compiler_fix': ["flang"], + 'compiler_f90': ["flang"], + 'linker_so': [None], + 'archiver': [ar_exe, "/verbose", "/OUT:"], + 'ranlib': None + } - return opt + library_switch = '/OUT:' # No space after /OUT:! + module_dir_switch = '-module ' # Don't remove ending space! - def get_flags(self): - return [] + def get_libraries(self): + opt = FCompiler.get_libraries(self) + opt.extend(['flang', 'flangrti', 'ompstub']) + return opt - def get_flags_free(self): - return [] + @functools.lru_cache(maxsize=128) + def get_library_dirs(self): + """List of compiler library directories.""" + opt = FCompiler.get_library_dirs(self) + flang_dir = dirname(self.executables['compiler_f77'][0]) + opt.append(normpath(join(flang_dir, '..', 'lib'))) - def get_flags_debug(self): - return ['-g'] + return opt - def get_flags_opt(self): - return ['-O3'] + def get_flags(self): + return [] - def get_flags_arch(self): - return [] + def get_flags_free(self): + return [] - def runtime_library_dir_option(self, dir): - raise NotImplementedError + def get_flags_debug(self): + return ['-g'] -else: - from numpy.distutils.fcompiler import CompilerNotFound + def get_flags_opt(self): + return ['-O3'] - # No point in supporting on older Pythons because not ABI compatible - class PGroupFlangCompiler(FCompiler): - compiler_type = 'flang' - description = 'Portland Group Fortran LLVM Compiler' + def get_flags_arch(self): + return [] - def get_version(self): - raise CompilerNotFound('Flang unsupported on Python < 3.5') + def runtime_library_dir_option(self, dir): + raise NotImplementedError if __name__ == '__main__': diff --git a/numpy/distutils/from_template.py b/numpy/distutils/from_template.py index b4dd05b5e..070b7d8b8 100644 --- a/numpy/distutils/from_template.py +++ b/numpy/distutils/from_template.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ process_file(filename) diff --git a/numpy/distutils/lib2def.py b/numpy/distutils/lib2def.py index c6d445d09..820ed71f5 100644 --- a/numpy/distutils/lib2def.py +++ b/numpy/distutils/lib2def.py @@ -22,7 +22,7 @@ __version__ = '0.1a' py_ver = "%d%d" % tuple(sys.version_info[:2]) -DEFAULT_NM = 'nm -Cs' +DEFAULT_NM = ['nm', '-Cs'] DEF_HEADER = """LIBRARY python%s.dll ;CODE PRELOAD MOVEABLE DISCARDABLE @@ -59,13 +59,16 @@ libfile, deffile = parse_cmd()""" deffile = None return libfile, deffile -def getnm(nm_cmd = ['nm', '-Cs', 'python%s.lib' % py_ver]): +def getnm(nm_cmd=['nm', '-Cs', 'python%s.lib' % py_ver], shell=True): """Returns the output of nm_cmd via a pipe. -nm_output = getnam(nm_cmd = 'nm -Cs py_lib')""" - f = subprocess.Popen(nm_cmd, shell=True, stdout=subprocess.PIPE, universal_newlines=True) - nm_output = f.stdout.read() - f.stdout.close() +nm_output = getnm(nm_cmd = 'nm -Cs py_lib')""" + p = subprocess.Popen(nm_cmd, shell=shell, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) + nm_output, nm_err = p.communicate() + if p.returncode != 0: + raise RuntimeError('failed to run "%s": "%s"' % ( + ' '.join(nm_cmd), nm_err)) return nm_output def parse_nm(nm_output): @@ -107,7 +110,7 @@ if __name__ == '__main__': deffile = sys.stdout else: deffile = open(deffile, 'w') - nm_cmd = [str(DEFAULT_NM), str(libfile)] - nm_output = getnm(nm_cmd) + nm_cmd = DEFAULT_NM + [str(libfile)] + nm_output = getnm(nm_cmd, shell=False) dlist, flist = parse_nm(nm_output) output_def(dlist, flist, DEF_HEADER, deffile) diff --git a/numpy/distutils/line_endings.py b/numpy/distutils/line_endings.py index 92a84e56a..686e5ebd9 100644 --- a/numpy/distutils/line_endings.py +++ b/numpy/distutils/line_endings.py @@ -1,7 +1,10 @@ """ Functions for converting from DOS to UNIX line endings """ -import sys, re, os +import os +import re +import sys + def dos2unix(file): "Replace CRLF with LF in argument files. Print names of changed files." diff --git a/numpy/distutils/log.py b/numpy/distutils/log.py index 79eec00a6..a8113b9c6 100644 --- a/numpy/distutils/log.py +++ b/numpy/distutils/log.py @@ -1,6 +1,6 @@ -# Colored log, requires Python 2.3 or up. +# Colored log import sys -from distutils.log import * +from distutils.log import * # noqa: F403 from distutils.log import Log as old_Log from distutils.log import _global_log diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index 475f73718..7cb6fadcc 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -14,7 +14,7 @@ import re import textwrap # Overwrite certain distutils.ccompiler functions: -import numpy.distutils.ccompiler +import numpy.distutils.ccompiler # noqa: F401 from numpy.distutils import log # NT stuff # 1. Make sure libpython<version>.a exists for gcc. If not, build it. @@ -26,8 +26,7 @@ import distutils.cygwinccompiler from distutils.version import StrictVersion from distutils.unixccompiler import UnixCCompiler from distutils.msvccompiler import get_build_version as get_build_msvc_version -from distutils.errors import (DistutilsExecError, CompileError, - UnknownFileError) +from distutils.errors import UnknownFileError from numpy.distutils.misc_util import (msvc_runtime_library, msvc_runtime_version, msvc_runtime_major, @@ -65,10 +64,10 @@ class Mingw32CCompiler(distutils.cygwinccompiler.CygwinCCompiler): # we need to support 3.2 which doesn't match the standard # get_versions methods regex if self.gcc_version is None: - p = subprocess.Popen(['gcc', '-dumpversion'], shell=True, - stdout=subprocess.PIPE) - out_string = p.stdout.read() - p.stdout.close() + try: + out_string = subprocess.check_output(['gcc', '-dumpversion']) + except (OSError, CalledProcessError): + out_string = "" # ignore failures to match old behavior result = re.search(r'(\d+\.\d+)', out_string) if result: self.gcc_version = StrictVersion(result.group(1)) @@ -279,8 +278,8 @@ def find_python_dll(): raise ValueError("%s not found in %s" % (dllname, lib_dirs)) def dump_table(dll): - st = subprocess.Popen(["objdump.exe", "-p", dll], stdout=subprocess.PIPE) - return st.stdout.readlines() + st = subprocess.check_output(["objdump.exe", "-p", dll]) + return st.split(b'\n') def generate_def(dll, dfile): """Given a dll file location, get all its exported symbols and dump them @@ -305,15 +304,14 @@ def generate_def(dll, dfile): if len(syms) == 0: log.warn('No symbols found in %s' % dll) - d = open(dfile, 'w') - d.write('LIBRARY %s\n' % os.path.basename(dll)) - d.write(';CODE PRELOAD MOVEABLE DISCARDABLE\n') - d.write(';DATA PRELOAD SINGLE\n') - d.write('\nEXPORTS\n') - for s in syms: - #d.write('@%d %s\n' % (s[0], s[1])) - d.write('%s\n' % s[1]) - d.close() + with open(dfile, 'w') as d: + d.write('LIBRARY %s\n' % os.path.basename(dll)) + d.write(';CODE PRELOAD MOVEABLE DISCARDABLE\n') + d.write(';DATA PRELOAD SINGLE\n') + d.write('\nEXPORTS\n') + for s in syms: + #d.write('@%d %s\n' % (s[0], s[1])) + d.write('%s\n' % s[1]) def find_dll(dll_name): @@ -466,7 +464,7 @@ def _build_import_library_amd64(): # generate import library from this symbol list cmd = ['dlltool', '-d', def_file, '-l', out_file] - subprocess.Popen(cmd) + subprocess.check_call(cmd) def _build_import_library_x86(): """ Build the import libraries for Mingw32-gcc on Windows @@ -500,16 +498,19 @@ def _build_import_library_x86(): def_name = "python%d%d.def" % tuple(sys.version_info[:2]) def_file = os.path.join(sys.prefix, 'libs', def_name) - nm_cmd = '%s %s' % (lib2def.DEFAULT_NM, lib_file) - nm_output = lib2def.getnm(nm_cmd) + nm_output = lib2def.getnm( + lib2def.DEFAULT_NM + [lib_file], shell=False) dlist, flist = lib2def.parse_nm(nm_output) - lib2def.output_def(dlist, flist, lib2def.DEF_HEADER, open(def_file, 'w')) + with open(def_file, 'w') as fid: + lib2def.output_def(dlist, flist, lib2def.DEF_HEADER, fid) dll_name = find_python_dll () - args = (dll_name, def_file, out_file) - cmd = 'dlltool --dllname "%s" --def "%s" --output-lib "%s"' % args - status = os.system(cmd) - # for now, fail silently + + cmd = ["dlltool", + "--dllname", dll_name, + "--def", def_file, + "--output-lib", out_file] + status = subprocess.check_output(cmd) if status: log.warn('Failed to build import library for gcc. Linking will fail.') return @@ -542,6 +543,8 @@ if sys.platform == 'win32': # Value from msvcrt.CRT_ASSEMBLY_VERSION under Python 3.3.0 # on Windows XP: _MSVCRVER_TO_FULLVER['100'] = "10.0.30319.460" + # Python 3.7 uses 1415, but get_build_version returns 140 ?? + _MSVCRVER_TO_FULLVER['140'] = "14.15.26726.0" if hasattr(msvcrt, "CRT_ASSEMBLY_VERSION"): major, minor, rest = msvcrt.CRT_ASSEMBLY_VERSION.split(".", 2) _MSVCRVER_TO_FULLVER[major + minor] = msvcrt.CRT_ASSEMBLY_VERSION diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index f9d2be716..9f9e9f1ac 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -32,7 +32,6 @@ def clean_up_temporary_directory(): atexit.register(clean_up_temporary_directory) -from numpy.compat import basestring from numpy.compat import npy_load_module __all__ = ['Configuration', 'get_numpy_include_dirs', 'default_config_dict', @@ -165,7 +164,6 @@ def get_path_from_frame(frame, parent_path=None): # we're probably running setup.py as execfile("setup.py") # (likely we're building an egg) d = os.path.abspath('.') - # hmm, should we use sys.argv[0] like in __builtin__ case? if parent_path is not None: d = rel_path(d, parent_path) @@ -450,7 +448,7 @@ def _get_f90_modules(source): return modules def is_string(s): - return isinstance(s, basestring) + return isinstance(s, str) def all_strings(lst): """Return True if all items in lst are string objects. """ @@ -923,18 +921,8 @@ class Configuration: else: pn = dot_join(*([parent_name] + subpackage_name.split('.')[:-1])) args = (pn,) - def fix_args_py2(args): - if setup_module.configuration.__code__.co_argcount > 1: - args = args + (self.top_path,) - return args - def fix_args_py3(args): - if setup_module.configuration.__code__.co_argcount > 1: - args = args + (self.top_path,) - return args - if sys.version_info[0] < 3: - args = fix_args_py2(args) - else: - args = fix_args_py3(args) + if setup_module.configuration.__code__.co_argcount > 1: + args = args + (self.top_path,) config = setup_module.configuration(*args) if config.name!=dot_join(parent_name, subpackage_name): self.warn('Subpackage %r configuration returned as %r' % \ @@ -1865,8 +1853,7 @@ class Configuration: """Return path's SVN revision number. """ try: - output = subprocess.check_output( - ['svnversion'], shell=True, cwd=path) + output = subprocess.check_output(['svnversion'], cwd=path) except (subprocess.CalledProcessError, OSError): pass else: @@ -1896,7 +1883,7 @@ class Configuration: """ try: output = subprocess.check_output( - ['hg identify --num'], shell=True, cwd=path) + ['hg', 'identify', '--num'], cwd=path) except (subprocess.CalledProcessError, OSError): pass else: @@ -2338,6 +2325,43 @@ def generate_config_py(target): return g.get(name, g.get(name + "_info", {})) def show(): + """ + Show libraries in the system on which NumPy was built. + + Print information about various resources (libraries, library + directories, include directories, etc.) in the system on which + NumPy was built. + + See Also + -------- + get_include : Returns the directory containing NumPy C + header files. + + Notes + ----- + Classes specifying the information to be printed are defined + in the `numpy.distutils.system_info` module. + + Information may include: + + * ``language``: language used to write the libraries (mostly + C or f77) + * ``libraries``: names of libraries found in the system + * ``library_dirs``: directories containing the libraries + * ``include_dirs``: directories containing library header files + * ``src_dirs``: directories containing library source files + * ``define_macros``: preprocessor macros used by + ``distutils.setup`` + + Examples + -------- + >>> np.show_config() + blas_opt_info: + language = c + define_macros = [('HAVE_CBLAS', None)] + libraries = ['openblas', 'openblas'] + library_dirs = ['/usr/local/lib'] + """ for name,info_dict in globals().items(): if name[0] == "_" or type(info_dict) is not type({}): continue print(name + ":") diff --git a/numpy/distutils/npy_pkg_config.py b/numpy/distutils/npy_pkg_config.py index 26a0437fb..951ce5fb8 100644 --- a/numpy/distutils/npy_pkg_config.py +++ b/numpy/distutils/npy_pkg_config.py @@ -375,7 +375,6 @@ def read_config(pkgname, dirs=None): # pkg-config simple emulator - useful for debugging, and maybe later to query # the system if __name__ == '__main__': - import sys from optparse import OptionParser import glob diff --git a/numpy/distutils/setup.py b/numpy/distutils/setup.py index 415d2adeb..69d35f5c2 100644 --- a/numpy/distutils/setup.py +++ b/numpy/distutils/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('distutils', parent_package, top_path) diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index f0641a688..3a6a7b29d 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -1,59 +1,8 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ This file defines a set of system_info classes for getting information about various resources (libraries, library directories, -include directories, etc.) in the system. Currently, the following -classes are available: - - atlas_info - atlas_threads_info - atlas_blas_info - atlas_blas_threads_info - lapack_atlas_info - lapack_atlas_threads_info - atlas_3_10_info - atlas_3_10_threads_info - atlas_3_10_blas_info, - atlas_3_10_blas_threads_info, - lapack_atlas_3_10_info - lapack_atlas_3_10_threads_info - flame_info - blas_info - lapack_info - openblas_info - openblas64__info - openblas_ilp64_info - blis_info - blas_opt_info # usage recommended - lapack_opt_info # usage recommended - blas_ilp64_opt_info # usage recommended (general ILP64 BLAS) - lapack_ilp64_opt_info # usage recommended (general ILP64 LAPACK) - blas_ilp64_plain_opt_info # usage recommended (general ILP64 BLAS, no symbol suffix) - lapack_ilp64_plain_opt_info # usage recommended (general ILP64 LAPACK, no symbol suffix) - blas64__opt_info # usage recommended (general ILP64 BLAS, 64_ symbol suffix) - lapack64__opt_info # usage recommended (general ILP64 LAPACK, 64_ symbol suffix) - fftw_info,dfftw_info,sfftw_info - fftw_threads_info,dfftw_threads_info,sfftw_threads_info - djbfft_info - x11_info - lapack_src_info - blas_src_info - numpy_info - numarray_info - numpy_info - boost_python_info - agg2_info - wx_info - gdk_pixbuf_xlib_2_info - gdk_pixbuf_2_info - gdk_x11_2_info - gtkp_x11_2_info - gtkp_2_info - xft_info - freetype2_info - umfpack_info - -Usage: +include directories, etc.) in the system. Usage: info_dict = get_info(<name>) where <name> is a string 'atlas','x11','fftw','lapack','blas', 'lapack_src', 'blas_src', etc. For a complete list of allowed names, @@ -81,19 +30,94 @@ The file 'site.cfg' is looked for in The first one found is used to get system configuration options The format is that used by ConfigParser (i.e., Windows .INI style). The -section ALL has options that are the default for each section. The -available sections are fftw, atlas, and x11. Appropriate defaults are -used if nothing is specified. +section ALL is not intended for general use. + +Appropriate defaults are used if nothing is specified. The order of finding the locations of resources is the following: 1. environment variable 2. section in site.cfg - 3. ALL section in site.cfg + 3. DEFAULT section in site.cfg + 4. System default search paths (see ``default_*`` variables below). Only the first complete match is returned. +Currently, the following classes are available, along with their section names: + + Numeric_info:Numeric + _numpy_info:Numeric + _pkg_config_info:None + accelerate_info:accelerate + agg2_info:agg2 + amd_info:amd + atlas_3_10_blas_info:atlas + atlas_3_10_blas_threads_info:atlas + atlas_3_10_info:atlas + atlas_3_10_threads_info:atlas + atlas_blas_info:atlas + atlas_blas_threads_info:atlas + atlas_info:atlas + atlas_threads_info:atlas + blas64__opt_info:ALL # usage recommended (general ILP64 BLAS, 64_ symbol suffix) + blas_ilp64_opt_info:ALL # usage recommended (general ILP64 BLAS) + blas_ilp64_plain_opt_info:ALL # usage recommended (general ILP64 BLAS, no symbol suffix) + blas_info:blas + blas_mkl_info:mkl + blas_opt_info:ALL # usage recommended + blas_src_info:blas_src + blis_info:blis + boost_python_info:boost_python + dfftw_info:fftw + dfftw_threads_info:fftw + djbfft_info:djbfft + f2py_info:ALL + fft_opt_info:ALL + fftw2_info:fftw + fftw3_info:fftw3 + fftw_info:fftw + fftw_threads_info:fftw + flame_info:flame + freetype2_info:freetype2 + gdk_2_info:gdk_2 + gdk_info:gdk + gdk_pixbuf_2_info:gdk_pixbuf_2 + gdk_pixbuf_xlib_2_info:gdk_pixbuf_xlib_2 + gdk_x11_2_info:gdk_x11_2 + gtkp_2_info:gtkp_2 + gtkp_x11_2_info:gtkp_x11_2 + lapack64__opt_info:ALL # usage recommended (general ILP64 LAPACK, 64_ symbol suffix) + lapack_atlas_3_10_info:atlas + lapack_atlas_3_10_threads_info:atlas + lapack_atlas_info:atlas + lapack_atlas_threads_info:atlas + lapack_ilp64_opt_info:ALL # usage recommended (general ILP64 LAPACK) + lapack_ilp64_plain_opt_info:ALL # usage recommended (general ILP64 LAPACK, no symbol suffix) + lapack_info:lapack + lapack_mkl_info:mkl + lapack_opt_info:ALL # usage recommended + lapack_src_info:lapack_src + mkl_info:mkl + numarray_info:numarray + numerix_info:numerix + numpy_info:numpy + openblas64__info:openblas64_ + openblas64__lapack_info:openblas64_ + openblas_clapack_info:openblas + openblas_ilp64_info:openblas_ilp64 + openblas_ilp64_lapack_info:openblas_ilp64 + openblas_info:openblas + openblas_lapack_info:openblas + sfftw_info:fftw + sfftw_threads_info:fftw + system_info:ALL + umfpack_info:umfpack + wx_info:wx + x11_info:x11 + xft_info:xft + Example: ---------- -[ALL] +[DEFAULT] +# default section library_dirs = /usr/lib:/usr/local/lib:/opt/lib include_dirs = /usr/include:/usr/local/include:/opt/include src_dirs = /usr/local/src:/opt/src @@ -153,7 +177,7 @@ from distutils.util import get_platform from numpy.distutils.exec_command import ( find_executable, filepath_from_subprocess_output, - get_pythonexe) + ) from numpy.distutils.misc_util import (is_sequence, is_string, get_shared_lib_extension) from numpy.distutils.command.config import config as cmd_config @@ -576,10 +600,14 @@ class system_info: """ get_info() is the only public method. Don't use others. """ - section = 'ALL' dir_env_var = None - search_static_first = 0 # XXX: disabled by default, may disappear in - # future unless it is proved to be useful. + # XXX: search_static_first is disabled by default, may disappear in + # future unless it is proved to be useful. + search_static_first = 0 + # The base-class section name is a random word "ALL" and is not really + # intended for general use. It cannot be None nor can it be DEFAULT as + # these break the ConfigParser. See gh-15338 + section = 'ALL' saved_results = {} notfounderror = NotFoundError diff --git a/numpy/distutils/tests/test_fcompiler.py b/numpy/distutils/tests/test_fcompiler.py index c559becf8..dd97f1e72 100644 --- a/numpy/distutils/tests/test_fcompiler.py +++ b/numpy/distutils/tests/test_fcompiler.py @@ -1,6 +1,4 @@ -import pytest - -from numpy.testing import assert_, suppress_warnings +from numpy.testing import assert_ import numpy.distutils.fcompiler customizable_flags = [ diff --git a/numpy/distutils/tests/test_mingw32ccompiler.py b/numpy/distutils/tests/test_mingw32ccompiler.py new file mode 100644 index 000000000..ebedacb32 --- /dev/null +++ b/numpy/distutils/tests/test_mingw32ccompiler.py @@ -0,0 +1,42 @@ +import shutil +import subprocess +import sys +import pytest + +from numpy.distutils import mingw32ccompiler + + +@pytest.mark.skipif(sys.platform != 'win32', reason='win32 only test') +def test_build_import(): + '''Test the mingw32ccompiler.build_import_library, which builds a + `python.a` from the MSVC `python.lib` + ''' + + # make sure `nm.exe` exists and supports the current python version. This + # can get mixed up when the PATH has a 64-bit nm but the python is 32-bit + try: + out = subprocess.check_output(['nm.exe', '--help']) + except FileNotFoundError: + pytest.skip("'nm.exe' not on path, is mingw installed?") + supported = out[out.find(b'supported targets:'):] + if sys.maxsize < 2**32: + if b'pe-i386' not in supported: + raise ValueError("'nm.exe' found but it does not support 32-bit " + "dlls when using 32-bit python. Supported " + "formats: '%s'" % supported) + elif b'pe-x86-64' not in supported: + raise ValueError("'nm.exe' found but it does not support 64-bit " + "dlls when using 64-bit python. Supported " + "formats: '%s'" % supported) + # Hide the import library to force a build + has_import_lib, fullpath = mingw32ccompiler._check_for_import_lib() + if has_import_lib: + shutil.move(fullpath, fullpath + '.bak') + + try: + # Whew, now we can actually test the function + mingw32ccompiler.build_import_library() + + finally: + if has_import_lib: + shutil.move(fullpath + '.bak', fullpath) diff --git a/numpy/distutils/tests/test_shell_utils.py b/numpy/distutils/tests/test_shell_utils.py index 754609a5d..32bd283e5 100644 --- a/numpy/distutils/tests/test_shell_utils.py +++ b/numpy/distutils/tests/test_shell_utils.py @@ -1,6 +1,5 @@ import pytest import subprocess -import os import json import sys diff --git a/numpy/distutils/tests/test_system_info.py b/numpy/distutils/tests/test_system_info.py index c40cdb7db..0768ffdde 100644 --- a/numpy/distutils/tests/test_system_info.py +++ b/numpy/distutils/tests/test_system_info.py @@ -7,7 +7,7 @@ from distutils.errors import DistutilsError from numpy.testing import assert_, assert_equal, assert_raises from numpy.distutils import ccompiler, customized_ccompiler -from numpy.distutils.system_info import system_info, ConfigParser +from numpy.distutils.system_info import system_info, ConfigParser, mkl_info from numpy.distutils.system_info import AliasedOptionError from numpy.distutils.system_info import default_lib_dirs, default_include_dirs from numpy.distutils import _shell_utils @@ -253,3 +253,35 @@ class TestSystemInfoReading: assert_(os.path.isfile(self._src2.replace('.c', '.o'))) finally: os.chdir(previousDir) + + def test_overrides(self): + previousDir = os.getcwd() + cfg = os.path.join(self._dir1, 'site.cfg') + shutil.copy(self._sitecfg, cfg) + try: + os.chdir(self._dir1) + # Check that the '[ALL]' section does not override + # missing values from other sections + info = mkl_info() + lib_dirs = info.cp['ALL']['library_dirs'].split(os.pathsep) + assert info.get_lib_dirs() != lib_dirs + + # But if we copy the values to a '[mkl]' section the value + # is correct + with open(cfg, 'r') as fid: + mkl = fid.read().replace('ALL', 'mkl') + with open(cfg, 'w') as fid: + fid.write(mkl) + info = mkl_info() + assert info.get_lib_dirs() == lib_dirs + + # Also, the values will be taken from a section named '[DEFAULT]' + with open(cfg, 'r') as fid: + dflt = fid.read().replace('mkl', 'DEFAULT') + with open(cfg, 'w') as fid: + fid.write(dflt) + info = mkl_info() + assert info.get_lib_dirs() == lib_dirs + finally: + os.chdir(previousDir) + diff --git a/numpy/distutils/unixccompiler.py b/numpy/distutils/unixccompiler.py index cf62cb019..5f36c439f 100644 --- a/numpy/distutils/unixccompiler.py +++ b/numpy/distutils/unixccompiler.py @@ -4,8 +4,8 @@ unixccompiler - can handle very long argument lists for ar. """ import os -from distutils.errors import DistutilsExecError, CompileError -from distutils.unixccompiler import * +from distutils.errors import CompileError, DistutilsExecError, LibError +from distutils.unixccompiler import UnixCCompiler from numpy.distutils.ccompiler import replace_method from numpy.distutils.misc_util import _commandline_dep_string from numpy.distutils import log diff --git a/numpy/doc/constants.py b/numpy/doc/constants.py index 96813b66c..2c629ad33 100644 --- a/numpy/doc/constants.py +++ b/numpy/doc/constants.py @@ -13,7 +13,8 @@ NumPy includes several constants: # # Note: the docstring is autogenerated. # -import textwrap, re +import re +import textwrap # Maintain same format as in numpy.add_newdocs constants = [] diff --git a/numpy/doc/glossary.py b/numpy/doc/glossary.py index 6d2e0010f..16a3b9241 100644 --- a/numpy/doc/glossary.py +++ b/numpy/doc/glossary.py @@ -159,7 +159,7 @@ Glossary field In a :term:`structured data type`, each sub-type is called a `field`. - The `field` has a name (a string), a type (any valid dtype, and + The `field` has a name (a string), a type (any valid dtype), and an optional `title`. See :ref:`arrays.dtypes` Fortran order diff --git a/numpy/doc/indexing.py b/numpy/doc/indexing.py index aa84e2b11..c7dda2790 100644 --- a/numpy/doc/indexing.py +++ b/numpy/doc/indexing.py @@ -285,14 +285,23 @@ Combining index arrays with slices Index arrays may be combined with slices. For example: :: - >>> y[np.array([0,2,4]),1:3] + >>> y[np.array([0, 2, 4]), 1:3] array([[ 1, 2], [15, 16], [29, 30]]) -In effect, the slice is converted to an index array -np.array([[1,2]]) (shape (1,2)) that is broadcast with the index array -to produce a resultant array of shape (3,2). +In effect, the slice and index array operation are independent. +The slice operation extracts columns with index 1 and 2, +(i.e. the 2nd and 3rd columns), +followed by the index array operation which extracts rows with +index 0, 2 and 4 (i.e the first, third and fifth rows). + +This is equivalent to:: + + >>> y[:, 1:3][np.array([0, 2, 4]), :] + array([[ 1, 2], + [15, 16], + [29, 30]]) Likewise, slicing can be combined with broadcasted boolean indices: :: @@ -372,8 +381,7 @@ exceptions (assigning complex to floats or ints): :: >>> x[1] 1 >>> x[1] = 1.2j - <type 'exceptions.TypeError'>: can't convert complex to long; use - long(abs(z)) + TypeError: can't convert complex to int Unlike some of the references (such as array and mask indices) diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index 5a54ddd90..7dc10e1c8 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -72,7 +72,7 @@ class, that points to the data in the original. There are other points in the use of ndarrays where we need such views, such as copying arrays (``c_arr.copy()``), creating ufunc output arrays (see also :ref:`array-wrap`), and reducing methods (like -``c_arr.mean()``. +``c_arr.mean()``). Relationship of view casting and new-from-template -------------------------------------------------- diff --git a/numpy/f2py/__init__.py b/numpy/f2py/__init__.py index 13ffef6fc..949bac0ff 100644 --- a/numpy/f2py/__init__.py +++ b/numpy/f2py/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Fortran to Python Interface Generator. """ @@ -8,8 +8,6 @@ import sys import subprocess import os -import numpy as np - from . import f2py2e from . import f2py_testing from . import diagnose @@ -87,7 +85,7 @@ def compile(source, args = ['-c', '-m', modulename, f.name] - if isinstance(extra_args, np.compat.basestring): + if isinstance(extra_args, str): is_posix = (os.name == 'posix') extra_args = shlex.split(extra_args, posix=is_posix) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index 31802621e..80b150655 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Auxiliary functions for f2py2e. diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index 6f9ff7bc6..fabbfc4c2 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Copyright 1999,2000 Pearu Peterson all rights reserved, @@ -19,11 +19,10 @@ f2py_version = __version__.version import copy import re import os -import sys from .crackfortran import markoutercomma from . import cb_rules -# The eviroment provided by auxfuncs.py is needed for some calls to eval. +# The environment provided by auxfuncs.py is needed for some calls to eval. # As the needed functions cannot be determined by static inspection of the # code, it is safest to use import * pending a major refactoring of f2py. from .auxfuncs import * @@ -147,11 +146,7 @@ c2buildvalue_map = {'double': 'd', 'complex_float': 'N', 'complex_double': 'N', 'complex_long_double': 'N', - 'string': 'z'} - -if sys.version_info[0] >= 3: - # Bytes, not Unicode strings - c2buildvalue_map['string'] = 'y' + 'string': 'y'} if using_newcore: # c2buildvalue_map=??? @@ -323,7 +318,6 @@ def getstrlength(var): def getarrdims(a, var, verbose=0): - global depargs ret = {} if isstring(var) and not isarray(var): ret['dims'] = getstrlength(var) @@ -519,7 +513,6 @@ def sign2map(a, var): varrfromat intent """ - global lcb_map, cb_map out_a = a if isintent_out(var): for k in var['intent']: diff --git a/numpy/f2py/cb_rules.py b/numpy/f2py/cb_rules.py index dc178078d..87887c152 100644 --- a/numpy/f2py/cb_rules.py +++ b/numpy/f2py/cb_rules.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Build call-back mechanism for f2py2e. @@ -436,7 +436,6 @@ cb_map = {} def buildcallbacks(m): - global cb_map cb_map[m['name']] = [] for bi in m['body']: if bi['block'] == 'interface': @@ -448,7 +447,6 @@ def buildcallbacks(m): def buildcallback(rout, um): - global cb_map from . import capi_maps outmess('\tConstructing call-back function "cb_%s_in_%s"\n' % diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index 28b2c0670..f1ac214d4 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ C declarations, CPP macros, and C functions for f2py2e. @@ -1033,6 +1033,8 @@ cfuncs[ 'try_pyarr_from_complex_double'] = 'static int try_pyarr_from_complex_double(PyObject* obj,complex_double* v) {\n TRYCOMPLEXPYARRAYTEMPLATE(double,\'D\');\n}\n' needs['create_cb_arglist'] = ['CFUNCSMESS', 'PRINTPYOBJERR', 'MINMAX'] + +# create the list of arguments to be used when calling back to python cfuncs['create_cb_arglist'] = """\ static int create_cb_arglist(PyObject* fun,PyTupleObject* xa,const int maxnofargs,const int nofoptargs,int *nofargs,PyTupleObject **args,const char *errmess) { PyObject *tmp = NULL; @@ -1058,6 +1060,10 @@ static int create_cb_arglist(PyObject* fun,PyTupleObject* xa,const int maxnofarg tmp_fun = fun; /* built-in function */ Py_INCREF(tmp_fun); tot = maxnofargs; + if (PyCFunction_Check(fun)) { + /* In case the function has a co_argcount (like on PyPy) */ + di = 0; + } if (xa != NULL) tot += PyTuple_Size((PyObject *)xa); } @@ -1156,7 +1162,7 @@ def buildcfuncs(): ############ Auxiliary functions for sorting needs ################### def append_needs(need, flag=1): - global outneeds, needs + # This function modifies the contents of the global `outneeds` dict. if isinstance(need, list): for n in need: append_needs(n, flag) @@ -1223,7 +1229,7 @@ def append_needs(need, flag=1): def get_needs(): - global outneeds, needs + # This function modifies the contents of the global `outneeds` dict. res = {} for n in outneeds.keys(): out = [] diff --git a/numpy/f2py/common_rules.py b/numpy/f2py/common_rules.py index 31aefcda9..90483e55b 100644 --- a/numpy/f2py/common_rules.py +++ b/numpy/f2py/common_rules.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Build common block mechanism for f2py2e. diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index 09bab11bd..3d2f97a56 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ crackfortran --- read fortran (77,90) code and extract declaration information. @@ -148,7 +148,7 @@ import platform from . import __version__ -# The eviroment provided by auxfuncs.py is needed for some calls to eval. +# The environment provided by auxfuncs.py is needed for some calls to eval. # As the needed functions cannot be determined by static inspection of the # code, it is safest to use import * pending a major refactoring of f2py. from .auxfuncs import * @@ -579,8 +579,8 @@ publicpattern = re.compile( beforethisafter % ('', 'public', 'public', '.*'), re.I), 'public' privatepattern = re.compile( beforethisafter % ('', 'private', 'private', '.*'), re.I), 'private' -intrisicpattern = re.compile( - beforethisafter % ('', 'intrisic', 'intrisic', '.*'), re.I), 'intrisic' +intrinsicpattern = re.compile( + beforethisafter % ('', 'intrinsic', 'intrinsic', '.*'), re.I), 'intrinsic' intentpattern = re.compile(beforethisafter % ( '', 'intent|depend|note|check', 'intent|depend|note|check', r'\s*\(.*?\).*'), re.I), 'intent' parameterpattern = re.compile( @@ -705,7 +705,7 @@ def crackline(line, reset=0): for pat in [dimensionpattern, externalpattern, intentpattern, optionalpattern, requiredpattern, parameterpattern, datapattern, publicpattern, privatepattern, - intrisicpattern, + intrinsicpattern, endifpattern, endpattern, formatpattern, beginpattern, functionpattern, subroutinepattern, @@ -1097,7 +1097,7 @@ def analyzeline(m, case, line): last_name = updatevars(typespec, selector, attr, edecl) if last_name is not None: previous_context = ('variable', last_name, groupcounter) - elif case in ['dimension', 'intent', 'optional', 'required', 'external', 'public', 'private', 'intrisic']: + elif case in ['dimension', 'intent', 'optional', 'required', 'external', 'public', 'private', 'intrinsic']: edecl = groupcache[groupcounter]['vars'] ll = m.group('after').strip() i = ll.find('::') @@ -1157,7 +1157,7 @@ def analyzeline(m, case, line): else: errmess('analyzeline: intent(callback) %s is already' ' in argument list' % (k)) - if case in ['optional', 'required', 'public', 'external', 'private', 'intrisic']: + if case in ['optional', 'required', 'public', 'external', 'private', 'intrinsic']: ap = case if 'attrspec' in edecl[k]: edecl[k]['attrspec'].append(ap) @@ -1749,10 +1749,12 @@ def setattrspec(decl, attr, force=0): decl['attrspec'].append(attr) elif attr == 'automatic' and 'static' not in decl['attrspec']: decl['attrspec'].append(attr) - elif attr == 'public' and 'private' not in decl['attrspec']: - decl['attrspec'].append(attr) - elif attr == 'private' and 'public' not in decl['attrspec']: - decl['attrspec'].append(attr) + elif attr == 'public': + if 'private' not in decl['attrspec']: + decl['attrspec'].append(attr) + elif attr == 'private': + if 'public' not in decl['attrspec']: + decl['attrspec'].append(attr) else: decl['attrspec'].append(attr) return decl diff --git a/numpy/f2py/diagnose.py b/numpy/f2py/diagnose.py index 092368c82..21ee399f0 100644 --- a/numpy/f2py/diagnose.py +++ b/numpy/f2py/diagnose.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os import sys import tempfile diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index a6751154b..71a049e41 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ f2py2e - Fortran to Python C/API generator. 2nd Edition. @@ -169,7 +169,7 @@ Extra options (only effective with -c): Version: %s numpy Version: %s -Requires: Python 2.3 or higher. +Requires: Python 3.5 or higher. License: NumPy license (see LICENSE.txt in the NumPy source code) Copyright 1999 - 2011 Pearu Peterson all rights reserved. http://cens.ioc.ee/projects/f2py2e/""" % (f2py_version, numpy_version) diff --git a/numpy/f2py/f90mod_rules.py b/numpy/f2py/f90mod_rules.py index f4f1bf1a9..122fa8939 100644 --- a/numpy/f2py/f90mod_rules.py +++ b/numpy/f2py/f90mod_rules.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Build F90 module support for f2py2e. @@ -23,7 +23,7 @@ from . import capi_maps from . import func2subr from .crackfortran import undo_rmbadname, undo_rmbadname1 -# The eviroment provided by auxfuncs.py is needed for some calls to eval. +# The environment provided by auxfuncs.py is needed for some calls to eval. # As the needed functions cannot be determined by static inspection of the # code, it is safest to use import * pending a major refactoring of f2py. from .auxfuncs import * @@ -85,7 +85,6 @@ fgetdims2_sa = """\ def buildhooks(pymod): - global fgetdims1, fgetdims2 from . import rules ret = {'f90modhooks': [], 'initf90modhooks': [], 'body': [], 'need': ['F_FUNC', 'arrayobject.h'], diff --git a/numpy/f2py/func2subr.py b/numpy/f2py/func2subr.py index 8e18a3236..e9976f43c 100644 --- a/numpy/f2py/func2subr.py +++ b/numpy/f2py/func2subr.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Rules for building C/API module with f2py2e. diff --git a/numpy/f2py/rules.py b/numpy/f2py/rules.py index 459c87aa3..6750bf705 100755 --- a/numpy/f2py/rules.py +++ b/numpy/f2py/rules.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Rules for building C/API module with f2py2e. @@ -276,7 +276,7 @@ static PyObject *#apiname#(const PyObject *capi_self, f2py_start_clock(); #endif \tif (!PyArg_ParseTupleAndKeywords(capi_args,capi_keywds,\\ -\t\t\"#argformat##keyformat##xaformat#:#pyname#\",\\ +\t\t\"#argformat#|#keyformat##xaformat#:#pyname#\",\\ \t\tcapi_kwlist#args_capi##keys_capi##keys_xa#))\n\t\treturn NULL; #frompyobj# /*end of frompyobj*/ @@ -1157,7 +1157,6 @@ def buildmodule(m, um): """ Return """ - global f2py_version, options outmess('\tBuilding module "%s"...\n' % (m['name'])) ret = {} mod_rules = defmod_rules[:] @@ -1445,16 +1444,6 @@ def buildapi(rout): ['\\begin{description}'] + rd[k][1:] +\ ['\\end{description}'] - # Workaround for Python 2.6, 2.6.1 bug: https://bugs.python.org/issue4720 - if rd['keyformat'] or rd['xaformat']: - argformat = rd['argformat'] - if isinstance(argformat, list): - argformat.append('|') - else: - assert isinstance(argformat, str), repr( - (argformat, type(argformat))) - rd['argformat'] += '|' - ar = applyrules(routine_rules, rd) if ismoduleroutine(rout): outmess('\t\t\t %s\n' % (ar['docshort'])) diff --git a/numpy/f2py/setup.py b/numpy/f2py/setup.py index 5e4d7cd56..6314c5af3 100644 --- a/numpy/f2py/setup.py +++ b/numpy/f2py/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ setup.py for installing F2PY diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index 644339218..b3a04bcf0 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -426,21 +426,13 @@ fortran_repr(PyFortranObject *fp) PyTypeObject PyFortran_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "fortran", /*tp_name*/ - sizeof(PyFortranObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)fortran_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - (getattrfunc)fortran_getattr, /*tp_getattr*/ - (setattrfunc)fortran_setattr, /*tp_setattr*/ - 0, /*tp_compare/tp_reserved*/ - (reprfunc)fortran_repr, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - (ternaryfunc)fortran_call, /*tp_call*/ + .tp_name ="fortran", + .tp_basicsize = sizeof(PyFortranObject), + .tp_dealloc = (destructor)fortran_dealloc, + .tp_getattr = (getattrfunc)fortran_getattr, + .tp_setattr = (setattrfunc)fortran_setattr, + .tp_repr = (reprfunc)fortran_repr, + .tp_call = (ternaryfunc)fortran_call, }; /************************* f2py_report_atexit *******************************/ diff --git a/numpy/f2py/tests/test_block_docstring.py b/numpy/f2py/tests/test_block_docstring.py index 03660f021..e431f5ba6 100644 --- a/numpy/f2py/tests/test_block_docstring.py +++ b/numpy/f2py/tests/test_block_docstring.py @@ -16,7 +16,8 @@ class TestBlockDocString(util.F2PyTest): @pytest.mark.skipif(sys.platform=='win32', reason='Fails with MinGW64 Gfortran (Issue #9673)') - @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") + @pytest.mark.xfail(IS_PYPY, + reason="PyPy cannot modify tp_doc after PyType_Ready") def test_block_docstring(self): expected = "'i'-array(2,3)\n" assert_equal(self.module.block.__doc__, expected) diff --git a/numpy/f2py/tests/test_callback.py b/numpy/f2py/tests/test_callback.py index 7629df605..4e29ab9fc 100644 --- a/numpy/f2py/tests/test_callback.py +++ b/numpy/f2py/tests/test_callback.py @@ -4,7 +4,7 @@ import sys import pytest import numpy as np -from numpy.testing import assert_, assert_equal +from numpy.testing import assert_, assert_equal, IS_PYPY from . import util @@ -59,12 +59,12 @@ cf2py intent(out) a end """ - @pytest.mark.slow @pytest.mark.parametrize('name', 't,t2'.split(',')) def test_all(self, name): self.check_function(name) - @pytest.mark.slow + @pytest.mark.xfail(IS_PYPY, + reason="PyPy cannot modify tp_doc after PyType_Ready") def test_docstring(self): expected = textwrap.dedent("""\ a = t(fun,[fun_extra_args]) diff --git a/numpy/f2py/tests/test_crackfortran.py b/numpy/f2py/tests/test_crackfortran.py index 796965e6f..735804024 100644 --- a/numpy/f2py/tests/test_crackfortran.py +++ b/numpy/f2py/tests/test_crackfortran.py @@ -1,8 +1,9 @@ -import pytest - import numpy as np from numpy.testing import assert_array_equal from . import util +from numpy.f2py import crackfortran +import tempfile +import textwrap class TestNoSpace(util.F2PyTest): @@ -35,3 +36,53 @@ class TestNoSpace(util.F2PyTest): self.module.subc([w, k]) assert_array_equal(k, w + 1) assert self.module.t0(23) == b'2' + +class TestPublicPrivate(): + def test_defaultPrivate(self, tmp_path): + f_path = tmp_path / "mod.f90" + with f_path.open('w') as ff: + ff.write(textwrap.dedent("""\ + module foo + private + integer :: a + public :: setA + integer :: b + contains + subroutine setA(v) + integer, intent(in) :: v + a = v + end subroutine setA + end module foo + """)) + mod = crackfortran.crackfortran([str(f_path)]) + assert len(mod) == 1 + mod = mod[0] + assert 'private' in mod['vars']['a']['attrspec'] + assert 'public' not in mod['vars']['a']['attrspec'] + assert 'private' in mod['vars']['b']['attrspec'] + assert 'public' not in mod['vars']['b']['attrspec'] + assert 'private' not in mod['vars']['seta']['attrspec'] + assert 'public' in mod['vars']['seta']['attrspec'] + + def test_defaultPublic(self, tmp_path): + f_path = tmp_path / "mod.f90" + with f_path.open('w') as ff: + ff.write(textwrap.dedent("""\ + module foo + public + integer, private :: a + public :: setA + contains + subroutine setA(v) + integer, intent(in) :: v + a = v + end subroutine setA + end module foo + """)) + mod = crackfortran.crackfortran([str(f_path)]) + assert len(mod) == 1 + mod = mod[0] + assert 'private' in mod['vars']['a']['attrspec'] + assert 'public' not in mod['vars']['a']['attrspec'] + assert 'private' not in mod['vars']['seta']['attrspec'] + assert 'public' in mod['vars']['seta']['attrspec'] diff --git a/numpy/f2py/tests/test_mixed.py b/numpy/f2py/tests/test_mixed.py index fc00ccc43..04266ca5b 100644 --- a/numpy/f2py/tests/test_mixed.py +++ b/numpy/f2py/tests/test_mixed.py @@ -2,7 +2,7 @@ import os import textwrap import pytest -from numpy.testing import assert_, assert_equal +from numpy.testing import assert_, assert_equal, IS_PYPY from . import util @@ -15,13 +15,13 @@ class TestMixed(util.F2PyTest): _path('src', 'mixed', 'foo_fixed.f90'), _path('src', 'mixed', 'foo_free.f90')] - @pytest.mark.slow def test_all(self): assert_(self.module.bar11() == 11) assert_(self.module.foo_fixed.bar12() == 12) assert_(self.module.foo_free.bar13() == 13) - @pytest.mark.slow + @pytest.mark.xfail(IS_PYPY, + reason="PyPy cannot modify tp_doc after PyType_Ready") def test_docstring(self): expected = textwrap.dedent("""\ a = bar11() diff --git a/numpy/f2py/tests/test_quoted_character.py b/numpy/f2py/tests/test_quoted_character.py index d89ef1385..20c77666c 100644 --- a/numpy/f2py/tests/test_quoted_character.py +++ b/numpy/f2py/tests/test_quoted_character.py @@ -2,7 +2,6 @@ """ import sys -from importlib import import_module import pytest from numpy.testing import assert_equal diff --git a/numpy/f2py/tests/test_return_character.py b/numpy/f2py/tests/test_return_character.py index 6cb95a8b6..429e69bb4 100644 --- a/numpy/f2py/tests/test_return_character.py +++ b/numpy/f2py/tests/test_return_character.py @@ -3,12 +3,13 @@ import pytest from numpy import array from numpy.testing import assert_ from . import util +import platform +IS_S390X = platform.machine() == 's390x' class TestReturnCharacter(util.F2PyTest): - def check_function(self, t): - tname = t.__doc__.split()[0] + def check_function(self, t, tname): if tname in ['t0', 't1', 's0', 's1']: assert_(t(23) == b'2') r = t('ab') @@ -79,10 +80,10 @@ cf2py intent(out) ts end """ - @pytest.mark.slow + @pytest.mark.xfail(IS_S390X, reason="calback returns ' '") @pytest.mark.parametrize('name', 't0,t1,t5,s0,s1,s5,ss'.split(',')) def test_all(self, name): - self.check_function(getattr(self.module, name)) + self.check_function(getattr(self.module, name), name) class TestF90ReturnCharacter(TestReturnCharacter): @@ -138,7 +139,7 @@ module f90_return_char end module f90_return_char """ - @pytest.mark.slow + @pytest.mark.xfail(IS_S390X, reason="calback returns ' '") @pytest.mark.parametrize('name', 't0,t1,t5,ts,s0,s1,s5,ss'.split(',')) def test_all(self, name): - self.check_function(getattr(self.module.f90_return_char, name)) + self.check_function(getattr(self.module.f90_return_char, name), name) diff --git a/numpy/f2py/tests/test_return_complex.py b/numpy/f2py/tests/test_return_complex.py index 9063695bc..3d2e2b94f 100644 --- a/numpy/f2py/tests/test_return_complex.py +++ b/numpy/f2py/tests/test_return_complex.py @@ -1,22 +1,20 @@ import pytest from numpy import array -from numpy.compat import long from numpy.testing import assert_, assert_raises from . import util class TestReturnComplex(util.F2PyTest): - def check_function(self, t): - tname = t.__doc__.split()[0] + def check_function(self, t, tname): if tname in ['t0', 't8', 's0', 's8']: err = 1e-5 else: err = 0.0 assert_(abs(t(234j) - 234.0j) <= err) assert_(abs(t(234.6) - 234.6) <= err) - assert_(abs(t(long(234)) - 234.0) <= err) + assert_(abs(t(234) - 234.0) <= err) assert_(abs(t(234.6 + 3j) - (234.6 + 3j)) <= err) #assert_( abs(t('234')-234.)<=err) #assert_( abs(t('234.6')-234.6)<=err) @@ -102,10 +100,9 @@ cf2py intent(out) td end """ - @pytest.mark.slow @pytest.mark.parametrize('name', 't0,t8,t16,td,s0,s8,s16,sd'.split(',')) def test_all(self, name): - self.check_function(getattr(self.module, name)) + self.check_function(getattr(self.module, name), name) class TestF90ReturnComplex(TestReturnComplex): @@ -161,7 +158,6 @@ module f90_return_complex end module f90_return_complex """ - @pytest.mark.slow @pytest.mark.parametrize('name', 't0,t8,t16,td,s0,s8,s16,sd'.split(',')) def test_all(self, name): - self.check_function(getattr(self.module.f90_return_complex, name)) + self.check_function(getattr(self.module.f90_return_complex, name), name) diff --git a/numpy/f2py/tests/test_return_integer.py b/numpy/f2py/tests/test_return_integer.py index 35f32e37d..0a8121dc1 100644 --- a/numpy/f2py/tests/test_return_integer.py +++ b/numpy/f2py/tests/test_return_integer.py @@ -1,17 +1,15 @@ import pytest from numpy import array -from numpy.compat import long from numpy.testing import assert_, assert_raises from . import util class TestReturnInteger(util.F2PyTest): - def check_function(self, t): + def check_function(self, t, tname): assert_(t(123) == 123, repr(t(123))) assert_(t(123.6) == 123) - assert_(t(long(123)) == 123) assert_(t('123') == 123) assert_(t(-123) == -123) assert_(t([123]) == 123) @@ -36,7 +34,7 @@ class TestReturnInteger(util.F2PyTest): assert_raises(Exception, t, t) assert_raises(Exception, t, {}) - if t.__doc__.split()[0] in ['t8', 's8']: + if tname in ['t8', 's8']: assert_raises(OverflowError, t, 100000000000000000000000) assert_raises(OverflowError, t, 10000000011111111111111.23) @@ -101,11 +99,10 @@ cf2py intent(out) t8 end """ - @pytest.mark.slow @pytest.mark.parametrize('name', 't0,t1,t2,t4,t8,s0,s1,s2,s4,s8'.split(',')) def test_all(self, name): - self.check_function(getattr(self.module, name)) + self.check_function(getattr(self.module, name), name) class TestF90ReturnInteger(TestReturnInteger): @@ -172,8 +169,7 @@ module f90_return_integer end module f90_return_integer """ - @pytest.mark.slow @pytest.mark.parametrize('name', 't0,t1,t2,t4,t8,s0,s1,s2,s4,s8'.split(',')) def test_all(self, name): - self.check_function(getattr(self.module.f90_return_integer, name)) + self.check_function(getattr(self.module.f90_return_integer, name), name) diff --git a/numpy/f2py/tests/test_return_logical.py b/numpy/f2py/tests/test_return_logical.py index 3139e0df7..9db939c7e 100644 --- a/numpy/f2py/tests/test_return_logical.py +++ b/numpy/f2py/tests/test_return_logical.py @@ -1,7 +1,6 @@ import pytest from numpy import array -from numpy.compat import long from numpy.testing import assert_, assert_raises from . import util @@ -18,7 +17,6 @@ class TestReturnLogical(util.F2PyTest): assert_(t(1j) == 1) assert_(t(234) == 1) assert_(t(234.6) == 1) - assert_(t(long(234)) == 1) assert_(t(234.6 + 3j) == 1) assert_(t('234') == 1) assert_(t('aaa') == 1) diff --git a/numpy/f2py/tests/test_return_real.py b/numpy/f2py/tests/test_return_real.py index 1707aab45..8e5022a8e 100644 --- a/numpy/f2py/tests/test_return_real.py +++ b/numpy/f2py/tests/test_return_real.py @@ -2,21 +2,19 @@ import platform import pytest from numpy import array -from numpy.compat import long from numpy.testing import assert_, assert_raises from . import util class TestReturnReal(util.F2PyTest): - def check_function(self, t): - if t.__doc__.split()[0] in ['t0', 't4', 's0', 's4']: + def check_function(self, t, tname): + if tname in ['t0', 't4', 's0', 's4']: err = 1e-5 else: err = 0.0 assert_(abs(t(234) - 234.0) <= err) assert_(abs(t(234.6) - 234.6) <= err) - assert_(abs(t(long(234)) - 234.0) <= err) assert_(abs(t('234') - 234) <= err) assert_(abs(t('234.6') - 234.6) <= err) assert_(abs(t(-234) + 234) <= err) @@ -32,7 +30,7 @@ class TestReturnReal(util.F2PyTest): assert_(abs(t(array([234], 'B')) - 234.) <= err) assert_(abs(t(array([234], 'f')) - 234.) <= err) assert_(abs(t(array([234], 'd')) - 234.) <= err) - if t.__doc__.split()[0] in ['t0', 't4', 's0', 's4']: + if tname in ['t0', 't4', 's0', 's4']: assert_(t(1e200) == t(1e300)) # inf #assert_raises(ValueError, t, array([234], 'S1')) @@ -88,10 +86,9 @@ end interface end python module c_ext_return_real """ - @pytest.mark.slow @pytest.mark.parametrize('name', 't4,t8,s4,s8'.split(',')) def test_all(self, name): - self.check_function(getattr(self.module, name)) + self.check_function(getattr(self.module, name), name) class TestF77ReturnReal(TestReturnReal): @@ -143,10 +140,9 @@ cf2py intent(out) td end """ - @pytest.mark.slow @pytest.mark.parametrize('name', 't0,t4,t8,td,s0,s4,s8,sd'.split(',')) def test_all(self, name): - self.check_function(getattr(self.module, name)) + self.check_function(getattr(self.module, name), name) class TestF90ReturnReal(TestReturnReal): @@ -202,7 +198,6 @@ module f90_return_real end module f90_return_real """ - @pytest.mark.slow @pytest.mark.parametrize('name', 't0,t4,t8,td,s0,s4,s8,sd'.split(',')) def test_all(self, name): - self.check_function(getattr(self.module.f90_return_real, name)) + self.check_function(getattr(self.module.f90_return_real, name), name) diff --git a/numpy/f2py/tests/util.py b/numpy/f2py/tests/util.py index 6dcc2ed12..c5b06697d 100644 --- a/numpy/f2py/tests/util.py +++ b/numpy/f2py/tests/util.py @@ -19,10 +19,7 @@ from numpy.compat import asbytes, asstr from numpy.testing import temppath from importlib import import_module -try: - from hashlib import md5 -except ImportError: - from md5 import new as md5 # noqa: F401 +from hashlib import md5 # # Maintaining a temporary module directory diff --git a/numpy/f2py/use_rules.py b/numpy/f2py/use_rules.py index 268c7e81b..f1b71e83c 100644 --- a/numpy/f2py/use_rules.py +++ b/numpy/f2py/use_rules.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Build 'use others module data' mechanism for f2py2e. diff --git a/numpy/fft/_pocketfft.py b/numpy/fft/_pocketfft.py index f2510a6c2..e9f554fe7 100644 --- a/numpy/fft/_pocketfft.py +++ b/numpy/fft/_pocketfft.py @@ -59,12 +59,11 @@ def _raw_fft(a, n, axis, is_real, is_forward, inv_norm): if a.shape[axis] != n: s = list(a.shape) + index = [slice(None)]*len(s) if s[axis] > n: - index = [slice(None)]*len(s) index[axis] = slice(0, n) a = a[tuple(index)] else: - index = [slice(None)]*len(s) index[axis] = slice(0, s[axis]) s[axis] = n z = zeros(s, a.dtype.char) @@ -525,8 +524,8 @@ def hfft(a, n=None, axis=-1, norm=None): domain and is real in the frequency domain. So here it's `hfft` for which you must supply the length of the result if it is to be odd. - * even: ``ihfft(hfft(a, 2*len(a) - 2) == a``, within roundoff error, - * odd: ``ihfft(hfft(a, 2*len(a) - 1) == a``, within roundoff error. + * even: ``ihfft(hfft(a, 2*len(a) - 2)) == a``, within roundoff error, + * odd: ``ihfft(hfft(a, 2*len(a) - 1)) == a``, within roundoff error. The correct interpretation of the hermitian input depends on the length of the original data, as given by `n`. This is because each input shape could @@ -605,8 +604,8 @@ def ihfft(a, n=None, axis=-1, norm=None): domain and is real in the frequency domain. So here it's `hfft` for which you must supply the length of the result if it is to be odd: - * even: ``ihfft(hfft(a, 2*len(a) - 2) == a``, within roundoff error, - * odd: ``ihfft(hfft(a, 2*len(a) - 1) == a``, within roundoff error. + * even: ``ihfft(hfft(a, 2*len(a) - 2)) == a``, within roundoff error, + * odd: ``ihfft(hfft(a, 2*len(a) - 1)) == a``, within roundoff error. Examples -------- diff --git a/numpy/fft/setup.py b/numpy/fft/setup.py index 40d632ec5..e8204fcd3 100644 --- a/numpy/fft/setup.py +++ b/numpy/fft/setup.py @@ -1,3 +1,4 @@ +import sys def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration @@ -5,9 +6,12 @@ def configuration(parent_package='',top_path=None): config.add_data_dir('tests') + # AIX needs to be told to use large file support - at all times + defs = [('_LARGE_FILES', None)] if sys.platform[:3] == "aix" else [] # Configure pocketfft_internal config.add_extension('_pocketfft_internal', - sources=['_pocketfft.c'] + sources=['_pocketfft.c'], + define_macros=defs, ) return config diff --git a/numpy/fft/tests/test_helper.py b/numpy/fft/tests/test_helper.py index 2b457271b..68f5990af 100644 --- a/numpy/fft/tests/test_helper.py +++ b/numpy/fft/tests/test_helper.py @@ -4,7 +4,7 @@ Copied from fftpack.helper by Pearu Peterson, October 2005 """ import numpy as np -from numpy.testing import assert_array_almost_equal, assert_equal +from numpy.testing import assert_array_almost_equal from numpy import fft, pi diff --git a/numpy/fft/tests/test_pocketfft.py b/numpy/fft/tests/test_pocketfft.py index 14f92c081..7c3db0485 100644 --- a/numpy/fft/tests/test_pocketfft.py +++ b/numpy/fft/tests/test_pocketfft.py @@ -5,7 +5,6 @@ from numpy.testing import ( assert_array_equal, assert_raises, assert_allclose ) import threading -import sys import queue diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index 139b8c0ca..f5d0cc217 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -35,8 +35,6 @@ Example:: """ import os -import sys -import warnings import shutil import io from contextlib import closing @@ -305,10 +303,7 @@ class DataSource: """Test if path is a net location. Tests the scheme and netloc.""" # We do this here to reduce the 'import numpy' initial import time. - if sys.version_info[0] >= 3: - from urllib.parse import urlparse - else: - from urlparse import urlparse + from urllib.parse import urlparse # BUG : URLs require a scheme string ('http://') to be used. # www.google.com will fail. @@ -325,14 +320,10 @@ class DataSource: Creates a copy of the file in the datasource cache. """ - # We import these here because importing urllib2 is slow and + # We import these here because importing urllib is slow and # a significant fraction of numpy's total import time. - if sys.version_info[0] >= 3: - from urllib.request import urlopen - from urllib.error import URLError - else: - from urllib2 import urlopen - from urllib2 import URLError + from urllib.request import urlopen + from urllib.error import URLError upath = self.abspath(path) @@ -407,10 +398,7 @@ class DataSource: """ # We do this here to reduce the 'import numpy' initial import time. - if sys.version_info[0] >= 3: - from urllib.parse import urlparse - else: - from urlparse import urlparse + from urllib.parse import urlparse # TODO: This should be more robust. Handles case where path includes # the destpath, but not other sub-paths. Failing case: diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index dd6e9ec66..ff5b94342 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -5,7 +5,7 @@ __docformat__ = "restructuredtext en" import numpy as np import numpy.core.numeric as nx -from numpy.compat import asbytes, asunicode, bytes, basestring +from numpy.compat import asbytes, asunicode, bytes def _decode_line(line, encoding=None): @@ -175,7 +175,7 @@ class LineSplitter: self.comments = comments # Delimiter is a character - if (delimiter is None) or isinstance(delimiter, basestring): + if (delimiter is None) or isinstance(delimiter, str): delimiter = delimiter or None _handyman = self._delimited_splitter # Delimiter is a list of field widths @@ -345,7 +345,7 @@ class NameValidator: if (nbfields is None): return None names = [] - if isinstance(names, basestring): + if isinstance(names, str): names = [names, ] if nbfields is not None: nbnames = len(names) @@ -504,18 +504,23 @@ class StringConverter: """ # _mapper = [(nx.bool_, str2bool, False), - (nx.integer, int, -1)] + (nx.int_, int, -1),] # On 32-bit systems, we need to make sure that we explicitly include - # nx.int64 since ns.integer is nx.int32. - if nx.dtype(nx.integer).itemsize < nx.dtype(nx.int64).itemsize: + # nx.int64 since ns.int_ is nx.int32. + if nx.dtype(nx.int_).itemsize < nx.dtype(nx.int64).itemsize: _mapper.append((nx.int64, int, -1)) - _mapper.extend([(nx.floating, float, nx.nan), - (nx.complexfloating, complex, nx.nan + 0j), + _mapper.extend([(nx.float64, float, nx.nan), + (nx.complex128, complex, nx.nan + 0j), (nx.longdouble, nx.longdouble, nx.nan), (nx.unicode_, asunicode, '???'), - (nx.string_, asbytes, '???')]) + (nx.string_, asbytes, '???'), + # If a non-default dtype is passed, fall back to generic + # ones (should only be used for the converter) + (nx.integer, int, -1), + (nx.floating, float, nx.nan), + (nx.complexfloating, complex, nx.nan + 0j),]) (_defaulttype, _defaultfunc, _defaultfill) = zip(*_mapper) @@ -659,7 +664,7 @@ class StringConverter: if missing_values is None: self.missing_values = {''} else: - if isinstance(missing_values, basestring): + if isinstance(missing_values, str): missing_values = missing_values.split(",") self.missing_values = set(list(missing_values) + ['']) # @@ -706,6 +711,26 @@ class StringConverter: return self._callingfunction(value) # + def _do_upgrade(self): + # Raise an exception if we locked the converter... + if self._locked: + errmsg = "Converter is locked and cannot be upgraded" + raise ConverterLockError(errmsg) + _statusmax = len(self._mapper) + # Complains if we try to upgrade by the maximum + _status = self._status + if _status == _statusmax: + errmsg = "Could not find a valid conversion function" + raise ConverterError(errmsg) + elif _status < _statusmax - 1: + _status += 1 + self.type, self.func, default = self._mapper[_status] + self._status = _status + if self._initial_default is not None: + self.default = self._initial_default + else: + self.default = default + def upgrade(self, value): """ Find the best converter for a given string, and return the result. @@ -731,24 +756,7 @@ class StringConverter: try: return self._strict_call(value) except ValueError: - # Raise an exception if we locked the converter... - if self._locked: - errmsg = "Converter is locked and cannot be upgraded" - raise ConverterLockError(errmsg) - _statusmax = len(self._mapper) - # Complains if we try to upgrade by the maximum - _status = self._status - if _status == _statusmax: - errmsg = "Could not find a valid conversion function" - raise ConverterError(errmsg) - elif _status < _statusmax - 1: - _status += 1 - (self.type, self.func, default) = self._mapper[_status] - self._status = _status - if self._initial_default is not None: - self.default = self._initial_default - else: - self.default = default + self._do_upgrade() return self.upgrade(value) def iterupgrade(self, value): @@ -760,25 +768,7 @@ class StringConverter: for _m in value: _strict_call(_m) except ValueError: - # Raise an exception if we locked the converter... - if self._locked: - errmsg = "Converter is locked and cannot be upgraded" - raise ConverterLockError(errmsg) - _statusmax = len(self._mapper) - # Complains if we try to upgrade by the maximum - _status = self._status - if _status == _statusmax: - raise ConverterError( - "Could not find a valid conversion function" - ) - elif _status < _statusmax - 1: - _status += 1 - (self.type, self.func, default) = self._mapper[_status] - if self._initial_default is not None: - self.default = self._initial_default - else: - self.default = default - self._status = _status + self._do_upgrade() self.iterupgrade(value) def update(self, func, default=None, testing_value=None, @@ -834,7 +824,7 @@ class StringConverter: else: if not np.iterable(missing_values): missing_values = [missing_values] - if not all(isinstance(v, basestring) for v in missing_values): + if not all(isinstance(v, str) for v in missing_values): raise TypeError("missing_values must be strings or unicode") self.missing_values.update(missing_values) @@ -884,7 +874,7 @@ def easy_dtype(ndtype, names=None, defaultfmt="f%i", **validationargs): nbfields = len(ndtype) if names is None: names = [''] * len(ndtype) - elif isinstance(names, basestring): + elif isinstance(names, str): names = names.split(",") names = validate(names, nbfields=nbfields, defaultfmt=defaultfmt) ndtype = np.dtype(dict(formats=ndtype, names=names)) @@ -892,7 +882,7 @@ def easy_dtype(ndtype, names=None, defaultfmt="f%i", **validationargs): # Explicit names if names is not None: validate = NameValidator(**validationargs) - if isinstance(names, basestring): + if isinstance(names, str): names = names.split(",") # Simple dtype: repeat to match the nb of names if ndtype.names is None: diff --git a/numpy/lib/_version.py b/numpy/lib/_version.py index 6a7c5cba1..d4098acb5 100644 --- a/numpy/lib/_version.py +++ b/numpy/lib/_version.py @@ -7,8 +7,6 @@ work; they don't recognize anything like alpha/beta/rc/dev versions. """ import re -from numpy.compat import basestring - __all__ = ['NumpyVersion'] @@ -114,10 +112,10 @@ class NumpyVersion(): return vercmp def _compare(self, other): - if not isinstance(other, (basestring, NumpyVersion)): + if not isinstance(other, (str, NumpyVersion)): raise ValueError("Invalid object to compare with NumpyVersion.") - if isinstance(other, basestring): + if isinstance(other, str): other = NumpyVersion(other) vercmp = self._compare_version(other) diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index ad508e85d..22687b941 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -251,9 +251,9 @@ def unique(ar, return_index=False, return_inverse=False, >>> u array([1, 2, 3, 4, 6]) >>> indices - array([0, 1, 4, ..., 1, 2, 1]) + array([0, 1, 4, 3, 1, 2, 1]) >>> u[indices] - array([1, 2, 6, ..., 2, 3, 2]) + array([1, 2, 6, 4, 2, 3, 2]) """ ar = np.asanyarray(ar) @@ -270,20 +270,33 @@ def unique(ar, return_index=False, return_inverse=False, # Must reshape to a contiguous 2D array for this to work... orig_shape, orig_dtype = ar.shape, ar.dtype - ar = ar.reshape(orig_shape[0], -1) + ar = ar.reshape(orig_shape[0], np.prod(orig_shape[1:], dtype=np.intp)) ar = np.ascontiguousarray(ar) dtype = [('f{i}'.format(i=i), ar.dtype) for i in range(ar.shape[1])] + # At this point, `ar` has shape `(n, m)`, and `dtype` is a structured + # data type with `m` fields where each field has the data type of `ar`. + # In the following, we create the array `consolidated`, which has + # shape `(n,)` with data type `dtype`. try: - consolidated = ar.view(dtype) + if ar.shape[1] > 0: + consolidated = ar.view(dtype) + else: + # If ar.shape[1] == 0, then dtype will be `np.dtype([])`, which is + # a data type with itemsize 0, and the call `ar.view(dtype)` will + # fail. Instead, we'll use `np.empty` to explicitly create the + # array with shape `(len(ar),)`. Because `dtype` in this case has + # itemsize 0, the total size of the result is still 0 bytes. + consolidated = np.empty(len(ar), dtype=dtype) except TypeError: # There's no good way to do this for object arrays, etc... msg = 'The axis argument to unique is not supported for dtype {dt}' raise TypeError(msg.format(dt=ar.dtype)) def reshape_uniq(uniq): + n = len(uniq) uniq = uniq.view(orig_dtype) - uniq = uniq.reshape(-1, *orig_shape[1:]) + uniq = uniq.reshape(n, *orig_shape[1:]) uniq = np.moveaxis(uniq, 0, axis) return uniq @@ -783,4 +796,3 @@ def setdiff1d(ar1, ar2, assume_unique=False): ar1 = unique(ar1) ar2 = unique(ar2) return ar1[in1d(ar1, ar2, assume_unique=True, invert=True)] - diff --git a/numpy/lib/arrayterator.py b/numpy/lib/arrayterator.py index 15b0f8aa9..b9ea21f8e 100644 --- a/numpy/lib/arrayterator.py +++ b/numpy/lib/arrayterator.py @@ -10,8 +10,6 @@ a user-specified number of elements. from operator import mul from functools import reduce -from numpy.compat import long - __all__ = ['Arrayterator'] @@ -108,7 +106,7 @@ class Arrayterator: if slice_ is Ellipsis: fixed.extend([slice(None)] * (dims-length+1)) length = len(fixed) - elif isinstance(slice_, (int, long)): + elif isinstance(slice_, int): fixed.append(slice(slice_, slice_+1, 1)) else: fixed.append(slice_) @@ -161,8 +159,7 @@ class Arrayterator: """ for block in self: - for value in block.flat: - yield value + yield from block.flat @property def shape(self): diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 114bae287..2afa4ac10 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -166,7 +166,7 @@ import io import warnings from numpy.lib.utils import safe_eval from numpy.compat import ( - isfileobj, long, os_fspath, pickle + isfileobj, os_fspath, pickle ) @@ -295,7 +295,11 @@ def descr_to_dtype(descr): # subtype, will always have a shape descr[1] dt = descr_to_dtype(descr[0]) return numpy.dtype((dt, descr[1])) - fields = [] + + titles = [] + names = [] + formats = [] + offsets = [] offset = 0 for field in descr: if len(field) == 2: @@ -309,14 +313,13 @@ def descr_to_dtype(descr): # Once support for blank names is removed, only "if name == ''" needed) is_pad = (name == '' and dt.type is numpy.void and dt.names is None) if not is_pad: - fields.append((name, dt, offset)) - + title, name = name if isinstance(name, tuple) else (None, name) + titles.append(title) + names.append(name) + formats.append(dt) + offsets.append(offset) offset += dt.itemsize - names, formats, offsets = zip(*fields) - # names may be (title, names) tuples - nametups = (n if isinstance(n, tuple) else (None, n) for n in names) - titles, names = zip(*nametups) return numpy.dtype({'names': names, 'formats': formats, 'titles': titles, 'offsets': offsets, 'itemsize': offset}) @@ -591,7 +594,7 @@ def _read_array_header(fp, version): # Sanity-check the values. if (not isinstance(d['shape'], tuple) or - not numpy.all([isinstance(x, (int, long)) for x in d['shape']])): + not numpy.all([isinstance(x, int) for x in d['shape']])): msg = "shape is not valid: {!r}" raise ValueError(msg.format(d['shape'])) if not isinstance(d['fortran_order'], bool): @@ -599,7 +602,7 @@ def _read_array_header(fp, version): raise ValueError(msg.format(d['fortran_order'])) try: dtype = descr_to_dtype(d['descr']) - except TypeError as e: + except TypeError: msg = "descr is not a valid dtype descriptor: {!r}" raise ValueError(msg.format(d['descr'])) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index ef8a26fe3..7eeed7825 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1,9 +1,4 @@ -try: - # Accessing collections abstract classes from collections - # has been deprecated since Python 3.3 - import collections.abc as collections_abc -except ImportError: - import collections as collections_abc +import collections.abc import functools import re import sys @@ -34,7 +29,6 @@ from numpy.core.multiarray import ( interp as compiled_interp, interp_complex as compiled_interp_complex ) from numpy.core.umath import _add_newdoc_ufunc as add_newdoc_ufunc -from numpy.compat import long import builtins @@ -497,8 +491,7 @@ def _piecewise_dispatcher(x, condlist, funclist, *args, **kw): yield x # support the undocumented behavior of allowing scalars if np.iterable(condlist): - for c in condlist: - yield c + yield from condlist @array_function_dispatch(_piecewise_dispatcher) @@ -613,7 +606,7 @@ def piecewise(x, condlist, funclist, *args, **kw): y = zeros(x.shape, x.dtype) for k in range(n): item = funclist[k] - if not isinstance(item, collections_abc.Callable): + if not isinstance(item, collections.abc.Callable): y[condlist[k]] = item else: vals = x[condlist[k]] @@ -624,10 +617,8 @@ def piecewise(x, condlist, funclist, *args, **kw): def _select_dispatcher(condlist, choicelist, default=None): - for c in condlist: - yield c - for c in choicelist: - yield c + yield from condlist + yield from choicelist @array_function_dispatch(_select_dispatcher) @@ -716,12 +707,12 @@ def select(condlist, choicelist, default=0): return result -def _copy_dispatcher(a, order=None): +def _copy_dispatcher(a, order=None, subok=None): return (a,) @array_function_dispatch(_copy_dispatcher) -def copy(a, order='K'): +def copy(a, order='K', subok=False): """ Return an array copy of the given object. @@ -736,12 +727,21 @@ def copy(a, order='K'): as possible. (Note that this function and :meth:`ndarray.copy` are very similar, but have different default values for their order= arguments.) + subok : bool, optional + If True, then sub-classes will be passed-through, otherwise the + returned array will be forced to be a base-class array (defaults to False). + + .. versionadded:: 1.19.0 Returns ------- arr : ndarray Array interpretation of `a`. + See Also + -------- + ndarray.copy : Preferred method for creating an array copy + Notes ----- This is equivalent to: @@ -765,15 +765,14 @@ def copy(a, order='K'): False """ - return array(a, order=order, copy=True) + return array(a, order=order, subok=subok, copy=True) # Basic operations def _gradient_dispatcher(f, *varargs, axis=None, edge_order=None): yield f - for v in varargs: - yield v + yield from varargs @array_function_dispatch(_gradient_dispatcher) @@ -1430,6 +1429,11 @@ def angle(z, deg=False): arctan2 absolute + Notes + ----- + Although the angle of the complex number 0 is undefined, ``numpy.angle(0)`` + returns the value 0. + Examples -------- >>> np.angle([1.0, 1.0j, 1+1j]) # in radians @@ -3879,7 +3883,7 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False, "'midpoint', or 'nearest'") n = np.array(False, dtype=bool) # check for nan's flag - if indices.dtype == intp: # take the points along axis + if np.issubdtype(indices.dtype, np.integer): # take the points along axis # Check if the array contains any nan's if np.issubdtype(a.dtype, np.inexact): indices = concatenate((indices, [-1])) @@ -4207,12 +4211,17 @@ def delete(arr, obj, axis=None): Parameters ---------- arr : array_like - Input array. + Input array. obj : slice, int or array of ints - Indicate indices of sub-arrays to remove along the specified axis. + Indicate indices of sub-arrays to remove along the specified axis. + + .. versionchanged:: 1.19.0 + Boolean indices are now treated as a mask of elements to remove, + rather than being cast to the integers 0 and 1. + axis : int, optional - The axis along which to delete the subarray defined by `obj`. - If `axis` is None, `obj` is applied to the flattened array. + The axis along which to delete the subarray defined by `obj`. + If `axis` is None, `obj` is applied to the flattened array. Returns ------- @@ -4270,20 +4279,11 @@ def delete(arr, obj, axis=None): if axis is None: if ndim != 1: arr = arr.ravel() + # needed for np.matrix, which is still not 1d after being ravelled ndim = arr.ndim - axis = -1 - - if ndim == 0: - # 2013-09-24, 1.9 - warnings.warn( - "in the future the special handling of scalars will be removed " - "from delete and raise an error", DeprecationWarning, stacklevel=3) - if wrap: - return wrap(arr) - else: - return arr.copy(order=arrorder) - - axis = normalize_axis_index(axis, ndim) + axis = ndim - 1 + else: + axis = normalize_axis_index(axis, ndim) slobj = [slice(None)]*ndim N = arr.shape[axis] @@ -4339,18 +4339,8 @@ def delete(arr, obj, axis=None): else: return new - _obj = obj - obj = np.asarray(obj) - # After removing the special handling of booleans and out of - # bounds values, the conversion to the array can be removed. - if obj.dtype == bool: - warnings.warn("in the future insert will treat boolean arrays and " - "array-likes as boolean index instead of casting it " - "to integer", FutureWarning, stacklevel=3) - obj = obj.astype(intp) - if isinstance(_obj, (int, long, integer)): + if isinstance(obj, (int, integer)) and not isinstance(obj, bool): # optimization for a single value - obj = obj.item() if (obj < -N or obj >= N): raise IndexError( "index %i is out of bounds for axis %i with " @@ -4366,35 +4356,23 @@ def delete(arr, obj, axis=None): slobj2[axis] = slice(obj+1, None) new[tuple(slobj)] = arr[tuple(slobj2)] else: + _obj = obj + obj = np.asarray(obj) if obj.size == 0 and not isinstance(_obj, np.ndarray): obj = obj.astype(intp) - if not np.can_cast(obj, intp, 'same_kind'): - # obj.size = 1 special case always failed and would just - # give superfluous warnings. - # 2013-09-24, 1.9 - warnings.warn( - "using a non-integer array as obj in delete will result in an " - "error in the future", DeprecationWarning, stacklevel=3) - obj = obj.astype(intp) - keep = ones(N, dtype=bool) - # Test if there are out of bound indices, this is deprecated - inside_bounds = (obj < N) & (obj >= -N) - if not inside_bounds.all(): - # 2013-09-24, 1.9 - warnings.warn( - "in the future out of bounds indices will raise an error " - "instead of being ignored by `numpy.delete`.", - DeprecationWarning, stacklevel=3) - obj = obj[inside_bounds] - positive_indices = obj >= 0 - if not positive_indices.all(): - warnings.warn( - "in the future negative indices will not be ignored by " - "`numpy.delete`.", FutureWarning, stacklevel=3) - obj = obj[positive_indices] + if obj.dtype == bool: + if obj.shape != (N,): + raise ValueError('boolean array argument obj to delete ' + 'must be one dimensional and match the axis ' + 'length of {}'.format(N)) + + # optimization, the other branch is slower + keep = ~obj + else: + keep = ones(N, dtype=bool) + keep[obj,] = False - keep[obj, ] = False slobj[axis] = keep new = arr[tuple(slobj)] @@ -4510,19 +4488,9 @@ def insert(arr, obj, values, axis=None): if axis is None: if ndim != 1: arr = arr.ravel() + # needed for np.matrix, which is still not 1d after being ravelled ndim = arr.ndim axis = ndim - 1 - elif ndim == 0: - # 2013-09-24, 1.9 - warnings.warn( - "in the future the special handling of scalars will be removed " - "from insert and raise an error", DeprecationWarning, stacklevel=3) - arr = arr.copy(order=arrorder) - arr[...] = values - if wrap: - return wrap(arr) - else: - return arr else: axis = normalize_axis_index(axis, ndim) slobj = [slice(None)]*ndim @@ -4531,12 +4499,13 @@ def insert(arr, obj, values, axis=None): if isinstance(obj, slice): # turn it into a range object - indices = arange(*obj.indices(N), **{'dtype': intp}) + indices = arange(*obj.indices(N), dtype=intp) else: # need to copy obj, because indices will be changed in-place indices = np.array(obj) if indices.dtype == bool: # See also delete + # 2012-10-11, NumPy 1.8 warnings.warn( "in the future insert will treat boolean arrays and " "array-likes as a boolean index instead of casting it to " @@ -4586,13 +4555,6 @@ def insert(arr, obj, values, axis=None): # Can safely cast the empty list to intp indices = indices.astype(intp) - if not np.can_cast(indices, intp, 'same_kind'): - # 2013-09-24, 1.9 - warnings.warn( - "using a non-integer array as obj in insert will result in an " - "error in the future", DeprecationWarning, stacklevel=3) - indices = indices.astype(intp) - indices[indices < 0] += N numnew = len(indices) @@ -4663,7 +4625,9 @@ def append(arr, values, axis=None): >>> np.append([[1, 2, 3], [4, 5, 6]], [7, 8, 9], axis=0) Traceback (most recent call last): ... - ValueError: all the input arrays must have same number of dimensions + ValueError: all the input arrays must have same number of dimensions, but + the array at index 0 has 2 dimension(s) and the array at index 1 has 1 + dimension(s) """ arr = asanyarray(arr) diff --git a/numpy/lib/histograms.py b/numpy/lib/histograms.py index 70ecd6eb1..f080cc392 100644 --- a/numpy/lib/histograms.py +++ b/numpy/lib/histograms.py @@ -7,7 +7,6 @@ import operator import warnings import numpy as np -from numpy.compat.py3k import basestring from numpy.core import overrides __all__ = ['histogram', 'histogramdd', 'histogram_bin_edges'] @@ -208,7 +207,7 @@ def _hist_bin_fd(x, range): than the standard deviation, so it is less accurate, especially for long tailed distributions. - If the IQR is 0, this function returns 1 for the number of bins. + If the IQR is 0, this function returns 0 for the bin width. Binwidth is inversely proportional to the cube root of data size (asymptotically optimal). @@ -230,21 +229,21 @@ def _hist_bin_fd(x, range): def _hist_bin_auto(x, range): """ Histogram bin estimator that uses the minimum width of the - Freedman-Diaconis and Sturges estimators if the FD bandwidth is non zero - and the Sturges estimator if the FD bandwidth is 0. + Freedman-Diaconis and Sturges estimators if the FD bin width is non-zero. + If the bin width from the FD estimator is 0, the Sturges estimator is used. The FD estimator is usually the most robust method, but its width estimate tends to be too large for small `x` and bad for data with limited variance. The Sturges estimator is quite good for small (<1000) datasets - and is the default in the R language. This method gives good off the shelf + and is the default in the R language. This method gives good off-the-shelf behaviour. .. versionchanged:: 1.15.0 If there is limited variance the IQR can be 0, which results in the FD bin width being 0 too. This is not a valid bin width, so ``np.histogram_bin_edges`` chooses 1 bin instead, which may not be optimal. - If the IQR is 0, it's unlikely any variance based estimators will be of - use, so we revert to the sturges estimator, which only uses the size of the + If the IQR is 0, it's unlikely any variance-based estimators will be of + use, so we revert to the Sturges estimator, which only uses the size of the dataset in its calculation. Parameters @@ -383,7 +382,7 @@ def _get_bin_edges(a, bins, range, weights): n_equal_bins = None bin_edges = None - if isinstance(bins, basestring): + if isinstance(bins, str): bin_name = bins # if `bins` is a string for an automatic method, # this will replace it with the number of bins calculated @@ -418,9 +417,9 @@ def _get_bin_edges(a, bins, range, weights): elif np.ndim(bins) == 0: try: n_equal_bins = operator.index(bins) - except TypeError: + except TypeError as e: raise TypeError( - '`bins` must be an integer, a string, or an array') + '`bins` must be an integer, a string, or an array') from e if n_equal_bins < 1: raise ValueError('`bins` must be positive, when an integer') diff --git a/numpy/lib/index_tricks.py b/numpy/lib/index_tricks.py index 0560bd36d..d145477c3 100644 --- a/numpy/lib/index_tricks.py +++ b/numpy/lib/index_tricks.py @@ -368,8 +368,10 @@ class AxisConcatenator: if len(vec) == 3: trans1d = int(vec[2]) continue - except Exception: - raise ValueError("unknown special directive") + except Exception as e: + raise ValueError( + "unknown special directive {!r}".format(item) + ) from e try: axis = int(item) continue @@ -597,8 +599,6 @@ class ndenumerate: def __iter__(self): return self - next = __next__ - @set_module('numpy') class ndindex: @@ -665,8 +665,6 @@ class ndindex: next(self._it) return self._it.multi_index - next = __next__ - # You can do all this with slice() plus a few special objects, # but there's a lot to remember. This version is simpler because diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 29af488d2..f5a548433 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -23,7 +23,7 @@ from ._iotools import ( ) from numpy.compat import ( - asbytes, asstr, asunicode, bytes, basestring, os_fspath, os_PathLike, + asbytes, asstr, asunicode, bytes, os_fspath, os_PathLike, pickle, contextlib_nullcontext ) @@ -408,15 +408,14 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, pickle_kwargs = dict(encoding=encoding, fix_imports=fix_imports) - # TODO: Use contextlib.ExitStack once we drop Python 2 - if hasattr(file, 'read'): - fid = file - own_fid = False - else: - fid = open(os_fspath(file), "rb") - own_fid = True + with contextlib.ExitStack() as stack: + if hasattr(file, 'read'): + fid = file + own_fid = False + else: + fid = stack.enter_context(open(os_fspath(file), "rb")) + own_fid = True - try: # Code to distinguish from NumPy binary files and pickles. _ZIP_PREFIX = b'PK\x03\x04' _ZIP_SUFFIX = b'PK\x05\x06' # empty zip files start with this @@ -427,10 +426,10 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, fid.seek(-min(N, len(magic)), 1) # back-up if magic.startswith(_ZIP_PREFIX) or magic.startswith(_ZIP_SUFFIX): # zip-file (assume .npz) - # Transfer file ownership to NpzFile + # Potentially transfer file ownership to NpzFile + stack.pop_all() ret = NpzFile(fid, own_fid=own_fid, allow_pickle=allow_pickle, pickle_kwargs=pickle_kwargs) - own_fid = False return ret elif magic == format.MAGIC_PREFIX: # .npy file @@ -449,9 +448,6 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, except Exception: raise IOError( "Failed to interpret file %s as a pickle" % repr(file)) - finally: - if own_fid: - fid.close() def _save_dispatcher(file, arr, allow_pickle=None, fix_imports=None): @@ -519,30 +515,23 @@ def save(file, arr, allow_pickle=True, fix_imports=True): >>> print(a, b) # [1 2] [1 3] """ - own_fid = False if hasattr(file, 'write'): - fid = file + file_ctx = contextlib_nullcontext(file) else: file = os_fspath(file) if not file.endswith('.npy'): file = file + '.npy' - fid = open(file, "wb") - own_fid = True + file_ctx = open(file, "wb") - try: + with file_ctx as fid: arr = np.asanyarray(arr) format.write_array(fid, arr, allow_pickle=allow_pickle, pickle_kwargs=dict(fix_imports=fix_imports)) - finally: - if own_fid: - fid.close() def _savez_dispatcher(file, *args, **kwds): - for a in args: - yield a - for v in kwds.values(): - yield v + yield from args + yield from kwds.values() @array_function_dispatch(_savez_dispatcher) @@ -628,10 +617,8 @@ def savez(file, *args, **kwds): def _savez_compressed_dispatcher(file, *args, **kwds): - for a in args: - yield a - for v in kwds.values(): - yield v + yield from args + yield from kwds.values() @array_function_dispatch(_savez_compressed_dispatcher) @@ -805,7 +792,7 @@ _loadtxt_chunksize = 50000 def loadtxt(fname, dtype=float, comments='#', delimiter=None, converters=None, skiprows=0, usecols=None, unpack=False, ndmin=0, encoding='bytes', max_rows=None): - """ + r""" Load data from a text file. Each row in the text file must have the same number of values. @@ -897,28 +884,39 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, Examples -------- >>> from io import StringIO # StringIO behaves like a file object - >>> c = StringIO(u"0 1\\n2 3") + >>> c = StringIO("0 1\n2 3") >>> np.loadtxt(c) array([[0., 1.], [2., 3.]]) - >>> d = StringIO(u"M 21 72\\nF 35 58") + >>> d = StringIO("M 21 72\nF 35 58") >>> np.loadtxt(d, dtype={'names': ('gender', 'age', 'weight'), ... 'formats': ('S1', 'i4', 'f4')}) array([(b'M', 21, 72.), (b'F', 35, 58.)], dtype=[('gender', 'S1'), ('age', '<i4'), ('weight', '<f4')]) - >>> c = StringIO(u"1,0,2\\n3,0,4") + >>> c = StringIO("1,0,2\n3,0,4") >>> x, y = np.loadtxt(c, delimiter=',', usecols=(0, 2), unpack=True) >>> x array([1., 3.]) >>> y array([2., 4.]) + This example shows how `converters` can be used to convert a field + with a trailing minus sign into a negative number. + + >>> s = StringIO('10.01 31.25-\n19.22 64.31\n17.57- 63.94') + >>> def conv(fld): + ... return -float(fld[:-1]) if fld.endswith(b'-') else float(fld) + ... + >>> np.loadtxt(s, converters={0: conv, 1: conv}) + array([[ 10.01, -31.25], + [ 19.22, 64.31], + [-17.57, 63.94]]) """ # Type conversions for Py3 convenience if comments is not None: - if isinstance(comments, (basestring, bytes)): + if isinstance(comments, (str, bytes)): comments = [comments] comments = [_decode_line(x) for x in comments] # Compile regex for comments beforehand @@ -1319,7 +1317,7 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', delimiter = asstr(delimiter) class WriteWrap: - """Convert to unicode in py2 or to bytes on bytestream inputs. + """Convert to bytes on bytestream inputs. """ def __init__(self, fh, encoding): @@ -1391,7 +1389,7 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', if len(fmt) != ncol: raise AttributeError('fmt has wrong shape. %s' % str(fmt)) format = asstr(delimiter).join(map(asstr, fmt)) - elif isinstance(fmt, basestring): + elif isinstance(fmt, str): n_fmt_chars = fmt.count('%') error = ValueError('fmt has wrong number of %% formats: %s' % fmt) if n_fmt_chars == 1: @@ -1657,7 +1655,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, ----- * When spaces are used as delimiters, or when no delimiter has been given as input, there should not be any missing data between two fields. - * When the variables are named (either by a flexible dtype or with `names`, + * When the variables are named (either by a flexible dtype or with `names`), there must not be any header in the file (else a ValueError exception is raised). * Individual values are not stripped of spaces by default. @@ -1747,7 +1745,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, try: if isinstance(fname, os_PathLike): fname = os_fspath(fname) - if isinstance(fname, basestring): + if isinstance(fname, str): fid = np.lib._datasource.open(fname, 'rt', encoding=encoding) fid_ctx = contextlib.closing(fid) else: @@ -1889,7 +1887,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, if value not in entry: entry.append(value) # We have a string : apply it to all entries - elif isinstance(user_missing_values, basestring): + elif isinstance(user_missing_values, str): user_value = user_missing_values.split(",") for entry in missing_values: entry.extend(user_value) diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index af4cfa09d..a11d5f2c7 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -13,7 +13,6 @@ from numpy.ma import MaskedArray from numpy.ma.mrecords import MaskedRecords from numpy.core.overrides import array_function_dispatch from numpy.lib._iotools import _is_string_like -from numpy.compat import basestring from numpy.testing import suppress_warnings _check_fill_value = np.ma.core._check_fill_value @@ -286,8 +285,7 @@ def _izip_fields_flat(iterable): """ for element in iterable: if isinstance(element, np.void): - for f in _izip_fields_flat(tuple(element)): - yield f + yield from _izip_fields_flat(tuple(element)) else: yield element @@ -299,12 +297,11 @@ def _izip_fields(iterable): """ for element in iterable: if (hasattr(element, '__iter__') and - not isinstance(element, basestring)): - for f in _izip_fields(element): - yield f + not isinstance(element, str)): + yield from _izip_fields(element) elif isinstance(element, np.void) and len(tuple(element)) == 1: - for f in _izip_fields(element): - yield f + # this statement is the same from the previous expression + yield from _izip_fields(element) else: yield element @@ -658,8 +655,7 @@ def rename_fields(base, namemapper): def _append_fields_dispatcher(base, names, data, dtypes=None, fill_value=None, usemask=None, asrecarray=None): yield base - for d in data: - yield d + yield from data @array_function_dispatch(_append_fields_dispatcher) @@ -698,7 +694,7 @@ def append_fields(base, names, data, dtypes=None, if len(names) != len(data): msg = "The number of arrays does not match the number of names" raise ValueError(msg) - elif isinstance(names, basestring): + elif isinstance(names, str): names = [names, ] data = [data, ] # @@ -735,8 +731,7 @@ def append_fields(base, names, data, dtypes=None, def _rec_append_fields_dispatcher(base, names, data, dtypes=None): yield base - for d in data: - yield d + yield from data @array_function_dispatch(_rec_append_fields_dispatcher) @@ -1455,7 +1450,7 @@ def join_by(key, r1, r2, jointype='inner', r1postfix='1', r2postfix='2', "'outer' or 'leftouter' (got '%s' instead)" % jointype ) # If we have a single key, put it in a tuple - if isinstance(key, basestring): + if isinstance(key, str): key = (key,) # Check the keys diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py index 7634af010..72a7f79d7 100644 --- a/numpy/lib/shape_base.py +++ b/numpy/lib/shape_base.py @@ -269,8 +269,8 @@ def apply_along_axis(func1d, axis, arr, *args, **kwargs): """ Apply a function to 1-D slices along the given axis. - Execute `func1d(a, *args)` where `func1d` operates on 1-D arrays and `a` - is a 1-D slice of `arr` along `axis`. + Execute `func1d(a, *args, **kwargs)` where `func1d` operates on 1-D arrays + and `a` is a 1-D slice of `arr` along `axis`. This is equivalent to (but faster than) the following use of `ndindex` and `s_`, which sets each of ``ii``, ``jj``, and ``kk`` to a tuple of indices:: @@ -372,8 +372,10 @@ def apply_along_axis(func1d, axis, arr, *args, **kwargs): # invoke the function on the first item try: ind0 = next(inds) - except StopIteration: - raise ValueError('Cannot apply_along_axis when any iteration dimensions are 0') + except StopIteration as e: + raise ValueError( + 'Cannot apply_along_axis when any iteration dimensions are 0' + ) from None res = asanyarray(func1d(inarr_view[ind0], *args, **kwargs)) # build a buffer for storing evaluations of func1d. diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index d3bd88d7f..1ed7815d9 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -1,13 +1,10 @@ import os -import sys import pytest from tempfile import mkdtemp, mkstemp, NamedTemporaryFile from shutil import rmtree import numpy.lib._datasource as datasource -from numpy.testing import ( - assert_, assert_equal, assert_raises, assert_warns - ) +from numpy.testing import assert_, assert_equal, assert_raises import urllib.request as urllib_request from urllib.parse import urlparse diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index 1d69d869e..6964c1128 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -9,7 +9,6 @@ from numpy.lib._iotools import ( LineSplitter, NameValidator, StringConverter, has_nested_fields, easy_dtype, flatten_dtype ) -from numpy.compat import unicode class TestLineSplitter: @@ -179,10 +178,10 @@ class TestStringConverter: # note that the longdouble type has been skipped, so the # _status increases by 2. Everything should succeed with # unicode conversion (5). - for s in ['a', u'a', b'a']: + for s in ['a', b'a']: res = converter.upgrade(s) - assert_(type(res) is unicode) - assert_equal(res, u'a') + assert_(type(res) is str) + assert_equal(res, 'a') assert_equal(converter._status, 5 + status_offset) def test_missing(self): diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index 851fd31ea..81ba789e3 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -11,7 +11,6 @@ from numpy.lib.arraysetops import ( import pytest - class TestSetOps: def test_intersect1d(self): @@ -118,12 +117,13 @@ class TestSetOps: assert_array_equal([-1, 0], ediff1d(zero_elem, to_begin=-1, to_end=0)) assert_array_equal([], ediff1d(one_elem)) assert_array_equal([1], ediff1d(two_elem)) - assert_array_equal([7,1,9], ediff1d(two_elem, to_begin=7, to_end=9)) - assert_array_equal([5,6,1,7,8], ediff1d(two_elem, to_begin=[5,6], to_end=[7,8])) - assert_array_equal([1,9], ediff1d(two_elem, to_end=9)) - assert_array_equal([1,7,8], ediff1d(two_elem, to_end=[7,8])) - assert_array_equal([7,1], ediff1d(two_elem, to_begin=7)) - assert_array_equal([5,6,1], ediff1d(two_elem, to_begin=[5,6])) + assert_array_equal([7, 1, 9], ediff1d(two_elem, to_begin=7, to_end=9)) + assert_array_equal([5, 6, 1, 7, 8], + ediff1d(two_elem, to_begin=[5, 6], to_end=[7, 8])) + assert_array_equal([1, 9], ediff1d(two_elem, to_end=9)) + assert_array_equal([1, 7, 8], ediff1d(two_elem, to_end=[7, 8])) + assert_array_equal([7, 1], ediff1d(two_elem, to_begin=7)) + assert_array_equal([5, 6, 1], ediff1d(two_elem, to_begin=[5, 6])) @pytest.mark.parametrize("ary, prepend, append", [ # should fail because trying to cast @@ -156,27 +156,27 @@ class TestSetOps: to_end=append, to_begin=prepend) - @pytest.mark.parametrize("ary," - "prepend," - "append," - "expected", [ - (np.array([1, 2, 3], dtype=np.int16), - 2**16, # will be cast to int16 under same kind rule. - 2**16 + 4, - np.array([0, 1, 1, 4], dtype=np.int16)), - (np.array([1, 2, 3], dtype=np.float32), - np.array([5], dtype=np.float64), - None, - np.array([5, 1, 1], dtype=np.float32)), - (np.array([1, 2, 3], dtype=np.int32), - 0, - 0, - np.array([0, 1, 1, 0], dtype=np.int32)), - (np.array([1, 2, 3], dtype=np.int64), - 3, - -9, - np.array([3, 1, 1, -9], dtype=np.int64)), - ]) + @pytest.mark.parametrize( + "ary,prepend,append,expected", + [ + (np.array([1, 2, 3], dtype=np.int16), + 2**16, # will be cast to int16 under same kind rule. + 2**16 + 4, + np.array([0, 1, 1, 4], dtype=np.int16)), + (np.array([1, 2, 3], dtype=np.float32), + np.array([5], dtype=np.float64), + None, + np.array([5, 1, 1], dtype=np.float32)), + (np.array([1, 2, 3], dtype=np.int32), + 0, + 0, + np.array([0, 1, 1, 0], dtype=np.int32)), + (np.array([1, 2, 3], dtype=np.int64), + 3, + -9, + np.array([3, 1, 1, -9], dtype=np.int64)), + ] + ) def test_ediff1d_scalar_handling(self, ary, prepend, @@ -191,7 +191,6 @@ class TestSetOps: assert_equal(actual, expected) assert actual.dtype == expected.dtype - def test_isin(self): # the tests for in1d cover most of isin's behavior # if in1d is removed, would need to change those tests to test @@ -200,33 +199,34 @@ class TestSetOps: b = np.asarray(b).flatten().tolist() return a in b isin_slow = np.vectorize(_isin_slow, otypes=[bool], excluded={1}) + def assert_isin_equal(a, b): x = isin(a, b) y = isin_slow(a, b) assert_array_equal(x, y) - #multidimensional arrays in both arguments + # multidimensional arrays in both arguments a = np.arange(24).reshape([2, 3, 4]) b = np.array([[10, 20, 30], [0, 1, 3], [11, 22, 33]]) assert_isin_equal(a, b) - #array-likes as both arguments + # array-likes as both arguments c = [(9, 8), (7, 6)] d = (9, 7) assert_isin_equal(c, d) - #zero-d array: + # zero-d array: f = np.array(3) assert_isin_equal(f, b) assert_isin_equal(a, f) assert_isin_equal(f, f) - #scalar: + # scalar: assert_isin_equal(5, b) assert_isin_equal(a, 6) assert_isin_equal(5, 6) - #empty array-like: + # empty array-like: x = [] assert_isin_equal(x, b) assert_isin_equal(a, x) @@ -520,7 +520,8 @@ class TestUnique: a = [] a1_idx = np.unique(a, return_index=True)[1] a2_inv = np.unique(a, return_inverse=True)[1] - a3_idx, a3_inv = np.unique(a, return_index=True, return_inverse=True)[1:] + a3_idx, a3_inv = np.unique(a, return_index=True, + return_inverse=True)[1:] assert_equal(a1_idx.dtype, np.intp) assert_equal(a2_inv.dtype, np.intp) assert_equal(a3_idx.dtype, np.intp) @@ -563,9 +564,52 @@ class TestUnique: result = np.array([[-0.0, 0.0]]) assert_array_equal(unique(data, axis=0), result, msg) + @pytest.mark.parametrize("axis", [0, -1]) + def test_unique_1d_with_axis(self, axis): + x = np.array([4, 3, 2, 3, 2, 1, 2, 2]) + uniq = unique(x, axis=axis) + assert_array_equal(uniq, [1, 2, 3, 4]) + + def test_unique_axis_zeros(self): + # issue 15559 + single_zero = np.empty(shape=(2, 0), dtype=np.int8) + uniq, idx, inv, cnt = unique(single_zero, axis=0, return_index=True, + return_inverse=True, return_counts=True) + + # there's 1 element of shape (0,) along axis 0 + assert_equal(uniq.dtype, single_zero.dtype) + assert_array_equal(uniq, np.empty(shape=(1, 0))) + assert_array_equal(idx, np.array([0])) + assert_array_equal(inv, np.array([0, 0])) + assert_array_equal(cnt, np.array([2])) + + # there's 0 elements of shape (2,) along axis 1 + uniq, idx, inv, cnt = unique(single_zero, axis=1, return_index=True, + return_inverse=True, return_counts=True) + + assert_equal(uniq.dtype, single_zero.dtype) + assert_array_equal(uniq, np.empty(shape=(2, 0))) + assert_array_equal(idx, np.array([])) + assert_array_equal(inv, np.array([])) + assert_array_equal(cnt, np.array([])) + + # test a "complicated" shape + shape = (0, 2, 0, 3, 0, 4, 0) + multiple_zeros = np.empty(shape=shape) + for axis in range(len(shape)): + expected_shape = list(shape) + if shape[axis] == 0: + expected_shape[axis] = 0 + else: + expected_shape[axis] = 1 + + assert_array_equal(unique(multiple_zeros, axis=axis), + np.empty(shape=expected_shape)) + def test_unique_masked(self): # issue 8664 - x = np.array([64, 0, 1, 2, 3, 63, 63, 0, 0, 0, 1, 2, 0, 63, 0], dtype='uint8') + x = np.array([64, 0, 1, 2, 3, 63, 63, 0, 0, 0, 1, 2, 0, 63, 0], + dtype='uint8') y = np.ma.masked_equal(x, 0) v = np.unique(y) @@ -580,7 +624,7 @@ class TestUnique: # as unsigned byte strings. See gh-10495. fmt = "sort order incorrect for integer type '%s'" for dt in 'bhilq': - a = np.array([[-1],[0]], dt) + a = np.array([[-1], [0]], dt) b = np.unique(a, axis=0) assert_array_equal(a, b, fmt % dt) diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 24593d7b3..2dbaeb8cb 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -535,8 +535,10 @@ dt4 = np.dtype({'names': ['a', '', 'b'], 'formats': ['i4']*3}) # titles dt5 = np.dtype({'names': ['a', 'b'], 'formats': ['i4', 'i4'], 'offsets': [1, 6], 'titles': ['aa', 'bb']}) +# empty +dt6 = np.dtype({'names': [], 'formats': [], 'itemsize': 8}) -@pytest.mark.parametrize("dt", [dt1, dt2, dt3, dt4, dt5]) +@pytest.mark.parametrize("dt", [dt1, dt2, dt3, dt4, dt5, dt6]) def test_load_padded_dtype(dt): arr = np.zeros(3, dt) for i in range(3): diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 7953de15d..23bf3296d 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -21,7 +21,6 @@ from numpy.lib import ( select, setxor1d, sinc, trapz, trim_zeros, unwrap, unique, vectorize ) -from numpy.compat import long def get_mat(n): data = np.arange(n) @@ -273,6 +272,13 @@ class TestCopy: assert_(not a_fort_copy.flags.c_contiguous) assert_(a_fort_copy.flags.f_contiguous) + def test_subok(self): + mx = ma.ones(5) + assert_(not ma.isMaskedArray(np.copy(mx, subok=False))) + assert_(ma.isMaskedArray(np.copy(mx, subok=True))) + # Default behavior + assert_(not ma.isMaskedArray(np.copy(mx))) + class TestAverage: @@ -502,12 +508,11 @@ class TestInsert: insert(a, 1, a[:, 2, :], axis=1)) def test_0d(self): - # This is an error in the future a = np.array(1) - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', DeprecationWarning) - assert_equal(insert(a, [], 2, axis=0), np.array(2)) - assert_(w[0].category is DeprecationWarning) + with pytest.raises(np.AxisError): + insert(a, [], 2, axis=0) + with pytest.raises(TypeError): + insert(a, [], 2, axis="nonsense") def test_subclass(self): class SubClass(np.ndarray): @@ -537,6 +542,12 @@ class TestInsert: b = np.insert(a, [0, 2], val) assert_array_equal(b[[0, 3]], np.array(val, dtype=b.dtype)) + def test_index_floats(self): + with pytest.raises(IndexError): + np.insert([0, 1, 2], np.array([1.0, 2.0]), [10, 20]) + with pytest.raises(IndexError): + np.insert([0, 1, 2], np.array([], dtype=float), []) + class TestAmax: @@ -797,10 +808,6 @@ class TestDelete: a_del = delete(self.a, indices) nd_a_del = delete(self.nd_a, indices, axis=1) msg = 'Delete failed for obj: %r' % indices - # NOTE: The cast should be removed after warning phase for bools - if not isinstance(indices, (slice, int, long, np.integer)): - indices = np.asarray(indices, dtype=np.intp) - indices = indices[(indices >= 0) & (indices < 5)] assert_array_equal(setxor1d(a_del, self.a[indices, ]), self.a, err_msg=msg) xor = setxor1d(nd_a_del[0,:, 0], self.nd_a[0, indices, 0]) @@ -816,19 +823,25 @@ class TestDelete: self._check_inverse_of_slicing(s) def test_fancy(self): - # Deprecation/FutureWarning tests should be kept after change. self._check_inverse_of_slicing(np.array([[0, 1], [2, 1]])) - with warnings.catch_warnings(): - warnings.filterwarnings('error', category=DeprecationWarning) - assert_raises(DeprecationWarning, delete, self.a, [100]) - assert_raises(DeprecationWarning, delete, self.a, [-100]) - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', category=FutureWarning) - self._check_inverse_of_slicing([0, -1, 2, 2]) - obj = np.array([True, False, False], dtype=bool) - self._check_inverse_of_slicing(obj) - assert_(w[0].category is FutureWarning) - assert_(w[1].category is FutureWarning) + with pytest.raises(IndexError): + delete(self.a, [100]) + with pytest.raises(IndexError): + delete(self.a, [-100]) + + self._check_inverse_of_slicing([0, -1, 2, 2]) + + self._check_inverse_of_slicing([True, False, False, True, False]) + + # not legal, indexing with these would change the dimension + with pytest.raises(ValueError): + delete(self.a, True) + with pytest.raises(ValueError): + delete(self.a, False) + + # not enough items + with pytest.raises(ValueError): + delete(self.a, [False]*4) def test_single(self): self._check_inverse_of_slicing(0) @@ -836,10 +849,10 @@ class TestDelete: def test_0d(self): a = np.array(1) - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', DeprecationWarning) - assert_equal(delete(a, [], axis=0), a) - assert_(w[0].category is DeprecationWarning) + with pytest.raises(np.AxisError): + delete(a, [], axis=0) + with pytest.raises(TypeError): + delete(a, [], axis="nonsense") def test_subclass(self): class SubClass(np.ndarray): @@ -861,6 +874,12 @@ class TestDelete: assert_equal(m.flags.c_contiguous, k.flags.c_contiguous) assert_equal(m.flags.f_contiguous, k.flags.f_contiguous) + def test_index_floats(self): + with pytest.raises(IndexError): + np.delete([0, 1, 2], np.array([1.0, 2.0])) + with pytest.raises(IndexError): + np.delete([0, 1, 2], np.array([], dtype=float)) + class TestGradient: @@ -1899,7 +1918,7 @@ class TestCov: frequencies = np.array([1, 4, 1]) x2_repeats = np.array([[0.0], [1.0], [1.0], [1.0], [1.0], [2.0]]).T res2 = np.array([[0.4, -0.4], [-0.4, 0.4]]) - unit_frequencies = np.ones(3, dtype=np.integer) + unit_frequencies = np.ones(3, dtype=np.int_) weights = np.array([1.0, 4.0, 1.0]) res3 = np.array([[2. / 3., -2. / 3.], [-2. / 3., 2. / 3.]]) unit_weights = np.ones(3) @@ -1952,11 +1971,11 @@ class TestCov: self.res1) nonint = self.frequencies + 0.5 assert_raises(TypeError, cov, self.x1, fweights=nonint) - f = np.ones((2, 3), dtype=np.integer) + f = np.ones((2, 3), dtype=np.int_) assert_raises(RuntimeError, cov, self.x1, fweights=f) - f = np.ones(2, dtype=np.integer) + f = np.ones(2, dtype=np.int_) assert_raises(RuntimeError, cov, self.x1, fweights=f) - f = -1 * np.ones(3, dtype=np.integer) + f = -1 * np.ones(3, dtype=np.int_) assert_raises(ValueError, cov, self.x1, fweights=f) def test_aweights(self): @@ -2967,6 +2986,16 @@ class TestQuantile: assert_equal(np.quantile(x, 1), 3.5) assert_equal(np.quantile(x, 0.5), 1.75) + def test_correct_quantile_value(self): + a = np.array([True]) + tf_quant = np.quantile(True, False) + assert_equal(tf_quant, a[0]) + assert_equal(type(tf_quant), a.dtype) + a = np.array([False, True, True]) + quant_res = np.quantile(a, a) + assert_array_equal(quant_res, a) + assert_equal(a.dtype, quant_res.dtype) + def test_fraction(self): # fractional input, integral quantile x = [Fraction(i, 2) for i in range(8)] diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 2d6f39e40..9abde3e11 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -1,4 +1,5 @@ import sys +import gc import gzip import os import threading @@ -7,15 +8,17 @@ import warnings import io import re import pytest +from pathlib import Path from tempfile import NamedTemporaryFile from io import BytesIO, StringIO from datetime import datetime import locale +from multiprocessing import Process import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConversionWarning -from numpy.compat import asbytes, bytes, Path +from numpy.compat import asbytes, bytes from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, @@ -275,8 +278,7 @@ class TestSavezLoad(RoundtripTest): fp.seek(0) assert_(not fp.closed) - #FIXME: Is this still true? - @pytest.mark.skipif(IS_PYPY, reason="Missing context manager on PyPy") + @pytest.mark.slow_pypy def test_closing_fid(self): # Test that issue #1517 (too many opened files) remains closed # It might be a "weak" test since failed to get triggered on @@ -289,17 +291,18 @@ class TestSavezLoad(RoundtripTest): # numpy npz file returned by np.load when their reference count # goes to zero. Python 3 running in debug mode raises a # ResourceWarning when file closing is left to the garbage - # collector, so we catch the warnings. Because ResourceWarning - # is unknown in Python < 3.x, we take the easy way out and - # catch all warnings. + # collector, so we catch the warnings. with suppress_warnings() as sup: - sup.filter(Warning) # TODO: specify exact message + sup.filter(ResourceWarning) # TODO: specify exact message for i in range(1, 1025): try: np.load(tmp)["data"] except Exception as e: msg = "Failed to load data from a file: %s" % e raise AssertionError(msg) + finally: + if IS_PYPY: + gc.collect() def test_closing_zipfile_after_load(self): # Check that zipfile owns file and can close it. This needs to @@ -362,7 +365,6 @@ class TestSaveTxt: c.seek(0) assert_equal(c.readlines(), [b'1 3\n', b'4 6\n']) - @pytest.mark.skipif(Path is None, reason="No pathlib.Path") def test_multifield_view(self): a = np.ones(1, dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'f4')]) v = a[['x', 'z']] @@ -568,16 +570,21 @@ class TestSaveTxt: else: assert_equal(s.read(), b"%f\n" % 1.) - @pytest.mark.skipif(sys.platform=='win32', - reason="large files cause problems") + @pytest.mark.skipif(sys.platform=='win32', reason="files>4GB may not work") @pytest.mark.slow @requires_memory(free_bytes=7e9) def test_large_zip(self): - # The test takes at least 6GB of memory, writes a file larger than 4GB - test_data = np.asarray([np.random.rand(np.random.randint(50,100),4) - for i in range(800000)], dtype=object) - with tempdir() as tmpdir: - np.savez(os.path.join(tmpdir, 'test.npz'), test_data=test_data) + def check_large_zip(): + # The test takes at least 6GB of memory, writes a file larger than 4GB + test_data = np.asarray([np.random.rand(np.random.randint(50,100),4) + for i in range(800000)], dtype=object) + with tempdir() as tmpdir: + np.savez(os.path.join(tmpdir, 'test.npz'), test_data=test_data) + # run in a subprocess to ensure memory is released on PyPy, see gh-15775 + p = Process(target=check_large_zip) + p.start() + p.join() + assert p.exitcode == 0 class LoadTxtBase: def check_compressed(self, fopen, suffixes): @@ -2332,14 +2339,13 @@ M 33 21.99 assert_(test.dtype['f0'] == float) assert_(test.dtype['f1'] == np.int64) - assert_(test.dtype['f2'] == np.integer) + assert_(test.dtype['f2'] == np.int_) assert_allclose(test['f0'], 73786976294838206464.) assert_equal(test['f1'], 17179869184) assert_equal(test['f2'], 1024) -@pytest.mark.skipif(Path is None, reason="No pathlib.Path") class TestPathUsage: # Test that pathlib.Path can be used def test_loadtxt(self): diff --git a/numpy/lib/tests/test_mixins.py b/numpy/lib/tests/test_mixins.py index e184ffe19..632058763 100644 --- a/numpy/lib/tests/test_mixins.py +++ b/numpy/lib/tests/test_mixins.py @@ -1,6 +1,5 @@ import numbers import operator -import sys import numpy as np from numpy.testing import assert_, assert_equal, assert_raises @@ -81,7 +80,6 @@ _ALL_BINARY_OPERATORS = [ operator.mul, operator.truediv, operator.floordiv, - # TODO: test div on Python 2, only operator.mod, divmod, pow, diff --git a/numpy/lib/tests/test_regression.py b/numpy/lib/tests/test_regression.py index 37cc158ba..55df2a675 100644 --- a/numpy/lib/tests/test_regression.py +++ b/numpy/lib/tests/test_regression.py @@ -1,12 +1,10 @@ import os -import sys import numpy as np from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_array_almost_equal, assert_raises, _assert_valid_refcount, ) -from numpy.compat import unicode class TestRegression: @@ -181,7 +179,7 @@ class TestRegression: # related to ticket #1405. include_dirs = [np.get_include()] for path in include_dirs: - assert_(isinstance(path, (str, unicode))) + assert_(isinstance(path, str)) assert_(path != '') def test_polyder_return_type(self): diff --git a/numpy/lib/tests/test_type_check.py b/numpy/lib/tests/test_type_check.py index 47685550a..3f4ca6309 100644 --- a/numpy/lib/tests/test_type_check.py +++ b/numpy/lib/tests/test_type_check.py @@ -1,5 +1,4 @@ import numpy as np -from numpy.compat import long from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_raises ) @@ -86,7 +85,6 @@ class TestIsscalar: assert_(not np.isscalar([3])) assert_(not np.isscalar((3,))) assert_(np.isscalar(3j)) - assert_(np.isscalar(long(10))) assert_(np.isscalar(4.0)) diff --git a/numpy/lib/type_check.py b/numpy/lib/type_check.py index 13db9adc3..2a2982ab3 100644 --- a/numpy/lib/type_check.py +++ b/numpy/lib/type_check.py @@ -492,7 +492,8 @@ def _real_if_close_dispatcher(a, tol=None): @array_function_dispatch(_real_if_close_dispatcher) def real_if_close(a, tol=100): """ - If complex input returns a real array if complex parts are close to zero. + If input is complex with all imaginary parts close to zero, return + real parts. "Close to zero" is defined as `tol` * (machine epsilon of the type for `a`). @@ -527,10 +528,10 @@ def real_if_close(a, tol=100): >>> np.finfo(float).eps 2.2204460492503131e-16 # may vary - >>> np.real_if_close([2.1 + 4e-14j], tol=1000) - array([2.1]) - >>> np.real_if_close([2.1 + 4e-13j], tol=1000) - array([2.1+4.e-13j]) + >>> np.real_if_close([2.1 + 4e-14j, 5.2 + 3e-15j], tol=1000) + array([2.1, 5.2]) + >>> np.real_if_close([2.1 + 4e-13j, 5.2 + 3e-15j], tol=1000) + array([2.1+4.e-13j, 5.2 + 3e-15j]) """ a = asanyarray(a) diff --git a/numpy/lib/ufunclike.py b/numpy/lib/ufunclike.py index 8512669c2..1f26a1845 100644 --- a/numpy/lib/ufunclike.py +++ b/numpy/lib/ufunclike.py @@ -188,9 +188,9 @@ def isposinf(x, out=None): is_inf = nx.isinf(x) try: signbit = ~nx.signbit(x) - except TypeError: + except TypeError as e: raise TypeError('This operation is not supported for complex values ' - 'because it would be ambiguous.') + 'because it would be ambiguous.') from e else: return nx.logical_and(is_inf, signbit, out) @@ -259,8 +259,8 @@ def isneginf(x, out=None): is_inf = nx.isinf(x) try: signbit = nx.signbit(x) - except TypeError: + except TypeError as e: raise TypeError('This operation is not supported for complex values ' - 'because it would be ambiguous.') + 'because it would be ambiguous.') from e else: return nx.logical_and(is_inf, signbit, out) diff --git a/numpy/lib/user_array.py b/numpy/lib/user_array.py index 9c266fd6b..0e96b477e 100644 --- a/numpy/lib/user_array.py +++ b/numpy/lib/user_array.py @@ -11,7 +11,6 @@ from numpy.core import ( bitwise_xor, invert, less, less_equal, not_equal, equal, greater, greater_equal, shape, reshape, arange, sin, sqrt, transpose ) -from numpy.compat import long class container: @@ -196,9 +195,6 @@ class container: def __int__(self): return self._scalarfunc(int) - def __long__(self): - return self._scalarfunc(long) - def __hex__(self): return self._scalarfunc(hex) @@ -231,6 +227,10 @@ class container: "" return self.array.tostring() + def tobytes(self): + "" + return self.array.tobytes() + def byteswap(self): "" return self._rc(self.array.byteswap()) diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index 152322115..f233c7240 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -605,41 +605,6 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): ) print(" %s -- %s" % (meth, methstr), file=output) - elif (sys.version_info[0] < 3 - and isinstance(object, types.InstanceType)): - # check for __call__ method - # types.InstanceType is the type of the instances of oldstyle classes - print("Instance of class: ", object.__class__.__name__, file=output) - print(file=output) - if hasattr(object, '__call__'): - arguments = formatargspec( - *getargspec(object.__call__.__func__) - ) - arglist = arguments.split(', ') - if len(arglist) > 1: - arglist[1] = "("+arglist[1] - arguments = ", ".join(arglist[1:]) - else: - arguments = "()" - - if hasattr(object, 'name'): - name = "%s" % object.name - else: - name = "<name>" - if len(name+arguments) > maxwidth: - argstr = _split_line(name, arguments, maxwidth) - else: - argstr = name + arguments - - print(" " + argstr + "\n", file=output) - doc = inspect.getdoc(object.__call__) - if doc is not None: - print(inspect.getdoc(object.__call__), file=output) - print(inspect.getdoc(object), file=output) - - else: - print(inspect.getdoc(object), file=output) - elif inspect.ismethod(object): name = object.__name__ arguments = formatargspec( @@ -867,7 +832,6 @@ def _lookfor_generate_cache(module, import_modules, regenerate): or newly generated. """ - global _lookfor_caches # Local import to speed up numpy's import time. import inspect diff --git a/numpy/linalg/lapack_lite/README.rst b/numpy/linalg/lapack_lite/README.rst index ba30aa4ed..ed738ab86 100644 --- a/numpy/linalg/lapack_lite/README.rst +++ b/numpy/linalg/lapack_lite/README.rst @@ -20,7 +20,7 @@ The routines that ``lapack_litemodule.c`` wraps are listed in properly. Assuming that you have an unpacked LAPACK source tree in ``~/LAPACK``, you generate the new routines in this directory with:: -$ python2 ./make_lite.py wrapped_routines ~/LAPACK +$ python ./make_lite.py wrapped_routines ~/LAPACK This will grab the right routines, with dependencies, put them into the appropriate ``f2c_*.f`` files, run ``f2c`` over them, then do some scrubbing diff --git a/numpy/linalg/lapack_lite/clapack_scrub.py b/numpy/linalg/lapack_lite/clapack_scrub.py index 531d861cf..f3e7d25d2 100644 --- a/numpy/linalg/lapack_lite/clapack_scrub.py +++ b/numpy/linalg/lapack_lite/clapack_scrub.py @@ -1,6 +1,7 @@ -#!/usr/bin/env python -import sys, os +#!/usr/bin/env python3 +import os import re +import sys from io import StringIO from plex import Scanner, Str, Lexicon, Opt, Bol, State, AnyChar, TEXT, IGNORE diff --git a/numpy/linalg/lapack_lite/fortran.py b/numpy/linalg/lapack_lite/fortran.py index 388c88daa..3aaefb92f 100644 --- a/numpy/linalg/lapack_lite/fortran.py +++ b/numpy/linalg/lapack_lite/fortran.py @@ -44,8 +44,6 @@ class LineIterator: line = line.rstrip() return line - next = __next__ - class PushbackIterator: """PushbackIterator(iterable) @@ -71,8 +69,6 @@ class PushbackIterator: def pushback(self, item): self.buffer.append(item) - next = __next__ - def fortranSourceLines(fo): """Return an iterator over statement lines of a Fortran source file. diff --git a/numpy/linalg/lapack_lite/make_lite.py b/numpy/linalg/lapack_lite/make_lite.py index 960f5e2d8..23921acf4 100755 --- a/numpy/linalg/lapack_lite/make_lite.py +++ b/numpy/linalg/lapack_lite/make_lite.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Usage: make_lite.py <wrapped_routines_file> <lapack_dir> @@ -20,12 +20,7 @@ import shutil import fortran import clapack_scrub -PY2 = sys.version_info < (3, 0) - -if PY2: - from distutils.spawn import find_executable as which -else: - from shutil import which +from shutil import which # Arguments to pass to f2c. You'll always want -A for ANSI C prototypes # Others of interest: -a to not make variables static by default diff --git a/numpy/linalg/linalg.py b/numpy/linalg/linalg.py index 7c0b6facf..5ee326f3c 100644 --- a/numpy/linalg/linalg.py +++ b/numpy/linalg/linalg.py @@ -24,7 +24,7 @@ from numpy.core import ( add, multiply, sqrt, fastCopyAndTranspose, sum, isfinite, finfo, errstate, geterrobj, moveaxis, amin, amax, product, abs, atleast_2d, intp, asanyarray, object_, matmul, - swapaxes, divide, count_nonzero, isnan, sign + swapaxes, divide, count_nonzero, isnan, sign, argsort, sort ) from numpy.core.multiarray import normalize_axis_index from numpy.core.overrides import set_module @@ -37,13 +37,6 @@ array_function_dispatch = functools.partial( overrides.array_function_dispatch, module='numpy.linalg') -# For Python2/3 compatibility -_N = b'N' -_V = b'V' -_A = b'A' -_S = b'S' -_L = b'L' - fortran_int = intc @@ -629,8 +622,8 @@ def matrix_power(a, n): try: n = operator.index(n) - except TypeError: - raise TypeError("exponent must be an integer") + except TypeError as e: + raise TypeError("exponent must be an integer") from e # Fall back on dot for object arrays. Object arrays are not supported by # the current implementation of matmul using einsum @@ -1230,8 +1223,10 @@ def eig(a): Hermitian (conjugate symmetric) array. eigvalsh : eigenvalues of a real symmetric or complex Hermitian (conjugate symmetric) array. - scipy.linalg.eig : Similar function in SciPy (but also solves the - generalized eigenvalue problem). + scipy.linalg.eig : Similar function in SciPy that also solves the + generalized eigenvalue problem. + scipy.linalg.schur : Best choice for unitary and other non-Hermitian + normal matrices. Notes ----- @@ -1245,21 +1240,26 @@ def eig(a): the eigenvalues and eigenvectors of general square arrays. The number `w` is an eigenvalue of `a` if there exists a vector - `v` such that ``dot(a,v) = w * v``. Thus, the arrays `a`, `w`, and - `v` satisfy the equations ``dot(a[:,:], v[:,i]) = w[i] * v[:,i]`` + `v` such that ``a @ v = w * v``. Thus, the arrays `a`, `w`, and + `v` satisfy the equations ``a @ v[:,i] = w[i] * v[:,i]`` for :math:`i \\in \\{0,...,M-1\\}`. The array `v` of eigenvectors may not be of maximum rank, that is, some of the columns may be linearly dependent, although round-off error may obscure that fact. If the eigenvalues are all different, then theoretically - the eigenvectors are linearly independent. Likewise, the (complex-valued) - matrix of eigenvectors `v` is unitary if the matrix `a` is normal, i.e., - if ``dot(a, a.H) = dot(a.H, a)``, where `a.H` denotes the conjugate - transpose of `a`. + the eigenvectors are linearly independent and `a` can be diagonalized by + a similarity transformation using `v`, i.e, ``inv(v) @ a @ v`` is diagonal. + + For non-Hermitian normal matrices the SciPy function `scipy.linalg.schur` + is preferred because the matrix `v` is guaranteed to be unitary, which is + not the case when using `eig`. The Schur factorization produces an + upper triangular matrix rather than a diagonal matrix, but for normal + matrices only the diagonal of the upper triangular matrix is needed, the + rest is roundoff error. Finally, it is emphasized that `v` consists of the *right* (as in right-hand side) eigenvectors of `a`. A vector `y` satisfying - ``dot(y.T, a) = z * y.T`` for some number `z` is called a *left* + ``y.T @ a = z * y.T`` for some number `z` is called a *left* eigenvector of `a`, and, in general, the left and right eigenvectors of a matrix are not necessarily the (perhaps conjugate) transposes of each other. @@ -1615,24 +1615,29 @@ def svd(a, full_matrices=True, compute_uv=True, hermitian=False): True """ + import numpy as _nx a, wrap = _makearray(a) if hermitian: - # note: lapack returns eigenvalues in reverse order to our contract. - # reversing is cheap by design in numpy, so we do so to be consistent + # note: lapack svd returns eigenvalues with s ** 2 sorted descending, + # but eig returns s sorted ascending, so we re-order the eigenvalues + # and related arrays to have the correct order if compute_uv: s, u = eigh(a) - s = s[..., ::-1] - u = u[..., ::-1] - # singular values are unsigned, move the sign into v - vt = transpose(u * sign(s)[..., None, :]).conjugate() + sgn = sign(s) s = abs(s) + sidx = argsort(s)[..., ::-1] + sgn = _nx.take_along_axis(sgn, sidx, axis=-1) + s = _nx.take_along_axis(s, sidx, axis=-1) + u = _nx.take_along_axis(u, sidx[..., None, :], axis=-1) + # singular values are unsigned, move the sign into v + vt = transpose(u * sgn[..., None, :]).conjugate() return wrap(u), s, wrap(vt) else: s = eigvalsh(a) s = s[..., ::-1] s = abs(s) - return s + return sort(s)[..., ::-1] _assert_stacked_2d(a) t, result_t = _commonType(a) @@ -2167,13 +2172,13 @@ def lstsq(a, b, rcond="warn"): r""" Return the least-squares solution to a linear matrix equation. - Solves the equation :math:`a x = b` by computing a vector `x` that - minimizes the squared Euclidean 2-norm :math:`\| b - a x \|^2_2`. - The equation may be under-, well-, or over-determined (i.e., the - number of linearly independent rows of `a` can be less than, equal - to, or greater than its number of linearly independent columns). + Computes the vector x that approximatively solves the equation + ``a @ x = b``. The equation may be under-, well-, or over-determined + (i.e., the number of linearly independent rows of `a` can be less than, + equal to, or greater than its number of linearly independent columns). If `a` is square and of full rank, then `x` (but for round-off error) - is the "exact" solution of the equation. + is the "exact" solution of the equation. Else, `x` minimizes the + Euclidean 2-norm :math:`|| b - a x ||`. Parameters ---------- @@ -2429,6 +2434,9 @@ def norm(x, ord=None, axis=None, keepdims=False): The nuclear norm is the sum of the singular values. + Both the Frobenius and nuclear norm orders are only defined for + matrices and raise a ValueError when ``x.ndim != 2``. + References ---------- .. [1] G. H. Golub and C. F. Van Loan, *Matrix Computations*, @@ -2532,8 +2540,8 @@ def norm(x, ord=None, axis=None, keepdims=False): elif not isinstance(axis, tuple): try: axis = int(axis) - except Exception: - raise TypeError("'axis' must be None, an integer or a tuple of integers") + except Exception as e: + raise TypeError("'axis' must be None, an integer or a tuple of integers") from e axis = (axis,) if len(axis) == 1: @@ -2551,11 +2559,11 @@ def norm(x, ord=None, axis=None, keepdims=False): # special case for speedup s = (x.conj() * x).real return sqrt(add.reduce(s, axis=axis, keepdims=keepdims)) + # None of the str-type keywords for ord ('fro', 'nuc') + # are valid for vectors + elif isinstance(ord, str): + raise ValueError(f"Invalid norm order '{ord}' for vectors") else: - try: - ord + 1 - except TypeError: - raise ValueError("Invalid norm order for vectors.") absx = abs(x) absx **= ord ret = add.reduce(absx, axis=axis, keepdims=keepdims) @@ -2605,12 +2613,13 @@ def norm(x, ord=None, axis=None, keepdims=False): # multi_dot -def _multidot_dispatcher(arrays): - return arrays +def _multidot_dispatcher(arrays, *, out=None): + yield from arrays + yield out @array_function_dispatch(_multidot_dispatcher) -def multi_dot(arrays): +def multi_dot(arrays, *, out=None): """ Compute the dot product of two or more arrays in a single function call, while automatically selecting the fastest evaluation order. @@ -2634,6 +2643,15 @@ def multi_dot(arrays): If the first argument is 1-D it is treated as row vector. If the last argument is 1-D it is treated as column vector. The other arguments must be 2-D. + out : ndarray, optional + Output argument. This must have the exact kind that would be returned + if it was not used. In particular, it must have the right type, must be + C-contiguous, and its dtype must be the dtype that would be returned + for `dot(a, b)`. This is a performance feature. Therefore, if these + conditions are not met, an exception is raised, instead of attempting + to be flexible. + + .. versionadded:: 1.19.0 Returns ------- @@ -2691,7 +2709,7 @@ def multi_dot(arrays): if n < 2: raise ValueError("Expecting at least two arrays.") elif n == 2: - return dot(arrays[0], arrays[1]) + return dot(arrays[0], arrays[1], out=out) arrays = [asanyarray(a) for a in arrays] @@ -2707,10 +2725,10 @@ def multi_dot(arrays): # _multi_dot_three is much faster than _multi_dot_matrix_chain_order if n == 3: - result = _multi_dot_three(arrays[0], arrays[1], arrays[2]) + result = _multi_dot_three(arrays[0], arrays[1], arrays[2], out=out) else: order = _multi_dot_matrix_chain_order(arrays) - result = _multi_dot(arrays, order, 0, n - 1) + result = _multi_dot(arrays, order, 0, n - 1, out=out) # return proper shape if ndim_first == 1 and ndim_last == 1: @@ -2721,7 +2739,7 @@ def multi_dot(arrays): return result -def _multi_dot_three(A, B, C): +def _multi_dot_three(A, B, C, out=None): """ Find the best order for three arrays and do the multiplication. @@ -2737,9 +2755,9 @@ def _multi_dot_three(A, B, C): cost2 = a1b0 * c1 * (a0 + b1c0) if cost1 < cost2: - return dot(dot(A, B), C) + return dot(dot(A, B), C, out=out) else: - return dot(A, dot(B, C)) + return dot(A, dot(B, C), out=out) def _multi_dot_matrix_chain_order(arrays, return_costs=False): @@ -2783,10 +2801,14 @@ def _multi_dot_matrix_chain_order(arrays, return_costs=False): return (s, m) if return_costs else s -def _multi_dot(arrays, order, i, j): +def _multi_dot(arrays, order, i, j, out=None): """Actually do the multiplication with the given order.""" if i == j: + # the initial call with non-None out should never get here + assert out is None + return arrays[i] else: return dot(_multi_dot(arrays, order, i, order[i, j]), - _multi_dot(arrays, order, order[i, j] + 1, j)) + _multi_dot(arrays, order, order[i, j] + 1, j), + out=out) diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index 59c71d196..3f3bf9f70 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -680,6 +680,14 @@ class SVDHermitianCases(HermitianTestCase, HermitianGeneralizedTestCase): assert_allclose(a, dot_generalized(np.asarray(u) * np.asarray(s)[..., None, :], np.asarray(vt)), rtol=get_rtol(u.dtype)) + def hermitian(mat): + axes = list(range(mat.ndim)) + axes[-1], axes[-2] = axes[-2], axes[-1] + return np.conj(np.transpose(mat, axes=axes)) + + assert_almost_equal(np.matmul(u, hermitian(u)), np.broadcast_to(np.eye(u.shape[-1]), u.shape)) + assert_almost_equal(np.matmul(vt, hermitian(vt)), np.broadcast_to(np.eye(vt.shape[-1]), vt.shape)) + assert_equal(np.sort(s)[..., ::-1], s) assert_(consistent_subclass(u, a)) assert_(consistent_subclass(vt, a)) @@ -1472,11 +1480,12 @@ class _TestNorm2D(_TestNormBase): # Using `axis=<integer>` or passing in a 1-D array implies vector # norms are being computed, so also using `ord='fro'` - # or `ord='nuc'` raises a ValueError. + # or `ord='nuc'` or any other string raises a ValueError. assert_raises(ValueError, norm, A, 'fro', 0) assert_raises(ValueError, norm, A, 'nuc', 0) assert_raises(ValueError, norm, [3, 4], 'fro', None) assert_raises(ValueError, norm, [3, 4], 'nuc', None) + assert_raises(ValueError, norm, [3, 4], 'test', None) # Similarly, norm should raise an exception when ord is any finite # number other than 1, 2, -1 or -2 when computing matrix norms. @@ -1827,6 +1836,7 @@ def test_xerbla_override(): pytest.skip('Numpy xerbla not linked in.') +@pytest.mark.slow def test_sdot_bug_8577(): # Regression test that loading certain other libraries does not # result to wrong results in float32 linear algebra. @@ -1920,6 +1930,41 @@ class TestMultiDot: # the result should be a scalar assert_equal(multi_dot([A1d, B, C, D1d]).shape, ()) + def test_three_arguments_and_out(self): + # multi_dot with three arguments uses a fast hand coded algorithm to + # determine the optimal order. Therefore test it separately. + A = np.random.random((6, 2)) + B = np.random.random((2, 6)) + C = np.random.random((6, 2)) + + out = np.zeros((6, 2)) + ret = multi_dot([A, B, C], out=out) + assert out is ret + assert_almost_equal(out, A.dot(B).dot(C)) + assert_almost_equal(out, np.dot(A, np.dot(B, C))) + + def test_two_arguments_and_out(self): + # separate code path with two arguments + A = np.random.random((6, 2)) + B = np.random.random((2, 6)) + out = np.zeros((6, 6)) + ret = multi_dot([A, B], out=out) + assert out is ret + assert_almost_equal(out, A.dot(B)) + assert_almost_equal(out, np.dot(A, B)) + + def test_dynamic_programing_optimization_and_out(self): + # multi_dot with four or more arguments uses the dynamic programing + # optimization and therefore deserve a separate test + A = np.random.random((6, 2)) + B = np.random.random((2, 6)) + C = np.random.random((6, 2)) + D = np.random.random((2, 1)) + out = np.zeros((6, 1)) + ret = multi_dot([A, B, C, D], out=out) + assert out is ret + assert_almost_equal(out, A.dot(B).dot(C).dot(D)) + def test_dynamic_programming_logic(self): # Test for the dynamic programming part # This test is directly taken from Cormen page 376. diff --git a/numpy/linalg/tests/test_regression.py b/numpy/linalg/tests/test_regression.py index 1ae207b49..7ed932bc9 100644 --- a/numpy/linalg/tests/test_regression.py +++ b/numpy/linalg/tests/test_regression.py @@ -57,8 +57,8 @@ class TestRegression: assert_array_almost_equal(b, np.zeros((2, 2))) def test_norm_vector_badarg(self): - # Regression for #786: Froebenius norm for vectors raises - # TypeError. + # Regression for #786: Frobenius norm for vectors raises + # ValueError. assert_raises(ValueError, linalg.norm, array([1., 2., 3.]), 'fro') def test_lapack_endian(self): diff --git a/numpy/ma/bench.py b/numpy/ma/bench.py index a1363d4d9..83cc6aea7 100644 --- a/numpy/ma/bench.py +++ b/numpy/ma/bench.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import timeit diff --git a/numpy/ma/core.py b/numpy/ma/core.py index fcbd1d8d0..a7214f9bf 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -21,7 +21,6 @@ Released for unlimited redistribution. """ # pylint: disable-msg=E1002 import builtins -import sys import operator import warnings import textwrap @@ -35,7 +34,7 @@ from numpy import ndarray, amax, amin, iscomplexobj, bool_, _NoValue from numpy import array as narray from numpy.lib.function_base import angle from numpy.compat import ( - getargspec, formatargspec, long, basestring, unicode, bytes + getargspec, formatargspec, long, unicode, bytes ) from numpy import expand_dims from numpy.core.numeric import normalize_axis_tuple @@ -286,11 +285,10 @@ def _extremum_fill_value(obj, extremum, extremum_name): def _scalar_fill_value(dtype): try: return extremum[dtype] - except KeyError: + except KeyError as e: raise TypeError( - "Unsuitable type {} for calculating {}." - .format(dtype, extremum_name) - ) + f"Unsuitable type {dtype} for calculating {extremum_name}." + ) from None dtype = _get_dtype_of(obj) return _recursive_fill_value(dtype, _scalar_fill_value) @@ -456,7 +454,7 @@ def _check_fill_value(fill_value, ndtype): fill_value = np.array(_recursive_set_fill_value(fill_value, ndtype), dtype=ndtype) else: - if isinstance(fill_value, basestring) and (ndtype.char not in 'OSVU'): + if isinstance(fill_value, str) and (ndtype.char not in 'OSVU'): # Note this check doesn't work if fill_value is not a scalar err_msg = "Cannot set fill value of string with array of dtype %s" raise TypeError(err_msg % ndtype) @@ -781,9 +779,9 @@ def fix_invalid(a, mask=nomask, copy=True, fill_value=None): return a def is_string_or_list_of_strings(val): - return (isinstance(val, basestring) or + return (isinstance(val, str) or (isinstance(val, list) and val and - builtins.all(isinstance(s, basestring) for s in val))) + builtins.all(isinstance(s, str) for s in val))) ############################################################################### # Ufuncs # @@ -897,7 +895,7 @@ class _MaskedUFunc: self.__name__ = ufunc.__name__ def __str__(self): - return "Masked version of {}".format(self.f) + return f"Masked version of {self.f}" class _MaskedUnaryOperation(_MaskedUFunc): @@ -1803,8 +1801,7 @@ def flatten_mask(mask): try: for element in sequence: if hasattr(element, '__iter__'): - for f in _flatsequence(element): - yield f + yield from _flatsequence(element) else: yield element except TypeError: @@ -2531,8 +2528,7 @@ def flatten_structured_array(a): """ for elm in iter(iterable): if hasattr(elm, '__iter__'): - for f in flatten_sequence(elm): - yield f + yield from flatten_sequence(elm) else: yield elm @@ -2705,8 +2701,6 @@ class MaskedIterator: return masked return d - next = __next__ - class MaskedArray(ndarray): """ @@ -2761,6 +2755,52 @@ class MaskedArray(ndarray): in any order (either C-, Fortran-contiguous, or even discontiguous), unless a copy is required, in which case it will be C-contiguous. + Examples + -------- + + The ``mask`` can be initialized with an array of boolean values + with the same shape as ``data``. + + >>> data = np.arange(6).reshape((2, 3)) + >>> np.ma.MaskedArray(data, mask=[[False, True, False], + ... [False, False, True]]) + masked_array( + data=[[0, --, 2], + [3, 4, --]], + mask=[[False, True, False], + [False, False, True]], + fill_value=999999) + + Alternatively, the ``mask`` can be initialized to homogeneous boolean + array with the same shape as ``data`` by passing in a scalar + boolean value: + + >>> np.ma.MaskedArray(data, mask=False) + masked_array( + data=[[0, 1, 2], + [3, 4, 5]], + mask=[[False, False, False], + [False, False, False]], + fill_value=999999) + + >>> np.ma.MaskedArray(data, mask=True) + masked_array( + data=[[--, --, --], + [--, --, --]], + mask=[[ True, True, True], + [ True, True, True]], + fill_value=999999, + dtype=int64) + + .. note:: + The recommended practice for initializing ``mask`` with a scalar + boolean value is to use ``True``/``False`` rather than + ``np.True_``/``np.False_``. The reason is :attr:`nomask` + is represented internally as ``np.False_``. + + >>> np.False_ is np.ma.nomask + True + """ __array_priority__ = 15 @@ -3259,6 +3299,10 @@ class MaskedArray(ndarray): if self._fill_value is not None: dout._fill_value = self._fill_value[indx] + # Something like gh-15895 has happened if this check fails. + # _fill_value should always be an ndarray. + if not isinstance(dout._fill_value, np.ndarray): + raise RuntimeError('Internal NumPy error.') # If we're indexing a multidimensional field in a # structured array (such as dtype("(2,)i2,(2,)i1")), # dimensionality goes up (M[field].ndim == M.ndim + @@ -3266,19 +3310,21 @@ class MaskedArray(ndarray): # M[field] but problematic for M[field].fill_value # which should have shape () to avoid breaking several # methods. There is no great way out, so set to - # first element. See issue #6723. + # first element. See issue #6723. if dout._fill_value.ndim > 0: if not (dout._fill_value == dout._fill_value.flat[0]).all(): warnings.warn( "Upon accessing multidimensional field " - "{indx:s}, need to keep dimensionality " + f"{indx!s}, need to keep dimensionality " "of fill_value at 0. Discarding " "heterogeneous fill_value and setting " - "all to {fv!s}.".format(indx=indx, - fv=dout._fill_value[0]), + f"all to {dout._fill_value[0]!s}.", stacklevel=2) - dout._fill_value = dout._fill_value.flat[0] + # Need to use `.flat[0:1].squeeze(...)` instead of just + # `.flat[0]` to ensure the result is a 0d array and not + # a scalar. + dout._fill_value = dout._fill_value.flat[0:1].squeeze(axis=0) dout._isfield = True # Update the mask if needed if mout is not nomask: @@ -3300,7 +3346,7 @@ class MaskedArray(ndarray): raise MaskError('Cannot alter the masked element.') _data = self._data _mask = self._mask - if isinstance(indx, basestring): + if isinstance(indx, str): _data[indx] = value if _mask is nomask: self._mask = _mask = make_mask_none(self.shape, self.dtype) @@ -3891,10 +3937,6 @@ class MaskedArray(ndarray): def __str__(self): return str(self._insert_masked_print()) - if sys.version_info.major < 3: - def __unicode__(self): - return unicode(self._insert_masked_print()) - def __repr__(self): """ Literal string representation. @@ -3924,7 +3966,7 @@ class MaskedArray(ndarray): ) return _legacy_print_templates[key] % parameters - prefix = 'masked_{}('.format(name) + prefix = f"masked_{name}(" dtype_needed = ( not np.core.arrayprint.dtype_is_implied(self.dtype) or @@ -4338,17 +4380,6 @@ class MaskedArray(ndarray): raise MaskError('Cannot convert masked element to a Python int.') return int(self.item()) - def __long__(self): - """ - Convert to long. - """ - if self.size > 1: - raise TypeError("Only length-1 arrays can be converted " - "to Python scalars") - elif self._mask: - raise MaskError('Cannot convert masked element to a Python long.') - return long(self.item()) - @property def imag(self): """ @@ -4744,7 +4775,7 @@ class MaskedArray(ndarray): >>> x = np.ma.array([1, 2, 3]) >>> x.ids() - (166691080, 3083169284L) # may vary + (166691080, 3083169284) # may vary """ if self._mask is nomask: @@ -5867,13 +5898,13 @@ class MaskedArray(ndarray): def partition(self, *args, **kwargs): warnings.warn("Warning: 'partition' will ignore the 'mask' " - "of the {}.".format(self.__class__.__name__), + f"of the {self.__class__.__name__}.", stacklevel=2) return super(MaskedArray, self).partition(*args, **kwargs) def argpartition(self, *args, **kwargs): warnings.warn("Warning: 'argpartition' will ignore the 'mask' " - "of the {}.".format(self.__class__.__name__), + f"of the {self.__class__.__name__}.", stacklevel=2) return super(MaskedArray, self).argpartition(*args, **kwargs) @@ -5965,10 +5996,17 @@ class MaskedArray(ndarray): return result.tolist() def tostring(self, fill_value=None, order='C'): + r""" + A compatibility alias for `tobytes`, with exactly the same behavior. + + Despite its name, it returns `bytes` not `str`\ s. + + .. deprecated:: 1.19.0 """ - This function is a compatibility alias for tobytes. Despite its name it - returns bytes not strings. - """ + # 2020-03-30, Numpy 1.19.0 + warnings.warn( + "tostring() is deprecated. Use tobytes() instead.", + DeprecationWarning, stacklevel=2) return self.tobytes(fill_value, order=order) @@ -6219,8 +6257,7 @@ class mvoid(MaskedArray): "Defines an iterator for mvoid" (_data, _mask) = (self._data, self._mask) if _mask is nomask: - for d in _data: - yield d + yield from _data else: for (d, m) in zip(_data, _mask): if m: @@ -6397,6 +6434,21 @@ class MaskedConstant(MaskedArray): # it's a subclass, or something is wrong, make it obvious return object.__repr__(self) + def __format__(self, format_spec): + # Replace ndarray.__format__ with the default, which supports no format characters. + # Supporting format characters is unwise here, because we do not know what type + # the user was expecting - better to not guess. + try: + return object.__format__(self, format_spec) + except TypeError: + # 2020-03-23, NumPy 1.19.0 + warnings.warn( + "Format strings passed to MaskedConstant are ignored, but in future may " + "error or produce different behavior", + FutureWarning, stacklevel=2 + ) + return object.__format__(self, "") + def __reduce__(self): """Override of MaskedArray's __reduce__. """ @@ -6433,7 +6485,7 @@ class MaskedConstant(MaskedArray): return super(MaskedConstant, self).__setattr__(attr, value) elif self is self.__singleton: raise AttributeError( - "attributes of {!r} are not writeable".format(self)) + f"attributes of {self!r} are not writeable") else: # duplicate instance - we can end up here from __array_finalize__, # where we set the __class__ attribute @@ -6538,8 +6590,8 @@ class _extrema_operation(_MaskedUFunc): if b is None: # 2016-04-13, 1.13.0 warnings.warn( - "Single-argument form of np.ma.{0} is deprecated. Use " - "np.ma.{0}.reduce instead.".format(self.__name__), + f"Single-argument form of np.ma.{self.__name__} is deprecated. Use " + f"np.ma.{self.__name__}.reduce instead.", DeprecationWarning, stacklevel=2) return self.reduce(a) return where(self.compare(a, b), a, b) @@ -6552,11 +6604,9 @@ class _extrema_operation(_MaskedUFunc): if axis is np._NoValue and target.ndim > 1: # 2017-05-06, Numpy 1.13.0: warn on axis default warnings.warn( - "In the future the default for ma.{0}.reduce will be axis=0, " - "not the current None, to match np.{0}.reduce. " - "Explicitly pass 0 or None to silence this warning.".format( - self.__name__ - ), + f"In the future the default for ma.{self.__name__}.reduce will be axis=0, " + f"not the current None, to match np.{self.__name__}.reduce. " + "Explicitly pass 0 or None to silence this warning.", MaskedArrayFutureWarning, stacklevel=2) axis = None @@ -6783,7 +6833,18 @@ def argsort(a, axis=np._NoValue, kind=None, order=None, endwith=True, fill_value argsort.__doc__ = MaskedArray.argsort.__doc__ def sort(a, axis=-1, kind=None, order=None, endwith=True, fill_value=None): - "Function version of the eponymous method." + """ + Return a sorted copy of the masked array. + + Equivalent to creating a copy of the array + and applying the MaskedArray ``sort()`` method. + + Refer to ``MaskedArray.sort`` for the full documentation + + See Also + -------- + MaskedArray.sort : equivalent method + """ a = np.array(a, copy=True, subok=True) if axis is None: a = a.flatten() @@ -6795,7 +6856,6 @@ def sort(a, axis=-1, kind=None, order=None, endwith=True, fill_value=None): else: a.sort(axis=axis, kind=kind, order=order) return a -sort.__doc__ = MaskedArray.sort.__doc__ def compressed(x): @@ -7895,10 +7955,8 @@ def asanyarray(a, dtype=None): def _pickle_warn(method): # NumPy 1.15.0, 2017-12-10 warnings.warn( - "np.ma.{method} is deprecated, use pickle.{method} instead" - .format(method=method), - DeprecationWarning, - stacklevel=3) + f"np.ma.{method} is deprecated, use pickle.{method} instead", + DeprecationWarning, stacklevel=3) def fromfile(file, dtype=float, count=-1, sep=''): diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py index 31648fb2e..f86ebf551 100644 --- a/numpy/ma/extras.py +++ b/numpy/ma/extras.py @@ -811,7 +811,7 @@ def compress_nd(x, axis=None): ---------- x : array_like, MaskedArray The array to operate on. If not a MaskedArray instance (or if no array - elements are masked, `x` is interpreted as a MaskedArray with `mask` + elements are masked), `x` is interpreted as a MaskedArray with `mask` set to `nomask`. axis : tuple of ints or int, optional Which dimensions to suppress slices from can be configured with this diff --git a/numpy/ma/mrecords.py b/numpy/ma/mrecords.py index 4ff7866ab..cd93a9a14 100644 --- a/numpy/ma/mrecords.py +++ b/numpy/ma/mrecords.py @@ -13,11 +13,9 @@ and the masking of individual fields. # first place, and then rename the invalid fields with a trailing # underscore. Maybe we could just overload the parser function ? -import sys import warnings import numpy as np -from numpy.compat import basestring from numpy import ( bool_, dtype, ndarray, recarray, array as narray ) @@ -303,7 +301,7 @@ class MaskedRecords(MaskedArray): _mask = ndarray.__getattribute__(self, '_mask') _data = ndarray.view(self, _localdict['_baseclass']) # We want a field - if isinstance(indx, basestring): + if isinstance(indx, str): # Make sure _sharedmask is True to propagate back to _fieldmask # Don't use _set_mask, there are some copies being made that # break propagation Don't force the mask to nomask, that wreaks @@ -330,7 +328,7 @@ class MaskedRecords(MaskedArray): """ MaskedArray.__setitem__(self, indx, value) - if isinstance(indx, basestring): + if isinstance(indx, str): self._mask[indx] = ma.getmaskarray(value) def __str__(self): diff --git a/numpy/ma/setup.py b/numpy/ma/setup.py index af1e419b4..144a961c2 100644 --- a/numpy/ma/setup.py +++ b/numpy/ma/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('ma', parent_package, top_path) diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index bc8423188..98fc7dd97 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -447,6 +447,21 @@ class TestMaskedArray: assert_equal(copied.mask, [0, 0, 0]) assert_equal(a.mask, [0, 1, 0]) + def test_format(self): + a = array([0, 1, 2], mask=[False, True, False]) + assert_equal(format(a), "[0 -- 2]") + assert_equal(format(masked), "--") + assert_equal(format(masked, ""), "--") + + # Postponed from PR #15410, perhaps address in the future. + # assert_equal(format(masked, " >5"), " --") + # assert_equal(format(masked, " <5"), "-- ") + + # Expect a FutureWarning for using format_spec with MaskedElement + with assert_warns(FutureWarning): + with_format_string = format(masked, " >5") + assert_equal(with_format_string, "--") + def test_str_repr(self): a = array([0, 1, 2], mask=[False, True, False]) assert_equal(str(a), '[0 -- 2]') @@ -1556,7 +1571,11 @@ class TestMaskedArrayArithmetic: assert_equal(test.mask, [True, True]) assert_(test.fill_value == True) - # test = (a[0] == b) # doesn't work in Python2 + test = (a[0] == b) + assert_equal(test.data, [False, False]) + assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + test = (b == a[0]) assert_equal(test.data, [False, False]) assert_equal(test.mask, [True, False]) @@ -1584,7 +1603,11 @@ class TestMaskedArrayArithmetic: assert_equal(test.mask, [True, True]) assert_(test.fill_value == True) - # test = (a[0] != b) # doesn't work in Python2 + test = (a[0] != b) + assert_equal(test.data, [True, True]) + assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + test = (b != a[0]) assert_equal(test.data, [True, True]) assert_equal(test.mask, [True, False]) @@ -1613,7 +1636,11 @@ class TestMaskedArrayArithmetic: assert_equal(test.mask, [True, True]) assert_(test.fill_value == True) - # test = (a[0] == b) # doesn't work in Python2 + test = (a[0] == b) + assert_equal(test.data, [False, False]) + assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + test = (b == a[0]) assert_equal(test.data, [False, False]) assert_equal(test.mask, [True, False]) @@ -1642,7 +1669,11 @@ class TestMaskedArrayArithmetic: assert_equal(test.mask, [True, True]) assert_(test.fill_value == True) - # test = (a[0] != b) # doesn't work in Python2 + test = (a[0] != b) + assert_equal(test.data, [True, True]) + assert_equal(test.mask, [True, False]) + assert_(test.fill_value == True) + test = (b != a[0]) assert_equal(test.data, [True, True]) assert_equal(test.mask, [True, False]) diff --git a/numpy/ma/tests/test_extras.py b/numpy/ma/tests/test_extras.py index 1c8610625..d237829cb 100644 --- a/numpy/ma/tests/test_extras.py +++ b/numpy/ma/tests/test_extras.py @@ -64,6 +64,28 @@ class TestGeneric: control = array([[(1, (1, 1))]], mask=[[(1, (1, 1))]], dtype=dt) assert_equal(test, control) + def test_masked_all_with_object_nested(self): + # Test masked_all works with nested array with dtype of an 'object' + # refers to issue #15895 + my_dtype = np.dtype([('b', ([('c', object)], (1,)))]) + masked_arr = np.ma.masked_all((1,), my_dtype) + + assert_equal(type(masked_arr['b']), np.ma.core.MaskedArray) + assert_equal(type(masked_arr['b']['c']), np.ma.core.MaskedArray) + assert_equal(len(masked_arr['b']['c']), 1) + assert_equal(masked_arr['b']['c'].shape, (1, 1)) + assert_equal(masked_arr['b']['c']._fill_value.shape, ()) + + def test_masked_all_with_object(self): + # same as above except that the array is not nested + my_dtype = np.dtype([('b', (object, (1,)))]) + masked_arr = np.ma.masked_all((1,), my_dtype) + + assert_equal(type(masked_arr['b']), np.ma.core.MaskedArray) + assert_equal(len(masked_arr['b']), 1) + assert_equal(masked_arr['b'].shape, (1, 1)) + assert_equal(masked_arr['b']._fill_value.shape, ()) + def test_masked_all_like(self): # Tests masked_all # Standard dtype diff --git a/numpy/ma/tests/test_regression.py b/numpy/ma/tests/test_regression.py index 9f3368489..7e76eb054 100644 --- a/numpy/ma/tests/test_regression.py +++ b/numpy/ma/tests/test_regression.py @@ -86,6 +86,6 @@ class TestRegression: ma = np.ma.MaskedArray([(1, 1.), (2, 2.), (3, 3.)], dtype='i4,f4') assert_array_equal(ma[[]], ma[:0]) - def test_masked_array_tostring_fortran(self): + def test_masked_array_tobytes_fortran(self): ma = np.ma.arange(4).reshape((2,2)) - assert_array_equal(ma.tostring(order='F'), ma.T.tostring()) + assert_array_equal(ma.tobytes(order='F'), ma.T.tobytes()) diff --git a/numpy/ma/tests/test_subclassing.py b/numpy/ma/tests/test_subclassing.py index 02aeebd17..caa746740 100644 --- a/numpy/ma/tests/test_subclassing.py +++ b/numpy/ma/tests/test_subclassing.py @@ -105,8 +105,6 @@ class CSAIterator: def __next__(self): return next(self._dataiter).__array__().view(type(self._original)) - next = __next__ - class ComplicatedSubArray(SubArray): diff --git a/numpy/matlib.py b/numpy/matlib.py index f3eb8eb4b..bd6b63289 100644 --- a/numpy/matlib.py +++ b/numpy/matlib.py @@ -1,7 +1,20 @@ +import warnings + +# 2018-05-29, PendingDeprecationWarning added to matrix.__new__ +# 2020-01-23, numpy 1.19.0 PendingDeprecatonWarning +warnings.warn("Importing from numpy.matlib is deprecated since 1.19.0. " + "The matrix subclass is not the recommended way to represent " + "matrices or deal with linear algebra (see " + "https://docs.scipy.org/doc/numpy/user/numpy-for-matlab-users.html). " + "Please adjust your code to use regular ndarray. ", + PendingDeprecationWarning, stacklevel=2) + import numpy as np from numpy.matrixlib.defmatrix import matrix, asmatrix -# need * as we're copying the numpy namespace (FIXME: this makes little sense) -from numpy import * +# Matlib.py contains all functions in the numpy namespace with a few +# replacements. See doc/source/reference/routines.matlib.rst for details. +# Need * as we're copying the numpy namespace. +from numpy import * # noqa: F403 __version__ = np.__version__ diff --git a/numpy/matrixlib/defmatrix.py b/numpy/matrixlib/defmatrix.py index 12ac74cb2..d1a1211aa 100644 --- a/numpy/matrixlib/defmatrix.py +++ b/numpy/matrixlib/defmatrix.py @@ -802,7 +802,7 @@ class matrix(N.ndarray): ------- ret : matrix object If `self` is non-singular, `ret` is such that ``ret * self`` == - ``self * ret`` == ``np.matrix(np.eye(self[0,:].size)`` all return + ``self * ret`` == ``np.matrix(np.eye(self[0,:].size))`` all return ``True``. Raises diff --git a/numpy/matrixlib/setup.py b/numpy/matrixlib/setup.py index c4eee4be4..529d2a2eb 100644 --- a/numpy/matrixlib/setup.py +++ b/numpy/matrixlib/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('matrixlib', parent_package, top_path) diff --git a/numpy/matrixlib/tests/test_defmatrix.py b/numpy/matrixlib/tests/test_defmatrix.py index a8070898f..4cb5f3a37 100644 --- a/numpy/matrixlib/tests/test_defmatrix.py +++ b/numpy/matrixlib/tests/test_defmatrix.py @@ -1,9 +1,4 @@ -try: - # Accessing collections abstract classes from collections - # has been deprecated since Python 3.3 - import collections.abc as collections_abc -except ImportError: - import collections as collections_abc +import collections.abc import numpy as np from numpy import matrix, asmatrix, bmat @@ -297,7 +292,7 @@ class TestMatrixReturn: if attrib.startswith('_') or attrib in excluded_methods: continue f = getattr(a, attrib) - if isinstance(f, collections_abc.Callable): + if isinstance(f, collections.abc.Callable): # reset contents of a a.astype('f8') a.fill(1.0) diff --git a/numpy/matrixlib/tests/test_interaction.py b/numpy/matrixlib/tests/test_interaction.py index 608416ed7..5154bd621 100644 --- a/numpy/matrixlib/tests/test_interaction.py +++ b/numpy/matrixlib/tests/test_interaction.py @@ -324,24 +324,17 @@ class TestConcatenatorMatrix: def test_array_equal_error_message_matrix(): # 2018-04-29: moved here from testing.tests.test_utils. - try: + with pytest.raises(AssertionError) as exc_info: assert_equal(np.array([1, 2]), np.matrix([1, 2])) - except AssertionError as e: - msg = str(e) - msg2 = msg.replace("shapes (2L,), (1L, 2L)", "shapes (2,), (1, 2)") - msg_reference = textwrap.dedent("""\ - - Arrays are not equal - - (shapes (2,), (1, 2) mismatch) - x: array([1, 2]) - y: matrix([[1, 2]])""") - try: - assert_equal(msg, msg_reference) - except AssertionError: - assert_equal(msg2, msg_reference) - else: - raise AssertionError("Did not raise") + msg = str(exc_info.value) + msg_reference = textwrap.dedent("""\ + + Arrays are not equal + + (shapes (2,), (1, 2) mismatch) + x: array([1, 2]) + y: matrix([[1, 2]])""") + assert_equal(msg, msg_reference) def test_array_almost_equal_matrix(): diff --git a/numpy/polynomial/_polybase.py b/numpy/polynomial/_polybase.py index 28bd50ec6..53efbb90f 100644 --- a/numpy/polynomial/_polybase.py +++ b/numpy/polynomial/_polybase.py @@ -277,18 +277,16 @@ class ABCPolyBase(abc.ABC): self.window = window def __repr__(self): - format = "%s(%s, domain=%s, window=%s)" coef = repr(self.coef)[6:-1] domain = repr(self.domain)[6:-1] window = repr(self.window)[6:-1] name = self.__class__.__name__ - return format % (name, coef, domain, window) + return f"{name}({coef}, domain={domain}, window={window})" def __str__(self): - format = "%s(%s)" coef = str(self.coef) name = self.nickname - return format % (name, coef) + return f"{name}({coef})" @classmethod def _repr_latex_term(cls, i, arg_str, needs_parens): @@ -297,9 +295,7 @@ class ABCPolyBase(abc.ABC): "Subclasses must define either a basis name, or override " "_repr_latex_term(i, arg_str, needs_parens)") # since we always add parens, we don't care if the expression needs them - return "{{{basis}}}_{{{i}}}({arg_str})".format( - basis=cls.basis_name, i=i, arg_str=arg_str - ) + return f"{{{cls.basis_name}}}_{{{i}}}({arg_str})" @staticmethod def _repr_latex_scalar(x): @@ -314,19 +310,15 @@ class ABCPolyBase(abc.ABC): term = 'x' needs_parens = False elif scale == 1: - term = '{} + x'.format( - self._repr_latex_scalar(off) - ) + term = f"{self._repr_latex_scalar(off)} + x" needs_parens = True elif off == 0: - term = '{}x'.format( - self._repr_latex_scalar(scale) - ) + term = f"{self._repr_latex_scalar(scale)}x" needs_parens = True else: - term = '{} + {}x'.format( - self._repr_latex_scalar(off), - self._repr_latex_scalar(scale) + term = ( + f"{self._repr_latex_scalar(off)} + " + f"{self._repr_latex_scalar(scale)}x" ) needs_parens = True @@ -336,20 +328,20 @@ class ABCPolyBase(abc.ABC): for i, c in enumerate(self.coef): # prevent duplication of + and - signs if i == 0: - coef_str = '{}'.format(self._repr_latex_scalar(c)) + coef_str = f"{self._repr_latex_scalar(c)}" elif not isinstance(c, numbers.Real): - coef_str = ' + ({})'.format(self._repr_latex_scalar(c)) + coef_str = f" + ({self._repr_latex_scalar(c)})" elif not np.signbit(c): - coef_str = ' + {}'.format(self._repr_latex_scalar(c)) + coef_str = f" + {self._repr_latex_scalar(c)}" else: - coef_str = ' - {}'.format(self._repr_latex_scalar(-c)) + coef_str = f" - {self._repr_latex_scalar(-c)}" # produce the string for the term term_str = self._repr_latex_term(i, term, needs_parens) if term_str == '1': part = coef_str else: - part = r'{}\,{}'.format(coef_str, term_str) + part = rf"{coef_str}\,{term_str}" if c == 0: part = mute(part) @@ -362,7 +354,7 @@ class ABCPolyBase(abc.ABC): # in case somehow there are no coefficients at all body = '0' - return r'$x \mapsto {}$'.format(body) + return rf"$x \mapsto {body}$" @@ -423,17 +415,15 @@ class ABCPolyBase(abc.ABC): return NotImplemented return self.__class__(coef, self.domain, self.window) - def __div__(self, other): - # this can be removed when python 2 support is dropped. - return self.__floordiv__(other) - def __truediv__(self, other): # there is no true divide if the rhs is not a Number, although it # could return the first n elements of an infinite series. # It is hard to see where n would come from, though. if not isinstance(other, numbers.Number) or isinstance(other, bool): - form = "unsupported types for true division: '%s', '%s'" - raise TypeError(form % (type(self), type(other))) + raise TypeError( + f"unsupported types for true division: " + f"'{type(self)}', '{type(other)}'" + ) return self.__floordiv__(other) def __floordiv__(self, other): diff --git a/numpy/polynomial/chebyshev.py b/numpy/polynomial/chebyshev.py index 0bb297807..4ddb0c688 100644 --- a/numpy/polynomial/chebyshev.py +++ b/numpy/polynomial/chebyshev.py @@ -1,5 +1,7 @@ """ -Objects for dealing with Chebyshev series. +==================================================== +Chebyshev Series (:mod:`numpy.polynomial.chebyshev`) +==================================================== This module provides a number of objects (mostly functions) useful for dealing with Chebyshev series, including a `Chebyshev` class that @@ -7,57 +9,75 @@ encapsulates the usual arithmetic operations. (General information on how this module represents and works with such polynomials is in the docstring for its "parent" sub-package, `numpy.polynomial`). +Classes +------- + +.. autosummary:: + :toctree: generated/ + + Chebyshev + + Constants --------- -- `chebdomain` -- Chebyshev series default domain, [-1,1]. -- `chebzero` -- (Coefficients of the) Chebyshev series that evaluates - identically to 0. -- `chebone` -- (Coefficients of the) Chebyshev series that evaluates - identically to 1. -- `chebx` -- (Coefficients of the) Chebyshev series for the identity map, - ``f(x) = x``. + +.. autosummary:: + :toctree: generated/ + + chebdomain + chebzero + chebone + chebx Arithmetic ---------- -- `chebadd` -- add two Chebyshev series. -- `chebsub` -- subtract one Chebyshev series from another. -- `chebmulx` -- multiply a Chebyshev series in ``P_i(x)`` by ``x``. -- `chebmul` -- multiply two Chebyshev series. -- `chebdiv` -- divide one Chebyshev series by another. -- `chebpow` -- raise a Chebyshev series to a positive integer power. -- `chebval` -- evaluate a Chebyshev series at given points. -- `chebval2d` -- evaluate a 2D Chebyshev series at given points. -- `chebval3d` -- evaluate a 3D Chebyshev series at given points. -- `chebgrid2d` -- evaluate a 2D Chebyshev series on a Cartesian product. -- `chebgrid3d` -- evaluate a 3D Chebyshev series on a Cartesian product. + +.. autosummary:: + :toctree: generated/ + + chebadd + chebsub + chebmulx + chebmul + chebdiv + chebpow + chebval + chebval2d + chebval3d + chebgrid2d + chebgrid3d Calculus -------- -- `chebder` -- differentiate a Chebyshev series. -- `chebint` -- integrate a Chebyshev series. + +.. autosummary:: + :toctree: generated/ + + chebder + chebint Misc Functions -------------- -- `chebfromroots` -- create a Chebyshev series with specified roots. -- `chebroots` -- find the roots of a Chebyshev series. -- `chebvander` -- Vandermonde-like matrix for Chebyshev polynomials. -- `chebvander2d` -- Vandermonde-like matrix for 2D power series. -- `chebvander3d` -- Vandermonde-like matrix for 3D power series. -- `chebgauss` -- Gauss-Chebyshev quadrature, points and weights. -- `chebweight` -- Chebyshev weight function. -- `chebcompanion` -- symmetrized companion matrix in Chebyshev form. -- `chebfit` -- least-squares fit returning a Chebyshev series. -- `chebpts1` -- Chebyshev points of the first kind. -- `chebpts2` -- Chebyshev points of the second kind. -- `chebtrim` -- trim leading coefficients from a Chebyshev series. -- `chebline` -- Chebyshev series representing given straight line. -- `cheb2poly` -- convert a Chebyshev series to a polynomial. -- `poly2cheb` -- convert a polynomial to a Chebyshev series. -- `chebinterpolate` -- interpolate a function at the Chebyshev points. -Classes -------- -- `Chebyshev` -- A Chebyshev series class. +.. autosummary:: + :toctree: generated/ + + chebfromroots + chebroots + chebvander + chebvander2d + chebvander3d + chebgauss + chebweight + chebcompanion + chebfit + chebpts1 + chebpts2 + chebtrim + chebline + cheb2poly + poly2cheb + chebinterpolate See also -------- @@ -87,7 +107,6 @@ References (preprint: https://www.math.hmc.edu/~benjamin/papers/CombTrig.pdf, pg. 4) """ -import warnings import numpy as np import numpy.linalg as la from numpy.core.multiarray import normalize_axis_index @@ -1058,7 +1077,6 @@ def chebint(c, m=1, k=[], lbnd=0, scl=1, axis=0): if n > 1: tmp[2] = c[1]/4 for j in range(2, n): - t = c[j]/(2*j + 1) # FIXME: t never used tmp[j + 1] = c[j]/(2*(j + 1)) tmp[j - 1] -= c[j]/(2*(j - 1)) tmp[0] += k[i] - chebval(lbnd, tmp) @@ -1453,7 +1471,7 @@ def chebvander2d(x, y, deg): ------- vander2d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)`. The dtype will be the same + :math:`order = (deg[0]+1)*(deg[1]+1)`. The dtype will be the same as the converted `x` and `y`. See Also @@ -1507,7 +1525,7 @@ def chebvander3d(x, y, z, deg): ------- vander3d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)*(deg[2]+1)`. The dtype will + :math:`order = (deg[0]+1)*(deg[1]+1)*(deg[2]+1)`. The dtype will be the same as the converted `x`, `y`, and `z`. See Also diff --git a/numpy/polynomial/hermite.py b/numpy/polynomial/hermite.py index cb98b7e1f..487d8dfdb 100644 --- a/numpy/polynomial/hermite.py +++ b/numpy/polynomial/hermite.py @@ -1,5 +1,7 @@ """ -Objects for dealing with Hermite series. +============================================================== +Hermite Series, "Physicists" (:mod:`numpy.polynomial.hermite`) +============================================================== This module provides a number of objects (mostly functions) useful for dealing with Hermite series, including a `Hermite` class that @@ -7,58 +9,72 @@ encapsulates the usual arithmetic operations. (General information on how this module represents and works with such polynomials is in the docstring for its "parent" sub-package, `numpy.polynomial`). +Classes +------- +.. autosummary:: + :toctree: generated/ + + Hermite + Constants --------- -- `hermdomain` -- Hermite series default domain, [-1,1]. -- `hermzero` -- Hermite series that evaluates identically to 0. -- `hermone` -- Hermite series that evaluates identically to 1. -- `hermx` -- Hermite series for the identity map, ``f(x) = x``. +.. autosummary:: + :toctree: generated/ + + hermdomain + hermzero + hermone + hermx Arithmetic ---------- -- `hermadd` -- add two Hermite series. -- `hermsub` -- subtract one Hermite series from another. -- `hermmulx` -- multiply a Hermite series in ``P_i(x)`` by ``x``. -- `hermmul` -- multiply two Hermite series. -- `hermdiv` -- divide one Hermite series by another. -- `hermpow` -- raise a Hermite series to a positive integer power. -- `hermval` -- evaluate a Hermite series at given points. -- `hermval2d` -- evaluate a 2D Hermite series at given points. -- `hermval3d` -- evaluate a 3D Hermite series at given points. -- `hermgrid2d` -- evaluate a 2D Hermite series on a Cartesian product. -- `hermgrid3d` -- evaluate a 3D Hermite series on a Cartesian product. +.. autosummary:: + :toctree: generated/ + + hermadd + hermsub + hermmulx + hermmul + hermdiv + hermpow + hermval + hermval2d + hermval3d + hermgrid2d + hermgrid3d Calculus -------- -- `hermder` -- differentiate a Hermite series. -- `hermint` -- integrate a Hermite series. +.. autosummary:: + :toctree: generated/ + + hermder + hermint Misc Functions -------------- -- `hermfromroots` -- create a Hermite series with specified roots. -- `hermroots` -- find the roots of a Hermite series. -- `hermvander` -- Vandermonde-like matrix for Hermite polynomials. -- `hermvander2d` -- Vandermonde-like matrix for 2D power series. -- `hermvander3d` -- Vandermonde-like matrix for 3D power series. -- `hermgauss` -- Gauss-Hermite quadrature, points and weights. -- `hermweight` -- Hermite weight function. -- `hermcompanion` -- symmetrized companion matrix in Hermite form. -- `hermfit` -- least-squares fit returning a Hermite series. -- `hermtrim` -- trim leading coefficients from a Hermite series. -- `hermline` -- Hermite series of given straight line. -- `herm2poly` -- convert a Hermite series to a polynomial. -- `poly2herm` -- convert a polynomial to a Hermite series. - -Classes -------- -- `Hermite` -- A Hermite series class. +.. autosummary:: + :toctree: generated/ + + hermfromroots + hermroots + hermvander + hermvander2d + hermvander3d + hermgauss + hermweight + hermcompanion + hermfit + hermtrim + hermline + herm2poly + poly2herm See also -------- `numpy.polynomial` """ -import warnings import numpy as np import numpy.linalg as la from numpy.core.multiarray import normalize_axis_index @@ -1178,7 +1194,7 @@ def hermvander2d(x, y, deg): ------- vander2d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)`. The dtype will be the same + :math:`order = (deg[0]+1)*(deg[1]+1)`. The dtype will be the same as the converted `x` and `y`. See Also @@ -1232,7 +1248,7 @@ def hermvander3d(x, y, z, deg): ------- vander3d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)*(deg[2]+1)`. The dtype will + :math:`order = (deg[0]+1)*(deg[1]+1)*(deg[2]+1)`. The dtype will be the same as the converted `x`, `y`, and `z`. See Also @@ -1353,8 +1369,8 @@ def hermfit(x, y, deg, rcond=None, full=False, w=None): Fits using Hermite series are probably most useful when the data can be approximated by ``sqrt(w(x)) * p(x)``, where `w(x)` is the Hermite - weight. In that case the weight ``sqrt(w(x[i])`` should be used - together with data values ``y[i]/sqrt(w(x[i])``. The weight function is + weight. In that case the weight ``sqrt(w(x[i]))`` should be used + together with data values ``y[i]/sqrt(w(x[i]))``. The weight function is available as `hermweight`. References diff --git a/numpy/polynomial/hermite_e.py b/numpy/polynomial/hermite_e.py index 1f4a93c24..cbec15184 100644 --- a/numpy/polynomial/hermite_e.py +++ b/numpy/polynomial/hermite_e.py @@ -1,5 +1,7 @@ """ -Objects for dealing with Hermite_e series. +=================================================================== +HermiteE Series, "Probabilists" (:mod:`numpy.polynomial.hermite_e`) +=================================================================== This module provides a number of objects (mostly functions) useful for dealing with Hermite_e series, including a `HermiteE` class that @@ -7,58 +9,72 @@ encapsulates the usual arithmetic operations. (General information on how this module represents and works with such polynomials is in the docstring for its "parent" sub-package, `numpy.polynomial`). +Classes +------- +.. autosummary:: + :toctree: generated/ + + HermiteE + Constants --------- -- `hermedomain` -- Hermite_e series default domain, [-1,1]. -- `hermezero` -- Hermite_e series that evaluates identically to 0. -- `hermeone` -- Hermite_e series that evaluates identically to 1. -- `hermex` -- Hermite_e series for the identity map, ``f(x) = x``. +.. autosummary:: + :toctree: generated/ + + hermedomain + hermezero + hermeone + hermex Arithmetic ---------- -- `hermeadd` -- add two Hermite_e series. -- `hermesub` -- subtract one Hermite_e series from another. -- `hermemulx` -- multiply a Hermite_e series in ``P_i(x)`` by ``x``. -- `hermemul` -- multiply two Hermite_e series. -- `hermediv` -- divide one Hermite_e series by another. -- `hermepow` -- raise a Hermite_e series to a positive integer power. -- `hermeval` -- evaluate a Hermite_e series at given points. -- `hermeval2d` -- evaluate a 2D Hermite_e series at given points. -- `hermeval3d` -- evaluate a 3D Hermite_e series at given points. -- `hermegrid2d` -- evaluate a 2D Hermite_e series on a Cartesian product. -- `hermegrid3d` -- evaluate a 3D Hermite_e series on a Cartesian product. +.. autosummary:: + :toctree: generated/ + + hermeadd + hermesub + hermemulx + hermemul + hermediv + hermepow + hermeval + hermeval2d + hermeval3d + hermegrid2d + hermegrid3d Calculus -------- -- `hermeder` -- differentiate a Hermite_e series. -- `hermeint` -- integrate a Hermite_e series. +.. autosummary:: + :toctree: generated/ + + hermeder + hermeint Misc Functions -------------- -- `hermefromroots` -- create a Hermite_e series with specified roots. -- `hermeroots` -- find the roots of a Hermite_e series. -- `hermevander` -- Vandermonde-like matrix for Hermite_e polynomials. -- `hermevander2d` -- Vandermonde-like matrix for 2D power series. -- `hermevander3d` -- Vandermonde-like matrix for 3D power series. -- `hermegauss` -- Gauss-Hermite_e quadrature, points and weights. -- `hermeweight` -- Hermite_e weight function. -- `hermecompanion` -- symmetrized companion matrix in Hermite_e form. -- `hermefit` -- least-squares fit returning a Hermite_e series. -- `hermetrim` -- trim leading coefficients from a Hermite_e series. -- `hermeline` -- Hermite_e series of given straight line. -- `herme2poly` -- convert a Hermite_e series to a polynomial. -- `poly2herme` -- convert a polynomial to a Hermite_e series. - -Classes -------- -- `HermiteE` -- A Hermite_e series class. +.. autosummary:: + :toctree: generated/ + + hermefromroots + hermeroots + hermevander + hermevander2d + hermevander3d + hermegauss + hermeweight + hermecompanion + hermefit + hermetrim + hermeline + herme2poly + poly2herme See also -------- `numpy.polynomial` """ -import warnings import numpy as np import numpy.linalg as la from numpy.core.multiarray import normalize_axis_index @@ -1171,7 +1187,7 @@ def hermevander2d(x, y, deg): ------- vander2d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)`. The dtype will be the same + :math:`order = (deg[0]+1)*(deg[1]+1)`. The dtype will be the same as the converted `x` and `y`. See Also @@ -1225,7 +1241,7 @@ def hermevander3d(x, y, z, deg): ------- vander3d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)*(deg[2]+1)`. The dtype will + :math:`order = (deg[0]+1)*(deg[1]+1)*(deg[2]+1)`. The dtype will be the same as the converted `x`, `y`, and `z`. See Also @@ -1346,8 +1362,8 @@ def hermefit(x, y, deg, rcond=None, full=False, w=None): Fits using HermiteE series are probably most useful when the data can be approximated by ``sqrt(w(x)) * p(x)``, where `w(x)` is the HermiteE - weight. In that case the weight ``sqrt(w(x[i])`` should be used - together with data values ``y[i]/sqrt(w(x[i])``. The weight function is + weight. In that case the weight ``sqrt(w(x[i]))`` should be used + together with data values ``y[i]/sqrt(w(x[i]))``. The weight function is available as `hermeweight`. References diff --git a/numpy/polynomial/laguerre.py b/numpy/polynomial/laguerre.py index bf8e11623..5b66d943e 100644 --- a/numpy/polynomial/laguerre.py +++ b/numpy/polynomial/laguerre.py @@ -1,5 +1,7 @@ """ -Objects for dealing with Laguerre series. +================================================== +Laguerre Series (:mod:`numpy.polynomial.laguerre`) +================================================== This module provides a number of objects (mostly functions) useful for dealing with Laguerre series, including a `Laguerre` class that @@ -7,58 +9,72 @@ encapsulates the usual arithmetic operations. (General information on how this module represents and works with such polynomials is in the docstring for its "parent" sub-package, `numpy.polynomial`). +Classes +------- +.. autosummary:: + :toctree: generated/ + + Laguerre + Constants --------- -- `lagdomain` -- Laguerre series default domain, [-1,1]. -- `lagzero` -- Laguerre series that evaluates identically to 0. -- `lagone` -- Laguerre series that evaluates identically to 1. -- `lagx` -- Laguerre series for the identity map, ``f(x) = x``. +.. autosummary:: + :toctree: generated/ + + lagdomain + lagzero + lagone + lagx Arithmetic ---------- -- `lagadd` -- add two Laguerre series. -- `lagsub` -- subtract one Laguerre series from another. -- `lagmulx` -- multiply a Laguerre series in ``P_i(x)`` by ``x``. -- `lagmul` -- multiply two Laguerre series. -- `lagdiv` -- divide one Laguerre series by another. -- `lagpow` -- raise a Laguerre series to a positive integer power. -- `lagval` -- evaluate a Laguerre series at given points. -- `lagval2d` -- evaluate a 2D Laguerre series at given points. -- `lagval3d` -- evaluate a 3D Laguerre series at given points. -- `laggrid2d` -- evaluate a 2D Laguerre series on a Cartesian product. -- `laggrid3d` -- evaluate a 3D Laguerre series on a Cartesian product. +.. autosummary:: + :toctree: generated/ + + lagadd + lagsub + lagmulx + lagmul + lagdiv + lagpow + lagval + lagval2d + lagval3d + laggrid2d + laggrid3d Calculus -------- -- `lagder` -- differentiate a Laguerre series. -- `lagint` -- integrate a Laguerre series. +.. autosummary:: + :toctree: generated/ + + lagder + lagint Misc Functions -------------- -- `lagfromroots` -- create a Laguerre series with specified roots. -- `lagroots` -- find the roots of a Laguerre series. -- `lagvander` -- Vandermonde-like matrix for Laguerre polynomials. -- `lagvander2d` -- Vandermonde-like matrix for 2D power series. -- `lagvander3d` -- Vandermonde-like matrix for 3D power series. -- `laggauss` -- Gauss-Laguerre quadrature, points and weights. -- `lagweight` -- Laguerre weight function. -- `lagcompanion` -- symmetrized companion matrix in Laguerre form. -- `lagfit` -- least-squares fit returning a Laguerre series. -- `lagtrim` -- trim leading coefficients from a Laguerre series. -- `lagline` -- Laguerre series of given straight line. -- `lag2poly` -- convert a Laguerre series to a polynomial. -- `poly2lag` -- convert a polynomial to a Laguerre series. - -Classes -------- -- `Laguerre` -- A Laguerre series class. +.. autosummary:: + :toctree: generated/ + + lagfromroots + lagroots + lagvander + lagvander2d + lagvander3d + laggauss + lagweight + lagcompanion + lagfit + lagtrim + lagline + lag2poly + poly2lag See also -------- `numpy.polynomial` """ -import warnings import numpy as np import numpy.linalg as la from numpy.core.multiarray import normalize_axis_index @@ -1178,7 +1194,7 @@ def lagvander2d(x, y, deg): ------- vander2d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)`. The dtype will be the same + :math:`order = (deg[0]+1)*(deg[1]+1)`. The dtype will be the same as the converted `x` and `y`. See Also @@ -1232,7 +1248,7 @@ def lagvander3d(x, y, z, deg): ------- vander3d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)*(deg[2]+1)`. The dtype will + :math:`order = (deg[0]+1)*(deg[1]+1)*(deg[2]+1)`. The dtype will be the same as the converted `x`, `y`, and `z`. See Also @@ -1353,8 +1369,8 @@ def lagfit(x, y, deg, rcond=None, full=False, w=None): Fits using Laguerre series are probably most useful when the data can be approximated by ``sqrt(w(x)) * p(x)``, where `w(x)` is the Laguerre - weight. In that case the weight ``sqrt(w(x[i])`` should be used - together with data values ``y[i]/sqrt(w(x[i])``. The weight function is + weight. In that case the weight ``sqrt(w(x[i]))`` should be used + together with data values ``y[i]/sqrt(w(x[i]))``. The weight function is available as `lagweight`. References diff --git a/numpy/polynomial/legendre.py b/numpy/polynomial/legendre.py index d74b87d5a..47e47a7b6 100644 --- a/numpy/polynomial/legendre.py +++ b/numpy/polynomial/legendre.py @@ -1,8 +1,7 @@ """ -Legendre Series (:mod: `numpy.polynomial.legendre`) -=================================================== - -.. currentmodule:: numpy.polynomial.polynomial +================================================== +Legendre Series (:mod:`numpy.polynomial.legendre`) +================================================== This module provides a number of objects (mostly functions) useful for dealing with Legendre series, including a `Legendre` class that @@ -10,16 +9,23 @@ encapsulates the usual arithmetic operations. (General information on how this module represents and works with such polynomials is in the docstring for its "parent" sub-package, `numpy.polynomial`). +Classes +------- +.. autosummary:: + :toctree: generated/ + + Legendre + Constants --------- .. autosummary:: :toctree: generated/ - legdomain Legendre series default domain, [-1,1]. - legzero Legendre series that evaluates identically to 0. - legone Legendre series that evaluates identically to 1. - legx Legendre series for the identity map, ``f(x) = x``. + legdomain + legzero + legone + legx Arithmetic ---------- @@ -27,17 +33,17 @@ Arithmetic .. autosummary:: :toctree: generated/ - legadd add two Legendre series. - legsub subtract one Legendre series from another. - legmulx multiply a Legendre series in ``P_i(x)`` by ``x``. - legmul multiply two Legendre series. - legdiv divide one Legendre series by another. - legpow raise a Legendre series to a positive integer power. - legval evaluate a Legendre series at given points. - legval2d evaluate a 2D Legendre series at given points. - legval3d evaluate a 3D Legendre series at given points. - leggrid2d evaluate a 2D Legendre series on a Cartesian product. - leggrid3d evaluate a 3D Legendre series on a Cartesian product. + legadd + legsub + legmulx + legmul + legdiv + legpow + legval + legval2d + legval3d + leggrid2d + leggrid3d Calculus -------- @@ -45,8 +51,8 @@ Calculus .. autosummary:: :toctree: generated/ - legder differentiate a Legendre series. - legint integrate a Legendre series. + legder + legint Misc Functions -------------- @@ -54,34 +60,25 @@ Misc Functions .. autosummary:: :toctree: generated/ - legfromroots create a Legendre series with specified roots. - legroots find the roots of a Legendre series. - legvander Vandermonde-like matrix for Legendre polynomials. - legvander2d Vandermonde-like matrix for 2D power series. - legvander3d Vandermonde-like matrix for 3D power series. - leggauss Gauss-Legendre quadrature, points and weights. - legweight Legendre weight function. - legcompanion symmetrized companion matrix in Legendre form. - legfit least-squares fit returning a Legendre series. - legtrim trim leading coefficients from a Legendre series. - legline Legendre series representing given straight line. - leg2poly convert a Legendre series to a polynomial. - poly2leg convert a polynomial to a Legendre series. - -Classes -------- - Legendre A Legendre series class. + legfromroots + legroots + legvander + legvander2d + legvander3d + leggauss + legweight + legcompanion + legfit + legtrim + legline + leg2poly + poly2leg See also -------- -numpy.polynomial.polynomial -numpy.polynomial.chebyshev -numpy.polynomial.laguerre -numpy.polynomial.hermite -numpy.polynomial.hermite_e +numpy.polynomial """ -import warnings import numpy as np import numpy.linalg as la from numpy.core.multiarray import normalize_axis_index @@ -1214,7 +1211,7 @@ def legvander2d(x, y, deg): ------- vander2d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)`. The dtype will be the same + :math:`order = (deg[0]+1)*(deg[1]+1)`. The dtype will be the same as the converted `x` and `y`. See Also @@ -1268,7 +1265,7 @@ def legvander3d(x, y, z, deg): ------- vander3d : ndarray The shape of the returned matrix is ``x.shape + (order,)``, where - :math:`order = (deg[0]+1)*(deg([1]+1)*(deg[2]+1)`. The dtype will + :math:`order = (deg[0]+1)*(deg[1]+1)*(deg[2]+1)`. The dtype will be the same as the converted `x`, `y`, and `z`. See Also diff --git a/numpy/polynomial/polynomial.py b/numpy/polynomial/polynomial.py index 92fdc06fa..2fb032db3 100644 --- a/numpy/polynomial/polynomial.py +++ b/numpy/polynomial/polynomial.py @@ -1,5 +1,7 @@ """ -Objects for dealing with polynomials. +================================================= +Power Series (:mod:`numpy.polynomial.polynomial`) +================================================= This module provides a number of objects (mostly functions) useful for dealing with polynomials, including a `Polynomial` class that @@ -7,48 +9,63 @@ encapsulates the usual arithmetic operations. (General information on how this module represents and works with polynomial objects is in the docstring for its "parent" sub-package, `numpy.polynomial`). +Classes +------- +.. autosummary:: + :toctree: generated/ + + Polynomial + Constants --------- -- `polydomain` -- Polynomial default domain, [-1,1]. -- `polyzero` -- (Coefficients of the) "zero polynomial." -- `polyone` -- (Coefficients of the) constant polynomial 1. -- `polyx` -- (Coefficients of the) identity map polynomial, ``f(x) = x``. +.. autosummary:: + :toctree: generated/ + + polydomain + polyzero + polyone + polyx Arithmetic ---------- -- `polyadd` -- add two polynomials. -- `polysub` -- subtract one polynomial from another. -- `polymulx` -- multiply a polynomial in ``P_i(x)`` by ``x``. -- `polymul` -- multiply two polynomials. -- `polydiv` -- divide one polynomial by another. -- `polypow` -- raise a polynomial to a positive integer power. -- `polyval` -- evaluate a polynomial at given points. -- `polyval2d` -- evaluate a 2D polynomial at given points. -- `polyval3d` -- evaluate a 3D polynomial at given points. -- `polygrid2d` -- evaluate a 2D polynomial on a Cartesian product. -- `polygrid3d` -- evaluate a 3D polynomial on a Cartesian product. +.. autosummary:: + :toctree: generated/ + + polyadd + polysub + polymulx + polymul + polydiv + polypow + polyval + polyval2d + polyval3d + polygrid2d + polygrid3d Calculus -------- -- `polyder` -- differentiate a polynomial. -- `polyint` -- integrate a polynomial. +.. autosummary:: + :toctree: generated/ + + polyder + polyint Misc Functions -------------- -- `polyfromroots` -- create a polynomial with specified roots. -- `polyroots` -- find the roots of a polynomial. -- `polyvalfromroots` -- evaluate a polynomial at given points from roots. -- `polyvander` -- Vandermonde-like matrix for powers. -- `polyvander2d` -- Vandermonde-like matrix for 2D power series. -- `polyvander3d` -- Vandermonde-like matrix for 3D power series. -- `polycompanion` -- companion matrix in power series form. -- `polyfit` -- least-squares fit returning a polynomial. -- `polytrim` -- trim leading coefficients from a polynomial. -- `polyline` -- polynomial representing given straight line. - -Classes -------- -- `Polynomial` -- polynomial class. +.. autosummary:: + :toctree: generated/ + + polyfromroots + polyroots + polyvalfromroots + polyvander + polyvander2d + polyvander3d + polycompanion + polyfit + polytrim + polyline See Also -------- @@ -62,7 +79,6 @@ __all__ = [ 'polyfit', 'polytrim', 'polyroots', 'Polynomial', 'polyval2d', 'polyval3d', 'polygrid2d', 'polygrid3d', 'polyvander2d', 'polyvander3d'] -import warnings import numpy as np import numpy.linalg as la from numpy.core.multiarray import normalize_axis_index @@ -1482,10 +1498,10 @@ class Polynomial(ABCPolyBase): @staticmethod def _repr_latex_term(i, arg_str, needs_parens): if needs_parens: - arg_str = r'\left({}\right)'.format(arg_str) + arg_str = rf"\left({arg_str}\right)" if i == 0: return '1' elif i == 1: return arg_str else: - return '{}^{{{}}}'.format(arg_str, i) + return f"{arg_str}^{{{i}}}" diff --git a/numpy/polynomial/polyutils.py b/numpy/polynomial/polyutils.py index 9b8e9fc42..b1cf07e8a 100644 --- a/numpy/polynomial/polyutils.py +++ b/numpy/polynomial/polyutils.py @@ -193,8 +193,8 @@ def as_series(alist, trim=True): else: try: dtype = np.common_type(*arrays) - except Exception: - raise ValueError("Coefficient arrays have no common type") + except Exception as e: + raise ValueError("Coefficient arrays have no common type") from e ret = [np.array(a, copy=True, dtype=dtype) for a in arrays] return ret @@ -468,10 +468,10 @@ def _vander_nd(vander_fs, points, degrees): n_dims = len(vander_fs) if n_dims != len(points): raise ValueError( - "Expected {} dimensions of sample points, got {}".format(n_dims, len(points))) + f"Expected {n_dims} dimensions of sample points, got {len(points)}") if n_dims != len(degrees): raise ValueError( - "Expected {} dimensions of degrees, got {}".format(n_dims, len(degrees))) + f"Expected {n_dims} dimensions of degrees, got {len(degrees)}") if n_dims == 0: raise ValueError("Unable to guess a dtype or shape when no points are given") @@ -777,7 +777,7 @@ def _deprecate_as_int(x, desc): """ try: return operator.index(x) - except TypeError: + except TypeError as e: # Numpy 1.17.0, 2019-03-11 try: ix = int(x) @@ -786,12 +786,11 @@ def _deprecate_as_int(x, desc): else: if ix == x: warnings.warn( - "In future, this will raise TypeError, as {} will need to " - "be an integer not just an integral float." - .format(desc), + f"In future, this will raise TypeError, as {desc} will " + "need to be an integer not just an integral float.", DeprecationWarning, stacklevel=3 ) return ix - raise TypeError("{} must be an integer".format(desc)) + raise TypeError(f"{desc} must be an integer") from e diff --git a/numpy/polynomial/tests/test_chebyshev.py b/numpy/polynomial/tests/test_chebyshev.py index ec0a7839a..2f54bebfd 100644 --- a/numpy/polynomial/tests/test_chebyshev.py +++ b/numpy/polynomial/tests/test_chebyshev.py @@ -65,7 +65,7 @@ class TestArithmetic: def test_chebadd(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] += 1 @@ -75,7 +75,7 @@ class TestArithmetic: def test_chebsub(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] -= 1 @@ -93,7 +93,7 @@ class TestArithmetic: def test_chebmul(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(i + j + 1) tgt[i + j] += .5 tgt[abs(i - j)] += .5 @@ -103,7 +103,7 @@ class TestArithmetic: def test_chebdiv(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" ci = [0]*i + [1] cj = [0]*j + [1] tgt = cheb.chebadd(ci, cj) @@ -114,7 +114,7 @@ class TestArithmetic: def test_chebpow(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" c = np.arange(i + 1) tgt = reduce(cheb.chebmul, [c]*j, np.array([1])) res = cheb.chebpow(c, j) @@ -139,7 +139,7 @@ class TestEvaluation: x = np.linspace(-1, 1) y = [polyval(x, c) for c in Tlist] for i in range(10): - msg = "At i=%d" % i + msg = f"At i={i}" tgt = y[i] res = cheb.chebval(x, [0]*i + [1]) assert_almost_equal(res, tgt, err_msg=msg) diff --git a/numpy/polynomial/tests/test_classes.py b/numpy/polynomial/tests/test_classes.py index 68656bc98..8e71a1945 100644 --- a/numpy/polynomial/tests/test_classes.py +++ b/numpy/polynomial/tests/test_classes.py @@ -13,7 +13,6 @@ from numpy.polynomial import ( from numpy.testing import ( assert_almost_equal, assert_raises, assert_equal, assert_, ) -from numpy.compat import long from numpy.polynomial.polyutils import RankWarning # @@ -42,7 +41,7 @@ def assert_poly_almost_equal(p1, p2, msg=""): assert_(np.all(p1.window == p2.window)) assert_almost_equal(p1.coef, p2.coef) except AssertionError: - msg = "Result: %s\nTarget: %s", (p1, p2) + msg = f"Result: {p1}\nTarget: {p2}" raise AssertionError(msg) @@ -315,7 +314,7 @@ def test_truediv(Poly): s = stype(5) assert_poly_almost_equal(op.truediv(p2, s), p1) assert_raises(TypeError, op.truediv, s, p2) - for stype in (int, long, float): + for stype in (int, float): s = stype(5) assert_poly_almost_equal(op.truediv(p2, s), p1) assert_raises(TypeError, op.truediv, s, p2) @@ -571,56 +570,6 @@ def test_ufunc_override(Poly): assert_raises(TypeError, np.add, x, p) - -class TestLatexRepr: - """Test the latex repr used by ipython """ - - def as_latex(self, obj): - # right now we ignore the formatting of scalars in our tests, since - # it makes them too verbose. Ideally, the formatting of scalars will - # be fixed such that tests below continue to pass - obj._repr_latex_scalar = lambda x: str(x) - try: - return obj._repr_latex_() - finally: - del obj._repr_latex_scalar - - def test_simple_polynomial(self): - # default input - p = Polynomial([1, 2, 3]) - assert_equal(self.as_latex(p), - r'$x \mapsto 1.0 + 2.0\,x + 3.0\,x^{2}$') - - # translated input - p = Polynomial([1, 2, 3], domain=[-2, 0]) - assert_equal(self.as_latex(p), - r'$x \mapsto 1.0 + 2.0\,\left(1.0 + x\right) + 3.0\,\left(1.0 + x\right)^{2}$') - - # scaled input - p = Polynomial([1, 2, 3], domain=[-0.5, 0.5]) - assert_equal(self.as_latex(p), - r'$x \mapsto 1.0 + 2.0\,\left(2.0x\right) + 3.0\,\left(2.0x\right)^{2}$') - - # affine input - p = Polynomial([1, 2, 3], domain=[-1, 0]) - assert_equal(self.as_latex(p), - r'$x \mapsto 1.0 + 2.0\,\left(1.0 + 2.0x\right) + 3.0\,\left(1.0 + 2.0x\right)^{2}$') - - def test_basis_func(self): - p = Chebyshev([1, 2, 3]) - assert_equal(self.as_latex(p), - r'$x \mapsto 1.0\,{T}_{0}(x) + 2.0\,{T}_{1}(x) + 3.0\,{T}_{2}(x)$') - # affine input - check no surplus parens are added - p = Chebyshev([1, 2, 3], domain=[-1, 0]) - assert_equal(self.as_latex(p), - r'$x \mapsto 1.0\,{T}_{0}(1.0 + 2.0x) + 2.0\,{T}_{1}(1.0 + 2.0x) + 3.0\,{T}_{2}(1.0 + 2.0x)$') - - def test_multichar_basis_func(self): - p = HermiteE([1, 2, 3]) - assert_equal(self.as_latex(p), - r'$x \mapsto 1.0\,{He}_{0}(x) + 2.0\,{He}_{1}(x) + 3.0\,{He}_{2}(x)$') - - # # Test class method that only exists for some classes # diff --git a/numpy/polynomial/tests/test_hermite.py b/numpy/polynomial/tests/test_hermite.py index 4b67c1b18..53ee0844e 100644 --- a/numpy/polynomial/tests/test_hermite.py +++ b/numpy/polynomial/tests/test_hermite.py @@ -49,7 +49,7 @@ class TestArithmetic: def test_hermadd(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] += 1 @@ -59,7 +59,7 @@ class TestArithmetic: def test_hermsub(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] -= 1 @@ -80,7 +80,7 @@ class TestArithmetic: pol1 = [0]*i + [1] val1 = herm.hermval(self.x, pol1) for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" pol2 = [0]*j + [1] val2 = herm.hermval(self.x, pol2) pol3 = herm.hermmul(pol1, pol2) @@ -91,7 +91,7 @@ class TestArithmetic: def test_hermdiv(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" ci = [0]*i + [1] cj = [0]*j + [1] tgt = herm.hermadd(ci, cj) @@ -102,7 +102,7 @@ class TestArithmetic: def test_hermpow(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" c = np.arange(i + 1) tgt = reduce(herm.hermmul, [c]*j, np.array([1])) res = herm.hermpow(c, j) @@ -127,7 +127,7 @@ class TestEvaluation: x = np.linspace(-1, 1) y = [polyval(x, c) for c in Hlist] for i in range(10): - msg = "At i=%d" % i + msg = f"At i={i}" tgt = y[i] res = herm.hermval(x, [0]*i + [1]) assert_almost_equal(res, tgt, err_msg=msg) diff --git a/numpy/polynomial/tests/test_hermite_e.py b/numpy/polynomial/tests/test_hermite_e.py index 3052500cc..2d262a330 100644 --- a/numpy/polynomial/tests/test_hermite_e.py +++ b/numpy/polynomial/tests/test_hermite_e.py @@ -49,7 +49,7 @@ class TestArithmetic: def test_hermeadd(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] += 1 @@ -59,7 +59,7 @@ class TestArithmetic: def test_hermesub(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] -= 1 @@ -80,7 +80,7 @@ class TestArithmetic: pol1 = [0]*i + [1] val1 = herme.hermeval(self.x, pol1) for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" pol2 = [0]*j + [1] val2 = herme.hermeval(self.x, pol2) pol3 = herme.hermemul(pol1, pol2) @@ -91,7 +91,7 @@ class TestArithmetic: def test_hermediv(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" ci = [0]*i + [1] cj = [0]*j + [1] tgt = herme.hermeadd(ci, cj) @@ -102,7 +102,7 @@ class TestArithmetic: def test_hermepow(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" c = np.arange(i + 1) tgt = reduce(herme.hermemul, [c]*j, np.array([1])) res = herme.hermepow(c, j) @@ -127,7 +127,7 @@ class TestEvaluation: x = np.linspace(-1, 1) y = [polyval(x, c) for c in Helist] for i in range(10): - msg = "At i=%d" % i + msg = f"At i={i}" tgt = y[i] res = herme.hermeval(x, [0]*i + [1]) assert_almost_equal(res, tgt, err_msg=msg) diff --git a/numpy/polynomial/tests/test_laguerre.py b/numpy/polynomial/tests/test_laguerre.py index ec103c258..227ef3c55 100644 --- a/numpy/polynomial/tests/test_laguerre.py +++ b/numpy/polynomial/tests/test_laguerre.py @@ -46,7 +46,7 @@ class TestArithmetic: def test_lagadd(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] += 1 @@ -56,7 +56,7 @@ class TestArithmetic: def test_lagsub(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] -= 1 @@ -77,7 +77,7 @@ class TestArithmetic: pol1 = [0]*i + [1] val1 = lag.lagval(self.x, pol1) for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" pol2 = [0]*j + [1] val2 = lag.lagval(self.x, pol2) pol3 = lag.lagmul(pol1, pol2) @@ -88,7 +88,7 @@ class TestArithmetic: def test_lagdiv(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" ci = [0]*i + [1] cj = [0]*j + [1] tgt = lag.lagadd(ci, cj) @@ -99,7 +99,7 @@ class TestArithmetic: def test_lagpow(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" c = np.arange(i + 1) tgt = reduce(lag.lagmul, [c]*j, np.array([1])) res = lag.lagpow(c, j) @@ -124,7 +124,7 @@ class TestEvaluation: x = np.linspace(-1, 1) y = [polyval(x, c) for c in Llist] for i in range(7): - msg = "At i=%d" % i + msg = f"At i={i}" tgt = y[i] res = lag.lagval(x, [0]*i + [1]) assert_almost_equal(res, tgt, err_msg=msg) diff --git a/numpy/polynomial/tests/test_legendre.py b/numpy/polynomial/tests/test_legendre.py index 8846ca6f2..a2a212c24 100644 --- a/numpy/polynomial/tests/test_legendre.py +++ b/numpy/polynomial/tests/test_legendre.py @@ -49,7 +49,7 @@ class TestArithmetic: def test_legadd(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] += 1 @@ -59,7 +59,7 @@ class TestArithmetic: def test_legsub(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] -= 1 @@ -81,7 +81,7 @@ class TestArithmetic: pol1 = [0]*i + [1] val1 = leg.legval(self.x, pol1) for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" pol2 = [0]*j + [1] val2 = leg.legval(self.x, pol2) pol3 = leg.legmul(pol1, pol2) @@ -92,7 +92,7 @@ class TestArithmetic: def test_legdiv(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" ci = [0]*i + [1] cj = [0]*j + [1] tgt = leg.legadd(ci, cj) @@ -103,7 +103,7 @@ class TestArithmetic: def test_legpow(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" c = np.arange(i + 1) tgt = reduce(leg.legmul, [c]*j, np.array([1])) res = leg.legpow(c, j) @@ -128,7 +128,7 @@ class TestEvaluation: x = np.linspace(-1, 1) y = [polyval(x, c) for c in Llist] for i in range(10): - msg = "At i=%d" % i + msg = f"At i={i}" tgt = y[i] res = leg.legval(x, [0]*i + [1]) assert_almost_equal(res, tgt, err_msg=msg) diff --git a/numpy/polynomial/tests/test_polynomial.py b/numpy/polynomial/tests/test_polynomial.py index 50973c480..5fd1a82a2 100644 --- a/numpy/polynomial/tests/test_polynomial.py +++ b/numpy/polynomial/tests/test_polynomial.py @@ -47,7 +47,7 @@ class TestArithmetic: def test_polyadd(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] += 1 @@ -57,7 +57,7 @@ class TestArithmetic: def test_polysub(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(max(i, j) + 1) tgt[i] += 1 tgt[j] -= 1 @@ -75,7 +75,7 @@ class TestArithmetic: def test_polymul(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" tgt = np.zeros(i + j + 1) tgt[i + j] += 1 res = poly.polymul([0]*i + [1], [0]*j + [1]) @@ -94,7 +94,7 @@ class TestArithmetic: # check rest. for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" ci = [0]*i + [1, 2] cj = [0]*j + [1, 2] tgt = poly.polyadd(ci, cj) @@ -105,7 +105,7 @@ class TestArithmetic: def test_polypow(self): for i in range(5): for j in range(5): - msg = "At i=%d, j=%d" % (i, j) + msg = f"At i={i}, j={j}" c = np.arange(i + 1) tgt = reduce(poly.polymul, [c]*j, np.array([1])) res = poly.polypow(c, j) diff --git a/numpy/polynomial/tests/test_printing.py b/numpy/polynomial/tests/test_printing.py index 049d3af2f..bbd5502af 100644 --- a/numpy/polynomial/tests/test_printing.py +++ b/numpy/polynomial/tests/test_printing.py @@ -64,3 +64,52 @@ class TestRepr: res = repr(poly.Laguerre([0, 1])) tgt = 'Laguerre([0., 1.], domain=[0, 1], window=[0, 1])' assert_equal(res, tgt) + + +class TestLatexRepr: + """Test the latex repr used by Jupyter""" + + def as_latex(self, obj): + # right now we ignore the formatting of scalars in our tests, since + # it makes them too verbose. Ideally, the formatting of scalars will + # be fixed such that tests below continue to pass + obj._repr_latex_scalar = lambda x: str(x) + try: + return obj._repr_latex_() + finally: + del obj._repr_latex_scalar + + def test_simple_polynomial(self): + # default input + p = poly.Polynomial([1, 2, 3]) + assert_equal(self.as_latex(p), + r'$x \mapsto 1.0 + 2.0\,x + 3.0\,x^{2}$') + + # translated input + p = poly.Polynomial([1, 2, 3], domain=[-2, 0]) + assert_equal(self.as_latex(p), + r'$x \mapsto 1.0 + 2.0\,\left(1.0 + x\right) + 3.0\,\left(1.0 + x\right)^{2}$') + + # scaled input + p = poly.Polynomial([1, 2, 3], domain=[-0.5, 0.5]) + assert_equal(self.as_latex(p), + r'$x \mapsto 1.0 + 2.0\,\left(2.0x\right) + 3.0\,\left(2.0x\right)^{2}$') + + # affine input + p = poly.Polynomial([1, 2, 3], domain=[-1, 0]) + assert_equal(self.as_latex(p), + r'$x \mapsto 1.0 + 2.0\,\left(1.0 + 2.0x\right) + 3.0\,\left(1.0 + 2.0x\right)^{2}$') + + def test_basis_func(self): + p = poly.Chebyshev([1, 2, 3]) + assert_equal(self.as_latex(p), + r'$x \mapsto 1.0\,{T}_{0}(x) + 2.0\,{T}_{1}(x) + 3.0\,{T}_{2}(x)$') + # affine input - check no surplus parens are added + p = poly.Chebyshev([1, 2, 3], domain=[-1, 0]) + assert_equal(self.as_latex(p), + r'$x \mapsto 1.0\,{T}_{0}(1.0 + 2.0x) + 2.0\,{T}_{1}(1.0 + 2.0x) + 3.0\,{T}_{2}(1.0 + 2.0x)$') + + def test_multichar_basis_func(self): + p = poly.HermiteE([1, 2, 3]) + assert_equal(self.as_latex(p), + r'$x \mapsto 1.0\,{He}_{0}(x) + 2.0\,{He}_{1}(x) + 3.0\,{He}_{2}(x)$') diff --git a/numpy/random/__init__.pxd b/numpy/random/__init__.pxd index 05e073876..1f9057296 100644 --- a/numpy/random/__init__.pxd +++ b/numpy/random/__init__.pxd @@ -11,4 +11,4 @@ cdef extern from "numpy/random/bitgen.h": ctypedef bitgen bitgen_t -from numpy.random._bit_generator cimport BitGenerator, SeedSequence +from numpy.random.bit_generator cimport BitGenerator, SeedSequence diff --git a/numpy/random/__init__.py b/numpy/random/__init__.py index 0b80999d8..7efa5c07f 100644 --- a/numpy/random/__init__.py +++ b/numpy/random/__init__.py @@ -181,7 +181,7 @@ from . import _common from . import _bounded_integers from ._generator import Generator, default_rng -from ._bit_generator import SeedSequence, BitGenerator +from .bit_generator import SeedSequence, BitGenerator from ._mt19937 import MT19937 from ._pcg64 import PCG64 from ._philox import Philox diff --git a/numpy/random/_bounded_integers.pyx.in b/numpy/random/_bounded_integers.pyx.in index 9e639b53b..9f46685d3 100644 --- a/numpy/random/_bounded_integers.pyx.in +++ b/numpy/random/_bounded_integers.pyx.in @@ -51,16 +51,6 @@ cdef extern from "numpy/random/distributions.h": np.npy_bool *out) nogil - -_integers_types = {'bool': (0, 2), - 'int8': (-2**7, 2**7), - 'int16': (-2**15, 2**15), - 'int32': (-2**31, 2**31), - 'int64': (-2**63, 2**63), - 'uint8': (0, 2**8), - 'uint16': (0, 2**16), - 'uint32': (0, 2**32), - 'uint64': (0, 2**64)} {{ py: type_info = (('uint32', 'uint32', 'uint64', 'NPY_UINT64', 0, 0, 0, '0X100000000ULL'), diff --git a/numpy/random/_examples/cython/extending.pyx b/numpy/random/_examples/cython/extending.pyx index 7a0dfe078..3a7f81aa0 100644 --- a/numpy/random/_examples/cython/extending.pyx +++ b/numpy/random/_examples/cython/extending.pyx @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #cython: language_level=3 from libc.stdint cimport uint32_t diff --git a/numpy/random/_examples/cython/extending_distributions.pyx b/numpy/random/_examples/cython/extending_distributions.pyx index 1bef506ef..d908e92d0 100644 --- a/numpy/random/_examples/cython/extending_distributions.pyx +++ b/numpy/random/_examples/cython/extending_distributions.pyx @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #cython: language_level=3 """ This file shows how the to use a BitGenerator to create a distribution. @@ -10,6 +10,8 @@ from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer from libc.stdint cimport uint16_t, uint64_t from numpy.random cimport bitgen_t from numpy.random import PCG64 +from numpy.random.c_distributions cimport ( + random_standard_uniform_fill, random_standard_uniform_fill_f) @cython.boundscheck(False) @@ -40,7 +42,7 @@ def uniforms(Py_ssize_t n): randoms = np.asarray(random_values) return randoms - + # cython example 2 @cython.boundscheck(False) @cython.wraparound(False) @@ -72,3 +74,44 @@ def uint10_uniforms(Py_ssize_t n): randoms = np.asarray(random_values) return randoms +# cython example 3 +def uniforms_ex(bit_generator, Py_ssize_t n, dtype=np.float64): + """ + Create an array of `n` uniformly distributed doubles via a "fill" function. + + A 'real' distribution would want to process the values into + some non-uniform distribution + + Parameters + ---------- + bit_generator: BitGenerator instance + n: int + Output vector length + dtype: {str, dtype}, optional + Desired dtype, either 'd' (or 'float64') or 'f' (or 'float32'). The + default dtype value is 'd' + """ + cdef Py_ssize_t i + cdef bitgen_t *rng + cdef const char *capsule_name = "BitGenerator" + cdef np.ndarray randoms + + capsule = bit_generator.capsule + # Optional check that the capsule if from a BitGenerator + if not PyCapsule_IsValid(capsule, capsule_name): + raise ValueError("Invalid pointer to anon_func_state") + # Cast the pointer + rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name) + + _dtype = np.dtype(dtype) + randoms = np.empty(n, dtype=_dtype) + if _dtype == np.float32: + with bit_generator.lock: + random_standard_uniform_fill_f(rng, n, <float*>np.PyArray_DATA(randoms)) + elif _dtype == np.float64: + with bit_generator.lock: + random_standard_uniform_fill(rng, n, <double*>np.PyArray_DATA(randoms)) + else: + raise TypeError('Unsupported dtype %r for random' % _dtype) + return randoms + diff --git a/numpy/random/_examples/cython/setup.py b/numpy/random/_examples/cython/setup.py index 20cedc4e3..42425c2c1 100644 --- a/numpy/random/_examples/cython/setup.py +++ b/numpy/random/_examples/cython/setup.py @@ -12,7 +12,11 @@ from setuptools.extension import Extension from os.path import join, dirname path = dirname(__file__) +src_dir = join(dirname(path), '..', 'src') defs = [('NPY_NO_DEPRECATED_API', 0)] +inc_path = np.get_include() +# not so nice. We need the random/lib library from numpy +lib_path = join(np.get_include(), '..', '..', 'random', 'lib') extending = Extension("extending", sources=[join(path, 'extending.pyx')], @@ -24,7 +28,9 @@ extending = Extension("extending", ) distributions = Extension("extending_distributions", sources=[join(path, 'extending_distributions.pyx')], - include_dirs=[np.get_include()], + include_dirs=[inc_path], + library_dirs=[lib_path], + libraries=['npyrandom'], define_macros=defs, ) diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index 32eda25f7..27cb2859e 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -11,13 +11,13 @@ import numpy as np cimport numpy as np from numpy.core.multiarray import normalize_axis_index +from .c_distributions cimport * from libc cimport string from libc.stdint cimport (uint8_t, uint16_t, uint32_t, uint64_t, int32_t, int64_t, INT64_MAX, SIZE_MAX) from ._bounded_integers cimport (_rand_bool, _rand_int32, _rand_int64, _rand_int16, _rand_int8, _rand_uint64, _rand_uint32, _rand_uint16, _rand_uint8, _gen_mask) -from ._bounded_integers import _integers_types from ._pcg64 import PCG64 from numpy.random cimport bitgen_t from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE, @@ -27,117 +27,8 @@ from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE, check_array_constraint, check_constraint, disc, discrete_broadcast_iii, ) - -cdef extern from "numpy/random/distributions.h": - - struct s_binomial_t: - int has_binomial - double psave - int64_t nsave - double r - double q - double fm - int64_t m - double p1 - double xm - double xl - double xr - double c - double laml - double lamr - double p2 - double p3 - double p4 - - ctypedef s_binomial_t binomial_t - - double random_standard_uniform(bitgen_t *bitgen_state) nogil - void random_standard_uniform_fill(bitgen_t* bitgen_state, np.npy_intp cnt, double *out) nogil - double random_standard_exponential(bitgen_t *bitgen_state) nogil - double random_standard_exponential_f(bitgen_t *bitgen_state) nogil - void random_standard_exponential_fill(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil - void random_standard_exponential_fill_f(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil - void random_standard_exponential_inv_fill(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil - void random_standard_exponential_inv_fill_f(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil - double random_standard_normal(bitgen_t* bitgen_state) nogil - void random_standard_normal_fill(bitgen_t *bitgen_state, np.npy_intp count, double *out) nogil - void random_standard_normal_fill_f(bitgen_t *bitgen_state, np.npy_intp count, float *out) nogil - double random_standard_gamma(bitgen_t *bitgen_state, double shape) nogil - - float random_standard_uniform_f(bitgen_t *bitgen_state) nogil - void random_standard_uniform_fill_f(bitgen_t* bitgen_state, np.npy_intp cnt, float *out) nogil - float random_standard_normal_f(bitgen_t* bitgen_state) nogil - float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) nogil - - int64_t random_positive_int64(bitgen_t *bitgen_state) nogil - int32_t random_positive_int32(bitgen_t *bitgen_state) nogil - int64_t random_positive_int(bitgen_t *bitgen_state) nogil - uint64_t random_uint(bitgen_t *bitgen_state) nogil - - double random_normal(bitgen_t *bitgen_state, double loc, double scale) nogil - - double random_gamma(bitgen_t *bitgen_state, double shape, double scale) nogil - float random_gamma_f(bitgen_t *bitgen_state, float shape, float scale) nogil - - double random_exponential(bitgen_t *bitgen_state, double scale) nogil - double random_uniform(bitgen_t *bitgen_state, double lower, double range) nogil - double random_beta(bitgen_t *bitgen_state, double a, double b) nogil - double random_chisquare(bitgen_t *bitgen_state, double df) nogil - double random_f(bitgen_t *bitgen_state, double dfnum, double dfden) nogil - double random_standard_cauchy(bitgen_t *bitgen_state) nogil - double random_pareto(bitgen_t *bitgen_state, double a) nogil - double random_weibull(bitgen_t *bitgen_state, double a) nogil - double random_power(bitgen_t *bitgen_state, double a) nogil - double random_laplace(bitgen_t *bitgen_state, double loc, double scale) nogil - double random_gumbel(bitgen_t *bitgen_state, double loc, double scale) nogil - double random_logistic(bitgen_t *bitgen_state, double loc, double scale) nogil - double random_lognormal(bitgen_t *bitgen_state, double mean, double sigma) nogil - double random_rayleigh(bitgen_t *bitgen_state, double mode) nogil - double random_standard_t(bitgen_t *bitgen_state, double df) nogil - double random_noncentral_chisquare(bitgen_t *bitgen_state, double df, - double nonc) nogil - double random_noncentral_f(bitgen_t *bitgen_state, double dfnum, - double dfden, double nonc) nogil - double random_wald(bitgen_t *bitgen_state, double mean, double scale) nogil - double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) nogil - double random_triangular(bitgen_t *bitgen_state, double left, double mode, - double right) nogil - - int64_t random_poisson(bitgen_t *bitgen_state, double lam) nogil - int64_t random_negative_binomial(bitgen_t *bitgen_state, double n, double p) nogil - int64_t random_binomial(bitgen_t *bitgen_state, double p, int64_t n, binomial_t *binomial) nogil - int64_t random_logseries(bitgen_t *bitgen_state, double p) nogil - int64_t random_geometric_search(bitgen_t *bitgen_state, double p) nogil - int64_t random_geometric_inversion(bitgen_t *bitgen_state, double p) nogil - int64_t random_geometric(bitgen_t *bitgen_state, double p) nogil - int64_t random_zipf(bitgen_t *bitgen_state, double a) nogil - int64_t random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, - int64_t sample) nogil - - uint64_t random_interval(bitgen_t *bitgen_state, uint64_t max) nogil - - # Generate random uint64 numbers in closed interval [off, off + rng]. - uint64_t random_bounded_uint64(bitgen_t *bitgen_state, - uint64_t off, uint64_t rng, - uint64_t mask, bint use_masked) nogil - - void random_multinomial(bitgen_t *bitgen_state, int64_t n, int64_t *mnix, - double *pix, np.npy_intp d, binomial_t *binomial) nogil - - int random_multivariate_hypergeometric_count(bitgen_t *bitgen_state, - int64_t total, - size_t num_colors, int64_t *colors, - int64_t nsample, - size_t num_variates, int64_t *variates) nogil - void random_multivariate_hypergeometric_marginals(bitgen_t *bitgen_state, - int64_t total, - size_t num_colors, int64_t *colors, - int64_t nsample, - size_t num_variates, int64_t *variates) nogil - np.import_array() - cdef int64_t _safe_sum_nonneg_int64(size_t num_colors, int64_t *colors): """ Sum the values in the array `colors`. @@ -262,7 +153,7 @@ cdef class Generator: def random(self, size=None, dtype=np.float64, out=None): """ - random(size=None, dtype='d', out=None) + random(size=None, dtype=np.float64, out=None) Return random floats in the half-open interval [0.0, 1.0). @@ -278,10 +169,9 @@ cdef class Generator: Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. Default is None, in which case a single value is returned. - dtype : {str, dtype}, optional - Desired dtype of the result, either 'd' (or 'float64') or 'f' - (or 'float32'). All dtypes are determined by their name. The - default value is 'd'. + dtype : dtype, optional + Desired dtype of the result, only `float64` and `float32` are supported. + Byteorder must be native. The default value is np.float64. out : ndarray, optional Alternative output array in which to place the result. If size is not None, it must have the same shape as the provided size and must match the type of @@ -312,13 +202,13 @@ cdef class Generator: """ cdef double temp - key = np.dtype(dtype).name - if key == 'float64': + _dtype = np.dtype(dtype) + if _dtype == np.float64: return double_fill(&random_standard_uniform_fill, &self._bitgen, size, self.lock, out) - elif key == 'float32': + elif _dtype == np.float32: return float_fill(&random_standard_uniform_fill_f, &self._bitgen, size, self.lock, out) else: - raise TypeError('Unsupported dtype "%s" for random' % key) + raise TypeError('Unsupported dtype %r for random' % _dtype) def beta(self, a, b, size=None): """ @@ -417,7 +307,7 @@ cdef class Generator: def standard_exponential(self, size=None, dtype=np.float64, method=u'zig', out=None): """ - standard_exponential(size=None, dtype='d', method='zig', out=None) + standard_exponential(size=None, dtype=np.float64, method='zig', out=None) Draw samples from the standard exponential distribution. @@ -431,9 +321,8 @@ cdef class Generator: ``m * n * k`` samples are drawn. Default is None, in which case a single value is returned. dtype : dtype, optional - Desired dtype of the result, either 'd' (or 'float64') or 'f' - (or 'float32'). All dtypes are determined by their name. The - default value is 'd'. + Desired dtype of the result, only `float64` and `float32` are supported. + Byteorder must be native. The default value is np.float64. method : str, optional Either 'inv' or 'zig'. 'inv' uses the default inverse CDF method. 'zig' uses the much faster Ziggurat method of Marsaglia and Tsang. @@ -454,24 +343,24 @@ cdef class Generator: >>> n = np.random.default_rng().standard_exponential((3, 8000)) """ - key = np.dtype(dtype).name - if key == 'float64': + _dtype = np.dtype(dtype) + if _dtype == np.float64: if method == u'zig': return double_fill(&random_standard_exponential_fill, &self._bitgen, size, self.lock, out) else: return double_fill(&random_standard_exponential_inv_fill, &self._bitgen, size, self.lock, out) - elif key == 'float32': + elif _dtype == np.float32: if method == u'zig': return float_fill(&random_standard_exponential_fill_f, &self._bitgen, size, self.lock, out) else: return float_fill(&random_standard_exponential_inv_fill_f, &self._bitgen, size, self.lock, out) else: - raise TypeError('Unsupported dtype "%s" for standard_exponential' - % key) + raise TypeError('Unsupported dtype %r for standard_exponential' + % _dtype) def integers(self, low, high=None, size=None, dtype=np.int64, endpoint=False): """ - integers(low, high=None, size=None, dtype='int64', endpoint=False) + integers(low, high=None, size=None, dtype=np.int64, endpoint=False) Return random integers from `low` (inclusive) to `high` (exclusive), or if endpoint=True, `low` (inclusive) to `high` (inclusive). Replaces @@ -496,11 +385,9 @@ cdef class Generator: Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. Default is None, in which case a single value is returned. - dtype : {str, dtype}, optional - Desired dtype of the result. All dtypes are determined by their - name, i.e., 'int64', 'int', etc, so byteorder is not available - and a specific precision may have different C types depending - on the platform. The default value is `np.int_`. + dtype : dtype, optional + Desired dtype of the result. Byteorder must be native. + The default value is np.int64. endpoint : bool, optional If true, sample from the interval [low, high] instead of the default [low, high) @@ -559,39 +446,39 @@ cdef class Generator: high = low low = 0 - dt = np.dtype(dtype) - key = dt.name - if key not in _integers_types: - raise TypeError('Unsupported dtype "%s" for integers' % key) - if not dt.isnative: - raise ValueError('Providing a dtype with a non-native byteorder ' - 'is not supported. If you require ' - 'platform-independent byteorder, call byteswap ' - 'when required.') + _dtype = np.dtype(dtype) # Implementation detail: the old API used a masked method to generate # bounded uniform integers. Lemire's method is preferable since it is # faster. randomgen allows a choice, we will always use the faster one. cdef bint _masked = False - if key == 'int32': + if _dtype == np.int32: ret = _rand_int32(low, high, size, _masked, endpoint, &self._bitgen, self.lock) - elif key == 'int64': + elif _dtype == np.int64: ret = _rand_int64(low, high, size, _masked, endpoint, &self._bitgen, self.lock) - elif key == 'int16': + elif _dtype == np.int16: ret = _rand_int16(low, high, size, _masked, endpoint, &self._bitgen, self.lock) - elif key == 'int8': + elif _dtype == np.int8: ret = _rand_int8(low, high, size, _masked, endpoint, &self._bitgen, self.lock) - elif key == 'uint64': + elif _dtype == np.uint64: ret = _rand_uint64(low, high, size, _masked, endpoint, &self._bitgen, self.lock) - elif key == 'uint32': + elif _dtype == np.uint32: ret = _rand_uint32(low, high, size, _masked, endpoint, &self._bitgen, self.lock) - elif key == 'uint16': + elif _dtype == np.uint16: ret = _rand_uint16(low, high, size, _masked, endpoint, &self._bitgen, self.lock) - elif key == 'uint8': + elif _dtype == np.uint8: ret = _rand_uint8(low, high, size, _masked, endpoint, &self._bitgen, self.lock) - elif key == 'bool': + elif _dtype == np.bool_: ret = _rand_bool(low, high, size, _masked, endpoint, &self._bitgen, self.lock) + elif not _dtype.isnative: + raise ValueError('Providing a dtype with a non-native byteorder ' + 'is not supported. If you require ' + 'platform-independent byteorder, call byteswap ' + 'when required.') + else: + raise TypeError('Unsupported dtype %r for integers' % _dtype) + if size is None and dtype in (bool, int, np.compat.long): if np.array(ret).shape == (): @@ -980,7 +867,7 @@ cdef class Generator: # Complicated, continuous distributions: def standard_normal(self, size=None, dtype=np.float64, out=None): """ - standard_normal(size=None, dtype='d', out=None) + standard_normal(size=None, dtype=np.float64, out=None) Draw samples from a standard Normal distribution (mean=0, stdev=1). @@ -990,10 +877,9 @@ cdef class Generator: Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. Default is None, in which case a single value is returned. - dtype : {str, dtype}, optional - Desired dtype of the result, either 'd' (or 'float64') or 'f' - (or 'float32'). All dtypes are determined by their name. The - default value is 'd'. + dtype : dtype, optional + Desired dtype of the result, only `float64` and `float32` are supported. + Byteorder must be native. The default value is np.float64. out : ndarray, optional Alternative output array in which to place the result. If size is not None, it must have the same shape as the provided size and must match the type of @@ -1041,14 +927,13 @@ cdef class Generator: [ 0.39924804, 4.68456316, 4.99394529, 4.84057254]]) # random """ - key = np.dtype(dtype).name - if key == 'float64': + _dtype = np.dtype(dtype) + if _dtype == np.float64: return double_fill(&random_standard_normal_fill, &self._bitgen, size, self.lock, out) - elif key == 'float32': + elif _dtype == np.float32: return float_fill(&random_standard_normal_fill_f, &self._bitgen, size, self.lock, out) - else: - raise TypeError('Unsupported dtype "%s" for standard_normal' % key) + raise TypeError('Unsupported dtype %r for standard_normal' % _dtype) def normal(self, loc=0.0, scale=1.0, size=None): """ @@ -1154,7 +1039,7 @@ cdef class Generator: def standard_gamma(self, shape, size=None, dtype=np.float64, out=None): """ - standard_gamma(shape, size=None, dtype='d', out=None) + standard_gamma(shape, size=None, dtype=np.float64, out=None) Draw samples from a standard Gamma distribution. @@ -1170,10 +1055,9 @@ cdef class Generator: ``m * n * k`` samples are drawn. If size is ``None`` (default), a single value is returned if ``shape`` is a scalar. Otherwise, ``np.array(shape).size`` samples are drawn. - dtype : {str, dtype}, optional - Desired dtype of the result, either 'd' (or 'float64') or 'f' - (or 'float32'). All dtypes are determined by their name. The - default value is 'd'. + dtype : dtype, optional + Desired dtype of the result, only `float64` and `float32` are supported. + Byteorder must be native. The default value is np.float64. out : ndarray, optional Alternative output array in which to place the result. If size is not None, it must have the same shape as the provided size and @@ -1230,19 +1114,19 @@ cdef class Generator: """ cdef void *func - key = np.dtype(dtype).name - if key == 'float64': + _dtype = np.dtype(dtype) + if _dtype == np.float64: return cont(&random_standard_gamma, &self._bitgen, size, self.lock, 1, shape, 'shape', CONS_NON_NEGATIVE, 0.0, '', CONS_NONE, 0.0, '', CONS_NONE, out) - if key == 'float32': + if _dtype == np.float32: return cont_f(&random_standard_gamma_f, &self._bitgen, size, self.lock, shape, 'shape', CONS_NON_NEGATIVE, out) else: - raise TypeError('Unsupported dtype "%s" for standard_gamma' % key) + raise TypeError('Unsupported dtype %r for standard_gamma' % _dtype) def gamma(self, shape, scale=1.0, size=None): """ @@ -2960,14 +2844,14 @@ cdef class Generator: Samples are drawn from a negative binomial distribution with specified parameters, `n` successes and `p` probability of success where `n` - is > 0 and `p` is in the interval [0, 1]. + is > 0 and `p` is in the interval (0, 1]. Parameters ---------- n : float or array_like of floats Parameter of the distribution, > 0. p : float or array_like of floats - Parameter of the distribution, >= 0 and <=1. + Parameter of the distribution. Must satisfy 0 < p <= 1. size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. If size is ``None`` (default), @@ -3025,7 +2909,7 @@ cdef class Generator: """ return disc(&random_negative_binomial, &self._bitgen, size, self.lock, 2, 0, n, 'n', CONS_POSITIVE_NOT_NAN, - p, 'p', CONS_BOUNDED_0_1, + p, 'p', CONS_BOUNDED_GT_0_1, 0.0, '', CONS_NONE) def poisson(self, lam=1.0, size=None): @@ -3653,10 +3537,9 @@ cdef class Generator: # approximately zero or when the covariance is not positive-semidefinite _factor = u * np.sqrt(abs(s)) else: - _factor = np.sqrt(s)[:, None] * vh + _factor = u * np.sqrt(s) - x = np.dot(x, _factor) - x += mean + x = mean + x @ _factor.T x.shape = tuple(final_shape) return x @@ -3759,8 +3642,8 @@ cdef class Generator: d = len(pvals) on = <np.ndarray>np.PyArray_FROM_OTF(n, np.NPY_INT64, np.NPY_ALIGNED) - parr = <np.ndarray>np.PyArray_FROM_OTF( - pvals, np.NPY_DOUBLE, np.NPY_ALIGNED | np.NPY_ARRAY_C_CONTIGUOUS) + parr = <np.ndarray>np.PyArray_FROMANY( + pvals, np.NPY_DOUBLE, 1, 1, np.NPY_ARRAY_ALIGNED | np.NPY_ARRAY_C_CONTIGUOUS) pix = <double*>np.PyArray_DATA(parr) check_array_constraint(parr, 'pvals', CONS_BOUNDED_0_1) if kahan_sum(pix, d-1) > (1.0 + 1e-12): @@ -4040,23 +3923,23 @@ cdef class Generator: Parameters ---------- - alpha : array - Parameter of the distribution (k dimension for sample of - dimension k). + alpha : sequence of floats, length k + Parameter of the distribution (length ``k`` for sample of + length ``k``). size : int or tuple of ints, optional - Output shape. If the given shape is, e.g., ``(m, n, k)``, then + Output shape. If the given shape is, e.g., ``(m, n)``, then ``m * n * k`` samples are drawn. Default is None, in which case a - single value is returned. + vector of length ``k`` is returned. Returns ------- samples : ndarray, - The drawn samples, of shape (size, alpha.ndim). + The drawn samples, of shape ``(size, k)``. Raises ------- ValueError - If any value in alpha is less than or equal to zero + If any value in ``alpha`` is less than or equal to zero Notes ----- @@ -4124,14 +4007,17 @@ cdef class Generator: # return val cdef np.npy_intp k, totsize, i, j - cdef np.ndarray alpha_arr, val_arr + cdef np.ndarray alpha_arr, val_arr, alpha_csum_arr + cdef double csum cdef double *alpha_data + cdef double *alpha_csum_data cdef double *val_data - cdef double acc, invacc + cdef double acc, invacc, v k = len(alpha) - alpha_arr = <np.ndarray>np.PyArray_FROM_OTF( - alpha, np.NPY_DOUBLE, np.NPY_ALIGNED | np.NPY_ARRAY_C_CONTIGUOUS) + alpha_arr = <np.ndarray>np.PyArray_FROMANY( + alpha, np.NPY_DOUBLE, 1, 1, + np.NPY_ARRAY_ALIGNED | np.NPY_ARRAY_C_CONTIGUOUS) if np.any(np.less_equal(alpha_arr, 0)): raise ValueError('alpha <= 0') alpha_data = <double*>np.PyArray_DATA(alpha_arr) @@ -4150,17 +4036,74 @@ cdef class Generator: i = 0 totsize = np.PyArray_SIZE(val_arr) - with self.lock, nogil: - while i < totsize: - acc = 0.0 - for j in range(k): - val_data[i+j] = random_standard_gamma(&self._bitgen, - alpha_data[j]) - acc = acc + val_data[i + j] - invacc = 1/acc - for j in range(k): - val_data[i + j] = val_data[i + j] * invacc - i = i + k + + # Select one of the following two algorithms for the generation + # of Dirichlet random variates (RVs) + # + # A) Small alpha case: Use the stick-breaking approach with beta + # random variates (RVs). + # B) Standard case: Perform unit normalisation of a vector + # of gamma random variates + # + # A) prevents NaNs resulting from 0/0 that may occur in B) + # when all values in the vector ':math:\\alpha' are smaller + # than 1, then there is a nonzero probability that all + # generated gamma RVs will be 0. When that happens, the + # normalization process ends up computing 0/0, giving nan. A) + # does not use divisions, so that a situation in which 0/0 has + # to be computed cannot occur. A) is slower than B) as + # generation of beta RVs is slower than generation of gamma + # RVs. A) is selected whenever `alpha.max() < t`, where `t < + # 1` is a threshold that controls the probability of + # generating a NaN value when B) is used. For a given + # threshold `t` this probability can be bounded by + # `gammainc(t, d)` where `gammainc` is the regularized + # incomplete gamma function and `d` is the smallest positive + # floating point number that can be represented with a given + # precision. For the chosen threshold `t=0.1` this probability + # is smaller than `1.8e-31` for double precision floating + # point numbers. + + if (k > 0) and (alpha_arr.max() < 0.1): + # Small alpha case: Use stick-breaking approach with beta + # random variates (RVs). + # alpha_csum_data will hold the cumulative sum, right to + # left, of alpha_arr. + # Use a numpy array for memory management only. We could just as + # well have malloc'd alpha_csum_data. alpha_arr is a C-contiguous + # double array, therefore so is alpha_csum_arr. + alpha_csum_arr = np.empty_like(alpha_arr) + alpha_csum_data = <double*>np.PyArray_DATA(alpha_csum_arr) + csum = 0.0 + for j in range(k - 1, -1, -1): + csum += alpha_data[j] + alpha_csum_data[j] = csum + + with self.lock, nogil: + while i < totsize: + acc = 1. + for j in range(k - 1): + v = random_beta(&self._bitgen, alpha_data[j], + alpha_csum_data[j + 1]) + val_data[i + j] = acc * v + acc *= (1. - v) + val_data[i + k - 1] = acc + i = i + k + + else: + # Standard case: Unit normalisation of a vector of gamma random + # variates + with self.lock, nogil: + while i < totsize: + acc = 0. + for j in range(k): + val_data[i + j] = random_standard_gamma(&self._bitgen, + alpha_data[j]) + acc = acc + val_data[i + j] + invacc = 1. / acc + for j in range(k): + val_data[i + j] = val_data[i + j] * invacc + i = i + k return diric @@ -4345,7 +4288,7 @@ cdef class Generator: >>> rng.permutation("abc") Traceback (most recent call last): ... - numpy.AxisError: x must be an integer or at least 1-dimensional + numpy.AxisError: axis 0 is out of bounds for array of dimension 0 >>> arr = np.arange(9).reshape((3, 3)) >>> rng.permutation(arr, axis=1) diff --git a/numpy/random/_pcg64.pyx b/numpy/random/_pcg64.pyx index 05d27db5c..605aae4bc 100644 --- a/numpy/random/_pcg64.pyx +++ b/numpy/random/_pcg64.pyx @@ -38,7 +38,7 @@ cdef double pcg64_double(void* st) nogil: cdef class PCG64(BitGenerator): """ - PCG64(seed_seq=None) + PCG64(seed=None) BitGenerator for the PCG-64 pseudo-random number generator. diff --git a/numpy/random/_bit_generator.pxd b/numpy/random/bit_generator.pxd index bd5e47a20..bd5e47a20 100644 --- a/numpy/random/_bit_generator.pxd +++ b/numpy/random/bit_generator.pxd diff --git a/numpy/random/_bit_generator.pyx b/numpy/random/bit_generator.pyx index 21d21e6bb..f145ec13d 100644 --- a/numpy/random/_bit_generator.pyx +++ b/numpy/random/bit_generator.pyx @@ -114,7 +114,7 @@ def _coerce_to_uint32_array(x): Examples -------- >>> import numpy as np - >>> from numpy.random._bit_generator import _coerce_to_uint32_array + >>> from numpy.random.bit_generator import _coerce_to_uint32_array >>> _coerce_to_uint32_array(12345) array([12345], dtype=uint32) >>> _coerce_to_uint32_array('12345') diff --git a/numpy/random/c_distributions.pxd b/numpy/random/c_distributions.pxd new file mode 100644 index 000000000..6f905edc1 --- /dev/null +++ b/numpy/random/c_distributions.pxd @@ -0,0 +1,114 @@ +#!python +#cython: wraparound=False, nonecheck=False, boundscheck=False, cdivision=True, language_level=3 +from numpy cimport npy_intp + +from libc.stdint cimport (uint64_t, int32_t, int64_t) +from numpy.random cimport bitgen_t + +cdef extern from "numpy/random/distributions.h": + + struct s_binomial_t: + int has_binomial + double psave + int64_t nsave + double r + double q + double fm + int64_t m + double p1 + double xm + double xl + double xr + double c + double laml + double lamr + double p2 + double p3 + double p4 + + ctypedef s_binomial_t binomial_t + + double random_standard_uniform(bitgen_t *bitgen_state) nogil + void random_standard_uniform_fill(bitgen_t* bitgen_state, npy_intp cnt, double *out) nogil + double random_standard_exponential(bitgen_t *bitgen_state) nogil + double random_standard_exponential_f(bitgen_t *bitgen_state) nogil + void random_standard_exponential_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) nogil + void random_standard_exponential_fill_f(bitgen_t *bitgen_state, npy_intp cnt, double *out) nogil + void random_standard_exponential_inv_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) nogil + void random_standard_exponential_inv_fill_f(bitgen_t *bitgen_state, npy_intp cnt, double *out) nogil + double random_standard_normal(bitgen_t* bitgen_state) nogil + void random_standard_normal_fill(bitgen_t *bitgen_state, npy_intp count, double *out) nogil + void random_standard_normal_fill_f(bitgen_t *bitgen_state, npy_intp count, float *out) nogil + double random_standard_gamma(bitgen_t *bitgen_state, double shape) nogil + + float random_standard_uniform_f(bitgen_t *bitgen_state) nogil + void random_standard_uniform_fill_f(bitgen_t* bitgen_state, npy_intp cnt, float *out) nogil + float random_standard_normal_f(bitgen_t* bitgen_state) nogil + float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) nogil + + int64_t random_positive_int64(bitgen_t *bitgen_state) nogil + int32_t random_positive_int32(bitgen_t *bitgen_state) nogil + int64_t random_positive_int(bitgen_t *bitgen_state) nogil + uint64_t random_uint(bitgen_t *bitgen_state) nogil + + double random_normal(bitgen_t *bitgen_state, double loc, double scale) nogil + + double random_gamma(bitgen_t *bitgen_state, double shape, double scale) nogil + float random_gamma_f(bitgen_t *bitgen_state, float shape, float scale) nogil + + double random_exponential(bitgen_t *bitgen_state, double scale) nogil + double random_uniform(bitgen_t *bitgen_state, double lower, double range) nogil + double random_beta(bitgen_t *bitgen_state, double a, double b) nogil + double random_chisquare(bitgen_t *bitgen_state, double df) nogil + double random_f(bitgen_t *bitgen_state, double dfnum, double dfden) nogil + double random_standard_cauchy(bitgen_t *bitgen_state) nogil + double random_pareto(bitgen_t *bitgen_state, double a) nogil + double random_weibull(bitgen_t *bitgen_state, double a) nogil + double random_power(bitgen_t *bitgen_state, double a) nogil + double random_laplace(bitgen_t *bitgen_state, double loc, double scale) nogil + double random_gumbel(bitgen_t *bitgen_state, double loc, double scale) nogil + double random_logistic(bitgen_t *bitgen_state, double loc, double scale) nogil + double random_lognormal(bitgen_t *bitgen_state, double mean, double sigma) nogil + double random_rayleigh(bitgen_t *bitgen_state, double mode) nogil + double random_standard_t(bitgen_t *bitgen_state, double df) nogil + double random_noncentral_chisquare(bitgen_t *bitgen_state, double df, + double nonc) nogil + double random_noncentral_f(bitgen_t *bitgen_state, double dfnum, + double dfden, double nonc) nogil + double random_wald(bitgen_t *bitgen_state, double mean, double scale) nogil + double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) nogil + double random_triangular(bitgen_t *bitgen_state, double left, double mode, + double right) nogil + + int64_t random_poisson(bitgen_t *bitgen_state, double lam) nogil + int64_t random_negative_binomial(bitgen_t *bitgen_state, double n, double p) nogil + int64_t random_binomial(bitgen_t *bitgen_state, double p, int64_t n, binomial_t *binomial) nogil + int64_t random_logseries(bitgen_t *bitgen_state, double p) nogil + int64_t random_geometric_search(bitgen_t *bitgen_state, double p) nogil + int64_t random_geometric_inversion(bitgen_t *bitgen_state, double p) nogil + int64_t random_geometric(bitgen_t *bitgen_state, double p) nogil + int64_t random_zipf(bitgen_t *bitgen_state, double a) nogil + int64_t random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, + int64_t sample) nogil + + uint64_t random_interval(bitgen_t *bitgen_state, uint64_t max) nogil + + # Generate random uint64 numbers in closed interval [off, off + rng]. + uint64_t random_bounded_uint64(bitgen_t *bitgen_state, + uint64_t off, uint64_t rng, + uint64_t mask, bint use_masked) nogil + + void random_multinomial(bitgen_t *bitgen_state, int64_t n, int64_t *mnix, + double *pix, npy_intp d, binomial_t *binomial) nogil + + int random_multivariate_hypergeometric_count(bitgen_t *bitgen_state, + int64_t total, + size_t num_colors, int64_t *colors, + int64_t nsample, + size_t num_variates, int64_t *variates) nogil + void random_multivariate_hypergeometric_marginals(bitgen_t *bitgen_state, + int64_t total, + size_t num_colors, int64_t *colors, + int64_t nsample, + size_t num_variates, int64_t *variates) nogil + diff --git a/numpy/random/mtrand.pyx b/numpy/random/mtrand.pyx index 95921a8ea..0c0c611af 100644 --- a/numpy/random/mtrand.pyx +++ b/numpy/random/mtrand.pyx @@ -15,7 +15,6 @@ from libc.stdint cimport int64_t, uint64_t from ._bounded_integers cimport (_rand_bool, _rand_int32, _rand_int64, _rand_int16, _rand_int8, _rand_uint64, _rand_uint32, _rand_uint16, _rand_uint8,) -from ._bounded_integers import _integers_types from ._mt19937 import MT19937 as _MT19937 from numpy.random cimport bitgen_t from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE, @@ -252,6 +251,12 @@ cdef class RandomState: For more details, see `set_state`. + Parameters + ---------- + legacy : bool, optional + Flag indicating to return a legacy tuple state when the BitGenerator + is MT19937, instead of a dict. + Returns ------- out : {tuple(str, ndarray of 624 uints, int, int, float), dict} @@ -263,13 +268,9 @@ cdef class RandomState: 4. an integer ``has_gauss``. 5. a float ``cached_gaussian``. - If `legacy` is False, or the BitGenerator is not NT19937, then + If `legacy` is False, or the BitGenerator is not MT19937, then state is returned as a dictionary. - legacy : bool - Flag indicating the return a legacy tuple state when the BitGenerator - is MT19937. - See Also -------- set_state @@ -641,7 +642,7 @@ cdef class RandomState: def randint(self, low, high=None, size=None, dtype=int): """ - randint(low, high=None, size=None, dtype='l') + randint(low, high=None, size=None, dtype=int) Return random integers from `low` (inclusive) to `high` (exclusive). @@ -668,10 +669,8 @@ cdef class RandomState: ``m * n * k`` samples are drawn. Default is None, in which case a single value is returned. dtype : dtype, optional - Desired dtype of the result. All dtypes are determined by their - name, i.e., 'int64', 'int', etc, so byteorder is not available - and a specific precision may have different C types depending - on the platform. The default value is `np.int_`. + Desired dtype of the result. Byteorder must be native. + The default value is int. .. versionadded:: 1.11.0 @@ -722,17 +721,16 @@ cdef class RandomState: high = low low = 0 - dt = np.dtype(dtype) - key = dt.name - if key not in _integers_types: - raise TypeError('Unsupported dtype "%s" for randint' % key) - if not dt.isnative: + _dtype = np.dtype(dtype) + + if not _dtype.isnative: # numpy 1.17.0, 2019-05-28 warnings.warn('Providing a dtype with a non-native byteorder is ' 'not supported. If you require platform-independent ' 'byteorder, call byteswap when required.\nIn future ' 'version, providing byteorder will raise a ' 'ValueError', DeprecationWarning) + _dtype = _dtype.newbyteorder() # Implementation detail: the use a masked method to generate # bounded uniform integers. Lemire's method is preferable since it is @@ -741,24 +739,26 @@ cdef class RandomState: cdef bint _masked = True cdef bint _endpoint = False - if key == 'int32': + if _dtype == np.int32: ret = _rand_int32(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) - elif key == 'int64': + elif _dtype == np.int64: ret = _rand_int64(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) - elif key == 'int16': + elif _dtype == np.int16: ret = _rand_int16(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) - elif key == 'int8': + elif _dtype == np.int8: ret = _rand_int8(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) - elif key == 'uint64': + elif _dtype == np.uint64: ret = _rand_uint64(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) - elif key == 'uint32': + elif _dtype == np.uint32: ret = _rand_uint32(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) - elif key == 'uint16': + elif _dtype == np.uint16: ret = _rand_uint16(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) - elif key == 'uint8': + elif _dtype == np.uint8: ret = _rand_uint8(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) - elif key == 'bool': + elif _dtype == np.bool_: ret = _rand_bool(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) + else: + raise TypeError('Unsupported dtype %r for randint' % _dtype) if size is None and dtype in (bool, int, np.compat.long): if np.array(ret).shape == (): @@ -848,6 +848,11 @@ cdef class RandomState: randint, shuffle, permutation Generator.choice: which should be used in new code + Notes + ----- + Sampling random rows from a 2-D array is not possible with this function, + but is possible with `Generator.choice` through its ``axis`` keyword. + Examples -------- Generate a uniform random sample from np.arange(5) of size 3: @@ -948,7 +953,7 @@ cdef class RandomState: raise ValueError("Cannot take a larger sample than " "population when 'replace=False'") elif size < 0: - raise ValueError("negative dimensions are not allowed") + raise ValueError("Negative dimensions are not allowed") if p is not None: if np.count_nonzero(p > 0) < size: @@ -1017,7 +1022,7 @@ cdef class RandomState: greater than or equal to low. The default value is 0. high : float or array_like of floats Upper boundary of the output interval. All values generated will be - less than high. The default value is 1.0. + less than or equal to high. The default value is 1.0. size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. If size is ``None`` (default), @@ -1053,7 +1058,14 @@ cdef class RandomState: If ``high`` < ``low``, the results are officially undefined and may eventually raise an error, i.e. do not rely on this function to behave when passed arguments satisfying that - inequality condition. + inequality condition. The ``high`` limit may be included in the + returned array of floats due to floating-point rounding in the + equation ``low + (high-low) * random_sample()``. For example: + + >>> x = np.float32(5*0.99999999) + >>> x + 5.0 + Examples -------- @@ -4192,8 +4204,8 @@ cdef class RandomState: cdef long ni d = len(pvals) - parr = <np.ndarray>np.PyArray_FROM_OTF( - pvals, np.NPY_DOUBLE, np.NPY_ALIGNED | np.NPY_ARRAY_C_CONTIGUOUS) + parr = <np.ndarray>np.PyArray_FROMANY( + pvals, np.NPY_DOUBLE, 1, 1, np.NPY_ARRAY_ALIGNED | np.NPY_ARRAY_C_CONTIGUOUS) pix = <double*>np.PyArray_DATA(parr) check_array_constraint(parr, 'pvals', CONS_BOUNDED_0_1) if kahan_sum(pix, d-1) > (1.0 + 1e-12): @@ -4239,23 +4251,23 @@ cdef class RandomState: Parameters ---------- - alpha : array - Parameter of the distribution (k dimension for sample of - dimension k). + alpha : sequence of floats, length k + Parameter of the distribution (length ``k`` for sample of + length ``k``). size : int or tuple of ints, optional - Output shape. If the given shape is, e.g., ``(m, n, k)``, then + Output shape. If the given shape is, e.g., ``(m, n)``, then ``m * n * k`` samples are drawn. Default is None, in which case a - single value is returned. + vector of length ``k`` is returned. Returns ------- samples : ndarray, - The drawn samples, of shape (size, alpha.ndim). + The drawn samples, of shape ``(size, k)``. Raises ------- ValueError - If any value in alpha is less than or equal to zero + If any value in ``alpha`` is less than or equal to zero See Also -------- @@ -4333,8 +4345,9 @@ cdef class RandomState: cdef double acc, invacc k = len(alpha) - alpha_arr = <np.ndarray>np.PyArray_FROM_OTF( - alpha, np.NPY_DOUBLE, np.NPY_ALIGNED | np.NPY_ARRAY_C_CONTIGUOUS) + alpha_arr = <np.ndarray>np.PyArray_FROMANY( + alpha, np.NPY_DOUBLE, 1, 1, + np.NPY_ARRAY_ALIGNED | np.NPY_ARRAY_C_CONTIGUOUS) if np.any(np.less_equal(alpha_arr, 0)): raise ValueError('alpha <= 0') alpha_data = <double*>np.PyArray_DATA(alpha_arr) diff --git a/numpy/random/setup.py b/numpy/random/setup.py index 42c00ee5e..90ec42671 100644 --- a/numpy/random/setup.py +++ b/numpy/random/setup.py @@ -35,8 +35,10 @@ def configuration(parent_package='', top_path=None): config.add_data_dir('_examples') EXTRA_LINK_ARGS = [] - # Math lib - EXTRA_LIBRARIES = ['m'] if os.name != 'nt' else [] + EXTRA_LIBRARIES = ['npyrandom'] + if os.name != 'nt': + # Math lib + EXTRA_LIBRARIES.append('m') # Some bit generators exclude GCC inlining EXTRA_COMPILE_ARGS = ['-U__GNUC_GNU_INLINE__'] @@ -52,78 +54,88 @@ def configuration(parent_package='', top_path=None): PCG64_DEFS = [] # One can force emulated 128-bit arithmetic if one wants. #PCG64_DEFS += [('PCG_FORCE_EMULATED_128BIT_MATH', '1')] + depends = ['__init__.pxd', 'c_distributions.pxd', 'bit_generator.pxd'] + + # npyrandom - a library like npymath + npyrandom_sources = [ + 'src/distributions/logfactorial.c', + 'src/distributions/distributions.c', + 'src/distributions/random_mvhg_count.c', + 'src/distributions/random_mvhg_marginals.c', + 'src/distributions/random_hypergeometric.c', + ] + config.add_installed_library('npyrandom', + sources=npyrandom_sources, + install_dir='lib', + build_info={ + 'include_dirs' : [], # empty list required for creating npyrandom.h + 'extra_compiler_args' : (['/GL-'] if is_msvc else []), + }) for gen in ['mt19937']: # gen.pyx, src/gen/gen.c, src/gen/gen-jump.c - config.add_extension('_{0}'.format(gen), - sources=['_{0}.c'.format(gen), - 'src/{0}/{0}.c'.format(gen), - 'src/{0}/{0}-jump.c'.format(gen)], + config.add_extension(f'_{gen}', + sources=[f'_{gen}.c', + f'src/{gen}/{gen}.c', + f'src/{gen}/{gen}-jump.c'], include_dirs=['.', 'src', join('src', gen)], libraries=EXTRA_LIBRARIES, extra_compile_args=EXTRA_COMPILE_ARGS, extra_link_args=EXTRA_LINK_ARGS, - depends=['_%s.pyx' % gen], + depends=depends + [f'_{gen}.pyx'], define_macros=defs, ) for gen in ['philox', 'pcg64', 'sfc64']: # gen.pyx, src/gen/gen.c _defs = defs + PCG64_DEFS if gen == 'pcg64' else defs - config.add_extension('_{0}'.format(gen), - sources=['_{0}.c'.format(gen), - 'src/{0}/{0}.c'.format(gen)], + config.add_extension(f'_{gen}', + sources=[f'_{gen}.c', + f'src/{gen}/{gen}.c'], include_dirs=['.', 'src', join('src', gen)], libraries=EXTRA_LIBRARIES, extra_compile_args=EXTRA_COMPILE_ARGS, extra_link_args=EXTRA_LINK_ARGS, - depends=['_%s.pyx' % gen, '_bit_generator.pyx', - '_bit_generator.pxd'], + depends=depends + [f'_{gen}.pyx', + 'bit_generator.pyx', 'bit_generator.pxd'], define_macros=_defs, ) - for gen in ['_common', '_bit_generator']: + for gen in ['_common', 'bit_generator']: # gen.pyx config.add_extension(gen, - sources=['{0}.c'.format(gen)], + sources=[f'{gen}.c'], libraries=EXTRA_LIBRARIES, extra_compile_args=EXTRA_COMPILE_ARGS, extra_link_args=EXTRA_LINK_ARGS, include_dirs=['.', 'src'], - depends=['%s.pyx' % gen, '%s.pxd' % gen,], + depends=depends + [f'{gen}.pyx', f'{gen}.pxd',], define_macros=defs, ) - config.add_data_files('{0}.pxd'.format(gen)) - other_srcs = [ - 'src/distributions/logfactorial.c', - 'src/distributions/distributions.c', - 'src/distributions/random_mvhg_count.c', - 'src/distributions/random_mvhg_marginals.c', - 'src/distributions/random_hypergeometric.c', - ] + config.add_data_files(f'{gen}.pxd') for gen in ['_generator', '_bounded_integers']: # gen.pyx, src/distributions/distributions.c config.add_extension(gen, - sources=['{0}.c'.format(gen)] + other_srcs, + sources=[f'{gen}.c'], libraries=EXTRA_LIBRARIES, extra_compile_args=EXTRA_COMPILE_ARGS, include_dirs=['.', 'src'], extra_link_args=EXTRA_LINK_ARGS, - depends=['%s.pyx' % gen], + depends=depends + [f'{gen}.pyx'], define_macros=defs, ) config.add_data_files('_bounded_integers.pxd') config.add_extension('mtrand', sources=['mtrand.c', 'src/legacy/legacy-distributions.c', - 'src/distributions/logfactorial.c', - 'src/distributions/distributions.c'], + 'src/distributions/distributions.c', + ], include_dirs=['.', 'src', 'src/legacy'], - libraries=EXTRA_LIBRARIES, + libraries=['m'] if os.name != 'nt' else [], extra_compile_args=EXTRA_COMPILE_ARGS, extra_link_args=EXTRA_LINK_ARGS, - depends=['mtrand.pyx'], + depends=depends + ['mtrand.pyx'], define_macros=defs + LEGACY_DEFS, ) - config.add_data_files('__init__.pxd') + config.add_data_files(*depends) return config diff --git a/numpy/random/src/distributions/distributions.c b/numpy/random/src/distributions/distributions.c index 0b46dc6d8..586e38aa5 100644 --- a/numpy/random/src/distributions/distributions.c +++ b/numpy/random/src/distributions/distributions.c @@ -6,6 +6,8 @@ #include <intrin.h> #endif +#include <assert.h> + /* Inline generators for internal use */ static NPY_INLINE uint32_t next_uint32(bitgen_t *bitgen_state) { return bitgen_state->next_uint32(bitgen_state->state); @@ -1149,6 +1151,8 @@ static NPY_INLINE uint64_t bounded_lemire_uint64(bitgen_t *bitgen_state, */ const uint64_t rng_excl = rng + 1; + assert(rng != 0xFFFFFFFFFFFFFFFFULL); + #if __SIZEOF_INT128__ /* 128-bit uint available (e.g. GCC/clang). `m` is the __uint128_t scaled * integer. */ @@ -1239,6 +1243,8 @@ static NPY_INLINE uint32_t buffered_bounded_lemire_uint32( uint64_t m; uint32_t leftover; + assert(rng != 0xFFFFFFFFUL); + /* Generate a scaled random number. */ m = ((uint64_t)next_uint32(bitgen_state)) * rng_excl; @@ -1273,6 +1279,8 @@ static NPY_INLINE uint16_t buffered_bounded_lemire_uint16( uint32_t m; uint16_t leftover; + assert(rng != 0xFFFFU); + /* Generate a scaled random number. */ m = ((uint32_t)buffered_uint16(bitgen_state, bcnt, buf)) * rng_excl; @@ -1308,6 +1316,9 @@ static NPY_INLINE uint8_t buffered_bounded_lemire_uint8(bitgen_t *bitgen_state, uint16_t m; uint8_t leftover; + assert(rng != 0xFFU); + + /* Generate a scaled random number. */ m = ((uint16_t)buffered_uint8(bitgen_state, bcnt, buf)) * rng_excl; @@ -1337,6 +1348,14 @@ uint64_t random_bounded_uint64(bitgen_t *bitgen_state, uint64_t off, return off; } else if (rng <= 0xFFFFFFFFUL) { /* Call 32-bit generator if range in 32-bit. */ + if (rng == 0xFFFFFFFFUL) { + /* + * The 32-bit Lemire method does not handle rng=0xFFFFFFFF, so we'll + * call next_uint32 directly. This also works when use_masked is True, + * so we handle both cases here. + */ + return off + (uint64_t) next_uint32(bitgen_state); + } if (use_masked) { return off + buffered_bounded_masked_uint32(bitgen_state, rng, mask, NULL, NULL); @@ -1450,22 +1469,34 @@ void random_bounded_uint64_fill(bitgen_t *bitgen_state, uint64_t off, out[i] = off; } } else if (rng <= 0xFFFFFFFFUL) { - uint32_t buf = 0; - int bcnt = 0; - /* Call 32-bit generator if range in 32-bit. */ - if (use_masked) { - /* Smallest bit mask >= max */ - uint64_t mask = gen_mask(rng); + /* + * The 32-bit Lemire method does not handle rng=0xFFFFFFFF, so we'll + * call next_uint32 directly. This also works when use_masked is True, + * so we handle both cases here. + */ + if (rng == 0xFFFFFFFFUL) { for (i = 0; i < cnt; i++) { - out[i] = off + buffered_bounded_masked_uint32(bitgen_state, rng, mask, - &bcnt, &buf); + out[i] = off + (uint64_t) next_uint32(bitgen_state); } } else { - for (i = 0; i < cnt; i++) { - out[i] = off + - buffered_bounded_lemire_uint32(bitgen_state, rng, &bcnt, &buf); + uint32_t buf = 0; + int bcnt = 0; + + if (use_masked) { + /* Smallest bit mask >= max */ + uint64_t mask = gen_mask(rng); + + for (i = 0; i < cnt; i++) { + out[i] = off + buffered_bounded_masked_uint32(bitgen_state, rng, mask, + &bcnt, &buf); + } + } else { + for (i = 0; i < cnt; i++) { + out[i] = off + + buffered_bounded_lemire_uint32(bitgen_state, rng, &bcnt, &buf); + } } } } else if (rng == 0xFFFFFFFFFFFFFFFFULL) { diff --git a/numpy/random/tests/test_direct.py b/numpy/random/tests/test_direct.py index 4fa69a402..dad12c8a8 100644 --- a/numpy/random/tests/test_direct.py +++ b/numpy/random/tests/test_direct.py @@ -127,7 +127,7 @@ def gauss_from_uint(x, n, bits): return gauss[:n] def test_seedsequence(): - from numpy.random._bit_generator import (ISeedSequence, + from numpy.random.bit_generator import (ISeedSequence, ISpawnableSeedSequence, SeedlessSeedSequence) diff --git a/numpy/random/tests/test_extending.py b/numpy/random/tests/test_extending.py index 23a874fb1..f7efafba9 100644 --- a/numpy/random/tests/test_extending.py +++ b/numpy/random/tests/test_extending.py @@ -1,8 +1,10 @@ -import os, sys +import os import pytest -import warnings import shutil import subprocess +import sys +import warnings +import numpy as np try: import cffi @@ -41,16 +43,43 @@ else: @pytest.mark.skipif(cython is None, reason="requires cython") @pytest.mark.slow def test_cython(tmp_path): - examples = os.path.join(os.path.dirname(__file__), '..', '_examples') - shutil.copytree(examples, tmp_path / '_examples') - subprocess.check_call([sys.executable, 'setup.py', 'build'], - cwd=str(tmp_path / '_examples' / 'cython')) + srcdir = os.path.join(os.path.dirname(__file__), '..') + shutil.copytree(srcdir, tmp_path / 'random') + # build the examples and "install" them into a temporary directory + env = os.environ.copy() + subprocess.check_call([sys.executable, 'setup.py', 'build', 'install', + '--prefix', str(tmp_path / 'installdir'), + '--single-version-externally-managed', + '--record', str(tmp_path/ 'tmp_install_log.txt'), + ], + cwd=str(tmp_path / 'random' / '_examples' / 'cython'), + env=env) + # get the path to the so's + so1 = so2 = None + with open(tmp_path /'tmp_install_log.txt') as fid: + for line in fid: + if 'extending.' in line: + so1 = line.strip() + if 'extending_distributions' in line: + so2 = line.strip() + assert so1 is not None + assert so2 is not None + # import the so's without adding the directory to sys.path + from importlib.machinery import ExtensionFileLoader + extending = ExtensionFileLoader('extending', so1).load_module() + extending_distributions = ExtensionFileLoader('extending_distributions', so2).load_module() + + # actually test the cython c-extension + from numpy.random import PCG64 + values = extending_distributions.uniforms_ex(PCG64(0), 10, 'd') + assert values.shape == (10,) + assert values.dtype == np.float64 @pytest.mark.skipif(numba is None or cffi is None, reason="requires numba and cffi") def test_numba(): - from numpy.random._examples.numba import extending + from numpy.random._examples.numba import extending # noqa: F401 @pytest.mark.skipif(cffi is None, reason="requires cffi") def test_cffi(): - from numpy.random._examples.cffi import extending + from numpy.random._examples.cffi import extending # noqa: F401 diff --git a/numpy/random/tests/test_generator_mt19937.py b/numpy/random/tests/test_generator_mt19937.py index 6f4407373..a28b7ca11 100644 --- a/numpy/random/tests/test_generator_mt19937.py +++ b/numpy/random/tests/test_generator_mt19937.py @@ -115,6 +115,12 @@ class TestMultinomial: contig = random.multinomial(100, pvals=np.ascontiguousarray(pvals)) assert_array_equal(non_contig, contig) + def test_multidimensional_pvals(self): + assert_raises(ValueError, random.multinomial, 10, [[0, 1]]) + assert_raises(ValueError, random.multinomial, 10, [[0], [1]]) + assert_raises(ValueError, random.multinomial, 10, [[[0], [1]], [[1], [0]]]) + assert_raises(ValueError, random.multinomial, 10, np.array([[0, 1], [1, 0]])) + class TestMultivariateHypergeometric: @@ -514,6 +520,44 @@ class TestIntegers: assert_array_equal(val, val_bc) + @pytest.mark.parametrize( + 'bound, expected', + [(2**32 - 1, np.array([517043486, 1364798665, 1733884389, 1353720612, + 3769704066, 1170797179, 4108474671])), + (2**32, np.array([517043487, 1364798666, 1733884390, 1353720613, + 3769704067, 1170797180, 4108474672])), + (2**32 + 1, np.array([517043487, 1733884390, 3769704068, 4108474673, + 1831631863, 1215661561, 3869512430]))] + ) + def test_repeatability_32bit_boundary(self, bound, expected): + for size in [None, len(expected)]: + random = Generator(MT19937(1234)) + x = random.integers(bound, size=size) + assert_equal(x, expected if size is not None else expected[0]) + + def test_repeatability_32bit_boundary_broadcasting(self): + desired = np.array([[[1622936284, 3620788691, 1659384060], + [1417365545, 760222891, 1909653332], + [3788118662, 660249498, 4092002593]], + [[3625610153, 2979601262, 3844162757], + [ 685800658, 120261497, 2694012896], + [1207779440, 1586594375, 3854335050]], + [[3004074748, 2310761796, 3012642217], + [2067714190, 2786677879, 1363865881], + [ 791663441, 1867303284, 2169727960]], + [[1939603804, 1250951100, 298950036], + [1040128489, 3791912209, 3317053765], + [3155528714, 61360675, 2305155588]], + [[ 817688762, 1335621943, 3288952434], + [1770890872, 1102951817, 1957607470], + [3099996017, 798043451, 48334215]]]) + for size in [None, (5, 3, 3)]: + random = Generator(MT19937(12345)) + x = random.integers([[-1], [0], [1]], + [2**32 - 1, 2**32, 2**32 + 1], + size=size) + assert_array_equal(x, desired if size is not None else desired[0]) + def test_int64_uint64_broadcast_exceptions(self, endpoint): configs = {np.uint64: ((0, 2**65), (-1, 2**62), (10, 9), (0, 0)), np.int64: ((0, 2**64), (-(2**64), 2**62), (10, 9), (0, 0), @@ -1043,6 +1087,12 @@ class TestRandomDist: alpha = np.array([5.4e-01, -1.0e-16]) assert_raises(ValueError, random.dirichlet, alpha) + # gh-15876 + assert_raises(ValueError, random.dirichlet, [[5, 1]]) + assert_raises(ValueError, random.dirichlet, [[5], [1]]) + assert_raises(ValueError, random.dirichlet, [[[5], [1]], [[1], [5]]]) + assert_raises(ValueError, random.dirichlet, np.array([[5, 1], [1, 5]])) + def test_dirichlet_alpha_non_contiguous(self): a = np.array([51.72840233779265162, -1.0, 39.74494232180943953]) alpha = a[::2] @@ -1053,6 +1103,31 @@ class TestRandomDist: size=(3, 2)) assert_array_almost_equal(non_contig, contig) + def test_dirichlet_small_alpha(self): + eps = 1.0e-9 # 1.0e-10 -> runtime x 10; 1e-11 -> runtime x 200, etc. + alpha = eps * np.array([1., 1.0e-3]) + random = Generator(MT19937(self.seed)) + actual = random.dirichlet(alpha, size=(3, 2)) + expected = np.array([ + [[1., 0.], + [1., 0.]], + [[1., 0.], + [1., 0.]], + [[1., 0.], + [1., 0.]] + ]) + assert_array_almost_equal(actual, expected, decimal=15) + + @pytest.mark.slow + def test_dirichlet_moderately_small_alpha(self): + # Use alpha.max() < 0.1 to trigger stick breaking code path + alpha = np.array([0.02, 0.04, 0.03]) + exact_mean = alpha / alpha.sum() + random = Generator(MT19937(self.seed)) + sample = random.dirichlet(alpha, size=20000000) + sample_mean = sample.mean(axis=0) + assert_allclose(sample_mean, exact_mean, rtol=1e-3) + def test_exponential(self): random = Generator(MT19937(self.seed)) actual = random.exponential(1.1234, size=(3, 2)) @@ -1242,6 +1317,17 @@ class TestRandomDist: assert_raises(ValueError, random.multivariate_normal, mean, cov, check_valid='raise', method='eigh') + # check degenerate samples from singular covariance matrix + cov = [[1, 1], [1, 1]] + if method in ('svd', 'eigh'): + samples = random.multivariate_normal(mean, cov, size=(3, 2), + method=method) + assert_array_almost_equal(samples[..., 0], samples[..., 1], + decimal=6) + else: + assert_raises(LinAlgError, random.multivariate_normal, mean, cov, + method='cholesky') + cov = np.array([[1, 0.1], [0.1, 1]], dtype=np.float32) with suppress_warnings() as sup: random.multivariate_normal(mean, cov, method=method) @@ -1259,6 +1345,19 @@ class TestRandomDist: assert_raises(ValueError, random.multivariate_normal, mu, np.eye(3)) + @pytest.mark.parametrize("method", ["svd", "eigh", "cholesky"]) + def test_multivariate_normal_basic_stats(self, method): + random = Generator(MT19937(self.seed)) + n_s = 1000 + mean = np.array([1, 2]) + cov = np.array([[2, 1], [1, 2]]) + s = random.multivariate_normal(mean, cov, size=(n_s,), method=method) + s_center = s - mean + cov_emp = (s_center.T @ s_center) / (n_s - 1) + # these are pretty loose and are only designed to detect major errors + assert np.all(np.abs(s_center.mean(-2)) < 0.1) + assert np.all(np.abs(cov_emp - cov) < 0.2) + def test_negative_binomial(self): random = Generator(MT19937(self.seed)) actual = random.negative_binomial(n=100, p=.12345, size=(3, 2)) @@ -1273,6 +1372,11 @@ class TestRandomDist: assert_raises(ValueError, random.negative_binomial, 100, [np.nan] * 10) + def test_negative_binomial_p0_exception(self): + # Verify that p=0 raises an exception. + with assert_raises(ValueError): + x = random.negative_binomial(1, 0) + def test_noncentral_chisquare(self): random = Generator(MT19937(self.seed)) actual = random.noncentral_chisquare(df=5, nonc=5, size=(3, 2)) diff --git a/numpy/random/tests/test_generator_mt19937_regressions.py b/numpy/random/tests/test_generator_mt19937_regressions.py index f0b502d06..456c932d4 100644 --- a/numpy/random/tests/test_generator_mt19937_regressions.py +++ b/numpy/random/tests/test_generator_mt19937_regressions.py @@ -1,5 +1,4 @@ from numpy.testing import (assert_, assert_array_equal) -from numpy.compat import long import numpy as np import pytest from numpy.random import Generator, MT19937 @@ -41,13 +40,6 @@ class TestRegression: msg = "Frequency was %f, should be < 0.23" % freq assert_(freq < 0.23, msg) - def test_permutation_longs(self): - mt19937 = Generator(MT19937(1234)) - a = mt19937.permutation(12) - mt19937 = Generator(MT19937(1234)) - b = mt19937.permutation(long(12)) - assert_array_equal(a, b) - def test_shuffle_mixed_dimension(self): # Test for trac ticket #2074 for t in [[1, 2, 3, None], diff --git a/numpy/random/tests/test_random.py b/numpy/random/tests/test_random.py index a9aa15083..276b5bc81 100644 --- a/numpy/random/tests/test_random.py +++ b/numpy/random/tests/test_random.py @@ -91,6 +91,12 @@ class TestMultinomial: assert_raises(TypeError, np.random.multinomial, 1, p, float(1)) + def test_multidimensional_pvals(self): + assert_raises(ValueError, np.random.multinomial, 10, [[0, 1]]) + assert_raises(ValueError, np.random.multinomial, 10, [[0], [1]]) + assert_raises(ValueError, np.random.multinomial, 10, [[[0], [1]], [[1], [0]]]) + assert_raises(ValueError, np.random.multinomial, 10, np.array([[0, 1], [1, 0]])) + class TestSetState: def setup(self): @@ -558,6 +564,12 @@ class TestRandomDist: alpha = np.array([5.4e-01, -1.0e-16]) assert_raises(ValueError, np.random.mtrand.dirichlet, alpha) + # gh-15876 + assert_raises(ValueError, random.dirichlet, [[5, 1]]) + assert_raises(ValueError, random.dirichlet, [[5], [1]]) + assert_raises(ValueError, random.dirichlet, [[[5], [1]], [[1], [5]]]) + assert_raises(ValueError, random.dirichlet, np.array([[5, 1], [1, 5]])) + def test_exponential(self): np.random.seed(self.seed) actual = np.random.exponential(1.1234, size=(3, 2)) diff --git a/numpy/random/tests/test_randomstate.py b/numpy/random/tests/test_randomstate.py index ebe8558ba..edd7811bf 100644 --- a/numpy/random/tests/test_randomstate.py +++ b/numpy/random/tests/test_randomstate.py @@ -350,6 +350,30 @@ class TestRandint: res = hashlib.md5(val).hexdigest() assert_(tgt[np.dtype(bool).name] == res) + @pytest.mark.skipif(np.iinfo('l').max < 2**32, + reason='Cannot test with 32-bit C long') + def test_repeatability_32bit_boundary_broadcasting(self): + desired = np.array([[[3992670689, 2438360420, 2557845020], + [4107320065, 4142558326, 3216529513], + [1605979228, 2807061240, 665605495]], + [[3211410639, 4128781000, 457175120], + [1712592594, 1282922662, 3081439808], + [3997822960, 2008322436, 1563495165]], + [[1398375547, 4269260146, 115316740], + [3414372578, 3437564012, 2112038651], + [3572980305, 2260248732, 3908238631]], + [[2561372503, 223155946, 3127879445], + [ 441282060, 3514786552, 2148440361], + [1629275283, 3479737011, 3003195987]], + [[ 412181688, 940383289, 3047321305], + [2978368172, 764731833, 2282559898], + [ 105711276, 720447391, 3596512484]]]) + for size in [None, (5, 3, 3)]: + random.seed(12345) + x = self.rfunc([[-1], [0], [1]], [2**32 - 1, 2**32, 2**32 + 1], + size=size) + assert_array_equal(x, desired if size is not None else desired[0]) + def test_int64_uint64_corner_case(self): # When stored in Numpy arrays, `lbnd` is casted # as np.int64, and `ubnd` is casted as np.uint64. diff --git a/numpy/random/tests/test_randomstate_regression.py b/numpy/random/tests/test_randomstate_regression.py index 1d8a0ed5a..4eb82fc4c 100644 --- a/numpy/random/tests/test_randomstate_regression.py +++ b/numpy/random/tests/test_randomstate_regression.py @@ -5,7 +5,6 @@ import pytest from numpy.testing import ( assert_, assert_array_equal, assert_raises, ) -from numpy.compat import long import numpy as np from numpy import random @@ -52,13 +51,6 @@ class TestRegression: msg = "Frequency was %f, should be < 0.23" % freq assert_(freq < 0.23, msg) - def test_permutation_longs(self): - random.seed(1234) - a = random.permutation(12) - random.seed(1234) - b = random.permutation(long(12)) - assert_array_equal(a, b) - def test_shuffle_mixed_dimension(self): # Test for trac ticket #2074 for t in [[1, 2, 3, None], diff --git a/numpy/random/tests/test_regression.py b/numpy/random/tests/test_regression.py index cd8f3891c..278622287 100644 --- a/numpy/random/tests/test_regression.py +++ b/numpy/random/tests/test_regression.py @@ -3,7 +3,6 @@ from numpy.testing import ( assert_, assert_array_equal, assert_raises, ) from numpy import random -from numpy.compat import long import numpy as np @@ -48,13 +47,6 @@ class TestRegression: msg = "Frequency was %f, should be < 0.23" % freq assert_(freq < 0.23, msg) - def test_permutation_longs(self): - np.random.seed(1234) - a = np.random.permutation(12) - np.random.seed(1234) - b = np.random.permutation(long(12)) - assert_array_equal(a, b) - def test_shuffle_mixed_dimension(self): # Test for trac ticket #2074 for t in [[1, 2, 3, None], diff --git a/numpy/setup.py b/numpy/setup.py index 742de2cae..fb9b36b78 100644 --- a/numpy/setup.py +++ b/numpy/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration diff --git a/numpy/testing/_private/decorators.py b/numpy/testing/_private/decorators.py index 2012b80d3..b4b6259a0 100644 --- a/numpy/testing/_private/decorators.py +++ b/numpy/testing/_private/decorators.py @@ -13,12 +13,7 @@ function name, setup and teardown functions and so on - see ``nose.tools`` for more information. """ -try: - # Accessing collections abstract classes from collections - # has been deprecated since Python 3.3 - import collections.abc as collections_abc -except ImportError: - import collections as collections_abc +import collections.abc from .utils import SkipTest, assert_warns, HAS_REFCOUNT @@ -129,7 +124,7 @@ def skipif(skip_condition, msg=None): import nose # Allow for both boolean or callable skip conditions. - if isinstance(skip_condition, collections_abc.Callable): + if isinstance(skip_condition, collections.abc.Callable): skip_val = lambda: skip_condition() else: skip_val = lambda: skip_condition @@ -157,8 +152,7 @@ def skipif(skip_condition, msg=None): if skip_val(): raise SkipTest(get_msg(f, msg)) else: - for x in f(*args, **kwargs): - yield x + yield from f(*args, **kwargs) # Choose the right skipper to use when building the actual decorator. if nose.util.isgenerator(f): @@ -205,7 +199,7 @@ def knownfailureif(fail_condition, msg=None): msg = 'Test skipped due to known failure' # Allow for both boolean or callable known failure conditions. - if isinstance(fail_condition, collections_abc.Callable): + if isinstance(fail_condition, collections.abc.Callable): fail_val = lambda: fail_condition() else: fail_val = lambda: fail_condition @@ -260,7 +254,7 @@ def deprecated(conditional=True): with assert_warns(DeprecationWarning): f(*args, **kwargs) - if isinstance(conditional, collections_abc.Callable): + if isinstance(conditional, collections.abc.Callable): cond = conditional() else: cond = conditional diff --git a/numpy/testing/_private/noseclasses.py b/numpy/testing/_private/noseclasses.py index 493bacfdd..69e19e959 100644 --- a/numpy/testing/_private/noseclasses.py +++ b/numpy/testing/_private/noseclasses.py @@ -210,7 +210,7 @@ class NumpyDoctest(npd.Doctest): # starting Python and executing "import numpy as np", and, # for SciPy packages, an additional import of the local # package (so that scipy.linalg.basic.py's doctests have an - # implicit "from scipy import linalg" as well. + # implicit "from scipy import linalg" as well). # # Note: __file__ allows the doctest in NoseTester to run # without producing an error diff --git a/numpy/testing/_private/nosetester.py b/numpy/testing/_private/nosetester.py index 45a582bb6..bd6d002aa 100644 --- a/numpy/testing/_private/nosetester.py +++ b/numpy/testing/_private/nosetester.py @@ -7,7 +7,6 @@ This module implements ``test()`` and ``bench()`` functions for NumPy modules. import os import sys import warnings -from numpy.compat import basestring import numpy as np from .utils import import_nose, suppress_warnings @@ -212,7 +211,7 @@ class NoseTester: ''' argv = [__file__, self.package_path, '-s'] if label and label != 'full': - if not isinstance(label, basestring): + if not isinstance(label, str): raise TypeError('Selection label should be a string') if label == 'fast': label = 'not slow' @@ -419,7 +418,7 @@ class NoseTester: _warn_opts = dict(develop=(Warning,), release=()) - if isinstance(raise_warnings, basestring): + if isinstance(raise_warnings, str): raise_warnings = _warn_opts[raise_warnings] with suppress_warnings("location") as sup: diff --git a/numpy/testing/_private/parameterized.py b/numpy/testing/_private/parameterized.py index 086b292e2..3bd8ede91 100644 --- a/numpy/testing/_private/parameterized.py +++ b/numpy/testing/_private/parameterized.py @@ -31,7 +31,6 @@ either expressed or implied, of David Wolever. """ import re -import sys import inspect import warnings from functools import wraps @@ -262,9 +261,6 @@ def detect_runner(): if module in _test_runners: _test_runner_guess = module break - if record[1].endswith("python2.6/unittest.py"): - _test_runner_guess = "unittest" - break else: _test_runner_guess = None return _test_runner_guess diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 8b098f1d1..4097a6738 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -284,7 +284,7 @@ def assert_equal(actual, desired, err_msg='', verbose=True): the scalar. This function handles NaN comparisons as if NaN was a "normal" number. - That is, no assertion is raised if both objects have NaNs in the same + That is, AssertionError is not raised if both objects have NaNs in the same positions. This is in contrast to the IEEE standard on NaNs, which says that NaN compared to anything must return False. @@ -530,7 +530,8 @@ def assert_almost_equal(actual,desired,decimal=7,err_msg='',verbose=True): ... AssertionError: Arrays are not almost equal to 9 decimals - Mismatch: 50% + <BLANKLINE> + Mismatched elements: 1 / 2 (50%) Max absolute difference: 6.66669964e-09 Max relative difference: 2.85715698e-09 x: array([1. , 2.333333333]) @@ -904,7 +905,8 @@ def assert_array_equal(x, y, err_msg='', verbose=True): ... AssertionError: Arrays are not equal - Mismatch: 33.3% + <BLANKLINE> + Mismatched elements: 1 / 3 (33.3%) Max absolute difference: 4.4408921e-16 Max relative difference: 1.41357986e-16 x: array([1. , 3.141593, nan]) @@ -987,7 +989,8 @@ def assert_array_almost_equal(x, y, decimal=6, err_msg='', verbose=True): ... AssertionError: Arrays are not almost equal to 5 decimals - Mismatch: 33.3% + <BLANKLINE> + Mismatched elements: 1 / 3 (33.3%) Max absolute difference: 6.e-05 Max relative difference: 2.57136612e-05 x: array([1. , 2.33333, nan]) @@ -999,6 +1002,7 @@ def assert_array_almost_equal(x, y, decimal=6, err_msg='', verbose=True): ... AssertionError: Arrays are not almost equal to 5 decimals + <BLANKLINE> x and y nan location mismatch: x: array([1. , 2.33333, nan]) y: array([1. , 2.33333, 5. ]) @@ -1086,7 +1090,8 @@ def assert_array_less(x, y, err_msg='', verbose=True): ... AssertionError: Arrays are not less-ordered - Mismatch: 33.3% + <BLANKLINE> + Mismatched elements: 1 / 3 (33.3%) Max absolute difference: 1. Max relative difference: 0.5 x: array([ 1., 1., nan]) @@ -1097,7 +1102,8 @@ def assert_array_less(x, y, err_msg='', verbose=True): ... AssertionError: Arrays are not less-ordered - Mismatch: 50% + <BLANKLINE> + Mismatched elements: 1 / 2 (50%) Max absolute difference: 2. Max relative difference: 0.66666667 x: array([1., 4.]) @@ -1108,6 +1114,7 @@ def assert_array_less(x, y, err_msg='', verbose=True): ... AssertionError: Arrays are not less-ordered + <BLANKLINE> (shapes (3,), (1,) mismatch) x: array([1., 2., 3.]) y: array([4]) @@ -1444,7 +1451,9 @@ def _assert_valid_refcount(op): """ if not HAS_REFCOUNT: return True - import numpy as np, gc + + import gc + import numpy as np b = np.arange(100*100).reshape(100, 100) c = b @@ -1607,6 +1616,12 @@ def assert_array_max_ulp(a, b, maxulp=1, dtype=None): AssertionError If one or more elements differ by more than `maxulp`. + Notes + ----- + For computing the ULP difference, this API does not differentiate between + various representations of NAN (ULP difference between 0x7fc00000 and 0xffc00000 + is zero). + See Also -------- assert_array_almost_equal_nulp : Compare two arrays relatively to their @@ -1647,6 +1662,12 @@ def nulp_diff(x, y, dtype=None): number of representable floating point numbers between each item in x and y. + Notes + ----- + For computing the ULP difference, this API does not differentiate between + various representations of NAN (ULP difference between 0x7fc00000 and 0xffc00000 + is zero). + Examples -------- # By definition, epsilon is the smallest number such as 1 + eps != 1, so @@ -1666,8 +1687,11 @@ def nulp_diff(x, y, dtype=None): if np.iscomplexobj(x) or np.iscomplexobj(y): raise NotImplementedError("_nulp not implemented for complex array") - x = np.array(x, dtype=t) - y = np.array(y, dtype=t) + x = np.array([x], dtype=t) + y = np.array([y], dtype=t) + + x[np.isnan(x)] = np.nan + y[np.isnan(y)] = np.nan if not x.shape == y.shape: raise ValueError("x and y do not have the same shape: %s - %s" % diff --git a/numpy/testing/print_coercion_tables.py b/numpy/testing/print_coercion_tables.py index 84d46b59b..8024df128 100755 --- a/numpy/testing/print_coercion_tables.py +++ b/numpy/testing/print_coercion_tables.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Prints type-coercion tables for the built-in NumPy types """ diff --git a/numpy/testing/setup.py b/numpy/testing/setup.py index c061b688a..f4970991c 100755 --- a/numpy/testing/setup.py +++ b/numpy/testing/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration diff --git a/numpy/testing/tests/test_decorators.py b/numpy/testing/tests/test_decorators.py index 77f8b66ba..b60d6dfbc 100644 --- a/numpy/testing/tests/test_decorators.py +++ b/numpy/testing/tests/test_decorators.py @@ -106,8 +106,7 @@ class TestNoseDecorators: def test_skip_generators_hardcoded(self): @dec.knownfailureif(True, "This test is known to fail") def g1(x): - for i in range(x): - yield i + yield from range(x) try: for j in g1(10): @@ -119,8 +118,7 @@ class TestNoseDecorators: @dec.knownfailureif(False, "This test is NOT known to fail") def g2(x): - for i in range(x): - yield i + yield from range(x) raise self.DidntSkipException('FAIL') try: @@ -137,8 +135,7 @@ class TestNoseDecorators: @dec.knownfailureif(skip_tester, "This test is known to fail") def g1(x): - for i in range(x): - yield i + yield from range(x) try: skip_flag = 'skip me!' @@ -151,8 +148,7 @@ class TestNoseDecorators: @dec.knownfailureif(skip_tester, "This test is NOT known to fail") def g2(x): - for i in range(x): - yield i + yield from range(x) raise self.DidntSkipException('FAIL') try: diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py index 232ca0e83..b899e94f4 100644 --- a/numpy/testing/tests/test_utils.py +++ b/numpy/testing/tests/test_utils.py @@ -2,7 +2,6 @@ import warnings import sys import os import itertools -import textwrap import pytest import weakref @@ -341,24 +340,6 @@ class TestEqual(TestArrayEqual): self._assert_func(x, x) self._test_not_equal(x, y) - def test_error_message(self): - with pytest.raises(AssertionError) as exc_info: - self._assert_func(np.array([1, 2]), np.array([[1, 2]])) - msg = str(exc_info.value) - msg2 = msg.replace("shapes (2L,), (1L, 2L)", "shapes (2,), (1, 2)") - msg_reference = textwrap.dedent("""\ - - Arrays are not equal - - (shapes (2,), (1, 2) mismatch) - x: array([1, 2]) - y: array([[1, 2]])""") - - try: - assert_equal(msg, msg_reference) - except AssertionError: - assert_equal(msg2, msg_reference) - def test_object(self): #gh-12942 import datetime diff --git a/numpy/tests/test_matlib.py b/numpy/tests/test_matlib.py index e04947a2e..0e93c4848 100644 --- a/numpy/tests/test_matlib.py +++ b/numpy/tests/test_matlib.py @@ -1,11 +1,3 @@ -# As we are testing matrices, we ignore its PendingDeprecationWarnings -try: - import pytest - pytestmark = pytest.mark.filterwarnings( - 'ignore:the matrix subclass is not:PendingDeprecationWarning') -except ImportError: - pass - import numpy as np import numpy.matlib from numpy.testing import assert_array_equal, assert_ diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index b4aa7ec3d..fb7ec5d83 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -94,6 +94,12 @@ def test_import_lazy_import(name): assert name in dir(np) +def test_dir_testing(): + """Assert that output of dir has only one "testing/tester" + attribute without duplicate""" + assert len(dir(np)) == len(set(dir(np))) + + def test_numpy_linalg(): bad_results = check_dir(np.linalg) assert bad_results == {} @@ -292,6 +298,7 @@ PRIVATE_BUT_PRESENT_MODULES = ['numpy.' + s for s in [ "matrixlib", "matrixlib.defmatrix", "random.mtrand", + "random.bit_generator", "testing.print_coercion_tables", "testing.utils", ]] diff --git a/numpy/tests/test_reloading.py b/numpy/tests/test_reloading.py index a6d2e62a9..860832be8 100644 --- a/numpy/tests/test_reloading.py +++ b/numpy/tests/test_reloading.py @@ -1,5 +1,3 @@ -import sys - from numpy.testing import assert_raises, assert_, assert_equal from numpy.compat import pickle diff --git a/numpy/tests/test_scripts.py b/numpy/tests/test_scripts.py index 20447bcf3..a0f2ba70a 100644 --- a/numpy/tests/test_scripts.py +++ b/numpy/tests/test_scripts.py @@ -9,8 +9,7 @@ from os.path import join as pathjoin, isfile, dirname import subprocess import numpy as np -from numpy.compat.py3k import basestring -from numpy.testing import assert_, assert_equal +from numpy.testing import assert_equal is_inplace = isfile(pathjoin(dirname(np.__file__), '..', 'setup.py')) diff --git a/numpy/tests/test_warnings.py b/numpy/tests/test_warnings.py index ff75681dc..d7a6d880c 100644 --- a/numpy/tests/test_warnings.py +++ b/numpy/tests/test_warnings.py @@ -2,7 +2,6 @@ Tests which scan for certain occurrences in the code, they may not find all of these occurrences but should catch almost all. """ -import sys import pytest from pathlib import Path @@ -35,7 +34,7 @@ class FindFuncs(ast.NodeVisitor): if p.ls[-1] == 'simplefilter' or p.ls[-1] == 'filterwarnings': if node.args[0].s == "ignore": raise AssertionError( - "ignore filter should not be used; found in " + "warnings should have an appropriate stacklevel; found in " "{} on line {}".format(self.__filename, node.lineno)) if p.ls[-1] == 'warn' and ( |