diff options
Diffstat (limited to 'numpy')
| -rw-r--r-- | numpy/conftest.py | 18 | ||||
| -rw-r--r-- | numpy/lib/tests/test_io.py | 7 | ||||
| -rw-r--r-- | numpy/linalg/tests/test_linalg.py | 16 | ||||
| -rw-r--r-- | numpy/testing/_private/decorators.py | 4 | ||||
| -rw-r--r-- | numpy/testing/_private/utils.py | 74 |
5 files changed, 112 insertions, 7 deletions
diff --git a/numpy/conftest.py b/numpy/conftest.py index 18d5d1ce9..1baf4adda 100644 --- a/numpy/conftest.py +++ b/numpy/conftest.py @@ -3,6 +3,8 @@ Pytest configuration and fixtures for the Numpy test suite. """ from __future__ import division, absolute_import, print_function +import os + import pytest import numpy @@ -22,6 +24,22 @@ def pytest_configure(config): "slow: Tests that are very slow.") +def pytest_addoption(parser): + parser.addoption("--available-memory", action="store", default=None, + help=("Set amount of memory available for running the " + "test suite. This can result to tests requiring " + "especially large amounts of memory to be skipped. " + "Equivalent to setting environment variable " + "NPY_AVAILABLE_MEM. Default: determined" + "automatically.")) + + +def pytest_sessionstart(session): + available_mem = session.config.getoption('available_memory') + if available_mem is not None: + os.environ['NPY_AVAILABLE_MEM'] = available_mem + + #FIXME when yield tests are gone. @pytest.hookimpl() def pytest_itemcollected(item): diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 6e2291ca3..66d6a5b00 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -24,6 +24,7 @@ from numpy.testing import ( assert_allclose, assert_array_equal, temppath, tempdir, IS_PYPY, HAS_REFCOUNT, suppress_warnings, assert_no_gc_cycles, assert_no_warnings ) +from numpy.testing._private.utils import requires_memory class TextIO(BytesIO): @@ -575,13 +576,9 @@ class TestSaveTxt(object): @pytest.mark.skipif(sys.platform=='win32', reason="large files cause problems") @pytest.mark.slow + @requires_memory(7e9) def test_large_zip(self): # The test takes at least 6GB of memory, writes a file larger than 4GB - try: - a = 'a' * 6 * 1024 * 1024 * 1024 - del a - except (MemoryError, OverflowError): - pytest.skip("Cannot allocate enough memory for test") test_data = np.asarray([np.random.rand(np.random.randint(50,100),4) for i in range(800000)]) with tempdir() as tmpdir: diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index 173e81e9c..e1590f1e7 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -20,8 +20,9 @@ from numpy.linalg.linalg import _multi_dot_matrix_chain_order from numpy.testing import ( assert_, assert_equal, assert_raises, assert_array_equal, assert_almost_equal, assert_allclose, suppress_warnings, - assert_raises_regex, + assert_raises_regex, HAS_LAPACK64, ) +from numpy.testing._private.utils import requires_memory def consistent_subclass(out, in_): @@ -2002,3 +2003,16 @@ def test_unsupported_commontype(): arr = np.array([[1, -2], [2, 5]], dtype='float16') with assert_raises_regex(TypeError, "unsupported in linalg"): linalg.cholesky(arr) + + +@pytest.mark.slow +@pytest.mark.xfail(not HAS_LAPACK64, run=False, + reason="Numpy not compiled with 64-bit BLAS/LAPACK") +@requires_memory(16e9) +def test_blas64_dot(): + n = 2**32 + a = np.zeros([1, n], dtype=np.float32) + b = np.ones([1, 1], dtype=np.float32) + a[0,-1] = 1 + c = np.dot(b, a) + assert_equal(c[0,-1], 1) diff --git a/numpy/testing/_private/decorators.py b/numpy/testing/_private/decorators.py index 24c4e385d..eab40e7c9 100644 --- a/numpy/testing/_private/decorators.py +++ b/numpy/testing/_private/decorators.py @@ -15,6 +15,10 @@ function name, setup and teardown functions and so on - see """ from __future__ import division, absolute_import, print_function +import sys +import os +import re + try: # Accessing collections abstract classes from collections # has been deprecated since Python 3.3 diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index b14c776d9..4642cc0f8 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -21,6 +21,7 @@ import pprint from numpy.core import( intp, float32, empty, arange, array_repr, ndarray, isnat, array) +import numpy.__config__ if sys.version_info[0] >= 3: from io import StringIO @@ -39,7 +40,7 @@ __all__ = [ 'SkipTest', 'KnownFailureException', 'temppath', 'tempdir', 'IS_PYPY', 'HAS_REFCOUNT', 'suppress_warnings', 'assert_array_compare', '_assert_valid_refcount', '_gen_alignment_data', 'assert_no_gc_cycles', - 'break_cycles', + 'break_cycles', 'HAS_LAPACK64' ] @@ -53,6 +54,7 @@ verbose = 0 IS_PYPY = platform.python_implementation() == 'PyPy' HAS_REFCOUNT = getattr(sys, 'getrefcount', None) is not None +HAS_LAPACK64 = hasattr(numpy.__config__, 'lapack64__opt_info') def import_nose(): @@ -2380,3 +2382,73 @@ def break_cycles(): gc.collect() # one more, just to make sure gc.collect() + + +def requires_memory(free_bytes): + """Decorator to skip a test if not enough memory is available""" + import pytest + + env_var = 'NPY_AVAILABLE_MEM' + env_value = os.environ.get(env_var) + if env_value is not None: + try: + mem_free = _parse_size(env_value) + except ValueError as exc: + raise ValueError('Invalid environment variable {}: {!s}'.format( + env_var, exc)) + + msg = ('{0} GB memory required, but environment variable ' + 'NPY_AVAILABLE_MEM={1} set'.format( + free_bytes/1e9, env_value)) + else: + mem_free = _get_mem_available() + + if mem_free is None: + msg = ("Could not determine available memory; set NPY_AVAILABLE_MEM " + "environment variable (e.g. NPY_AVAILABLE_MEM=16GB) to run " + "the test.") + mem_free = -1 + else: + msg = '{0} GB memory required, but {1} GB available'.format( + free_bytes/1e9, mem_free/1e9) + + return pytest.mark.skipif(mem_free < free_bytes, reason=msg) + + +def _parse_size(size_str): + """Convert memory size strings ('12 GB' etc.) to float""" + suffixes = {'': 1.0, 'b': 1.0, + 'k': 1e3, 'm': 1e6, 'g': 1e9, 't': 1e12, + 'kb': 1e3, 'mb': 1e6, 'gb': 1e9, 'tb': 1e12} + + size_re = re.compile(r'^\s*(\d+|\d+\.\d+)\s*({0})\s*$'.format( + '|'.join(suffixes.keys())), re.I) + + m = size_re.match(size_str.lower()) + if not m or m.group(2) not in suffixes: + raise ValueError("value {!r} not a valid size".format(size_str)) + return float(m.group(1)) * suffixes[m.group(2)] + + +def _get_mem_available(): + """Return available memory in bytes, or None if unknown.""" + try: + import psutil + return psutil.virtual_memory().available + except (ImportError, AttributeError): + pass + + if sys.platform.startswith('linux'): + info = {} + with open('/proc/meminfo', 'r') as f: + for line in f: + p = line.split() + info[p[0].strip(':').lower()] = float(p[1]) * 1e3 + + if 'memavailable' in info: + # Linux >= 3.14 + return info['memavailable'] + else: + return info['memfree'] + info['cached'] + + return None |