From 9765987c2ade250c82bfeaba49d1f9301211d63a Mon Sep 17 00:00:00 2001 From: Antonio Larrosa <33935697+antlarr-suse@users.noreply.github.com> Date: Sat, 18 Jul 2020 22:20:10 +0200 Subject: TST, BUG: Re-raise MemoryError exception in test_large_zip's process (gh-16890) Since #15893, test_large_zip's actual test is run in a child process, so when this test raises a MemoryError exception, the exception is lost and the @requires_memory decorator can't catch it to return an xfail. This commit uses a boolean variable in shared memory to flag if the exception was raised, and in that case, re-raise it in the parent process. Fixes #16889 --- numpy/lib/tests/test_io.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) (limited to 'numpy/lib/tests/test_io.py') diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 664bfe6e5..959e63fa2 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -13,7 +13,8 @@ from tempfile import NamedTemporaryFile from io import BytesIO, StringIO from datetime import datetime import locale -from multiprocessing import Process +from multiprocessing import Process, Value +from ctypes import c_bool import numpy as np import numpy.ma as ma @@ -574,16 +575,29 @@ class TestSaveTxt: @pytest.mark.slow @requires_memory(free_bytes=7e9) def test_large_zip(self): - def check_large_zip(): - # The test takes at least 6GB of memory, writes a file larger than 4GB - test_data = np.asarray([np.random.rand(np.random.randint(50,100),4) - for i in range(800000)], dtype=object) - with tempdir() as tmpdir: - np.savez(os.path.join(tmpdir, 'test.npz'), test_data=test_data) + def check_large_zip(memoryerror_raised): + memoryerror_raised.value = False + try: + # The test takes at least 6GB of memory, writes a file larger + # than 4GB + test_data = np.asarray([np.random.rand( + np.random.randint(50,100),4) + for i in range(800000)], dtype=object) + with tempdir() as tmpdir: + np.savez(os.path.join(tmpdir, 'test.npz'), + test_data=test_data) + except MemoryError: + memoryerror_raised.value = True + raise # run in a subprocess to ensure memory is released on PyPy, see gh-15775 - p = Process(target=check_large_zip) + # Use an object in shared memory to re-raise the MemoryError exception + # in our process if needed, see gh-16889 + memoryerror_raised = Value(c_bool) + p = Process(target=check_large_zip, args=(memoryerror_raised,)) p.start() p.join() + if memoryerror_raised.value: + raise MemoryError("Child process raised a MemoryError exception") assert p.exitcode == 0 class LoadTxtBase: -- cgit v1.2.1 From cf5e7665fdd764a6647f2b62d2740bb431180794 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 28 Jul 2020 15:43:38 +0300 Subject: TST: fix tests for windows + PyPy --- numpy/lib/tests/test_io.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'numpy/lib/tests/test_io.py') diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 959e63fa2..a23c6b007 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -24,7 +24,8 @@ from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, assert_allclose, assert_array_equal, temppath, tempdir, IS_PYPY, - HAS_REFCOUNT, suppress_warnings, assert_no_gc_cycles, assert_no_warnings + HAS_REFCOUNT, suppress_warnings, assert_no_gc_cycles, assert_no_warnings, + break_cycles ) from numpy.testing._private.utils import requires_memory @@ -2387,6 +2388,9 @@ class TestPathUsage: assert_array_equal(data, a) # close the mem-mapped file del data + if IS_PYPY: + break_cycles() + break_cycles() def test_save_load_memmap_readwrite(self): # Test that pathlib.Path instances can be written mem-mapped. @@ -2398,6 +2402,9 @@ class TestPathUsage: a[0][0] = 5 b[0][0] = 5 del b # closes the file + if IS_PYPY: + break_cycles() + break_cycles() data = np.load(path) assert_array_equal(data, a) -- cgit v1.2.1 From 8a4fafa5278c2d894cf4efd40a342b33e941e373 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 31 Aug 2020 10:23:54 +0100 Subject: MAINT: Remove users of `numpy.compat.bytes` Some more Python 2 cleanup. --- numpy/lib/tests/test_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'numpy/lib/tests/test_io.py') diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index a23c6b007..38d698df4 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -19,7 +19,7 @@ from ctypes import c_bool import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConversionWarning -from numpy.compat import asbytes, bytes +from numpy.compat import asbytes from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, -- cgit v1.2.1 From 3329d26c5d1d53f7fca1dfe253fdc43e93f0f6aa Mon Sep 17 00:00:00 2001 From: Andrew Eckart Date: Fri, 11 Sep 2020 11:31:03 -0500 Subject: ENH: Allow genfromtxt to unpack structured arrays (#16650) * ENH: Allow genfromtxt to unpack structured arrays genfromtxt failed to transpose output when unpack=True and `dtype` was structured (or None). This patch resolves the issue by returning a list of arrays, as in `loadtxt`. Co-authored-by: Matti Picus Co-authored-by: Eric Wieser Co-authored-by: Sebastian Berg Co-authored-by: Ross Barnowski --- numpy/lib/tests/test_io.py | 47 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) (limited to 'numpy/lib/tests/test_io.py') diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 38d698df4..aa4499764 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -1026,7 +1026,7 @@ class TestLoadTxt(LoadTxtBase): a = np.array([b'start ', b' ', b'']) assert_array_equal(x['comment'], a) - def test_structure_unpack(self): + def test_unpack_structured(self): txt = TextIO("M 21 72\nF 35 58") dt = {'names': ('a', 'b', 'c'), 'formats': ('|S1', '