diff options
Diffstat (limited to 'numpy/lib/tests/test_io.py')
| -rw-r--r-- | numpy/lib/tests/test_io.py | 107 |
1 files changed, 50 insertions, 57 deletions
diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 1181fe986..8ce20a116 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -1,6 +1,5 @@ -from __future__ import division, absolute_import, print_function - import sys +import gc import gzip import os import threading @@ -9,21 +8,24 @@ import warnings import io import re import pytest +from pathlib import Path from tempfile import NamedTemporaryFile from io import BytesIO, StringIO from datetime import datetime import locale +from multiprocessing import Process import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConversionWarning -from numpy.compat import asbytes, bytes, Path +from numpy.compat import asbytes, bytes from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, assert_allclose, assert_array_equal, temppath, tempdir, IS_PYPY, HAS_REFCOUNT, suppress_warnings, assert_no_gc_cycles, assert_no_warnings ) +from numpy.testing._private.utils import requires_memory class TextIO(BytesIO): @@ -45,7 +47,6 @@ class TextIO(BytesIO): BytesIO.writelines(self, [asbytes(s) for s in lines]) -MAJVER, MINVER = sys.version_info[:2] IS_64BIT = sys.maxsize > 2**32 try: import bz2 @@ -70,7 +71,7 @@ def strptime(s, fmt=None): return datetime(*time.strptime(s, fmt)[:3]) -class RoundtripTest(object): +class RoundtripTest: def roundtrip(self, save_func, *args, **kwargs): """ save_func : callable @@ -277,8 +278,6 @@ class TestSavezLoad(RoundtripTest): fp.seek(0) assert_(not fp.closed) - #FIXME: Is this still true? - @pytest.mark.skipif(IS_PYPY, reason="Missing context manager on PyPy") def test_closing_fid(self): # Test that issue #1517 (too many opened files) remains closed # It might be a "weak" test since failed to get triggered on @@ -291,17 +290,18 @@ class TestSavezLoad(RoundtripTest): # numpy npz file returned by np.load when their reference count # goes to zero. Python 3 running in debug mode raises a # ResourceWarning when file closing is left to the garbage - # collector, so we catch the warnings. Because ResourceWarning - # is unknown in Python < 3.x, we take the easy way out and - # catch all warnings. + # collector, so we catch the warnings. with suppress_warnings() as sup: - sup.filter(Warning) # TODO: specify exact message + sup.filter(ResourceWarning) # TODO: specify exact message for i in range(1, 1025): try: np.load(tmp)["data"] except Exception as e: msg = "Failed to load data from a file: %s" % e raise AssertionError(msg) + finally: + if IS_PYPY: + gc.collect() def test_closing_zipfile_after_load(self): # Check that zipfile owns file and can close it. This needs to @@ -317,7 +317,7 @@ class TestSavezLoad(RoundtripTest): assert_(fp.closed) -class TestSaveTxt(object): +class TestSaveTxt: def test_array(self): a = np.array([[1, 2], [3, 4]], float) fmt = "%.18e" @@ -364,7 +364,6 @@ class TestSaveTxt(object): c.seek(0) assert_equal(c.readlines(), [b'1 3\n', b'4 6\n']) - @pytest.mark.skipif(Path is None, reason="No pathlib.Path") def test_multifield_view(self): a = np.ones(1, dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'f4')]) v = a[['x', 'z']] @@ -518,7 +517,7 @@ class TestSaveTxt(object): def test_unicode(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) with tempdir() as tmpdir: # set encoding as on windows it may not be unicode even on py3 np.savetxt(os.path.join(tmpdir, 'test.csv'), a, fmt=['%s'], @@ -526,26 +525,24 @@ class TestSaveTxt(object): def test_unicode_roundtrip(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) # our gz wrapper support encoding suffixes = ['', '.gz'] - # stdlib 2 versions do not support encoding - if MAJVER > 2: - if HAS_BZ2: - suffixes.append('.bz2') - if HAS_LZMA: - suffixes.extend(['.xz', '.lzma']) + if HAS_BZ2: + suffixes.append('.bz2') + if HAS_LZMA: + suffixes.extend(['.xz', '.lzma']) with tempdir() as tmpdir: for suffix in suffixes: np.savetxt(os.path.join(tmpdir, 'test.csv' + suffix), a, fmt=['%s'], encoding='UTF-16-LE') b = np.loadtxt(os.path.join(tmpdir, 'test.csv' + suffix), - encoding='UTF-16-LE', dtype=np.unicode) + encoding='UTF-16-LE', dtype=np.unicode_) assert_array_equal(a, b) def test_unicode_bytestream(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) s = BytesIO() np.savetxt(s, a, fmt=['%s'], encoding='UTF-8') s.seek(0) @@ -553,7 +550,7 @@ class TestSaveTxt(object): def test_unicode_stringstream(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) s = StringIO() np.savetxt(s, a, fmt=['%s'], encoding='UTF-8') s.seek(0) @@ -572,22 +569,23 @@ class TestSaveTxt(object): else: assert_equal(s.read(), b"%f\n" % 1.) - @pytest.mark.skipif(sys.platform=='win32', - reason="large files cause problems") + @pytest.mark.skipif(sys.platform=='win32', reason="files>4GB may not work") @pytest.mark.slow + @requires_memory(free_bytes=7e9) def test_large_zip(self): - # The test takes at least 6GB of memory, writes a file larger than 4GB - try: - a = 'a' * 6 * 1024 * 1024 * 1024 - del a - except (MemoryError, OverflowError): - pytest.skip("Cannot allocate enough memory for test") - test_data = np.asarray([np.random.rand(np.random.randint(50,100),4) - for i in range(800000)]) - with tempdir() as tmpdir: - np.savez(os.path.join(tmpdir, 'test.npz'), test_data=test_data) - -class LoadTxtBase(object): + def check_large_zip(): + # The test takes at least 6GB of memory, writes a file larger than 4GB + test_data = np.asarray([np.random.rand(np.random.randint(50,100),4) + for i in range(800000)], dtype=object) + with tempdir() as tmpdir: + np.savez(os.path.join(tmpdir, 'test.npz'), test_data=test_data) + # run in a subprocess to ensure memory is released on PyPy, see gh-15775 + p = Process(target=check_large_zip) + p.start() + p.join() + assert p.exitcode == 0 + +class LoadTxtBase: def check_compressed(self, fopen, suffixes): # Test that we can load data from a compressed file wanted = np.arange(6).reshape((2, 3)) @@ -604,18 +602,14 @@ class LoadTxtBase(object): res = self.loadfunc(f) assert_array_equal(res, wanted) - # Python2 .open does not support encoding - @pytest.mark.skipif(MAJVER == 2, reason="Needs Python version >= 3") def test_compressed_gzip(self): self.check_compressed(gzip.open, ('.gz',)) @pytest.mark.skipif(not HAS_BZ2, reason="Needs bz2") - @pytest.mark.skipif(MAJVER == 2, reason="Needs Python version >= 3") def test_compressed_bz2(self): self.check_compressed(bz2.open, ('.bz2',)) @pytest.mark.skipif(not HAS_LZMA, reason="Needs lzma") - @pytest.mark.skipif(MAJVER == 2, reason="Needs Python version >= 3") def test_compressed_lzma(self): self.check_compressed(lzma.open, ('.xz', '.lzma')) @@ -632,12 +626,12 @@ class LoadTxtBase(object): with temppath() as path: with open(path, "wb") as f: f.write(nonascii.encode("UTF-16")) - x = self.loadfunc(path, encoding="UTF-16", dtype=np.unicode) + x = self.loadfunc(path, encoding="UTF-16", dtype=np.unicode_) assert_array_equal(x, nonascii) def test_binary_decode(self): utf16 = b'\xff\xfeh\x04 \x00i\x04 \x00j\x04' - v = self.loadfunc(BytesIO(utf16), dtype=np.unicode, encoding='UTF-16') + v = self.loadfunc(BytesIO(utf16), dtype=np.unicode_, encoding='UTF-16') assert_array_equal(v, np.array(utf16.decode('UTF-16').split())) def test_converters_decode(self): @@ -645,7 +639,7 @@ class LoadTxtBase(object): c = TextIO() c.write(b'\xcf\x96') c.seek(0) - x = self.loadfunc(c, dtype=np.unicode, + x = self.loadfunc(c, dtype=np.unicode_, converters={0: lambda x: x.decode('UTF-8')}) a = np.array([b'\xcf\x96'.decode('UTF-8')]) assert_array_equal(x, a) @@ -656,7 +650,7 @@ class LoadTxtBase(object): with temppath() as path: with io.open(path, 'wt', encoding='UTF-8') as f: f.write(utf8) - x = self.loadfunc(path, dtype=np.unicode, + x = self.loadfunc(path, dtype=np.unicode_, converters={0: lambda x: x + 't'}, encoding='UTF-8') a = np.array([utf8 + 't']) @@ -829,7 +823,7 @@ class TestLoadTxt(LoadTxtBase): assert_array_equal(x, a[:, 1]) # Testing with some crazy custom integer type - class CrazyInt(object): + class CrazyInt: def __index__(self): return 1 @@ -1104,7 +1098,7 @@ class TestLoadTxt(LoadTxtBase): with open(path, "wb") as f: f.write(butf8) with open(path, "rb") as f: - x = np.loadtxt(f, encoding="UTF-8", dtype=np.unicode) + x = np.loadtxt(f, encoding="UTF-8", dtype=np.unicode_) assert_array_equal(x, sutf8) # test broken latin1 conversion people now rely on with open(path, "rb") as f: @@ -1161,7 +1155,7 @@ class TestLoadTxt(LoadTxtBase): a = np.array([[1, 2, 3, 5], [4, 5, 7, 8], [2, 1, 4, 5]], int) assert_array_equal(x, a) -class Testfromregex(object): +class Testfromregex: def test_record(self): c = TextIO() c.write('1.312 foo\n1.534 bar\n4.444 qux') @@ -1587,7 +1581,7 @@ M 33 21.99 with open(path, 'wb') as f: f.write(b'skip,skip,2001-01-01' + utf8 + b',1.0,skip') test = np.genfromtxt(path, delimiter=",", names=None, dtype=float, - usecols=(2, 3), converters={2: np.unicode}, + usecols=(2, 3), converters={2: np.compat.unicode}, encoding='UTF-8') control = np.array([('2001-01-01' + utf8.decode('UTF-8'), 1.)], dtype=[('', '|U11'), ('', float)]) @@ -2126,7 +2120,7 @@ M 33 21.99 ctl = np.array([ ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"], ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"]], - dtype=np.unicode) + dtype=np.unicode_) assert_array_equal(test, ctl) # test a mixed dtype @@ -2169,7 +2163,7 @@ M 33 21.99 ["norm1", "norm2", "norm3"], ["norm1", latin1, "norm3"], ["test1", "testNonethe" + utf8, "test3"]], - dtype=np.unicode) + dtype=np.unicode_) assert_array_equal(test, ctl) def test_recfromtxt(self): @@ -2344,15 +2338,14 @@ M 33 21.99 assert_(test.dtype['f0'] == float) assert_(test.dtype['f1'] == np.int64) - assert_(test.dtype['f2'] == np.integer) + assert_(test.dtype['f2'] == np.int_) assert_allclose(test['f0'], 73786976294838206464.) assert_equal(test['f1'], 17179869184) assert_equal(test['f2'], 1024) -@pytest.mark.skipif(Path is None, reason="No pathlib.Path") -class TestPathUsage(object): +class TestPathUsage: # Test that pathlib.Path can be used def test_loadtxt(self): with temppath(suffix='.txt') as path: @@ -2485,7 +2478,7 @@ def test_gzip_load(): # These next two classes encode the minimal API needed to save()/load() arrays. # The `test_ducktyping` ensures they work correctly -class JustWriter(object): +class JustWriter: def __init__(self, base): self.base = base @@ -2495,7 +2488,7 @@ class JustWriter(object): def flush(self): return self.base.flush() -class JustReader(object): +class JustReader: def __init__(self, base): self.base = base |