diff options
Diffstat (limited to 'numpy/lib')
| -rw-r--r-- | numpy/lib/npyio.py | 38 | ||||
| -rw-r--r-- | numpy/lib/tests/test_io.py | 25 |
2 files changed, 47 insertions, 16 deletions
diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index f37228a22..8c4a1a823 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -1343,10 +1343,10 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, try: values = [values[_] for _ in usecols] except IndexError: - append_to_invalid((i, nbvalues)) + append_to_invalid((i + skip_header + 1, nbvalues)) continue elif nbvalues != nbcols: - append_to_invalid((i, nbvalues)) + append_to_invalid((i + skip_header + 1, nbvalues)) continue # Store the values append_to_rows(tuple(values)) @@ -1354,11 +1354,6 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, append_to_masks(tuple([v.strip() in m for (v, m) in zip(values, missing_values)])) - # Strip the last skip_footer data - if skip_footer > 0: - rows = rows[:-skip_footer] - if usemask: - masks = masks[:-skip_footer] # Upgrade the converters (if needed) if dtype is None: @@ -1378,17 +1373,23 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, raise ConverterError(errmsg) # Check that we don't have invalid values - if len(invalid) > 0: - nbrows = len(rows) + nbinvalid = len(invalid) + if nbinvalid > 0: + nbrows = len(rows) + nbinvalid - skip_footer # Construct the error message template = " Line #%%i (got %%i columns instead of %i)" % nbcols if skip_footer > 0: - nbrows -= skip_footer - errmsg = [template % (i + skip_header + 1, nb) - for (i, nb) in invalid if i < nbrows] - else: - errmsg = [template % (i + skip_header + 1, nb) - for (i, nb) in invalid] + nbinvalid_skipped = len([_ for _ in invalid + if _[0] > nbrows + skip_header]) + invalid = invalid[:nbinvalid - nbinvalid_skipped] + skip_footer -= nbinvalid_skipped +# +# nbrows -= skip_footer +# errmsg = [template % (i, nb) +# for (i, nb) in invalid if i < nbrows] +# else: + errmsg = [template % (i, nb) + for (i, nb) in invalid] if len(errmsg): errmsg.insert(0, "Some errors were detected !") errmsg = "\n".join(errmsg) @@ -1399,6 +1400,13 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, else: warnings.warn(errmsg, ConversionWarning) + # Strip the last skip_footer data + if skip_footer > 0: + rows = rows[:-skip_footer] + if usemask: + masks = masks[:-skip_footer] + + # Convert each value according to the converter: # We want to modify the list in place to avoid creating a new one... # if loose: diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 25fa6faf4..c1a19f5a4 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -539,10 +539,33 @@ class TestFromTxt(TestCase): data[-1] = "99,99" kwargs = dict(delimiter=",", names=True, skip_header=5, skip_footer=10) test = np.genfromtxt(StringIO(asbytes("\n".join(data))), **kwargs) - ctrl = np.array([("%f" % i, "%f" % i, "%f" % i) for i in range(40)], + ctrl = np.array([("%f" % i, "%f" % i, "%f" % i) for i in range(41)], dtype=[(_, float) for _ in "ABC"]) assert_equal(test, ctrl) + def test_skip_footer_with_invalid(self): + import warnings + basestr = '1 1\n2 2\n3 3\n4 4\n5 \n6 \n7 \n' + warnings.filterwarnings("ignore") + # Footer too small to get rid of all invalid values + assert_raises(ValueError, np.genfromtxt, + StringIO(basestr), skip_footer=1) +# except ValueError: +# pass + a = np.genfromtxt(StringIO(basestr), skip_footer=1, invalid_raise=False) + assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]])) + # + a = np.genfromtxt(StringIO(basestr), skip_footer=3) + assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]])) + # + basestr = '1 1\n2 \n3 3\n4 4\n5 \n6 6\n7 7\n' + a = np.genfromtxt(StringIO(basestr), skip_footer=1, invalid_raise=False) + assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.], [6., 6.]])) + a = np.genfromtxt(StringIO(basestr), skip_footer=3, invalid_raise=False) + assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.]])) + warnings.resetwarnings() + + def test_header(self): "Test retrieving a header" data = StringIO('gender age weight\nM 64.0 75.0\nF 25.0 60.0') |