BUG: genfromtxt gave OverflorError for large integers

Fix StringConverter to avoid OverflowError in genfromtxt. Before, int(2**66) would work
(and return a ‘long’) but then np.array([2**66], dtype=np.integer) would not work and
return an OverflowError which would propagate to genfromtxt. This commit fixes this by
ensuring testing in advance whether an OverflowError will occur. In addition, this adds
an explicit np.int64 entry on systems where integer means int32. Values larger than
2**63-1 will be cast as float. This includes a regression test and adds an entry to the
release notes.

2 files changed, 43 insertions, 4 deletions

diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py
index 060f815d5..e0a917a21 100644
--- a/numpy/lib/tests/test__iotools.py
+++ b/numpy/lib/tests/test__iotools.py
@@ -152,17 +152,31 @@ class TestStringConverter(TestCase):
 
     def test_upgrade(self):
         "Tests the upgrade method."
+
         converter = StringConverter()
         assert_equal(converter._status, 0)
+
         # test int
         assert_equal(converter.upgrade(asbytes('0')), 0)
         assert_equal(converter._status, 1)
+
+        # On systems where integer defaults to 32-bit, the statuses will be
+        # offset by one, so we check for this here.
+        import numpy.core.numeric as nx
+        status_offset = int(nx.dtype(nx.integer).itemsize < nx.dtype(nx.int64).itemsize)
+
+        # test int > 2**32
+        assert_equal(converter.upgrade(asbytes('17179869184')), 17179869184)
+        assert_equal(converter._status, 1 + status_offset)
+
         # test float
         assert_allclose(converter.upgrade(asbytes('0.')), 0.0)
-        assert_equal(converter._status, 2)
+        assert_equal(converter._status, 2 + status_offset)
+
         # test complex
         assert_equal(converter.upgrade(asbytes('0j')), complex('0j'))
-        assert_equal(converter._status, 3)
+        assert_equal(converter._status, 3 + status_offset)
+
         # test str
         assert_equal(converter.upgrade(asbytes('a')), asbytes('a'))
         assert_equal(converter._status, len(converter._mapper) - 1)
diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py
index 7054ab1fe..2598a6cfb 100644
--- a/numpy/lib/tests/test_io.py
+++ b/numpy/lib/tests/test_io.py
@@ -18,7 +18,7 @@ from numpy.lib._iotools import (ConverterError, ConverterLockError,
 from numpy.compat import asbytes, asbytes_nested, bytes, asstr
 from nose import SkipTest
 from numpy.ma.testutils import (
-    TestCase, assert_equal, assert_array_equal,
+    TestCase, assert_equal, assert_array_equal, assert_allclose,
     assert_raises, assert_raises_regex, run_module_suite
 )
 from numpy.testing import assert_warns, assert_, build_err_msg
@@ -216,7 +216,7 @@ class TestSavezLoad(RoundtripTest, TestCase):
         l = np.load(c)
         assert_equal(a, l['file_a'])
         assert_equal(b, l['file_b'])
-    
+
     def test_BagObj(self):
         a = np.array([[1, 2], [3, 4]], float)
         b = np.array([[1 + 2j, 2 + 7j], [3 - 6j, 4 + 12j]], complex)
@@ -1762,6 +1762,31 @@ M   33  21.99
         res = np.genfromtxt(count())
         assert_array_equal(res, np.arange(10))
 
+    def test_auto_dtype_largeint(self):
+        """
+        Regression test for numpy/numpy#5635 whereby large integers could
+        cause OverflowErrors.
+        """
+        "Test the automatic definition of the output dtype"
+
+        # 2**66 = 73786976294838206464 => should convert to float
+        # 2**34 = 17179869184 => should convert to int64
+        # 2**10 = 1024 => should convert to int (int32 on 32-bit systems,
+        #                 int64 on 64-bit systems)
+
+        data = TextIO('73786976294838206464 17179869184 1024')
+
+        test = np.ndfromtxt(data, dtype=None)
+
+        assert_equal(test.dtype.names, ['f0', 'f1', 'f2'])
+
+        assert test.dtype['f0'] == np.float
+        assert test.dtype['f1'] == np.int64
+        assert test.dtype['f2'] == np.integer
+
+        assert_allclose(test['f0'], 73786976294838206464.)
+        assert_equal(test['f1'], 17179869184)
+        assert_equal(test['f2'], 1024)
 
 def test_gzip_load():
     a = np.random.random((5, 5))