genfromtxt: Taking more adequate testing values when updating converters: bug #1665

3 files changed, 21 insertions, 3 deletions

diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py
index 8a0697d58..e24583ec1 100644
--- a/numpy/lib/_iotools.py
+++ b/numpy/lib/_iotools.py
@@ -707,8 +707,8 @@ class StringConverter:
             self._status = _status
             self.iterupgrade(value)
 
-    def update(self, func, default=None, missing_values=asbytes(''),
-               locked=False):
+    def update(self, func, default=None, testing_value=None,
+               missing_values=asbytes(''), locked=False):
         """
         Set StringConverter attributes directly.
 
@@ -720,6 +720,9 @@ class StringConverter:
             Value to return by default, that is, when the string to be converted
             is flagged as missing. If not given, `StringConverter` tries to supply
             a reasonable default value.
+        testing_value : str, optional
+            A string representing a standard input value of the converter.
+            This string is used to help defining a reasonable default value.
         missing_values : sequence of str, optional
             Sequence of strings indicating a missing value.
         locked : bool, optional
@@ -741,7 +744,7 @@ class StringConverter:
             self.type = self._getsubdtype(default)
         else:
             try:
-                tester = func(asbytes('1'))
+                tester = func(testing_value or asbytes('1'))
             except (TypeError, ValueError):
                 tester = None
             self.type = self._getsubdtype(tester)
diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py
index 8221f5c09..016a9d48f 100644
--- a/numpy/lib/npyio.py
+++ b/numpy/lib/npyio.py
@@ -1399,7 +1399,13 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None,
             except ValueError:
                 # Unused converter specified
                 continue
+        # Find the value to test:
+        if len(first_line):
+            testing_value = first_values[i]
+        else:
+            testing_value = None
         converters[i].update(conv, locked=True,
+                             testing_value=testing_value,
                              default=filling_values[i],
                              missing_values=missing_values[i],)
         uc_update.append((i, conv))
diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py
index 81e15119b..a85b01909 100644
--- a/numpy/lib/tests/test_io.py
+++ b/numpy/lib/tests/test_io.py
@@ -709,6 +709,15 @@ M   33  21.99
                       dtype=None)
         assert_raises(ConverterError, np.genfromtxt, s, **kwargs)
 
+    def test_tricky_converter_bug1666(self):
+        "Test some corner case"
+        s = StringIO('q1,2\nq3,4')
+        cnv = lambda s:float(s[1:])
+        test = np.genfromtxt(s, delimiter=',', converters={0:cnv})
+        control = np.array([[1., 2.], [3., 4.]])
+        assert_equal(test, control)
+
+
 
     def test_dtype_with_converters(self):
         dstr = "2009; 23; 46"