BUG: restore api for file npy_PyFile_Dup and npy_PyFile_DupClose

breaking the api breaks matplotlib build and pip installation.
Introduce npy_PyFile_Dup2 and npy_PyFile_DupClose2 as replacements

4 files changed, 59 insertions, 10 deletions

diff --git a/doc/release/1.9.0-notes.rst b/doc/release/1.9.0-notes.rst
index cf33cdb1c..e7d4a7406 100644
--- a/doc/release/1.9.0-notes.rst
+++ b/doc/release/1.9.0-notes.rst
@@ -242,7 +242,12 @@ For example ``np.float_(2) * [1]`` will be an error in the future.
 C-API
 ~~~~~
 
-None
+The utility function npy_PyFile_Dup and npy_PyFile_DupClose are broken by the
+internal buffering python 3 applies to its file objects.
+To fix this two new functions npy_PyFile_Dup2 and npy_PyFile_DupClose2 are
+declared in npy_3kcompat.h and the old functions are deprecated.
+Due to the fragile nature of these functions it is recommended to instead use
+the python API when possible.
 
 
 New Features
diff --git a/numpy/core/include/numpy/npy_3kcompat.h b/numpy/core/include/numpy/npy_3kcompat.h
index 0d33de17a..36b1def4b 100644
--- a/numpy/core/include/numpy/npy_3kcompat.h
+++ b/numpy/core/include/numpy/npy_3kcompat.h
@@ -141,12 +141,11 @@ PyUnicode_Concat2(PyObject **left, PyObject *right)
  * PyFile_* compatibility
  */
 #if defined(NPY_PY3K)
-
 /*
  * Get a FILE* handle to the file represented by the Python object
  */
 static NPY_INLINE FILE*
-npy_PyFile_Dup(PyObject *file, char *mode, npy_off_t *orig_pos)
+npy_PyFile_Dup2(PyObject *file, char *mode, npy_off_t *orig_pos)
 {
     int fd, fd2;
     PyObject *ret, *os;
@@ -221,7 +220,7 @@ npy_PyFile_Dup(PyObject *file, char *mode, npy_off_t *orig_pos)
  * Close the dup-ed file handle, and seek the Python one to the current position
  */
 static NPY_INLINE int
-npy_PyFile_DupClose(PyObject *file, FILE* handle, npy_off_t orig_pos)
+npy_PyFile_DupClose2(PyObject *file, FILE* handle, npy_off_t orig_pos)
 {
     int fd;
     PyObject *ret;
@@ -269,10 +268,55 @@ npy_PyFile_Check(PyObject *file)
     return 1;
 }
 
+/*
+ * DEPRECATED DO NOT USE
+ * use npy_PyFile_Dup2 instead
+ * this function will mess ups python3 internal file object buffering
+ * Get a FILE* handle to the file represented by the Python object
+ */
+static NPY_INLINE FILE*
+npy_PyFile_Dup(PyObject *file, char *mode)
+{
+    npy_off_t orig;
+    if (DEPRECATE("npy_PyFile_Dup is deprecated, use "
+                  "npy_PyFile_Dup2") < 0) {
+        return NULL;
+    }
+
+    return npy_PyFile_Dup2(file, mode, &orig);
+}
+
+/*
+ * DEPRECATED DO NOT USE
+ * use npy_PyFile_DupClose2 instead
+ * this function will mess ups python3 internal file object buffering
+ * Close the dup-ed file handle, and seek the Python one to the current position
+ */
+static NPY_INLINE int
+npy_PyFile_DupClose(PyObject *file, FILE* handle)
+{
+    PyObject *ret;
+    Py_ssize_t position;
+    position = npy_ftell(handle);
+    fclose(handle);
+
+    ret = PyObject_CallMethod(file, "seek", NPY_SSIZE_T_PYFMT "i", position, 0);
+    if (ret == NULL) {
+        return -1;
+    }
+    Py_DECREF(ret);
+    return 0;
+}
+
+
 #else
 
-#define npy_PyFile_Dup(file, mode, orig_pos_p) PyFile_AsFile(file)
-#define npy_PyFile_DupClose(file, handle, orig_pos) (0)
+/* DEPRECATED DO NOT USE */
+#define npy_PyFile_Dup(file, mode) PyFile_AsFile(file)
+#define npy_PyFile_DupClose(file, handle) (0)
+/* use these */
+#define npy_PyFile_Dup2(file, mode, orig_pos_p) PyFile_AsFile(file)
+#define npy_PyFile_DupClose2(file, handle, orig_pos) (0)
 #define npy_PyFile_Check PyFile_Check
 
 #endif
diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c
index 66d68e5b3..91f639650 100644
--- a/numpy/core/src/multiarray/methods.c
+++ b/numpy/core/src/multiarray/methods.c
@@ -588,7 +588,7 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds)
         own = 0;
     }
 
-    fd = npy_PyFile_Dup(file, "wb", &orig_pos);
+    fd = npy_PyFile_Dup2(file, "wb", &orig_pos);
     if (fd == NULL) {
         PyErr_SetString(PyExc_IOError,
                 "first argument must be a string or open file");
@@ -597,7 +597,7 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds)
     if (PyArray_ToFile(self, fd, sep, format) < 0) {
         goto fail;
     }
-    if (npy_PyFile_DupClose(file, fd, orig_pos) < 0) {
+    if (npy_PyFile_DupClose2(file, fd, orig_pos) < 0) {
         goto fail;
     }
     if (own && npy_PyFile_CloseFile(file) < 0) {
diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c
index b3fdd040f..cfaf7799a 100644
--- a/numpy/core/src/multiarray/multiarraymodule.c
+++ b/numpy/core/src/multiarray/multiarraymodule.c
@@ -1999,7 +1999,7 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds)
         Py_INCREF(file);
         own = 0;
     }
-    fp = npy_PyFile_Dup(file, "rb", &orig_pos);
+    fp = npy_PyFile_Dup2(file, "rb", &orig_pos);
     if (fp == NULL) {
         PyErr_SetString(PyExc_IOError,
                 "first argument must be an open file");
@@ -2011,7 +2011,7 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds)
     }
     ret = PyArray_FromFile(fp, type, (npy_intp) nin, sep);
 
-    if (npy_PyFile_DupClose(file, fp, orig_pos) < 0) {
+    if (npy_PyFile_DupClose2(file, fp, orig_pos) < 0) {
         goto fail;
     }
     if (own && npy_PyFile_CloseFile(file) < 0) {