diff options
| -rw-r--r-- | Misc/NEWS | 4 | ||||
| -rw-r--r-- | Modules/_fileio.c | 31 |
2 files changed, 31 insertions, 4 deletions
@@ -12,6 +12,10 @@ What's New in Python 3.1 alpha 0 Core and Builtins ----------------- +- Issue #4533: File read operation was dreadfully slow due to a slowly + growing read buffer. Fixed by using the same growth rate algorithm as + Python 2.x. + Library ------- diff --git a/Modules/_fileio.c b/Modules/_fileio.c index 0a34eb3be5..c23d5a3c16 100644 --- a/Modules/_fileio.c +++ b/Modules/_fileio.c @@ -27,6 +27,20 @@ #include <windows.h> #endif +#if BUFSIZ < (8*1024) +#define SMALLCHUNK (8*1024) +#elif (BUFSIZ >= (2 << 25)) +#error "unreasonable BUFSIZ > 64MB defined" +#else +#define SMALLCHUNK BUFSIZ +#endif + +#if SIZEOF_INT < 4 +#define BIGCHUNK (512 * 32) +#else +#define BIGCHUNK (512 * 1024) +#endif + typedef struct { PyObject_HEAD int fd; @@ -387,8 +401,6 @@ fileio_readinto(PyFileIOObject *self, PyObject *args) return PyLong_FromSsize_t(n); } -#define DEFAULT_BUFFER_SIZE (8*1024) - static PyObject * fileio_readall(PyFileIOObject *self) { @@ -396,12 +408,23 @@ fileio_readall(PyFileIOObject *self) Py_ssize_t total = 0; int n; - result = PyBytes_FromStringAndSize(NULL, DEFAULT_BUFFER_SIZE); + result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); if (result == NULL) return NULL; while (1) { - Py_ssize_t newsize = total + DEFAULT_BUFFER_SIZE; + Py_ssize_t newsize = (total < SMALLCHUNK) ? SMALLCHUNK : total; + + /* Keep doubling until we reach BIGCHUNK; + then keep adding BIGCHUNK. */ + if (newsize <= BIGCHUNK) { + newsize += newsize; + } + else { + /* NOTE: overflow impossible due to limits on BUFSIZ */ + newsize += BIGCHUNK; + } + if (PyBytes_GET_SIZE(result) < newsize) { if (_PyBytes_Resize(&result, newsize) < 0) { if (total == 0) { |