ENH: iter: Add support for buffering string and unicode arrays

4 files changed, 292 insertions, 74 deletions

diff --git a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src
index 2eb0229a8..f02021c86 100644
--- a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src
+++ b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src
@@ -115,7 +115,7 @@
 static void
 @prefix@_@oper@_size@elsize@(char *dst, npy_intp dst_stride,
                         char *src, npy_intp src_stride,
-                        npy_intp N, npy_intp NPY_UNUSED(itemsize),
+                        npy_intp N, npy_intp NPY_UNUSED(src_itemsize),
                         void *NPY_UNUSED(data))
 {
     /*printf("fn @prefix@_@oper@_size@elsize@\n");*/
@@ -175,7 +175,7 @@ static void
 @prefix@_@oper@_size@elsize@_srcstride0(char *dst,
                         npy_intp dst_stride,
                         char *src, npy_intp NPY_UNUSED(src_stride),
-                        npy_intp N, npy_intp NPY_UNUSED(itemsize),
+                        npy_intp N, npy_intp NPY_UNUSED(src_itemsize),
                         void *NPY_UNUSED(data))
 {
 #if @elsize@ != 16
@@ -219,11 +219,11 @@ static void
 static void
 _strided_to_strided(char *dst, npy_intp dst_stride,
                         char *src, npy_intp src_stride,
-                        npy_intp N, npy_intp itemsize,
+                        npy_intp N, npy_intp src_itemsize,
                         void *NPY_UNUSED(data))
 {
     while (N > 0) {
-        memcpy(dst, src, itemsize);
+        memcpy(dst, src, src_itemsize);
         dst += dst_stride;
         src += src_stride;
         --N;
@@ -233,16 +233,16 @@ _strided_to_strided(char *dst, npy_intp dst_stride,
 static void
 _swap_strided_to_strided(char *dst, npy_intp dst_stride,
                         char *src, npy_intp src_stride,
-                        npy_intp N, npy_intp itemsize,
+                        npy_intp N, npy_intp src_itemsize,
                         void *NPY_UNUSED(data))
 {
     char *a, *b, c;
 
     while (N > 0) {
-        memcpy(dst, src, itemsize);
+        memcpy(dst, src, src_itemsize);
         /* general in-place swap */
         a = dst;
-        b = dst + itemsize - 1;
+        b = dst + src_itemsize - 1;
         while (a < b) {
             c = *a;
             *a = *b;
@@ -258,14 +258,14 @@ _swap_strided_to_strided(char *dst, npy_intp dst_stride,
 static void
 _swap_pair_strided_to_strided(char *dst, npy_intp dst_stride,
                         char *src, npy_intp src_stride,
-                        npy_intp N, npy_intp itemsize,
+                        npy_intp N, npy_intp src_itemsize,
                         void *NPY_UNUSED(data))
 {
     char *a, *b, c;
-    npy_intp itemsize_half = itemsize / 2;
+    npy_intp itemsize_half = src_itemsize / 2;
 
     while (N > 0) {
-        memcpy(dst, src, itemsize);
+        memcpy(dst, src, src_itemsize);
         /* general in-place swap */
         a = dst;
         b = dst + itemsize_half - 1;
@@ -293,12 +293,12 @@ _swap_pair_strided_to_strided(char *dst, npy_intp dst_stride,
 static void
 _strided_to_contig(char *dst, npy_intp NPY_UNUSED(dst_stride),
                         char *src, npy_intp src_stride,
-                        npy_intp N, npy_intp itemsize,
+                        npy_intp N, npy_intp src_itemsize,
                         void *NPY_UNUSED(data))
 {
     while (N > 0) {
-        memcpy(dst, src, itemsize);
-        dst += itemsize;
+        memcpy(dst, src, src_itemsize);
+        dst += src_itemsize;
         src += src_stride;
         --N;
     }
@@ -307,13 +307,13 @@ _strided_to_contig(char *dst, npy_intp NPY_UNUSED(dst_stride),
 static void
 _contig_to_strided(char *dst, npy_intp dst_stride,
                         char *src, npy_intp NPY_UNUSED(src_stride),
-                        npy_intp N, npy_intp itemsize,
+                        npy_intp N, npy_intp src_itemsize,
                         void *NPY_UNUSED(data))
 {
     while (N > 0) {
-        memcpy(dst, src, itemsize);
+        memcpy(dst, src, src_itemsize);
         dst += dst_stride;
-        src += itemsize;
+        src += src_itemsize;
         --N;
     }
 }
@@ -321,10 +321,10 @@ _contig_to_strided(char *dst, npy_intp dst_stride,
 static void
 _contig_to_contig(char *dst, npy_intp NPY_UNUSED(dst_stride),
                         char *src, npy_intp NPY_UNUSED(src_stride),
-                        npy_intp N, npy_intp itemsize,
+                        npy_intp N, npy_intp src_itemsize,
                         void *NPY_UNUSED(data))
 {
-    memcpy(dst, src, itemsize*N);
+    memcpy(dst, src, src_itemsize*N);
 }
 
 
@@ -656,6 +656,85 @@ NPY_NO_EXPORT PyArray_StridedTransferFn
 
 /**end repeat**/
 
+/* Does a zero-padded copy */
+typedef struct {
+    void *freefunc, *copyfunc;
+    npy_intp dst_itemsize;
+} _strided_zero_pad_data;
+
+/* zero-padded data copy function */
+_strided_zero_pad_data *_strided_zero_pad_data_copy(
+                                        _strided_zero_pad_data *data)
+{
+    _strided_zero_pad_data *newdata = 
+            (_strided_zero_pad_data *)PyArray_malloc(
+                                    sizeof(_strided_zero_pad_data));
+    if (newdata == NULL) {
+        return NULL;
+    }
+
+    memcpy(newdata, data, sizeof(_strided_zero_pad_data));
+
+    return newdata;
+}
+
+/*
+ * Does a strided to strided zero-padded copy for the case where
+ * dst_itemsize > src_itemsize
+ */
+static void
+_strided_to_strided_zero_pad_copy(char *dst, npy_intp dst_stride,
+                        char *src, npy_intp src_stride,
+                        npy_intp N, npy_intp src_itemsize,
+                        void *data)
+{
+    _strided_zero_pad_data *d = (_strided_zero_pad_data *)data;
+    npy_intp dst_itemsize = d->dst_itemsize;
+    npy_intp zero_size = dst_itemsize-src_itemsize;
+
+    while (N > 0) {
+        memcpy(dst, src, src_itemsize);
+        memset(dst + src_itemsize, 0, zero_size);
+        src += src_stride;
+        dst += dst_stride;
+        --N;
+    }
+}
+
+NPY_NO_EXPORT int
+PyArray_GetStridedZeroPadCopyFn(npy_intp aligned,
+                            npy_intp src_stride, npy_intp dst_stride,
+                            npy_intp src_itemsize, npy_intp dst_itemsize,
+                            PyArray_StridedTransferFn *outstransfer,
+                            void **outtransferdata)
+{
+    if (src_itemsize >= dst_itemsize) {
+        /* If the sizes are different, the alignment flag isn't trustworthy */
+        if (src_itemsize != dst_itemsize) {
+            aligned = 0;
+        }
+        *outstransfer = PyArray_GetStridedCopyFn(aligned, src_stride,
+                                dst_stride, dst_itemsize);
+        *outtransferdata = NULL;
+        return (*outstransfer == NULL) ? NPY_FAIL : NPY_SUCCEED;
+    }
+    else {
+        _strided_zero_pad_data *d = PyArray_malloc(
+                                        sizeof(_strided_zero_pad_data));
+        if (d == NULL) {
+            PyErr_NoMemory();
+            return NPY_FAIL;
+        }
+        d->dst_itemsize = dst_itemsize;
+        d->freefunc = &PyArray_free;
+        d->copyfunc = &_strided_zero_pad_data_copy;
+
+        *outstransfer = &_strided_to_strided_zero_pad_copy;
+        *outtransferdata = d;
+        return NPY_SUCCEED;
+    }
+}
+
 /* Does a simple aligned cast */
 typedef struct {
     void *freefunc, *copyfunc;
@@ -679,7 +758,7 @@ _strided_cast_data *_strided_cast_data_copy(_strided_cast_data *data)
 static void
 _aligned_strided_to_strided_cast(char *dst, npy_intp dst_stride,
                         char *src, npy_intp src_stride,
-                        npy_intp N, npy_intp itemsize,
+                        npy_intp N, npy_intp src_itemsize,
                         void *data)
 {
     PyArray_VectorUnaryFunc *castfunc = ((_strided_cast_data *)data)->castfunc;
@@ -754,14 +833,14 @@ _align_wrap_data *_align_wrap_data_copy(_align_wrap_data *data)
 static void
 _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride,
                         char *src, npy_intp src_stride,
-                        npy_intp N, npy_intp itemsize,
+                        npy_intp N, npy_intp src_itemsize,
                         void *data)
 {
     _align_wrap_data *d = (_align_wrap_data *)data;
     PyArray_StridedTransferFn wrapped = d->wrapped,
             tobuffer = d->tobuffer,
             frombuffer = d->frombuffer;
-    npy_intp src_itemsize = d->src_itemsize, dst_itemsize = d->dst_itemsize;
+    npy_intp dst_itemsize = d->dst_itemsize;
     void *wrappeddata = d->wrappeddata;
     char *bufferin = d->bufferin, *bufferout = d->bufferout;
 
@@ -770,7 +849,7 @@ _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride,
             tobuffer(bufferin, src_itemsize, src, src_stride, 32,
                                             src_itemsize, NULL);
             wrapped(bufferout, dst_itemsize, bufferin, src_itemsize, 32,
-                                            itemsize, wrappeddata);
+                                            src_itemsize, wrappeddata);
             frombuffer(dst, dst_stride, bufferout, dst_itemsize, 32,
                                             dst_itemsize, NULL);
             N -= 32;
@@ -781,7 +860,7 @@ _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride,
             tobuffer(bufferin, src_itemsize, src, src_stride, N,
                                             src_itemsize, NULL);
             wrapped(bufferout, dst_itemsize, bufferin, src_itemsize, N,
-                                            itemsize, wrappeddata);
+                                            src_itemsize, wrappeddata);
             frombuffer(dst, dst_stride, bufferout, dst_itemsize, N,
                                             dst_itemsize, NULL);
             return;
@@ -878,6 +957,27 @@ PyArray_GetDTypeTransferFunction(int aligned,
             return (*outstransfer == NULL) ? NPY_FAIL : NPY_SUCCEED;
         }
     }
+    else if (src_dtype->type_num == dst_dtype->type_num) {
+        switch (src_dtype->type_num) {
+            case NPY_VOID:
+                /*
+                 * If it's not a structured type or subarray,
+                 * do a simple copy
+                 */
+                if (src_dtype->fields != NULL ||
+                            dst_dtype->fields != NULL ||
+                            src_dtype->subarray != NULL ||
+                            dst_dtype->subarray != NULL) {
+                    break;
+                }
+            case NPY_STRING:
+            case NPY_UNICODE:
+                return PyArray_GetStridedZeroPadCopyFn(0,
+                                    src_stride, dst_stride,
+                                    src_dtype->elsize, dst_dtype->elsize,
+                                    outstransfer, outtransferdata);
+        }
+    }
 
     /* Check whether a simple cast and some swaps will suffice */
     if (src_type_num < NPY_OBJECT && dst_type_num < NPY_OBJECT) {
@@ -1012,7 +1112,7 @@ PyArray_TransferNDimToStrided(npy_intp ndim,
                 char *src, npy_intp *src_strides, npy_intp src_strides_inc,
                 npy_intp *coords, npy_intp coords_inc,
                 npy_intp *shape, npy_intp shape_inc,
-                npy_intp count, npy_intp itemsize,
+                npy_intp count, npy_intp src_itemsize,
                 PyArray_StridedTransferFn stransfer,
                 void *data)
 {
@@ -1024,10 +1124,10 @@ PyArray_TransferNDimToStrided(npy_intp ndim,
     src_stride0 = src_strides[0];
     N = shape0 - coord0;
     if (N >= count) {
-        stransfer(dst, dst_stride, src, src_stride0, count, itemsize, data);
+        stransfer(dst, dst_stride, src, src_stride0, count, src_itemsize, data);
         return 0;
     }
-    stransfer(dst, dst_stride, src, src_stride0, N, itemsize, data);
+    stransfer(dst, dst_stride, src, src_stride0, N, src_itemsize, data);
     count -= N;
 
     /* If it's 1-dimensional, there's no more to copy */
@@ -1048,12 +1148,12 @@ PyArray_TransferNDimToStrided(npy_intp ndim,
     for (i = 0; i < M; ++i) {
         if (shape0 >= count) {
             stransfer(dst, dst_stride, src, src_stride0,
-                        count, itemsize, data);
+                        count, src_itemsize, data);
             return 0;
         }
         else {
             stransfer(dst, dst_stride, src, src_stride0,
-                        shape0, itemsize, data);
+                        shape0, src_itemsize, data);
         }
         count -= shape0;
         src += src_stride1;
@@ -1109,12 +1209,12 @@ PyArray_TransferNDimToStrided(npy_intp ndim,
             for (i = 0; i < shape1; ++i) {
                 if (shape0 >= count) {
                     stransfer(dst, dst_stride, src, src_stride0,
-                                count, itemsize, data);
+                                count, src_itemsize, data);
                     return 0;
                 }
                 else {
                     stransfer(dst, dst_stride, src, src_stride0,
-                                shape0, itemsize, data);
+                                shape0, src_itemsize, data);
                 }
                 count -= shape0;
                 src += src_stride1;
@@ -1130,7 +1230,7 @@ PyArray_TransferStridedToNDim(npy_intp ndim,
                 char *src, npy_intp src_stride,
                 npy_intp *coords, npy_intp coords_inc,
                 npy_intp *shape, npy_intp shape_inc,
-                npy_intp count, npy_intp itemsize,
+                npy_intp count, npy_intp src_itemsize,
                 PyArray_StridedTransferFn stransfer,
                 void *data)
 {
@@ -1142,10 +1242,10 @@ PyArray_TransferStridedToNDim(npy_intp ndim,
     dst_stride0 = dst_strides[0];
     N = shape0 - coord0;
     if (N >= count) {
-        stransfer(dst, dst_stride0, src, src_stride, count, itemsize, data);
+        stransfer(dst, dst_stride0, src, src_stride, count, src_itemsize, data);
         return 0;
     }
-    stransfer(dst, dst_stride0, src, src_stride, N, itemsize, data);
+    stransfer(dst, dst_stride0, src, src_stride, N, src_itemsize, data);
     count -= N;
 
     /* If it's 1-dimensional, there's no more to copy */
@@ -1166,12 +1266,12 @@ PyArray_TransferStridedToNDim(npy_intp ndim,
     for (i = 0; i < M; ++i) {
         if (shape0 >= count) {
             stransfer(dst, dst_stride0, src, src_stride,
-                        count, itemsize, data);
+                        count, src_itemsize, data);
             return 0;
         }
         else {
             stransfer(dst, dst_stride0, src, src_stride,
-                        shape0, itemsize, data);
+                        shape0, src_itemsize, data);
         }
         count -= shape0;
         dst += dst_stride1;
@@ -1227,12 +1327,12 @@ PyArray_TransferStridedToNDim(npy_intp ndim,
             for (i = 0; i < shape1; ++i) {
                 if (shape0 >= count) {
                     stransfer(dst, dst_stride0, src, src_stride,
-                                count, itemsize, data);
+                                count, src_itemsize, data);
                     return 0;
                 }
                 else {
                     stransfer(dst, dst_stride0, src, src_stride,
-                                shape0, itemsize, data);
+                                shape0, src_itemsize, data);
                 }
                 count -= shape0;
                 dst += dst_stride1;
diff --git a/numpy/core/src/multiarray/lowlevel_strided_loops.h b/numpy/core/src/multiarray/lowlevel_strided_loops.h
index 52a941b1b..5c6374493 100644
--- a/numpy/core/src/multiarray/lowlevel_strided_loops.h
+++ b/numpy/core/src/multiarray/lowlevel_strided_loops.h
@@ -19,7 +19,7 @@
  */
 typedef void (*PyArray_StridedTransferFn)(char *dst, npy_intp dst_stride,
                                     char *src, npy_intp src_stride,
-                                    npy_intp N, npy_intp itemsize,
+                                    npy_intp N, npy_intp src_itemsize,
                                     void *transferdata);
 
 /*
@@ -80,6 +80,20 @@ PyArray_GetStridedCopySwapPairFn(npy_intp aligned, npy_intp src_stride,
                              npy_intp dst_stride, npy_intp itemsize);
 
 /*
+ * Gives back a transfer function and transfer data pair which copies
+ * the data from source to dest, truncating it if the data doesn't
+ * fit, and padding with zero bytes if there's too much space.
+ *
+ * Returns NPY_SUCCEED or NPY_FAIL
+ */
+NPY_NO_EXPORT int
+PyArray_GetStridedZeroPadCopyFn(npy_intp aligned,
+                            npy_intp src_stride, npy_intp dst_stride,
+                            npy_intp src_itemsize, npy_intp dst_itemsize,
+                            PyArray_StridedTransferFn *outstransfer,
+                            void **outtransferdata);
+
+/*
  * If it's possible, gives back a transfer function which casts and/or
  * byte swaps data with the dtype 'from' into data with the dtype 'to'.
  * If the outtransferdata is populated with a non-NULL value, it
@@ -126,11 +140,11 @@ PyArray_GetDTypeTransferFunction(int aligned,
  *      How much to add to the shape pointer to get to the next shape entry.
  * count:
  *      How many elements to transfer
- * itemsize:
+ * src_itemsize:
  *      How big each element is.  If transfering between elements of different
  *      sizes, for example a casting operation, the 'stransfer' function
- *      should be specialized for that, in which case 'stransfer' will ignore
- *      this parameter.
+ *      should be specialized for that, in which case 'stransfer' will use
+ *      this parameter as the source item size.
  * stransfer:
  *      The strided transfer function.
  * transferdata:
@@ -144,7 +158,7 @@ PyArray_TransferNDimToStrided(npy_intp ndim,
                 char *src, npy_intp *src_strides, npy_intp src_strides_inc,
                 npy_intp *coords, npy_intp coords_inc,
                 npy_intp *shape, npy_intp shape_inc,
-                npy_intp count, npy_intp itemsize,
+                npy_intp count, npy_intp src_itemsize,
                 PyArray_StridedTransferFn stransfer,
                 void *transferdata);
 
@@ -154,7 +168,7 @@ PyArray_TransferStridedToNDim(npy_intp ndim,
                 char *src, npy_intp src_stride,
                 npy_intp *coords, npy_intp coords_inc,
                 npy_intp *shape, npy_intp shape_inc,
-                npy_intp count, npy_intp itemsize,
+                npy_intp count, npy_intp src_itemsize,
                 PyArray_StridedTransferFn stransfer,
                 void *transferdata);
 
diff --git a/numpy/core/src/multiarray/new_iterator.c.src b/numpy/core/src/multiarray/new_iterator.c.src
index bbbc295c9..25491166b 100644
--- a/numpy/core/src/multiarray/new_iterator.c.src
+++ b/numpy/core/src/multiarray/new_iterator.c.src
@@ -293,6 +293,8 @@ npyiter_get_priority_subtype(npy_intp niter, PyArrayObject **op,
                             double *subtype_priority, PyTypeObject **subtype);
 
 static int
+npyiter_allocate_transfer_functions(NpyIter *iter);
+static int
 npyiter_allocate_buffers(NpyIter *iter, char **errmsg);
 static void npyiter_goto_iterindex(NpyIter *iter, npy_intp iterindex);
 static void
@@ -567,6 +569,10 @@ NpyIter_MultiNew(npy_intp niter, PyArrayObject **op_in, npy_uint32 flags,
 
     /* If buffering is set without delayed allocation */
     if (itflags&NPY_ITFLAG_BUFFER) {
+        if (!npyiter_allocate_transfer_functions(iter)) {
+            NpyIter_Deallocate(iter);
+            return NULL;
+        }
         if (itflags&NPY_ITFLAG_DELAYBUF) {
             /* Make the data pointers NULL */
             memset(NBF_PTRS(bufferdata), 0, niter*NPY_SIZEOF_INTP);
@@ -2357,12 +2363,19 @@ npyiter_prepare_one_operand(PyArrayObject **op,
             return 0;
         }
         /* Reading should be disabled */
+        /* Disable this check for now, since except in one case (buffering
+         * without delayed buffer allocation), the caller can use
+         * NpyIter_GetObjectArray to initialize it after the iterator
+         * is created.
+         * TODO: Maybe reenable it when buffering is on, but delayed buffer
+         * allocation is off.
         if ((*op_itflags)&NPY_OP_ITFLAG_READ) {
             PyErr_SetString(PyExc_ValueError,
                     "Automatic allocation was requested for an iterator "
                     "operand, but it wasn't flagged as write only");
             return 0;
         }
+        */
         *op_dataptr = NULL;
         /* If a requested dtype was provided, use it, otherwise NULL */
         Py_XINCREF(op_request_dtype);
@@ -2416,9 +2429,41 @@ npyiter_prepare_one_operand(PyArrayObject **op,
          * requested type.
          */
         if (op_request_dtype != NULL) {
+            /* We just have a borrowed reference to op_request_dtype */
+            Py_INCREF(op_request_dtype);
+            /* If it's a data type without a size, set the size */
+            if (op_request_dtype->elsize == 0) {
+                PyArray_DESCR_REPLACE(op_request_dtype);
+                if (op_request_dtype == NULL) {
+                    return 0;
+                }
+
+                if (op_request_dtype->type_num == NPY_STRING) {
+                    switch((*op_dtype)->type_num) {
+                        case NPY_STRING:
+                            op_request_dtype->elsize = (*op_dtype)->elsize;
+                            break;
+                        case NPY_UNICODE:
+                            op_request_dtype->elsize = (*op_dtype)->elsize >> 2;
+                            break;
+                    }
+                }
+                else if (op_request_dtype->type_num == NPY_UNICODE) {
+                    switch((*op_dtype)->type_num) {
+                        case NPY_STRING:
+                            op_request_dtype->elsize = (*op_dtype)->elsize << 2;
+                            break;
+                        case NPY_UNICODE:
+                            op_request_dtype->elsize = (*op_dtype)->elsize;
+                            break;
+                    }
+                }
+                else if (op_request_dtype->type_num == NPY_VOID) {
+                    op_request_dtype->elsize = (*op_dtype)->elsize;
+                }
+            }
             /* Store the requested dtype */
             Py_DECREF(*op_dtype);
-            Py_INCREF(op_request_dtype);
             *op_dtype = op_request_dtype;
         }
 
@@ -2568,7 +2613,18 @@ npyiter_can_cast(PyArray_Descr *from, PyArray_Descr *to, NPY_CASTING casting)
                 Py_DECREF(nbo_to);
             }
             else {
-                ret = PyArray_EquivTypes(from, to);
+                if (from->type_num == NPY_STRING ||
+                                    from->type_num == NPY_UNICODE) {
+                    if (casting == NPY_SAME_KIND_CASTING) {
+                        ret = 1;
+                    }
+                    else {
+                        ret = (from->elsize <= to->elsize);
+                    }
+                }
+                else {
+                    ret = PyArray_EquivTypes(from, to);
+                }
             }
             return ret;
         }
@@ -3889,7 +3945,7 @@ npyiter_get_priority_subtype(npy_intp niter, PyArrayObject **op,
     npy_intp iiter;
 
     for (iiter = 0; iiter < niter; ++iiter) {
-        if (op_itflags[iiter]&NPY_OP_ITFLAG_READ) {
+        if (op[iiter] != NULL && op_itflags[iiter]&NPY_OP_ITFLAG_READ) {
             double priority = PyArray_GetPriority((PyObject *)op[iiter], 0.0);
             if (priority > *subtype_priority) {
                 *subtype_priority = priority;
@@ -4076,15 +4132,8 @@ npyiter_promote_types(int type1, int type2)
     return NPY_NOTYPE;
 }
 
-/*
- *
- * If errmsg is non-NULL, it should point to a variable which will
- * receive the error message, and no Python exception will be set.
- * This is so that the function can be called from code not holding
- * the GIL.
- */
 static int
-npyiter_allocate_buffers(NpyIter *iter, char **errmsg)
+npyiter_allocate_transfer_functions(NpyIter *iter)
 {
     npy_uint32 itflags = NIT_ITFLAGS(iter);
     npy_intp ndim = NIT_NDIM(iter);
@@ -4097,8 +4146,6 @@ npyiter_allocate_buffers(NpyIter *iter, char **errmsg)
     PyArrayObject **op = NIT_OBJECTS(iter);
     PyArray_Descr **op_dtype = NIT_DTYPES(iter);
     npy_intp *strides = NAD_STRIDES(axisdata), op_stride;
-    npy_intp buffersize = NBF_BUFFERSIZE(bufferdata);
-    char *buffer, **buffers = NBF_BUFFERS(bufferdata);
     PyArray_StridedTransferFn *readtransferfn = NBF_READTRANSFERFN(bufferdata),
                         *writetransferfn = NBF_WRITETRANSFERFN(bufferdata);
     void **readtransferdata = NBF_READTRANSFERDATA(bufferdata),
@@ -4113,17 +4160,9 @@ npyiter_allocate_buffers(NpyIter *iter, char **errmsg)
 
         /*
          * If we have determined that a buffer may be needed,
-         * allocate one.
+         * allocate the appropriate transfer functions
          */
         if (!(flags&NPY_OP_ITFLAG_BUFNEVER)) {
-            npy_intp itemsize = op_dtype[iiter]->elsize;
-            buffer = PyArray_malloc(itemsize*buffersize);
-            if (buffer == NULL) {
-                goto fail;
-            }
-            buffers[iiter] = buffer;
-
-            /* Also need to get an appropriate transfer functions */
             if (flags&NPY_OP_ITFLAG_READ) {
                 if (PyArray_GetDTypeTransferFunction(
                                         (flags&NPY_OP_ITFLAG_ALIGNED) != 0,
@@ -4169,10 +4208,6 @@ npyiter_allocate_buffers(NpyIter *iter, char **errmsg)
 
 fail:
     for (i = 0; i < iiter; ++i) {
-        if (buffers[i] != NULL) {
-            PyArray_free(buffers[i]);
-            buffers[i] = NULL;
-        }
         if (readtransferdata[iiter] != NULL) {
             PyArray_FreeStridedTransferData(readtransferdata[iiter]);
             readtransferdata[iiter] = NULL;
@@ -4182,11 +4217,61 @@ fail:
             writetransferdata[iiter] = NULL;
         }
     }
-    if (errmsg == NULL) {
-        PyErr_NoMemory();
+    return 0;
+}
+
+/*
+ *
+ * If errmsg is non-NULL, it should point to a variable which will
+ * receive the error message, and no Python exception will be set.
+ * This is so that the function can be called from code not holding
+ * the GIL.
+ */
+static int
+npyiter_allocate_buffers(NpyIter *iter, char **errmsg)
+{
+    /*npy_uint32 itflags = NIT_ITFLAGS(iter);*/
+    npy_intp ndim = NIT_NDIM(iter);
+    npy_intp iiter = 0, niter = NIT_NITER(iter);
+
+    npy_intp i;
+    char *op_itflags = NIT_OPITFLAGS(iter);
+    NpyIter_BufferData *bufferdata = NIT_BUFFERDATA(iter);
+    PyArray_Descr **op_dtype = NIT_DTYPES(iter);
+    npy_intp buffersize = NBF_BUFFERSIZE(bufferdata);
+    char *buffer, **buffers = NBF_BUFFERS(bufferdata);
+
+    for (iiter = 0; iiter < niter; ++iiter) {
+        char flags = op_itflags[iiter];
+
+        /*
+         * If we have determined that a buffer may be needed,
+         * allocate one.
+         */
+        if (!(flags&NPY_OP_ITFLAG_BUFNEVER)) {
+            npy_intp itemsize = op_dtype[iiter]->elsize;
+            buffer = PyArray_malloc(itemsize*buffersize);
+            if (buffer == NULL) {
+                if (errmsg == NULL) {
+                    PyErr_NoMemory();
+                }
+                else {
+                    *errmsg = "out of memory";
+                }
+                goto fail;
+            }
+            buffers[iiter] = buffer;
+        }
     }
-    else {
-        *errmsg = "out of memory";
+
+    return 1;
+
+fail:
+    for (i = 0; i < iiter; ++i) {
+        if (buffers[i] != NULL) {
+            PyArray_free(buffers[i]);
+            buffers[i] = NULL;
+        }
     }
     return 0;
 }
@@ -4345,6 +4430,7 @@ npyiter_copy_to_buffers(NpyIter *iter)
     NpyIter_AxisData *axisdata = NIT_AXISDATA(iter);
 
     PyArray_Descr **dtypes = NIT_DTYPES(iter);
+    PyArrayObject **operands = NIT_OBJECTS(iter);
     npy_intp *strides = NBF_STRIDES(bufferdata),
              *ad_strides = NAD_STRIDES(axisdata);
     char **ptrs = NBF_PTRS(bufferdata), **ad_ptrs = NAD_PTRS(axisdata);
@@ -4430,7 +4516,7 @@ npyiter_copy_to_buffers(NpyIter *iter)
                     ad_ptrs[iiter], &ad_strides[iiter], axisdata_incr,
                     &NAD_COORD(axisdata), axisdata_incr,
                     &NAD_SHAPE(axisdata), axisdata_incr,
-                    transfersize, dtypes[iiter]->elsize,
+                    transfersize, PyArray_DESCR(operands[iiter])->elsize,
                     stransfer,
                     transferdata);
         }
diff --git a/numpy/core/tests/test_new_iterator.py b/numpy/core/tests/test_new_iterator.py
index 06aa34560..d012ed2f5 100644
--- a/numpy/core/tests/test_new_iterator.py
+++ b/numpy/core/tests/test_new_iterator.py
@@ -1495,6 +1495,24 @@ def test_iter_buffering_badwriteback():
                         [['readwrite'],['writeonly']],
                         order='C')
 
+def test_iter_buffering_string():
+    # Safe casting disallows shrinking strings
+    a = np.array(['abc', 'a', 'abcd'], dtype=np.str)
+    assert_equal(a.dtype, np.dtype('S4'));
+    assert_raises(TypeError,newiter,a,['buffered'],['readonly'],
+                    op_dtypes='S2')
+    i = newiter(a, ['buffered'], ['readonly'], op_dtypes='S6')
+    assert_equal(i[0], 'abc')
+    assert_equal(i[0].dtype, np.dtype('S6'))
+
+    a = np.array(['abc', 'a', 'abcd'], dtype=np.unicode)
+    assert_equal(a.dtype, np.dtype('U4'));
+    assert_raises(TypeError,newiter,a,['buffered'],['readonly'],
+                    op_dtypes='U2')
+    i = newiter(a, ['buffered'], ['readonly'], op_dtypes='U6')
+    assert_equal(i[0], u'abc')
+    assert_equal(i[0].dtype, np.dtype('U6'))
+
 def test_iter_buffering_growinner():
     # Test that the inner loop grows when no buffering is needed
     a = np.arange(30)