diff options
Diffstat (limited to 'numpy')
| -rw-r--r-- | numpy/core/src/multiarray/lowlevel_strided_loops.c.src | 66 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/lowlevel_strided_loops.h | 14 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/new_iterator.c.src | 106 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/new_iterator.h | 8 | ||||
| -rw-r--r-- | numpy/core/src/multiarray/new_iterator_pywrap.c | 40 | ||||
| -rw-r--r-- | numpy/core/tests/test_new_iterator.py | 60 |
6 files changed, 282 insertions, 12 deletions
diff --git a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src index 880659f2a..2eb0229a8 100644 --- a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src +++ b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src @@ -658,10 +658,24 @@ NPY_NO_EXPORT PyArray_StridedTransferFn /* Does a simple aligned cast */ typedef struct { - void *freefunc; + void *freefunc, *copyfunc; PyArray_VectorUnaryFunc *castfunc; } _strided_cast_data; +/* strided cast data copy function */ +_strided_cast_data *_strided_cast_data_copy(_strided_cast_data *data) +{ + _strided_cast_data *newdata = + (_strided_cast_data *)PyArray_malloc(sizeof(_strided_cast_data)); + if (newdata == NULL) { + return NULL; + } + + memcpy(newdata, data, sizeof(_strided_cast_data)); + + return newdata; +} + static void _aligned_strided_to_strided_cast(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, @@ -691,7 +705,7 @@ _aligned_contig_to_contig_cast(char *dst, npy_intp NPY_UNUSED(dst_stride), /* Wraps a transfer function + data in alignment code */ typedef struct { - void *freefunc; + void *freefunc, *copyfunc; PyArray_StridedTransferFn wrapped, tobuffer, frombuffer; void *wrappeddata; @@ -706,6 +720,37 @@ void _align_wrap_data_free(_align_wrap_data *data) PyArray_free(data); } +/* transfer data copy function */ +_align_wrap_data *_align_wrap_data_copy(_align_wrap_data *data) +{ + _align_wrap_data *newdata; + npy_intp basedatasize, datasize; + + /* Round up the structure size to 16-byte boundary */ + basedatasize = (sizeof(_align_wrap_data)+15)&(-0x10); + /* Add space for two 32-element buffers */ + datasize = basedatasize + 32*data->src_itemsize + 32*data->dst_itemsize; + + /* Allocate the data, and populate it */ + newdata = (_align_wrap_data *)PyArray_malloc(datasize); + if (newdata == NULL) { + return NULL; + } + memcpy(newdata, data, basedatasize); + newdata->bufferin = (char *)newdata + basedatasize; + newdata->bufferout = newdata->bufferin + 32*newdata->src_itemsize; + if (newdata->wrappeddata != NULL) { + newdata->wrappeddata = + PyArray_CopyStridedTransferData(data->wrappeddata); + if (newdata->wrappeddata == NULL) { + PyArray_free(newdata); + return NULL; + } + } + + return newdata; +} + static void _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, @@ -775,6 +820,7 @@ PyArray_WrapTransferFunction(npy_intp src_itemsize, npy_intp dst_itemsize, /* Allocate the data, and populate it */ data = (_align_wrap_data *)PyArray_malloc(datasize); data->freefunc = (void *)&_align_wrap_data_free; + data->copyfunc = (void *)&_align_wrap_data_copy; data->tobuffer = tobuffer; data->frombuffer = frombuffer; data->wrapped = wrapped; @@ -847,7 +893,7 @@ PyArray_GetDTypeTransferFunction(int aligned, } /* Allocate the data for the casting */ - data = PyArray_malloc(sizeof(_strided_cast_data)); + data = (_strided_cast_data *)PyArray_malloc(sizeof(_strided_cast_data)); if (data == NULL) { PyErr_NoMemory(); *outstransfer = NULL; @@ -855,6 +901,7 @@ PyArray_GetDTypeTransferFunction(int aligned, return NPY_FAIL; } data->freefunc = (void*)&(PyArray_free); + data->copyfunc = (void*)&(_strided_cast_data_copy); data->castfunc = castfunc; @@ -944,6 +991,19 @@ PyArray_FreeStridedTransferData(void *transferdata) } } +typedef void *(*_npy_stridedtransfer_copy)(void *); +NPY_NO_EXPORT void * +PyArray_CopyStridedTransferData(void *transferdata) +{ + if (transferdata != NULL) { + _npy_stridedtransfer_copy copy = + *((_npy_stridedtransfer_copy *)transferdata + 1); + return copy(transferdata); + } + + return NULL; +} + NPY_NO_EXPORT npy_intp diff --git a/numpy/core/src/multiarray/lowlevel_strided_loops.h b/numpy/core/src/multiarray/lowlevel_strided_loops.h index 66cf60f02..52a941b1b 100644 --- a/numpy/core/src/multiarray/lowlevel_strided_loops.h +++ b/numpy/core/src/multiarray/lowlevel_strided_loops.h @@ -11,9 +11,10 @@ * and a casting operation, * * The 'transferdata' parameter is slightly special, and must always contain - * a pointer to a deallocation routine at its beginning. The function - * PyArray_FreeStridedTransferFn should be used to deallocate such - * pointers, and calls the function pointer. + * pointer to deallocation and copying routines at its beginning. The function + * PyArray_FreeStridedTransferData should be used to deallocate such + * pointers, and calls the first function pointer, while the function + * PyArray_CopyStridedTransferData should be used to copy it. * */ typedef void (*PyArray_StridedTransferFn)(char *dst, npy_intp dst_stride, @@ -29,6 +30,13 @@ NPY_NO_EXPORT void PyArray_FreeStridedTransferData(void *transferdata); /* + * Copies a PyArray_StridedTransferFunction data object. See + * the comment with the function typedef for more details. + */ +NPY_NO_EXPORT void * +PyArray_CopyStridedTransferData(void *transferdata); + +/* * Gives back a function pointer to a specialized function for copying * strided memory. Returns NULL if there is a problem with the inputs. * diff --git a/numpy/core/src/multiarray/new_iterator.c.src b/numpy/core/src/multiarray/new_iterator.c.src index b7f5a04fd..78d45d2c1 100644 --- a/numpy/core/src/multiarray/new_iterator.c.src +++ b/numpy/core/src/multiarray/new_iterator.c.src @@ -72,7 +72,7 @@ struct NpyIter_InternalOnly { npy_uint32 itflags; npy_uint16 ndim, niter; npy_intp itersize, iterstart, iterend; - /* iterindex is only used if NPY_ITER_RANGED was set */ + /* iterindex is only used if RANGED or BUFFERED was set */ npy_intp iterindex; /* The rest is variable */ char iter_flexdata; @@ -302,7 +302,7 @@ static void npyiter_copy_to_buffers(NpyIter *iter); /* The constructor for an iterator over multiple objects */ -NpyIter* +NpyIter * NpyIter_MultiNew(npy_intp niter, PyArrayObject **op_in, npy_uint32 flags, NPY_ORDER order, NPY_CASTING casting, npy_uint32 *op_flags, @@ -578,7 +578,7 @@ NpyIter_MultiNew(npy_intp niter, PyArrayObject **op_in, npy_uint32 flags, } /* The constructor for an iterator over one object */ -NpyIter* +NpyIter * NpyIter_New(PyArrayObject *op, npy_uint32 flags, NPY_ORDER order, NPY_CASTING casting, PyArray_Descr* dtype, @@ -600,6 +600,106 @@ NpyIter_New(PyArrayObject *op, npy_uint32 flags, } } +NpyIter * +NpyIter_Copy(NpyIter *iter) +{ + npy_uint32 itflags = NIT_ITFLAGS(iter); + npy_intp ndim = NIT_NDIM(iter); + npy_intp iiter, niter = NIT_NITER(iter); + int out_of_memory = 0; + + npy_intp size; + NpyIter *newiter; + PyArrayObject **objects; + PyArray_Descr **dtypes; + + /* Allocate memory for the new iterator */ + size = NIT_SIZEOF_ITERATOR(itflags, ndim, niter); + newiter = (NpyIter*)PyArray_malloc(size); + + /* Copy the raw values to the new iterator */ + memcpy(newiter, iter, size); + + /* Take ownership of references to the operands and dtypes */ + objects = NIT_OBJECTS(newiter); + dtypes = NIT_DTYPES(newiter); + for (iiter = 0; iiter < niter; ++iiter) { + Py_INCREF(objects[iiter]); + Py_INCREF(dtypes[iiter]); + } + + /* Allocate buffers and make copies of the transfer data if necessary */ + if (itflags&NPY_ITFLAG_BUFFER) { + NpyIter_BufferData *bufferdata; + npy_intp buffersize, itemsize; + char **buffers; + void **readtransferdata, **writetransferdata; + + bufferdata = NIT_BUFFERDATA(newiter); + buffers = NBF_BUFFERS(bufferdata); + readtransferdata = NBF_READTRANSFERDATA(bufferdata); + writetransferdata = NBF_WRITETRANSFERDATA(bufferdata); + buffersize = NBF_BUFFERSIZE(bufferdata); + + for (iiter = 0; iiter < niter; ++iiter) { + if (buffers[iiter] != NULL) { + if (out_of_memory) { + buffers[iiter] = NULL; + } + else { + itemsize = dtypes[iiter]->elsize; + buffers[iiter] = PyArray_malloc(itemsize*buffersize); + if (buffers[iiter] == NULL) { + out_of_memory = 1; + } + } + } + + if (readtransferdata[iiter] != NULL) { + if (out_of_memory) { + readtransferdata[iiter] = NULL; + } + else { + readtransferdata[iiter] = + PyArray_CopyStridedTransferData(readtransferdata[iiter]); + if (readtransferdata[iiter] == NULL) { + out_of_memory = 1; + } + } + } + + if (writetransferdata[iiter] != NULL) { + if (out_of_memory) { + writetransferdata[iiter] = NULL; + } + else { + writetransferdata[iiter] = + PyArray_CopyStridedTransferData(writetransferdata[iiter]); + if (writetransferdata[iiter] == NULL) { + out_of_memory = 1; + } + } + } + } + + /* Initialize the buffers to the current iterindex */ + if (!out_of_memory && NBF_SIZE(bufferdata) > 0) { + npyiter_goto_iterindex(newiter, NIT_ITERINDEX(newiter)); + + /* Prepare the next buffers and set iterend/size */ + npyiter_copy_to_buffers(newiter); + } + } + + if (out_of_memory) { + NpyIter_Deallocate(newiter); + PyErr_NoMemory(); + return NULL; + } + + return newiter; +} + int NpyIter_Deallocate(NpyIter *iter) { npy_uint32 itflags = NIT_ITFLAGS(iter); diff --git a/numpy/core/src/multiarray/new_iterator.h b/numpy/core/src/multiarray/new_iterator.h index 89d0be055..72e7d87ea 100644 --- a/numpy/core/src/multiarray/new_iterator.h +++ b/numpy/core/src/multiarray/new_iterator.h @@ -28,19 +28,23 @@ typedef enum { /* Allocate a new iterator for one array object */ -NpyIter* +NpyIter * NpyIter_New(PyArrayObject* op, npy_uint32 flags, NPY_ORDER order, NPY_CASTING casting, PyArray_Descr* dtype, npy_intp a_ndim, npy_intp *axes, npy_intp buffersize); /* Allocate a new iterator for multiple array objects */ -NpyIter* +NpyIter * NpyIter_MultiNew(npy_intp niter, PyArrayObject **op_in, npy_uint32 flags, NPY_ORDER order, NPY_CASTING casting, npy_uint32 *op_flags, PyArray_Descr **op_request_dtypes, npy_intp oa_ndim, npy_intp **op_axes, npy_intp buffersize); +/* Makes a copy of the iterator */ +NpyIter * +NpyIter_Copy(NpyIter *iter); + /* Deallocate an iterator */ int NpyIter_Deallocate(NpyIter* iter); diff --git a/numpy/core/src/multiarray/new_iterator_pywrap.c b/numpy/core/src/multiarray/new_iterator_pywrap.c index 49c6c3d6c..9a0b6cbfd 100644 --- a/numpy/core/src/multiarray/new_iterator_pywrap.c +++ b/numpy/core/src/multiarray/new_iterator_pywrap.c @@ -1126,6 +1126,43 @@ npyiter_reset(NewNpyArrayIterObject *self) Py_RETURN_NONE; } +/* + * Makes a copy of the iterator. Note that the nesting is not + * copied. + */ +static PyObject * +npyiter_copy(NewNpyArrayIterObject *self) +{ + NewNpyArrayIterObject *iter; + + if (self->iter == NULL) { + PyErr_SetString(PyExc_ValueError, + "Iterator is invalid"); + return NULL; + } + + /* Allocate the iterator */ + iter = (NewNpyArrayIterObject *)npyiter_new(&NpyIter_Type, NULL, NULL); + if (iter == NULL) { + return NULL; + } + + /* Copy the C iterator */ + iter->iter = NpyIter_Copy(self->iter); + if (iter->iter == NULL) { + Py_DECREF(iter); + return NULL; + } + + /* Cache some values for the member functions to use */ + npyiter_cache_values(iter); + + iter->started = self->started; + iter->finished = self->finished; + + return (PyObject *)iter; +} + static PyObject * npyiter_iternext(NewNpyArrayIterObject *self) { @@ -1550,7 +1587,6 @@ static PyObject *npyiter_iterrange_get(NewNpyArrayIterObject *self) static int npyiter_iterrange_set(NewNpyArrayIterObject *self, PyObject *value) { npy_intp istart = 0, iend = 0; - PyObject *item; if (self->iter == NULL) { PyErr_SetString(PyExc_ValueError, @@ -1869,6 +1905,8 @@ npyiter_seq_ass_item(NewNpyArrayIterObject *self, Py_ssize_t i, PyObject *v) static PyMethodDef npyiter_methods[] = { {"reset", (PyCFunction)npyiter_reset, METH_NOARGS, NULL}, + {"copy", (PyCFunction)npyiter_copy, METH_NOARGS, NULL}, + {"__copy__", (PyCFunction)npyiter_copy, METH_NOARGS, NULL}, {"iternext", (PyCFunction)npyiter_iternext, METH_NOARGS, NULL}, {"remove_coords", (PyCFunction)npyiter_remove_coords, METH_NOARGS, NULL}, {"remove_inner_loop", (PyCFunction)npyiter_remove_inner_loop, diff --git a/numpy/core/tests/test_new_iterator.py b/numpy/core/tests/test_new_iterator.py index 3d492cab5..b2b9bcb59 100644 --- a/numpy/core/tests/test_new_iterator.py +++ b/numpy/core/tests/test_new_iterator.py @@ -29,6 +29,7 @@ def iter_iterindices(i): def test_iter_refcount(): # Make sure the iterator doesn't leak + # Basic a = arange(6) dt = np.dtype('f4').newbyteorder() rc_a = sys.getrefcount(a) @@ -43,6 +44,26 @@ def test_iter_refcount(): assert_equal(sys.getrefcount(a), rc_a) assert_equal(sys.getrefcount(dt), rc_dt) + # With a copy + a = arange(6, dtype='f4') + dt = np.dtype('f4') + rc_a = sys.getrefcount(a) + rc_dt = sys.getrefcount(dt) + it = newiter(a, [], + [['readwrite']], + op_dtypes=[dt]) + rc2_a = sys.getrefcount(a) + rc2_dt = sys.getrefcount(dt) + it2 = it.copy() + assert_(sys.getrefcount(a) > rc2_a) + assert_(sys.getrefcount(dt) > rc2_dt) + it = None + assert_equal(sys.getrefcount(a), rc2_a) + assert_equal(sys.getrefcount(dt), rc2_dt) + it2 = None + assert_equal(sys.getrefcount(a), rc_a) + assert_equal(sys.getrefcount(dt), rc_dt) + def test_iter_best_order(): # The iterator should always find the iteration order # with increasing memory addresses @@ -976,6 +997,45 @@ def test_iter_op_axes_errors(): assert_raises(ValueError, newiter, [a,a], [], [['readonly']]*2, op_axes=[[0,1],[1,0]]) +def test_iter_copy(): + # Check that copying the iterator works correctly + a = arange(24).reshape(2,3,4) + + # Simple iterator + i = newiter(a) + j = i.copy() + assert_equal([x[()] for x in i], [x[()] for x in j]) + + i.iterindex = 3 + j = i.copy() + assert_equal([x[()] for x in i], [x[()] for x in j]) + + # Buffered iterator + i = newiter(a, ['buffered','ranged'], order='F', buffersize=3) + j = i.copy() + assert_equal([x[()] for x in i], [x[()] for x in j]) + + i.iterindex = 3 + j = i.copy() + assert_equal([x[()] for x in i], [x[()] for x in j]) + + i.iterrange = (3,9) + j = i.copy() + assert_equal([x[()] for x in i], [x[()] for x in j]) + + i.iterrange = (2,18) + i.next(); i.next() + j = i.copy() + assert_equal([x[()] for x in i], [x[()] for x in j]) + + # Casting iterator + i = newiter(a, ['buffered'], order='F', casting='unsafe', + op_dtypes='f8', buffersize=5) + j = i.copy() + i = None + assert_equal([x[()] for x in j], a.ravel(order='F')) + + def test_iter_allocate_output_simple(): # Check that the iterator will properly allocate outputs |