From 65b4a8a12e13d6451b089c6ca5f55c9da684a0ac Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 10 Mar 2011 10:52:24 -0800 Subject: DOC: Add some missing documentation, hyper-link the iterator documentation --- doc/source/reference/c-api.array.rst | 177 +++++++++++++++-- doc/source/reference/c-api.iterator.rst | 335 ++++++++++++++++++-------------- 2 files changed, 348 insertions(+), 164 deletions(-) (limited to 'doc/source/reference') diff --git a/doc/source/reference/c-api.array.rst b/doc/source/reference/c-api.array.rst index a548144bc..c89d686c6 100644 --- a/doc/source/reference/c-api.array.rst +++ b/doc/source/reference/c-api.array.rst @@ -160,15 +160,18 @@ From scratch .. cfunction:: PyObject* PyArray_NewLikeArray(PyArrayObject* prototype, NPY_ORDER order, PyArray_Descr* descr) + .. versionadded:: 1.6 + This function steals a reference to *descr* if it is not NULL. This array creation routine allows for the convenient creation of a new array matching an existing array's shapes and memory layout, possibly changing the layout and/or data type. - When *order* is NPY_ANYORDER, the result order is NPY_FORTRANORDER if - *prototype* is a fortran array, NPY_CORDER otherwise. When *order* is - NPY_KEEPORDER, the result order matches that of *prototype*, even + When *order* is :cdata:`NPY_ANYORDER`, the result order is + :cdata:`NPY_FORTRANORDER` if *prototype* is a fortran array, + :cdata:`NPY_CORDER` otherwise. When *order* is + :cdata:`NPY_KEEPORDER`, the result order matches that of *prototype*, even when the axes of *prototype* aren't in C or Fortran order. If *descr* is NULL, the data type of *prototype* is used. @@ -393,6 +396,68 @@ From other objects :cdata:`NPY_F_CONTIGUOUS` \| :cdata:`NPY_WRITEABLE` \| :cdata:`NPY_ALIGNED` \| :cdata:`NPY_UPDATEIFCOPY` +.. cfunction:: int PyArray_GetArrayParamsFromObject(PyObject* op, PyArray_Descr* requested_dtype, npy_bool writeable, PyArray_Descr** out_dtype, int* out_ndim, npy_intp* out_dims, PyArrayObject** out_arr, PyObject* context) + + .. versionadded:: 1.6 + + Retrieves the array parameters for viewing/converting an arbitrary + PyObject* to a NumPy array. This allows the "innate type and shape" + of Python list-of-lists to be discovered without + actually converting to an array. PyArray_FromAny calls this function + to analyze its input. + + In some cases, such as structured arrays and the __array__ interface, + a data type needs to be used to make sense of the object. When + this is needed, provide a Descr for 'requested_dtype', otherwise + provide NULL. This reference is not stolen. Also, if the requested + dtype doesn't modify the interpretation of the input, out_dtype will + still get the "innate" dtype of the object, not the dtype passed + in 'requested_dtype'. + + If writing to the value in 'op' is desired, set the boolean + 'writeable' to 1. This raises an error when 'op' is a scalar, list + of lists, or other non-writeable 'op'. This differs from passing + NPY_WRITEABLE to PyArray_FromAny, where the writeable array may + be a copy of the input. + + When success (0 return value) is returned, either out_arr + is filled with a non-NULL PyArrayObject and + the rest of the parameters are untouched, or out_arr is + filled with NULL, and the rest of the parameters are filled. + + Typical usage: + + .. code-block:: c + + PyArrayObject *arr = NULL; + PyArray_Descr *dtype = NULL; + int ndim = 0; + npy_intp dims[NPY_MAXDIMS]; + + if (PyArray_GetArrayParamsFromObject(op, NULL, 1, &dtype, + &ndim, &dims, &arr, NULL) < 0) { + return NULL; + } + if (arr == NULL) { + ... validate/change dtype, validate flags, ndim, etc ... + // Could make custom strides here too + arr = PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, + dims, NULL, + fortran ? NPY_F_CONTIGUOUS : 0, + NULL); + if (arr == NULL) { + return NULL; + } + if (PyArray_CopyObject(arr, op) < 0) { + Py_DECREF(arr); + return NULL; + } + } + else { + ... in this case the other parameters weren't filled, just + validate and possibly copy arr itself ... + } + ... use arr ... .. cfunction:: PyObject* PyArray_CheckFromAny(PyObject* op, PyArray_Descr* dtype, int min_depth, int max_depth, int requirements, PyObject* context) @@ -869,16 +934,20 @@ Converting data types .. cfunction:: int PyArray_CanCastTypeTo(PyArray_Descr* fromtype, PyArray_Descr* totype, NPY_CASTING casting) + .. versionadded:: 1.6 + Returns non-zero if an array of data type *fromtype* (which can include flexible types) can be cast safely to an array of data type *totype* (which can include flexible types) according to - the casting rule *casting*. For simple types with NPY_SAFE_CASTING, + the casting rule *casting*. For simple types with :cdata:`NPY_SAFE_CASTING`, this is basically a wrapper around :cfunc:`PyArray_CanCastSafely`, but for flexible types such as strings or unicode, it produces results taking into account their sizes. .. cfunction:: int PyArray_CanCastArrayTo(PyArrayObject* arr, PyArray_Descr* totype, NPY_CASTING casting) + .. versionadded:: 1.6 + Returns non-zero if *arr* can be cast to *totype* according to the casting rule given in *casting*. If *arr* is an array scalar, its value is taken into account, and non-zero is also @@ -887,6 +956,8 @@ Converting data types .. cfunction:: PyArray_Descr* PyArray_MinScalarType(PyArrayObject* arr) + .. versionadded:: 1.6 + If *arr* is an array, returns its data type descriptor, but if *arr* is an array scalar (has 0 dimensions), it finds the data type of smallest kind and size to which the value may be converted @@ -894,11 +965,15 @@ Converting data types .. cfunction:: PyArray_Descr* PyArray_PromoteTypes(PyArray_Descr* type1, PyArray_Descr* type2) + .. versionadded:: 1.6 + Finds the data type of smallest size and kind to which *type1* and *type2* may be safely converted. .. cfunction:: PyArray_Descr* PyArray_ResultType(npy_intp narrs, PyArrayObject**arrs, npy_intp ndtypes, PyArray_Descr**dtypes) + .. versionadded:: 1.6 + This applies PyArray_PromoteTypes to all the inputs, along with using the NumPy rules for combining scalars and arrays, to determine the output type of a set of operands. This is the @@ -929,9 +1004,9 @@ Converting data types .. cfunction:: PyArrayObject** PyArray_ConvertToCommonType(PyObject* op, int* n) - May be deprecated in the future. Using the newly introduced iterator - with flag NPY_ITER_COMMON_DTYPE or with the same dtype parameter for - all operands is preferred to this method. + May be deprecated in the future. Using the newly introduced + :ctype:`NpyIter` with flag :cdata:`NPY_ITER_COMMON_DTYPE` or with + the same dtype parameter for all operands is preferred to this method. Convert a sequence of Python objects contained in *op* to an array of ndarrays each having the same data type. The type is selected @@ -1596,6 +1671,8 @@ Item selection and manipulation .. cfunction:: npy_intp PyArray_CountNonzero(PyArrayObject* self) + .. versionadded:: 1.6 + Counts the number of non-zero elements in the array object *self*. .. cfunction:: PyObject* PyArray_Nonzero(PyArrayObject* self) @@ -1813,7 +1890,7 @@ Array Functions second-to-last dimension of *obj2*. For 2-d arrays this is a matrix-product. Neither array is conjugated. -.. cfunction:: PyObject* PyArray_MatrixProduct2(PyObject* obj1, PyObject* obj, PyObject *out) +.. cfunction:: PyObject* PyArray_MatrixProduct2(PyObject* obj1, PyObject* obj, PyObject* out) .. versionadded:: 1.6 @@ -1821,6 +1898,21 @@ Array Functions output array must have the correct shape, type, and be C-contiguous, or an exception is raised. +.. cfunction:: PyObject* PyArray_EinsteinSum(char* subscripts, npy_intp nop, PyArrayObject** op_in, PyArray_Descr* dtype, NPY_ORDER order, NPY_CASTING casting, PyArrayObject* out) + + .. versionadded:: 1.6 + + Applies the einstein summation convention to the array operands + provided, returning a new array or placing the result in *out*. + The string in *subscripts* is a comma separated list of index + letters. The number of operands is in *nop*, and *op_in* is an + array containing those operands. The data type of the output can + be forced with *dtype*, the output order can be forced with *order* + (:cdata:`NPY_KEEPORDER` is recommended), and when *dtype* is specified, + *casting* indicates how permissive the data conversion should be. + + See the :func:`einsum` function for more details. + .. cfunction:: PyObject* PyArray_CopyAndTranspose(PyObject \* op) A specialized copy and transpose function that works only for 2-d @@ -1901,6 +1993,9 @@ Other functions Array Iterators --------------- +As of Numpy 1.6, these array iterators are deprecated in favor of +the new array iterator, :ctype:`NpyIter`. + An array iterator is a simple way to access the elements of an N-dimensional array quickly and efficiently. Section `2 <#sec-array-iterator>`__ provides more description and examples of @@ -2415,15 +2510,16 @@ to. .. cfunction:: int PyArray_OrderConverter(PyObject* obj, NPY_ORDER* order) - Convert the Python strings 'C', 'F', 'A', and 'K' into the NPY_ORDER - enumeration NPY_CORDER, NPY_FORTRANORDER, NPY_ANYORDER, and NPY_KEEPORDER. + Convert the Python strings 'C', 'F', 'A', and 'K' into the :ctype:`NPY_ORDER` + enumeration :cdata:`NPY_CORDER`, :cdata:`NPY_FORTRANORDER`, + :cdata:`NPY_ANYORDER`, and :cdata:`NPY_KEEPORDER`. .. cfunction:: int PyArray_CastingConverter(PyObject* obj, NPY_CASTING* casting) Convert the Python strings 'no', 'equiv', 'safe', 'same_kind', and - 'unsafe' into the NPY_CASTING enumeration NPY_NO_CASTING, - NPY_EQUIV_CASTING, NPY_SAFE_CASTING, NPY_SAME_KIND_CASTING, and - NPY_UNSAFE_CASTING. + 'unsafe' into the NPY_CASTING enumeration :cdata:`NPY_NO_CASTING`, + :cdata:`NPY_EQUIV_CASTING`, :cdata:`NPY_SAFE_CASTING`, + :cdata:`NPY_SAME_KIND_CASTING`, and :cdata:`NPY_UNSAFE_CASTING`. Other conversions ^^^^^^^^^^^^^^^^^ @@ -2887,7 +2983,6 @@ Enumerated Types **INTNEG_SCALAR**, **FLOAT_SCALAR**, **COMPLEX_SCALAR**, **OBJECT_SCALAR** - .. cvar:: NPY_NSCALARKINDS Defined to be the number of scalar kinds @@ -2895,11 +2990,27 @@ Enumerated Types .. ctype:: NPY_ORDER - A variable type indicating the order that an array should be - interpreted in. The value of a variable of this type can be - :cdata:`NPY_{ORDER}` where ``{ORDER}`` is + An enumeration type indicating the element order that an array should be + interpreted in. When a brand new array is created, generally + only **NPY_CORDER** and **NPY_FORTRANORDER** are used, whereas + when one or more inputs are provided, the order can be based on them. + + .. cvar:: NPY_ANYORDER + + Fortran order if all the inputs are Fortran, C otherwise. + + .. cvar:: NPY_CORDER - **ANYORDER**, **CORDER**, **FORTRANORDER** + C order. + + .. cvar:: NPY_FORTRANORDER + + Fortran order. + + .. cvar:: NPY_KEEPORDER + + An order as close to the order of the inputs as possible, even + if the input is in neither C nor Fortran order. .. ctype:: NPY_CLIPMODE @@ -2909,5 +3020,35 @@ Enumerated Types **CLIP**, **WRAP**, **RAISE** +.. ctype:: NPY_CASTING + + .. versionadded:: 1.6 + + An enumeration type indicating how permissive data conversions should + be. This is used by the iterator added in NumPy 1.6, and is intended + to be used more broadly in a future version. + + .. cvar:: NPY_NO_CASTING + + Only allow identical types. + + .. cvar:: NPY_EQUIV_CASTING + + Allow identical and casts involving byte swapping. + + .. cvar:: NPY_SAFE_CASTING + + Only allow casts which will not cause values to be rounded, + truncated, or otherwise changed. + + .. cvar:: NPY_SAME_KIND_CASTING + + Allow any safe casts, and casts between types of the same kind. + For example, float64 -> float32 is permitted with this rule. + + .. cvar:: NPY_UNSAFE_CASTING + + Allow any cast, no matter what kind of data loss may occur. + .. index:: pair: ndarray; C-API diff --git a/doc/source/reference/c-api.iterator.rst b/doc/source/reference/c-api.iterator.rst index 62f179a4d..610f5aa1b 100644 --- a/doc/source/reference/c-api.iterator.rst +++ b/doc/source/reference/c-api.iterator.rst @@ -3,6 +3,10 @@ Array Iterator API .. sectionauthor:: Mark Wiebe +.. index:: + pair: iterator; C-API + pair: C-API; iterator + .. versionadded:: 1.6 Array Iterator @@ -31,52 +35,45 @@ existing iterator should be replaceable with the new iterator without significant effort. In 1.6, the major exception to this is the neighborhood iterator, which does not have corresponding features in this iterator. -Here is a conversion table for the regular iterator: - -=============================== ============================================= -``PyArray_IterNew`` ``NpyIter_New`` -``PyArray_IterAllButAxis`` ``NpyIter_New`` + ``axes`` parameter **or** - Iterator flag ``NPY_ITER_NO_INNER_ITERATION`` -``PyArray_BroadcastToShape`` **NOT SUPPORTED** (Use the support for - multiple operands instead.) -``PyArrayIter_Check`` Will need to add this in Python exposure -``PyArray_ITER_RESET`` ``NpyIter_Reset`` -``PyArray_ITER_NEXT`` Function pointer from ``NpyIter_GetIterNext`` -``PyArray_ITER_DATA`` ``NpyIter_GetDataPtrArray`` -``PyArray_ITER_GOTO`` ``NpyIter_GotoCoords`` -``PyArray_ITER_GOTO1D`` ``NpyIter_GotoIndex`` or - ``NpyIter_GotoIterIndex`` -``PyArray_ITER_NOTDONE`` Return value of ``iternext`` function pointer -=============================== ============================================= - -For the multi-iterator: - -=============================== ============================================= -``PyArray_MultiIterNew`` ``NpyIter_MultiNew`` -``PyArray_MultiIter_RESET`` ``NpyIter_Reset`` -``PyArray_MultiIter_NEXT`` Function pointer from ``NpyIter_GetIterNext`` -``PyArray_MultiIter_DATA`` ``NpyIter_GetDataPtrArray`` -``PyArray_MultiIter_NEXTi`` **NOT SUPPORTED** (always lock-step iteration) -``PyArray_MultiIter_GOTO`` ``NpyIter_GotoCoords`` -``PyArray_MultiIter_GOTO1D`` ``NpyIter_GotoIndex`` or - ``NpyIter_GotoIterIndex`` -``PyArray_MultiIter_NOTDONE`` Return value of ``iternext`` function pointer -``PyArray_Broadcast`` Handled by ``NpyIter_MultiNew`` -``PyArray_RemoveSmallest`` Iterator flag ``NPY_ITER_NO_INNER_ITERATION`` -=============================== ============================================= - -For other API calls: - -=============================== ============================================= -``PyArray_ConvertToCommonType`` Iterator flag ``NPY_ITER_COMMON_DTYPE`` -=============================== ============================================= +Here is a conversion table for which funcitons to use with the new iterator: + +===================================== ============================================= +*Iterator Functions* +:cfunc:`PyArray_IterNew` :cfunc:`NpyIter_New` +:cfunc:`PyArray_IterAllButAxis` :cfunc:`NpyIter_New` + ``axes`` parameter **or** + Iterator flag :cdata:`NPY_ITER_NO_INNER_ITERATION` +:cfunc:`PyArray_BroadcastToShape` **NOT SUPPORTED** (Use the support for + multiple operands instead.) +:cfunc:`PyArrayIter_Check` Will need to add this in Python exposure +:cfunc:`PyArray_ITER_RESET` :cfunc:`NpyIter_Reset` +:cfunc:`PyArray_ITER_NEXT` Function pointer from :cfunc:`NpyIter_GetIterNext` +:cfunc:`PyArray_ITER_DATA` :cfunc:`NpyIter_GetDataPtrArray` +:cfunc:`PyArray_ITER_GOTO` :cfunc:`NpyIter_GotoCoords` +:cfunc:`PyArray_ITER_GOTO1D` :cfunc:`NpyIter_GotoIndex` or + :cfunc:`NpyIter_GotoIterIndex` +:cfunc:`PyArray_ITER_NOTDONE` Return value of ``iternext`` function pointer +*Multi-iterator Functions* +:cfunc:`PyArray_MultiIterNew` :cfunc:`NpyIter_MultiNew` +:cfunc:`PyArray_MultiIter_RESET` :cfunc:`NpyIter_Reset` +:cfunc:`PyArray_MultiIter_NEXT` Function pointer from :cfunc:`NpyIter_GetIterNext` +:cfunc:`PyArray_MultiIter_DATA` :cfunc:`NpyIter_GetDataPtrArray` +:cfunc:`PyArray_MultiIter_NEXTi` **NOT SUPPORTED** (always lock-step iteration) +:cfunc:`PyArray_MultiIter_GOTO` :cfunc:`NpyIter_GotoCoords` +:cfunc:`PyArray_MultiIter_GOTO1D` :cfunc:`NpyIter_GotoIndex` or + :cfunc:`NpyIter_GotoIterIndex` +:cfunc:`PyArray_MultiIter_NOTDONE` Return value of ``iternext`` function pointer +:cfunc:`PyArray_Broadcast` Handled by :cfunc:`NpyIter_MultiNew` +:cfunc:`PyArray_RemoveSmallest` Iterator flag :cdata:`NPY_ITER_NO_INNER_ITERATION` +*Other Functions* +:cfunc:`PyArray_ConvertToCommonType` Iterator flag :cdata:`NPY_ITER_COMMON_DTYPE` +===================================== ============================================= Simple Iteration Example ------------------------ The best way to become familiar with the iterator is to look at its usage within the NumPy codebase itself. For example, here is a slightly -tweaked version of the code for ``PyArray_CountNonzero``, which counts the +tweaked version of the code for :cfunc:`PyArray_CountNonzero`, which counts the number of non-zero elements in an array. .. code-block:: c @@ -164,7 +161,7 @@ Simple Multi-Iteration Example Here is a simple copy function using the iterator. The ``order`` parameter is used to control the memory layout of the allocated result, typically -NPY_KEEPORDER is desired. +:cdata:`NPY_KEEPORDER` is desired. .. code-block:: c @@ -247,16 +244,26 @@ NPY_KEEPORDER is desired. } -Iterator Pointer Type +Iterator Data Types --------------------- The iterator layout is an internal detail, and user code only sees an incomplete struct. -.. code-block:: c +.. ctype:: NpyIter + + This is an opaque pointer type for the iterator. Access to its contents + can only be done through the iterator API. + +.. ctype:: NpyIter_IterNextFunc - typedef struct NpyIter_InternalOnly NpyIter; + This is a function pointer for the iteration loop, returned by + :cfunc:`NpyIter_GetIterNext`. +.. ctype:: NpyIter_GetCoordsFunc + + This is a function pointer for getting iterator coordinates, returned by + :cfunc:`NpyIter_GetGetCoords`. Construction and Destruction ---------------------------- @@ -267,22 +274,22 @@ Construction and Destruction Flags that may be passed in ``flags`` are any combination of the global and per-operand flags documented in - ``NpyIter_MultiNew``, except for ``NPY_ITER_ALLOCATE``. + :cfunc:`NpyIter_MultiNew`, except for :cdata:`NPY_ITER_ALLOCATE`. - Any of the ``NPY_ORDER`` enum values may be passed to ``order``. For - efficient iteration, ``NPY_KEEPORDER`` is the best option, and the other - orders enforce the particular iteration pattern. + Any of the :ctype:`NPY_ORDER` enum values may be passed to ``order``. For + efficient iteration, :ctype:`NPY_KEEPORDER` is the best option, and + the other orders enforce the particular iteration pattern. - Any of the ``NPY_CASTING`` enum values may be passed to ``casting``. - The values include ``NPY_NO_CASTING``, ``NPY_EQUIV_CASTING``, - ``NPY_SAFE_CASTING``, ``NPY_SAME_KIND_CASTING``, and - ``NPY_UNSAFE_CASTING``. To allow the casts to occur, copying or + Any of the :ctype:`NPY_CASTING` enum values may be passed to ``casting``. + The values include :cdata:`NPY_NO_CASTING`, :cdata:`NPY_EQUIV_CASTING`, + :cdata:`NPY_SAFE_CASTING`, :cdata:`NPY_SAME_KIND_CASTING`, and + :cdata:`NPY_UNSAFE_CASTING`. To allow the casts to occur, copying or buffering must also be enabled. If ``dtype`` isn't ``NULL``, then it requires that data type. If copying is allowed, it will make a temporary copy if the data - is castable. If ``UPDATEIFCOPY`` is enabled, it will also copy - the data back with another cast upon iterator destruction. + is castable. If :cdata:`NPY_ITER_UPDATEIFCOPY` is enabled, it will + also copy the data back with another cast upon iterator destruction. If ``a_ndim`` is greater than zero, ``axes`` must also be provided. In this case, ``axes`` is an ``a_ndim``-sized array of ``op``'s axes. @@ -327,17 +334,17 @@ Construction and Destruction See below for a description of these parameters, which allow for custom manual broadcasting as well as reordering and leaving out axes. - Any of the ``NPY_ORDER`` enum values may be passed to ``order``. For - efficient iteration, ``NPY_KEEPORDER`` is the best option, and the other - orders enforce the particular iteration pattern. When using - ``NPY_KEEPORDER``, if you also want to ensure that the iteration is + Any of the :ctype:`NPY_ORDER` enum values may be passed to ``order``. For + efficient iteration, :cdata:`NPY_KEEPORDER` is the best option, and the + other orders enforce the particular iteration pattern. When using + :cdata:`NPY_KEEPORDER`, if you also want to ensure that the iteration is not reversed along an axis, you should pass the flag - ``NPY_ITER_DONT_NEGATE_STRIDES``. + :cdata:`NPY_ITER_DONT_NEGATE_STRIDES`. - Any of the ``NPY_CASTING`` enum values may be passed to ``casting``. - The values include ``NPY_NO_CASTING``, ``NPY_EQUIV_CASTING``, - ``NPY_SAFE_CASTING``, ``NPY_SAME_KIND_CASTING``, and - ``NPY_UNSAFE_CASTING``. To allow the casts to occur, copying or + Any of the :ctype:`NPY_CASTING` enum values may be passed to ``casting``. + The values include :cdata:`NPY_NO_CASTING`, :cdata:`NPY_EQUIV_CASTING`, + :cdata:`NPY_SAFE_CASTING`, :cdata:`NPY_SAME_KIND_CASTING`, and + :cdata:`NPY_UNSAFE_CASTING`. To allow the casts to occur, copying or buffering must also be enabled. If ``op_dtypes`` isn't ``NULL``, it specifies a data type or ``NULL`` @@ -375,37 +382,42 @@ Construction and Destruction Flags that may be passed in ``flags``, applying to the whole iterator, are: - ``NPY_ITER_C_INDEX``, ``NPY_ITER_F_INDEX`` + .. cvar:: NPY_ITER_C_INDEX + + Causes the iterator to track a raveled flat index matching C + order. This option cannot be used with :cdata:`NPY_ITER_F_INDEX`. - Causes the iterator to track an index matching C or - Fortran order. These options are mutually exclusive. + .. cvar:: NPY_ITER_F_INDEX - ``NPY_ITER_COORDS`` + Causes the iterator to track a raveled flat index matching Fortran + order. This option cannot be used with :cdata:`NPY_ITER_C_INDEX`. + + .. cvar:: NPY_ITER_COORDS Causes the iterator to track array coordinates. This prevents the iterator from coalescing axes to produce bigger inner loops. - ``NPY_ITER_NO_INNER_ITERATION`` + .. cvar:: NPY_ITER_NO_INNER_ITERATION Causes the iterator to skip iteration of the innermost loop, allowing the user of the iterator to handle it. - This flag is incompatible with ``NPY_ITER_C_INDEX``, - ``NPY_ITER_F_INDEX``, and ``NPY_ITER_COORDS``. + This flag is incompatible with :cdata:`NPY_ITER_C_INDEX`, + :cdata:`NPY_ITER_F_INDEX`, and :cdata:`NPY_ITER_COORDS`. - ``NPY_ITER_DONT_NEGATE_STRIDES`` + .. cvar:: NPY_ITER_DONT_NEGATE_STRIDES - This only affects the iterator when NPY_KEEPORDER is specified - for the order parameter. By default with NPY_KEEPORDER, the - iterator reverses axes which have negative strides, so that - memory is traversed in a forward direction. This disables - this step. Use this flag if you want to use the underlying - memory-ordering of the axes, but don't want an axis reversed. - This is the behavior of ``numpy.ravel(a, order='K')``, for - instance. + This only affects the iterator when :ctype:`NPY_KEEPORDER` is + specified for the order parameter. By default with + :ctype:`NPY_KEEPORDER`, the iterator reverses axes which have + negative strides, so that memory is traversed in a forward + direction. This disables this step. Use this flag if you + want to use the underlying memory-ordering of the axes, + but don't want an axis reversed. This is the behavior of + ``numpy.ravel(a, order='K')``, for instance. - ``NPY_ITER_COMMON_DTYPE`` + .. cvar:: NPY_ITER_COMMON_DTYPE Causes the iterator to convert all the operands to a common data type, calculated based on the ufunc type promotion rules. @@ -414,23 +426,23 @@ Construction and Destruction If the common data type is known ahead of time, don't use this flag. Instead, set the requested dtype for all the operands. - ``NPY_ITER_REFS_OK`` + .. cvar:: NPY_ITER_REFS_OK Indicates that arrays with reference types (object arrays or structured arrays containing an object type) may be accepted and used in the iterator. If this flag is enabled, the caller must be sure to check whether - ``NpyIter_IterationNeedsAPI(iter)`` is true, in which case + :cfunc:`NpyIter_IterationNeedsAPI`(iter) is true, in which case it may not release the GIL during iteration. - ``NPY_ITER_ZEROSIZE_OK`` + .. cvar:: NPY_ITER_ZEROSIZE_OK Indicates that arrays with a size of zero should be permitted. Since the typical iteration loop does not naturally work with zero-sized arrays, you must check that the IterSize is non-zero before entering the iteration loop. - ``NPY_ITER_REDUCE_OK`` + .. cvar:: NPY_ITER_REDUCE_OK Permits writeable operands with a dimension with zero stride and size greater than one. Note that such operands @@ -441,55 +453,56 @@ Construction and Destruction not trample on values being reduced. Note that if you want to do a reduction on an automatically - allocated output, you must use ``NpyIter_GetOperandArray`` + allocated output, you must use :cfunc:`NpyIter_GetOperandArray` to get its reference, then set every value to the reduction unit before doing the iteration loop. In the case of a buffered reduction, this means you must also specify the - flag ``NPY_ITER_DELAY_BUFALLOC``, then reset the iterator + flag :cdata:`NPY_ITER_DELAY_BUFALLOC`, then reset the iterator after initializing the allocated operand to prepare the buffers. - ``NPY_ITER_RANGED`` + .. cvar:: NPY_ITER_RANGED Enables support for iteration of sub-ranges of the full ``iterindex`` range ``[0, NpyIter_IterSize(iter))``. Use - the function ``NpyIter_ResetToIterIndexRange`` to specify + the function :cfunc:`NpyIter_ResetToIterIndexRange` to specify a range for iteration. - This flag can only be used with ``NPY_ITER_NO_INNER_ITERATION`` - when ``NPY_ITER_BUFFERED`` is enabled. This is because + This flag can only be used with :cdata:`NPY_ITER_NO_INNER_ITERATION` + when :cdata:`NPY_ITER_BUFFERED` is enabled. This is because without buffering, the inner loop is always the size of the innermost iteration dimension, and allowing it to get cut up would require special handling, effectively making it more like the buffered version. - ``NPY_ITER_BUFFERED`` + .. cvar:: NPY_ITER_BUFFERED Causes the iterator to store buffering data, and use buffering to satisfy data type, alignment, and byte-order requirements. - To buffer an operand, do not specify the ``NPY_ITER_COPY`` - or ``NPY_ITER_UPDATEIFCOPY`` flags, because they will + To buffer an operand, do not specify the :cdata:`NPY_ITER_COPY` + or :cdata:`NPY_ITER_UPDATEIFCOPY` flags, because they will override buffering. Buffering is especially useful for Python code using the iterator, allowing for larger chunks of data at once to amortize the Python interpreter overhead. - If used with ``NPY_ITER_NO_INNER_ITERATION``, the inner loop + If used with :cdata:`NPY_ITER_NO_INNER_ITERATION`, the inner loop for the caller may get larger chunks than would be possible without buffering, because of how the strides are laid out. - Note that if an operand is given the flag ``NPY_ITER_COPY`` - or ``NPY_ITER_UPDATEIFCOPY``, a copy will be made in preference + Note that if an operand is given the flag :cdata:`NPY_ITER_COPY` + or :cdata:`NPY_ITER_UPDATEIFCOPY`, a copy will be made in preference to buffering. Buffering will still occur when the array was broadcast so elements need to be duplicated to get a constant stride. In normal buffering, the size of each inner loop is equal - to the buffer size, or possibly larger if ``NPY_ITER_GROWINNER`` - is specified. If ``NPY_ITER_REDUCE_OK`` is enabled and - a reduction occurs, the inner loops may become smaller depending + to the buffer size, or possibly larger if + :cdata:`NPY_ITER_GROWINNER` is specified. If + :cdata:`NPY_ITER_REDUCE_OK` is enabled and a reduction occurs, + the inner loops may become smaller depending on the structure of the reduction. - ``NPY_ITER_GROWINNER`` + .. cvar:: NPY_ITER_GROWINNER When buffering is enabled, this allows the size of the inner loop to grow when buffering isn't necessary. This option @@ -497,10 +510,10 @@ Construction and Destruction data, rather than anything with small cache-friendly arrays of temporary values for each inner loop. - ``NPY_ITER_DELAY_BUFALLOC`` + .. cvar:: NPY_ITER_DELAY_BUFALLOC When buffering is enabled, this delays allocation of the - buffers until one of the ``NpyIter_Reset*`` functions is + buffers until :cfunc:`NpyIter_Reset` or another reset function is called. This flag exists to avoid wasteful copying of buffer data when making multiple copies of a buffered iterator for multi-threaded iteration. @@ -509,27 +522,29 @@ Construction and Destruction After the iterator is created, and a reduction output is allocated automatically by the iterator (be sure to use READWRITE access), its value may be initialized to the reduction - unit. Use ``NpyIter_GetOperandArray`` to get the object. - Then, call ``NpyIter_Reset`` to allocate and fill the buffers + unit. Use :cfunc:`NpyIter_GetOperandArray` to get the object. + Then, call :cfunc:`NpyIter_Reset` to allocate and fill the buffers with their initial values. Flags that may be passed in ``op_flags[i]``, where ``0 <= i < niter``: - ``NPY_ITER_READWRITE``, ``NPY_ITER_READONLY``, ``NPY_ITER_WRITEONLY`` + .. cvar:: NPY_ITER_READWRITE + .. cvar:: NPY_ITER_READONLY + .. cvar:: NPY_ITER_WRITEONLY Indicate how the user of the iterator will read or write to ``op[i]``. Exactly one of these flags must be specified per operand. - ``NPY_ITER_COPY`` + .. cvar:: NPY_ITER_COPY Allow a copy of ``op[i]`` to be made if it does not meet the data type or alignment requirements as specified by the constructor flags and parameters. - ``NPY_ITER_UPDATEIFCOPY`` + .. cvar:: NPY_ITER_UPDATEIFCOPY - Triggers ``NPY_ITER_COPY``, and when an array operand + Triggers :cdata:`NPY_ITER_COPY`, and when an array operand is flagged for writing and is copied, causes the data in a copy to be copied back to ``op[i]`` when the iterator is destroyed. @@ -539,7 +554,9 @@ Construction and Destruction to back to ``op[i]`` on destruction, instead of doing the unecessary copy operation. - ``NPY_ITER_NBO``, ``NPY_ITER_ALIGNED``, ``NPY_ITER_CONTIG`` + .. cvar:: NPY_ITER_NBO + .. cvar:: NPY_ITER_ALIGNED + .. cvar:: NPY_ITER_CONTIG Causes the iterator to provide data for ``op[i]`` that is in native byte order, aligned according to @@ -558,10 +575,10 @@ Construction and Destruction the NBO flag overrides it and the requested data type is converted to be in native byte order. - ``NPY_ITER_ALLOCATE`` + .. cvar:: NPY_ITER_ALLOCATE This is for output arrays, and requires that the flag - ``NPY_ITER_WRITEONLY`` be set. If ``op[i]`` is NULL, + :cdata:`NPY_ITER_WRITEONLY` be set. If ``op[i]`` is NULL, creates a new array with the final broadcast dimensions, and a layout matching the iteration order of the iterator. @@ -576,20 +593,20 @@ Construction and Destruction output will be in native byte order. After being allocated with this flag, the caller may retrieve - the new array by calling ``NpyIter_GetOperandArray`` and + the new array by calling :cfunc:`NpyIter_GetOperandArray` and getting the i-th object in the returned C array. The caller must call Py_INCREF on it to claim a reference to the array. - ``NPY_ITER_NO_SUBTYPE`` + .. cvar:: NPY_ITER_NO_SUBTYPE - For use with ``NPY_ITER_ALLOCATE``, this flag disables + For use with :cdata:`NPY_ITER_ALLOCATE`, this flag disables allocating an array subtype for the output, forcing it to be a straight ndarray. TODO: Maybe it would be better to introduce a function ``NpyIter_GetWrappedOutput`` and remove this flag? - ``NPY_ITER_NO_BROADCAST`` + .. cvar:: NPY_ITER_NO_BROADCAST Ensures that the input or output matches the iteration dimensions exactly. @@ -603,19 +620,19 @@ Construction and Destruction The recommended approach to multithreaded iteration is to first create an iterator with the flags - ``NPY_ITER_NO_INNER_ITERATION``, ``NPY_ITER_RANGED``, - ``NPY_ITER_BUFFERED``, ``NPY_ITER_DELAY_BUFALLOC``, and - possibly ``NPY_ITER_GROWINNER``. Create a copy of this iterator + :cdata:`NPY_ITER_NO_INNER_ITERATION`, :cdata:`NPY_ITER_RANGED`, + :cdata:`NPY_ITER_BUFFERED`, :cdata:`NPY_ITER_DELAY_BUFALLOC`, and + possibly :cdata:`NPY_ITER_GROWINNER`. Create a copy of this iterator for each thread (minus one for the first iterator). Then, take the iteration index range ``[0, NpyIter_GetIterSize(iter))`` and split it up into tasks, for example using a TBB parallel_for loop. When a thread gets a task to execute, it then uses its copy of - the iterator by calling ``NpyIter_ResetToIterIndexRange`` and + the iterator by calling :cfunc:`NpyIter_ResetToIterIndexRange` and iterating over the full range. When using the iterator in multi-threaded code or in code not holding the Python GIL, care must be taken to only call functions - which are safe in that context. ``NpyIter_Copy`` cannot be safely + which are safe in that context. :cfunc:`NpyIter_Copy` cannot be safely called without the Python GIL, because it increments Python references. The ``Reset*`` and some other functions may be safely called by passing in the ``errmsg`` parameter as non-NULL, so that @@ -625,7 +642,7 @@ Construction and Destruction .. cfunction:: int NpyIter_RemoveAxis(NpyIter* iter, int axis)`` Removes an axis from iteration. This requires that - ``NPY_ITER_COORDS`` was set for iterator creation, and does not work + :cdata:`NPY_ITER_COORDS` was set for iterator creation, and does not work if buffering is enabled or an index is being tracked. This function also resets the iterator to its initial state. @@ -653,7 +670,7 @@ Construction and Destruction the iterator. Any cached functions or pointers from the iterator must be retrieved again! - After calling this function, ``NpyIter_HasCoords(iter)`` will + After calling this function, :cfunc:`NpyIter_HasCoords`(iter) will return false. Returns ``NPY_SUCCEED`` or ``NPY_FAIL``. @@ -661,9 +678,9 @@ Construction and Destruction .. cfunction:: int NpyIter_RemoveInnerLoop(NpyIter* iter) If RemoveCoords was used, you may want to specify the - flag ``NPY_ITER_NO_INNER_ITERATION``. This flag is not permitted - together with ``NPY_ITER_COORDS``, so this function is provided - to enable the feature after ``NpyIter_RemoveCoords`` is called. + flag :cdata:`NPY_ITER_NO_INNER_ITERATION`. This flag is not permitted + together with :cdata:`NPY_ITER_COORDS`, so this function is provided + to enable the feature after :cfunc:`NpyIter_RemoveCoords` is called. This function also resets the iterator to its initial state. **WARNING**: This function changes the internal logic of the iterator. @@ -693,9 +710,9 @@ Construction and Destruction .. cfunction:: int NpyIter_ResetToIterIndexRange(NpyIter* iter, npy_intp istart, npy_intp iend, char** errmsg) Resets the iterator and restricts it to the ``iterindex`` range - ``[istart, iend)``. See ``NpyIter_Copy`` for an explanation of + ``[istart, iend)``. See :cfunc:`NpyIter_Copy` for an explanation of how to use this for multi-threaded iteration. This requires that - the flag ``NPY_ITER_RANGED`` was passed to the iterator constructor. + the flag :cdata:`NPY_ITER_RANGED` was passed to the iterator constructor. If you want to reset both the ``iterindex`` range and the base pointers at the same time, you can do the following to avoid @@ -735,12 +752,12 @@ Construction and Destruction Creating iterators for nested iteration requires some care. All the iterator operands must match exactly, or the calls to - ``NpyIter_ResetBasePointers`` will be invalid. This means that + :cfunc:`NpyIter_ResetBasePointers` will be invalid. This means that automatic copies and output allocation should not be used haphazardly. It is possible to still use the automatic data conversion and casting features of the iterator by creating one of the iterators with all the conversion parameters enabled, then grabbing the allocated - operands with the ``NpyIter_GetOperandArray`` function and passing + operands with the :cfunc:`NpyIter_GetOperandArray` function and passing them into the constructors for the rest of the iterators. **WARNING**: When creating iterators for nested iteration, @@ -792,9 +809,9 @@ Construction and Destruction Adjusts the iterator to point to the ``index`` specified. If the iterator was constructed with the flag - ``NPY_ITER_C_INDEX``, ``index`` is the C-order index, + :cdata:`NPY_ITER_C_INDEX`, ``index`` is the C-order index, and if the iterator was constructed with the flag - ``NPY_ITER_F_INDEX``, ``index`` is the Fortran-order + :cdata:`NPY_ITER_F_INDEX`, ``index`` is the Fortran-order index. Returns an error if there is no index being tracked, the index is out of bounds, or inner loop iteration is disabled. @@ -813,7 +830,7 @@ Construction and Destruction .. cfunction:: void NpyIter_GetIterIndexRange(NpyIter* iter, npy_intp* istart, npy_intp* iend) Gets the ``iterindex`` sub-range that is being iterated. If - ``NPY_ITER_RANGED`` was not specified, this always returns the + :cdata:`NPY_ITER_RANGED` was not specified, this always returns the range ``[0, NpyIter_IterSize(iter))``. .. cfunction:: int NpyIter_GotoIterIndex(NpyIter* iter, npy_intp iterindex) @@ -825,32 +842,44 @@ Construction and Destruction Returns ``NPY_SUCCEED`` or ``NPY_FAIL``. +.. cfunction:: npy_bool NpyIter_HasDelayedBufAlloc(NpyIter* iter) + + Returns 1 if the flag :cdata:`NPY_ITER_DELAY_BUFALLOC` was passed + to the iterator constructor, and no call to one of the Reset + functions has been done yet, 0 otherwise. + .. cfunction:: npy_bool NpyIter_HasInnerLoop(NpyIter* iter) Returns 1 if the iterator handles the inner loop, or 0 if the caller needs to handle it. This is controlled - by the constructor flag ``NPY_ITER_NO_INNER_ITERATION``. + by the constructor flag :cdata:`NPY_ITER_NO_INNER_ITERATION`. .. cfunction:: npy_bool NpyIter_HasCoords(NpyIter* iter) Returns 1 if the iterator was created with the - ``NPY_ITER_COORDS`` flag, 0 otherwise. + :cdata:`NPY_ITER_COORDS` flag, 0 otherwise. .. cfunction:: npy_bool NpyIter_HasIndex(NpyIter* iter) Returns 1 if the iterator was created with the - ``NPY_ITER_C_INDEX`` or ``NPY_ITER_F_INDEX`` + :cdata:`NPY_ITER_C_INDEX` or :cdata:`NPY_ITER_F_INDEX` flag, 0 otherwise. +.. cfunction:: npy_bool NpyIter_RequiresBuffering(NpyIter* iter) + + Returns 1 if the iterator requires buffering, which occurs + when an operand needs conversion or alignment and so cannot + be used directly. + .. cfunction:: npy_bool NpyIter_IsBuffered(NpyIter* iter) Returns 1 if the iterator was created with the - ``NPY_ITER_BUFFERED`` flag, 0 otherwise. + :cdata:`NPY_ITER_BUFFERED` flag, 0 otherwise. .. cfunction:: npy_bool NpyIter_IsGrowInner(NpyIter* iter) Returns 1 if the iterator was created with the - ``NPY_ITER_GROWINNER`` flag, 0 otherwise. + :cdata:`NPY_ITER_GROWINNER` flag, 0 otherwise. .. cfunction:: npy_intp NpyIter_GetBufferSize(NpyIter* iter) @@ -875,7 +904,7 @@ Construction and Destruction be enabled. This may be used when you want to match up operand axes in - some fashion, then remove them with ``NpyIter_RemoveAxis`` to + some fashion, then remove them with :cfunc:`NpyIter_RemoveAxis` to handle their processing manually. By calling this function before removing the axes, you can get the strides for the manual processing. @@ -907,7 +936,7 @@ Construction and Destruction .. cfunction:: PyObject* NpyIter_GetIterView(NpyIter* iter, npy_intp i) This gives back a reference to a new ndarray view, which is a view - into the i-th object in the array ``NpyIter_GetOperandArray()``, + into the i-th object in the array :cfunc:`NpyIter_GetOperandArray`(), whose dimensions and strides match the internal optimized iteration pattern. A C-order iteration of this view is equivalent to the iterator's iteration order. @@ -955,7 +984,7 @@ Functions For Iteration /* use the addresses dataptr[0], ... dataptr[niter-1] */ } while(iternext(iter)); - When ``NPY_ITER_NO_INNER_ITERATION`` is specified, the typical + When :cdata:`NPY_ITER_NO_INNER_ITERATION` is specified, the typical inner loop construct is as follows. .. code-block:: c @@ -982,11 +1011,11 @@ Functions For Iteration with fresh values, not incrementally updated. If a compile-time fixed buffer is being used (both flags - ``NPY_ITER_BUFFERED`` and ``NPY_ITER_NO_INNER_ITERATION``), the + :cdata:`NPY_ITER_BUFFERED` and :cdata:`NPY_ITER_NO_INNER_ITERATION`), the inner size may be used as a signal as well. The size is guaranteed to become zero when ``iternext()`` returns false, enabling the following loop construct. Note that if you use this construct, - you should not pass ``NPY_ITER_GROWINNER`` as a flag, because it + you should not pass :cdata:`NPY_ITER_GROWINNER` as a flag, because it will cause larger sizes under some circumstances. .. code-block:: c @@ -1045,7 +1074,7 @@ Functions For Iteration .. cfunction:: char** NpyIter_GetDataPtrArray(NpyIter* iter) This gives back a pointer to the ``niter`` data pointers. If - ``NPY_ITER_NO_INNER_ITERATION`` was not specified, each data + :cdata:`NPY_ITER_NO_INNER_ITERATION` was not specified, each data pointer points to the current data item of the iterator. If no inner iteration was specified, it points to the first data item of the inner loop. @@ -1054,14 +1083,25 @@ Functions For Iteration ``iternext`` will not change it. This function may be safely called without holding the Python GIL. +.. cfunction:: char** NpyIter_GetInitialDataPtrArray(NpyIter* iter) + + Gets the array of data pointers directly into the arrays (never + into the buffers), corresponding to iteration index 0. + + These pointers are different from the pointers accepted by + ``NpyIter_ResetBasePointers``, because the direction along + some axes may have been reversed. + + This function may be safely called without holding the Python GIL. + .. cfunction:: npy_intp* NpyIter_GetIndexPtr(NpyIter* iter) This gives back a pointer to the index being tracked, or NULL if no index is being tracked. It is only useable if one of - the flags ``NPY_ITER_C_INDEX`` or ``NPY_ITER_F_INDEX`` + the flags :cdata:`NPY_ITER_C_INDEX` or :cdata:`NPY_ITER_F_INDEX` were specified during construction. -When the flag ``NPY_ITER_NO_INNER_ITERATION`` is used, the code +When the flag :cdata:`NPY_ITER_NO_INNER_ITERATION` is used, the code needs to know the parameters for doing the inner loop. These functions provide that information. @@ -1091,7 +1131,7 @@ functions provide that information. NPY_MAX_INTP is placed in the stride. Once the iterator is prepared for iteration (after a reset if - ``NPY_DELAY_BUFALLOC`` was used), call this to get the strides + :cdata:`NPY_DELAY_BUFALLOC` was used), call this to get the strides which may be used to select a fast inner loop function. For example, if the stride is 0, that means the inner loop can always load its value into a variable once, then use the variable throughout the loop, @@ -1099,3 +1139,6 @@ functions provide that information. operand may be used. This function may be safely called without holding the Python GIL. + +.. index:: + pair: iterator; C-API -- cgit v1.2.1