MAINT: Add a PyDataType_ISUNSIZED macro

This allows us to change how flexible types with no length are represented in future, to allow zero-size dtypes (#8970).

9 files changed, 49 insertions, 32 deletions

diff --git a/doc/source/reference/c-api.array.rst b/doc/source/reference/c-api.array.rst
index 79a886e71..7e42d3c5f 100644
--- a/doc/source/reference/c-api.array.rst
+++ b/doc/source/reference/c-api.array.rst
@@ -947,6 +947,12 @@ argument must be a :c:type:`PyObject *<PyObject>` that can be directly interpret
     Type represents one of the flexible array types ( :c:data:`NPY_STRING`,
     :c:data:`NPY_UNICODE`, or :c:data:`NPY_VOID` ).
 
+.. c:function:: PyDataType_ISUNSIZED(descr):
+
+    Type has no size information attached, and can be resized. Should only be
+    called on flexible dtypes. Types that are attached to an array will always
+    be sized, hence the array form of this macro not existing.
+
 .. c:function:: PyTypeNum_ISUSERDEF(num)
 
 .. c:function:: PyDataType_ISUSERDEF(descr)
diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h
index e0df189f9..8c5d855df 100644
--- a/numpy/core/include/numpy/ndarraytypes.h
+++ b/numpy/core/include/numpy/ndarraytypes.h
@@ -1676,6 +1676,8 @@ PyArray_CLEARFLAGS(PyArrayObject *arr, int flags)
 #define PyDataType_ISOBJECT(obj) PyTypeNum_ISOBJECT(((PyArray_Descr*)(obj))->type_num)
 #define PyDataType_HASFIELDS(obj) (((PyArray_Descr *)(obj))->names != NULL)
 #define PyDataType_HASSUBARRAY(dtype) ((dtype)->subarray != NULL)
+#define PyDataType_ISUNSIZED(dtype) ((dtype)->elsize == 0)
+#define PyDataType_MAKEUNSIZED(dtype) ((dtype)->elsize = 0)
 
 #define PyArray_ISBOOL(obj) PyTypeNum_ISBOOL(PyArray_TYPE(obj))
 #define PyArray_ISUNSIGNED(obj) PyTypeNum_ISUNSIGNED(PyArray_TYPE(obj))
diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src
index 6023365ed..dc89a2318 100644
--- a/numpy/core/src/multiarray/arraytypes.c.src
+++ b/numpy/core/src/multiarray/arraytypes.c.src
@@ -4746,6 +4746,15 @@ set_typeinfo(PyObject *dict)
 
     /**end repeat**/
 
+
+    /**begin repeat
+      * #name = STRING, UNICODE, VOID#
+      */
+
+    PyDataType_MAKEUNSIZED(&@name@_Descr);
+
+    /**end repeat**/
+
     /* Set a dictionary with type information */
     infodict = PyDict_New();
     if (infodict == NULL) return -1;
diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c
index f0f18eead..9927ffb6f 100644
--- a/numpy/core/src/multiarray/convert_datatype.c
+++ b/numpy/core/src/multiarray/convert_datatype.c
@@ -167,7 +167,7 @@ PyArray_AdaptFlexibleDType(PyObject *data_obj, PyArray_Descr *data_dtype,
     flex_type_num = (*flex_dtype)->type_num;
 
     /* Flexible types with expandable size */
-    if ((*flex_dtype)->elsize == 0) {
+    if (PyDataType_ISUNSIZED(*flex_dtype)) {
         /* First replace the flex dtype */
         PyArray_DESCR_REPLACE(*flex_dtype);
         if (*flex_dtype == NULL) {
@@ -526,7 +526,7 @@ PyArray_CanCastTo(PyArray_Descr *from, PyArray_Descr *to)
             }
 
             ret = 0;
-            if (to->elsize == 0) {
+            if (PyDataType_ISUNSIZED(to)) {
                 ret = 1;
             }
             /* 
@@ -1152,7 +1152,7 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2)
             else if (PyTypeNum_ISNUMBER(type_num2)) {
                 PyArray_Descr *ret = NULL;
                 PyArray_Descr *temp = PyArray_DescrNew(type1);
-                temp->elsize = 0;
+                PyDataType_MAKEUNSIZED(temp);
                 PyArray_AdaptFlexibleDType(NULL, type2, &temp);
                 if (temp->elsize > type1->elsize) {
                     ret = ensure_dtype_nbo(temp);
@@ -1190,7 +1190,7 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2)
             else if (PyTypeNum_ISNUMBER(type_num2)) {
                 PyArray_Descr *ret = NULL;
                 PyArray_Descr *temp = PyArray_DescrNew(type1);
-                temp->elsize = 0;
+                PyDataType_MAKEUNSIZED(temp);
                 PyArray_AdaptFlexibleDType(NULL, type2, &temp);
                 if (temp->elsize > type1->elsize) {
                     ret = ensure_dtype_nbo(temp);
@@ -1238,7 +1238,7 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2)
             if (PyTypeNum_ISNUMBER(type_num1)) {
                 PyArray_Descr *ret = NULL;
                 PyArray_Descr *temp = PyArray_DescrNew(type2);
-                temp->elsize = 0;
+                PyDataType_MAKEUNSIZED(temp);
                 PyArray_AdaptFlexibleDType(NULL, type1, &temp);
                 if (temp->elsize > type2->elsize) {
                     ret = ensure_dtype_nbo(temp);
@@ -1255,7 +1255,7 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2)
             if (PyTypeNum_ISNUMBER(type_num1)) {
                 PyArray_Descr *ret = NULL;
                 PyArray_Descr *temp = PyArray_DescrNew(type2);
-                temp->elsize = 0;
+                PyDataType_MAKEUNSIZED(temp);
                 PyArray_AdaptFlexibleDType(NULL, type1, &temp);
                 if (temp->elsize > type2->elsize) {
                     ret = ensure_dtype_nbo(temp);
diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c
index 701f1da73..d31a9cf74 100644
--- a/numpy/core/src/multiarray/ctors.c
+++ b/numpy/core/src/multiarray/ctors.c
@@ -934,7 +934,7 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
 
     /* Check datatype element size */
     nbytes = descr->elsize;
-    if (nbytes == 0) {
+    if (PyDataType_ISUNSIZED(descr)) {
         if (!PyDataType_ISFLEXIBLE(descr)) {
             PyErr_SetString(PyExc_TypeError, "Empty data-type");
             Py_DECREF(descr);
@@ -1255,7 +1255,7 @@ PyArray_New(PyTypeObject *subtype, int nd, npy_intp *dims, int type_num,
     if (descr == NULL) {
         return NULL;
     }
-    if (descr->elsize == 0) {
+    if (PyDataType_ISUNSIZED(descr)) {
         if (itemsize < 1) {
             PyErr_SetString(PyExc_ValueError,
                             "data type must provide an itemsize");
@@ -1617,7 +1617,7 @@ PyArray_GetArrayParamsFromObject(PyObject *op,
         }
 
         /* If the type is flexible, determine its size */
-        if ((*out_dtype)->elsize == 0 &&
+        if (PyDataType_ISUNSIZED(*out_dtype) &&
                             PyTypeNum_ISEXTENDED((*out_dtype)->type_num)) {
             int itemsize = 0;
             int string_type = 0;
@@ -1886,7 +1886,6 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags)
 {
 
     PyArrayObject *ret = NULL;
-    int itemsize;
     int copy = 0;
     int arrflags;
     PyArray_Descr *oldtype;
@@ -1905,14 +1904,12 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags)
         newtype = oldtype;
         Py_INCREF(oldtype);
     }
-    itemsize = newtype->elsize;
-    if (itemsize == 0) {
+    if (PyDataType_ISUNSIZED(newtype)) {
         PyArray_DESCR_REPLACE(newtype);
         if (newtype == NULL) {
             return NULL;
         }
         newtype->elsize = oldtype->elsize;
-        itemsize = newtype->elsize;
     }
 
     /* If the casting if forced, use the 'unsafe' casting rule */
@@ -3460,7 +3457,7 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type,
         Py_DECREF(type);
         return NULL;
     }
-    if (type->elsize == 0) {
+    if (PyDataType_ISUNSIZED(type)) {
         PyErr_SetString(PyExc_ValueError,
                         "itemsize cannot be zero in type");
         Py_DECREF(type);
@@ -3677,12 +3674,13 @@ PyArray_FromIter(PyObject *obj, PyArray_Descr *dtype, npy_intp count)
     if (iter == NULL) {
         goto done;
     }
-    elcount = (count < 0) ? 0 : count;
-    if ((elsize = dtype->elsize) == 0) {
+    if (PyDataType_ISUNSIZED(dtype)) {
         PyErr_SetString(PyExc_ValueError,
                 "Must specify length when using variable-size data-type.");
         goto done;
     }
+    elcount = (count < 0) ? 0 : count;
+    elsize = dtype->elsize;
 
     /*
      * We would need to alter the memory RENEW code to decrement any
diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c
index 49f086d21..e3c49b314 100644
--- a/numpy/core/src/multiarray/descriptor.c
+++ b/numpy/core/src/multiarray/descriptor.c
@@ -267,7 +267,7 @@ _convert_from_tuple(PyObject *obj)
      * We get here if res was NULL but errflag wasn't set
      * --- i.e. the conversion to a data-descr failed in _use_inherit
      */
-    if (type->elsize == 0) {
+    if (PyDataType_ISUNSIZED(type)) {
         /* interpret next item as a typesize */
         int itemsize = PyArray_PyIntAsInt(PyTuple_GET_ITEM(obj,1));
 
@@ -846,15 +846,18 @@ _use_inherit(PyArray_Descr *type, PyObject *newobj, int *errflag)
     if (new == NULL) {
         goto fail;
     }
-    if (new->elsize && new->elsize != conv->elsize) {
+    if (PyDataType_ISUNSIZED(new)) {
+        new->elsize = conv->elsize;
+    }
+    else if (new->elsize != conv->elsize) {
         PyErr_SetString(PyExc_ValueError,
                 "mismatch in size of old and new data-descriptor");
         goto fail;
     }
-    if (new->elsize && invalid_union_object_dtype(new, conv)) {
+    else if (invalid_union_object_dtype(new, conv)) {
         goto fail;
     }
-    new->elsize = conv->elsize;
+
     if (PyDataType_HASFIELDS(conv)) {
         Py_XDECREF(new->fields);
         new->fields = conv->fields;
@@ -1646,7 +1649,7 @@ finish:
         goto fail;
     }
 
-    if (((*at)->elsize == 0) && (elsize != 0)) {
+    if (PyDataType_ISUNSIZED(*at) && (*at)->elsize != elsize) {
         PyArray_DESCR_REPLACE(*at);
         (*at)->elsize = elsize;
     }
@@ -1896,7 +1899,7 @@ arraydescr_typename_get(PyArray_Descr *self)
         len -= suffix_len;
         res = PyUString_FromStringAndSize(typeobj->tp_name+prefix_len, len);
     }
-    if (PyTypeNum_ISFLEXIBLE(self->type_num) && self->elsize != 0) {
+    if (PyTypeNum_ISFLEXIBLE(self->type_num) && !PyDataType_ISUNSIZED(self)) {
         PyObject *p;
         p = PyUString_FromFormat("%d", self->elsize * 8);
         PyUString_ConcatAndDel(&res, p);
@@ -3551,7 +3554,7 @@ arraydescr_construction_repr(PyArray_Descr *dtype, int includealignflag,
             return PyUString_FromString("'O'");
 
         case NPY_STRING:
-            if (dtype->elsize == 0) {
+            if (PyDataType_ISUNSIZED(dtype)) {
                 return PyUString_FromString("'S'");
             }
             else {
@@ -3559,7 +3562,7 @@ arraydescr_construction_repr(PyArray_Descr *dtype, int includealignflag,
             }
 
         case NPY_UNICODE:
-            if (dtype->elsize == 0) {
+            if (PyDataType_ISUNSIZED(dtype)) {
                 return PyUString_FromFormat("'%sU'", byteorder);
             }
             else {
@@ -3568,7 +3571,7 @@ arraydescr_construction_repr(PyArray_Descr *dtype, int includealignflag,
             }
 
         case NPY_VOID:
-            if (dtype->elsize == 0) {
+            if (PyDataType_ISUNSIZED(dtype)) {
                 return PyUString_FromString("'V'");
             }
             else {
diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c
index d58071239..a43675040 100644
--- a/numpy/core/src/multiarray/getset.c
+++ b/numpy/core/src/multiarray/getset.c
@@ -470,12 +470,12 @@ array_descr_set(PyArrayObject *self, PyObject *arg)
     }
 
     /*
-     * Treat V0 as resizable void - unless the destination is already V0, then
-     * don't allow np.void to be duplicated
+     * Viewing as an unsized void implies a void dtype matching the size of the
+     * current dtype.
      */
     if (newtype->type_num == NPY_VOID &&
-            newtype->elsize == 0 &&
-            PyArray_DESCR(self)->elsize != 0) {
+            PyDataType_ISUNSIZED(newtype) &&
+            newtype->elsize != PyArray_DESCR(self)->elsize) {
         PyArray_DESCR_REPLACE(newtype);
         if (newtype == NULL) {
             return -1;
@@ -483,7 +483,6 @@ array_descr_set(PyArrayObject *self, PyObject *arg)
         newtype->elsize = PyArray_DESCR(self)->elsize;
     }
 
-
     /* Changing the size of the dtype results in a shape change */
     if (newtype->elsize != PyArray_DESCR(self)->elsize) {
         int axis;
diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c
index 0cb6b072d..1941f849f 100644
--- a/numpy/core/src/multiarray/scalarapi.c
+++ b/numpy/core/src/multiarray/scalarapi.c
@@ -567,7 +567,7 @@ PyArray_DescrFromScalar(PyObject *sc)
     }
 
     descr = PyArray_DescrFromTypeObject((PyObject *)Py_TYPE(sc));
-    if (descr->elsize == 0) {
+    if (PyDataType_ISUNSIZED(descr)) {
         PyArray_DESCR_REPLACE(descr);
         type_num = descr->type_num;
         if (type_num == NPY_STRING) {
diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c
index c32a710de..8e8090002 100644
--- a/numpy/core/src/multiarray/usertypes.c
+++ b/numpy/core/src/multiarray/usertypes.c
@@ -146,7 +146,7 @@ PyArray_RegisterDataType(PyArray_Descr *descr)
     }
     typenum = NPY_USERDEF + NPY_NUMUSERTYPES;
     descr->type_num = typenum;
-    if (descr->elsize == 0) {
+    if (PyDataType_ISUNSIZED(descr)) {
         PyErr_SetString(PyExc_ValueError, "cannot register a" \
                         "flexible data-type");
         return -1;