Fixup for scalar kind, and ensure OBJECT is special for assignment

5 files changed, 86 insertions, 41 deletions

diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h
index ab8ffaa84..0eeaed599 100644
--- a/numpy/core/include/numpy/ndarraytypes.h
+++ b/numpy/core/include/numpy/ndarraytypes.h
@@ -1822,8 +1822,14 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
     typedef PyArray_Descr *(discover_descr_from_pyobject_function)(
             PyArray_DTypeMeta *cls, PyObject *obj);
 
-    typedef int (is_known_scalar_function)(
-            PyArray_DTypeMeta *cls, PyObject *obj);
+    /*
+     * Before making this public, we should decide whether it should pass
+     * the type, or allow looking at the object. A possible use-case:
+     * `np.array(np.array([0]), dtype=np.ndarray)`
+     * Could consider arrays that are not `dtype=ndarray` "scalars".
+     */
+    typedef int (is_known_scalar_type_function)(
+            PyArray_DTypeMeta *cls, PyTypeObject *obj);
 
     typedef PyArray_Descr *(default_descr_function)(PyArray_DTypeMeta *cls);
 
@@ -1879,7 +1885,7 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
 
         /* DType methods, these could be moved into its own struct */
         discover_descr_from_pyobject_function *discover_descr_from_pyobject;
-        is_known_scalar_function *is_known_scalar;
+        is_known_scalar_type_function *is_known_scalar_type;
         default_descr_function *default_descr;
     };
 
diff --git a/numpy/core/src/multiarray/abstractdtypes.c b/numpy/core/src/multiarray/abstractdtypes.c
index 5cb4df85b..b5294505c 100644
--- a/numpy/core/src/multiarray/abstractdtypes.c
+++ b/numpy/core/src/multiarray/abstractdtypes.c
@@ -95,9 +95,9 @@ initialize_abstract_dtypes_and_map_others()
 
     /*
      * Map str, bytes, and bool, for which we do not need abstract versions
-     * to the NumPy DTypes. This is done here using the `is_known_scalar`
+     * to the NumPy DTypes. This is done here using the `is_known_scalar_type`
      * function.
-     * TODO: The `is_known_scalar` function was considered preliminary,
+     * TODO: The `is_known_scalar_type` function was considered preliminary,
      *       the same could be achieved e.g. with additional abstract DTypes.
      */
     PyArray_DTypeMeta *dtype;
diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c
index 9fbc0fa0f..8c31ca6a5 100644
--- a/numpy/core/src/multiarray/array_coercion.c
+++ b/numpy/core/src/multiarray/array_coercion.c
@@ -188,7 +188,9 @@ _PyArray_MapPyTypeToDType(
 static NPY_INLINE PyArray_DTypeMeta *
 discover_dtype_from_pytype(PyTypeObject *pytype)
 {
-    PyObject *weakref = PyDict_GetItem(
+    PyObject *weakref;
+
+    weakref = PyDict_GetItem(
             _global_pytype_to_type_dict, (PyObject *)pytype);
 
     if (weakref == NULL) {
@@ -216,18 +218,22 @@ discover_dtype_from_pytype(PyTypeObject *pytype)
         }
     }
     else {
-        assert(PyObject_IsInstance(DType, (PyObject *)&PyArrayDTypeMeta_Type));
+        assert(PyObject_TypeCheck(DType, (PyTypeObject *)&PyArrayDTypeMeta_Type));
     }
     return (PyArray_DTypeMeta *)DType;
 }
 
 
 /**
- * Find the correct DType class for the given python type.
+ * Find the correct DType class for the given python type. If flags is NULL
+ * this is not used to discover a dtype, but only for conversion to an
+ * existing dtype. In that case the Python (not NumPy) scalar subclass
+ * checks are skipped.
  *
  * @param obj The python object, mainly type(pyobj) is used, the object
  *        is passed to reuse existing code at this time only.
- * @param flags Flags used to know if warnings were already given.
+ * @param flags Flags used to know if warnings were already given. If
+ *        flags is NULL, this is not
  * @param fixed_DType if not NULL, will be checked first for whether or not
  *        it can/wants to handle the (possible) scalar value.
  * @return New reference to either a DType class, Py_None, or NULL
@@ -249,15 +255,8 @@ discover_dtype_from_pyobject(
          *   4. NULL in case of an error.
          */
         if ((Py_TYPE(obj) == fixed_DType->scalar_type) ||
-                (fixed_DType->is_known_scalar != NULL &&
-                 fixed_DType->is_known_scalar(fixed_DType, obj))) {
-            /*
-             * There are some corner cases, where we want to make sure a
-             * sequence is considered a scalar. In particular tuples with
-             * structured/void dtype and strings.
-             * The type check is simply a fast (and simple default) path
-             * which could capture some special dtypes, such as polynomials.
-             */
+                (fixed_DType->is_known_scalar_type != NULL &&
+                 fixed_DType->is_known_scalar_type(fixed_DType, Py_TYPE(obj)))) {
             Py_INCREF(fixed_DType);
             return fixed_DType;
         }
@@ -279,6 +278,10 @@ discover_dtype_from_pyobject(
             return NULL;
         }
     }
+    else if (flags == NULL) {
+        Py_INCREF(Py_None);
+        return (PyArray_DTypeMeta *)Py_None;
+    }
     else if (PyBytes_Check(obj)) {
         legacy_descr = PyArray_DescrFromType(NPY_BYTE);
     }
@@ -307,7 +310,6 @@ discover_dtype_from_pyobject(
         }
         return DType;
     }
-
     Py_INCREF(Py_None);
     return (PyArray_DTypeMeta *)Py_None;
 }
@@ -330,7 +332,7 @@ cast_descriptor_to_fixed_dtype(
          */
         return fixed_DType->default_descr(fixed_DType);
     }
-    if (PyObject_IsInstance((PyObject *)descr, (PyObject *)fixed_DType)) {
+    if (PyObject_TypeCheck((PyObject *)descr, (PyTypeObject *)fixed_DType)) {
         Py_INCREF(descr);
         return descr;
     }
@@ -458,12 +460,23 @@ PyArray_Pack(PyArray_Descr *descr, char *item, PyObject *value)
             .ob_base.ob_type = &PyArrayDescr_Type,
             .flags = NPY_ARRAY_BEHAVED,
         };
-    enum _dtype_discovery_flags flags = GAVE_SUBCLASS_WARNING;
+
+    if (NPY_UNLIKELY(descr->type_num == NPY_OBJECT)) {
+        /*
+         * We always have store objects directly, casting will lose some
+         * type information. Any other dtype discards the type information...
+         */
+        return descr->f->setitem(value, item, &arr_fields);
+    }
+
     PyArray_DTypeMeta *DType = discover_dtype_from_pyobject(
-            value, &flags, NPY_DTYPE(descr));
+            value, NULL, NPY_DTYPE(descr));
+
+    if (DType == NULL) {
+        return -1;
+    }
 
-    if (DType == NPY_DTYPE(descr) || DType == (PyArray_DTypeMeta *)Py_None ||
-            DType == NULL) {
+    if (DType == NPY_DTYPE(descr) || DType == (PyArray_DTypeMeta *)Py_None) {
         Py_XDECREF(DType);
         /* We can set the element directly (luckily) */
         arr_fields.descr = descr;
@@ -566,7 +579,7 @@ npy_new_coercion_cache(
         PyObject *converted_obj, PyObject *arr_or_sequence, npy_bool sequence,
         coercion_cache_obj ***next_ptr, int ndim)
 {
-    coercion_cache_obj *cache = PyArray_malloc(sizeof(coercion_cache_obj));
+    coercion_cache_obj *cache = PyObject_MALLOC(sizeof(coercion_cache_obj));
     if (cache == NULL) {
         PyErr_NoMemory();
         return -1;
@@ -591,7 +604,7 @@ npy_free_coercion_cache(coercion_cache_obj *next) {
         next = current->next;
 
         Py_DECREF(current->arr_or_sequence);
-        PyArray_free(current);
+        PyObject_FREE(current);
     }
 }
 
@@ -760,6 +773,7 @@ PyArray_DiscoverDTypeAndShape_Recursive(
         if (update_shape(curr_dims, &max_dims, out_shape,
                 PyArray_NDIM(arr), PyArray_SHAPE(arr), NPY_FALSE) < 0) {
             *flags |= FOUND_RAGGED_ARRAY;
+            Py_XSETREF(*out_descr, PyArray_DescrFromType(NPY_OBJECT));
             return max_dims;
         }
 
@@ -966,8 +980,8 @@ PyArray_DiscoverDTypeAndShape(
 
     /* Validate input of requested descriptor and DType */
     if (fixed_DType != NULL) {
-        assert(PyObject_IsInstance(
-                (PyObject *)fixed_DType, (PyObject *)&PyArrayDTypeMeta_Type));
+        assert(PyObject_TypeCheck(
+                (PyObject *)fixed_DType, (PyTypeObject *)&PyArrayDTypeMeta_Type));
     }
     if (requested_descr != NULL) {
         assert(!descr_is_legacy_parametric_instance(requested_descr));
@@ -1084,7 +1098,7 @@ PyArray_DiscoverDTypeAndShape(
                 Py_DECREF(current->arr_or_sequence);
                 coercion_cache_obj *_old = current;
                 current = current->next;
-                PyArray_free(_old);
+                PyObject_FREE(_old);
                 continue;
             }
             /* advance both prev and next, and set prev->next to new item */
@@ -1153,13 +1167,13 @@ PyArray_ExtractDTypeAndDescriptor(PyObject *dtype,
     *out_descr = NULL;
 
     if (dtype != NULL) {
-        if (PyObject_IsInstance(dtype, (PyObject *)&PyArrayDTypeMeta_Type)) {
+        if (PyObject_TypeCheck(dtype, (PyTypeObject *)&PyArrayDTypeMeta_Type)) {
             assert(dtype != (PyObject * )&PyArrayDescr_Type);  /* not np.dtype */
             *out_DType = (PyArray_DTypeMeta *)dtype;
             Py_INCREF(*out_DType);
         }
-        else if (PyObject_IsInstance((PyObject *)Py_TYPE(dtype),
-                    (PyObject *)&PyArrayDTypeMeta_Type)) {
+        else if (PyObject_TypeCheck((PyObject *)Py_TYPE(dtype),
+                    (PyTypeObject *)&PyArrayDTypeMeta_Type)) {
             *out_DType = NPY_DTYPE(dtype);
             Py_INCREF(*out_DType);
             if (!descr_is_legacy_parametric_instance((PyArray_Descr *)dtype)) {
diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c
index a07d884b5..704c9b399 100644
--- a/numpy/core/src/multiarray/ctors.c
+++ b/numpy/core/src/multiarray/ctors.c
@@ -637,7 +637,7 @@ PyArray_AssignFromCache_Recursive(
     int depth = (*cache)->depth;
     coercion_cache_obj *_old = *cache;
     *cache = (*cache)->next;
-    PyArray_free(_old);
+    PyObject_FREE(_old);
 
     /*
      * The maximum depth is special (specifically for objects), but usually
@@ -2113,14 +2113,14 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth,
     }
 
     /* Got the correct parameters, but the cache may already hold the result */
-    if (cache != NULL && (cache->converted_obj == op && !(cache->sequence))) {
+    if (cache != NULL && !(cache->sequence)) {
         /*
-         * There is only a single array and it was converted, it
+         * There is only a single array-like and it was converted, it
          * may still have the incorrect type, but that is handled below.
          */
+        assert(cache->converted_obj == op);
         arr = (PyArrayObject *)(cache->arr_or_sequence);
-        /* we may need to cast or assert flags: */
-
+        /* we may need to cast or assert flags (e.g. copy) */
         PyObject *res = PyArray_FromArray(arr, dtype, flags);
         npy_free_coercion_cache(cache);
         return res;
diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c
index 2ca63b6a9..ecdf0f53e 100644
--- a/numpy/core/src/multiarray/dtypemeta.c
+++ b/numpy/core/src/multiarray/dtypemeta.c
@@ -208,10 +208,35 @@ flexible_default_descr(PyArray_DTypeMeta *cls)
 }
 
 static int
-python_classes_are_known_scalars(
-        PyArray_DTypeMeta *NPY_UNUSED(cls), PyObject *obj)
+python_builtins_are_known_scalar_types(
+        PyArray_DTypeMeta *NPY_UNUSED(cls), PyTypeObject *pytype)
 {
-    return PyArray_IsPythonScalar(obj);
+    /*
+     * Always accept the common Python types, this ensures that we do not
+     * convert pyfloat->float64->integers. Subclasses are hopefully rejected
+     * as being discovered.
+     * This is necessary only for python scalar classes which we discover
+     * as valid DTypes.
+     */
+    if (pytype == &PyFloat_Type) {
+        return 1;
+    }
+    if (pytype == &PyLong_Type) {
+        return 1;
+    }
+    if (pytype == &PyBool_Type) {
+        return 1;
+    }
+    if (pytype == &PyComplex_Type) {
+        return 1;
+    }
+    if (pytype == &PyUnicode_Type) {
+        return 1;
+    }
+    if (pytype == &PyBytes_Type) {
+        return 1;
+    }
+    return 0;
 }
 
 
@@ -333,7 +358,7 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr)
     dtype_class->kind = descr->kind;
 
     /* Strings and voids have (strange) logic around scalars. */
-    dtype_class-> is_known_scalar = python_classes_are_known_scalars;
+    dtype_class->is_known_scalar_type = python_builtins_are_known_scalar_types;
 
     if (PyTypeNum_ISDATETIME(descr->type_num)) {
         /* Datetimes are flexible, but were not considered previously */