Merge pull request #7463 from seberg/array-too-large

BUG: fix array too big error for wide dtypes.

4 files changed, 40 insertions, 25 deletions

diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c
index 79c2b16b1..0017de0ad 100644
--- a/numpy/core/src/multiarray/ctors.c
+++ b/numpy/core/src/multiarray/ctors.c
@@ -901,8 +901,7 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
 {
     PyArrayObject_fields *fa;
     int i;
-    size_t sd;
-    npy_intp size;
+    npy_intp nbytes;
 
     if (descr->subarray) {
         PyObject *ret;
@@ -929,10 +928,9 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
         return NULL;
     }
 
-    /* Check dimensions */
-    size = 1;
-    sd = (size_t) descr->elsize;
-    if (sd == 0) {
+    /* Check datatype element size */
+    nbytes = descr->elsize;
+    if (nbytes == 0) {
         if (!PyDataType_ISSTRING(descr)) {
             PyErr_SetString(PyExc_TypeError, "Empty data-type");
             Py_DECREF(descr);
@@ -943,13 +941,14 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
             return NULL;
         }
         if (descr->type_num == NPY_STRING) {
-            sd = descr->elsize = 1;
+            nbytes = descr->elsize = 1;
         }
         else {
-            sd = descr->elsize = sizeof(npy_ucs4);
+            nbytes = descr->elsize = sizeof(npy_ucs4);
         }
     }
 
+    /* Check dimensions and multiply them to nbytes */
     for (i = 0; i < nd; i++) {
         npy_intp dim = dims[i];
 
@@ -974,9 +973,10 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
          * multiplying the dimensions together to get the total size of the
          * array.
          */
-        if (npy_mul_with_overflow_intp(&size, size, dim)) {
+        if (npy_mul_with_overflow_intp(&nbytes, nbytes, dim)) {
             PyErr_SetString(PyExc_ValueError,
-                            "array is too big.");
+                "array is too big; `arr.size * arr.dtype.itemsize` "
+                "is larger than the maximum possible size.");
             Py_DECREF(descr);
             return NULL;
         }
@@ -1015,9 +1015,9 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
         }
         fa->strides = fa->dimensions + nd;
         memcpy(fa->dimensions, dims, sizeof(npy_intp)*nd);
-        if (strides == NULL) { /* fill it in */
-            sd = _array_fill_strides(fa->strides, dims, nd, sd,
-                                     flags, &(fa->flags));
+        if (strides == NULL) {  /* fill it in */
+            _array_fill_strides(fa->strides, dims, nd, descr->elsize,
+                                flags, &(fa->flags));
         }
         else {
             /*
@@ -1025,7 +1025,6 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
              * the memory, but be careful with this...
              */
             memcpy(fa->strides, strides, sizeof(npy_intp)*nd);
-            sd *= size;
         }
     }
     else {
@@ -1039,19 +1038,18 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
          * e.g. shape=(0,) -- otherwise buffer exposure
          * (a.data) doesn't work as it should.
          */
-
-        if (sd == 0) {
-            sd = descr->elsize;
+        if (nbytes == 0) {
+            nbytes = descr->elsize;
         }
         /*
          * It is bad to have uninitialized OBJECT pointers
          * which could also be sub-fields of a VOID array
          */
         if (zeroed || PyDataType_FLAGCHK(descr, NPY_NEEDS_INIT)) {
-            data = npy_alloc_cache_zero(sd);
+            data = npy_alloc_cache_zero(nbytes);
         }
         else {
-            data = npy_alloc_cache(sd);
+            data = npy_alloc_cache(nbytes);
         }
         if (data == NULL) {
             PyErr_NoMemory();
@@ -3772,9 +3770,11 @@ PyArray_FromIter(PyObject *obj, PyArray_Descr *dtype, npy_intp count)
  * If data is not given but created here, then flags will be NPY_ARRAY_DEFAULT
  * and a non-zero flags argument can be used to indicate a FORTRAN style
  * array is desired.
+ *
+ * Dimensions and itemsize must have been checked for validity.
  */
 
-NPY_NO_EXPORT size_t
+NPY_NO_EXPORT void
 _array_fill_strides(npy_intp *strides, npy_intp *dims, int nd, size_t itemsize,
                     int inflag, int *objflags)
 {
@@ -3853,7 +3853,7 @@ _array_fill_strides(npy_intp *strides, npy_intp *dims, int nd, size_t itemsize,
             *objflags |= (NPY_ARRAY_C_CONTIGUOUS|NPY_ARRAY_F_CONTIGUOUS);
         }
     }
-    return itemsize;
+    return;
 }
 
 /*
diff --git a/numpy/core/src/multiarray/ctors.h b/numpy/core/src/multiarray/ctors.h
index 757362fb8..783818def 100644
--- a/numpy/core/src/multiarray/ctors.h
+++ b/numpy/core/src/multiarray/ctors.h
@@ -51,7 +51,7 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src,
                                 NPY_ORDER order);
 
 /* FIXME: remove those from here */
-NPY_NO_EXPORT size_t
+NPY_NO_EXPORT void
 _array_fill_strides(npy_intp *strides, npy_intp *dims, int nd, size_t itemsize,
                     int inflag, int *objflags);
 
diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c
index 2d8b6f602..cc02d15f9 100644
--- a/numpy/core/src/multiarray/shape.c
+++ b/numpy/core/src/multiarray/shape.c
@@ -152,9 +152,9 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck,
     }
 
     /* make new_strides variable */
-    sd = (size_t) PyArray_DESCR(self)->elsize;
-    sd = (size_t) _array_fill_strides(new_strides, new_dimensions, new_nd, sd,
-            PyArray_FLAGS(self), &(((PyArrayObject_fields *)self)->flags));
+    _array_fill_strides(
+        new_strides, new_dimensions, new_nd, PyArray_DESCR(self)->elsize,
+        PyArray_FLAGS(self), &(((PyArrayObject_fields *)self)->flags));
     memmove(PyArray_DIMS(self), new_dimensions, new_nd*sizeof(npy_intp));
     memmove(PyArray_STRIDES(self), new_strides, new_nd*sizeof(npy_intp));
     Py_RETURN_NONE;
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
index ffdc50d2c..262fbc0c0 100644
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -730,6 +730,21 @@ class TestCreation(TestCase):
         d = A([1,2,3])
         assert_equal(len(np.array(d)), 3)
 
+    def test_array_too_big(self):
+        # Test that array creation succeeds for arrays addressable by intp
+        # on the byte level and fails for too large arrays.
+        buf = np.zeros(100)
+
+        max_bytes = np.iinfo(np.intp).max
+        for dtype in ["intp", "S20", "b"]:
+            dtype = np.dtype(dtype)
+            itemsize = dtype.itemsize
+
+            np.ndarray(buffer=buf, strides=(0,),
+                       shape=(max_bytes//itemsize,), dtype=dtype)
+            assert_raises(ValueError, np.ndarray, buffer=buf, strides=(0,),
+                          shape=(max_bytes//itemsize + 1,), dtype=dtype)
+
 
 class TestStructured(TestCase):
     def test_subarray_field_access(self):