Merge pull request #11694 from mattip/master

BUG: pickle and memoryview for datetime64, timedelta64 scalars

5 files changed, 68 insertions, 11 deletions

diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c
index d549a3029..c8e3da8bc 100644
--- a/numpy/core/src/multiarray/buffer.c
+++ b/numpy/core/src/multiarray/buffer.c
@@ -404,8 +404,8 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str,
         case NPY_CFLOAT:       if (_append_str(str, "Zf")) return -1; break;
         case NPY_CDOUBLE:      if (_append_str(str, "Zd")) return -1; break;
         case NPY_CLONGDOUBLE:  if (_append_str(str, "Zg")) return -1; break;
-        /* XXX: datetime */
-        /* XXX: timedelta */
+        /* XXX NPY_DATETIME */
+        /* XXX NPY_TIMEDELTA */
         case NPY_OBJECT:       if (_append_char(str, 'O')) return -1; break;
         case NPY_STRING: {
             char buf[128];
@@ -483,7 +483,29 @@ _buffer_info_new(PyObject *obj)
         goto fail;
     }
 
-    if (PyArray_IsScalar(obj, Generic)) {
+    if (PyArray_IsScalar(obj, Datetime) || PyArray_IsScalar(obj, Timedelta)) {
+        /*
+         * Special case datetime64 scalars to remain backward compatible.
+         * This will change in a future version.
+         * Note arrays of datetime64 and strutured arrays with datetime64
+         * fields will not hit this code path and are currently unsupported
+         * in _buffer_format_string.
+         */
+        _append_char(&fmt, 'B');
+        _append_char(&fmt, '\0');
+        info->ndim = 1;
+        info->shape = malloc(sizeof(Py_ssize_t) * 2);
+        if (info->shape == NULL) {
+            PyErr_NoMemory();
+            goto fail;
+        }
+        info->strides = info->shape + info->ndim;
+        info->shape[0] = 8;
+        info->strides[0] = 1;
+        info->format = fmt.s;
+        return info;
+    }
+    else if (PyArray_IsScalar(obj, Generic)) {
         descr = PyArray_DescrFromScalar(obj);
         if (descr == NULL) {
             goto fail;
@@ -809,8 +831,6 @@ gentype_getbuffer(PyObject *self, Py_buffer *view, int flags)
     /* Fill in information */
     info = _buffer_get_info(self);
     if (info == NULL) {
-        PyErr_SetString(PyExc_BufferError,
-                        "could not get scalar buffer information");
         goto fail;
     }
 
@@ -833,6 +853,9 @@ gentype_getbuffer(PyObject *self, Py_buffer *view, int flags)
     }
 #endif
     view->len = elsize;
+    if (PyArray_IsScalar(self, Datetime) || PyArray_IsScalar(self, Timedelta)) {
+        elsize = 1; /* descr->elsize,char is 8,'M', but we return 1,'B' */
+    }
     view->itemsize = elsize;
 
     Py_DECREF(descr);
diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c
index a2cf17f4e..f1b8a0209 100644
--- a/numpy/core/src/multiarray/ctors.c
+++ b/numpy/core/src/multiarray/ctors.c
@@ -1391,10 +1391,12 @@ _array_from_buffer_3118(PyObject *memoryview)
 
         if (!is_ctypes) {
             /* This object has no excuse for a broken PEP3118 buffer */
-            PyErr_SetString(
+            PyErr_Format(
                     PyExc_RuntimeError,
-                    "Item size computed from the PEP 3118 buffer format "
-                    "string does not match the actual item size.");
+                   "Item size %zd for PEP 3118 buffer format "
+                    "string %s does not match the dtype %c item size %d.",
+                    view->itemsize, view->format, descr->type,
+                    descr->elsize);
             Py_DECREF(descr);
             return NULL;
         }
diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py
index 942554cae..b1b1e87c1 100644
--- a/numpy/core/tests/test_datetime.py
+++ b/numpy/core/tests/test_datetime.py
@@ -620,6 +620,10 @@ class TestDateTime(object):
         assert_equal(pickle.loads(pickle.dumps(dt)), dt)
         dt = np.dtype('M8[W]')
         assert_equal(pickle.loads(pickle.dumps(dt)), dt)
+        scalar = np.datetime64('2016-01-01T00:00:00.000000000')
+        assert_equal(pickle.loads(pickle.dumps(scalar)), scalar)
+        delta = scalar - np.datetime64('2015-01-01T00:00:00.000000000')
+        assert_equal(pickle.loads(pickle.dumps(delta)), delta)
 
         # Check that loading pickles from 1.6 works
         pkl = b"cnumpy\ndtype\np0\n(S'M8'\np1\nI0\nI1\ntp2\nRp3\n" + \
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
index d751657db..003d03fdc 100644
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -6487,6 +6487,15 @@ class TestNewBufferProtocol(object):
         # Issue #4015.
         self._check_roundtrip(0)
 
+    def test_invalid_buffer_format(self):
+        # datetime64 cannot be used fully in a buffer yet
+        # Should be fixed in the next Numpy major release
+        dt = np.dtype([('a', 'uint16'), ('b', 'M8[s]')])
+        a = np.empty(3, dt)
+        assert_raises((ValueError, BufferError), memoryview, a)
+        assert_raises((ValueError, BufferError), memoryview, np.array((3), 'M8[D]'))
+
+
     def test_export_simple_1d(self):
         x = np.array([1, 2, 3, 4, 5], dtype='i')
         y = memoryview(x)
diff --git a/numpy/core/tests/test_scalarbuffer.py b/numpy/core/tests/test_scalarbuffer.py
index 9e75fec3e..cb6c521e1 100644
--- a/numpy/core/tests/test_scalarbuffer.py
+++ b/numpy/core/tests/test_scalarbuffer.py
@@ -78,8 +78,27 @@ class TestScalarPEP3118(object):
         assert_equal(mv_x.itemsize, mv_a.itemsize)
         assert_equal(mv_x.format, mv_a.format)
 
-    def test_invalid_buffer_format(self):
-        # datetime64 cannot be used in a buffer yet
-        dt = np.dtype([('a', int), ('b', 'M8[s]')])
+    def test_datetime_memoryview(self):
+        # gh-11656
+        # Values verified with v1.13.3, shape is not () as in test_scalar_dim
+        def as_dict(m):
+            return dict(strides=m.strides, shape=m.shape, itemsize=m.itemsize,
+                        ndim=m.ndim, format=m.format)
+
+        dt1 = np.datetime64('2016-01-01')
+        dt2 = np.datetime64('2017-01-01')
+        expected = {'strides': (1,), 'itemsize': 1, 'ndim': 1,
+                    'shape': (8,), 'format': 'B'}
+        v = memoryview(dt1)
+        res = as_dict(v) 
+        assert_equal(res, expected)
+
+        v = memoryview(dt2 - dt1)
+        res = as_dict(v)
+        assert_equal(res, expected)
+
+        dt = np.dtype([('a', 'uint16'), ('b', 'M8[s]')])
         a = np.empty(1, dt)
+        # Fails to create a PEP 3118 valid buffer
         assert_raises((ValueError, BufferError), memoryview, a[0])
+