ENH: implement PEP 3118 buffer consumer on Py2.6, adding a simple Memoryview object

To use PEP 3118 buffers as ndarray bases, we need to have a way to
release the buffers on garbage collection. The easiest way to do this is
to use a Memoryview object -- but they are not available on Python 2.6.

Hence, we implement a minimal memory view object that only handles the
garbage collection, and use it on Python 2.6.  It also implements the
new buffer protocol, to allow testing the functionality.

10 files changed, 371 insertions, 16 deletions

diff --git a/numpy/core/SConscript b/numpy/core/SConscript
index e9c4be217..b030c0044 100644
--- a/numpy/core/SConscript
+++ b/numpy/core/SConscript
@@ -447,6 +447,7 @@ if ENABLE_SEPARATE_COMPILATION:
         pjoin('src', 'multiarray', 'conversion_utils.c'),
         pjoin('src', 'multiarray', 'usertypes.c'),
         pjoin('src', 'multiarray', 'buffer.c'),
+        pjoin('src', 'multiarray', 'numpymemoryview.c'),
         pjoin('src', 'multiarray', 'scalarapi.c')]
     multiarray_src.extend(arraytypes_src)
     multiarray_src.extend(scalartypes_src)
diff --git a/numpy/core/setup.py b/numpy/core/setup.py
index 66b6bffef..a5802f502 100644
--- a/numpy/core/setup.py
+++ b/numpy/core/setup.py
@@ -703,6 +703,7 @@ def configuration(parent_package='',top_path=None):
             join('src', 'multiarray', 'mapping.h'),
             join('src', 'multiarray', 'methods.h'),
             join('src', 'multiarray', 'multiarraymodule.h'),
+            join('src', 'multiarray', 'numpymemoryview.h'),
             join('src', 'multiarray', 'number.h'),
             join('src', 'multiarray', 'numpyos.h'),
             join('src', 'multiarray', 'refcount.h'),
@@ -715,6 +716,7 @@ def configuration(parent_package='',top_path=None):
     multiarray_src = [join('src', 'multiarray', 'multiarraymodule.c'),
         join('src', 'multiarray', 'hashdescr.c'),
         join('src', 'multiarray', 'arrayobject.c'),
+        join('src', 'multiarray', 'numpymemoryview.c'),
         join('src', 'multiarray', 'buffer.c'),
         join('src', 'multiarray', 'datetime.c'),
         join('src', 'multiarray', 'numpyos.c'),
diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c
index 17ea107f4..fdf89e883 100644
--- a/numpy/core/src/multiarray/arrayobject.c
+++ b/numpy/core/src/multiarray/arrayobject.c
@@ -1317,6 +1317,9 @@ NPY_NO_EXPORT PyTypeObject PyArray_Type = {
 #if !defined(NPY_PY3K)
      | Py_TPFLAGS_CHECKTYPES
 #endif
+#if (PY_VERSION_HEX >= 0x02060000) && (PY_VERSION_HEX < 0x03000000)
+     | Py_TPFLAGS_HAVE_NEWBUFFER
+#endif
      | Py_TPFLAGS_BASETYPE),                  /* tp_flags */
     0,                                          /* tp_doc */
 
diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c
index 5fa13c1e2..45d40f4b9 100644
--- a/numpy/core/src/multiarray/buffer.c
+++ b/numpy/core/src/multiarray/buffer.c
@@ -636,8 +636,8 @@ _descriptor_from_pep3118_format(char *s)
         return NULL;
     }
     str = PyUString_FromStringAndSize(buf, strlen(buf));
-    descr = PyObject_CallMethod(_numpy_internal, "_dtype_from_pep3118",
-                                "O", str);
+    descr = (PyArray_Descr*)PyObject_CallMethod(
+        _numpy_internal, "_dtype_from_pep3118", "O", str);
     Py_DECREF(str);
     if (descr == NULL) {
         PyErr_Format(PyExc_ValueError,
diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c
index b66f8e208..712a500bd 100644
--- a/numpy/core/src/multiarray/common.c
+++ b/numpy/core/src/multiarray/common.c
@@ -153,7 +153,7 @@ _array_find_type(PyObject *op, PyArray_Descr *minitype, int max)
     PyObject *ip;
     PyArray_Descr *chktype = NULL;
     PyArray_Descr *outtype;
-#if PY_VERSION_HEX >= 0x02070000
+#if PY_VERSION_HEX >= 0x02060000
     Py_buffer buffer_view;
 #endif
 
@@ -210,7 +210,7 @@ _array_find_type(PyObject *op, PyArray_Descr *minitype, int max)
         goto finish;
     }
 
-#if PY_VERSION_HEX >= 0x02070000
+#if PY_VERSION_HEX >= 0x02060000
     /* PEP 3118 buffer interface */
     memset(&buffer_view, 0, sizeof(Py_buffer));
     if (PyObject_GetBuffer(op, &buffer_view, PyBUF_FORMAT|PyBUF_STRIDES) == 0 ||
diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c
index 2a5c2c4e9..50ed7b31c 100644
--- a/numpy/core/src/multiarray/ctors.c
+++ b/numpy/core/src/multiarray/ctors.c
@@ -17,6 +17,8 @@
 
 #include "ctors.h"
 
+#include "numpymemoryview.h"
+
 /*
  * Reading from a file or a string.
  *
@@ -1083,7 +1085,7 @@ discover_depth(PyObject *s, int max, int stop_at_string, int stop_at_tuple)
 {
     int d = 0;
     PyObject *e;
-#if PY_VERSION_HEX >= 0x02070000
+#if PY_VERSION_HEX >= 0x02060000
     Py_buffer buffer_view;
 #endif
 
@@ -1117,7 +1119,7 @@ discover_depth(PyObject *s, int max, int stop_at_string, int stop_at_tuple)
     if (stop_at_tuple && PyTuple_Check(s)) {
         return 0;
     }
-#if PY_VERSION_HEX >= 0x02070000
+#if PY_VERSION_HEX >= 0x02060000
     /* PEP 3118 buffer interface */
     memset(&buffer_view, 0, sizeof(Py_buffer));
     if (PyObject_GetBuffer(s, &buffer_view, PyBUF_STRIDES) == 0 ||
@@ -1631,12 +1633,7 @@ PyArray_New(PyTypeObject *subtype, int nd, intp *dims, int type_num,
 NPY_NO_EXPORT int
 _array_from_buffer_3118(PyObject *obj, PyObject **out)
 {
-#if PY_VERSION_HEX >= 0x02070000
-    /* XXX: Pythons < 2.7 do not have the memory view object. This makes
-     *      using buffers somewhat difficult, since one cannot release them
-     *      using the garbage collection
-     */
-
+#if PY_VERSION_HEX >= 0x02060000
     /* PEP 3118 */
     PyObject *memoryview;
     Py_buffer *view;
@@ -1654,7 +1651,7 @@ _array_from_buffer_3118(PyObject *obj, PyObject **out)
 
     view = PyMemoryView_GET_BUFFER(memoryview);
     if (view->format != NULL) {
-        descr = _descriptor_from_pep3118_format(view->format);
+        descr = (PyObject*)_descriptor_from_pep3118_format(view->format);
         if (descr == NULL) {
             goto fail;
         }
diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c
index 5b8420656..ef031e0f9 100644
--- a/numpy/core/src/multiarray/multiarraymodule.c
+++ b/numpy/core/src/multiarray/multiarraymodule.c
@@ -41,6 +41,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0;
 #include "calculation.h"
 #include "number.h"
 #include "scalartypes.h"
+#include "numpymemoryview.h"
 
 /*NUMPY_API
  * Get Priority from object
@@ -3051,6 +3052,14 @@ PyMODINIT_FUNC initmultiarray(void) {
         goto err;
     }
 
+    /* Initialize types in numpymemoryview.c */
+    if (_numpymemoryview_init(&s) < 0) {
+        return RETVAL;
+    }
+    if (s != NULL) {
+        PyDict_SetItemString(d, "memorysimpleview", s);
+    }
+
     /*
      * PyExc_Exception should catch all the standard errors that are
      * now raised instead of the string exception "multiarray.error"
diff --git a/numpy/core/src/multiarray/numpymemoryview.c b/numpy/core/src/multiarray/numpymemoryview.c
new file mode 100644
index 000000000..7b60c94b1
--- /dev/null
+++ b/numpy/core/src/multiarray/numpymemoryview.c
@@ -0,0 +1,310 @@
+/*
+ * Simple PyMemoryView'ish object for Python 2.6 compatibility.
+ *
+ * On Python >= 2.7, we can use the actual PyMemoryView objects.
+ *
+ * Some code copied from the CPython implementation.
+ */
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include "structmember.h"
+
+#define _MULTIARRAYMODULE
+#define NPY_NO_PREFIX
+#include "numpy/arrayobject.h"
+#include "numpy/arrayscalars.h"
+
+#include "npy_config.h"
+#include "npy_3kcompat.h"
+
+#include "numpymemoryview.h"
+
+
+#if (PY_VERSION_HEX >= 0x02060000) && (PY_VERSION_HEX < 0x02070000)
+
+/*
+ * Memory allocation
+ */
+
+static int
+memorysimpleview_traverse(PyMemorySimpleViewObject *self,
+                          visitproc visit, void *arg)
+{
+    if (self->base != NULL)
+        Py_VISIT(self->base);
+    if (self->view.obj != NULL)
+        Py_VISIT(self->view.obj);
+    return 0;
+}
+
+static int
+memorysimpleview_clear(PyMemorySimpleViewObject *self)
+{
+    Py_CLEAR(self->base);
+    PyBuffer_Release(&self->view);
+    self->view.obj = NULL;
+    return 0;
+}
+
+static void
+memorysimpleview_dealloc(PyMemorySimpleViewObject *self)
+{
+    PyObject_GC_UnTrack(self);
+    Py_CLEAR(self->base);
+    if (self->view.obj != NULL) {
+        PyBuffer_Release(&self->view);
+        self->view.obj = NULL;
+    }
+    PyObject_GC_Del(self);
+}
+
+static PyObject *
+memorysimpleview_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)
+{
+    PyObject *obj;
+    static char *kwlist[] = {"object", 0};
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:memorysimpleview", kwlist,
+                                     &obj)) {
+        return NULL;
+    }
+    return PyMemorySimpleView_FromObject(obj);
+}
+
+
+/*
+ * Buffer interface
+ */
+
+static int
+memorysimpleview_getbuffer(PyMemorySimpleViewObject *self,
+                           Py_buffer *view, int flags)
+{
+    return PyObject_GetBuffer(self->base, view, flags);
+}
+
+static void
+memorysimpleview_releasebuffer(PyMemorySimpleViewObject *self,
+                               Py_buffer *view)
+{
+    PyBuffer_Release(view);
+}
+
+static PyBufferProcs memorysimpleview_as_buffer = {
+    (readbufferproc)0,       /*bf_getreadbuffer*/
+    (writebufferproc)0,     /*bf_getwritebuffer*/
+    (segcountproc)0,        /*bf_getsegcount*/
+    (charbufferproc)0,       /*bf_getcharbuffer*/
+    (getbufferproc)memorysimpleview_getbuffer, /* bf_getbuffer */
+    (releasebufferproc)memorysimpleview_releasebuffer, /* bf_releasebuffer */
+};
+
+
+/*
+ * Getters
+ */
+
+static PyObject *
+_IntTupleFromSsizet(int len, Py_ssize_t *vals)
+{
+    int i;
+    PyObject *o;
+    PyObject *intTuple;
+
+    if (vals == NULL) {
+        Py_INCREF(Py_None);
+        return Py_None;
+    }
+    intTuple = PyTuple_New(len);
+    if (!intTuple) return NULL;
+    for(i=0; i<len; i++) {
+        o = PyInt_FromSsize_t(vals[i]);
+        if (!o) {
+            Py_DECREF(intTuple);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(intTuple, i, o);
+    }
+    return intTuple;
+}
+
+static PyObject *
+memorysimpleview_format_get(PyMemorySimpleViewObject *self)
+{
+    return PyUnicode_FromString(self->view.format);
+}
+
+static PyObject *
+memorysimpleview_itemsize_get(PyMemorySimpleViewObject *self)
+{
+    return PyLong_FromSsize_t(self->view.itemsize);
+}
+
+static PyObject *
+memorysimpleview_shape_get(PyMemorySimpleViewObject *self)
+{
+    return _IntTupleFromSsizet(self->view.ndim, self->view.shape);
+}
+
+static PyObject *
+memorysimpleview_strides_get(PyMemorySimpleViewObject *self)
+{
+    return _IntTupleFromSsizet(self->view.ndim, self->view.strides);
+}
+
+static PyObject *
+memorysimpleview_suboffsets_get(PyMemorySimpleViewObject *self)
+{
+    return _IntTupleFromSsizet(self->view.ndim, self->view.suboffsets);
+}
+
+static PyObject *
+memorysimpleview_readonly_get(PyMemorySimpleViewObject *self)
+{
+    return PyBool_FromLong(self->view.readonly);
+}
+
+static PyObject *
+memorysimpleview_ndim_get(PyMemorySimpleViewObject *self)
+{
+    return PyLong_FromLong(self->view.ndim);
+}
+
+
+static PyGetSetDef memorysimpleview_getsets[] =
+{
+    {"format", (getter)memorysimpleview_format_get, NULL, NULL, NULL},
+    {"itemsize", (getter)memorysimpleview_itemsize_get, NULL, NULL, NULL},
+    {"shape", (getter)memorysimpleview_shape_get, NULL, NULL, NULL},
+    {"strides", (getter)memorysimpleview_strides_get, NULL, NULL, NULL},
+    {"suboffsets", (getter)memorysimpleview_suboffsets_get, NULL, NULL, NULL},
+    {"readonly", (getter)memorysimpleview_readonly_get, NULL, NULL, NULL},
+    {"ndim", (getter)memorysimpleview_ndim_get, NULL, NULL, NULL},
+    {NULL, NULL, NULL, NULL}
+};
+
+NPY_NO_EXPORT PyTypeObject PyMemorySimpleView_Type = {
+#if defined(NPY_PY3K)
+    PyVarObject_HEAD_INIT(NULL, 0)
+#else
+    PyObject_HEAD_INIT(NULL)
+    0,                                          /* ob_size */
+#endif
+    "numpy.memorysimpleview",
+    sizeof(PyMemorySimpleViewObject),
+    0,                                          /* tp_itemsize */
+    /* methods */
+    (destructor)memorysimpleview_dealloc,       /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+#if defined(NPY_PY3K)
+    0,                                          /* tp_reserved */
+#else
+    (cmpfunc)0,                                 /* tp_compare */
+#endif
+    (reprfunc)0,                                /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    (reprfunc)0,                                /* tp_str */
+    0,                                          /* tp_getattro */
+    0,                                          /* tp_setattro */
+    &memorysimpleview_as_buffer,                /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
+    | Py_TPFLAGS_HAVE_NEWBUFFER,                /* tp_flags */
+    0,                                          /* tp_doc */
+    (traverseproc)memorysimpleview_traverse,    /* tp_traverse */
+    (inquiry)memorysimpleview_clear,            /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+    memorysimpleview_getsets,                   /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    0,                                          /* tp_dictoffset */
+    0,                                          /* tp_init */
+    0,                                          /* tp_alloc */
+    memorysimpleview_new,                       /* tp_new */
+    0,                                          /* tp_free */
+    0,                                          /* tp_is_gc */
+    0,                                          /* tp_bases */
+    0,                                          /* tp_mro */
+    0,                                          /* tp_cache */
+    0,                                          /* tp_subclasses */
+    0,                                          /* tp_weaklist */
+    0,                                          /* tp_del */
+#if PY_VERSION_HEX >= 0x02060000
+    0,                                          /* tp_version_tag */
+#endif
+};
+
+
+/*
+ * Factory
+ */
+NPY_NO_EXPORT PyObject *
+PyMemorySimpleView_FromObject(PyObject *base)
+{
+    PyMemorySimpleViewObject *mview = NULL;
+    Py_buffer view;
+
+    if (Py_TYPE(base)->tp_as_buffer == NULL ||
+        Py_TYPE(base)->tp_as_buffer->bf_getbuffer == NULL) {
+
+        PyErr_SetString(PyExc_TypeError,
+            "cannot make memory view because object does "
+            "not have the buffer interface");
+        return NULL;
+    }
+
+    memset(&view, 0, sizeof(Py_buffer));
+    if (PyObject_GetBuffer(base, &view, PyBUF_FULL_RO) < 0)
+        return NULL;
+
+    mview = (PyMemorySimpleViewObject *)
+        PyObject_GC_New(PyMemorySimpleViewObject, &PyMemorySimpleView_Type);
+    if (mview == NULL) {
+        PyBuffer_Release(&view);
+        return NULL;
+    }
+    memcpy(&mview->view, &view, sizeof(Py_buffer));
+    mview->base = base;
+    Py_INCREF(base);
+
+    PyObject_GC_Track(mview);
+    return (PyObject *)mview;
+}
+
+
+/*
+ * Module initialization
+ */
+
+NPY_NO_EXPORT int
+_numpymemoryview_init(PyObject **typeobject)
+{
+    if (PyType_Ready(&PyMemorySimpleView_Type) < 0) {
+        return -1;
+    }
+    *typeobject = (PyObject*)&PyMemorySimpleView_Type;
+    return 0;
+}
+
+#else
+
+NPY_NO_EXPORT int
+_numpymemoryview_init(PyObject **typeobject)
+{
+    *typeobject = NULL;
+    return 0;
+}
+
+#endif
diff --git a/numpy/core/src/multiarray/numpymemoryview.h b/numpy/core/src/multiarray/numpymemoryview.h
new file mode 100644
index 000000000..3a2661754
--- /dev/null
+++ b/numpy/core/src/multiarray/numpymemoryview.h
@@ -0,0 +1,29 @@
+#ifndef _NPY_PRIVATE_NUMPYMEMORYVIEW_H_
+#define _NPY_PRIVATE_NUMPYMEMORYVIEW_H_
+
+/*
+ * Memoryview is introduced to 2.x series only in 2.7, so for supporting 2.6,
+ * we need to have a minimal implementation here.
+ */
+#if (PY_VERSION_HEX >= 0x02060000) && (PY_VERSION_HEX < 0x02070000)
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *base;
+    Py_buffer view;
+} PyMemorySimpleViewObject;
+
+NPY_NO_EXPORT PyObject *
+PyMemorySimpleView_FromObject(PyObject *base);
+
+#define PyMemorySimpleView_GET_BUFFER(op) (&((PyMemorySimpleViewObject *)(op))->view)
+
+#define PyMemoryView_FromObject PyMemorySimpleView_FromObject
+#define PyMemoryView_GET_BUFFER PyMemorySimpleView_GET_BUFFER
+
+#endif
+
+NPY_NO_EXPORT int
+_numpymemoryview_init(PyObject **typeobject);
+
+#endif
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
index 0c814589c..426d123b5 100644
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -1451,7 +1451,11 @@ class TestWarnings(object):
         assert_raises(np.ComplexWarning, x.__setitem__, slice(None), y)
         warnings.simplefilter("default", np.ComplexWarning)
 
-if sys.version_info >= (2, 7):
+if sys.version_info >= (2, 6):
+
+    if sys.version_info[:2] == (2, 6):
+        from numpy.core.multiarray import memorysimpleview as memoryview
+
     class TestNewBufferProtocol(object):
         def _check_roundtrip(self, obj):
             obj = np.asarray(obj)
@@ -1556,9 +1560,9 @@ if sys.version_info >= (2, 7):
             x = np.array(([[1,2],[3,4]],), dtype=[('a', (int, (2,2)))])
             y = memoryview(x)
             assert y.format == 'T{(2,2)=l:a:}'
-            assert y.shape == ()
+            assert y.shape is None
             assert y.ndim == 0
-            assert y.strides == ()
+            assert y.strides is None
             assert y.suboffsets is None
             assert y.itemsize == 16