Fix memmap some. Fix reductions on object arrays.

7 files changed, 97 insertions, 40 deletions

diff --git a/scipy/base/include/scipy/arrayobject.h b/scipy/base/include/scipy/arrayobject.h
index 97b2e022e..367eea20c 100644
--- a/scipy/base/include/scipy/arrayobject.h
+++ b/scipy/base/include/scipy/arrayobject.h
@@ -891,7 +891,9 @@ typedef struct {
 #define BEHAVED_FLAGS ALIGNED | NOTSWAPPED | WRITEABLE
 #define BEHAVED_FLAGS_RO ALIGNED | NOTSWAPPED
 #define CARRAY_FLAGS CONTIGUOUS | BEHAVED_FLAGS
+#define CARRAY_FLAGS_RO CONTIGUOUS | BEHAVED_FLAGS_RO
 #define FARRAY_FLAGS FORTRAN | BEHAVED_FLAGS
+#define FARRAY_FLAGS_RO FORTRAN | BEHAVED_FLAGS_RO
 #define DEFAULT_FLAGS CARRAY_FLAGS
 
 #define UPDATE_ALL_FLAGS CONTIGUOUS | FORTRAN | ALIGNED
@@ -909,7 +911,9 @@ typedef struct {
 #define PyArray_ISALIGNED(m) PyArray_CHKFLAGS(m, ALIGNED)
 
 #define PyArray_ISCARRAY(m) PyArray_CHKFLAGS(m, CARRAY_FLAGS)
+#define PyArray_ISCARRAY_RO(m) PyArray_CHKFLAGS(m, CARRAY_FLAGS_RO)
 #define PyArray_ISFARRAY(m) PyArray_CHKFLAGS(m, FARRAY_FLAGS)
+#define PyArray_ISFARRAY_RO(m) PyArray_CHKFLAGS(m, FARRAY_FLAGS_RO)
 #define PyArray_ISBEHAVED(m) PyArray_CHKFLAGS(m, BEHAVED_FLAGS)
 #define PyArray_ISBEHAVED_RO(m) PyArray_CHKFLAGS(m, BEHAVED_FLAGS_RO)
 
diff --git a/scipy/base/memmap.py b/scipy/base/memmap.py
index f43372cb7..e591566a3 100644
--- a/scipy/base/memmap.py
+++ b/scipy/base/memmap.py
@@ -230,7 +230,7 @@ import types
 import operator
 import copy_reg
 import copy
-from numeric import frombuffer, newbuffer
+from numeric import frombuffer, newbuffer, getbuffer
 
 valid_filemodes = ["r", "c", "r+", "w+"]
 writeable_filemodes = ["r+","w+"]
@@ -327,13 +327,10 @@ class memmap:
             raise RuntimeError("memmap no longer valid.  (closed?)")
         if end is None:
             end = len(self)
-        obj = frombuffer(self._mmap)
-        if self._readonly:
-            obj.flags.writeable=False
-        return obj.data
+        return getbuffer(self._mmap, begin, end)    
 
     def _chkOverlaps(self, begin, end):
-        """_chkOverlaps(self, begin, end) is called to raise an exception
+        """_chkOverlaps(self, begin,  end) is called to raise an exception
         if the requested slice will overlap any slices which have already
         been taken.
         """
@@ -621,7 +618,7 @@ class memmapslice:
         return "<memmapslice of length:%d %s>" % (len(self), s)
 
     def __len__(self):
-        if self._buffer:
+        if self._buffer is not None:
             return len(self.__buffer__())
         else:
             return 0
@@ -787,9 +784,8 @@ def test():
 def proveit(N, filename="memmap.dat", pagesize=1024, mode="r"):
     """proveit is a diagnostic function which creates a file of size 'N',
     memory maps it, and then reads one byte at 'pagesize' intervals."""
-    
+
     import numeric as num
-    import os
     
     f = _open(filename, "w+")
     f.seek(N-1)
diff --git a/scipy/base/numeric.py b/scipy/base/numeric.py
index 0c4a3cb4c..f37947897 100644
--- a/scipy/base/numeric.py
+++ b/scipy/base/numeric.py
@@ -1,6 +1,6 @@
 __all__ = ['newaxis', 'ndarray', 'bigndarray', 'flatiter', 'ufunc',
            'arange', 'array', 'zeros', 'empty', 'multiter',
-           'fromstring', 'fromfile', 'frombuffer',
+           'fromstring', 'fromfile', 'frombuffer','newbuffer','getbuffer',
            'where', 'concatenate', 'fastCopyAndTranspose',
            'register_dtype', 'set_numeric_ops', 'can_cast',
            'asarray', 'asanyarray', 'isfortran', 'zeros_like', 'empty_like',
@@ -58,6 +58,7 @@ fromstring = multiarray.fromstring
 fromfile = multiarray.fromfile
 frombuffer = multiarray.frombuffer
 newbuffer = multiarray.newbuffer
+getbuffer = multiarray.getbuffer
 where = multiarray.where
 concatenate = multiarray.concatenate
 fastCopyAndTranspose = multiarray._fastCopyAndTranspose
@@ -65,6 +66,7 @@ register_dtype = multiarray.register_dtype
 set_numeric_ops = multiarray.set_numeric_ops
 can_cast = multiarray.can_cast
 
+
 def asarray(a, dtype=None, fortran=False):
     """returns a as an array.  Unlike array(),
     no copy is performed if a is already an array.  Subclasses are converted
diff --git a/scipy/base/src/arrayobject.c b/scipy/base/src/arrayobject.c
index d32ab9c7d..ca85fe4bf 100644
--- a/scipy/base/src/arrayobject.c
+++ b/scipy/base/src/arrayobject.c
@@ -3056,7 +3056,6 @@ static Bool
 _IsWriteable(PyArrayObject *ap)
 {
 	PyObject *base=ap->base;
-	PyBufferProcs *pb;
 	void *dummy;
 	int n;
 
@@ -4965,6 +4964,8 @@ _bufferedcast(PyArrayObject *out, PyArrayObject *in)
 	
 	inbuffer = PyDataMem_NEW(PyArray_BUFSIZE*elsize);
 	if (inbuffer == NULL) return -1;
+	if (PyArray_ISOBJECT(in)) 
+		memset(inbuffer, 0, PyArray_BUFSIZE*elsize);
 	it_in = (PyArrayIterObject *)PyArray_IterNew((PyObject *)in);
 	if (it_in == NULL) goto exit;
 
@@ -4973,7 +4974,9 @@ _bufferedcast(PyArrayObject *out, PyArrayObject *in)
 			    PyArray_ISNOTSWAPPED(out));
 		outbuffer = PyDataMem_NEW(PyArray_BUFSIZE*oelsize);
 		if (outbuffer == NULL) goto exit;
-
+		if (PyArray_ISOBJECT(out))
+			memset(outbuffer, 0, PyArray_BUFSIZE*oelsize);
+		
 		it_out = (PyArrayIterObject *)PyArray_IterNew((PyObject *)out);
 		if (it_out == NULL) goto exit;
 
@@ -5115,8 +5118,8 @@ PyArray_CastTo(PyArrayObject *out, PyArrayObject *mp)
 				
 	}
 
-	simple =  ((PyArray_ISCARRAY(mp) && PyArray_ISCARRAY(out)) ||   \
-                   (PyArray_ISFARRAY(mp) && PyArray_ISFARRAY(out)));
+	simple =  ((PyArray_ISCARRAY_RO(mp) && PyArray_ISCARRAY(out)) ||   \
+                   (PyArray_ISFARRAY_RO(mp) && PyArray_ISFARRAY(out)));
 	
 	if (simple) {
 		char *inptr;
diff --git a/scipy/base/src/multiarraymodule.c b/scipy/base/src/multiarraymodule.c
index 8fce86372..b2d2e5562 100644
--- a/scipy/base/src/multiarraymodule.c
+++ b/scipy/base/src/multiarraymodule.c
@@ -3509,7 +3509,7 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Typecode *type,
 	/* Store a reference for decref on deallocation */
 	ret->base = buf;
 	PyArray_UpdateFlags(ret, ALIGNED);
-	return (PyObject *)ret; 	
+	return (PyObject *)ret;
 }
 
 static char doc_frombuffer[] = \
@@ -3869,8 +3869,8 @@ array_can_cast_safely(PyObject *dummy, PyObject *args, PyObject *kwds)
 }
 
 static char doc_new_buffer[] = \
-	"newbuffer(size) return a new (uninitialized) buffer object of "\
-	"size bytes.";
+	"newbuffer(size) return a new uninitialized buffer object of size "
+	"bytes";
 
 static PyObject *
 new_buffer(PyObject *dummy, PyObject *args)
@@ -3883,6 +3883,33 @@ new_buffer(PyObject *dummy, PyObject *args)
 	return PyBuffer_New(size);
 }
 
+static char doc_buffer_buffer[] = \
+	"getbuffer(obj [,offset[, size]]) create a buffer object from the "\
+	"given object\n  referencing a slice of length size starting at "\
+	"offset.  Default\n is the entire buffer. A read-write buffer is "\
+	"attempted followed by a read-only buffer.";
+
+static PyObject *
+buffer_buffer(PyObject *dummy, PyObject *args, PyObject *kwds)
+{
+	PyObject *obj;
+	int offset=0, size=Py_END_OF_BUFFER, n;
+	void *unused;
+	static char *kwlist[] = {"object", "offset", "size", NULL};
+	
+	if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|ii", kwlist, 
+					 &obj, &offset, &size))
+		return NULL;
+
+
+	if (PyObject_AsWriteBuffer(obj, &unused, &n) < 0) {
+		PyErr_Clear();
+		return PyBuffer_FromObject(obj, offset, size);		
+	}
+	else
+		return PyBuffer_FromReadWriteObject(obj, offset, size);
+}
+
 
 static struct PyMethodDef array_module_methods[] = {
 	{"set_string_function", (PyCFunction)array_set_string_function, 
@@ -3926,6 +3953,8 @@ static struct PyMethodDef array_module_methods[] = {
 	 METH_VARARGS | METH_KEYWORDS, doc_can_cast_safely},		
 	{"newbuffer", (PyCFunction)new_buffer,
 	 METH_VARARGS, doc_new_buffer},	
+	{"getbuffer", (PyCFunction)buffer_buffer,
+	 METH_VARARGS | METH_KEYWORDS, doc_buffer_buffer},	
 	{NULL,		NULL, 0}		/* sentinel */
 };
 
diff --git a/scipy/base/src/scalartypes.inc.src b/scipy/base/src/scalartypes.inc.src
index 135b0dcab..6c7124e6b 100644
--- a/scipy/base/src/scalartypes.inc.src
+++ b/scipy/base/src/scalartypes.inc.src
@@ -1258,7 +1258,7 @@ string_arrtype_dealloc(PyObject *v)
 static void
 object_arrtype_dealloc(PyObject *v)
 {
-	Py_DECREF(((PyObjectScalarObject *)v)->obval);
+	Py_XDECREF(((PyObjectScalarObject *)v)->obval);
 	v->ob_type->tp_free(v);
 }
 
diff --git a/scipy/base/src/ufuncobject.c b/scipy/base/src/ufuncobject.c
index 439974672..506ea6d98 100644
--- a/scipy/base/src/ufuncobject.c
+++ b/scipy/base/src/ufuncobject.c
@@ -1149,6 +1149,8 @@ construct_matrices(PyUFuncLoopObject *loop, PyObject *args, PyArrayObject **mps)
 		loop->buffer[0] = (char *)malloc(loop->bufsize*(cnt+cntcast));
 		/* fprintf(stderr, "Allocated buffer at %p of size %d, cnt=%d, cntcast=%d\n", loop->buffer[0], loop->bufsize * (cnt + cntcast), cnt, cntcast); */
 		if (loop->buffer[0] == NULL) return -1;
+		if (loop->obj) memset(loop->buffer[0], 0, 
+				      loop->bufsize*(cnt+cntcast));
 		castptr = loop->buffer[0] + loop->bufsize*cnt;
 		for (i=0; i<self->nargs; i++) {
 			if (i > 0)
@@ -1215,8 +1217,9 @@ construct_loop(PyUFuncObject *self, PyObject *args, PyArrayObject **mps)
 		PyErr_SetString(PyExc_ValueError, "function not supported");
 		return NULL;
 	}
-	loop = malloc(sizeof(PyUFuncLoopObject));
-	if (loop==NULL) return NULL;
+        if ((loop = malloc(sizeof(PyUFuncLoopObject)))==NULL) {
+                PyErr_NoMemory(); return loop;
+        }
 	
 	loop->index = 0;
 	loop->ufunc = self;
@@ -1621,6 +1624,12 @@ construct_reduce(PyUFuncObject *self, PyArrayObject **arr, int axis,
                 loop->swap = !(PyArray_ISNOTSWAPPED(aar));
         }
 
+        /* Determine if object arrays are involved */
+        if (otype == PyArray_OBJECT || aar->descr->type_num == PyArray_OBJECT)
+                loop->obj = 1;
+        else
+                loop->obj = 0;
+
         if (loop->meth == ZERODIM_REDUCELOOP) {
                 idarr = _getidentity(self, otype, str);
                 if (idarr == NULL) goto fail;
@@ -1721,33 +1730,31 @@ construct_reduce(PyUFuncObject *self, PyArrayObject **arr, int axis,
 	}
 	loop->steps[2] = loop->steps[1];
 	loop->bufptr[2] = loop->bufptr[1] + loop->steps[2];
+
 	
 	if (loop->meth == BUFFER_UFUNCLOOP) {
-
+		int _size;
 		loop->steps[0] = loop->outsize;
                 if (otype != aar->descr->type_num) {
-                        loop->buffer = (char *)malloc(loop->bufsize*\
-                                                      (loop->outsize + \
-                                                       aar->itemsize));
+			_size=loop->bufsize*(loop->outsize +		\
+					     aar->itemsize);
+                        loop->buffer = (char *)malloc(_size);
                         if (loop->buffer == NULL) goto fail;
+			if (loop->obj) memset(loop->buffer, 0, _size);
                         loop->castbuf = loop->buffer + \
                                 loop->bufsize*aar->itemsize;
                         loop->bufptr[0] = loop->castbuf;     
                         loop->cast = aar->descr->cast[otype];
                 }
                 else {
-                        loop->buffer = (char *)malloc(loop->bufsize*\
-                                                      loop->outsize);
+			_size = loop->bufsize * loop->outsize;
+                        loop->buffer = (char *)malloc(_size);
                         if (loop->buffer == NULL) goto fail;
+			if (loop->obj) memset(loop->buffer, 0, _size);
                         loop->bufptr[0] = loop->buffer;
                 }
 	}
 
-        /* Determine if object arrays are involved */
-        if (otype == PyArray_OBJECT || aar->descr->type_num == PyArray_OBJECT)
-                loop->obj = 1;
-        else
-                loop->obj = 0;
 
 	PyUFunc_clearfperr();
 	return loop;
@@ -1785,13 +1792,15 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, int axis, int otype)
         case ZERODIM_REDUCELOOP:
 		/* fprintf(stderr, "ZERO..%d\n", loop->size); */
 		for(i=0; i<loop->size; i++) {
+			if (loop->obj) Py_INCREF(*((PyObject **)loop->idptr));
 			memcpy(loop->bufptr[1], loop->idptr, loop->outsize);
 			loop->bufptr[1] += loop->outsize;
-		}               
+		}
                 break;
         case ONEDIM_REDUCELOOP:
 		/*fprintf(stderr, "ONEDIM..%d\n", loop->size); */
                 while(loop->index < loop->size) {
+			if (loop->obj) Py_INCREF(*((PyObject **)loop->it->dataptr));
                         memcpy(loop->bufptr[1], loop->it->dataptr, 
                                loop->outsize);
 			PyArray_ITER_NEXT(loop->it);
@@ -1803,6 +1812,8 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, int axis, int otype)
 		/*fprintf(stderr, "NOBUFFER..%d\n", loop->size); */
                 while(loop->index < loop->size) {
 			/* Copy first element to output */
+			if (loop->obj) 
+				Py_INCREF(*((PyObject **)loop->it->dataptr));
                         memcpy(loop->bufptr[1], loop->it->dataptr, 
                                loop->outsize);
 			/* Adjust input pointer */
@@ -1816,7 +1827,6 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, int axis, int otype)
                         loop->bufptr[1] += loop->outsize;
                         loop->bufptr[2] = loop->bufptr[1];
                         loop->index++; 
-			if (PyErr_Occurred()) goto fail;
                 }
                 break;
         case BUFFER_UFUNCLOOP:
@@ -1844,6 +1854,8 @@ PyUFunc_Reduce(PyUFuncObject *self, PyArrayObject *arr, int axis, int otype)
 						     loop->insize);
 				loop->cast(loop->buffer, loop->castbuf,
 					   1, NULL, NULL);
+				if (loop->obj) 
+					Py_INCREF(*((PyObject **)loop->castbuf));
 				memcpy(loop->bufptr[1], loop->castbuf,
 				       loop->outsize);
 			}
@@ -1919,6 +1931,8 @@ PyUFunc_Accumulate(PyUFuncObject *self, PyArrayObject *arr, int axis,
         case ZERODIM_REDUCELOOP: /* Accumulate */
 		/* fprintf(stderr, "ZERO..%d\n", loop->size); */
 		for(i=0; i<loop->size; i++) {
+			if (loop->obj) 
+				Py_INCREF(*((PyObject **)loop->idptr));
 			memcpy(loop->bufptr[1], loop->idptr, loop->outsize);
 			loop->bufptr[1] += loop->outsize;
 		}               
@@ -1926,6 +1940,8 @@ PyUFunc_Accumulate(PyUFuncObject *self, PyArrayObject *arr, int axis,
         case ONEDIM_REDUCELOOP: /* Accumulate */
 		/* fprintf(stderr, "ONEDIM..%d\n", loop->size); */
                 while(loop->index < loop->size) {
+			if (loop->obj) 
+				Py_INCREF(*((PyObject **)loop->it->dataptr));
                         memcpy(loop->bufptr[1], loop->it->dataptr, 
                                loop->outsize);
 			PyArray_ITER_NEXT(loop->it);
@@ -1937,6 +1953,8 @@ PyUFunc_Accumulate(PyUFuncObject *self, PyArrayObject *arr, int axis,
 		/* fprintf(stderr, "NOBUFFER..%d\n", loop->size); */
                 while(loop->index < loop->size) {
 			/* Copy first element to output */
+			if (loop->obj) 
+				Py_INCREF(*((PyObject **)loop->it->dataptr));
                         memcpy(loop->bufptr[1], loop->it->dataptr, 
                                loop->outsize);
 			/* Adjust input pointer */
@@ -1978,6 +1996,8 @@ PyUFunc_Accumulate(PyUFuncObject *self, PyArrayObject *arr, int axis,
 						     loop->insize);
 				loop->cast(loop->buffer, loop->castbuf,
 					   1, NULL, NULL);
+				if (loop->obj) 
+					Py_INCREF(*((PyObject **)loop->castbuf));
 				memcpy(loop->bufptr[1], loop->castbuf,
 				       loop->outsize);
 			}
@@ -2086,7 +2106,6 @@ PyUFunc_Reduceat(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *ind,
 	case ZERODIM_REDUCELOOP:
 		/* fprintf(stderr, "ZERO..\n"); */
 		break;
-
 	/* NOBUFFER -- behaved array and same type */
 	case NOBUFFER_UFUNCLOOP: 	                /* Reduceat */
 		/* fprintf(stderr, "NOBUFFER..%d\n", loop->size); */
@@ -2095,6 +2114,8 @@ PyUFunc_Reduceat(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *ind,
 			for (i=0; i<nn; i++) {
 				loop->bufptr[0] = loop->it->dataptr +	\
 					(*ptr)*loop->instrides;
+				if (loop->obj) 
+					Py_INCREF(*((PyObject **)loop->bufptr[0]));
 				memcpy(loop->bufptr[1], loop->bufptr[0],
 				       loop->outsize);
 				mm = (i==nn-1 ? arr->dimensions[axis]-*ptr : \
@@ -2123,6 +2144,8 @@ PyUFunc_Reduceat(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *ind,
 		while(loop->index < loop->size) {
 			ptr = (intp *)ind->data;
 			for (i=0; i<nn; i++) {
+				if (loop->obj) 
+					Py_INCREF(*((PyObject **)loop->idptr));
 				memcpy(loop->bufptr[1], loop->idptr, 
 				       loop->outsize);
 				n = 0;
@@ -2307,13 +2330,13 @@ PyUFunc_GenericReduction(PyUFuncObject *self, PyObject *args,
 	if (ret==NULL) return NULL;
 	if (op->ob_type != ret->ob_type) {
 		res = PyObject_CallMethod(op, "__array_wrap__", "O", ret);
+		if (res == NULL) PyErr_Clear();
+		else if (res == Py_None) Py_DECREF(res);
+		else {
+			Py_DECREF(ret);
+			return res;
+		}	
 	}
-	if (res == NULL) PyErr_Clear();
-	else if (res == Py_None) Py_DECREF(res);
-	else {
-		Py_DECREF(ret);
-		return res;
-	}	
 	return PyArray_Return(ret);
 	
 }