Fixed pickling to support arbitrary dtypedescr arrays.

8 files changed, 365 insertions, 73 deletions

diff --git a/scipy/base/__init__.py b/scipy/base/__init__.py
index 4bc197f6b..60c4fb723 100644
--- a/scipy/base/__init__.py
+++ b/scipy/base/__init__.py
@@ -21,6 +21,7 @@ from polynomial import *
 from machar import *
 from getlimits import *
 import ma
+import records as rec
 from records import *
 from memmap import *
 import convertcode
diff --git a/scipy/base/numeric.py b/scipy/base/numeric.py
index 895b5ce02..f81195e13 100644
--- a/scipy/base/numeric.py
+++ b/scipy/base/numeric.py
@@ -239,7 +239,7 @@ def array_repr(arr, max_line_width=None, precision=None, suppress_small=None):
         if issubclass(arr.dtype, flexible):
             if typename not in ['unicode','string','void']:
                 typename = arr.dtype.__name__
-            typename = "%s%d" % (typename, arr.itemsize)
+            typename = "(%s,%d)" % (typename, arr.itemsize)
         return cName + "(%s, dtype=%s)" % (lst, typename)
 
 def array_str(a, max_line_width=None, precision=None, suppress_small=None):
diff --git a/scipy/base/records.py b/scipy/base/records.py
index 25503826b..62eb76777 100644
--- a/scipy/base/records.py
+++ b/scipy/base/records.py
@@ -13,6 +13,7 @@ import re
 format_re = re.compile(r'(?P<repeat> *[(]?[ ,0-9]*[)]? *)(?P<dtype>[A-Za-z0-9.]*)')
 
 numfmt = nt.typeDict
+_typestr = nt._typestr
 
 def find_duplicate(list):
     """Find duplication in a list, return a list of duplicated elements"""
@@ -69,7 +70,7 @@ class format_parser:
 
         _alignment = nt._alignment
         _bytes = nt.nbytes
-        _typestr = nt._typestr
+
         if (type(formats) in [types.ListType, types.TupleType]):
             _fmt = formats[:]
         elif (type(formats) == types.StringType):
@@ -105,11 +106,14 @@ class format_parser:
 
             # Flexible types need special treatment
             _dtype = _dtype.strip()
-            if _dtype[0] in ['V','S','U']:
+            if _dtype[0] in ['V','S','U','a']:
                 self._itemsizes[i] = int(_dtype[1:])
                 if _dtype[0] == 'U':
                     self._itemsizes[i] *= unisize
-                _dtype = _dtype[0]
+                if _dtype[0] == 'a':
+                    _dtype = 'S'
+                else:
+                    _dtype = _dtype[0]
 
             if _repeat == '': 
                 _repeat = 1
@@ -214,7 +218,7 @@ class record(nt.void):
         all.sort(lambda x,y: cmp(x[1],y[1]))
 
         outlist = [self.getfield(item[0], item[1]) for item in all]
-        return str(outlist)
+        return str(tuple(outlist))
 
     def __getattribute__(self, attr):
         if attr in ['setfield', 'getfield', 'fields']:
@@ -252,8 +256,10 @@ class recarray(sb.ndarray):
     def __new__(subtype, shape, formats, names=None, titles=None,
                 buf=None, offset=0, strides=None, swap=0, aligned=0):
 
-        if isinstance(formats,str):
-            parsed = format_parser(formats, aligned, names, titles)
+        if isinstance(formats, sb.dtypedescr):
+            descr = formats   
+        elif isinstance(formats,str):
+            parsed = format_parser(formats, names, titles, aligned)
             descr = parsed._descr
         else:
             if aligned:
@@ -287,4 +293,107 @@ class recarray(sb.ndarray):
             return sb.ndarray.__setattr__(self,attr,val)
         
         return self.setfield(val,*res)
+
+
+def fromarrays(arrayList, formats=None, names=None, titles=None, shape=None,
+               swap=0, aligned=0):
+    """ create a record array from a (flat) list of arrays
+
+    >>> x1=array([1,2,3,4])
+    >>> x2=array(['a','dd','xyz','12'])
+    >>> x3=array([1.1,2,3,4])
+    >>> r=fromarrays([x1,x2,x3],names='a,b,c')
+    >>> print r[1]
+    (2, 'dd\x00', 2.0)
+    >>> x1[1]=34
+    >>> r.a
+    array([1, 2, 3, 4])
+    """
+
+    if shape is None or shape == 0:
+        shape = arrayList[0].shape
+            
+    if formats is None:
+        # go through each object in the list to see if it is an ndarray
+        # and determine the formats.
+        formats = ''
+        for obj in arrayList:
+            if not isinstance(obj, sb.ndarray):
+                raise ValueError, "item in the array list must be an ndarray."
+            if obj.ndim == 1: 
+                _repeat = ''
+            elif len(obj._shape) >= 2:
+                _repeat = `obj._shape[1:]`
+            formats += _repeat + _typestr[obj.dtype]
+            if issubclass(obj.dtype, nt.flexible):
+                formats += `obj.itemsize`
+            formats += ','
+        formats=formats[:-1]
+
+    for obj in arrayList:
+        if obj.shape != shape:
+            raise ValueError, "array has different shape"
+
+    parsed = format_parser(formats, names, titles, aligned)
+    _names = parsed._names
+    _array = recarray(shape, parsed._descr, swap=swap)
     
+    # populate the record array (makes a copy)
+    for i in range(len(arrayList)):
+        _array[_names[i]] = arrayList[i]
+
+    return _array
+
+def fromrecords(recList, formats=None, names=None, shape=0, swap=0, aligned=0):
+    """ create a Record Array from a list of records in text form
+
+        The data in the same field can be heterogeneous, they will be promoted
+        to the highest data type.  This method is intended for creating
+        smaller record arrays.  If used to create large array e.g.
+
+        r=fromrecords([[2,3.,'abc']]*100000)
+
+        it is slow.
+
+    >>> r=fromrecords([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
+    >>> print r[0]
+    (456, 'dbe', 1.2)
+    >>> r.field('col1')
+    array([456,   2])
+    >>> r.field('col2')
+    array(['dbe', 'de'])
+    >>> import cPickle
+    >>> print cPickle.loads(cPickle.dumps(r))
+    recarray[ 
+    (456, 'dbe', 1.2),
+    (2, 'de', 1.3)
+    ]
+    """
+
+    if shape == 0:
+        _shape = len(recList)
+    else:
+        _shape = shape
+
+    _nfields = len(recList[0])
+    for _rec in recList:
+        if len(_rec) != _nfields:
+            raise ValueError, "inconsistent number of objects in each record"
+    arrlist = [0]*_nfields
+    for col in range(_nfields):
+        tmp = [0]*_shape
+        for row in range(_shape):
+            tmp[row] = recList[row][col]
+        try:
+            arrlist[col] = num.array(tmp)
+        except:
+            try:
+                arrlist[col] = chararray.array(tmp)
+            except:
+                raise ValueError, "inconsistent data at row %d,field %d" % (row, col)
+    _array = fromarrays(arrlist, formats=formats, shape=_shape, names=names,
+                byteorder=byteorder, aligned=aligned)
+    del arrlist
+    del tmp
+    return _array
+
diff --git a/scipy/base/src/arraymethods.c b/scipy/base/src/arraymethods.c
index 9530b1520..0cca98896 100644
--- a/scipy/base/src/arraymethods.c
+++ b/scipy/base/src/arraymethods.c
@@ -854,6 +854,8 @@ array_reduce(PyArrayObject *self, PyObject *args)
 {
 	PyObject *ret=NULL, *state=NULL, *obj=NULL, *mod=NULL;
 	PyObject *mybool, *thestr=NULL;
+	PyArray_Descr *descr;
+	char endian;
 
 	/* Return a tuple of (callable object, arguments, object's state) */
 	/*  We will put everything in the object's state, so that on UnPickle
@@ -862,10 +864,10 @@ array_reduce(PyArrayObject *self, PyObject *args)
 	ret = PyTuple_New(3);
 	if (ret == NULL) return NULL;
 	mod = PyImport_ImportModule("scipy.base.multiarray");
-	if (mod == NULL) return NULL;
+	if (mod == NULL) {Py_DECREF(ret); return NULL;}
 	obj = PyObject_GetAttrString(mod, "empty");
 	Py_DECREF(mod);
-	if (obj == NULL) return NULL;
+	if (obj == NULL) {Py_DECREF(ret); return NULL;}
 	PyTuple_SET_ITEM(ret, 0, obj);
 	PyTuple_SET_ITEM(ret, 1, 
 			 Py_BuildValue("NNN",
@@ -873,14 +875,13 @@ array_reduce(PyArrayObject *self, PyObject *args)
 						     PyInt_FromLong(0)),
 				       PyObject_GetAttrString((PyObject *)self,
 							      "dtypechar"),
-				       PyInt_FromLong((long) 0)));
+				       PyInt_FromLong((long) self->descr->elsize)));
 	
 	/* Now fill in object's state.  This is a tuple with 
 	   4 arguments
 
 	   1) a Tuple giving the shape
-	   2) a string giving the typestr (> or < indicates byte order
-	      if swap is possibly needed)
+	   2) a PyArray_Descr Object (with correct bytorder set)
 	   3) a Bool stating if Fortran or not
 	   4) a binary string with the data (or a list for Object arrays)
 
@@ -893,9 +894,24 @@ array_reduce(PyArrayObject *self, PyObject *args)
 		Py_DECREF(ret); return NULL;
 	}
 	PyTuple_SET_ITEM(state, 0, PyObject_GetAttrString((PyObject *)self, 
-							  "shape"));	
-	PyTuple_SET_ITEM(state, 1, PyObject_GetAttrString((PyObject *)self, 
-							  "dtypestr"));
+							  "shape"));
+	endian = self->descr->byteorder;
+	if ((endian == '=') || (PyArray_ISNOTSWAPPED(self) !=		\
+				PyArray_IsNativeByteOrder(endian))) {
+		descr = PyArray_DescrNew(self->descr);
+		endian = '<';
+		if (!PyArray_IsNativeByteOrder(endian)) endian = '>';
+		if (!PyArray_ISNOTSWAPPED(self)) {
+			if (endian == '<') endian = '>';
+			else endian = '<';
+		}
+		descr->byteorder = endian;
+	}
+	else {
+		descr = self->descr;
+		Py_INCREF(descr);
+	}
+	PyTuple_SET_ITEM(state, 1, (PyObject *)descr);
 	mybool = (PyArray_ISFORTRAN(self) ? Py_True : Py_False);
 	Py_INCREF(mybool);
 	PyTuple_SET_ITEM(state, 2, mybool);
@@ -919,8 +935,7 @@ static char doc_setstate[] = "a.__setstate__(tuple) for unpickling.";
 
 /*
 	   1) a Tuple giving the shape
-	   2) a string giving the typestr (> or < indicates byte order
-	      if swap is possibly needed)
+	   2) a PyArray_Descr Object
 	   3) a Bool stating if Fortran or not
 	   4) a binary string with the data (or a list if Object array) 
 */
@@ -939,25 +954,21 @@ array_setstate(PyArrayObject *self, PyObject *args)
 	long fortran;
 	PyObject *rawdata;
 	char *datastr;
-	const char *typestr;
-	int len, typestrlen;
-	int swap;
+	int len;
 	intp dimensions[MAX_DIMS];
 	int nd;
 	
 	/* This will free any memory associated with a and
 	   use the string in setstate as the (writeable) memory.
 	*/
-	if (!PyArg_ParseTuple(args, "(O!z#iO)", &PyTuple_Type,
-			      &shape, &typestr, &typestrlen,
+	if (!PyArg_ParseTuple(args, "(O!O!iO)", &PyTuple_Type,
+			      &shape, &PyArrayDescr_Type, &typecode, 
 			      &fortran, &rawdata))
 		return NULL;
 
-	typecode = _array_typedescr_fromstr((char *)typestr, &swap);
-	if (typecode == NULL) return NULL;
-
 	Py_XDECREF(self->descr);
 	self->descr = typecode;
+	Py_INCREF(typecode);
 	nd = PyArray_IntpFromSequence(shape, dimensions, MAX_DIMS);
 	if (typecode->type_num == PyArray_OBJECT) {
 		if (!PyList_Check(rawdata)) {
@@ -1030,7 +1041,8 @@ array_setstate(PyArrayObject *self, PyObject *args)
 			self->base = rawdata;
 			Py_INCREF(self->base);
 		}
-		if (swap) self->flags &= ~NOTSWAPPED;
+		if (!PyArray_IsNativeByteOrder(typecode->byteorder))
+			self->flags &= ~NOTSWAPPED;
 	}
 	else {
 		self->data = PyDataMem_NEW(PyArray_NBYTES(self));
diff --git a/scipy/base/src/arrayobject.c b/scipy/base/src/arrayobject.c
index 603747779..ba4d17762 100644
--- a/scipy/base/src/arrayobject.c
+++ b/scipy/base/src/arrayobject.c
@@ -3538,9 +3538,11 @@ PyArray_NewFromDescr(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
 					   this to be reset if truly
 					   desired */
         }
-	if (descr->byteorder != '=' &&				\
-	    !PyArray_IsNativeByteOrder(descr->byteorder)) {
-		self->flags &= ~NOTSWAPPED;
+	if ((descr->byteorder != '=') && (descr->byteorder != '|')) {
+		if (PyArray_IsNativeByteOrder(descr->byteorder)) 
+			self->flags |= NOTSWAPPED;
+		else
+			self->flags &= ~NOTSWAPPED;
 	}
 
         self->data = data;
@@ -4706,10 +4708,22 @@ array_alloc(PyTypeObject *type, int nitems)
 
 static char Arraytype__doc__[] = 
         "A array object represents a multidimensional, homogeneous array\n"
-	"  of basic values.  Arrays are sequence, mapping and numeric\n"
-	"  objects.  More information is available in the scipy module and\n"
-	"  by looking at the methods and attributes of an array.";
-
+	"  of fixed-size items.  An associated data-type-descriptor object details\n"
+	"  the data-type in an array (including any fields).  An array can be\n"
+	"  constructed using the scipy.array command. Arrays are sequence, mapping\n"
+	"  and numeric objects.  More information is available in the scipy module\n"
+	"  and by looking at the methods and attributes of an array.\n\n"
+	"  ndarray.__new__(subtype, shape=, dtype=long_, buffer=None, offset=0,\n"
+        "                  strides=None, swap=0, fortran=False)\n\n"
+	"   There are two modes of creating an array using the __new__ method:\n"
+	"   1) If buffer is None, then only shape, dtype, and fortran are used\n"
+	"   2) If buffer is an object exporting the buffer interface, then all\n"
+	"      keywords are interpreted.\n"
+	"   The dtype parameter can be any object that can be interpreted as a\n"
+	"   scipy.dtypedescr object.\n\n"
+	"   No __init__ method is needed because the array is fully initialized\n"
+	"      after the __new__ method.";
+	
 static PyTypeObject PyBigArray_Type = { 
         PyObject_HEAD_INIT(NULL)
         0,					  /*ob_size*/
@@ -8019,28 +8033,150 @@ static PyObject *
 arraydescr_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)
 {
 	PyObject *odescr;
-	PyArray_Descr *descr;
+	PyArray_Descr *descr, *conv;
 	int align=0;
 	
 	if (!PyArg_ParseTuple(args, "O|i", &odescr, &align))
 		return NULL;
-
+	
 	if (align) {
 		if PyDict_Check(odescr) 
 			return (PyObject *)_convert_from_dict(odescr, 1);
 		if PyList_Check(odescr) 
 			return (PyObject *)_convert_from_list(odescr, 1);
 	}
-	if (!PyArray_DescrConverter(odescr, &descr)) 
+	if (!PyArray_DescrConverter(odescr, &conv)) 
 		return NULL;
+	/* Get a new copy of it */
+	descr = PyArray_DescrNew(conv);
+	Py_DECREF(conv);
 	return (PyObject *)descr;
 }
 
+static char doc_arraydescr_reduce[] = "self.__reduce__()  for pickling.";
+
+/* return a tuple of (callable object, args, state) */
+static PyObject *
+arraydescr_reduce(PyArray_Descr *self, PyObject *args)
+{
+	PyObject *ret, *mod, *obj;
+	PyObject *state;
+	char endian;
+	int elsize, alignment;
+
+	ret = PyTuple_New(3);
+	if (ret == NULL) return NULL;
+	mod = PyImport_ImportModule("scipy.base.multiarray");
+	if (mod == NULL) {Py_DECREF(ret); return NULL;}
+	obj = PyObject_GetAttrString(mod, "dtypedescr");
+	Py_DECREF(mod);
+	if (obj == NULL) {Py_DECREF(ret); return NULL;}
+	PyTuple_SET_ITEM(ret, 0, obj);
+	if (PyTypeNum_ISUSERDEF(self->type_num)) {
+		obj = (PyObject *)self->typeobj;
+		Py_INCREF(obj);
+	}
+	else {
+		obj = PyString_FromFormat("%c%d",self->kind, self->elsize);
+	}
+	PyTuple_SET_ITEM(ret, 1, Py_BuildValue("(N)", obj));
+
+	/* Now return the state which is at least 
+	   byteorder, subarray, and fields */
+	endian = self->byteorder;
+	if (endian == '=') {
+		endian = '<';
+		if (!PyArray_IsNativeByteOrder(endian)) endian = '>';
+	}
+	state = PyTuple_New(5);
+	PyTuple_SET_ITEM(state, 0, PyString_FromFormat("%c", endian));
+	PyTuple_SET_ITEM(state, 1, arraydescr_subdescr_get(self));
+	if (self->fields) {
+		Py_INCREF(self->fields);
+		PyTuple_SET_ITEM(state, 2, self->fields);
+	}
+	else {
+		PyTuple_SET_ITEM(state, 2, Py_None);
+		Py_INCREF(Py_None);
+	}
+
+	/* for extended types it also includes elsize and alignment */
+	if (PyTypeNum_ISEXTENDED(self->type_num)) {
+		elsize = self->elsize;
+		alignment = self->alignment;
+	}
+	else {elsize = -1; alignment = -1;}
+
+	PyTuple_SET_ITEM(state, 3, PyInt_FromLong(elsize));
+	PyTuple_SET_ITEM(state, 4, PyInt_FromLong(alignment));
+
+	PyTuple_SET_ITEM(ret, 2, state);
+	return ret;
+}
+
+
+
+/* state is at least byteorder, subarray, and fields but could include elsize 
+   and alignment for EXTENDED arrays 
+*/
+static char doc_arraydescr_setstate[] = "self.__setstate__()  for pickling.";
+
+static PyObject *
+arraydescr_setstate(PyArray_Descr *self, PyObject *args)
+{
+	int elsize = -1, alignment = -1;
+	char endian;
+	PyObject *subarray, *fields;
+
+	if (!PyArg_ParseTuple(args, "(cOOii)", &endian, &subarray, &fields,
+			      &elsize, &alignment)) return NULL;
+	
+	if (PyArray_IsNativeByteOrder(endian)) endian = '=';
+
+	self->byteorder = endian;
+	if (self->subarray) {
+		Py_XDECREF(self->subarray->base);
+		Py_XDECREF(self->subarray->shape);
+		free(self->subarray);
+	}
+	self->subarray = NULL;
+
+	if (subarray != Py_None) {
+		self->subarray = malloc(sizeof(PyArray_ArrayDescr));
+		self->subarray->base = (PyArray_Descr *)PyTuple_GET_ITEM(subarray, 0);
+		Py_INCREF(self->subarray->base);
+		self->subarray->shape = PyTuple_GET_ITEM(subarray, 1);
+		Py_INCREF(self->subarray->shape);
+	}
+
+	if (fields != Py_None) {
+		Py_XDECREF(self->fields);
+		self->fields = fields;
+		Py_INCREF(fields);
+	}
+	
+	if (PyTypeNum_ISEXTENDED(self->type_num)) {
+		self->elsize = elsize;
+		self->alignment = alignment;
+	}
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+
+
+static PyMethodDef arraydescr_methods[] = {
+        /* for pickling */
+        {"__reduce__", (PyCFunction)arraydescr_reduce, METH_VARARGS, 
+	 doc_arraydescr_reduce},
+	{"__setstate__", (PyCFunction)arraydescr_setstate, METH_VARARGS,
+	 doc_arraydescr_setstate},
+        {NULL,		NULL}		/* sentinel */
+};
 
 static PyTypeObject PyArrayDescr_Type = {
         PyObject_HEAD_INIT(NULL)
         0,					 /* ob_size */
-        "scipy.datadescr",	 	         /* tp_name */
+        "scipy.dtypedescr",	 	         /* tp_name */
         sizeof(PyArray_Descr),                   /* tp_basicsize */
         0,					 /* tp_itemsize */
         /* methods */
@@ -8067,7 +8203,7 @@ static PyTypeObject PyArrayDescr_Type = {
         0,					/* tp_weaklistoffset */
         0,         	                        /* tp_iter */
         0,	                           	/* tp_iternext */
-        0,				        /* tp_methods */
+        arraydescr_methods,	       	        /* tp_methods */
         arraydescr_members,	                /* tp_members */
         arraydescr_getsets,                     /* tp_getset */
         0,					  /* tp_base */
diff --git a/scipy/base/src/arraytypes.inc.src b/scipy/base/src/arraytypes.inc.src
index 87a93abe9..39ba44e66 100644
--- a/scipy/base/src/arraytypes.inc.src
+++ b/scipy/base/src/arraytypes.inc.src
@@ -1452,7 +1452,31 @@ OBJECT_dot(char *ip1, intp is1, char *ip2, intp is2, char *op, intp n,
 }
 
 #define BOOL_fill NULL
-#define OBJECT_fill NULL
+
+/* this requires buffer to be filled with objects or NULL */
+static void
+OBJECT_fill(PyObject **buffer, intp length, void *ignored)
+{
+	intp i;
+	PyObject *start = buffer[0];
+	PyObject *delta = buffer[1];
+	delta = PyNumber_Subtract(delta, start);
+	if (!delta) return;
+	start = PyNumber_Add(start, delta);
+	if (!start) goto finish;
+	buffer += 2;
+	
+	for (i=2; i<length; i++, buffer++) {
+		start = PyNumber_Add(start, delta);
+		if (!start) goto finish;
+		Py_XDECREF(*buffer);
+		*buffer = start;
+	}
+
+ finish:
+	Py_DECREF(delta);
+	return;	
+}
 
 /**begin repeat 
 #NAME=BYTE,UBYTE,SHORT,USHORT,INT,UINT,LONG,ULONG,LONGLONG,ULONGLONG,FLOAT,DOUBLE,LONGDOUBLE#
diff --git a/scipy/base/src/multiarraymodule.c b/scipy/base/src/multiarraymodule.c
index 7c98f44e2..290226d56 100644
--- a/scipy/base/src/multiarraymodule.c
+++ b/scipy/base/src/multiarraymodule.c
@@ -2677,7 +2677,13 @@ PyArray_TypestrConvert(int itemsize, int gentype)
 {
 	register int newtype = gentype;
 	
-	if (gentype == PyArray_SIGNEDLTR) {
+	if (gentype == PyArray_GENBOOLLTR) {
+		if (itemsize == 1)
+			newtype = PyArray_BOOL;
+		else 
+			newtype = PyArray_NOTYPE;
+	}
+	else if (gentype == PyArray_SIGNEDLTR) {
 		switch(itemsize) {
 		case 1:
 			newtype = PyArray_INT8;
@@ -3310,24 +3316,23 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at)
 			   more than one byte and itemsize must be
 			   the number of bytes.
 			*/
-			if (check_num == PyArray_UNICODELTR)
+			if (elsize == 0) {
+				check_num = PyArray_NOTYPE+10;
+			}
+			else if (check_num == PyArray_UNICODELTR) {
 				elsize *= sizeof(Py_UNICODE);
-			
+			}
 			/* Support for generic processing 
 			   c4, i4, f8, etc...
 			*/
 			else if ((check_num != PyArray_STRINGLTR) &&
 				 (check_num != PyArray_VOIDLTR)) {
-				if (elsize == 0) {
-					/* reset because string conversion failed */
-					check_num = PyArray_NOTYPE+10;
-				}
-				else {
-					check_num =			\
-						PyArray_TypestrConvert(elsize,
-								       check_num);
-					elsize = 0;
-				}
+				check_num =				\
+					PyArray_TypestrConvert(elsize,
+							       check_num);
+				if (check_num == PyArray_NOTYPE)
+					check_num += 10;
+				elsize = 0;
 			}
 		}
 	}
@@ -3551,34 +3556,29 @@ array_empty(PyObject *ignored, PyObject *args, PyObject *kwds)
 	return ret;
 }
 
-static char doc_scalar[] = "scalar(dtypestr,obj) will return a new scalar array of the given type initialized with obj. Mainly for pickle support. typestr must be a valid data typestr (complete with < > or |).  If dtypestr is object, then obj can be any object, otherwise obj must be a string. If obj is not given it will be interpreted as None for object type and zeros for all other types.";
+static char doc_scalar[] = "scalar(dtypedescr,obj) will return a new scalar array of the given type initialized with obj. Mainly for pickle support.  The dtypedescr must be a valid data-type descriptor.  If dtypedescr corresponds to an OBJECT descriptor, then obj can be any object, otherwise obj must be a string. If obj is not given it will be interpreted as None for object type and zeros for all other types.";
 
 static PyObject *
 array_scalar(PyObject *ignored, PyObject *args, PyObject *kwds) 
 {
         
-	static char *kwlist[] = {"dtypestr","obj", NULL};
+	static char *kwlist[] = {"dtypedescr","obj", NULL};
 	PyArray_Descr *typecode;
 	PyObject *obj=NULL;
-	char *typestr;
-	int typestrlen;
 	int swap, alloc=0;
 	void *dptr;
 	PyObject *ret;
 
 	
-	if (!PyArg_ParseTupleAndKeywords(args, kwds, "z#|O",
-					 kwlist, &typestr, &typestrlen,
+	if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|O",
+					 kwlist, &PyArrayDescr_Type, 
+					 &typecode,
 					 &obj)) 
 		return NULL;
-	
-	if (!(typecode=_array_typedescr_fromstr(typestr, &swap)))
-		return NULL;
-	
+		
 	if (typecode->elsize == 0) {
 		PyErr_SetString(PyExc_ValueError,		\
 				"itemsize cannot be zero");
-		Py_DECREF(typecode);
 		return NULL;
 	}
 
@@ -3591,7 +3591,6 @@ array_scalar(PyObject *ignored, PyObject *args, PyObject *kwds)
 		if (obj == NULL) {
 			dptr = malloc(typecode->elsize);
 			if (dptr == NULL) {
-				Py_DECREF(typecode);		
 				return PyErr_NoMemory();
 			}
 			memset(dptr, '\0', typecode->elsize);
@@ -3602,22 +3601,20 @@ array_scalar(PyObject *ignored, PyObject *args, PyObject *kwds)
 				PyErr_SetString(PyExc_TypeError, 
 						"initializing object must "\
 						"be a string");
-				Py_DECREF(typecode);				
 				return NULL;
 			}
 			if (PyString_GET_SIZE(obj) < typecode->elsize) {
 				PyErr_SetString(PyExc_ValueError,
 						"initialization string is too"\
 						" small");
-				Py_DECREF(typecode);				
 				return NULL;
 			}
 			dptr = PyString_AS_STRING(obj);
 		}
+		swap = (!PyArray_IsNativeByteOrder(typecode->byteorder));
 	}
 
 	ret = PyArray_Scalar(dptr, swap, typecode, NULL);
-	Py_DECREF(typecode);
 	
 	/* free dptr which contains zeros */
 	if (alloc) free(dptr);
@@ -4190,6 +4187,7 @@ PyArray_Arange(double start, double stop, double step, int type_num)
 	PyObject *range;
 	PyArray_ArrFuncs *funcs;
 	PyObject *obj;
+	int ret;
 
 	length = (intp ) ceil((stop - start)/step);
     
@@ -4209,14 +4207,16 @@ PyArray_Arange(double start, double stop, double step, int type_num)
 	/* if length > 2, then call the inner loop, otherwise stop */
 
 	obj = PyFloat_FromDouble(start);
-	funcs->setitem(obj, PyArray_DATA(range), (PyArrayObject *)range);
+	ret = funcs->setitem(obj, PyArray_DATA(range), (PyArrayObject *)range);
 	Py_DECREF(obj);
+	if (ret < 0) goto fail;
 	if (length == 1) return range;
 
 	obj = PyFloat_FromDouble(start + step);
-	funcs->setitem(obj, PyArray_DATA(range)+PyArray_ITEMSIZE(range), 
-		       (PyArrayObject *)range);
+	ret = funcs->setitem(obj, PyArray_DATA(range)+PyArray_ITEMSIZE(range), 
+			     (PyArrayObject *)range);
 	Py_DECREF(obj);
+	if (ret < 0) goto fail;
 	if (length == 2) return range;
 
 	if (!funcs->fill) {
@@ -4225,8 +4225,13 @@ PyArray_Arange(double start, double stop, double step, int type_num)
 		return NULL;
 	}
 	funcs->fill(PyArray_DATA(range), length, (PyArrayObject *)range);
+	if (PyErr_Occurred()) goto fail;
 	
 	return range;
+
+ fail:
+	Py_DECREF(range);
+	return NULL;
 }
 
 /* the formula is 
@@ -4343,6 +4348,7 @@ PyArray_ArangeObj(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr
 		goto fail;
 	}
 	funcs->fill(PyArray_DATA(range), length, (PyArrayObject *)range);
+	if (PyErr_Occurred()) goto fail;
 
  finish:
 	Py_DECREF(start);
diff --git a/scipy/base/src/scalartypes.inc.src b/scipy/base/src/scalartypes.inc.src
index cac065ea6..5dfa914b6 100644
--- a/scipy/base/src/scalartypes.inc.src
+++ b/scipy/base/src/scalartypes.inc.src
@@ -1151,7 +1151,7 @@ gentype_reduce(PyObject *self, PyObject *args)
 	Py_DECREF(mod);
 	if (obj == NULL) return NULL;
 	PyTuple_SET_ITEM(ret, 0, obj);
-	obj = PyObject_GetAttrString((PyObject *)self, "dtypestr");
+	obj = PyObject_GetAttrString((PyObject *)self, "dtypedescr");
 	if PyArray_IsScalar(self, Object) {
 		mod = ((PyObjectScalarObject *)self)->obval;
 		PyTuple_SET_ITEM(ret, 1,
@@ -1847,9 +1847,11 @@ PyArray_DescrFromTypeObject(PyObject *type)
 	PyArray_Descr *new, *conv=NULL;
 
 	/* if it's a builtin type, then use the typenumber */
-	typenum = _typenum_fromtypeobj(type,0);	
+	typenum = _typenum_fromtypeobj(type,1);	
 	if (typenum != PyArray_NOTYPE) {
-		return PyArray_DescrFromType(typenum);
+		new = PyArray_DescrFromType(typenum);
+		if (PyTypeNum_ISUSERDEF(typenum)) goto finish;
+		return new;
 	}
 
 	/* Check the generic types */
@@ -1879,6 +1881,8 @@ PyArray_DescrFromTypeObject(PyObject *type)
 	*/
 	/* look for a dtypedescr attribute */
 	new = PyArray_DescrNewFromType(PyArray_VOID);
+
+ finish:
 	conv = _arraydescr_fromobj(type);
 	if (conv) {
 		new->fields = conv->fields;