testutils

* improved check on object/record arrays

core 
* fixed filled for flexible types
* fixed the definition of the mask for flexible types

mrecords:
* fixed a bug w/ titles/formats in __new__ and __array_finalize__

5 files changed, 170 insertions, 64 deletions

diff --git a/numpy/ma/core.py b/numpy/ma/core.py
index 21c2f44d4..118f8f6df 100644
--- a/numpy/ma/core.py
+++ b/numpy/ma/core.py
@@ -38,7 +38,7 @@ __all__ = ['MAError', 'MaskType', 'MaskedArray',
            'isMA', 'isMaskedArray', 'is_mask', 'is_masked', 'isarray',
            'left_shift', 'less', 'less_equal', 'load', 'loads', 'log', 'log10',
            'logical_and', 'logical_not', 'logical_or', 'logical_xor',
-           'make_mask', 'make_mask_none', 'mask_or', 'masked',
+           'make_mask', 'make_mask_descr', 'make_mask_none', 'mask_or', 'masked',
            'masked_array', 'masked_equal', 'masked_greater',
            'masked_greater_equal', 'masked_inside', 'masked_invalid',
            'masked_less','masked_less_equal', 'masked_not_equal',
@@ -125,7 +125,10 @@ def default_fill_value(obj):
     if hasattr(obj,'dtype'):
         defval = default_filler[obj.dtype.kind]
     elif isinstance(obj, np.dtype):
-        defval = default_filler[obj.kind]
+        if obj.subdtype:
+            defval = default_filler[obj.subdtype[0].kind]
+        else:
+            defval = default_filler[obj.kind]
     elif isinstance(obj, float):
         defval = default_filler['f']
     elif isinstance(obj, int) or isinstance(obj, long):
@@ -185,26 +188,28 @@ def maximum_fill_value(obj):
 
 def _check_fill_value(fill_value, ndtype):
     ndtype = np.dtype(ndtype)
-    nbfields = len(ndtype)
+    fields = ndtype.fields
     if fill_value is None:
-        if nbfields >= 1:
-            fill_value = np.array(tuple([default_fill_value(np.dtype(d))
-                                         for (_, d) in ndtype.descr]),
-                                  dtype=ndtype)
+        if fields:
+            fdtype = [(_[0], _[1]) for _ in ndtype.descr]
+            fill_value = np.array(tuple([default_fill_value(fields[n][0])
+                                         for n in ndtype.names]),
+                                  dtype=fdtype)
         else:
             fill_value = default_fill_value(ndtype)
-    elif nbfields >= 1:
+    elif fields:
+        fdtype = [(_[0], _[1]) for _ in ndtype.descr]
         if isinstance(fill_value, ndarray):
             try:
-                fill_value = np.array(fill_value, copy=False, dtype=ndtype)
+                fill_value = np.array(fill_value, copy=False, dtype=fdtype)
             except ValueError:
                 err_msg = "Unable to transform %s to dtype %s"
-                raise ValueError(err_msg % (fill_value,ndtype))
+                raise ValueError(err_msg % (fill_value,fdtype))
         else:
-            fval = np.resize(fill_value, nbfields)
-            fill_value = tuple([np.asarray(f).astype(d).item()
-                                for (f, (_, d)) in zip(fval, ndtype.descr)])
-            fill_value = np.array(fill_value, copy=False, dtype=ndtype)
+            fval = np.resize(fill_value, len(ndtype.descr))
+            fill_value = [np.asarray(f).astype(desc[1]).item()
+                          for (f, desc) in zip(fval, ndtype.descr)]
+            fill_value = np.array(tuple(fill_value), copy=False, dtype=fdtype)
     else:
         if isinstance(fill_value, basestring) and (ndtype.char not in 'SV'):
             fill_value = default_fill_value(ndtype)
@@ -726,6 +731,20 @@ fmod = _DomainedBinaryOperation(umath.fmod, _DomainSafeDivide(), 0, 1)
 #####--------------------------------------------------------------------------
 #---- --- Mask creation functions ---
 #####--------------------------------------------------------------------------
+
+def make_mask_descr(ndtype):
+    """Constructs a dtype description list from a given dtype.
+    Each field is set to a bool.
+    
+    """
+    if ndtype.names:
+        mdescr = [list(_) for _ in ndtype.descr]
+        for m in mdescr:
+            m[1] = '|b1'
+        return [tuple(_) for _ in mdescr]
+    else:
+        return MaskType
+
 def get_mask(a):
     """Return the mask of a, if any, or nomask.
 
@@ -796,21 +815,21 @@ def make_mask(m, copy=False, shrink=True, flag=None):
     else:
         return result
 
-def make_mask_none(newshape, fields=None):
+def make_mask_none(newshape, dtype=None):
     """Return a mask of shape s, filled with False.
 
     Parameters
     ----------
     news : tuple
         A tuple indicating the shape of the final mask.
-    fields: {None, string sequence}, optional
-        A list of field names, if needed.
+    dtype: {None, dtype}, optional
+        A dtype.
 
     """
-    if not fields:
+    if dtype is None:
         result = np.zeros(newshape, dtype=MaskType)
     else:
-        result = np.zeros(newshape, dtype=[(n, MaskType) for n in fields])
+        result = np.zeros(newshape, dtype=make_mask_descr(dtype))
     return result
 
 def mask_or (m1, m2, copy=False, shrink=True):
@@ -1219,7 +1238,7 @@ class MaskedArray(ndarray):
         names_ = _data.dtype.names or ()
         # Type of the mask
         if names_:
-            mdtype = [(n, MaskType) for n in names_]
+            mdtype = make_mask_descr(_data.dtype)
         else:
             mdtype = MaskType
         # Case 1. : no mask in input ............
@@ -1441,17 +1460,18 @@ class MaskedArray(ndarray):
             return
         #........................................
 #        ndgetattr = ndarray.__getattribute__
-        _names = ndarray.__getattribute__(self,'dtype').names or ()
         _data = self._data
+        _dtype = ndarray.__getattribute__(_data,'dtype')
         _mask = ndarray.__getattribute__(self,'_mask')
+        nbfields = len(_dtype.names or ())
         #........................................
         if value is masked:
             # The mask wasn't set: create a full version...
             if _mask is nomask:
-                _mask = self._mask = make_mask_none(self.shape, _names)
+                _mask = self._mask = make_mask_none(self.shape, _dtype)
             # Now, set the mask to its value.
-            if _names:
-                _mask[indx] = tuple([True,] * len(_names))
+            if nbfields:
+                _mask[indx] = tuple([True,] * nbfields)
             else:
                 _mask[indx] = True
             if not self._isfield:
@@ -1462,13 +1482,13 @@ class MaskedArray(ndarray):
         dval = value
         # Get the _mask part of the new value
         mval = getattr(value, '_mask', nomask)
-        if _names and mval is nomask:
-            mval = tuple([False] * len(_names))
+        if nbfields and mval is nomask:
+            mval = tuple([False] * nbfields)
         if _mask is nomask:
             # Set the data, then the mask
             ndarray.__setitem__(_data, indx, dval)
             if mval is not nomask:
-                _mask = self._mask = make_mask_none(self.shape, _names)
+                _mask = self._mask = make_mask_none(self.shape, _dtype)
                 ndarray.__setitem__(_mask, indx, mval)
         elif not self._hardmask:
             # Unshare the mask if necessary to avoid propagation
@@ -1482,7 +1502,7 @@ class MaskedArray(ndarray):
             indx = indx * umath.logical_not(_mask)
             ndarray.__setitem__(_data,indx,dval)
         else:
-            if _names:
+            if nbfields:
                 err_msg = "Flexible 'hard' masks are not yet supported..."
                 raise NotImplementedError(err_msg)
             mindx = mask_or(_mask[indx], mval, copy=True)
@@ -1517,7 +1537,7 @@ class MaskedArray(ndarray):
         """Set the mask.
 
         """
-        names = ndarray.__getattribute__(self,'dtype').names
+        idtype = ndarray.__getattribute__(self,'dtype')
         current_mask = ndarray.__getattribute__(self,'_mask')
         if mask is masked:
             mask = True
@@ -1526,9 +1546,9 @@ class MaskedArray(ndarray):
             # Just don't do anything is there's nothing to do...
             if mask is nomask:
                 return
-            current_mask = self._mask = make_mask_none(self.shape, names)
+            current_mask = self._mask = make_mask_none(self.shape, idtype)
         # No named fields.........
-        if names is None:
+        if idtype.names is None:
             # Hardmask: don't unmask the data
             if self._hardmask:
                 current_mask |= mask
@@ -1559,7 +1579,7 @@ class MaskedArray(ndarray):
                                     dtype=mdtype)
             # Hardmask: don't unmask the data
             if self._hardmask:
-                for n in names:
+                for n in idtype.names:
                     current_mask[n] |= mask[n]
             # Softmask: set everything to False
             else:
@@ -1584,13 +1604,20 @@ class MaskedArray(ndarray):
     A record is masked when all the fields are masked.
 
         """
-        if self.dtype.names is None:
-            return self._mask
-        elif self.size > 1:
-            return self._mask.view((bool_, len(self.dtype))).all(1)
+        _mask = ndarray.__getattribute__(self, '_mask').view(ndarray)
+        if _mask.dtype.names is None:
+            return _mask
+        if _mask.size > 1:
+            axis = 1
         else:
-            return self._mask.view((bool_, len(self.dtype))).all()
-    
+            axis=None
+        #
+        try:
+            return _mask.view((bool_, len(self.dtype))).all(axis)
+        except ValueError:
+            return np.all([[f[n].all() for n in _mask.dtype.names]
+                           for f in _mask], axis=axis)
+
     def _setrecordmask(self):
         """Return the mask of the records.
     A record is masked when all the fields are masked.
@@ -1707,15 +1734,17 @@ class MaskedArray(ndarray):
         #
         if fill_value is None:
             fill_value = self.fill_value
+        else:
+            fill_value = _check_fill_value(fill_value, self.dtype)
         #
         if self is masked_singleton:
             return np.asanyarray(fill_value)
         #
-        if len(self.dtype):
+        if m.dtype.names:
             result = self._data.copy()
             for n in result.dtype.names:
                 field = result[n]
-                np.putmask(field, self._mask[n], self.fill_value[n])
+                np.putmask(field, self._mask[n], fill_value[n])
         elif not m.any():
             return self._data
         else:
@@ -3515,7 +3544,7 @@ def putmask(a, mask, values): #, mode='raise'):
     if getmask(a) is nomask:
         if valmask is not nomask:
             a._sharedmask = True
-            a._mask = make_mask_none(a.shape, a.dtype.names)
+            a._mask = make_mask_none(a.shape, a.dtype)
             np.putmask(a._mask, mask, valmask)
     elif a._hardmask:
         if valmask is not nomask:
@@ -3899,3 +3928,5 @@ ones = _convert2ma('ones')
 zeros = _convert2ma('zeros')
 
 ###############################################################################
+
+
diff --git a/numpy/ma/mrecords.py b/numpy/ma/mrecords.py
index ba4b0f915..3870fceb9 100644
--- a/numpy/ma/mrecords.py
+++ b/numpy/ma/mrecords.py
@@ -119,10 +119,11 @@ class MaskedRecords(MaskedArray, object):
                 **options):
         #
         self = recarray.__new__(cls, shape, dtype=dtype, buf=buf, offset=offset,
-                                strides=strides, formats=formats,
-                                byteorder=byteorder, aligned=aligned,)
+                                strides=strides, formats=formats, names=names,
+                                titles=titles, byteorder=byteorder,
+                                aligned=aligned,)
         #
-        mdtype = [(k,'|b1') for (k,_) in self.dtype.descr]
+        mdtype = ma.make_mask_descr(self.dtype)
         if mask is nomask or not np.size(mask):
             if not keep_mask:
                 self._mask = tuple([False]*len(mdtype))
@@ -156,12 +157,11 @@ class MaskedRecords(MaskedArray, object):
         # Make sure we have a _fieldmask by default ..
         _fieldmask = getattr(obj, '_fieldmask', None)
         if _fieldmask is None:
-            mdescr = [(n,'|b1') for (n,_) in self.dtype.descr]
             objmask = getattr(obj, '_mask', nomask)
             if objmask is nomask:
-                _mask = np.empty(self.shape, dtype=mdescr).view(recarray)
-                _mask.flat = tuple([False]*len(mdescr))
+                _mask = ma.make_mask_none(self.shape, dtype=self.dtype)
             else:
+                mdescr = ma.make_mask_descr(self.dtype)
                 _mask = narray([tuple([m]*len(mdescr)) for m in objmask],
                                dtype=mdescr).view(recarray)
         else:
diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py
index 7c79ef1cd..98dbaab20 100644
--- a/numpy/ma/tests/test_core.py
+++ b/numpy/ma/tests/test_core.py
@@ -141,7 +141,7 @@ class TestMaskedArray(TestCase):
 
     def test_creation_maskcreation(self):
         "Tests how masks are initialized at the creation of Maskedarrays."
-        data = arange(24, dtype=float_)
+        data = arange(24, dtype=float)
         data[[3,6,15]] = masked
         dma_1 = MaskedArray(data)
         assert_equal(dma_1.mask, data.mask)
@@ -422,7 +422,19 @@ class TestMaskedArray(TestCase):
         idx = atest.mask
         atest[idx] = btest[idx]
         assert_equal(atest,[20])
-    #........................
+
+
+    def test_filled_w_flexible_dtype(self):
+        "Test filled w/ flexible dtype"
+        flexi = array([(1,1,1)], dtype=[('i',int), ('s','|S3'), ('f',float)])
+        flexi[0] = masked
+        assert_equal(flexi.filled(),
+                     np.array([(default_fill_value(0),
+                                default_fill_value('0'),
+                                default_fill_value(0.),)], dtype=flexi.dtype))
+        flexi[0] = masked
+        assert_equal(flexi.filled(1),
+                     np.array([(1, '1', 1.)], dtype=flexi.dtype))
 
 #------------------------------------------------------------------------------
 
@@ -845,12 +857,12 @@ class TestFillingValues(TestCase):
         assert_equal(fval.item(), [default_fill_value(0),
                                    default_fill_value(0.),
                                    default_fill_value("0")])
-        #.....Using a flexi-ndarray as fill_value should work
+        #.....Using a flexible type as fill_value should work
         fill_val = np.array((-999,-999.9,"???"),dtype=ndtype)
         fval = _check_fill_value(fill_val, ndtype)
         assert(isinstance(fval,ndarray))
         assert_equal(fval.item(), [-999,-999.9,"???"])
-        #.....Using a flexi-ndarray w/ a different type shouldn't matter
+        #.....Using a flexible type w/ a different type shouldn't matter
         fill_val = np.array((-999,-999.9,"???"),
                             dtype=[("A",int),("B",float),("C","|S3")])
         fval = _check_fill_value(fill_val, ndtype)
@@ -866,7 +878,7 @@ class TestFillingValues(TestCase):
         fval = _check_fill_value(fill_val, ndtype)
         assert(isinstance(fval,ndarray))
         assert_equal(fval.item(), [-999,-999.9,"???"])
-        #.....One-field-only flexi-ndarray should work as well
+        #.....One-field-only flexible type should work as well
         ndtype = [("a",int)]
         fval = _check_fill_value(-999, ndtype)
         assert(isinstance(fval,ndarray))
@@ -904,7 +916,7 @@ class TestFillingValues(TestCase):
         series = data[[0,2,1]]
         assert_equal(series._fill_value, data._fill_value)
         #
-        mtype = [('f',float_),('s','|S3')]
+        mtype = [('f',float),('s','|S3')]
         x = array([(1,'a'),(2,'b'),(pi,'pi')], dtype=mtype)
         x.fill_value=999
         assert_equal(x.fill_value.item(),[999.,'999'])
@@ -918,10 +930,37 @@ class TestFillingValues(TestCase):
         #
         x = array([1,2,3.1])
         x.fill_value = 999
-        assert_equal(np.asarray(x.fill_value).dtype, float_)
+        assert_equal(np.asarray(x.fill_value).dtype, float)
         assert_equal(x.fill_value, 999.)
 
 
+    def test_fillvalue_exotic_dtype(self):
+        "Tests yet more exotic flexible dtypes"
+        _check_fill_value = np.ma.core._check_fill_value
+        ndtype = [('i',int), ('s','|S3'), ('f',float)]
+        control = np.array((default_fill_value(0),
+                            default_fill_value('0'),
+                            default_fill_value(0.),),
+                            dtype=ndtype)
+        assert_equal(_check_fill_value(None, ndtype), control)
+        # The shape shouldn't matter
+        ndtype = [('f0', float, (2, 2))]
+        control = np.array((default_fill_value(0.),),
+                           dtype=[('f0',float)])
+        assert_equal(_check_fill_value(None, ndtype), control)
+        control = np.array((0,), dtype=[('f0',float)])
+        assert_equal(_check_fill_value(0, ndtype), control)
+        # 
+        ndtype = np.dtype("int, (2,3)float, float")
+        control = np.array((default_fill_value(0),
+                            default_fill_value(0.),
+                            default_fill_value(0.),),
+                           dtype="int, float, float")
+        test = _check_fill_value(None, ndtype)
+        assert_equal(test, control)
+        control = np.array((0,0,0), dtype="int, float, float")
+        assert_equal(_check_fill_value(0, ndtype), control)
+
 #------------------------------------------------------------------------------
 
 class TestUfuncs(TestCase):
@@ -1038,7 +1077,7 @@ class TestMaskedArrayInPlaceArithmetics(TestCase):
         """Test of inplace subtractions"""
         (x, y, xm) = self.floatdata
         m = xm.mask
-        a = arange(10, dtype=float_)
+        a = arange(10, dtype=float)
         a[-1] = masked
         x -= a
         xm -= a
@@ -1058,7 +1097,7 @@ class TestMaskedArrayInPlaceArithmetics(TestCase):
         """Test of inplace multiplication"""
         (x, y, xm) = self.floatdata
         m = xm.mask
-        a = arange(10, dtype=float_)
+        a = arange(10, dtype=float)
         a[-1] = masked
         x *= a
         xm *= a
@@ -1089,7 +1128,7 @@ class TestMaskedArrayInPlaceArithmetics(TestCase):
         """Test of inplace division"""
         (x, y, xm) = self.floatdata
         m = xm.mask
-        a = arange(10, dtype=float_)
+        a = arange(10, dtype=float)
         a[-1] = masked
         x /= a
         xm /= a
@@ -1341,7 +1380,7 @@ class TestMaskedArrayMethods(TestCase):
 
     def test_empty(self):
         "Tests empty/like"
-        datatype = [('a',int_),('b',float_),('c','|S8')]
+        datatype = [('a',int_),('b',float),('c','|S8')]
         a = masked_array([(1,1.1,'1.1'),(2,2.2,'2.2'),(3,3.3,'3.3')],
                          dtype=datatype)
         assert_equal(len(a.fill_value.item()), len(datatype))
@@ -1606,7 +1645,7 @@ class TestMaskedArrayMethods(TestCase):
         x = array(zip([1,2,3],
                       [1.1,2.2,3.3],
                       ['one','two','thr']),
-                  dtype=[('a',int_),('b',float_),('c','|S8')])
+                  dtype=[('a',int_),('b',float),('c','|S8')])
         x[-1] = masked
         assert_equal(x.tolist(), [(1,1.1,'one'),(2,2.2,'two'),(None,None,None)])
 
@@ -1690,8 +1729,8 @@ class TestMaskArrayMathMethod(TestCase):
         (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d
         (n,m) = X.shape
         assert_equal(mx.ptp(),mx.compressed().ptp())
-        rows = np.zeros(n,np.float_)
-        cols = np.zeros(m,np.float_)
+        rows = np.zeros(n,np.float)
+        cols = np.zeros(m,np.float)
         for k in range(m):
             cols[k] = mX[:,k].compressed().ptp()
         for k in range(n):
@@ -1857,7 +1896,7 @@ class TestMaskedArrayFunctions(TestCase):
 
     def test_masked_where_oddities(self):
         """Tests some generic features."""
-        atest = ones((10,10,10), dtype=float_)
+        atest = ones((10,10,10), dtype=float)
         btest = zeros(atest.shape, MaskType)
         ctest = masked_where(btest,atest)
         assert_equal(atest,ctest)
diff --git a/numpy/ma/tests/test_mrecords.py b/numpy/ma/tests/test_mrecords.py
index 7b5092b9c..882f448ff 100644
--- a/numpy/ma/tests/test_mrecords.py
+++ b/numpy/ma/tests/test_mrecords.py
@@ -276,7 +276,7 @@ class TestMRecords(TestCase):
         mbase._mask = nomask
         # So, the mask of a field is no longer set to nomask...
         assert_equal_records(mbase._mask, 
-                             ma.make_mask_none(base.shape,base.dtype.names))
+                             ma.make_mask_none(base.shape,base.dtype))
         assert(ma.make_mask(mbase['b']._mask) is nomask)
         assert_equal(mbase['a']._mask,mbase['b']._mask)
     #
@@ -317,6 +317,33 @@ class TestMRecords(TestCase):
         assert_equal(mrec.tolist(),
                      [(1,1.1,None),(2,2.2,'two'),(None,None,'three')])
 
+
+    #
+    def test_withnames(self):
+        "Test the creation w/ format and names"
+        x = mrecarray(1, formats=float, names='base')
+        x[0]['base'] = 10
+        assert_equal(x['base'][0], 10)
+    #
+    def test_exotic_formats(self):
+        "Test that 'exotic' formats are processed properly"
+        easy = mrecarray(1, dtype=[('i',int), ('s','|S3'), ('f',float)])
+        easy[0] = masked
+        assert_equal(easy.filled(1).item(), (1,'1',1.))
+        #
+        solo = mrecarray(1, dtype=[('f0', '<f8', (2, 2))])
+        solo[0] = masked
+        assert_equal(solo.filled(1).item(), 
+                     np.array((1,), dtype=solo.dtype).item())
+        #
+        mult = mrecarray(2, dtype= "i4, (2,3)float, float")
+        mult[0] = masked
+        mult[1] = (1, 1, 1)
+        mult.filled(0)
+        assert_equal(mult.filled(0),
+                     np.array([(0,0,0),(1,1,1)], dtype=mult.dtype))
+
+
 ################################################################################
 class TestMRecordsImport(TestCase):
     "Base test class for MaskedArrays."
diff --git a/numpy/ma/testutils.py b/numpy/ma/testutils.py
index a2fde4405..33030489c 100644
--- a/numpy/ma/testutils.py
+++ b/numpy/ma/testutils.py
@@ -104,7 +104,16 @@ def assert_equal(actual,desired,err_msg=''):
         raise ValueError(msg)
     actual = np.array(actual, copy=False, subok=True)
     desired = np.array(desired, copy=False, subok=True)
-    if actual.dtype.char in "OSV" and desired.dtype.char in "OSV":
+    (actual_dtype, desired_dtype) = (actual.dtype, desired.dtype)
+    if actual_dtype.char == "S" and desired_dtype.char == "S":
+        return _assert_equal_on_sequences(actual.tolist(),
+                                          desired.tolist(),
+                                          err_msg='')
+    elif actual_dtype.char in "OV" and desired_dtype.char in "OV":
+        if (actual_dtype != desired_dtype) and actual_dtype:
+            msg = build_err_msg([actual_dtype, desired_dtype],
+                                err_msg, header='', names=('actual', 'desired'))
+            raise ValueError(msg)
         return _assert_equal_on_sequences(actual.tolist(),
                                           desired.tolist(),
                                           err_msg='')