diff options
| -rw-r--r-- | THANKS.txt | 1 | ||||
| -rw-r--r-- | numpy/__init__.py | 18 | ||||
| -rw-r--r-- | numpy/core/__init__.py | 3 | ||||
| -rw-r--r-- | numpy/core/ma.py | 2255 | ||||
| -rw-r--r-- | numpy/core/tests/test_regression.py | 4 | ||||
| -rw-r--r-- | numpy/ma/API_CHANGES.txt | 68 | ||||
| -rw-r--r-- | numpy/ma/LICENSE | 24 | ||||
| -rw-r--r-- | numpy/ma/__init__.py | 22 | ||||
| -rw-r--r-- | numpy/ma/bench.py | 165 | ||||
| -rw-r--r-- | numpy/ma/core.py | 3307 | ||||
| -rw-r--r-- | numpy/ma/extras.py | 802 | ||||
| -rw-r--r-- | numpy/ma/morestats.py | 406 | ||||
| -rw-r--r-- | numpy/ma/mrecords.py | 791 | ||||
| -rw-r--r-- | numpy/ma/mstats.py | 433 | ||||
| -rw-r--r-- | numpy/ma/setup.py | 18 | ||||
| -rw-r--r-- | numpy/ma/tests/test_core.py | 1449 | ||||
| -rw-r--r-- | numpy/ma/tests/test_extras.py | 331 | ||||
| -rw-r--r-- | numpy/ma/tests/test_morestats.py | 114 | ||||
| -rw-r--r-- | numpy/ma/tests/test_mrecords.py | 348 | ||||
| -rw-r--r-- | numpy/ma/tests/test_mstats.py | 174 | ||||
| -rw-r--r-- | numpy/ma/tests/test_old_ma.py (renamed from numpy/core/tests/test_ma.py) | 42 | ||||
| -rw-r--r-- | numpy/ma/tests/test_subclassing.py | 183 | ||||
| -rw-r--r-- | numpy/ma/testutils.py | 219 | ||||
| -rw-r--r-- | numpy/ma/timer_comparison.py | 457 | ||||
| -rw-r--r-- | numpy/ma/version.py | 11 | ||||
| -rw-r--r-- | numpy/setup.py | 1 |
26 files changed, 9348 insertions, 2298 deletions
diff --git a/THANKS.txt b/THANKS.txt index 339a498ba..b068f2aaf 100644 --- a/THANKS.txt +++ b/THANKS.txt @@ -35,3 +35,4 @@ Albert Strasheim for documentation, bug-fixes, regression tests and Stefan van der Walt for documentation, bug-fixes and regression-tests. Andrew Straw for help with http://www.scipy.org, documentation, and testing. David Cournapeau for documentation, bug-fixes, and code contributions including fast_clipping. +Pierre Gerard-Marchant for his rewrite of the masked array functionality. diff --git a/numpy/__init__.py b/numpy/__init__.py index f07d862a5..3b611f4da 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -38,6 +38,9 @@ else: loader = PackageLoader(infunc=True) return loader(*packages, **options) + import add_newdocs + __all__ = ['add_newdocs',] + pkgload.__doc__ = PackageLoader.__call__.__doc__ import testing from testing import ScipyTest, NumpyTest @@ -49,6 +52,7 @@ else: import fft import random import ctypeslib + import ma # Make these accessible from numpy name-space # but not imported in from numpy import * @@ -56,11 +60,11 @@ else: object, unicode, str from core import round, abs, max, min - __all__ = ['__version__', 'pkgload', 'PackageLoader', - 'ScipyTest', 'NumpyTest', 'show_config'] - __all__ += core.__all__ - __all__ += lib.__all__ - __all__ += ['linalg', 'fft', 'random', 'ctypeslib'] + __all__.extend(['__version__', 'pkgload', 'PackageLoader', + 'ScipyTest', 'NumpyTest', 'show_config']) + __all__.extend(core.__all__) + __all__.extend(lib.__all__) + __all__.extend(['linalg', 'fft', 'random', 'ctypeslib']) if __doc__ is not None: __doc__ += """ @@ -97,10 +101,6 @@ testing --> NumpyTest return NumpyTest().test(*args, **kw) test.__doc__ = NumpyTest.test.__doc__ - import add_newdocs - - __all__.extend(['add_newdocs']) - if __doc__ is not None: __doc__ += """ diff --git a/numpy/core/__init__.py b/numpy/core/__init__.py index 4d22394d5..37ed7213e 100644 --- a/numpy/core/__init__.py +++ b/numpy/core/__init__.py @@ -11,7 +11,6 @@ import _sort from numeric import * from fromnumeric import * from defmatrix import * -import ma import defchararray as char import records as rec from records import * @@ -24,7 +23,7 @@ from fromnumeric import amax as max, amin as min, \ round_ as round from numeric import absolute as abs -__all__ = ['char','rec','memmap','ma'] +__all__ = ['char','rec','memmap'] __all__ += numeric.__all__ __all__ += fromnumeric.__all__ __all__ += defmatrix.__all__ diff --git a/numpy/core/ma.py b/numpy/core/ma.py deleted file mode 100644 index 79941b7f8..000000000 --- a/numpy/core/ma.py +++ /dev/null @@ -1,2255 +0,0 @@ -"""MA: a facility for dealing with missing observations -MA is generally used as a numpy.array look-alike. -by Paul F. Dubois. - -Copyright 1999, 2000, 2001 Regents of the University of California. -Released for unlimited redistribution. -Adapted for numpy_core 2005 by Travis Oliphant and -(mainly) Paul Dubois. -""" -import types, sys - -import umath -import fromnumeric -from numeric import newaxis, ndarray, inf -from fromnumeric import amax, amin -from numerictypes import bool_, typecodes -import numeric -import warnings - -# Ufunc domain lookup for __array_wrap__ -ufunc_domain = {} -# Ufunc fills lookup for __array__ -ufunc_fills = {} - -MaskType = bool_ -nomask = MaskType(0) -divide_tolerance = 1.e-35 - -class MAError (Exception): - def __init__ (self, args=None): - "Create an exception" - - # The .args attribute must be a tuple. - if not isinstance(args, tuple): - args = (args,) - self.args = args - def __str__(self): - "Calculate the string representation" - return str(self.args[0]) - __repr__ = __str__ - -class _MaskedPrintOption: - "One instance of this class, masked_print_option, is created." - def __init__ (self, display): - "Create the masked print option object." - self.set_display(display) - self._enabled = 1 - - def display (self): - "Show what prints for masked values." - return self._display - - def set_display (self, s): - "set_display(s) sets what prints for masked values." - self._display = s - - def enabled (self): - "Is the use of the display value enabled?" - return self._enabled - - def enable(self, flag=1): - "Set the enabling flag to flag." - self._enabled = flag - - def __str__ (self): - return str(self._display) - - __repr__ = __str__ - -#if you single index into a masked location you get this object. -masked_print_option = _MaskedPrintOption('--') - -# Use single element arrays or scalars. -default_real_fill_value = 1.e20 -default_complex_fill_value = 1.e20 + 0.0j -default_character_fill_value = '-' -default_integer_fill_value = 999999 -default_object_fill_value = '?' - -def default_fill_value (obj): - "Function to calculate default fill value for an object." - if isinstance(obj, types.FloatType): - return default_real_fill_value - elif isinstance(obj, types.IntType) or isinstance(obj, types.LongType): - return default_integer_fill_value - elif isinstance(obj, types.StringType): - return default_character_fill_value - elif isinstance(obj, types.ComplexType): - return default_complex_fill_value - elif isinstance(obj, MaskedArray) or isinstance(obj, ndarray): - x = obj.dtype.char - if x in typecodes['Float']: - return default_real_fill_value - if x in typecodes['Integer']: - return default_integer_fill_value - if x in typecodes['Complex']: - return default_complex_fill_value - if x in typecodes['Character']: - return default_character_fill_value - if x in typecodes['UnsignedInteger']: - return umath.absolute(default_integer_fill_value) - return default_object_fill_value - else: - return default_object_fill_value - -def minimum_fill_value (obj): - "Function to calculate default fill value suitable for taking minima." - if isinstance(obj, types.FloatType): - return numeric.inf - elif isinstance(obj, types.IntType) or isinstance(obj, types.LongType): - return sys.maxint - elif isinstance(obj, MaskedArray) or isinstance(obj, ndarray): - x = obj.dtype.char - if x in typecodes['Float']: - return numeric.inf - if x in typecodes['Integer']: - return sys.maxint - if x in typecodes['UnsignedInteger']: - return sys.maxint - else: - raise TypeError, 'Unsuitable type for calculating minimum.' - -def maximum_fill_value (obj): - "Function to calculate default fill value suitable for taking maxima." - if isinstance(obj, types.FloatType): - return -inf - elif isinstance(obj, types.IntType) or isinstance(obj, types.LongType): - return -sys.maxint - elif isinstance(obj, MaskedArray) or isinstance(obj, ndarray): - x = obj.dtype.char - if x in typecodes['Float']: - return -inf - if x in typecodes['Integer']: - return -sys.maxint - if x in typecodes['UnsignedInteger']: - return 0 - else: - raise TypeError, 'Unsuitable type for calculating maximum.' - -def set_fill_value (a, fill_value): - "Set fill value of a if it is a masked array." - if isMaskedArray(a): - a.set_fill_value (fill_value) - -def getmask (a): - """Mask of values in a; could be nomask. - Returns nomask if a is not a masked array. - To get an array for sure use getmaskarray.""" - if isinstance(a, MaskedArray): - return a.raw_mask() - else: - return nomask - -def getmaskarray (a): - """Mask of values in a; an array of zeros if mask is nomask - or not a masked array, and is a byte-sized integer. - Do not try to add up entries, for example. - """ - m = getmask(a) - if m is nomask: - return make_mask_none(shape(a)) - else: - return m - -def is_mask (m): - """Is m a legal mask? Does not check contents, only type. - """ - try: - return m.dtype.type is MaskType - except AttributeError: - return False - -def make_mask (m, copy=0, flag=0): - """make_mask(m, copy=0, flag=0) - return m as a mask, creating a copy if necessary or requested. - Can accept any sequence of integers or nomask. Does not check - that contents must be 0s and 1s. - if flag, return nomask if m contains no true elements. - """ - if m is nomask: - return nomask - elif isinstance(m, ndarray): - if m.dtype.type is MaskType: - if copy: - result = numeric.array(m, dtype=MaskType, copy=copy) - else: - result = m - else: - result = m.astype(MaskType) - else: - result = filled(m, True).astype(MaskType) - - if flag and not fromnumeric.sometrue(fromnumeric.ravel(result)): - return nomask - else: - return result - -def make_mask_none (s): - "Return a mask of all zeros of shape s." - result = numeric.zeros(s, dtype=MaskType) - result.shape = s - return result - -def mask_or (m1, m2): - """Logical or of the mask candidates m1 and m2, treating nomask as false. - Result may equal m1 or m2 if the other is nomask. - """ - if m1 is nomask: return make_mask(m2) - if m2 is nomask: return make_mask(m1) - if m1 is m2 and is_mask(m1): return m1 - return make_mask(umath.logical_or(m1, m2)) - -def filled (a, value = None): - """a as a contiguous numeric array with any masked areas replaced by value - if value is None or the special element "masked", get_fill_value(a) - is used instead. - - If a is already a contiguous numeric array, a itself is returned. - - filled(a) can be used to be sure that the result is numeric when - passing an object a to other software ignorant of MA, in particular to - numeric itself. - """ - if isinstance(a, MaskedArray): - return a.filled(value) - elif isinstance(a, ndarray) and a.flags['CONTIGUOUS']: - return a - elif isinstance(a, types.DictType): - return numeric.array(a, 'O') - else: - return numeric.array(a) - -def get_fill_value (a): - """ - The fill value of a, if it has one; otherwise, the default fill value - for that type. - """ - if isMaskedArray(a): - result = a.fill_value() - else: - result = default_fill_value(a) - return result - -def common_fill_value (a, b): - "The common fill_value of a and b, if there is one, or None" - t1 = get_fill_value(a) - t2 = get_fill_value(b) - if t1 == t2: return t1 - return None - -# Domain functions return 1 where the argument(s) are not in the domain. -class domain_check_interval: - "domain_check_interval(a,b)(x) = true where x < a or y > b" - def __init__(self, y1, y2): - "domain_check_interval(a,b)(x) = true where x < a or y > b" - self.y1 = y1 - self.y2 = y2 - - def __call__ (self, x): - "Execute the call behavior." - return umath.logical_or(umath.greater (x, self.y2), - umath.less(x, self.y1) - ) - -class domain_tan: - "domain_tan(eps) = true where abs(cos(x)) < eps)" - def __init__(self, eps): - "domain_tan(eps) = true where abs(cos(x)) < eps)" - self.eps = eps - - def __call__ (self, x): - "Execute the call behavior." - return umath.less(umath.absolute(umath.cos(x)), self.eps) - -class domain_greater: - "domain_greater(v)(x) = true where x <= v" - def __init__(self, critical_value): - "domain_greater(v)(x) = true where x <= v" - self.critical_value = critical_value - - def __call__ (self, x): - "Execute the call behavior." - return umath.less_equal (x, self.critical_value) - -class domain_greater_equal: - "domain_greater_equal(v)(x) = true where x < v" - def __init__(self, critical_value): - "domain_greater_equal(v)(x) = true where x < v" - self.critical_value = critical_value - - def __call__ (self, x): - "Execute the call behavior." - return umath.less (x, self.critical_value) - -class masked_unary_operation: - def __init__ (self, aufunc, fill=0, domain=None): - """ masked_unary_operation(aufunc, fill=0, domain=None) - aufunc(fill) must be defined - self(x) returns aufunc(x) - with masked values where domain(x) is true or getmask(x) is true. - """ - self.f = aufunc - self.fill = fill - self.domain = domain - self.__doc__ = getattr(aufunc, "__doc__", str(aufunc)) - self.__name__ = getattr(aufunc, "__name__", str(aufunc)) - ufunc_domain[aufunc] = domain - ufunc_fills[aufunc] = fill, - - def __call__ (self, a, *args, **kwargs): - "Execute the call behavior." -# numeric tries to return scalars rather than arrays when given scalars. - m = getmask(a) - d1 = filled(a, self.fill) - if self.domain is not None: - m = mask_or(m, self.domain(d1)) - result = self.f(d1, *args, **kwargs) - return masked_array(result, m) - - def __str__ (self): - return "Masked version of " + str(self.f) - - -class domain_safe_divide: - def __init__ (self, tolerance=divide_tolerance): - self.tolerance = tolerance - def __call__ (self, a, b): - return umath.absolute(a) * self.tolerance >= umath.absolute(b) - -class domained_binary_operation: - """Binary operations that have a domain, like divide. These are complicated - so they are a separate class. They have no reduce, outer or accumulate. - """ - def __init__ (self, abfunc, domain, fillx=0, filly=0): - """abfunc(fillx, filly) must be defined. - abfunc(x, filly) = x for all x to enable reduce. - """ - self.f = abfunc - self.domain = domain - self.fillx = fillx - self.filly = filly - self.__doc__ = getattr(abfunc, "__doc__", str(abfunc)) - self.__name__ = getattr(abfunc, "__name__", str(abfunc)) - ufunc_domain[abfunc] = domain - ufunc_fills[abfunc] = fillx, filly - - def __call__(self, a, b): - "Execute the call behavior." - ma = getmask(a) - mb = getmask(b) - d1 = filled(a, self.fillx) - d2 = filled(b, self.filly) - t = self.domain(d1, d2) - - if fromnumeric.sometrue(t, None): - d2 = where(t, self.filly, d2) - mb = mask_or(mb, t) - m = mask_or(ma, mb) - result = self.f(d1, d2) - return masked_array(result, m) - - def __str__ (self): - return "Masked version of " + str(self.f) - -class masked_binary_operation: - def __init__ (self, abfunc, fillx=0, filly=0): - """abfunc(fillx, filly) must be defined. - abfunc(x, filly) = x for all x to enable reduce. - """ - self.f = abfunc - self.fillx = fillx - self.filly = filly - self.__doc__ = getattr(abfunc, "__doc__", str(abfunc)) - ufunc_domain[abfunc] = None - ufunc_fills[abfunc] = fillx, filly - - def __call__ (self, a, b, *args, **kwargs): - "Execute the call behavior." - m = mask_or(getmask(a), getmask(b)) - d1 = filled(a, self.fillx) - d2 = filled(b, self.filly) - result = self.f(d1, d2, *args, **kwargs) - if isinstance(result, ndarray) \ - and m.ndim != 0 \ - and m.shape != result.shape: - m = mask_or(getmaskarray(a), getmaskarray(b)) - return masked_array(result, m) - - def reduce (self, target, axis=0, dtype=None): - """Reduce target along the given axis with this function.""" - m = getmask(target) - t = filled(target, self.filly) - if t.shape == (): - t = t.reshape(1) - if m is not nomask: - m = make_mask(m, copy=1) - m.shape = (1,) - if m is nomask: - t = self.f.reduce(t, axis) - else: - t = masked_array (t, m) - # XXX: "or t.dtype" below is a workaround for what appears - # XXX: to be a bug in reduce. - t = self.f.reduce(filled(t, self.filly), axis, - dtype=dtype or t.dtype) - m = umath.logical_and.reduce(m, axis) - if isinstance(t, ndarray): - return masked_array(t, m, get_fill_value(target)) - elif m: - return masked - else: - return t - - def outer (self, a, b): - "Return the function applied to the outer product of a and b." - ma = getmask(a) - mb = getmask(b) - if ma is nomask and mb is nomask: - m = nomask - else: - ma = getmaskarray(a) - mb = getmaskarray(b) - m = logical_or.outer(ma, mb) - d = self.f.outer(filled(a, self.fillx), filled(b, self.filly)) - return masked_array(d, m) - - def accumulate (self, target, axis=0): - """Accumulate target along axis after filling with y fill value.""" - t = filled(target, self.filly) - return masked_array (self.f.accumulate (t, axis)) - def __str__ (self): - return "Masked version of " + str(self.f) - -sqrt = masked_unary_operation(umath.sqrt, 0.0, domain_greater_equal(0.0)) -log = masked_unary_operation(umath.log, 1.0, domain_greater(0.0)) -log10 = masked_unary_operation(umath.log10, 1.0, domain_greater(0.0)) -exp = masked_unary_operation(umath.exp) -conjugate = masked_unary_operation(umath.conjugate) -sin = masked_unary_operation(umath.sin) -cos = masked_unary_operation(umath.cos) -tan = masked_unary_operation(umath.tan, 0.0, domain_tan(1.e-35)) -arcsin = masked_unary_operation(umath.arcsin, 0.0, domain_check_interval(-1.0, 1.0)) -arccos = masked_unary_operation(umath.arccos, 0.0, domain_check_interval(-1.0, 1.0)) -arctan = masked_unary_operation(umath.arctan) -# Missing from numeric -arcsinh = masked_unary_operation(umath.arcsinh) -arccosh = masked_unary_operation(umath.arccosh, 1.0, domain_greater_equal(1.0)) -arctanh = masked_unary_operation(umath.arctanh, 0.0, domain_check_interval(-1.0+1e-15, 1.0-1e-15)) -sinh = masked_unary_operation(umath.sinh) -cosh = masked_unary_operation(umath.cosh) -tanh = masked_unary_operation(umath.tanh) -absolute = masked_unary_operation(umath.absolute) -fabs = masked_unary_operation(umath.fabs) -negative = masked_unary_operation(umath.negative) - -def nonzero(a): - """returns the indices of the elements of a which are not zero - and not masked - """ - return numeric.asarray(filled(a, 0).nonzero()) - -around = masked_unary_operation(fromnumeric.round_) -floor = masked_unary_operation(umath.floor) -ceil = masked_unary_operation(umath.ceil) -logical_not = masked_unary_operation(umath.logical_not) - -add = masked_binary_operation(umath.add) -subtract = masked_binary_operation(umath.subtract) -subtract.reduce = None -multiply = masked_binary_operation(umath.multiply, 1, 1) -divide = domained_binary_operation(umath.divide, domain_safe_divide(), 0, 1) -true_divide = domained_binary_operation(umath.true_divide, domain_safe_divide(), 0, 1) -floor_divide = domained_binary_operation(umath.floor_divide, domain_safe_divide(), 0, 1) -remainder = domained_binary_operation(umath.remainder, domain_safe_divide(), 0, 1) -fmod = domained_binary_operation(umath.fmod, domain_safe_divide(), 0, 1) -hypot = masked_binary_operation(umath.hypot) -arctan2 = masked_binary_operation(umath.arctan2, 0.0, 1.0) -arctan2.reduce = None -equal = masked_binary_operation(umath.equal) -equal.reduce = None -not_equal = masked_binary_operation(umath.not_equal) -not_equal.reduce = None -less_equal = masked_binary_operation(umath.less_equal) -less_equal.reduce = None -greater_equal = masked_binary_operation(umath.greater_equal) -greater_equal.reduce = None -less = masked_binary_operation(umath.less) -less.reduce = None -greater = masked_binary_operation(umath.greater) -greater.reduce = None -logical_and = masked_binary_operation(umath.logical_and) -alltrue = masked_binary_operation(umath.logical_and, 1, 1).reduce -logical_or = masked_binary_operation(umath.logical_or) -sometrue = logical_or.reduce -logical_xor = masked_binary_operation(umath.logical_xor) -bitwise_and = masked_binary_operation(umath.bitwise_and) -bitwise_or = masked_binary_operation(umath.bitwise_or) -bitwise_xor = masked_binary_operation(umath.bitwise_xor) - -def rank (object): - return fromnumeric.rank(filled(object)) - -def shape (object): - return fromnumeric.shape(filled(object)) - -def size (object, axis=None): - return fromnumeric.size(filled(object), axis) - -class MaskedArray (object): - """Arrays with possibly masked values. - Masked values of 1 exclude the corresponding element from - any computation. - - Construction: - x = array(data, dtype=None, copy=True, order=False, - mask = nomask, fill_value=None) - - If copy=False, every effort is made not to copy the data: - If data is a MaskedArray, and argument mask=nomask, - then the candidate data is data.data and the - mask used is data.mask. If data is a numeric array, - it is used as the candidate raw data. - If dtype is not None and - is != data.dtype.char then a data copy is required. - Otherwise, the candidate is used. - - If a data copy is required, raw data stored is the result of: - numeric.array(data, dtype=dtype.char, copy=copy) - - If mask is nomask there are no masked values. Otherwise mask must - be convertible to an array of booleans with the same shape as x. - - fill_value is used to fill in masked values when necessary, - such as when printing and in method/function filled(). - The fill_value is not used for computation within this module. - """ - __array_priority__ = 10.1 - def __init__(self, data, dtype=None, copy=True, order=False, - mask=nomask, fill_value=None): - """array(data, dtype=None, copy=True, order=False, mask=nomask, fill_value=None) - If data already a numeric array, its dtype becomes the default value of dtype. - """ - if dtype is None: - tc = None - else: - tc = numeric.dtype(dtype) - need_data_copied = copy - if isinstance(data, MaskedArray): - c = data.data - if tc is None: - tc = c.dtype - elif tc != c.dtype: - need_data_copied = True - if mask is nomask: - mask = data.mask - elif mask is not nomask: #attempting to change the mask - need_data_copied = True - - elif isinstance(data, ndarray): - c = data - if tc is None: - tc = c.dtype - elif tc != c.dtype: - need_data_copied = True - else: - need_data_copied = False #because I'll do it now - c = numeric.array(data, dtype=tc, copy=True, order=order) - tc = c.dtype - - if need_data_copied: - if tc == c.dtype: - self._data = numeric.array(c, dtype=tc, copy=True, order=order) - else: - self._data = c.astype(tc) - else: - self._data = c - - if mask is nomask: - self._mask = nomask - self._shared_mask = 0 - else: - self._mask = make_mask (mask) - if self._mask is nomask: - self._shared_mask = 0 - else: - self._shared_mask = (self._mask is mask) - nm = size(self._mask) - nd = size(self._data) - if nm != nd: - if nm == 1: - self._mask = fromnumeric.resize(self._mask, self._data.shape) - self._shared_mask = 0 - elif nd == 1: - self._data = fromnumeric.resize(self._data, self._mask.shape) - self._data.shape = self._mask.shape - else: - raise MAError, "Mask and data not compatible." - elif nm == 1 and shape(self._mask) != shape(self._data): - self.unshare_mask() - self._mask.shape = self._data.shape - - self.set_fill_value(fill_value) - - def __array__ (self, t=None, context=None): - "Special hook for numeric. Converts to numeric if possible." - if self._mask is not nomask: - if fromnumeric.ravel(self._mask).any(): - if context is None: - warnings.warn("Cannot automatically convert masked array to "\ - "numeric because data\n is masked in one or "\ - "more locations."); - return self._data - #raise MAError, \ - # """Cannot automatically convert masked array to numeric because data - # is masked in one or more locations. - # """ - else: - func, args, i = context - fills = ufunc_fills.get(func) - if fills is None: - raise MAError, "%s not known to ma" % func - return self.filled(fills[i]) - else: # Mask is all false - # Optimize to avoid future invocations of this section. - self._mask = nomask - self._shared_mask = 0 - if t: - return self._data.astype(t) - else: - return self._data - - def __array_wrap__ (self, array, context=None): - """Special hook for ufuncs. - - Wraps the numpy array and sets the mask according to - context. - """ - if context is None: - return MaskedArray(array, copy=False, mask=nomask) - func, args = context[:2] - domain = ufunc_domain[func] - m = reduce(mask_or, [getmask(a) for a in args]) - if domain is not None: - m = mask_or(m, domain(*[getattr(a, '_data', a) - for a in args])) - if m is not nomask: - try: - shape = array.shape - except AttributeError: - pass - else: - if m.shape != shape: - m = reduce(mask_or, [getmaskarray(a) for a in args]) - - return MaskedArray(array, copy=False, mask=m) - - def _get_shape(self): - "Return the current shape." - return self._data.shape - - def _set_shape (self, newshape): - "Set the array's shape." - self._data.shape = newshape - if self._mask is not nomask: - self._mask = self._mask.copy() - self._mask.shape = newshape - - def _get_flat(self): - """Calculate the flat value. - """ - if self._mask is nomask: - return masked_array(self._data.ravel(), mask=nomask, - fill_value = self.fill_value()) - else: - return masked_array(self._data.ravel(), - mask=self._mask.ravel(), - fill_value = self.fill_value()) - - def _set_flat (self, value): - "x.flat = value" - y = self.ravel() - y[:] = value - - def _get_real(self): - "Get the real part of a complex array." - if self._mask is nomask: - return masked_array(self._data.real, mask=nomask, - fill_value = self.fill_value()) - else: - return masked_array(self._data.real, mask=self._mask, - fill_value = self.fill_value()) - - def _set_real (self, value): - "x.real = value" - y = self.real - y[...] = value - - def _get_imaginary(self): - "Get the imaginary part of a complex array." - if self._mask is nomask: - return masked_array(self._data.imag, mask=nomask, - fill_value = self.fill_value()) - else: - return masked_array(self._data.imag, mask=self._mask, - fill_value = self.fill_value()) - - def _set_imaginary (self, value): - "x.imaginary = value" - y = self.imaginary - y[...] = value - - def __str__(self): - """Calculate the str representation, using masked for fill if - it is enabled. Otherwise fill with fill value. - """ - if masked_print_option.enabled(): - f = masked_print_option - # XXX: Without the following special case masked - # XXX: would print as "[--]", not "--". Can we avoid - # XXX: checks for masked by choosing a different value - # XXX: for the masked singleton? 2005-01-05 -- sasha - if self is masked: - return str(f) - m = self._mask - if m is not nomask and m.shape == () and m: - return str(f) - # convert to object array to make filled work - self = self.astype(object) - else: - f = self.fill_value() - res = self.filled(f) - return str(res) - - def __repr__(self): - """Calculate the repr representation, using masked for fill if - it is enabled. Otherwise fill with fill value. - """ - with_mask = """\ -array(data = - %(data)s, - mask = - %(mask)s, - fill_value=%(fill)s) -""" - with_mask1 = """\ -array(data = %(data)s, - mask = %(mask)s, - fill_value=%(fill)s) -""" - without_mask = """array( - %(data)s)""" - without_mask1 = """array(%(data)s)""" - - n = len(self.shape) - if self._mask is nomask: - if n <= 1: - return without_mask1 % {'data':str(self.filled())} - return without_mask % {'data':str(self.filled())} - else: - if n <= 1: - return with_mask % { - 'data': str(self.filled()), - 'mask': str(self._mask), - 'fill': str(self.fill_value()) - } - return with_mask % { - 'data': str(self.filled()), - 'mask': str(self._mask), - 'fill': str(self.fill_value()) - } - without_mask1 = """array(%(data)s)""" - if self._mask is nomask: - return without_mask % {'data':str(self.filled())} - else: - return with_mask % { - 'data': str(self.filled()), - 'mask': str(self._mask), - 'fill': str(self.fill_value()) - } - - def __float__(self): - "Convert self to float." - self.unmask() - if self._mask is not nomask: - raise MAError, 'Cannot convert masked element to a Python float.' - return float(self.data.item()) - - def __int__(self): - "Convert self to int." - self.unmask() - if self._mask is not nomask: - raise MAError, 'Cannot convert masked element to a Python int.' - return int(self.data.item()) - - def __getitem__(self, i): - "Get item described by i. Not a copy as in previous versions." - self.unshare_mask() - m = self._mask - dout = self._data[i] - if m is nomask: - try: - if dout.size == 1: - return dout - else: - return masked_array(dout, fill_value=self._fill_value) - except AttributeError: - return dout - mi = m[i] - if mi.size == 1: - if mi: - return masked - else: - return dout - else: - return masked_array(dout, mi, fill_value=self._fill_value) - -# -------- -# setitem and setslice notes -# note that if value is masked, it means to mask those locations. -# setting a value changes the mask to match the value in those locations. - - def __setitem__(self, index, value): - "Set item described by index. If value is masked, mask those locations." - d = self._data - if self is masked: - raise MAError, 'Cannot alter masked elements.' - if value is masked: - if self._mask is nomask: - self._mask = make_mask_none(d.shape) - self._shared_mask = False - else: - self.unshare_mask() - self._mask[index] = True - return - m = getmask(value) - value = filled(value).astype(d.dtype) - d[index] = value - if m is nomask: - if self._mask is not nomask: - self.unshare_mask() - self._mask[index] = False - else: - if self._mask is nomask: - self._mask = make_mask_none(d.shape) - self._shared_mask = True - else: - self.unshare_mask() - self._mask[index] = m - - def __nonzero__(self): - """returns true if any element is non-zero or masked - - """ - # XXX: This changes bool conversion logic from MA. - # XXX: In MA bool(a) == len(a) != 0, but in numpy - # XXX: scalars do not have len - m = self._mask - d = self._data - return bool(m is not nomask and m.any() - or d is not nomask and d.any()) - - def __len__ (self): - """Return length of first dimension. This is weird but Python's - slicing behavior depends on it.""" - return len(self._data) - - def __and__(self, other): - "Return bitwise_and" - return bitwise_and(self, other) - - def __or__(self, other): - "Return bitwise_or" - return bitwise_or(self, other) - - def __xor__(self, other): - "Return bitwise_xor" - return bitwise_xor(self, other) - - __rand__ = __and__ - __ror__ = __or__ - __rxor__ = __xor__ - - def __abs__(self): - "Return absolute(self)" - return absolute(self) - - def __neg__(self): - "Return negative(self)" - return negative(self) - - def __pos__(self): - "Return array(self)" - return array(self) - - def __add__(self, other): - "Return add(self, other)" - return add(self, other) - - __radd__ = __add__ - - def __mod__ (self, other): - "Return remainder(self, other)" - return remainder(self, other) - - def __rmod__ (self, other): - "Return remainder(other, self)" - return remainder(other, self) - - def __lshift__ (self, n): - return left_shift(self, n) - - def __rshift__ (self, n): - return right_shift(self, n) - - def __sub__(self, other): - "Return subtract(self, other)" - return subtract(self, other) - - def __rsub__(self, other): - "Return subtract(other, self)" - return subtract(other, self) - - def __mul__(self, other): - "Return multiply(self, other)" - return multiply(self, other) - - __rmul__ = __mul__ - - def __div__(self, other): - "Return divide(self, other)" - return divide(self, other) - - def __rdiv__(self, other): - "Return divide(other, self)" - return divide(other, self) - - def __truediv__(self, other): - "Return divide(self, other)" - return true_divide(self, other) - - def __rtruediv__(self, other): - "Return divide(other, self)" - return true_divide(other, self) - - def __floordiv__(self, other): - "Return divide(self, other)" - return floor_divide(self, other) - - def __rfloordiv__(self, other): - "Return divide(other, self)" - return floor_divide(other, self) - - def __pow__(self, other, third=None): - "Return power(self, other, third)" - return power(self, other, third) - - def __sqrt__(self): - "Return sqrt(self)" - return sqrt(self) - - def __iadd__(self, other): - "Add other to self in place." - t = self._data.dtype.char - f = filled(other, 0) - t1 = f.dtype.char - if t == t1: - pass - elif t in typecodes['Integer']: - if t1 in typecodes['Integer']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - elif t in typecodes['Float']: - if t1 in typecodes['Integer']: - f = f.astype(t) - elif t1 in typecodes['Float']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - elif t in typecodes['Complex']: - if t1 in typecodes['Integer']: - f = f.astype(t) - elif t1 in typecodes['Float']: - f = f.astype(t) - elif t1 in typecodes['Complex']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - else: - raise TypeError, 'Incorrect type for in-place operation.' - - if self._mask is nomask: - self._data += f - m = getmask(other) - self._mask = m - self._shared_mask = m is not nomask - else: - result = add(self, masked_array(f, mask=getmask(other))) - self._data = result.data - self._mask = result.mask - self._shared_mask = 1 - return self - - def __imul__(self, other): - "Add other to self in place." - t = self._data.dtype.char - f = filled(other, 0) - t1 = f.dtype.char - if t == t1: - pass - elif t in typecodes['Integer']: - if t1 in typecodes['Integer']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - elif t in typecodes['Float']: - if t1 in typecodes['Integer']: - f = f.astype(t) - elif t1 in typecodes['Float']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - elif t in typecodes['Complex']: - if t1 in typecodes['Integer']: - f = f.astype(t) - elif t1 in typecodes['Float']: - f = f.astype(t) - elif t1 in typecodes['Complex']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - else: - raise TypeError, 'Incorrect type for in-place operation.' - - if self._mask is nomask: - self._data *= f - m = getmask(other) - self._mask = m - self._shared_mask = m is not nomask - else: - result = multiply(self, masked_array(f, mask=getmask(other))) - self._data = result.data - self._mask = result.mask - self._shared_mask = 1 - return self - - def __isub__(self, other): - "Subtract other from self in place." - t = self._data.dtype.char - f = filled(other, 0) - t1 = f.dtype.char - if t == t1: - pass - elif t in typecodes['Integer']: - if t1 in typecodes['Integer']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - elif t in typecodes['Float']: - if t1 in typecodes['Integer']: - f = f.astype(t) - elif t1 in typecodes['Float']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - elif t in typecodes['Complex']: - if t1 in typecodes['Integer']: - f = f.astype(t) - elif t1 in typecodes['Float']: - f = f.astype(t) - elif t1 in typecodes['Complex']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - else: - raise TypeError, 'Incorrect type for in-place operation.' - - if self._mask is nomask: - self._data -= f - m = getmask(other) - self._mask = m - self._shared_mask = m is not nomask - else: - result = subtract(self, masked_array(f, mask=getmask(other))) - self._data = result.data - self._mask = result.mask - self._shared_mask = 1 - return self - - - - def __idiv__(self, other): - "Divide self by other in place." - t = self._data.dtype.char - f = filled(other, 0) - t1 = f.dtype.char - if t == t1: - pass - elif t in typecodes['Integer']: - if t1 in typecodes['Integer']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - elif t in typecodes['Float']: - if t1 in typecodes['Integer']: - f = f.astype(t) - elif t1 in typecodes['Float']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - elif t in typecodes['Complex']: - if t1 in typecodes['Integer']: - f = f.astype(t) - elif t1 in typecodes['Float']: - f = f.astype(t) - elif t1 in typecodes['Complex']: - f = f.astype(t) - else: - raise TypeError, 'Incorrect type for in-place operation.' - else: - raise TypeError, 'Incorrect type for in-place operation.' - mo = getmask(other) - result = divide(self, masked_array(f, mask=mo)) - self._data = result.data - dm = result.raw_mask() - if dm is not self._mask: - self._mask = dm - self._shared_mask = 1 - return self - - def __eq__(self, other): - return equal(self,other) - - def __ne__(self, other): - return not_equal(self,other) - - def __lt__(self, other): - return less(self,other) - - def __le__(self, other): - return less_equal(self,other) - - def __gt__(self, other): - return greater(self,other) - - def __ge__(self, other): - return greater_equal(self,other) - - def astype (self, tc): - "return self as array of given type." - d = self._data.astype(tc) - return array(d, mask=self._mask) - - def byte_swapped(self): - """Returns the raw data field, byte_swapped. Included for consistency - with numeric but doesn't make sense in this context. - """ - return self._data.byte_swapped() - - def compressed (self): - "A 1-D array of all the non-masked data." - d = fromnumeric.ravel(self._data) - if self._mask is nomask: - return array(d) - else: - m = 1 - fromnumeric.ravel(self._mask) - c = fromnumeric.compress(m, d) - return array(c, copy=0) - - def count (self, axis = None): - "Count of the non-masked elements in a, or along a certain axis." - m = self._mask - s = self._data.shape - ls = len(s) - if m is nomask: - if ls == 0: - return 1 - if ls == 1: - return s[0] - if axis is None: - return reduce(lambda x, y:x*y, s) - else: - n = s[axis] - t = list(s) - del t[axis] - return ones(t) * n - if axis is None: - w = fromnumeric.ravel(m).astype(int) - n1 = size(w) - if n1 == 1: - n2 = w[0] - else: - n2 = umath.add.reduce(w) - return n1 - n2 - else: - n1 = size(m, axis) - n2 = sum(m.astype(int), axis) - return n1 - n2 - - def dot (self, other): - "s.dot(other) = innerproduct(s, other)" - return innerproduct(self, other) - - def fill_value(self): - "Get the current fill value." - return self._fill_value - - def filled (self, fill_value=None): - """A numeric array with masked values filled. If fill_value is None, - use self.fill_value(). - - If mask is nomask, copy data only if not contiguous. - Result is always a contiguous, numeric array. -# Is contiguous really necessary now? - """ - d = self._data - m = self._mask - if m is nomask: - if d.flags['CONTIGUOUS']: - return d - else: - return d.copy() - else: - if fill_value is None: - value = self._fill_value - else: - value = fill_value - - if self is masked: - result = numeric.array(value) - else: - try: - result = numeric.array(d, dtype=d.dtype, copy=1) - result[m] = value - except (TypeError, AttributeError): - #ok, can't put that value in here - value = numeric.array(value, dtype=object) - d = d.astype(object) - result = fromnumeric.choose(m, (d, value)) - return result - - def ids (self): - """Return the ids of the data and mask areas""" - return (id(self._data), id(self._mask)) - - def iscontiguous (self): - "Is the data contiguous?" - return self._data.flags['CONTIGUOUS'] - - def itemsize(self): - "Item size of each data item." - return self._data.itemsize - - - def outer(self, other): - "s.outer(other) = outerproduct(s, other)" - return outerproduct(self, other) - - def put (self, values): - """Set the non-masked entries of self to filled(values). - No change to mask - """ - iota = numeric.arange(self.size) - d = self._data - if self._mask is nomask: - ind = iota - else: - ind = fromnumeric.compress(1 - self._mask, iota) - d[ind] = filled(values).astype(d.dtype) - - def putmask (self, values): - """Set the masked entries of self to filled(values). - Mask changed to nomask. - """ - d = self._data - if self._mask is not nomask: - d[self._mask] = filled(values).astype(d.dtype) - self._shared_mask = 0 - self._mask = nomask - - def ravel (self): - """Return a 1-D view of self.""" - if self._mask is nomask: - return masked_array(self._data.ravel()) - else: - return masked_array(self._data.ravel(), self._mask.ravel()) - - def raw_data (self): - """ Obsolete; use data property instead. - The raw data; portions may be meaningless. - May be noncontiguous. Expert use only.""" - return self._data - data = property(fget=raw_data, - doc="The data, but values at masked locations are meaningless.") - - def raw_mask (self): - """ Obsolete; use mask property instead. - May be noncontiguous. Expert use only. - """ - return self._mask - mask = property(fget=raw_mask, - doc="The mask, may be nomask. Values where mask true are meaningless.") - - def reshape (self, *s): - """This array reshaped to shape s""" - d = self._data.reshape(*s) - if self._mask is nomask: - return masked_array(d) - else: - m = self._mask.reshape(*s) - return masked_array(d, m) - - def set_fill_value (self, v=None): - "Set the fill value to v. Omit v to restore default." - if v is None: - v = default_fill_value (self.raw_data()) - self._fill_value = v - - def _get_ndim(self): - return self._data.ndim - ndim = property(_get_ndim, doc=numeric.ndarray.ndim.__doc__) - - def _get_size (self): - return self._data.size - size = property(fget=_get_size, doc="Number of elements in the array.") -## CHECK THIS: signature of numeric.array.size? - - def _get_dtype(self): - return self._data.dtype - dtype = property(fget=_get_dtype, doc="type of the array elements.") - - def item(self, *args): - "Return Python scalar if possible" - if self._mask is not nomask: - m = self._mask.item(*args) - try: - if m[0]: - return masked - except IndexError: - return masked - return self._data.item(*args) - - def itemset(self, *args): - "Set Python scalar into array" - item = args[-1] - args = args[:-1] - self[args] = item - - def tolist(self, fill_value=None): - "Convert to list" - return self.filled(fill_value).tolist() - - def tostring(self, fill_value=None): - "Convert to string" - return self.filled(fill_value).tostring() - - def unmask (self): - "Replace the mask by nomask if possible." - if self._mask is nomask: return - m = make_mask(self._mask, flag=1) - if m is nomask: - self._mask = nomask - self._shared_mask = 0 - - def unshare_mask (self): - "If currently sharing mask, make a copy." - if self._shared_mask: - self._mask = make_mask (self._mask, copy=1, flag=0) - self._shared_mask = 0 - - def _get_ctypes(self): - return self._data.ctypes - - def _get_T(self): - if (self.ndim < 2): - return self - return self.transpose() - - shape = property(_get_shape, _set_shape, - doc = 'tuple giving the shape of the array') - - flat = property(_get_flat, _set_flat, - doc = 'Access array in flat form.') - - real = property(_get_real, _set_real, - doc = 'Access the real part of the array') - - imaginary = property(_get_imaginary, _set_imaginary, - doc = 'Access the imaginary part of the array') - - imag = imaginary - - ctypes = property(_get_ctypes, None, doc="ctypes") - - T = property(_get_T, None, doc="get transpose") - -#end class MaskedArray - -array = MaskedArray - -def isMaskedArray (x): - "Is x a masked array, that is, an instance of MaskedArray?" - return isinstance(x, MaskedArray) - -isarray = isMaskedArray -isMA = isMaskedArray #backward compatibility - -def allclose (a, b, fill_value=1, rtol=1.e-5, atol=1.e-8): - """ Returns true if all components of a and b are equal - subject to given tolerances. - If fill_value is 1, masked values considered equal. - If fill_value is 0, masked values considered unequal. - The relative error rtol should be positive and << 1.0 - The absolute error atol comes into play for those elements - of b that are very small or zero; it says how small a must be also. - """ - m = mask_or(getmask(a), getmask(b)) - d1 = filled(a) - d2 = filled(b) - x = filled(array(d1, copy=0, mask=m), fill_value).astype(float) - y = filled(array(d2, copy=0, mask=m), 1).astype(float) - d = umath.less_equal(umath.absolute(x-y), atol + rtol * umath.absolute(y)) - return fromnumeric.alltrue(fromnumeric.ravel(d)) - -def allequal (a, b, fill_value=1): - """ - True if all entries of a and b are equal, using - fill_value as a truth value where either or both are masked. - """ - m = mask_or(getmask(a), getmask(b)) - if m is nomask: - x = filled(a) - y = filled(b) - d = umath.equal(x, y) - return fromnumeric.alltrue(fromnumeric.ravel(d)) - elif fill_value: - x = filled(a) - y = filled(b) - d = umath.equal(x, y) - dm = array(d, mask=m, copy=0) - return fromnumeric.alltrue(fromnumeric.ravel(filled(dm, 1))) - else: - return 0 - -def masked_values (data, value, rtol=1.e-5, atol=1.e-8, copy=1): - """ - masked_values(data, value, rtol=1.e-5, atol=1.e-8) - Create a masked array; mask is nomask if possible. - If copy==0, and otherwise possible, result - may share data values with original array. - Let d = filled(data, value). Returns d - masked where abs(data-value)<= atol + rtol * abs(value) - if d is of a floating point type. Otherwise returns - masked_object(d, value, copy) - """ - abs = umath.absolute - d = filled(data, value) - if issubclass(d.dtype.type, numeric.floating): - m = umath.less_equal(abs(d-value), atol+rtol*abs(value)) - m = make_mask(m, flag=1) - return array(d, mask = m, copy=copy, - fill_value=value) - else: - return masked_object(d, value, copy=copy) - -def masked_object (data, value, copy=1): - "Create array masked where exactly data equal to value" - d = filled(data, value) - dm = make_mask(umath.equal(d, value), flag=1) - return array(d, mask=dm, copy=copy, fill_value=value) - -def arange(start, stop=None, step=1, dtype=None): - """Just like range() except it returns a array whose type can be specified - by the keyword argument dtype. - """ - return array(numeric.arange(start, stop, step, dtype)) - -arrayrange = arange - -def fromstring (s, t): - "Construct a masked array from a string. Result will have no mask." - return masked_array(numeric.fromstring(s, t)) - -def left_shift (a, n): - "Left shift n bits" - m = getmask(a) - if m is nomask: - d = umath.left_shift(filled(a), n) - return masked_array(d) - else: - d = umath.left_shift(filled(a, 0), n) - return masked_array(d, m) - -def right_shift (a, n): - "Right shift n bits" - m = getmask(a) - if m is nomask: - d = umath.right_shift(filled(a), n) - return masked_array(d) - else: - d = umath.right_shift(filled(a, 0), n) - return masked_array(d, m) - -def resize (a, new_shape): - """resize(a, new_shape) returns a new array with the specified shape. - The original array's total size can be any size.""" - m = getmask(a) - if m is not nomask: - m = fromnumeric.resize(m, new_shape) - result = array(fromnumeric.resize(filled(a), new_shape), mask=m) - result.set_fill_value(get_fill_value(a)) - return result - -def repeat(a, repeats, axis=None): - """repeat elements of a repeats times along axis - repeats is a sequence of length a.shape[axis] - telling how many times to repeat each element. - """ - af = filled(a) - if isinstance(repeats, types.IntType): - if axis is None: - num = af.size - else: - num = af.shape[axis] - repeats = tuple([repeats]*num) - - m = getmask(a) - if m is not nomask: - m = fromnumeric.repeat(m, repeats, axis) - d = fromnumeric.repeat(af, repeats, axis) - result = masked_array(d, m) - result.set_fill_value(get_fill_value(a)) - return result - -def identity(n): - """identity(n) returns the identity matrix of shape n x n. - """ - return array(numeric.identity(n)) - -def indices (dimensions, dtype=None): - """indices(dimensions,dtype=None) returns an array representing a grid - of indices with row-only, and column-only variation. - """ - return array(numeric.indices(dimensions, dtype)) - -def zeros (shape, dtype=float): - """zeros(n, dtype=float) = - an array of all zeros of the given length or shape.""" - return array(numeric.zeros(shape, dtype)) - -def ones (shape, dtype=float): - """ones(n, dtype=float) = - an array of all ones of the given length or shape.""" - return array(numeric.ones(shape, dtype)) - -def count (a, axis = None): - "Count of the non-masked elements in a, or along a certain axis." - a = masked_array(a) - return a.count(axis) - -def power (a, b, third=None): - "a**b" - if third is not None: - raise MAError, "3-argument power not supported." - ma = getmask(a) - mb = getmask(b) - m = mask_or(ma, mb) - fa = filled(a, 1) - fb = filled(b, 1) - if fb.dtype.char in typecodes["Integer"]: - return masked_array(umath.power(fa, fb), m) - md = make_mask(umath.less(fa, 0), flag=1) - m = mask_or(m, md) - if m is nomask: - return masked_array(umath.power(fa, fb)) - else: - fa = numeric.where(m, 1, fa) - return masked_array(umath.power(fa, fb), m) - -def masked_array (a, mask=nomask, fill_value=None): - """masked_array(a, mask=nomask) = - array(a, mask=mask, copy=0, fill_value=fill_value) - """ - return array(a, mask=mask, copy=0, fill_value=fill_value) - -def sum (target, axis=None, dtype=None): - if axis is None: - target = ravel(target) - axis = 0 - return add.reduce(target, axis, dtype) - -def product (target, axis=None, dtype=None): - if axis is None: - target = ravel(target) - axis = 0 - return multiply.reduce(target, axis, dtype) - -def average (a, axis=None, weights=None, returned = 0): - """average(a, axis=None, weights=None) - Computes average along indicated axis. - If axis is None, average over the entire array - Inputs can be integer or floating types; result is of type float. - - If weights are given, result is sum(a*weights,axis=0)/(sum(weights,axis=0)*1.0) - weights must have a's shape or be the 1-d with length the size - of a in the given axis. - - If returned, return a tuple: the result and the sum of the weights - or count of values. Results will have the same shape. - - masked values in the weights will be set to 0.0 - """ - a = masked_array(a) - mask = a.mask - ash = a.shape - if ash == (): - ash = (1,) - if axis is None: - if mask is nomask: - if weights is None: - n = add.reduce(a.raw_data().ravel()) - d = reduce(lambda x, y: x * y, ash, 1.0) - else: - w = filled(weights, 0.0).ravel() - n = umath.add.reduce(a.raw_data().ravel() * w) - d = umath.add.reduce(w) - del w - else: - if weights is None: - n = add.reduce(a.ravel()) - w = fromnumeric.choose(mask, (1.0, 0.0)).ravel() - d = umath.add.reduce(w) - del w - else: - w = array(filled(weights, 0.0), float, mask=mask).ravel() - n = add.reduce(a.ravel() * w) - d = add.reduce(w) - del w - else: - if mask is nomask: - if weights is None: - d = ash[axis] * 1.0 - n = umath.add.reduce(a.raw_data(), axis) - else: - w = filled(weights, 0.0) - wsh = w.shape - if wsh == (): - wsh = (1,) - if wsh == ash: - w = numeric.array(w, float, copy=0) - n = add.reduce(a*w, axis) - d = add.reduce(w, axis) - del w - elif wsh == (ash[axis],): - r = [newaxis]*len(ash) - r[axis] = slice(None, None, 1) - w = eval ("w["+ repr(tuple(r)) + "] * ones(ash, float)") - n = add.reduce(a*w, axis) - d = add.reduce(w, axis) - del w, r - else: - raise ValueError, 'average: weights wrong shape.' - else: - if weights is None: - n = add.reduce(a, axis) - w = numeric.choose(mask, (1.0, 0.0)) - d = umath.add.reduce(w, axis) - del w - else: - w = filled(weights, 0.0) - wsh = w.shape - if wsh == (): - wsh = (1,) - if wsh == ash: - w = array(w, float, mask=mask, copy=0) - n = add.reduce(a*w, axis) - d = add.reduce(w, axis) - elif wsh == (ash[axis],): - r = [newaxis]*len(ash) - r[axis] = slice(None, None, 1) - w = eval ("w["+ repr(tuple(r)) + "] * masked_array(ones(ash, float), mask)") - n = add.reduce(a*w, axis) - d = add.reduce(w, axis) - else: - raise ValueError, 'average: weights wrong shape.' - del w - #print n, d, repr(mask), repr(weights) - if n is masked or d is masked: return masked - result = divide (n, d) - del n - - if isinstance(result, MaskedArray): - result.unmask() - if returned: - if not isinstance(d, MaskedArray): - d = masked_array(d) - if not d.shape == result.shape: - d = ones(result.shape, float) * d - d.unmask() - if returned: - return result, d - else: - return result - -def where (condition, x, y): - """where(condition, x, y) is x where condition is nonzero, y otherwise. - condition must be convertible to an integer array. - Answer is always the shape of condition. - The type depends on x and y. It is integer if both x and y are - the value masked. - """ - fc = filled(not_equal(condition, 0), 0) - xv = filled(x) - xm = getmask(x) - yv = filled(y) - ym = getmask(y) - d = numeric.choose(fc, (yv, xv)) - md = numeric.choose(fc, (ym, xm)) - m = getmask(condition) - m = make_mask(mask_or(m, md), copy=0, flag=1) - return masked_array(d, m) - -def choose (indices, t, out=None, mode='raise'): - "Returns array shaped like indices with elements chosen from t" - def fmask (x): - if x is masked: return 1 - return filled(x) - def nmask (x): - if x is masked: return 1 - m = getmask(x) - if m is nomask: return 0 - return m - c = filled(indices, 0) - masks = [nmask(x) for x in t] - a = [fmask(x) for x in t] - d = numeric.choose(c, a) - m = numeric.choose(c, masks) - m = make_mask(mask_or(m, getmask(indices)), copy=0, flag=1) - return masked_array(d, m) - -def masked_where(condition, x, copy=1): - """Return x as an array masked where condition is true. - Also masked where x or condition masked. - """ - cm = filled(condition,1) - m = mask_or(getmask(x), cm) - return array(filled(x), copy=copy, mask=m) - -def masked_greater(x, value, copy=1): - "masked_greater(x, value) = x masked where x > value" - return masked_where(greater(x, value), x, copy) - -def masked_greater_equal(x, value, copy=1): - "masked_greater_equal(x, value) = x masked where x >= value" - return masked_where(greater_equal(x, value), x, copy) - -def masked_less(x, value, copy=1): - "masked_less(x, value) = x masked where x < value" - return masked_where(less(x, value), x, copy) - -def masked_less_equal(x, value, copy=1): - "masked_less_equal(x, value) = x masked where x <= value" - return masked_where(less_equal(x, value), x, copy) - -def masked_not_equal(x, value, copy=1): - "masked_not_equal(x, value) = x masked where x != value" - d = filled(x, 0) - c = umath.not_equal(d, value) - m = mask_or(c, getmask(x)) - return array(d, mask=m, copy=copy) - -def masked_equal(x, value, copy=1): - """masked_equal(x, value) = x masked where x == value - For floating point consider masked_values(x, value) instead. - """ - d = filled(x, 0) - c = umath.equal(d, value) - m = mask_or(c, getmask(x)) - return array(d, mask=m, copy=copy) - -def masked_inside(x, v1, v2, copy=1): - """x with mask of all values of x that are inside [v1,v2] - v1 and v2 can be given in either order. - """ - if v2 < v1: - t = v2 - v2 = v1 - v1 = t - d = filled(x, 0) - c = umath.logical_and(umath.less_equal(d, v2), umath.greater_equal(d, v1)) - m = mask_or(c, getmask(x)) - return array(d, mask = m, copy=copy) - -def masked_outside(x, v1, v2, copy=1): - """x with mask of all values of x that are outside [v1,v2] - v1 and v2 can be given in either order. - """ - if v2 < v1: - t = v2 - v2 = v1 - v1 = t - d = filled(x, 0) - c = umath.logical_or(umath.less(d, v1), umath.greater(d, v2)) - m = mask_or(c, getmask(x)) - return array(d, mask = m, copy=copy) - -def reshape (a, *newshape): - "Copy of a with a new shape." - m = getmask(a) - d = filled(a).reshape(*newshape) - if m is nomask: - return masked_array(d) - else: - return masked_array(d, mask=numeric.reshape(m, *newshape)) - -def ravel (a): - "a as one-dimensional, may share data and mask" - m = getmask(a) - d = fromnumeric.ravel(filled(a)) - if m is nomask: - return masked_array(d) - else: - return masked_array(d, mask=numeric.ravel(m)) - -def concatenate (arrays, axis=0): - "Concatenate the arrays along the given axis" - d = [] - for x in arrays: - d.append(filled(x)) - d = numeric.concatenate(d, axis) - for x in arrays: - if getmask(x) is not nomask: break - else: - return masked_array(d) - dm = [] - for x in arrays: - dm.append(getmaskarray(x)) - dm = numeric.concatenate(dm, axis) - return masked_array(d, mask=dm) - -def swapaxes (a, axis1, axis2): - m = getmask(a) - d = masked_array(a).data - if m is nomask: - return masked_array(data=numeric.swapaxes(d, axis1, axis2)) - else: - return masked_array(data=numeric.swapaxes(d, axis1, axis2), - mask=numeric.swapaxes(m, axis1, axis2),) - - -def take (a, indices, axis=None, out=None, mode='raise'): - "returns selection of items from a." - m = getmask(a) - # d = masked_array(a).raw_data() - d = masked_array(a).data - if m is nomask: - return masked_array(numeric.take(d, indices, axis)) - else: - return masked_array(numeric.take(d, indices, axis), - mask = numeric.take(m, indices, axis)) - -def transpose(a, axes=None): - "reorder dimensions per tuple axes" - m = getmask(a) - d = filled(a) - if m is nomask: - return masked_array(numeric.transpose(d, axes)) - else: - return masked_array(numeric.transpose(d, axes), - mask = numeric.transpose(m, axes)) - - -def put(a, indices, values, mode='raise'): - """sets storage-indexed locations to corresponding values. - - Values and indices are filled if necessary. - - """ - d = a.raw_data() - ind = filled(indices) - v = filled(values) - numeric.put (d, ind, v) - m = getmask(a) - if m is not nomask: - a.unshare_mask() - numeric.put(a.raw_mask(), ind, 0) - -def putmask(a, mask, values): - "putmask(a, mask, values) sets a where mask is true." - if mask is nomask: - return - numeric.putmask(a.raw_data(), mask, values) - m = getmask(a) - if m is nomask: return - a.unshare_mask() - numeric.putmask(a.raw_mask(), mask, 0) - -def inner(a, b): - """inner(a,b) returns the dot product of two arrays, which has - shape a.shape[:-1] + b.shape[:-1] with elements computed by summing the - product of the elements from the last dimensions of a and b. - Masked elements are replace by zeros. - """ - fa = filled(a, 0) - fb = filled(b, 0) - if len(fa.shape) == 0: fa.shape = (1,) - if len(fb.shape) == 0: fb.shape = (1,) - return masked_array(numeric.inner(fa, fb)) - -innerproduct = inner - -def outer(a, b): - """outer(a,b) = {a[i]*b[j]}, has shape (len(a),len(b))""" - fa = filled(a, 0).ravel() - fb = filled(b, 0).ravel() - d = numeric.outer(fa, fb) - ma = getmask(a) - mb = getmask(b) - if ma is nomask and mb is nomask: - return masked_array(d) - ma = getmaskarray(a) - mb = getmaskarray(b) - m = make_mask(1-numeric.outer(1-ma, 1-mb), copy=0) - return masked_array(d, m) - -outerproduct = outer - -def dot(a, b): - """dot(a,b) returns matrix-multiplication between a and b. The product-sum - is over the last dimension of a and the second-to-last dimension of b. - Masked values are replaced by zeros. See also innerproduct. - """ - return innerproduct(filled(a, 0), numeric.swapaxes(filled(b, 0), -1, -2)) - -def compress(condition, x, dimension=-1, out=None): - """Select those parts of x for which condition is true. - Masked values in condition are considered false. - """ - c = filled(condition, 0) - m = getmask(x) - if m is not nomask: - m = numeric.compress(c, m, dimension) - d = numeric.compress(c, filled(x), dimension) - return masked_array(d, m) - -class _minimum_operation: - "Object to calculate minima" - def __init__ (self): - """minimum(a, b) or minimum(a) - In one argument case returns the scalar minimum. - """ - pass - - def __call__ (self, a, b=None): - "Execute the call behavior." - if b is None: - m = getmask(a) - if m is nomask: - d = amin(filled(a).ravel()) - return d - ac = a.compressed() - if len(ac) == 0: - return masked - else: - return amin(ac.raw_data()) - else: - return where(less(a, b), a, b) - - def reduce (self, target, axis=0): - """Reduce target along the given axis.""" - m = getmask(target) - if m is nomask: - t = filled(target) - return masked_array (umath.minimum.reduce (t, axis)) - else: - t = umath.minimum.reduce(filled(target, minimum_fill_value(target)), axis) - m = umath.logical_and.reduce(m, axis) - return masked_array(t, m, get_fill_value(target)) - - def outer (self, a, b): - "Return the function applied to the outer product of a and b." - ma = getmask(a) - mb = getmask(b) - if ma is nomask and mb is nomask: - m = nomask - else: - ma = getmaskarray(a) - mb = getmaskarray(b) - m = logical_or.outer(ma, mb) - d = umath.minimum.outer(filled(a), filled(b)) - return masked_array(d, m) - -minimum = _minimum_operation () - -class _maximum_operation: - "Object to calculate maxima" - def __init__ (self): - """maximum(a, b) or maximum(a) - In one argument case returns the scalar maximum. - """ - pass - - def __call__ (self, a, b=None): - "Execute the call behavior." - if b is None: - m = getmask(a) - if m is nomask: - d = amax(filled(a).ravel()) - return d - ac = a.compressed() - if len(ac) == 0: - return masked - else: - return amax(ac.raw_data()) - else: - return where(greater(a, b), a, b) - - def reduce (self, target, axis=0): - """Reduce target along the given axis.""" - m = getmask(target) - if m is nomask: - t = filled(target) - return masked_array (umath.maximum.reduce (t, axis)) - else: - t = umath.maximum.reduce(filled(target, maximum_fill_value(target)), axis) - m = umath.logical_and.reduce(m, axis) - return masked_array(t, m, get_fill_value(target)) - - def outer (self, a, b): - "Return the function applied to the outer product of a and b." - ma = getmask(a) - mb = getmask(b) - if ma is nomask and mb is nomask: - m = nomask - else: - ma = getmaskarray(a) - mb = getmaskarray(b) - m = logical_or.outer(ma, mb) - d = umath.maximum.outer(filled(a), filled(b)) - return masked_array(d, m) - -maximum = _maximum_operation () - -def sort (x, axis = -1, fill_value=None): - """If x does not have a mask, return a masked array formed from the - result of numeric.sort(x, axis). - Otherwise, fill x with fill_value. Sort it. - Set a mask where the result is equal to fill_value. - Note that this may have unintended consequences if the data contains the - fill value at a non-masked site. - - If fill_value is not given the default fill value for x's type will be - used. - """ - if fill_value is None: - fill_value = default_fill_value (x) - d = filled(x, fill_value) - s = fromnumeric.sort(d, axis) - if getmask(x) is nomask: - return masked_array(s) - return masked_values(s, fill_value, copy=0) - -def diagonal(a, k = 0, axis1=0, axis2=1): - """diagonal(a,k=0,axis1=0, axis2=1) = the k'th diagonal of a""" - d = fromnumeric.diagonal(filled(a), k, axis1, axis2) - m = getmask(a) - if m is nomask: - return masked_array(d, m) - else: - return masked_array(d, fromnumeric.diagonal(m, k, axis1, axis2)) - -def trace (a, offset=0, axis1=0, axis2=1, dtype=None, out=None): - """trace(a,offset=0, axis1=0, axis2=1) returns the sum along diagonals - (defined by the last two dimenions) of the array. - """ - return diagonal(a, offset, axis1, axis2).sum(dtype=dtype) - -def argsort (x, axis = -1, out=None, fill_value=None): - """Treating masked values as if they have the value fill_value, - return sort indices for sorting along given axis. - if fill_value is None, use get_fill_value(x) - Returns a numpy array. - """ - d = filled(x, fill_value) - return fromnumeric.argsort(d, axis) - -def argmin (x, axis = -1, out=None, fill_value=None): - """Treating masked values as if they have the value fill_value, - return indices for minimum values along given axis. - if fill_value is None, use get_fill_value(x). - Returns a numpy array if x has more than one dimension. - Otherwise, returns a scalar index. - """ - d = filled(x, fill_value) - return fromnumeric.argmin(d, axis) - -def argmax (x, axis = -1, out=None, fill_value=None): - """Treating masked values as if they have the value fill_value, - return sort indices for maximum along given axis. - if fill_value is None, use -get_fill_value(x) if it exists. - Returns a numpy array if x has more than one dimension. - Otherwise, returns a scalar index. - """ - if fill_value is None: - fill_value = default_fill_value (x) - try: - fill_value = - fill_value - except: - pass - d = filled(x, fill_value) - return fromnumeric.argmax(d, axis) - -def fromfunction (f, s): - """apply f to s to create array as in umath.""" - return masked_array(numeric.fromfunction(f, s)) - -def asarray(data, dtype=None): - """asarray(data, dtype) = array(data, dtype, copy=0) - """ - if isinstance(data, MaskedArray) and \ - (dtype is None or dtype == data.dtype): - return data - return array(data, dtype=dtype, copy=0) - -# Add methods to support ndarray interface -# XXX: I is better to to change the masked_*_operation adaptors -# XXX: to wrap ndarray methods directly to create ma.array methods. -from types import MethodType -def _m(f): - return MethodType(f, None, array) -def not_implemented(*args, **kwds): - raise NotImplementedError, "not yet implemented for numpy.ma arrays" -array.all = _m(alltrue) -array.any = _m(sometrue) -array.argmax = _m(argmax) -array.argmin = _m(argmin) -array.argsort = _m(argsort) -array.base = property(_m(not_implemented)) -array.byteswap = _m(not_implemented) - -def _choose(self, *args, **kwds): - return choose(self, args) -array.choose = _m(_choose) -del _choose - -def _clip(self,a_min,a_max,out=None): - return MaskedArray(data = self.data.clip(asarray(a_min).data, - asarray(a_max).data), - mask = mask_or(self.mask, - mask_or(getmask(a_min),getmask(a_max)))) -array.clip = _m(_clip) - -def _compress(self, cond, axis=None, out=None): - return compress(cond, self, axis) -array.compress = _m(_compress) -del _compress - -array.conj = array.conjugate = _m(conjugate) -array.copy = _m(not_implemented) - -def _cumprod(self, axis=None, dtype=None, out=None): - m = self.mask - if m is not nomask: - m = umath.logical_or.accumulate(self.mask, axis) - return MaskedArray(data = self.filled(1).cumprod(axis, dtype), mask=m) -array.cumprod = _m(_cumprod) - -def _cumsum(self, axis=None, dtype=None, out=None): - m = self.mask - if m is not nomask: - m = umath.logical_or.accumulate(self.mask, axis) - return MaskedArray(data=self.filled(0).cumsum(axis, dtype), mask=m) -array.cumsum = _m(_cumsum) - -array.diagonal = _m(diagonal) -array.dump = _m(not_implemented) -array.dumps = _m(not_implemented) -array.fill = _m(not_implemented) -array.flags = property(_m(not_implemented)) -array.flatten = _m(ravel) -array.getfield = _m(not_implemented) - -def _max(a, axis=None, out=None): - if out is not None: - raise TypeError("Output arrays Unsupported for masked arrays") - if axis is None: - return maximum(a) - else: - return maximum.reduce(a, axis) -array.max = _m(_max) -del _max -def _min(a, axis=None, out=None): - if out is not None: - raise TypeError("Output arrays Unsupported for masked arrays") - if axis is None: - return minimum(a) - else: - return minimum.reduce(a, axis) -array.min = _m(_min) -del _min -array.mean = _m(average) -array.nbytes = property(_m(not_implemented)) -array.newbyteorder = _m(not_implemented) -array.nonzero = _m(nonzero) -array.prod = _m(product) - -def _ptp(a,axis=None,out=None): - return a.max(axis,out)-a.min(axis) -array.ptp = _m(_ptp) -array.repeat = _m(repeat) -array.resize = _m(resize) -array.searchsorted = _m(not_implemented) -array.setfield = _m(not_implemented) -array.setflags = _m(not_implemented) -array.sort = _m(not_implemented) # NB: ndarray.sort is inplace - -def _squeeze(self): - try: - result = MaskedArray(data = self.data.squeeze(), - mask = self.mask.squeeze()) - except AttributeError: - result = _wrapit(self, 'squeeze') - return result -array.squeeze = _m(_squeeze) - -array.strides = property(_m(not_implemented)) -array.sum = _m(sum) -def _swapaxes(self,axis1,axis2): - return MaskedArray(data = self.data.swapaxes(axis1, axis2), - mask = self.mask.swapaxes(axis1, axis2)) -array.swapaxes = _m(_swapaxes) -array.take = _m(take) -array.tofile = _m(not_implemented) -array.trace = _m(trace) -array.transpose = _m(transpose) - -def _var(self,axis=None,dtype=None, out=None): - if axis is None: - return numeric.asarray(self.compressed()).var() - a = self.swapaxes(axis,0) - a = a - a.mean(axis=0) - a *= a - a /= a.count(axis=0) - return a.swapaxes(0,axis).sum(axis) -def _std(self,axis=None, dtype=None, out=None): - return (self.var(axis,dtype))**0.5 -array.var = _m(_var) -array.std = _m(_std) - -array.view = _m(not_implemented) -array.round = _m(around) -del _m, MethodType, not_implemented - - -masked = MaskedArray(0, int, mask=1) diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 26cbfece1..b8fdff56f 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -110,11 +110,11 @@ class TestRegression(NumpyTestCase): def check_masked_array(self,level=rlevel): """Ticket #61""" - x = np.core.ma.array(1,mask=[1]) + x = np.ma.array(1,mask=[1]) def check_mem_masked_where(self,level=rlevel): """Ticket #62""" - from numpy.core.ma import masked_where, MaskType + from numpy.ma import masked_where, MaskType a = np.zeros((1,1)) b = np.zeros(a.shape, MaskType) c = masked_where(b,a) diff --git a/numpy/ma/API_CHANGES.txt b/numpy/ma/API_CHANGES.txt new file mode 100644 index 000000000..551aba33c --- /dev/null +++ b/numpy/ma/API_CHANGES.txt @@ -0,0 +1,68 @@ +.. -*- rest -*- + +================================================== +API changes in the new masked array implementation +================================================== + +``put``, ``putmask`` behave like their ndarray counterparts +----------------------------------------------------------- + +Previously, ``putmask`` was used like this:: + + mask = [False,True,True] + x = array([1,4,7],mask=mask) + putmask(x,mask,[3]) + +which translated to:: + + x[~mask] = [3] + +(Note that a ``True``-value in a mask suppresses a value.) + +In other words, the mask had the same length as ``x``, whereas +``values`` had ``sum(~mask)`` elements. + +Now, the behaviour is similar to that of ``ndarray.putmask``, where +the mask and the values are both the same length as ``x``, i.e. + +:: + + putmask(x,mask,[3,0,0]) + + +``fill_value`` is a property +---------------------------- + +``fill_value`` is no longer a method, but a property:: + + >>> print x.fill_value + 999999 + +``cumsum`` and ``cumprod`` ignore missing values +------------------------------------------------ + +Missing values are assumed to be the identity element, i.e. 0 for +``cumsum`` and 1 for ``cumprod``:: + + >>> x = N.ma.array([1,2,3,4],mask=[False,True,False,False]) + >>> print x + [1 -- 3 4] + >>> print x.cumsum() + [1 -- 4 8] + >> print x.cumprod() + [1 -- 3 12] + +``bool(x)`` raises a ValueError +------------------------------- + +Masked arrays now behave like regular ``ndarrays``, in that they cannot be +converted to booleans: + +:: + + >>> x = N.ma.array([1,2,3]) + >>> bool(x) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all() + diff --git a/numpy/ma/LICENSE b/numpy/ma/LICENSE new file mode 100644 index 000000000..b41aae0c8 --- /dev/null +++ b/numpy/ma/LICENSE @@ -0,0 +1,24 @@ +* Copyright (c) 2006, University of Georgia and Pierre G.F. Gerard-Marchant +* All rights reserved. +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* * Neither the name of the University of Georgia nor the +* names of its contributors may be used to endorse or promote products +* derived from this software without specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY +* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file diff --git a/numpy/ma/__init__.py b/numpy/ma/__init__.py new file mode 100644 index 000000000..ccbb6d3c6 --- /dev/null +++ b/numpy/ma/__init__.py @@ -0,0 +1,22 @@ +"""Masked arrays add-ons. + +A collection of utilities for maskedarray + +:author: Pierre GF Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: __init__.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import core +from core import * + +import extras +from extras import * + +__all__ = ['core', 'extras'] +__all__ += core.__all__ +__all__ += extras.__all__ diff --git a/numpy/ma/bench.py b/numpy/ma/bench.py new file mode 100644 index 000000000..19fbe9461 --- /dev/null +++ b/numpy/ma/bench.py @@ -0,0 +1,165 @@ +#! python +# encoding: utf-8 + +import timeit +#import IPython.ipapi +#ip = IPython.ipapi.get() +#from IPython import ipmagic +import numpy +from numpy import ma +from numpy.ma import filled +from numpy.ma.testutils import assert_equal + + +#####--------------------------------------------------------------------------- +#---- --- Global variables --- +#####--------------------------------------------------------------------------- + +# Small arrays .................................. +xs = numpy.random.uniform(-1,1,6).reshape(2,3) +ys = numpy.random.uniform(-1,1,6).reshape(2,3) +zs = xs + 1j * ys +m1 = [[True, False, False], [False, False, True]] +m2 = [[True, False, True], [False, False, True]] +nmxs = numpy.ma.array(xs, mask=m1) +nmys = numpy.ma.array(ys, mask=m2) +nmzs = numpy.ma.array(zs, mask=m1) +# Big arrays .................................... +xl = numpy.random.uniform(-1,1,100*100).reshape(100,100) +yl = numpy.random.uniform(-1,1,100*100).reshape(100,100) +zl = xl + 1j * yl +maskx = xl > 0.8 +masky = yl < -0.8 +nmxl = numpy.ma.array(xl, mask=maskx) +nmyl = numpy.ma.array(yl, mask=masky) +nmzl = numpy.ma.array(zl, mask=maskx) + +#####--------------------------------------------------------------------------- +#---- --- Functions --- +#####--------------------------------------------------------------------------- + +def timer(s, v='', nloop=500, nrep=3): + units = ["s", "ms", "µs", "ns"] + scaling = [1, 1e3, 1e6, 1e9] + print "%s : %-50s : " % (v,s), + varnames = ["%ss,nm%ss,%sl,nm%sl" % tuple(x*4) for x in 'xyz'] + setup = 'from __main__ import numpy, ma, %s' % ','.join(varnames) + Timer = timeit.Timer(stmt=s, setup=setup) + best = min(Timer.repeat(nrep, nloop)) / nloop + if best > 0.0: + order = min(-int(numpy.floor(numpy.log10(best)) // 3), 3) + else: + order = 3 + print "%d loops, best of %d: %.*g %s per loop" % (nloop, nrep, + 3, + best * scaling[order], + units[order]) +# ip.magic('timeit -n%i %s' % (nloop,s)) + + + +def compare_functions_1v(func, nloop=500, + xs=xs, nmxs=nmxs, xl=xl, nmxl=nmxl): + funcname = func.__name__ + print "-"*50 + print "%s on small arrays" % funcname + module, data = "numpy.ma","nmxs" + timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) + # + print "%s on large arrays" % funcname + module, data = "numpy.ma","nmxl" + timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) + return + +def compare_methods(methodname, args, vars='x', nloop=500, test=True, + xs=xs, nmxs=nmxs, xl=xl, nmxl=nmxl): + print "-"*50 + print "%s on small arrays" % methodname + data, ver = "nm%ss" % vars, 'numpy.ma' + timer("%(data)s.%(methodname)s(%(args)s)" % locals(), v=ver, nloop=nloop) + # + print "%s on large arrays" % methodname + data, ver = "nm%sl" % vars, 'numpy.ma' + timer("%(data)s.%(methodname)s(%(args)s)" % locals(), v=ver, nloop=nloop) + return + +def compare_functions_2v(func, nloop=500, test=True, + xs=xs, nmxs=nmxs, + ys=ys, nmys=nmys, + xl=xl, nmxl=nmxl, + yl=yl, nmyl=nmyl): + funcname = func.__name__ + print "-"*50 + print "%s on small arrays" % funcname + module, data = "numpy.ma","nmxs,nmys" + timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) + # + print "%s on large arrays" % funcname + module, data = "numpy.ma","nmxl,nmyl" + timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) + return + + +############################################################################### + + +################################################################################ +if __name__ == '__main__': +# # Small arrays .................................. +# xs = numpy.random.uniform(-1,1,6).reshape(2,3) +# ys = numpy.random.uniform(-1,1,6).reshape(2,3) +# zs = xs + 1j * ys +# m1 = [[True, False, False], [False, False, True]] +# m2 = [[True, False, True], [False, False, True]] +# nmxs = numpy.ma.array(xs, mask=m1) +# nmys = numpy.ma.array(ys, mask=m2) +# nmzs = numpy.ma.array(zs, mask=m1) +# mmxs = maskedarray.array(xs, mask=m1) +# mmys = maskedarray.array(ys, mask=m2) +# mmzs = maskedarray.array(zs, mask=m1) +# # Big arrays .................................... +# xl = numpy.random.uniform(-1,1,100*100).reshape(100,100) +# yl = numpy.random.uniform(-1,1,100*100).reshape(100,100) +# zl = xl + 1j * yl +# maskx = xl > 0.8 +# masky = yl < -0.8 +# nmxl = numpy.ma.array(xl, mask=maskx) +# nmyl = numpy.ma.array(yl, mask=masky) +# nmzl = numpy.ma.array(zl, mask=maskx) +# mmxl = maskedarray.array(xl, mask=maskx, shrink=True) +# mmyl = maskedarray.array(yl, mask=masky, shrink=True) +# mmzl = maskedarray.array(zl, mask=maskx, shrink=True) +# + compare_functions_1v(numpy.sin) + compare_functions_1v(numpy.log) + compare_functions_1v(numpy.sqrt) + #.................................................................... + compare_functions_2v(numpy.multiply) + compare_functions_2v(numpy.divide) + compare_functions_2v(numpy.power) + #.................................................................... + compare_methods('ravel','', nloop=1000) + compare_methods('conjugate','','z', nloop=1000) + compare_methods('transpose','', nloop=1000) + compare_methods('compressed','', nloop=1000) + compare_methods('__getitem__','0', nloop=1000) + compare_methods('__getitem__','(0,0)', nloop=1000) + compare_methods('__getitem__','[0,-1]', nloop=1000) + compare_methods('__setitem__','0, 17', nloop=1000, test=False) + compare_methods('__setitem__','(0,0), 17', nloop=1000, test=False) + #.................................................................... + print "-"*50 + print "__setitem__ on small arrays" + timer('nmxs.__setitem__((-1,0),numpy.ma.masked)', 'numpy.ma ',nloop=10000) + + print "-"*50 + print "__setitem__ on large arrays" + timer('nmxl.__setitem__((-1,0),numpy.ma.masked)', 'numpy.ma ',nloop=10000) + + #.................................................................... + print "-"*50 + print "where on small arrays" + timer('numpy.ma.where(nmxs>2,nmxs,nmys)', 'numpy.ma ',nloop=1000) + print "-"*50 + print "where on large arrays" + timer('numpy.ma.where(nmxl>2,nmxl,nmyl)', 'numpy.ma ',nloop=100) diff --git a/numpy/ma/core.py b/numpy/ma/core.py new file mode 100644 index 000000000..ada1a554a --- /dev/null +++ b/numpy/ma/core.py @@ -0,0 +1,3307 @@ +# pylint: disable-msg=E1002 +"""MA: a facility for dealing with missing observations +MA is generally used as a numpy.array look-alike. +by Paul F. Dubois. + +Copyright 1999, 2000, 2001 Regents of the University of California. +Released for unlimited redistribution. +Adapted for numpy_core 2005 by Travis Oliphant and +(mainly) Paul Dubois. + +Subclassing of the base ndarray 2006 by Pierre Gerard-Marchant. +pgmdevlist_AT_gmail_DOT_com +Improvements suggested by Reggie Dugard (reggie_AT_merfinllc_DOT_com) + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +""" +__author__ = "Pierre GF Gerard-Marchant" +__docformat__ = "restructuredtext en" + +__all__ = ['MAError', 'MaskType', 'MaskedArray', + 'bool_', 'complex_', 'float_', 'int_', 'object_', + 'abs', 'absolute', 'add', 'all', 'allclose', 'allequal', 'alltrue', + 'amax', 'amin', 'anom', 'anomalies', 'any', 'arange', + 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctan2', + 'arctanh', 'argmax', 'argmin', 'argsort', 'around', + 'array', 'asarray','asanyarray', + 'bitwise_and', 'bitwise_or', 'bitwise_xor', + 'ceil', 'choose', 'compress', 'compressed', 'concatenate', + 'conjugate', 'cos', 'cosh', 'count', + 'default_fill_value', 'diagonal', 'divide', 'dump', 'dumps', + 'empty', 'empty_like', 'equal', 'exp', + 'fabs', 'fmod', 'filled', 'floor', 'floor_divide','fix_invalid', + 'getdata','getmask', 'getmaskarray', 'greater', 'greater_equal', + 'hypot', + 'ids', 'inner', 'innerproduct', + 'isMA', 'isMaskedArray', 'is_mask', 'is_masked', 'isarray', + 'left_shift', 'less', 'less_equal', 'load', 'loads', 'log', 'log10', + 'logical_and', 'logical_not', 'logical_or', 'logical_xor', + 'make_mask', 'make_mask_none', 'mask_or', 'masked', + 'masked_array', 'masked_equal', 'masked_greater', + 'masked_greater_equal', 'masked_inside', 'masked_less', + 'masked_less_equal', 'masked_not_equal', 'masked_object', + 'masked_outside', 'masked_print_option', 'masked_singleton', + 'masked_values', 'masked_where', 'max', 'maximum', 'mean', 'min', + 'minimum', 'multiply', + 'negative', 'nomask', 'nonzero', 'not_equal', + 'ones', 'outer', 'outerproduct', + 'power', 'product', 'ptp', 'put', 'putmask', + 'rank', 'ravel', 'remainder', 'repeat', 'reshape', 'resize', + 'right_shift', 'round_', + 'shape', 'sin', 'sinh', 'size', 'sometrue', 'sort', 'sqrt', 'std', + 'subtract', 'sum', 'swapaxes', + 'take', 'tan', 'tanh', 'transpose', 'true_divide', + 'var', 'where', + 'zeros'] + +import sys +import types +import cPickle +import operator + +import numpy +from numpy.core import bool_, complex_, float_, int_, object_, str_ + +import numpy.core.umath as umath +import numpy.core.fromnumeric as fromnumeric +import numpy.core.numeric as numeric +import numpy.core.numerictypes as ntypes +from numpy import bool_, dtype, typecodes, amax, amin, ndarray +from numpy import expand_dims as n_expand_dims +from numpy import array as narray +import warnings + + +MaskType = bool_ +nomask = MaskType(0) + +divide_tolerance = 1.e-35 +numpy.seterr(all='ignore') + +def doc_note(note): + return "\nNotes\n-----\n%s" % note + +#####-------------------------------------------------------------------------- +#---- --- Exceptions --- +#####-------------------------------------------------------------------------- +class MAError(Exception): + "Class for MA related errors." + def __init__ (self, args=None): + "Creates an exception." + Exception.__init__(self,args) + self.args = args + def __str__(self): + "Calculates the string representation." + return str(self.args) + __repr__ = __str__ + +#####-------------------------------------------------------------------------- +#---- --- Filling options --- +#####-------------------------------------------------------------------------- +# b: boolean - c: complex - f: floats - i: integer - O: object - S: string +default_filler = {'b': True, + 'c' : 1.e20 + 0.0j, + 'f' : 1.e20, + 'i' : 999999, + 'O' : '?', + 'S' : 'N/A', + 'u' : 999999, + 'V' : '???', + } +max_filler = ntypes._minvals +max_filler.update([(k,-numpy.inf) for k in [numpy.float32, numpy.float64]]) +min_filler = ntypes._maxvals +min_filler.update([(k,numpy.inf) for k in [numpy.float32, numpy.float64]]) +if 'float128' in ntypes.typeDict: + max_filler.update([(numpy.float128,-numpy.inf)]) + min_filler.update([(numpy.float128, numpy.inf)]) + +def default_fill_value(obj): + """Calculate the default fill value for the argument object. + + """ + if hasattr(obj,'dtype'): + defval = default_filler[obj.dtype.kind] + elif isinstance(obj, numeric.dtype): + defval = default_filler[obj.kind] + elif isinstance(obj, float): + defval = default_filler['f'] + elif isinstance(obj, int) or isinstance(obj, long): + defval = default_filler['i'] + elif isinstance(obj, str): + defval = default_filler['S'] + elif isinstance(obj, complex): + defval = default_filler['c'] + else: + defval = default_filler['O'] + return defval + +def minimum_fill_value(obj): + """Calculate the default fill value suitable for taking the + minimum of ``obj``. + + """ + if hasattr(obj, 'dtype'): + objtype = obj.dtype + filler = min_filler[objtype] + if filler is None: + raise TypeError, 'Unsuitable type for calculating minimum.' + return filler + elif isinstance(obj, float): + return min_filler[ntypes.typeDict['float_']] + elif isinstance(obj, int): + return min_filler[ntypes.typeDict['int_']] + elif isinstance(obj, long): + return min_filler[ntypes.typeDict['uint']] + elif isinstance(obj, numeric.dtype): + return min_filler[obj] + else: + raise TypeError, 'Unsuitable type for calculating minimum.' + +def maximum_fill_value(obj): + """Calculate the default fill value suitable for taking the maximum + of ``obj``. + + """ + if hasattr(obj, 'dtype'): + objtype = obj.dtype + filler = max_filler[objtype] + if filler is None: + raise TypeError, 'Unsuitable type for calculating minimum.' + return filler + elif isinstance(obj, float): + return max_filler[ntypes.typeDict['float_']] + elif isinstance(obj, int): + return max_filler[ntypes.typeDict['int_']] + elif isinstance(obj, long): + return max_filler[ntypes.typeDict['uint']] + elif isinstance(obj, numeric.dtype): + return max_filler[obj] + else: + raise TypeError, 'Unsuitable type for calculating minimum.' + + +def _check_fill_value(fill_value, dtype): + descr = numpy.dtype(dtype).descr + if fill_value is None: + if len(descr) > 1: + fill_value = [default_fill_value(numeric.dtype(d[1])) + for d in descr] + else: + fill_value = default_fill_value(dtype) + else: + fill_value = narray(fill_value).tolist() + fval = numpy.resize(fill_value, len(descr)) + if len(descr) > 1: + fill_value = [numpy.asarray(f).astype(d[1]).item() + for (f,d) in zip(fval, descr)] + else: + fill_value = narray(fval, copy=False, dtype=dtype).item() + return fill_value + + +def set_fill_value(a, fill_value): + """Set the filling value of a, if a is a masked array. Otherwise, + do nothing. + + Returns + ------- + None + + """ + if isinstance(a, MaskedArray): + a._fill_value = _check_fill_value(fill_value, a.dtype) + return + +def get_fill_value(a): + """Return the filling value of a, if any. Otherwise, returns the + default filling value for that type. + + """ + if isinstance(a, MaskedArray): + result = a.fill_value + else: + result = default_fill_value(a) + return result + +def common_fill_value(a, b): + """Return the common filling value of a and b, if any. + If a and b have different filling values, returns None. + + """ + t1 = get_fill_value(a) + t2 = get_fill_value(b) + if t1 == t2: + return t1 + return None + + +#####-------------------------------------------------------------------------- +def filled(a, value = None): + """Return a as an array with masked data replaced by value. If + value is None, get_fill_value(a) is used instead. If a is already + a ndarray, a itself is returned. + + Parameters + ---------- + a : maskedarray or array_like + An input object. + value : {var}, optional + Filling value. If not given, the output of get_fill_value(a) + is used instead. + + Returns + ------- + a : array_like + + """ + if hasattr(a, 'filled'): + return a.filled(value) + elif isinstance(a, ndarray): + # Should we check for contiguity ? and a.flags['CONTIGUOUS']: + return a + elif isinstance(a, dict): + return narray(a, 'O') + else: + return narray(a) + +#####-------------------------------------------------------------------------- +def get_masked_subclass(*arrays): + """Return the youngest subclass of MaskedArray from a list of + (masked) arrays. In case of siblings, the first takes over. + + """ + if len(arrays) == 1: + arr = arrays[0] + if isinstance(arr, MaskedArray): + rcls = type(arr) + else: + rcls = MaskedArray + else: + arrcls = [type(a) for a in arrays] + rcls = arrcls[0] + if not issubclass(rcls, MaskedArray): + rcls = MaskedArray + for cls in arrcls[1:]: + if issubclass(cls, rcls): + rcls = cls + return rcls + +#####-------------------------------------------------------------------------- +def get_data(a, subok=True): + """Return the _data part of a (if any), or a as a ndarray. + + Parameters + ---------- + a : array_like + A ndarray or a subclass of. + subok : bool + Whether to force the output to a 'pure' ndarray (False) or to + return a subclass of ndarray if approriate (True). + + """ + data = getattr(a, '_data', numpy.array(a, subok=subok)) + if not subok: + return data.view(ndarray) + return data +getdata = get_data + +def fix_invalid(a, copy=True, fill_value=None): + """Return (a copy of) a where invalid data (nan/inf) are masked + and replaced by fill_value. + + Note that a copy is performed by default (just in case...). + + Parameters + ---------- + a : array_like + A (subclass of) ndarray. + copy : bool + Whether to use a copy of a (True) or to fix a in place (False). + fill_value : {var}, optional + Value used for fixing invalid data. If not given, the output + of get_fill_value(a) is used instead. + + Returns + ------- + b : MaskedArray + + """ + a = masked_array(a, copy=copy, subok=True) + invalid = (numpy.isnan(a._data) | numpy.isinf(a._data)) + a._mask |= invalid + if fill_value is None: + fill_value = a.fill_value + a._data[invalid] = fill_value + return a + + + +#####-------------------------------------------------------------------------- +#---- --- Ufuncs --- +#####-------------------------------------------------------------------------- +ufunc_domain = {} +ufunc_fills = {} + +class _DomainCheckInterval: + """Define a valid interval, so that : + + ``domain_check_interval(a,b)(x) = true`` where + ``x < a`` or ``x > b``. + + """ + def __init__(self, a, b): + "domain_check_interval(a,b)(x) = true where x < a or y > b" + if (a > b): + (a, b) = (b, a) + self.a = a + self.b = b + + def __call__ (self, x): + "Execute the call behavior." + return umath.logical_or(umath.greater (x, self.b), + umath.less(x, self.a)) +#............................ +class _DomainTan: + """Define a valid interval for the `tan` function, so that: + + ``domain_tan(eps) = True`` where ``abs(cos(x)) < eps`` + + """ + def __init__(self, eps): + "domain_tan(eps) = true where abs(cos(x)) < eps)" + self.eps = eps + def __call__ (self, x): + "Executes the call behavior." + return umath.less(umath.absolute(umath.cos(x)), self.eps) +#............................ +class _DomainSafeDivide: + """Define a domain for safe division.""" + def __init__ (self, tolerance=divide_tolerance): + self.tolerance = tolerance + def __call__ (self, a, b): + return umath.absolute(a) * self.tolerance >= umath.absolute(b) +#............................ +class _DomainGreater: + "DomainGreater(v)(x) = true where x <= v" + def __init__(self, critical_value): + "DomainGreater(v)(x) = true where x <= v" + self.critical_value = critical_value + + def __call__ (self, x): + "Executes the call behavior." + return umath.less_equal(x, self.critical_value) +#............................ +class _DomainGreaterEqual: + "DomainGreaterEqual(v)(x) = true where x < v" + def __init__(self, critical_value): + "DomainGreaterEqual(v)(x) = true where x < v" + self.critical_value = critical_value + + def __call__ (self, x): + "Executes the call behavior." + return umath.less(x, self.critical_value) + +#.............................................................................. +class _MaskedUnaryOperation: + """Defines masked version of unary operations, where invalid + values are pre-masked. + + Parameters + ---------- + f : callable + fill : + Default filling value (0). + domain : + Default domain (None). + + """ + def __init__ (self, mufunc, fill=0, domain=None): + """ _MaskedUnaryOperation(aufunc, fill=0, domain=None) + aufunc(fill) must be defined + self(x) returns aufunc(x) + with masked values where domain(x) is true or getmask(x) is true. + """ + self.f = mufunc + self.fill = fill + self.domain = domain + self.__doc__ = getattr(mufunc, "__doc__", str(mufunc)) + self.__name__ = getattr(mufunc, "__name__", str(mufunc)) + ufunc_domain[mufunc] = domain + ufunc_fills[mufunc] = fill + # + def __call__ (self, a, *args, **kwargs): + "Execute the call behavior." + # + m = getmask(a) + d1 = get_data(a) + # + if self.domain is not None: + dm = narray(self.domain(d1), copy=False) + m = numpy.logical_or(m, dm) + # The following two lines control the domain filling methods. + d1 = d1.copy() + # We could use smart indexing : d1[dm] = self.fill ... + # ... but numpy.putmask looks more efficient, despite the copy. + numpy.putmask(d1, dm, self.fill) + # Take care of the masked singletong first ... + if not m.ndim and m: + return masked + # Get the result class ....................... + if isinstance(a, MaskedArray): + subtype = type(a) + else: + subtype = MaskedArray + # Get the result as a view of the subtype ... + result = self.f(d1, *args, **kwargs).view(subtype) + # Fix the mask if we don't have a scalar + if result.ndim > 0: + result._mask = m + result._update_from(a) + return result + # + def __str__ (self): + return "Masked version of %s. [Invalid values are masked]" % str(self.f) + +#.............................................................................. +class _MaskedBinaryOperation: + """Define masked version of binary operations, where invalid + values are pre-masked. + + Parameters + ---------- + f : callable + fillx : + Default filling value for the first argument (0). + filly : + Default filling value for the second argument (0). + domain : + Default domain (None). + + """ + def __init__ (self, mbfunc, fillx=0, filly=0): + """abfunc(fillx, filly) must be defined. + abfunc(x, filly) = x for all x to enable reduce. + """ + self.f = mbfunc + self.fillx = fillx + self.filly = filly + self.__doc__ = getattr(mbfunc, "__doc__", str(mbfunc)) + self.__name__ = getattr(mbfunc, "__name__", str(mbfunc)) + ufunc_domain[mbfunc] = None + ufunc_fills[mbfunc] = (fillx, filly) + # + def __call__ (self, a, b, *args, **kwargs): + "Execute the call behavior." + m = mask_or(getmask(a), getmask(b)) + (d1, d2) = (get_data(a), get_data(b)) + result = self.f(d1, d2, *args, **kwargs).view(get_masked_subclass(a,b)) + if result.size > 1: + if m is not nomask: + result._mask = make_mask_none(result.shape) + result._mask.flat = m + if isinstance(a,MaskedArray): + result._update_from(a) + if isinstance(b,MaskedArray): + result._update_from(b) + elif m: + return masked + return result + # + def reduce (self, target, axis=0, dtype=None): + """Reduce `target` along the given `axis`.""" + if isinstance(target, MaskedArray): + tclass = type(target) + else: + tclass = MaskedArray + m = getmask(target) + t = filled(target, self.filly) + if t.shape == (): + t = t.reshape(1) + if m is not nomask: + m = make_mask(m, copy=1) + m.shape = (1,) + if m is nomask: + return self.f.reduce(t, axis).view(tclass) + t = t.view(tclass) + t._mask = m + tr = self.f.reduce(getdata(t), axis, dtype=dtype or t.dtype) + mr = umath.logical_and.reduce(m, axis) + tr = tr.view(tclass) + if mr.ndim > 0: + tr._mask = mr + return tr + elif mr: + return masked + return tr + + def outer (self, a, b): + """Return the function applied to the outer product of a and b. + + """ + ma = getmask(a) + mb = getmask(b) + if ma is nomask and mb is nomask: + m = nomask + else: + ma = getmaskarray(a) + mb = getmaskarray(b) + m = umath.logical_or.outer(ma, mb) + if (not m.ndim) and m: + return masked + rcls = get_masked_subclass(a,b) + # We could fill the arguments first, butis it useful ? + # d = self.f.outer(filled(a, self.fillx), filled(b, self.filly)).view(rcls) + d = self.f.outer(getdata(a), getdata(b)).view(rcls) + if d.ndim > 0: + d._mask = m + return d + + def accumulate (self, target, axis=0): + """Accumulate `target` along `axis` after filling with y fill + value. + + """ + if isinstance(target, MaskedArray): + tclass = type(target) + else: + tclass = masked_array + t = filled(target, self.filly) + return self.f.accumulate(t, axis).view(tclass) + + def __str__ (self): + return "Masked version of " + str(self.f) + +#.............................................................................. +class _DomainedBinaryOperation: + """Define binary operations that have a domain, like divide. + + They have no reduce, outer or accumulate. + + Parameters + ---------- + f : function. + domain : Default domain. + fillx : Default filling value for the first argument (0). + filly : Default filling value for the second argument (0). + + """ + def __init__ (self, dbfunc, domain, fillx=0, filly=0): + """abfunc(fillx, filly) must be defined. + abfunc(x, filly) = x for all x to enable reduce. + """ + self.f = dbfunc + self.domain = domain + self.fillx = fillx + self.filly = filly + self.__doc__ = getattr(dbfunc, "__doc__", str(dbfunc)) + self.__name__ = getattr(dbfunc, "__name__", str(dbfunc)) + ufunc_domain[dbfunc] = domain + ufunc_fills[dbfunc] = (fillx, filly) + + def __call__(self, a, b): + "Execute the call behavior." + ma = getmask(a) + mb = getmask(b) + d1 = getdata(a) + d2 = get_data(b) + t = narray(self.domain(d1, d2), copy=False) + if t.any(None): + mb = mask_or(mb, t) + # The following two lines control the domain filling + d2 = d2.copy() + numpy.putmask(d2, t, self.filly) + m = mask_or(ma, mb) + if (not m.ndim) and m: + return masked + result = self.f(d1, d2).view(get_masked_subclass(a,b)) + if result.ndim > 0: + result._mask = m + if isinstance(a,MaskedArray): + result._update_from(a) + if isinstance(b,MaskedArray): + result._update_from(b) + return result + + def __str__ (self): + return "Masked version of " + str(self.f) + +#.............................................................................. +# Unary ufuncs +exp = _MaskedUnaryOperation(umath.exp) +conjugate = _MaskedUnaryOperation(umath.conjugate) +sin = _MaskedUnaryOperation(umath.sin) +cos = _MaskedUnaryOperation(umath.cos) +tan = _MaskedUnaryOperation(umath.tan) +arctan = _MaskedUnaryOperation(umath.arctan) +arcsinh = _MaskedUnaryOperation(umath.arcsinh) +sinh = _MaskedUnaryOperation(umath.sinh) +cosh = _MaskedUnaryOperation(umath.cosh) +tanh = _MaskedUnaryOperation(umath.tanh) +abs = absolute = _MaskedUnaryOperation(umath.absolute) +fabs = _MaskedUnaryOperation(umath.fabs) +negative = _MaskedUnaryOperation(umath.negative) +floor = _MaskedUnaryOperation(umath.floor) +ceil = _MaskedUnaryOperation(umath.ceil) +around = _MaskedUnaryOperation(fromnumeric.round_) +logical_not = _MaskedUnaryOperation(umath.logical_not) +# Domained unary ufuncs ....................................................... +sqrt = _MaskedUnaryOperation(umath.sqrt, 0.0, + _DomainGreaterEqual(0.0)) +log = _MaskedUnaryOperation(umath.log, 1.0, + _DomainGreater(0.0)) +log10 = _MaskedUnaryOperation(umath.log10, 1.0, + _DomainGreater(0.0)) +tan = _MaskedUnaryOperation(umath.tan, 0.0, + _DomainTan(1.e-35)) +arcsin = _MaskedUnaryOperation(umath.arcsin, 0.0, + _DomainCheckInterval(-1.0, 1.0)) +arccos = _MaskedUnaryOperation(umath.arccos, 0.0, + _DomainCheckInterval(-1.0, 1.0)) +arccosh = _MaskedUnaryOperation(umath.arccosh, 1.0, + _DomainGreaterEqual(1.0)) +arctanh = _MaskedUnaryOperation(umath.arctanh, 0.0, + _DomainCheckInterval(-1.0+1e-15, 1.0-1e-15)) +# Binary ufuncs ............................................................... +add = _MaskedBinaryOperation(umath.add) +subtract = _MaskedBinaryOperation(umath.subtract) +multiply = _MaskedBinaryOperation(umath.multiply, 1, 1) +arctan2 = _MaskedBinaryOperation(umath.arctan2, 0.0, 1.0) +equal = _MaskedBinaryOperation(umath.equal) +equal.reduce = None +not_equal = _MaskedBinaryOperation(umath.not_equal) +not_equal.reduce = None +less_equal = _MaskedBinaryOperation(umath.less_equal) +less_equal.reduce = None +greater_equal = _MaskedBinaryOperation(umath.greater_equal) +greater_equal.reduce = None +less = _MaskedBinaryOperation(umath.less) +less.reduce = None +greater = _MaskedBinaryOperation(umath.greater) +greater.reduce = None +logical_and = _MaskedBinaryOperation(umath.logical_and) +alltrue = _MaskedBinaryOperation(umath.logical_and, 1, 1).reduce +logical_or = _MaskedBinaryOperation(umath.logical_or) +sometrue = logical_or.reduce +logical_xor = _MaskedBinaryOperation(umath.logical_xor) +bitwise_and = _MaskedBinaryOperation(umath.bitwise_and) +bitwise_or = _MaskedBinaryOperation(umath.bitwise_or) +bitwise_xor = _MaskedBinaryOperation(umath.bitwise_xor) +hypot = _MaskedBinaryOperation(umath.hypot) +# Domained binary ufuncs ...................................................... +divide = _DomainedBinaryOperation(umath.divide, _DomainSafeDivide(), 0, 1) +true_divide = _DomainedBinaryOperation(umath.true_divide, + _DomainSafeDivide(), 0, 1) +floor_divide = _DomainedBinaryOperation(umath.floor_divide, + _DomainSafeDivide(), 0, 1) +remainder = _DomainedBinaryOperation(umath.remainder, + _DomainSafeDivide(), 0, 1) +fmod = _DomainedBinaryOperation(umath.fmod, _DomainSafeDivide(), 0, 1) + + +#####-------------------------------------------------------------------------- +#---- --- Mask creation functions --- +#####-------------------------------------------------------------------------- +def get_mask(a): + """Return the mask of a, if any, or nomask. + + To get a full array of booleans of the same shape as a, use + getmaskarray. + + """ + return getattr(a, '_mask', nomask) +getmask = get_mask + +def getmaskarray(a): + """Return the mask of a, if any, or a boolean array of the shape + of a, full of False. + + """ + m = getmask(a) + if m is nomask: + m = make_mask_none(fromnumeric.shape(a)) + return m + +def is_mask(m): + """Return True if m is a legal mask. + + Does not check contents, only type. + + """ + try: + return m.dtype.type is MaskType + except AttributeError: + return False +# +def make_mask(m, copy=False, shrink=True, flag=None): + """Return m as a mask, creating a copy if necessary or requested. + + The function can accept any sequence of integers or nomask. Does + not check that contents must be 0s and 1s. + + Parameters + ---------- + m : array_like + Potential mask. + copy : bool + Whether to return a copy of m (True) or m itself (False). + shrink : bool + Whether to shrink m to nomask if all its values are False. + + """ + if flag is not None: + warnings.warn("The flag 'flag' is now called 'shrink'!", + DeprecationWarning) + shrink = flag + if m is nomask: + return nomask + elif isinstance(m, ndarray): + m = filled(m, True) + if m.dtype.type is MaskType: + if copy: + result = narray(m, dtype=MaskType, copy=copy) + else: + result = m + else: + result = narray(m, dtype=MaskType) + else: + result = narray(filled(m, True), dtype=MaskType) + # Bas les masques ! + if shrink and not result.any(): + return nomask + else: + return result + +def make_mask_none(s): + """Return a mask of shape s, filled with False. + + Parameters + ---------- + s : tuple + A tuple indicating the shape of the final mask. + + """ + result = numeric.zeros(s, dtype=MaskType) + return result + +def mask_or (m1, m2, copy=False, shrink=True): + """Return the combination of two masks m1 and m2. + + The masks are combined with the *logical_or* operator, treating + nomask as False. The result may equal m1 or m2 if the other is + nomask. + + Parameters + ---------- + m1 : array_like + First mask. + m2 : array_like + Second mask + copy : bool + Whether to return a copy. + shrink : bool + Whether to shrink m to nomask if all its values are False. + + """ + if m1 is nomask: + return make_mask(m2, copy=copy, shrink=shrink) + if m2 is nomask: + return make_mask(m1, copy=copy, shrink=shrink) + if m1 is m2 and is_mask(m1): + return m1 + return make_mask(umath.logical_or(m1, m2), copy=copy, shrink=shrink) + +#####-------------------------------------------------------------------------- +#--- --- Masking functions --- +#####-------------------------------------------------------------------------- +def masked_where(condition, a, copy=True): + """Return a as an array masked where condition is true. + + Masked values of a or condition are kept. + + Parameters + ---------- + condition : array_like + Masking condition. + a : array_like + Array to mask. + copy : bool + Whether to return a copy of a (True) or modify a in place. + + """ + cond = filled(condition,1) + a = narray(a, copy=copy, subok=True) + if hasattr(a, '_mask'): + cond = mask_or(cond, a._mask) + cls = type(a) + else: + cls = MaskedArray + result = a.view(cls) + result._mask = cond + return result + +def masked_greater(x, value, copy=True): + "Shortcut to masked_where, with condition = (x > value)." + return masked_where(greater(x, value), x, copy=copy) + +def masked_greater_equal(x, value, copy=True): + "Shortcut to masked_where, with condition = (x >= value)." + return masked_where(greater_equal(x, value), x, copy=copy) + +def masked_less(x, value, copy=True): + "Shortcut to masked_where, with condition = (x < value)." + return masked_where(less(x, value), x, copy=copy) + +def masked_less_equal(x, value, copy=True): + "Shortcut to masked_where, with condition = (x <= value)." + return masked_where(less_equal(x, value), x, copy=copy) + +def masked_not_equal(x, value, copy=True): + "Shortcut to masked_where, with condition = (x != value)." + return masked_where((x != value), x, copy=copy) + +# +def masked_equal(x, value, copy=True): + """Shortcut to masked_where, with condition = (x == value). For + floating point, consider `masked_values(x, value)` instead. + + """ + # An alternative implementation relies on filling first: probably not needed. + # d = filled(x, 0) + # c = umath.equal(d, value) + # m = mask_or(c, getmask(x)) + # return array(d, mask=m, copy=copy) + return masked_where((x == value), x, copy=copy) + +def masked_inside(x, v1, v2, copy=True): + """Shortcut to masked_where, where condition is True for x inside + the interval [v1,v2] (v1 <= x <= v2). The boundaries v1 and v2 + can be given in either order. + + Notes + ----- + The array x is prefilled with its filling value. + + """ + if v2 < v1: + (v1, v2) = (v2, v1) + xf = filled(x) + condition = (xf >= v1) & (xf <= v2) + return masked_where(condition, x, copy=copy) + +def masked_outside(x, v1, v2, copy=True): + """Shortcut to masked_where, where condition is True for x outside + the interval [v1,v2] (x < v1)|(x > v2). The boundaries v1 and v2 + can be given in either order. + + Notes + ----- + The array x is prefilled with its filling value. + + """ + if v2 < v1: + (v1, v2) = (v2, v1) + xf = filled(x) + condition = (xf < v1) | (xf > v2) + return masked_where(condition, x, copy=copy) + +# +def masked_object(x, value, copy=True): + """Mask the array x where the data are exactly equal to value. + + This function is suitable only for object arrays: for floating + point, please use ``masked_values`` instead. + + Notes + ----- + The mask is set to `nomask` if posible. + + """ + if isMaskedArray(x): + condition = umath.equal(x._data, value) + mask = x._mask + else: + condition = umath.equal(fromnumeric.asarray(x), value) + mask = nomask + mask = mask_or(mask, make_mask(condition, shrink=True)) + return masked_array(x, mask=mask, copy=copy, fill_value=value) + +def masked_values(x, value, rtol=1.e-5, atol=1.e-8, copy=True): + """Mask the array x where the data are approximately equal in + value, i.e. + + (abs(x - value) <= atol+rtol*abs(value)) + + Suitable only for floating points. For integers, please use + ``masked_equal``. The mask is set to nomask if posible. + + Parameters + ---------- + x : array_like + Array to fill. + value : float + Masking value. + rtol : float + Tolerance parameter. + atol : float + Tolerance parameter (1e-8). + copy : bool + Whether to return a copy of x. + + """ + abs = umath.absolute + xnew = filled(x, value) + if issubclass(xnew.dtype.type, numeric.floating): + condition = umath.less_equal(abs(xnew-value), atol+rtol*abs(value)) + mask = getattr(x, '_mask', nomask) + else: + condition = umath.equal(xnew, value) + mask = nomask + mask = mask_or(mask, make_mask(condition, shrink=True)) + return masked_array(xnew, mask=mask, copy=copy, fill_value=value) + +def masked_invalid(a, copy=True): + """Mask the array for invalid values (nans or infs). Any + preexisting mask is conserved. + + """ + a = narray(a, copy=copy, subok=True) + condition = (numpy.isnan(a) | numpy.isinf(a)) + if hasattr(a, '_mask'): + condition = mask_or(condition, a._mask) + cls = type(a) + else: + cls = MaskedArray + result = a.view(cls) + result._mask = condition + return result + + +#####-------------------------------------------------------------------------- +#---- --- Printing options --- +#####-------------------------------------------------------------------------- +class _MaskedPrintOption: + """Handle the string used to represent missing data in a masked + array. + + """ + def __init__ (self, display): + "Create the masked_print_option object." + self._display = display + self._enabled = True + + def display(self): + "Display the string to print for masked values." + return self._display + + def set_display (self, s): + "Set the string to print for masked values." + self._display = s + + def enabled(self): + "Is the use of the display value enabled?" + return self._enabled + + def enable(self, shrink=1): + "Set the enabling shrink to `shrink`." + self._enabled = shrink + + def __str__ (self): + return str(self._display) + + __repr__ = __str__ + +#if you single index into a masked location you get this object. +masked_print_option = _MaskedPrintOption('--') + +#####-------------------------------------------------------------------------- +#---- --- MaskedArray class --- +#####-------------------------------------------------------------------------- + +#............................................................................... +class _arraymethod(object): + """Define a wrapper for basic array methods. + + Upon call, returns a masked array, where the new _data array is + the output of the corresponding method called on the original + _data. + + If onmask is True, the new mask is the output of the method called + on the initial mask. Otherwise, the new mask is just a reference + to the initial mask. + + Parameters + ---------- + _name : String + Name of the function to apply on data. + _onmask : bool + Whether the mask must be processed also (True) or left + alone (False). Default: True. + obj : Object + The object calling the arraymethod. + + """ + def __init__(self, funcname, onmask=True): + self._name = funcname + self._onmask = onmask + self.obj = None + self.__doc__ = self.getdoc() + # + def getdoc(self): + "Return the doc of the function (from the doc of the method)." + methdoc = getattr(ndarray, self._name, None) + methdoc = getattr(numpy, self._name, methdoc) + if methdoc is not None: + return methdoc.__doc__ + # + def __get__(self, obj, objtype=None): + self.obj = obj + return self + # + def __call__(self, *args, **params): + methodname = self._name + data = self.obj._data + mask = self.obj._mask + cls = type(self.obj) + result = getattr(data, methodname)(*args, **params).view(cls) + result._update_from(self.obj) + if result.ndim: + if not self._onmask: + result.__setmask__(mask) + elif mask is not nomask: + result.__setmask__(getattr(mask, methodname)(*args, **params)) + else: + if mask.ndim and mask.all(): + return masked + return result +#.......................................................... + +class FlatIter(object): + "Define an interator." + def __init__(self, ma): + self.ma = ma + self.ma_iter = numpy.asarray(ma).flat + + if ma._mask is nomask: + self.maskiter = None + else: + self.maskiter = ma._mask.flat + + def __iter__(self): + return self + + ### This won't work is ravel makes a copy + def __setitem__(self, index, value): + a = self.ma.ravel() + a[index] = value + + def next(self): + d = self.ma_iter.next() + if self.maskiter is not None and self.maskiter.next(): + d = masked + return d + + +class MaskedArray(numeric.ndarray): + """Arrays with possibly masked values. Masked values of True + exclude the corresponding element from any computation. + + Construction: + x = MaskedArray(data, mask=nomask, dtype=None, copy=True, + fill_value=None, keep_mask=True, hard_mask=False, shrink=True) + + Parameters + ---------- + data : {var} + Input data. + mask : {nomask, sequence} + Mask. Must be convertible to an array of booleans with + the same shape as data: True indicates a masked (eg., + invalid) data. + dtype : dtype + Data type of the output. If None, the type of the data + argument is used. If dtype is not None and different from + data.dtype, a copy is performed. + copy : bool + Whether to copy the input data (True), or to use a + reference instead. Note: data are NOT copied by default. + subok : {True, boolean} + Whether to return a subclass of MaskedArray (if possible) + or a plain MaskedArray. + ndmin : {0, int} + Minimum number of dimensions + fill_value : {var} + Value used to fill in the masked values when necessary. If + None, a default based on the datatype is used. + keep_mask : {True, boolean} + Whether to combine mask with the mask of the input data, + if any (True), or to use only mask for the output (False). + hard_mask : {False, boolean} + Whether to use a hard mask or not. With a hard mask, + masked values cannot be unmasked. + shrink : {True, boolean} + Whether to force compression of an empty mask. + + """ + + __array_priority__ = 15 + _defaultmask = nomask + _defaulthardmask = False + _baseclass = numeric.ndarray + + def __new__(cls, data=None, mask=nomask, dtype=None, copy=False, + subok=True, ndmin=0, fill_value=None, + keep_mask=True, hard_mask=False, flag=None,shrink=True, + **options): + """Create a new masked array from scratch. + + Note: you can also create an array with the .view(MaskedArray) + method. + + """ + if flag is not None: + warnings.warn("The flag 'flag' is now called 'shrink'!", + DeprecationWarning) + shrink = flag + # Process data............ + _data = narray(data, dtype=dtype, copy=copy, subok=True, ndmin=ndmin) + _baseclass = getattr(data, '_baseclass', type(_data)) + _basedict = getattr(data, '_basedict', getattr(data, '__dict__', None)) + if not isinstance(data, MaskedArray) or not subok: + _data = _data.view(cls) + else: + _data = _data.view(type(data)) + # Backwards compatibility w/ numpy.core.ma ....... + if hasattr(data,'_mask') and not isinstance(data, ndarray): + _data._mask = data._mask + _sharedmask = True + # Process mask ........... + if mask is nomask: + if not keep_mask: + if shrink: + _data._mask = nomask + else: + _data._mask = make_mask_none(_data) + if copy: + _data._mask = _data._mask.copy() + _data._sharedmask = False + else: + _data._sharedmask = True + else: + mask = narray(mask, dtype=MaskType, copy=copy) + if mask.shape != _data.shape: + (nd, nm) = (_data.size, mask.size) + if nm == 1: + mask = numeric.resize(mask, _data.shape) + elif nm == nd: + mask = fromnumeric.reshape(mask, _data.shape) + else: + msg = "Mask and data not compatible: data size is %i, "+\ + "mask size is %i." + raise MAError, msg % (nd, nm) + copy = True + if _data._mask is nomask: + _data._mask = mask + _data._sharedmask = not copy + else: + if not keep_mask: + _data._mask = mask + _data._sharedmask = not copy + else: + _data._mask = umath.logical_or(mask, _data._mask) + _data._sharedmask = False + + # Update fill_value....... + _data._fill_value = _check_fill_value(fill_value, _data.dtype) + # Process extra options .. + _data._hardmask = hard_mask + _data._baseclass = _baseclass + _data._basedict = _basedict + return _data + # + def _update_from(self, obj): + """Copies some attributes of obj to self. + """ + self._hardmask = getattr(obj, '_hardmask', self._defaulthardmask) + self._sharedmask = getattr(obj, '_sharedmask', False) + if obj is not None: + self._baseclass = getattr(obj, '_baseclass', type(obj)) + else: + self._baseclass = ndarray + self._fill_value = getattr(obj, '_fill_value', None) + return + #........................ + def __array_finalize__(self,obj): + """Finalizes the masked array. + """ + # Get main attributes ......... + self._mask = getattr(obj, '_mask', nomask) + self._update_from(obj) + # Update special attributes ... + self._basedict = getattr(obj, '_basedict', getattr(obj, '__dict__', None)) + if self._basedict is not None: + self.__dict__.update(self._basedict) + # Finalize the mask ........... + if self._mask is not nomask: + self._mask.shape = self.shape + return + #.................................. + def __array_wrap__(self, obj, context=None): + """Special hook for ufuncs. + Wraps the numpy array and sets the mask according to context. + """ + result = obj.view(type(self)) + result._update_from(self) + #.......... + if context is not None: + result._mask = result._mask.copy() + (func, args, _) = context + m = reduce(mask_or, [getmaskarray(arg) for arg in args]) + # Get the domain mask................ + domain = ufunc_domain.get(func, None) + if domain is not None: + if len(args) > 2: + d = reduce(domain, args) + else: + d = domain(*args) + # Fill the result where the domain is wrong + try: + # Binary domain: take the last value + fill_value = ufunc_fills[func][-1] + except TypeError: + # Unary domain: just use this one + fill_value = ufunc_fills[func] + except KeyError: + # Domain not recognized, use fill_value instead + fill_value = self.fill_value + result = result.copy() + numpy.putmask(result, d, fill_value) + # Update the mask + if m is nomask: + if d is not nomask: + m = d + else: + m |= d + # Make sure the mask has the proper size + if result.shape == () and m: + return masked + else: + result._mask = m + result._sharedmask = False + #.... + return result + #............................................. + def __getitem__(self, indx): + """x.__getitem__(y) <==> x[y] + + Return the item described by i, as a masked array. + + """ + # This test is useful, but we should keep things light... +# if getmask(indx) is not nomask: +# msg = "Masked arrays must be filled before they can be used as indices!" +# raise IndexError, msg + dout = ndarray.__getitem__(self.view(ndarray), indx) + m = self._mask + if not getattr(dout,'ndim', False): + # Just a scalar............ + if m is not nomask and m[indx]: + return masked + else: + # Force dout to MA ........ + dout = dout.view(type(self)) + # Inherit attributes from self + dout._update_from(self) + # Check the fill_value .... + if isinstance(indx, basestring): + fvindx = list(self.dtype.names).index(indx) + dout._fill_value = self.fill_value[fvindx] + # Update the mask if needed + if m is not nomask: + if isinstance(indx, basestring): + dout._mask = m.reshape(dout.shape) + else: + dout._mask = ndarray.__getitem__(m, indx).reshape(dout.shape) +# Note: Don't try to check for m.any(), that'll take too long... +# mask = ndarray.__getitem__(m, indx).reshape(dout.shape) +# if self._shrinkmask and not m.any(): +# dout._mask = nomask +# else: +# dout._mask = mask + return dout + #........................ + def __setitem__(self, indx, value): + """x.__setitem__(i, y) <==> x[i]=y + + Set item described by index. If value is masked, masks those + locations. + + """ + if self is masked: + raise MAError, 'Cannot alter the masked element.' + # This test is useful, but we should keep things light... +# if getmask(indx) is not nomask: +# msg = "Masked arrays must be filled before they can be used as indices!" +# raise IndexError, msg + if isinstance(indx, basestring): + ndarray.__setitem__(self._data,indx, getdata(value)) + warnings.warn("The mask is NOT affected!") + return + #.... + if value is masked: + m = self._mask + if m is nomask: + m = numpy.zeros(self.shape, dtype=MaskType) + m[indx] = True + self._mask = m + self._sharedmask = False + return + #.... + dval = getdata(value).astype(self.dtype) + valmask = getmask(value) + if self._mask is nomask: + if valmask is not nomask: + self._mask = numpy.zeros(self.shape, dtype=MaskType) + self._mask[indx] = valmask + elif not self._hardmask: + # Unshare the mask if necessary to avoid propagation + self.unshare_mask() + self._mask[indx] = valmask + elif hasattr(indx, 'dtype') and (indx.dtype==bool_): + indx = indx * umath.logical_not(self._mask) + else: + mindx = mask_or(self._mask[indx], valmask, copy=True) + dindx = self._data[indx] + if dindx.size > 1: + dindx[~mindx] = dval + elif mindx is nomask: + dindx = dval + dval = dindx + self._mask[indx] = mindx + # Set data .......... + ndarray.__setitem__(self._data,indx,dval) + #............................................ + def __getslice__(self, i, j): + """x.__getslice__(i, j) <==> x[i:j] + + Return the slice described by (i, j). The use of negative + indices is not supported. + + """ + return self.__getitem__(slice(i,j)) + #........................ + def __setslice__(self, i, j, value): + """x.__setslice__(i, j, value) <==> x[i:j]=value + + Set the slice (i,j) of a to value. If value is masked, mask + those locations. + + """ + self.__setitem__(slice(i,j), value) + #............................................ + def __setmask__(self, mask, copy=False): + """Set the mask. + + """ + if mask is not nomask: + mask = narray(mask, copy=copy, dtype=MaskType) + # We could try to check whether shrinking is needed.. + # ... but we would waste some precious time +# if self._shrinkmask and not mask.any(): +# mask = nomask + if self._mask is nomask: + self._mask = mask + elif self._hardmask: + if mask is not nomask: + self._mask.__ior__(mask) + else: + # This one is tricky: if we set the mask that way, we may break the + # propagation. But if we don't, we end up with a mask full of False + # and a test on nomask fails... + if mask is nomask: + self._mask = nomask + else: + self.unshare_mask() + self._mask.flat = mask + if self._mask.shape: + self._mask = numeric.reshape(self._mask, self.shape) + _set_mask = __setmask__ + #.... + def _get_mask(self): + """Return the current mask. + + """ + # We could try to force a reshape, but that wouldn't work in some cases. +# return self._mask.reshape(self.shape) + return self._mask + mask = property(fget=_get_mask, fset=__setmask__, doc="Mask") + #............................................ + def harden_mask(self): + """Force the mask to hard. + + """ + self._hardmask = True + + def soften_mask(self): + """Force the mask to soft. + + """ + self._hardmask = False + + def unshare_mask(self): + """Copy the mask and set the sharedmask flag to False. + + """ + if self._sharedmask: + self._mask = self._mask.copy() + self._sharedmask = False + + def shrink_mask(self): + """Reduce a mask to nomask when possible. + + """ + m = self._mask + if m.ndim and not m.any(): + self._mask = nomask + + #............................................ + def _get_data(self): + """Return the current data, as a view of the original + underlying data. + + """ + return self.view(self._baseclass) + _data = property(fget=_get_data) + data = property(fget=_get_data) + + def raw_data(self): + """Return the _data part of the MaskedArray. + + DEPRECATED: You should really use ``.data`` instead... + + """ + warnings.warn('Use .data instead.', DeprecationWarning) + return self._data + #............................................ + def _get_flat(self): + """Return a flat iterator. + + """ + return FlatIter(self) + # + def _set_flat (self, value): + """Set a flattened version of self to value. + + """ + y = self.ravel() + y[:] = value + # + flat = property(fget=_get_flat, fset=_set_flat, + doc="Flat version of the array.") + #............................................ + def get_fill_value(self): + """Return the filling value. + + """ + if self._fill_value is None: + self._fill_value = _check_fill_value(None, self.dtype) + return self._fill_value + + def set_fill_value(self, value=None): + """Set the filling value to value. + + If value is None, use a default based on the data type. + + """ + self._fill_value = _check_fill_value(value,self.dtype) + + fill_value = property(fget=get_fill_value, fset=set_fill_value, + doc="Filling value.") + + def filled(self, fill_value=None): + """Return a copy of self._data, where masked values are filled + with fill_value. + + If fill_value is None, self.fill_value is used instead. + + Notes + ----- + + Subclassing is preserved + + The result is NOT a MaskedArray ! + + Examples + -------- + >>> x = array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999) + >>> x.filled() + array([1,2,-999,4,-999]) + >>> type(x.filled()) + <type 'numpy.ndarray'> + + """ + m = self._mask + if m is nomask or not m.any(): + return self._data + # + if fill_value is None: + fill_value = self.fill_value + # + if self is masked_singleton: + result = numeric.asanyarray(fill_value) + else: + result = self._data.copy() + try: + numpy.putmask(result, m, fill_value) + except (TypeError, AttributeError): + fill_value = narray(fill_value, dtype=object) + d = result.astype(object) + result = fromnumeric.choose(m, (d, fill_value)) + except IndexError: + #ok, if scalar + if self._data.shape: + raise + elif m: + result = narray(fill_value, dtype=self.dtype) + else: + result = self._data + return result + + def compressed(self): + """Return a 1-D array of all the non-masked data. + + """ + data = ndarray.ravel(self._data).view(type(self)) + data._update_from(self) + if self._mask is not nomask: + data = data[numpy.logical_not(ndarray.ravel(self._mask))] + return data + + + def compress(self, condition, axis=None, out=None): + """Return a where condition is True. + If condition is a MaskedArray, missing values are considered as False. + + Returns + ------- + A MaskedArray object. + + Notes + ----- + Please note the difference with compressed() ! + The output of compress has a mask, the output of compressed does not. + + """ + # Get the basic components + (_data, _mask) = (self._data, self._mask) + # Force the condition to a regular ndarray (forget the missing values...) + condition = narray(condition, copy=False, subok=False) + # + _new = _data.compress(condition, axis=axis, out=out).view(type(self)) + _new._update_from(self) + if _mask is not nomask: + _new._mask = _mask.compress(condition, axis=axis) + return _new + + #............................................ + def __str__(self): + """String representation. + + """ + if masked_print_option.enabled(): + f = masked_print_option + if self is masked: + return str(f) + m = self._mask + if m is nomask: + res = self._data + else: + if m.shape == (): + if m: + return str(f) + else: + return str(self._data) + # convert to object array to make filled work +#CHECK: the two lines below seem more robust than the self._data.astype +# res = numeric.empty(self._data.shape, object_) +# numeric.putmask(res,~m,self._data) + res = self._data.astype("|O8") + res[m] = f + else: + res = self.filled(self.fill_value) + return str(res) + + def __repr__(self): + """Literal string representation. + + """ + with_mask = """\ +masked_%(name)s(data = + %(data)s, + mask = + %(mask)s, + fill_value=%(fill)s) +""" + with_mask1 = """\ +masked_%(name)s(data = %(data)s, + mask = %(mask)s, + fill_value=%(fill)s) +""" + n = len(self.shape) + name = repr(self._data).split('(')[0] + if n <= 1: + return with_mask1 % { + 'name': name, + 'data': str(self), + 'mask': str(self._mask), + 'fill': str(self.fill_value), + } + return with_mask % { + 'name': name, + 'data': str(self), + 'mask': str(self._mask), + 'fill': str(self.fill_value), + } + #............................................ + def __add__(self, other): + "Add other to self, and return a new masked array." + return add(self, other) + # + def __sub__(self, other): + "Subtract other to self, and return a new masked array." + return subtract(self, other) + # + def __mul__(self, other): + "Multiply other by self, and return a new masked array." + return multiply(self, other) + # + def __div__(self, other): + "Divides other into self, and return a new masked array." + return divide(self, other) + # + def __truediv__(self, other): + "Divide other into self, and return a new masked array." + return true_divide(self, other) + # + def __floordiv__(self, other): + "Divide other into self, and return a new masked array." + return floor_divide(self, other) + + #............................................ + def __iadd__(self, other): + "Add other to self in-place." + ndarray.__iadd__(self._data, getdata(other)) + m = getmask(other) + if self._mask is nomask: + self._mask = m + elif m is not nomask: + self._mask += m + return self + #.... + def __isub__(self, other): + "Subtract other from self in-place." + ndarray.__isub__(self._data, getdata(other)) + m = getmask(other) + if self._mask is nomask: + self._mask = m + elif m is not nomask: + self._mask += m + return self + #.... + def __imul__(self, other): + "Multiply self by other in-place." + ndarray.__imul__(self._data, getdata(other)) + m = getmask(other) + if self._mask is nomask: + self._mask = m + elif m is not nomask: + self._mask += m + return self + #.... + def __idiv__(self, other): + "Divide self by other in-place." + other_data = getdata(other) + dom_mask = _DomainSafeDivide().__call__(self._data, other_data) + other_mask = getmask(other) + new_mask = mask_or(other_mask, dom_mask) + # The following 3 lines control the domain filling + if dom_mask.any(): + other_data = other_data.copy() + numpy.putmask(other_data, dom_mask, 1) + ndarray.__idiv__(self._data, other_data) + self._mask = mask_or(self._mask, new_mask) + return self + #............................................ + def __float__(self): + "Convert to float." + if self._mask is not nomask: + warnings.warn("Warning: converting a masked element to nan.") + return numpy.nan + return float(self.item()) + + def __int__(self): + "Convert to int." + if self._mask is not nomask: + raise MAError, 'Cannot convert masked element to a Python int.' + return int(self.item()) + #............................................ + def get_imag(self): + result = self._data.imag.view(type(self)) + result.__setmask__(self._mask) + return result + imag = property(fget=get_imag,doc="Imaginary part") + + def get_real(self): + result = self._data.real.view(type(self)) + result.__setmask__(self._mask) + return result + real = property(fget=get_real,doc="Real part") + + + #............................................ + def count(self, axis=None): + """Count the non-masked elements of the array along the given + axis. + + Parameters + ---------- + axis : int, optional + Axis along which to count the non-masked elements. If + not given, all the non masked elements are counted. + + Returns + ------- + A masked array where the mask is True where all data are + masked. If axis is None, returns either a scalar ot the + masked singleton if all values are masked. + + """ + m = self._mask + s = self.shape + ls = len(s) + if m is nomask: + if ls == 0: + return 1 + if ls == 1: + return s[0] + if axis is None: + return self.size + else: + n = s[axis] + t = list(s) + del t[axis] + return numeric.ones(t) * n + n1 = numpy.size(m, axis) + n2 = m.astype(int_).sum(axis) + if axis is None: + return (n1-n2) + else: + return masked_array(n1 - n2) + #............................................ + flatten = _arraymethod('flatten') + # + def ravel(self): + """Returns a 1D version of self, as a view.""" + r = ndarray.ravel(self._data).view(type(self)) + r._update_from(self) + if self._mask is not nomask: + r._mask = ndarray.ravel(self._mask).reshape(r.shape) + else: + r._mask = nomask + return r + # + repeat = _arraymethod('repeat') + # + def reshape (self, *s): + """Reshape the array to shape s. + + Returns + ------- + A new masked array. + + Notes + ----- + If you want to modify the shape in place, please use + ``a.shape = s`` + + """ + result = self._data.reshape(*s).view(type(self)) + result.__dict__.update(self.__dict__) + if result._mask is not nomask: + result._mask = self._mask.copy() + result._mask.shape = result.shape + return result + # + def resize(self, newshape, refcheck=True, order=False): + """Attempt to modify the size and the shape of the array in place. + + The array must own its own memory and not be referenced by + other arrays. + + Returns + ------- + None. + + """ + try: + self._data.resize(newshape, refcheck, order) + if self.mask is not nomask: + self._mask.resize(newshape, refcheck, order) + except ValueError: + raise ValueError("Cannot resize an array that has been referenced " + "or is referencing another array in this way.\n" + "Use the resize function.") + return None + # + def put(self, indices, values, mode='raise'): + """Set storage-indexed locations to corresponding values. + + a.put(values, indices, mode) sets a.flat[n] = values[n] for + each n in indices. If ``values`` is shorter than ``indices`` + then it will repeat. If ``values`` has some masked values, the + initial mask is updated in consequence, else the corresponding + values are unmasked. + + """ + m = self._mask + # Hard mask: Get rid of the values/indices that fall on masked data + if self._hardmask and self._mask is not nomask: + mask = self._mask[indices] + indices = narray(indices, copy=False) + values = narray(values, copy=False, subok=True) + values.resize(indices.shape) + indices = indices[~mask] + values = values[~mask] + #.... + self._data.put(indices, values, mode=mode) + #.... + if m is nomask: + m = getmask(values) + else: + m = m.copy() + if getmask(values) is nomask: + m.put(indices, False, mode=mode) + else: + m.put(indices, values._mask, mode=mode) + m = make_mask(m, copy=False, shrink=True) + self._mask = m + #............................................ + def ids (self): + """Return the addresses of the data and mask areas.""" + if self._mask is nomask: + return (self.ctypes.data, id(nomask)) + return (self.ctypes.data, self._mask.ctypes.data) + #............................................ + def all(self, axis=None, out=None): + """Return True if all entries along the given axis are True, + False otherwise. Masked values are considered as True during + computation. + + Parameter + ---------- + axis : int, optional + Axis along which the operation is performed. If None, + the operation is performed on a flatten array + out : {MaskedArray}, optional + Alternate optional output. If not None, out should be + a valid MaskedArray of the same shape as the output of + self._data.all(axis). + + Returns A masked array, where the mask is True if all data along + ------- + the axis are masked. + + Notes + ----- + An exception is raised if ``out`` is not None and not of the + same type as self. + + """ + if out is None: + d = self.filled(True).all(axis=axis).view(type(self)) + if d.ndim > 0: + d.__setmask__(self._mask.all(axis)) + return d + elif type(out) is not type(self): + raise TypeError("The external array should have " \ + "a type %s (got %s instead)" %\ + (type(self), type(out))) + self.filled(True).all(axis=axis, out=out) + if out.ndim: + out.__setmask__(self._mask.all(axis)) + return out + + + def any(self, axis=None, out=None): + """Returns True if at least one entry along the given axis is + True. + + Returns False if all entries are False. + Masked values are considered as True during computation. + + Parameter + ---------- + axis : int, optional + Axis along which the operation is performed. + If None, the operation is performed on a flatten array + out : {MaskedArray}, optional + Alternate optional output. If not None, out should be + a valid MaskedArray of the same shape as the output of + self._data.all(axis). + + Returns A masked array, where the mask is True if all data along + ------- + the axis are masked. + + Notes + ----- + An exception is raised if ``out`` is not None and not of the + same type as self. + + """ + if out is None: + d = self.filled(False).any(axis=axis).view(type(self)) + if d.ndim > 0: + d.__setmask__(self._mask.all(axis)) + return d + elif type(out) is not type(self): + raise TypeError("The external array should have a type %s "\ + "(got %s instead)" %\ + (type(self), type(out))) + self.filled(False).any(axis=axis, out=out) + if out.ndim: + out.__setmask__(self._mask.all(axis)) + return out + + + def nonzero(self): + """Return the indices of the elements of a that are not zero + nor masked, as a tuple of arrays. + + There are as many tuples as dimensions of a, each tuple + contains the indices of the non-zero elements in that + dimension. The corresponding non-zero values can be obtained + with ``a[a.nonzero()]``. + + To group the indices by element, rather than dimension, use + instead: ``transpose(a.nonzero())``. + + The result of this is always a 2d array, with a row for each + non-zero element. + + """ + return narray(self.filled(0), copy=False).nonzero() + #............................................ + def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None): + """a.trace(offset=0, axis1=0, axis2=1, dtype=None, out=None) + + Return the sum along the offset diagonal of the array's + indicated `axis1` and `axis2`. + + """ + # TODO: What are we doing with `out`? + m = self._mask + if m is nomask: + result = super(MaskedArray, self).trace(offset=offset, axis1=axis1, + axis2=axis2, out=out) + return result.astype(dtype) + else: + D = self.diagonal(offset=offset, axis1=axis1, axis2=axis2) + return D.astype(dtype).filled(0).sum(axis=None) + #............................................ + def sum(self, axis=None, dtype=None): + """Sum the array over the given axis. + + Masked elements are set to 0 internally. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : dtype, optional + Datatype for the intermediary computation. If not given, + the current dtype is used instead. + + """ + if self._mask is nomask: + mask = nomask + else: + mask = self._mask.all(axis) + if (not mask.ndim) and mask: + return masked + result = self.filled(0).sum(axis, dtype=dtype).view(type(self)) + if result.ndim > 0: + result.__setmask__(mask) + return result + + def cumsum(self, axis=None, dtype=None): + """Return the cumulative sum of the elements of the array + along the given axis. + + Masked values are set to 0 internally. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not + given, the current dtype is used instead. + + """ + result = self.filled(0).cumsum(axis=axis, dtype=dtype).view(type(self)) + result.__setmask__(self.mask) + return result + + def prod(self, axis=None, dtype=None): + """Return the product of the elements of the array along the + given axis. + + Masked elements are set to 1 internally. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not + given, the current dtype is used instead. + + """ + if self._mask is nomask: + mask = nomask + else: + mask = self._mask.all(axis) + if (not mask.ndim) and mask: + return masked + result = self.filled(1).prod(axis=axis, dtype=dtype).view(type(self)) + if result.ndim: + result.__setmask__(mask) + return result + + product = prod + + def cumprod(self, axis=None, dtype=None): + """Return the cumulative product of the elements of the array + along the given axis. + + Masked values are set to 1 internally. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not + given, the current dtype is used instead. + + """ + result = self.filled(1).cumprod(axis=axis, dtype=dtype).view(type(self)) + result.__setmask__(self.mask) + return result + + def mean(self, axis=None, dtype=None): + """Average the array over the given axis. Equivalent to + + a.sum(axis, dtype) / a.size(axis). + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not + given, the current dtype is used instead. + + """ + if self._mask is nomask: + return super(MaskedArray, self).mean(axis=axis, dtype=dtype) + else: + dsum = self.sum(axis=axis, dtype=dtype) + cnt = self.count(axis=axis) + return dsum*1./cnt + + def anom(self, axis=None, dtype=None): + """Return the anomalies (deviations from the average) along + the given axis. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not + given, the current dtype is used instead. + + """ + m = self.mean(axis, dtype) + if not axis: + return (self - m) + else: + return (self - expand_dims(m,axis)) + + def var(self, axis=None, dtype=None): + """Return the variance, a measure of the spread of a distribution. + + The variance is the average of the squared deviations from the + mean, i.e. var = mean((x - x.mean())**2). + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not + given, the current dtype is used instead. + + Notes + ----- + The value returned is a biased estimate of the true variance. + For the (more standard) unbiased estimate, use varu. + + """ + if self._mask is nomask: + # TODO: Do we keep super, or var _data and take a view ? + return super(MaskedArray, self).var(axis=axis, dtype=dtype) + else: + cnt = self.count(axis=axis) + danom = self.anom(axis=axis, dtype=dtype) + danom *= danom + dvar = narray(danom.sum(axis) / cnt).view(type(self)) + if axis is not None: + dvar._mask = mask_or(self._mask.all(axis), (cnt==1)) + dvar._update_from(self) + return dvar + + def std(self, axis=None, dtype=None): + """Return the standard deviation, a measure of the spread of a + distribution. + + The standard deviation is the square root of the average of + the squared deviations from the mean, i.e. + + std = sqrt(mean((x - x.mean())**2)). + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. + If not given, the current dtype is used instead. + + Notes + ----- + The value returned is a biased estimate of the true + standard deviation. For the more standard unbiased + estimate, use stdu. + + """ + dvar = self.var(axis,dtype) + if axis is not None or dvar is not masked: + dvar = sqrt(dvar) + return dvar + + #............................................ + def argsort(self, axis=None, fill_value=None, kind='quicksort', + order=None): + """Return an ndarray of indices that sort the array along the + specified axis. Masked values are filled beforehand to + fill_value. + + Parameters + ---------- + axis : int, optional + Axis to be indirectly sorted. + If not given, uses a flatten version of the array. + fill_value : {var} + Value used to fill in the masked values. + If not given, self.fill_value is used instead. + kind : {string} + Sorting algorithm (default 'quicksort') + Possible values: 'quicksort', 'mergesort', or 'heapsort' + + Notes + ----- + This method executes an indirect sort along the given axis + using the algorithm specified by the kind keyword. It returns + an array of indices of the same shape as 'a' that index data + along the given axis in sorted order. + + The various sorts are characterized by average speed, worst + case performance need for work space, and whether they are + stable. A stable sort keeps items with the same key in the + same relative order. The three available algorithms have the + following properties: + + |------------------------------------------------------| + | kind | speed | worst case | work space | stable| + |------------------------------------------------------| + |'quicksort'| 1 | O(n^2) | 0 | no | + |'mergesort'| 2 | O(n*log(n)) | ~n/2 | yes | + |'heapsort' | 3 | O(n*log(n)) | 0 | no | + |------------------------------------------------------| + + All the sort algorithms make temporary copies of the data when + the sort is not along the last axis. Consequently, sorts along + the last axis are faster and use less space than sorts along + other axis. + + """ + if fill_value is None: + fill_value = default_fill_value(self) + d = self.filled(fill_value).view(ndarray) + return d.argsort(axis=axis, kind=kind, order=order) + #........................ + def argmin(self, axis=None, fill_value=None): + """Return an ndarray of indices for the minimum values of a + along the specified axis. + + Masked values are treated as if they had the value fill_value. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. If None, the + output of minimum_fill_value(self._data) is used. + + """ + if fill_value is None: + fill_value = minimum_fill_value(self) + d = self.filled(fill_value).view(ndarray) + return d.argmin(axis) + #........................ + def argmax(self, axis=None, fill_value=None): + """Returns the array of indices for the maximum values of `a` + along the specified axis. + + Masked values are treated as if they had the value fill_value. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. If None, the + output of maximum_fill_value(self._data) is used. + + """ + if fill_value is None: + fill_value = maximum_fill_value(self._data) + d = self.filled(fill_value).view(ndarray) + return d.argmax(axis) + + def sort(self, axis=-1, kind='quicksort', order=None, + endwith=True, fill_value=None): + """Sort along the given axis. + + Parameters + ---------- + axis : int + Axis to be indirectly sorted. + kind : {string} + Sorting algorithm (default 'quicksort') + Possible values: 'quicksort', 'mergesort', or 'heapsort'. + order : {var} + If a has fields defined, then the order keyword can be + the field name to sort on or a list (or tuple) of + field names to indicate the order that fields should + be used to define the sort. + fill_value : {var} + Value used to fill in the masked values. If None, use + the the output of minimum_fill_value(). + endwith : bool + Whether missing values (if any) should be forced in + the upper indices (at the end of the array) (True) or + lower indices (at the beginning). + + Returns + ------- + When used as method, returns None. + When used as a function, returns an array. + + Notes + ----- + This method sorts 'a' in place along the given axis using + the algorithm specified by the kind keyword. + + The various sorts may characterized by average speed, + worst case performance need for work space, and whether + they are stable. A stable sort keeps items with the same + key in the same relative order and is most useful when + used w/ argsort where the key might differ from the items + being sorted. The three available algorithms have the + following properties: + + |------------------------------------------------------| + | kind | speed | worst case | work space | stable| + |------------------------------------------------------| + |'quicksort'| 1 | O(n^2) | 0 | no | + |'mergesort'| 2 | O(n*log(n)) | ~n/2 | yes | + |'heapsort' | 3 | O(n*log(n)) | 0 | no | + |------------------------------------------------------| + + """ + if self._mask is nomask: + ndarray.sort(self,axis=axis, kind=kind, order=order) + else: + if fill_value is None: + if endwith: + filler = minimum_fill_value(self) + else: + filler = maximum_fill_value(self) + else: + filler = fill_value + idx = numpy.indices(self.shape) + idx[axis] = self.filled(filler).argsort(axis=axis,kind=kind,order=order) + idx_l = idx.tolist() + tmp_mask = self._mask[idx_l].flat + tmp_data = self._data[idx_l].flat + self.flat = tmp_data + self._mask.flat = tmp_mask + return + + #............................................ + def min(self, axis=None, fill_value=None): + """Return the minimum of a along the given axis. + + Masked values are filled with fill_value. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. + If None, use the the output of minimum_fill_value(). + + """ + mask = self._mask + # Check all/nothing case ...... + if mask is nomask: + return super(MaskedArray, self).min(axis=axis) + elif (not mask.ndim) and mask: + return masked + # Get the mask ................ + if axis is None: + mask = umath.logical_and.reduce(mask.flat) + else: + mask = umath.logical_and.reduce(mask, axis=axis) + # Skip if all masked .......... + if not mask.ndim and mask: + return masked + # Get the fill value ........... + if fill_value is None: + fill_value = minimum_fill_value(self) + # Get the data ................ + result = self.filled(fill_value).min(axis=axis).view(type(self)) + if result.ndim > 0: + result._mask = mask + return result + + def mini(self, axis=None): + if axis is None: + return minimum(self) + else: + return minimum.reduce(self, axis) + + #........................ + def max(self, axis=None, fill_value=None): + """Return the maximum/a along the given axis. + + Masked values are filled with fill_value. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. + If None, use the the output of maximum_fill_value(). + """ + mask = self._mask + # Check all/nothing case ...... + if mask is nomask: + return super(MaskedArray, self).max(axis=axis) + elif (not mask.ndim) and mask: + return masked + # Check the mask .............. + if axis is None: + mask = umath.logical_and.reduce(mask.flat) + else: + mask = umath.logical_and.reduce(mask, axis=axis) + # Skip if all masked .......... + if not mask.ndim and mask: + return masked + # Get the fill value .......... + if fill_value is None: + fill_value = maximum_fill_value(self) + # Get the data ................ + result = self.filled(fill_value).max(axis=axis).view(type(self)) + if result.ndim > 0: + result._mask = mask + return result + #........................ + def ptp(self, axis=None, fill_value=None): + """Return the visible data range (max-min) along the given axis. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. If None, the + maximum uses the maximum default, the minimum uses the + minimum default. + + """ + return self.max(axis, fill_value) - self.min(axis, fill_value) + + # Array methods --------------------------------------- + copy = _arraymethod('copy') + diagonal = _arraymethod('diagonal') + take = _arraymethod('take') + transpose = _arraymethod('transpose') + T = property(fget=lambda self:self.transpose()) + swapaxes = _arraymethod('swapaxes') + clip = _arraymethod('clip', onmask=False) + copy = _arraymethod('copy') + squeeze = _arraymethod('squeeze') + #-------------------------------------------- + def tolist(self, fill_value=None): + """Copy the data portion of the array to a hierarchical python + list and returns that list. + + Data items are converted to the nearest compatible Python + type. Masked values are converted to fill_value. If + fill_value is None, the corresponding entries in the output + list will be ``None``. + + """ + if fill_value is not None: + return self.filled(fill_value).tolist() + result = self.filled().tolist() + # Set temps to save time when dealing w/ mrecarrays... + _mask = self._mask + if _mask is nomask: + return result + nbdims = self.ndim + dtypesize = len(self.dtype) + if nbdims == 0: + return tuple([None]*dtypesize) + elif nbdims == 1: + maskedidx = _mask.nonzero()[0].tolist() + if dtypesize: + nodata = tuple([None]*dtypesize) + else: + nodata = None + [operator.setitem(result,i,nodata) for i in maskedidx] + else: + for idx in zip(*[i.tolist() for i in _mask.nonzero()]): + tmp = result + for i in idx[:-1]: + tmp = tmp[i] + tmp[idx[-1]] = None + return result + #........................ + def tostring(self, fill_value=None, order='C'): + """Return a copy of array data as a Python string containing the raw + bytes in the array. + + Parameters + ---------- + fill_value : {var}, optional + Value used to fill in the masked values. + If None, uses self.fill_value instead. + order : {string} + Order of the data item in the copy {"C","F","A"}. + "C" -- C order (row major) + "Fortran" -- Fortran order (column major) + "Any" -- Current order of array. + None -- Same as "Any" + + """ + return self.filled(fill_value).tostring(order=order) + #-------------------------------------------- + # Pickling + def __getstate__(self): + """Return the internal state of the masked array, for pickling + purposes. + + """ + state = (1, + self.shape, + self.dtype, + self.flags.fnc, + self._data.tostring(), + getmaskarray(self).tostring(), + self._fill_value, + ) + return state + # + def __setstate__(self, state): + """Restore the internal state of the masked array, for + pickling purposes. ``state`` is typically the output of the + ``__getstate__`` output, and is a 5-tuple: + + - class name + - a tuple giving the shape of the data + - a typecode for the data + - a binary string for the data + - a binary string for the mask. + + """ + (ver, shp, typ, isf, raw, msk, flv) = state + ndarray.__setstate__(self, (shp, typ, isf, raw)) + self._mask.__setstate__((shp, dtype(bool), isf, msk)) + self.fill_value = flv + # + def __reduce__(self): + """Return a 3-tuple for pickling a MaskedArray. + + """ + return (_mareconstruct, + (self.__class__, self._baseclass, (0,), 'b', ), + self.__getstate__()) + + +def _mareconstruct(subtype, baseclass, baseshape, basetype,): + """Internal function that builds a new MaskedArray from the + information stored in a pickle. + + """ + _data = ndarray.__new__(baseclass, baseshape, basetype) + _mask = ndarray.__new__(ndarray, baseshape, 'b1') + return subtype.__new__(subtype, _data, mask=_mask, dtype=basetype,) + + +#####-------------------------------------------------------------------------- +#---- --- Shortcuts --- +#####--------------------------------------------------------------------------- +def isMaskedArray(x): + "Is x a masked array, that is, an instance of MaskedArray?" + return isinstance(x, MaskedArray) +isarray = isMaskedArray +isMA = isMaskedArray #backward compatibility +# We define the masked singleton as a float for higher precedence... +# Note that it can be tricky sometimes w/ type comparison +masked_singleton = MaskedArray(0, dtype=float_, mask=True) +masked = masked_singleton + +masked_array = MaskedArray + +def array(data, dtype=None, copy=False, order=False, + mask=nomask, fill_value=None, + keep_mask=True, hard_mask=False, shrink=True, subok=True, ndmin=0, + ): + """array(data, dtype=None, copy=False, order=False, mask=nomask, + fill_value=None, keep_mask=True, hard_mask=False, shrink=True, + subok=True, ndmin=0) + + Acts as shortcut to MaskedArray, with options in a different order + for convenience. And backwards compatibility... + + """ + #TODO: we should try to put 'order' somwehere + return MaskedArray(data, mask=mask, dtype=dtype, copy=copy, subok=subok, + keep_mask=keep_mask, hard_mask=hard_mask, + fill_value=fill_value, ndmin=ndmin, shrink=shrink) +array.__doc__ = masked_array.__doc__ + +def is_masked(x): + """Does x have masked values?""" + m = getmask(x) + if m is nomask: + return False + elif m.any(): + return True + return False + + +#####--------------------------------------------------------------------------- +#---- --- Extrema functions --- +#####--------------------------------------------------------------------------- +class _extrema_operation(object): + "Generic class for maximum/minimum functions." + def __call__(self, a, b=None): + "Executes the call behavior." + if b is None: + return self.reduce(a) + return where(self.compare(a, b), a, b) + #......... + def reduce(self, target, axis=None): + "Reduce target along the given axis." + target = narray(target, copy=False, subok=True) + m = getmask(target) + if axis is not None: + kargs = { 'axis' : axis } + else: + kargs = {} + target = target.ravel() + if not (m is nomask): + m = m.ravel() + if m is nomask: + t = self.ufunc.reduce(target, **kargs) + else: + target = target.filled(self.fill_value_func(target)).view(type(target)) + t = self.ufunc.reduce(target, **kargs) + m = umath.logical_and.reduce(m, **kargs) + if hasattr(t, '_mask'): + t._mask = m + elif m: + t = masked + return t + #......... + def outer (self, a, b): + "Return the function applied to the outer product of a and b." + ma = getmask(a) + mb = getmask(b) + if ma is nomask and mb is nomask: + m = nomask + else: + ma = getmaskarray(a) + mb = getmaskarray(b) + m = logical_or.outer(ma, mb) + result = self.ufunc.outer(filled(a), filled(b)) + result._mask = m + return result + +#............................ +class _minimum_operation(_extrema_operation): + "Object to calculate minima" + def __init__ (self): + """minimum(a, b) or minimum(a) +In one argument case, returns the scalar minimum. + """ + self.ufunc = umath.minimum + self.afunc = amin + self.compare = less + self.fill_value_func = minimum_fill_value + +#............................ +class _maximum_operation(_extrema_operation): + "Object to calculate maxima" + def __init__ (self): + """maximum(a, b) or maximum(a) + In one argument case returns the scalar maximum. + """ + self.ufunc = umath.maximum + self.afunc = amax + self.compare = greater + self.fill_value_func = maximum_fill_value + +#.......................................................... +def min(array, axis=None, out=None): + """Return the minima along the given axis. + + If `axis` is None, applies to the flattened array. + + """ + if out is not None: + raise TypeError("Output arrays Unsupported for masked arrays") + if axis is None: + return minimum(array) + else: + return minimum.reduce(array, axis) +min.__doc__ = MaskedArray.min.__doc__ +#............................ +def max(obj, axis=None, out=None): + if out is not None: + raise TypeError("Output arrays Unsupported for masked arrays") + if axis is None: + return maximum(obj) + else: + return maximum.reduce(obj, axis) +max.__doc__ = MaskedArray.max.__doc__ +#............................. +def ptp(obj, axis=None): + """a.ptp(axis=None) = a.max(axis)-a.min(axis)""" + try: + return obj.max(axis)-obj.min(axis) + except AttributeError: + return max(obj, axis=axis) - min(obj, axis=axis) +ptp.__doc__ = MaskedArray.ptp.__doc__ + + +#####--------------------------------------------------------------------------- +#---- --- Definition of functions from the corresponding methods --- +#####--------------------------------------------------------------------------- +class _frommethod: + """Define functions from existing MaskedArray methods. + + Parameters + ---------- + _methodname : string + Name of the method to transform. + + """ + def __init__(self, methodname): + self._methodname = methodname + self.__doc__ = self.getdoc() + def getdoc(self): + "Return the doc of the function (from the doc of the method)." + try: + return getattr(MaskedArray, self._methodname).__doc__ + except: + return getattr(numpy, self._methodname).__doc__ + def __call__(self, a, *args, **params): + if isinstance(a, MaskedArray): + return getattr(a, self._methodname).__call__(*args, **params) + #FIXME ---- + #As x is not a MaskedArray, we transform it to a ndarray with asarray + #... and call the corresponding method. + #Except that sometimes it doesn't work (try reshape([1,2,3,4],(2,2))) + #we end up with a "SystemError: NULL result without error in PyObject_Call" + #A dirty trick is then to call the initial numpy function... + method = getattr(narray(a, copy=False), self._methodname) + try: + return method(*args, **params) + except SystemError: + return getattr(numpy,self._methodname).__call__(a, *args, **params) + +all = _frommethod('all') +anomalies = anom = _frommethod('anom') +any = _frommethod('any') +conjugate = _frommethod('conjugate') +ids = _frommethod('ids') +nonzero = _frommethod('nonzero') +diagonal = _frommethod('diagonal') +maximum = _maximum_operation() +mean = _frommethod('mean') +minimum = _minimum_operation () +product = _frommethod('prod') +ptp = _frommethod('ptp') +ravel = _frommethod('ravel') +repeat = _frommethod('repeat') +std = _frommethod('std') +sum = _frommethod('sum') +swapaxes = _frommethod('swapaxes') +take = _frommethod('take') +var = _frommethod('var') +compress = _frommethod('compress') + +#.............................................................................. +def power(a, b, third=None): + """Computes a**b elementwise. + + Masked values are set to 1. + + """ + if third is not None: + raise MAError, "3-argument power not supported." + ma = getmask(a) + mb = getmask(b) + m = mask_or(ma, mb) + fa = getdata(a) + fb = getdata(b) + if fb.dtype.char in typecodes["Integer"]: + return masked_array(umath.power(fa, fb), m) + md = make_mask((fa < 0), shrink=True) + m = mask_or(m, md) + if m is nomask: + return masked_array(umath.power(fa, fb)) + else: + fa = fa.copy() + fa[(fa < 0)] = 1 + return masked_array(umath.power(fa, fb), m) + +#.............................................................................. +def argsort(a, axis=None, kind='quicksort', order=None, fill_value=None): + "Function version of the eponymous method." + if fill_value is None: + fill_value = default_fill_value(a) + d = filled(a, fill_value) + if axis is None: + return d.argsort(kind=kind, order=order) + return d.argsort(axis, kind=kind, order=order) +argsort.__doc__ = MaskedArray.argsort.__doc__ + +def argmin(a, axis=None, fill_value=None): + "Function version of the eponymous method." + if fill_value is None: + fill_value = default_fill_value(a) + d = filled(a, fill_value) + return d.argmin(axis=axis) +argmin.__doc__ = MaskedArray.argmin.__doc__ + +def argmax(a, axis=None, fill_value=None): + "Function version of the eponymous method." + if fill_value is None: + fill_value = default_fill_value(a) + try: + fill_value = - fill_value + except: + pass + d = filled(a, fill_value) + return d.argmax(axis=axis) +argmin.__doc__ = MaskedArray.argmax.__doc__ + +def sort(a, axis=-1, kind='quicksort', order=None, endwith=True, fill_value=None): + "Function version of the eponymous method." + a = narray(a, copy=False, subok=True) + if fill_value is None: + if endwith: + filler = minimum_fill_value(a) + else: + filler = maximum_fill_value(a) + else: + filler = fill_value +# return + indx = numpy.indices(a.shape).tolist() + indx[axis] = filled(a,filler).argsort(axis=axis,kind=kind,order=order) + return a[indx] +sort.__doc__ = MaskedArray.sort.__doc__ + + +def compressed(x): + """Return a 1-D array of all the non-masked data.""" + if getmask(x) is nomask: + return x + else: + return x.compressed() + +def concatenate(arrays, axis=0): + "Concatenate the arrays along the given axis." + d = numpy.concatenate([getdata(a) for a in arrays], axis) + rcls = get_masked_subclass(*arrays) + data = d.view(rcls) + # Check whether one of the arrays has a non-empty mask... + for x in arrays: + if getmask(x) is not nomask: + break + else: + return data + # OK, so we have to concatenate the masks + dm = numpy.concatenate([getmaskarray(a) for a in arrays], axis) + # If we decide to keep a '_shrinkmask' option, we want to check that ... + # ... all of them are True, and then check for dm.any() +# shrink = numpy.logical_or.reduce([getattr(a,'_shrinkmask',True) for a in arrays]) +# if shrink and not dm.any(): + if not dm.any(): + data._mask = nomask + else: + data._mask = dm.reshape(d.shape) + return data + +def count(a, axis = None): + return masked_array(a, copy=False).count(axis) +count.__doc__ = MaskedArray.count.__doc__ + + +def expand_dims(x,axis): + """Expand the shape of the array by including a new axis before + the given one. + + """ + result = n_expand_dims(x,axis) + if isinstance(x, MaskedArray): + new_shape = result.shape + result = x.view() + result.shape = new_shape + if result._mask is not nomask: + result._mask.shape = new_shape + return result + +#...................................... +def left_shift (a, n): + "Left shift n bits." + m = getmask(a) + if m is nomask: + d = umath.left_shift(filled(a), n) + return masked_array(d) + else: + d = umath.left_shift(filled(a, 0), n) + return masked_array(d, mask=m) + +def right_shift (a, n): + "Right shift n bits." + m = getmask(a) + if m is nomask: + d = umath.right_shift(filled(a), n) + return masked_array(d) + else: + d = umath.right_shift(filled(a, 0), n) + return masked_array(d, mask=m) + +#...................................... +def put(a, indices, values, mode='raise'): + """Set storage-indexed locations to corresponding values. + + Values and indices are filled if necessary. + + """ + # We can't use 'frommethod', the order of arguments is different + try: + return a.put(indices, values, mode=mode) + except AttributeError: + return narray(a, copy=False).put(indices, values, mode=mode) + +def putmask(a, mask, values): #, mode='raise'): + """Set a.flat[n] = values[n] for each n where mask.flat[n] is true. + + If values is not the same size of a and mask then it will repeat + as necessary. This gives different behavior than + a[mask] = values. + + Note: Using a masked array as values will NOT transform a ndarray in + a maskedarray. + + """ + # We can't use 'frommethod', the order of arguments is different + if not isinstance(a, MaskedArray): + a = a.view(MaskedArray) + (valdata, valmask) = (getdata(values), getmask(values)) + if getmask(a) is nomask: + if valmask is not nomask: + a._sharedmask = True + a.mask = numpy.zeros(a.shape, dtype=bool_) + numpy.putmask(a._mask, mask, valmask) + elif a._hardmask: + if valmask is not nomask: + m = a._mask.copy() + numpy.putmask(m, mask, valmask) + a.mask |= m + else: + if valmask is nomask: + valmask = getmaskarray(values) + numpy.putmask(a._mask, mask, valmask) + numpy.putmask(a._data, mask, valdata) + return + +def transpose(a,axes=None): + """Return a view of the array with dimensions permuted according to axes, + as a masked array. + + If ``axes`` is None (default), the output view has reversed + dimensions compared to the original. + + """ + #We can't use 'frommethod', as 'transpose' doesn't take keywords + try: + return a.transpose(axes) + except AttributeError: + return narray(a, copy=False).transpose(axes).view(MaskedArray) + +def reshape(a, new_shape): + """Change the shape of the array a to new_shape.""" + #We can't use 'frommethod', it whine about some parameters. Dmmit. + try: + return a.reshape(new_shape) + except AttributeError: + return narray(a, copy=False).reshape(new_shape).view(MaskedArray) + +def resize(x, new_shape): + """Return a new array with the specified shape. + + The total size of the original array can be any size. The new + array is filled with repeated copies of a. If a was masked, the + new array will be masked, and the new mask will be a repetition of + the old one. + + """ + # We can't use _frommethods here, as N.resize is notoriously whiny. + m = getmask(x) + if m is not nomask: + m = numpy.resize(m, new_shape) + result = numpy.resize(x, new_shape).view(get_masked_subclass(x)) + if result.ndim: + result._mask = m + return result + + +#................................................ +def rank(obj): + "maskedarray version of the numpy function." + return fromnumeric.rank(getdata(obj)) +rank.__doc__ = numpy.rank.__doc__ +# +def shape(obj): + "maskedarray version of the numpy function." + return fromnumeric.shape(getdata(obj)) +shape.__doc__ = numpy.shape.__doc__ +# +def size(obj, axis=None): + "maskedarray version of the numpy function." + return fromnumeric.size(getdata(obj), axis) +size.__doc__ = numpy.size.__doc__ +#................................................ + +#####-------------------------------------------------------------------------- +#---- --- Extra functions --- +#####-------------------------------------------------------------------------- +def where (condition, x=None, y=None): + """where(condition | x, y) + + Returns a (subclass of) masked array, shaped like condition, where + the elements are x when condition is True, and y otherwise. If + neither x nor y are given, returns a tuple of indices where + condition is True (a la condition.nonzero()). + + Parameters + ---------- + condition : {var} + The condition to meet. Must be convertible to an integer + array. + x : {var}, optional + Values of the output when the condition is met + y : {var}, optional + Values of the output when the condition is not met. + + """ + if x is None and y is None: + return filled(condition, 0).nonzero() + elif x is None or y is None: + raise ValueError, "Either both or neither x and y should be given." + # Get the condition ............... + fc = filled(condition, 0).astype(bool_) + notfc = numpy.logical_not(fc) + # Get the data ...................................... + xv = getdata(x) + yv = getdata(y) + if x is masked: + ndtype = yv.dtype + xm = numpy.ones(fc.shape, dtype=MaskType) + elif y is masked: + ndtype = xv.dtype + ym = numpy.ones(fc.shape, dtype=MaskType) + else: + ndtype = numpy.max([xv.dtype, yv.dtype]) + xm = getmask(x) + d = numpy.empty(fc.shape, dtype=ndtype).view(MaskedArray) + numpy.putmask(d._data, fc, xv.astype(ndtype)) + numpy.putmask(d._data, notfc, yv.astype(ndtype)) + d._mask = numpy.zeros(fc.shape, dtype=MaskType) + numpy.putmask(d._mask, fc, getmask(x)) + numpy.putmask(d._mask, notfc, getmask(y)) + d._mask |= getmaskarray(condition) + if not d._mask.any(): + d._mask = nomask + return d + +def choose (indices, t, out=None, mode='raise'): + "Return array shaped like indices with elements chosen from t" + #TODO: implement options `out` and `mode`, if possible. + def fmask (x): + "Returns the filled array, or True if masked." + if x is masked: + return 1 + return filled(x) + def nmask (x): + "Returns the mask, True if ``masked``, False if ``nomask``." + if x is masked: + return 1 + m = getmask(x) + if m is nomask: + return 0 + return m + c = filled(indices, 0) + masks = [nmask(x) for x in t] + a = [fmask(x) for x in t] + d = numpy.choose(c, a) + m = numpy.choose(c, masks) + m = make_mask(mask_or(m, getmask(indices)), copy=0, shrink=True) + return masked_array(d, mask=m) + +def round_(a, decimals=0, out=None): + """Return a copy of a, rounded to 'decimals' places. + + When 'decimals' is negative, it specifies the number of positions + to the left of the decimal point. The real and imaginary parts of + complex numbers are rounded separately. Nothing is done if the + array is not of float type and 'decimals' is greater than or equal + to 0. + + Parameters + ---------- + decimals : int + Number of decimals to round to. May be negative. + out : array_like + Existing array to use for output. + If not given, returns a default copy of a. + + Notes + ----- + If out is given and does not have a mask attribute, the mask of a + is lost! + + """ + if out is None: + result = fromnumeric.round_(getdata(a), decimals, out) + if isinstance(a,MaskedArray): + result = result.view(type(a)) + result._mask = a._mask + else: + result = result.view(MaskedArray) + return result + else: + fromnumeric.round_(getdata(a), decimals, out) + if hasattr(out, '_mask'): + out._mask = getmask(a) + return out + +def arange(stop, start=None, step=1, dtype=None): + "maskedarray version of the numpy function." + return numpy.arange(stop, start, step, dtype).view(MaskedArray) +arange.__doc__ = numpy.arange.__doc__ + +def inner(a, b): + "maskedarray version of the numpy function." + fa = filled(a, 0) + fb = filled(b, 0) + if len(fa.shape) == 0: + fa.shape = (1,) + if len(fb.shape) == 0: + fb.shape = (1,) + return numpy.inner(fa, fb).view(MaskedArray) +inner.__doc__ = numpy.inner.__doc__ +inner.__doc__ += doc_note("Masked values are replaced by 0.") +innerproduct = inner + +def outer(a, b): + "maskedarray version of the numpy function." + fa = filled(a, 0).ravel() + fb = filled(b, 0).ravel() + d = numeric.outer(fa, fb) + ma = getmask(a) + mb = getmask(b) + if ma is nomask and mb is nomask: + return masked_array(d) + ma = getmaskarray(a) + mb = getmaskarray(b) + m = make_mask(1-numeric.outer(1-ma, 1-mb), copy=0) + return masked_array(d, mask=m) +outer.__doc__ = numpy.outer.__doc__ +outer.__doc__ += doc_note("Masked values are replaced by 0.") +outerproduct = outer + +def allequal (a, b, fill_value=True): + """Return True if all entries of a and b are equal, using + fill_value as a truth value where either or both are masked. + + """ + m = mask_or(getmask(a), getmask(b)) + if m is nomask: + x = getdata(a) + y = getdata(b) + d = umath.equal(x, y) + return d.all() + elif fill_value: + x = getdata(a) + y = getdata(b) + d = umath.equal(x, y) + dm = array(d, mask=m, copy=False) + return dm.filled(True).all(None) + else: + return False + +def allclose (a, b, fill_value=True, rtol=1.e-5, atol=1.e-8): + """ Return True if all elements of a and b are equal subject to + given tolerances. + + If fill_value is True, masked values are considered equal. + If fill_value is False, masked values considered unequal. + The relative error rtol should be positive and << 1.0 + The absolute error atol comes into play for those elements of b + that are very small or zero; it says how small `a` must be also. + + """ + m = mask_or(getmask(a), getmask(b)) + d1 = getdata(a) + d2 = getdata(b) + x = filled(array(d1, copy=0, mask=m), fill_value).astype(float) + y = filled(array(d2, copy=0, mask=m), 1).astype(float) + d = umath.less_equal(umath.absolute(x-y), atol + rtol * umath.absolute(y)) + return fromnumeric.alltrue(fromnumeric.ravel(d)) + +#.............................................................................. +def asarray(a, dtype=None): + """asarray(data, dtype) = array(data, dtype, copy=0, subok=0) + + Return a as a MaskedArray object of the given dtype. + If dtype is not given or None, is is set to the dtype of a. + No copy is performed if a is already an array. + Subclasses are converted to the base class MaskedArray. + + """ + return masked_array(a, dtype=dtype, copy=False, keep_mask=True, subok=False) + +def asanyarray(a, dtype=None): + """asanyarray(data, dtype) = array(data, dtype, copy=0, subok=1) + + Return a as an masked array. + If dtype is not given or None, is is set to the dtype of a. + No copy is performed if a is already an array. + Subclasses are conserved. + + """ + return masked_array(a, dtype=dtype, copy=False, keep_mask=True, subok=True) + + +def empty(new_shape, dtype=float): + "maskedarray version of the numpy function." + return numpy.empty(new_shape, dtype).view(MaskedArray) +empty.__doc__ = numpy.empty.__doc__ + +def empty_like(a): + "maskedarray version of the numpy function." + return numpy.empty_like(a).view(MaskedArray) +empty_like.__doc__ = numpy.empty_like.__doc__ + +def ones(new_shape, dtype=float): + "maskedarray version of the numpy function." + return numpy.ones(new_shape, dtype).view(MaskedArray) +ones.__doc__ = numpy.ones.__doc__ + +def zeros(new_shape, dtype=float): + "maskedarray version of the numpy function." + return numpy.zeros(new_shape, dtype).view(MaskedArray) +zeros.__doc__ = numpy.zeros.__doc__ + +#####-------------------------------------------------------------------------- +#---- --- Pickling --- +#####-------------------------------------------------------------------------- +def dump(a,F): + """Pickle the MaskedArray `a` to the file `F`. `F` can either be + the handle of an exiting file, or a string representing a file + name. + + """ + if not hasattr(F,'readline'): + F = open(F,'w') + return cPickle.dump(a,F) + +def dumps(a): + """Return a string corresponding to the pickling of the + MaskedArray. + + """ + return cPickle.dumps(a) + +def load(F): + """Wrapper around ``cPickle.load`` which accepts either a + file-like object or a filename. + + """ + if not hasattr(F, 'readline'): + F = open(F,'r') + return cPickle.load(F) + +def loads(strg): + "Load a pickle from the current string.""" + return cPickle.loads(strg) + +################################################################################ diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py new file mode 100644 index 000000000..4ebb6ca2f --- /dev/null +++ b/numpy/ma/extras.py @@ -0,0 +1,802 @@ +"""Masked arrays add-ons. + +A collection of utilities for maskedarray + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: extras.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +__all__ = [ +'apply_along_axis', 'atleast_1d', 'atleast_2d', 'atleast_3d', 'average', +'vstack', 'hstack', 'dstack', 'row_stack', 'column_stack', +'compress_rowcols', 'compress_rows', 'compress_cols', 'count_masked', +'dot', 'hsplit', +'mask_rowcols','mask_rows','mask_cols','masked_all','masked_all_like', +'mediff1d', 'mr_', +'notmasked_edges','notmasked_contiguous', +'stdu', 'varu', + ] + +from itertools import groupby + +import core +from core import * + +import numpy +from numpy import float_ +import numpy.core.umath as umath +import numpy.core.numeric as numeric +from numpy.core.numeric import ndarray +from numpy.core.numeric import array as nxarray +from numpy.core.fromnumeric import asarray as nxasarray + +from numpy.lib.index_tricks import AxisConcatenator +import numpy.lib.function_base as function_base + +#............................................................................... +def issequence(seq): + """Is seq a sequence (ndarray, list or tuple)?""" + if isinstance(seq, ndarray): + return True + elif isinstance(seq, tuple): + return True + elif isinstance(seq, list): + return True + return False + +def count_masked(arr, axis=None): + """Count the number of masked elements along the given axis. + + Parameters + ---------- + axis : int, optional + Axis along which to count. + If None (default), a flattened version of the array is used. + + """ + m = getmaskarray(arr) + return m.sum(axis) + +def masked_all(shape, dtype=float_): + """Return an empty masked array of the given shape and dtype, + where all the data are masked. + + Parameters + ---------- + dtype : dtype, optional + Data type of the output. + + """ + a = masked_array(numeric.empty(shape, dtype), + mask=numeric.ones(shape, bool_)) + return a + +def masked_all_like(arr): + """Return an empty masked array of the same shape and dtype as + the array `a`, where all the data are masked. + + """ + a = masked_array(numeric.empty_like(arr), + mask=numeric.ones(arr.shape, bool_)) + return a + +#####-------------------------------------------------------------------------- +#---- --- New methods --- +#####-------------------------------------------------------------------------- +def varu(a, axis=None, dtype=None): + """Return an unbiased estimate of the variance. + i.e. var = sum((x - x.mean())**2)/(size(x,axis)-1) + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, + the current dtype is used instead. + + Notes + ----- + The value returned is an unbiased estimate of the true variance. + For the (less standard) biased estimate, use var. + + """ + a = asarray(a) + cnt = a.count(axis=axis) + anom = a.anom(axis=axis, dtype=dtype) + anom *= anom + dvar = anom.sum(axis) / (cnt-1) + if axis is None: + return dvar + dvar.__setmask__(mask_or(a._mask.all(axis), (cnt==1))) + return dvar +# return a.__class__(dvar, +# mask=mask_or(a._mask.all(axis), (cnt==1)), +# fill_value=a._fill_value) + +def stdu(a, axis=None, dtype=None): + """Return an unbiased estimate of the standard deviation. The + standard deviation is the square root of the average of the + squared deviations from the mean, i.e. stdu = sqrt(varu(x)). + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : dtype, optional + Datatype for the intermediary computation. + If not given, the current dtype is used instead. + + Notes + ----- + The value returned is an unbiased estimate of the true + standard deviation. For the biased estimate, + use std. + + """ + a = asarray(a) + dvar = a.varu(axis,dtype) + if axis is None: + if dvar is masked: + return masked + else: + # Should we use umath.sqrt instead ? + return sqrt(dvar) + return sqrt(dvar) + + +MaskedArray.stdu = stdu +MaskedArray.varu = varu + +#####-------------------------------------------------------------------------- +#---- --- Standard functions --- +#####-------------------------------------------------------------------------- +class _fromnxfunction: + """Defines a wrapper to adapt numpy functions to masked arrays.""" + def __init__(self, funcname): + self._function = funcname + self.__doc__ = self.getdoc() + def getdoc(self): + "Retrieves the __doc__ string from the function." + return getattr(numpy, self._function).__doc__ +\ + "*Notes*:\n (The function is applied to both the _data and the _mask, if any.)" + def __call__(self, *args, **params): + func = getattr(numpy, self._function) + if len(args)==1: + x = args[0] + if isinstance(x,ndarray): + _d = func(nxasarray(x), **params) + _m = func(getmaskarray(x), **params) + return masked_array(_d, mask=_m) + elif isinstance(x, tuple) or isinstance(x, list): + _d = func(tuple([nxasarray(a) for a in x]), **params) + _m = func(tuple([getmaskarray(a) for a in x]), **params) + return masked_array(_d, mask=_m) + else: + arrays = [] + args = list(args) + while len(args)>0 and issequence(args[0]): + arrays.append(args.pop(0)) + res = [] + for x in arrays: + _d = func(nxasarray(x), *args, **params) + _m = func(getmaskarray(x), *args, **params) + res.append(masked_array(_d, mask=_m)) + return res + +atleast_1d = _fromnxfunction('atleast_1d') +atleast_2d = _fromnxfunction('atleast_2d') +atleast_3d = _fromnxfunction('atleast_3d') + +vstack = row_stack = _fromnxfunction('vstack') +hstack = _fromnxfunction('hstack') +column_stack = _fromnxfunction('column_stack') +dstack = _fromnxfunction('dstack') + +hsplit = _fromnxfunction('hsplit') + +#####-------------------------------------------------------------------------- +#---- +#####-------------------------------------------------------------------------- +def flatten_inplace(seq): + """Flatten a sequence in place.""" + k = 0 + while (k != len(seq)): + while hasattr(seq[k],'__iter__'): + seq[k:(k+1)] = seq[k] + k += 1 + return seq + + +def apply_along_axis(func1d,axis,arr,*args,**kwargs): + """Execute func1d(arr[i],*args) where func1d takes 1-D arrays and + arr is an N-d array. i varies so as to apply the function along + the given axis for each 1-d subarray in arr. + """ + arr = core.array(arr, copy=False, subok=True) + nd = arr.ndim + if axis < 0: + axis += nd + if (axis >= nd): + raise ValueError("axis must be less than arr.ndim; axis=%d, rank=%d." + % (axis,nd)) + ind = [0]*(nd-1) + i = numeric.zeros(nd,'O') + indlist = range(nd) + indlist.remove(axis) + i[axis] = slice(None,None) + outshape = numeric.asarray(arr.shape).take(indlist) + i.put(indlist, ind) + j = i.copy() + res = func1d(arr[tuple(i.tolist())],*args,**kwargs) + # if res is a number, then we have a smaller output array + asscalar = numeric.isscalar(res) + if not asscalar: + try: + len(res) + except TypeError: + asscalar = True + # Note: we shouldn't set the dtype of the output from the first result... + #...so we force the type to object, and build a list of dtypes + #...we'll just take the largest, to avoid some downcasting + dtypes = [] + if asscalar: + dtypes.append(numeric.asarray(res).dtype) + outarr = zeros(outshape, object_) + outarr[tuple(ind)] = res + Ntot = numeric.product(outshape) + k = 1 + while k < Ntot: + # increment the index + ind[-1] += 1 + n = -1 + while (ind[n] >= outshape[n]) and (n > (1-nd)): + ind[n-1] += 1 + ind[n] = 0 + n -= 1 + i.put(indlist,ind) + res = func1d(arr[tuple(i.tolist())],*args,**kwargs) + outarr[tuple(ind)] = res + dtypes.append(asarray(res).dtype) + k += 1 + else: + res = core.array(res, copy=False, subok=True) + j = i.copy() + j[axis] = ([slice(None,None)] * res.ndim) + j.put(indlist, ind) + Ntot = numeric.product(outshape) + holdshape = outshape + outshape = list(arr.shape) + outshape[axis] = res.shape + dtypes.append(asarray(res).dtype) + outshape = flatten_inplace(outshape) + outarr = zeros(outshape, object_) + outarr[tuple(flatten_inplace(j.tolist()))] = res + k = 1 + while k < Ntot: + # increment the index + ind[-1] += 1 + n = -1 + while (ind[n] >= holdshape[n]) and (n > (1-nd)): + ind[n-1] += 1 + ind[n] = 0 + n -= 1 + i.put(indlist, ind) + j.put(indlist, ind) + res = func1d(arr[tuple(i.tolist())],*args,**kwargs) + outarr[tuple(flatten_inplace(j.tolist()))] = res + dtypes.append(asarray(res).dtype) + k += 1 + max_dtypes = numeric.dtype(numeric.asarray(dtypes).max()) + if not hasattr(arr, '_mask'): + result = numeric.asarray(outarr, dtype=max_dtypes) + else: + result = core.asarray(outarr, dtype=max_dtypes) + result.fill_value = core.default_fill_value(result) + return result + +def average(a, axis=None, weights=None, returned=False): + """Average the array over the given axis. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + weights : sequence, optional + Sequence of weights. + The weights must have the shape of a, or be 1D with length + the size of a along the given axis. + If no weights are given, weights are assumed to be 1. + returned : bool + Flag indicating whether a tuple (result, sum of weights/counts) + should be returned as output (True), or just the result (False). + + """ + a = asarray(a) + mask = a.mask + ash = a.shape + if ash == (): + ash = (1,) + if axis is None: + if mask is nomask: + if weights is None: + n = a.sum(axis=None) + d = float(a.size) + else: + w = filled(weights, 0.0).ravel() + n = umath.add.reduce(a._data.ravel() * w) + d = umath.add.reduce(w) + del w + else: + if weights is None: + n = a.filled(0).sum(axis=None) + d = umath.add.reduce((-mask).ravel().astype(int_)) + else: + w = array(filled(weights, 0.0), float, mask=mask).ravel() + n = add.reduce(a.ravel() * w) + d = add.reduce(w) + del w + else: + if mask is nomask: + if weights is None: + d = ash[axis] * 1.0 + n = add.reduce(a._data, axis, dtype=float_) + else: + w = filled(weights, 0.0) + wsh = w.shape + if wsh == (): + wsh = (1,) + if wsh == ash: + w = numeric.array(w, float_, copy=0) + n = add.reduce(a*w, axis) + d = add.reduce(w, axis) + del w + elif wsh == (ash[axis],): + ni = ash[axis] + r = [None]*len(ash) + r[axis] = slice(None, None, 1) + w = eval ("w["+ repr(tuple(r)) + "] * ones(ash, float)") + n = add.reduce(a*w, axis, dtype=float_) + d = add.reduce(w, axis, dtype=float_) + del w, r + else: + raise ValueError, 'average: weights wrong shape.' + else: + if weights is None: + n = add.reduce(a, axis, dtype=float_) + d = umath.add.reduce((-mask), axis=axis, dtype=float_) + else: + w = filled(weights, 0.0) + wsh = w.shape + if wsh == (): + wsh = (1,) + if wsh == ash: + w = array(w, dtype=float_, mask=mask, copy=0) + n = add.reduce(a*w, axis, dtype=float_) + d = add.reduce(w, axis, dtype=float_) + elif wsh == (ash[axis],): + ni = ash[axis] + r = [None]*len(ash) + r[axis] = slice(None, None, 1) + w = eval ("w["+ repr(tuple(r)) + \ + "] * masked_array(ones(ash, float), mask)") + n = add.reduce(a*w, axis, dtype=float_) + d = add.reduce(w, axis, dtype=float_) + else: + raise ValueError, 'average: weights wrong shape.' + del w + if n is masked or d is masked: + return masked + result = n/d + del n + + if isMaskedArray(result): + if ((axis is None) or (axis==0 and a.ndim == 1)) and \ + (result.mask is nomask): + result = result._data + if returned: + if not isMaskedArray(d): + d = masked_array(d) + if isinstance(d, ndarray) and (not d.shape == result.shape): + d = ones(result.shape, dtype=float_) * d + if returned: + return result, d + else: + return result + +#.............................................................................. +def compress_rowcols(x, axis=None): + """Suppress the rows and/or columns of a 2D array that contains + masked values. + + The suppression behavior is selected with the `axis`parameter. + - If axis is None, rows and columns are suppressed. + - If axis is 0, only rows are suppressed. + - If axis is 1 or -1, only columns are suppressed. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + + Returns + ------- + compressed_array : an ndarray. + + """ + x = asarray(x) + if x.ndim != 2: + raise NotImplementedError, "compress2d works for 2D arrays only." + m = getmask(x) + # Nothing is masked: return x + if m is nomask or not m.any(): + return x._data + # All is masked: return empty + if m.all(): + return nxarray([]) + # Builds a list of rows/columns indices + (idxr, idxc) = (range(len(x)), range(x.shape[1])) + masked = m.nonzero() + if not axis: + for i in function_base.unique(masked[0]): + idxr.remove(i) + if axis in [None, 1, -1]: + for j in function_base.unique(masked[1]): + idxc.remove(j) + return x._data[idxr][:,idxc] + +def compress_rows(a): + """Suppress whole rows of a 2D array that contain masked values. + + """ + return compress_rowcols(a,0) + +def compress_cols(a): + """Suppress whole columnss of a 2D array that contain masked values. + + """ + return compress_rowcols(a,1) + +def mask_rowcols(a, axis=None): + """Mask whole rows and/or columns of a 2D array that contain + masked values. The masking behavior is selected with the + `axis`parameter. + + - If axis is None, rows and columns are masked. + - If axis is 0, only rows are masked. + - If axis is 1 or -1, only columns are masked. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + + Returns + ------- + a *pure* ndarray. + + """ + a = asarray(a) + if a.ndim != 2: + raise NotImplementedError, "compress2d works for 2D arrays only." + m = getmask(a) + # Nothing is masked: return a + if m is nomask or not m.any(): + return a + maskedval = m.nonzero() + a._mask = a._mask.copy() + if not axis: + a[function_base.unique(maskedval[0])] = masked + if axis in [None, 1, -1]: + a[:,function_base.unique(maskedval[1])] = masked + return a + +def mask_rows(a, axis=None): + """Mask whole rows of a 2D array that contain masked values. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + """ + return mask_rowcols(a, 0) + +def mask_cols(a, axis=None): + """Mask whole columns of a 2D array that contain masked values. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + """ + return mask_rowcols(a, 1) + + +def dot(a,b, strict=False): + """Return the dot product of two 2D masked arrays a and b. + + Like the generic numpy equivalent, the product sum is over the + last dimension of a and the second-to-last dimension of b. If + strict is True, masked values are propagated: if a masked value + appears in a row or column, the whole row or column is considered + masked. + + Parameters + ---------- + strict : {boolean} + Whether masked data are propagated (True) or set to 0 for + the computation. + + Notes + ----- + The first argument is not conjugated. + + """ + #TODO: Works only with 2D arrays. There should be a way to get it to run with higher dimension + if strict and (a.ndim == 2) and (b.ndim == 2): + a = mask_rows(a) + b = mask_cols(b) + # + d = numpy.dot(filled(a, 0), filled(b, 0)) + # + am = (~getmaskarray(a)) + bm = (~getmaskarray(b)) + m = ~numpy.dot(am,bm) + return masked_array(d, mask=m) + +#............................................................................... +def mediff1d(array, to_end=None, to_begin=None): + """Return the differences between consecutive elements of an + array, possibly with prefixed and/or appended values. + + Parameters + ---------- + array : {array} + Input array, will be flattened before the difference is taken. + to_end : {number}, optional + If provided, this number will be tacked onto the end of the returned + differences. + to_begin : {number}, optional + If provided, this number will be taked onto the beginning of the + returned differences. + + Returns + ------- + ed : {array} + The differences. Loosely, this will be (ary[1:] - ary[:-1]). + + """ + a = masked_array(array, copy=True) + if a.ndim > 1: + a.reshape((a.size,)) + (d, m, n) = (a._data, a._mask, a.size-1) + dd = d[1:]-d[:-1] + if m is nomask: + dm = nomask + else: + dm = m[1:]-m[:-1] + # + if to_end is not None: + to_end = asarray(to_end) + nend = to_end.size + if to_begin is not None: + to_begin = asarray(to_begin) + nbegin = to_begin.size + r_data = numeric.empty((n+nend+nbegin,), dtype=a.dtype) + r_mask = numeric.zeros((n+nend+nbegin,), dtype=bool_) + r_data[:nbegin] = to_begin._data + r_mask[:nbegin] = to_begin._mask + r_data[nbegin:-nend] = dd + r_mask[nbegin:-nend] = dm + else: + r_data = numeric.empty((n+nend,), dtype=a.dtype) + r_mask = numeric.zeros((n+nend,), dtype=bool_) + r_data[:-nend] = dd + r_mask[:-nend] = dm + r_data[-nend:] = to_end._data + r_mask[-nend:] = to_end._mask + # + elif to_begin is not None: + to_begin = asarray(to_begin) + nbegin = to_begin.size + r_data = numeric.empty((n+nbegin,), dtype=a.dtype) + r_mask = numeric.zeros((n+nbegin,), dtype=bool_) + r_data[:nbegin] = to_begin._data + r_mask[:nbegin] = to_begin._mask + r_data[nbegin:] = dd + r_mask[nbegin:] = dm + # + else: + r_data = dd + r_mask = dm + return masked_array(r_data, mask=r_mask) + + + + +#####-------------------------------------------------------------------------- +#---- --- Concatenation helpers --- +#####-------------------------------------------------------------------------- + +class MAxisConcatenator(AxisConcatenator): + """Translate slice objects to concatenation along an axis. + + """ + + def __init__(self, axis=0): + AxisConcatenator.__init__(self, axis, matrix=False) + + def __getitem__(self,key): + if isinstance(key, str): + raise MAError, "Unavailable for masked array." + if type(key) is not tuple: + key = (key,) + objs = [] + scalars = [] + final_dtypedescr = None + for k in range(len(key)): + scalar = False + if type(key[k]) is slice: + step = key[k].step + start = key[k].start + stop = key[k].stop + if start is None: + start = 0 + if step is None: + step = 1 + if type(step) is type(1j): + size = int(abs(step)) + newobj = function_base.linspace(start, stop, num=size) + else: + newobj = numeric.arange(start, stop, step) + elif type(key[k]) is str: + if (key[k] in 'rc'): + self.matrix = True + self.col = (key[k] == 'c') + continue + try: + self.axis = int(key[k]) + continue + except (ValueError, TypeError): + raise ValueError, "Unknown special directive" + elif type(key[k]) in numeric.ScalarType: + newobj = asarray([key[k]]) + scalars.append(k) + scalar = True + else: + newobj = key[k] + objs.append(newobj) + if isinstance(newobj, numeric.ndarray) and not scalar: + if final_dtypedescr is None: + final_dtypedescr = newobj.dtype + elif newobj.dtype > final_dtypedescr: + final_dtypedescr = newobj.dtype + if final_dtypedescr is not None: + for k in scalars: + objs[k] = objs[k].astype(final_dtypedescr) + res = concatenate(tuple(objs),axis=self.axis) + return self._retval(res) + +class mr_class(MAxisConcatenator): + """Translate slice objects to concatenation along the first axis. + + For example: + >>> mr_[array([1,2,3]), 0, 0, array([4,5,6])] + array([1, 2, 3, 0, 0, 4, 5, 6]) + + """ + def __init__(self): + MAxisConcatenator.__init__(self, 0) + +mr_ = mr_class() + +#####-------------------------------------------------------------------------- +#---- --- +#####-------------------------------------------------------------------------- + +def flatnotmasked_edges(a): + """Find the indices of the first and last not masked values in a + 1D masked array. If all values are masked, returns None. + + """ + m = getmask(a) + if m is nomask or not numpy.any(m): + return [0,-1] + unmasked = numeric.flatnonzero(~m) + if len(unmasked) > 0: + return unmasked[[0,-1]] + else: + return None + +def notmasked_edges(a, axis=None): + """Find the indices of the first and last not masked values along + the given axis in a masked array. + + If all values are masked, return None. Otherwise, return a list + of 2 tuples, corresponding to the indices of the first and last + unmasked values respectively. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + """ + a = asarray(a) + if axis is None or a.ndim == 1: + return flatnotmasked_edges(a) + m = getmask(a) + idx = array(numpy.indices(a.shape), mask=nxasarray([m]*a.ndim)) + return [tuple([idx[i].min(axis).compressed() for i in range(a.ndim)]), + tuple([idx[i].max(axis).compressed() for i in range(a.ndim)]),] + +def flatnotmasked_contiguous(a): + """Find contiguous unmasked data in a flattened masked array. + + Return a sorted sequence of slices (start index, end index). + + """ + m = getmask(a) + if m is nomask: + return (a.size, [0,-1]) + unmasked = numeric.flatnonzero(~m) + if len(unmasked) == 0: + return None + result = [] + for k, group in groupby(enumerate(unmasked), lambda (i,x):i-x): + tmp = numpy.fromiter((g[1] for g in group), int_) +# result.append((tmp.size, tuple(tmp[[0,-1]]))) + result.append( slice(tmp[0],tmp[-1]) ) + result.sort() + return result + +def notmasked_contiguous(a, axis=None): + """Find contiguous unmasked data in a masked array along the given + axis. + + Parameters + ---------- + axis : int, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + + Returns + ------- + a sorted sequence of slices (start index, end index). + + Notes + ----- + Only accepts 2D arrays at most. + + """ + a = asarray(a) + nd = a.ndim + if nd > 2: + raise NotImplementedError,"Currently limited to atmost 2D array." + if axis is None or nd == 1: + return flatnotmasked_contiguous(a) + # + result = [] + # + other = (axis+1)%2 + idx = [0,0] + idx[axis] = slice(None,None) + # + for i in range(a.shape[other]): + idx[other] = i + result.append( flatnotmasked_contiguous(a[idx]) ) + return result + +################################################################################ diff --git a/numpy/ma/morestats.py b/numpy/ma/morestats.py new file mode 100644 index 000000000..e7085c240 --- /dev/null +++ b/numpy/ma/morestats.py @@ -0,0 +1,406 @@ +""" +Generic statistics functions, with support to MA. + +:author: Pierre GF Gerard-Marchant +:contact: pierregm_at_uga_edu +:date: $Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $ +:version: $Id: morestats.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + + +import numpy +from numpy import bool_, float_, int_, ndarray, \ + sqrt,\ + arange, empty,\ + r_ +from numpy import array as narray +import numpy.core.numeric as numeric +from numpy.core.numeric import concatenate + +import numpy.ma as MA +from numpy.ma.core import masked, nomask, MaskedArray, masked_array +from numpy.ma.extras import apply_along_axis, dot +from numpy.ma.mstats import trim_both, trimmed_stde, mquantiles, mmedian, stde_median + +from scipy.stats.distributions import norm, beta, t, binom +from scipy.stats.morestats import find_repeats + +__all__ = ['hdquantiles', 'hdmedian', 'hdquantiles_sd', + 'trimmed_mean_ci', 'mjci', 'rank_data'] + + +#####-------------------------------------------------------------------------- +#---- --- Quantiles --- +#####-------------------------------------------------------------------------- +def hdquantiles(data, prob=list([.25,.5,.75]), axis=None, var=False,): + """Computes quantile estimates with the Harrell-Davis method, where the estimates +are calculated as a weighted linear combination of order statistics. + +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + var : {boolean} + Whether to return the variance of the estimate. + +*Returns* + A (p,) array of quantiles (if ``var`` is False), or a (2,p) array of quantiles + and variances (if ``var`` is True), where ``p`` is the number of quantiles. + +:Note: + The function is restricted to 2D arrays. + """ + def _hd_1D(data,prob,var): + "Computes the HD quantiles for a 1D array. Returns nan for invalid data." + xsorted = numpy.squeeze(numpy.sort(data.compressed().view(ndarray))) + # Don't use length here, in case we have a numpy scalar + n = xsorted.size + #......... + hd = empty((2,len(prob)), float_) + if n < 2: + hd.flat = numpy.nan + if var: + return hd + return hd[0] + #......... + v = arange(n+1) / float(n) + betacdf = beta.cdf + for (i,p) in enumerate(prob): + _w = betacdf(v, (n+1)*p, (n+1)*(1-p)) + w = _w[1:] - _w[:-1] + hd_mean = dot(w, xsorted) + hd[0,i] = hd_mean + # + hd[1,i] = dot(w, (xsorted-hd_mean)**2) + # + hd[0, prob == 0] = xsorted[0] + hd[0, prob == 1] = xsorted[-1] + if var: + hd[1, prob == 0] = hd[1, prob == 1] = numpy.nan + return hd + return hd[0] + # Initialization & checks --------- + data = masked_array(data, copy=False, dtype=float_) + p = numpy.array(prob, copy=False, ndmin=1) + # Computes quantiles along axis (or globally) + if (axis is None) or (data.ndim == 1): + result = _hd_1D(data, p, var) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + result = apply_along_axis(_hd_1D, axis, data, p, var) + # + return masked_array(result, mask=numpy.isnan(result)) + +#.............................................................................. +def hdmedian(data, axis=-1, var=False): + """Returns the Harrell-Davis estimate of the median along the given axis. + +*Parameters* : + data: {ndarray} + Data array. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + var : {boolean} + Whether to return the variance of the estimate. + """ + result = hdquantiles(data,[0.5], axis=axis, var=var) + return result.squeeze() + + +#.............................................................................. +def hdquantiles_sd(data, prob=list([.25,.5,.75]), axis=None): + """Computes the standard error of the Harrell-Davis quantile estimates by jackknife. + + +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + +*Note*: + The function is restricted to 2D arrays. + """ + def _hdsd_1D(data,prob): + "Computes the std error for 1D arrays." + xsorted = numpy.sort(data.compressed()) + n = len(xsorted) + #......... + hdsd = empty(len(prob), float_) + if n < 2: + hdsd.flat = numpy.nan + #......... + vv = arange(n) / float(n-1) + betacdf = beta.cdf + # + for (i,p) in enumerate(prob): + _w = betacdf(vv, (n+1)*p, (n+1)*(1-p)) + w = _w[1:] - _w[:-1] + mx_ = numpy.fromiter([dot(w,xsorted[r_[range(0,k), + range(k+1,n)].astype(int_)]) + for k in range(n)], dtype=float_) + mx_var = numpy.array(mx_.var(), copy=False, ndmin=1) * n / float(n-1) + hdsd[i] = float(n-1) * sqrt(numpy.diag(mx_var).diagonal() / float(n)) + return hdsd + # Initialization & checks --------- + data = masked_array(data, copy=False, dtype=float_) + p = numpy.array(prob, copy=False, ndmin=1) + # Computes quantiles along axis (or globally) + if (axis is None): + result = _hdsd_1D(data.compressed(), p) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + result = apply_along_axis(_hdsd_1D, axis, data, p) + # + return masked_array(result, mask=numpy.isnan(result)).ravel() + + +#####-------------------------------------------------------------------------- +#---- --- Confidence intervals --- +#####-------------------------------------------------------------------------- + +def trimmed_mean_ci(data, proportiontocut=0.2, alpha=0.05, axis=None): + """Returns the selected confidence interval of the trimmed mean along the +given axis. + +*Parameters* : + data : {sequence} + Input data. The data is transformed to a masked array + proportiontocut : {float} + Proportion of the data to cut from each side of the data . + As a result, (2*proportiontocut*n) values are actually trimmed. + alpha : {float} + Confidence level of the intervals. + axis : {integer} + Axis along which to cut. If None, uses a flattened version of the input. + """ + data = masked_array(data, copy=False) + trimmed = trim_both(data, proportiontocut=proportiontocut, axis=axis) + tmean = trimmed.mean(axis) + tstde = trimmed_stde(data, proportiontocut=proportiontocut, axis=axis) + df = trimmed.count(axis) - 1 + tppf = t.ppf(1-alpha/2.,df) + return numpy.array((tmean - tppf*tstde, tmean+tppf*tstde)) + +#.............................................................................. +def mjci(data, prob=[0.25,0.5,0.75], axis=None): + """Returns the Maritz-Jarrett estimators of the standard error of selected +experimental quantiles of the data. + +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + """ + def _mjci_1D(data, p): + data = data.compressed() + sorted = numpy.sort(data) + n = data.size + prob = (numpy.array(p) * n + 0.5).astype(int_) + betacdf = beta.cdf + # + mj = empty(len(prob), float_) + x = arange(1,n+1, dtype=float_) / n + y = x - 1./n + for (i,m) in enumerate(prob): + (m1,m2) = (m-1, n-m) + W = betacdf(x,m-1,n-m) - betacdf(y,m-1,n-m) + C1 = numpy.dot(W,sorted) + C2 = numpy.dot(W,sorted**2) + mj[i] = sqrt(C2 - C1**2) + return mj + # + data = masked_array(data, copy=False) + assert data.ndim <= 2, "Array should be 2D at most !" + p = numpy.array(prob, copy=False, ndmin=1) + # Computes quantiles along axis (or globally) + if (axis is None): + return _mjci_1D(data, p) + else: + return apply_along_axis(_mjci_1D, axis, data, p) + +#.............................................................................. +def mquantiles_cimj(data, prob=[0.25,0.50,0.75], alpha=0.05, axis=None): + """Computes the alpha confidence interval for the selected quantiles of the +data, with Maritz-Jarrett estimators. + +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + alpha : {float} + Confidence level of the intervals. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + """ + alpha = min(alpha, 1-alpha) + z = norm.ppf(1-alpha/2.) + xq = mquantiles(data, prob, alphap=0, betap=0, axis=axis) + smj = mjci(data, prob, axis=axis) + return (xq - z * smj, xq + z * smj) + + +#............................................................................. +def median_cihs(data, alpha=0.05, axis=None): + """Computes the alpha-level confidence interval for the median of the data, +following the Hettmasperger-Sheather method. + +*Parameters* : + data : {sequence} + Input data. Masked values are discarded. The input should be 1D only, or + axis should be set to None. + alpha : {float} + Confidence level of the intervals. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + """ + def _cihs_1D(data, alpha): + data = numpy.sort(data.compressed()) + n = len(data) + alpha = min(alpha, 1-alpha) + k = int(binom._ppf(alpha/2., n, 0.5)) + gk = binom.cdf(n-k,n,0.5) - binom.cdf(k-1,n,0.5) + if gk < 1-alpha: + k -= 1 + gk = binom.cdf(n-k,n,0.5) - binom.cdf(k-1,n,0.5) + gkk = binom.cdf(n-k-1,n,0.5) - binom.cdf(k,n,0.5) + I = (gk - 1 + alpha)/(gk - gkk) + lambd = (n-k) * I / float(k + (n-2*k)*I) + lims = (lambd*data[k] + (1-lambd)*data[k-1], + lambd*data[n-k-1] + (1-lambd)*data[n-k]) + return lims + data = masked_array(data, copy=False) + # Computes quantiles along axis (or globally) + if (axis is None): + result = _cihs_1D(data.compressed(), p, var) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + result = apply_along_axis(_cihs_1D, axis, data, alpha) + # + return result + +#.............................................................................. +def compare_medians_ms(group_1, group_2, axis=None): + """Compares the medians from two independent groups along the given axis. + +The comparison is performed using the McKean-Schrader estimate of the standard +error of the medians. + +*Parameters* : + group_1 : {sequence} + First dataset. + group_2 : {sequence} + Second dataset. + axis : {integer} + Axis along which the medians are estimated. If None, the arrays are flattened. + +*Returns* : + A (p,) array of comparison values. + + """ + (med_1, med_2) = (mmedian(group_1, axis=axis), mmedian(group_2, axis=axis)) + (std_1, std_2) = (stde_median(group_1, axis=axis), + stde_median(group_2, axis=axis)) + W = abs(med_1 - med_2) / sqrt(std_1**2 + std_2**2) + return 1 - norm.cdf(W) + + +#####-------------------------------------------------------------------------- +#---- --- Ranking --- +#####-------------------------------------------------------------------------- + +#.............................................................................. +def rank_data(data, axis=None, use_missing=False): + """Returns the rank (also known as order statistics) of each data point +along the given axis. + +If some values are tied, their rank is averaged. +If some values are masked, their rank is set to 0 if use_missing is False, or +set to the average rank of the unmasked values if use_missing is True. + +*Parameters* : + data : {sequence} + Input data. The data is transformed to a masked array + axis : {integer} + Axis along which to perform the ranking. If None, the array is first + flattened. An exception is raised if the axis is specified for arrays + with a dimension larger than 2 + use_missing : {boolean} + Whether the masked values have a rank of 0 (False) or equal to the + average rank of the unmasked values (True). + """ + # + def _rank1d(data, use_missing=False): + n = data.count() + rk = numpy.empty(data.size, dtype=float_) + idx = data.argsort() + rk[idx[:n]] = numpy.arange(1,n+1) + # + if use_missing: + rk[idx[n:]] = (n+1)/2. + else: + rk[idx[n:]] = 0 + # + repeats = find_repeats(data) + for r in repeats[0]: + condition = (data==r).filled(False) + rk[condition] = rk[condition].mean() + return rk + # + data = masked_array(data, copy=False) + if axis is None: + if data.ndim > 1: + return _rank1d(data.ravel(), use_missing).reshape(data.shape) + else: + return _rank1d(data, use_missing) + else: + return apply_along_axis(_rank1d, axis, data, use_missing) + +############################################################################### +if __name__ == '__main__': + + if 0: + from numpy.ma.testutils import assert_almost_equal + data = [0.706560797,0.727229578,0.990399276,0.927065621,0.158953014, + 0.887764025,0.239407086,0.349638551,0.972791145,0.149789972, + 0.936947700,0.132359948,0.046041972,0.641675031,0.945530547, + 0.224218684,0.771450991,0.820257774,0.336458052,0.589113496, + 0.509736129,0.696838829,0.491323573,0.622767425,0.775189248, + 0.641461450,0.118455200,0.773029450,0.319280007,0.752229111, + 0.047841438,0.466295911,0.583850781,0.840581845,0.550086491, + 0.466470062,0.504765074,0.226855960,0.362641207,0.891620942, + 0.127898691,0.490094097,0.044882048,0.041441695,0.317976349, + 0.504135618,0.567353033,0.434617473,0.636243375,0.231803616, + 0.230154113,0.160011327,0.819464108,0.854706985,0.438809221, + 0.487427267,0.786907310,0.408367937,0.405534192,0.250444460, + 0.995309248,0.144389588,0.739947527,0.953543606,0.680051621, + 0.388382017,0.863530727,0.006514031,0.118007779,0.924024803, + 0.384236354,0.893687694,0.626534881,0.473051932,0.750134705, + 0.241843555,0.432947602,0.689538104,0.136934797,0.150206859, + 0.474335206,0.907775349,0.525869295,0.189184225,0.854284286, + 0.831089744,0.251637345,0.587038213,0.254475554,0.237781276, + 0.827928620,0.480283781,0.594514455,0.213641488,0.024194386, + 0.536668589,0.699497811,0.892804071,0.093835427,0.731107772] + # + assert_almost_equal(hdquantiles(data,[0., 1.]), + [0.006514031, 0.995309248]) + hdq = hdquantiles(data,[0.25, 0.5, 0.75]) + assert_almost_equal(hdq, [0.253210762, 0.512847491, 0.762232442,]) + hdq = hdquantiles_sd(data,[0.25, 0.5, 0.75]) + assert_almost_equal(hdq, [0.03786954, 0.03805389, 0.03800152,], 4) + # + data = numpy.array(data).reshape(10,10) + hdq = hdquantiles(data,[0.25,0.5,0.75],axis=0) diff --git a/numpy/ma/mrecords.py b/numpy/ma/mrecords.py new file mode 100644 index 000000000..4c0bdfa76 --- /dev/null +++ b/numpy/ma/mrecords.py @@ -0,0 +1,791 @@ +"""mrecords + +Defines the equivalent of recarrays for maskedarray. +Masked arrays already support named fields, but masking works only by records. +By comparison, mrecarrays support masking individual fields. + +:author: Pierre Gerard-Marchant +""" +#TODO: We should make sure that no field is called '_mask','mask','_fieldmask', +#TODO: ...or whatever restricted keywords. +#TODO: An idea would be to no bother in the first place, and then rename the +#TODO: invalid fields with a trailing underscore... +#TODO: Maybe we could just overload the parser function ? + + +__author__ = "Pierre GF Gerard-Marchant" + +import sys +import types + +import numpy +from numpy import bool_, complex_, float_, int_, str_, object_, dtype +from numpy import array as narray +import numpy.core.numeric as numeric +import numpy.core.numerictypes as ntypes +from numpy.core.defchararray import chararray +from numpy.core.records import find_duplicate, format_parser, record, recarray +from numpy.core.records import fromarrays as recfromarrays +from numpy.core.records import fromrecords as recfromrecords + +ndarray = numeric.ndarray +_byteorderconv = numpy.core.records._byteorderconv +_typestr = ntypes._typestr + +import numpy.ma +from numpy.ma import MAError, MaskedArray, masked, nomask, masked_array,\ + make_mask, mask_or, getdata, getmask, getmaskarray, filled +from numpy.ma.core import default_fill_value, masked_print_option +_check_fill_value = numpy.ma.core._check_fill_value + +import warnings + +__all__ = ['MaskedRecords','mrecarray', + 'fromarrays','fromrecords','fromtextfile','addfield', + ] + +reserved_fields = ['_data','_mask','_fieldmask', 'dtype'] + +def _getformats(data): + "Returns the formats of each array of arraylist as a comma-separated string." + if hasattr(data,'dtype'): + return ",".join([desc[1] for desc in data.dtype.descr]) + + formats = '' + for obj in data: + obj = numeric.asarray(obj) +# if not isinstance(obj, ndarray): +## if not isinstance(obj, ndarray): +# raise ValueError, "item in the array list must be an ndarray." + formats += _typestr[obj.dtype.type] + if issubclass(obj.dtype.type, ntypes.flexible): + formats += `obj.itemsize` + formats += ',' + return formats[:-1] + +def _checknames(descr, names=None): + """Checks that the field names of the descriptor ``descr`` are not some +reserved keywords. If this is the case, a default 'f%i' is substituted. +If the argument `names` is not None, updates the field names to valid names. + """ + ndescr = len(descr) + default_names = ['f%i' % i for i in range(ndescr)] + if names is None: + new_names = default_names + else: + if isinstance(names, (tuple, list)): + new_names = names + elif isinstance(names, str): + new_names = names.split(',') + else: + raise NameError, "illegal input names %s" % `names` + nnames = len(new_names) + if nnames < ndescr: + new_names += default_names[nnames:] + ndescr = [] + for (n, d, t) in zip(new_names, default_names, descr.descr): + if n in reserved_fields: + if t[0] in reserved_fields: + ndescr.append((d,t[1])) + else: + ndescr.append(t) + else: + ndescr.append((n,t[1])) + return numeric.dtype(ndescr) + + +def _get_fieldmask(self): + mdescr = [(n,'|b1') for n in self.dtype.names] + fdmask = numpy.empty(self.shape, dtype=mdescr) + fdmask.flat = tuple([False]*len(mdescr)) + return fdmask + + +class MaskedRecords(MaskedArray, object): + """ + +*IVariables*: + _data : {recarray} + Underlying data, as a record array. + _mask : {boolean array} + Mask of the records. A record is masked when all its fields are masked. + _fieldmask : {boolean recarray} + Record array of booleans, setting the mask of each individual field of each record. + _fill_value : {record} + Filling values for each field. + """ + _defaultfieldmask = nomask + _defaulthardmask = False + #............................................ + def __new__(cls, shape, dtype=None, buf=None, offset=0, strides=None, + formats=None, names=None, titles=None, + byteorder=None, aligned=False, + mask=nomask, hard_mask=False, fill_value=None, keep_mask=True, + copy=False, + **options): + # + self = recarray.__new__(cls, shape, dtype=dtype, buf=buf, offset=offset, + strides=strides, formats=formats, + byteorder=byteorder, aligned=aligned,) +# self = self.view(cls) + # + mdtype = [(k,'|b1') for (k,_) in self.dtype.descr] + if mask is nomask or not numpy.size(mask): + if not keep_mask: + self._fieldmask = tuple([False]*len(mdtype)) + else: + mask = narray(mask, copy=copy) + if mask.shape != self.shape: + (nd, nm) = (self.size, mask.size) + if nm == 1: + mask = numpy.resize(mask, self.shape) + elif nm == nd: + mask = numpy.reshape(mask, self.shape) + else: + msg = "Mask and data not compatible: data size is %i, "+\ + "mask size is %i." + raise MAError(msg % (nd, nm)) + copy = True + if not keep_mask: + self.__setmask__(mask) + self._sharedmask = True + else: + if mask.dtype == mdtype: + _fieldmask = mask + else: + _fieldmask = narray([tuple([m]*len(mdtype)) for m in mask], + dtype=mdtype) + self._fieldmask = _fieldmask + return self + #...................................................... + def __array_finalize__(self,obj): + # Make sure we have a _fieldmask by default .. + _fieldmask = getattr(obj, '_fieldmask', None) + if _fieldmask is None: + mdescr = [(n,'|b1') for (n,_) in self.dtype.descr] + _fieldmask = numpy.empty(self.shape, dtype=mdescr).view(recarray) + _fieldmask.flat = tuple([False]*len(mdescr)) + # Update some of the attributes + attrdict = dict(_fieldmask=_fieldmask, + _hardmask=getattr(obj,'_hardmask',False), + _fill_value=getattr(obj,'_fill_value',None)) + self.__dict__.update(attrdict) + # Finalize as a regular maskedarray ..... + # Note: we can't call MaskedArray.__array_finalize__, it chokes pickling + self._update_from(obj) + # Update special attributes ... + self._basedict = getattr(obj, '_basedict', getattr(obj, '__dict__', None)) + if self._basedict is not None: + self.__dict__.update(self._basedict) + return + #...................................................... + def _getdata(self): + "Returns the data as a recarray." + return self.view(recarray) + _data = property(fget=_getdata) + #...................................................... + def __setmask__(self, mask): + "Sets the mask and update the fieldmask." + names = self.dtype.names + fmask = self.__dict__['_fieldmask'] + newmask = make_mask(mask, copy=False) + if names is not None: + if self._hardmask: + for n in names: + fmask[n].__ior__(newmask) + else: + for n in names: + fmask[n].flat = newmask + return + _setmask = __setmask__ + # + def _getmask(self): + """Return the mask of the mrecord. + A record is masked when all the fields are masked. + + """ + if self.size > 1: + return self._fieldmask.view((bool_, len(self.dtype))).all(1) + else: + return self._fieldmask.view((bool_, len(self.dtype))).all() + _mask = property(fget=_getmask, fset=_setmask) + #...................................................... + def get_fill_value(self): + """Return the filling value. + + """ + if self._fill_value is None: + ddtype = self.dtype + fillval = _check_fill_value(None, ddtype) + self._fill_value = narray(tuple(fillval), dtype=ddtype) + return self._fill_value + + def set_fill_value(self, value=None): + """Set the filling value to value. + + If value is None, use a default based on the data type. + + """ + ddtype = self.dtype + fillval = _check_fill_value(value, ddtype) + self._fill_value = narray(tuple(fillval), dtype=ddtype) + + fill_value = property(fget=get_fill_value, fset=set_fill_value, + doc="Filling value.") + #...................................................... + def __len__(self): + "Returns the length" + # We have more than one record + if self.ndim: + return len(self._data) + # We have only one record: return the nb of fields + return len(self.dtype) + #...................................................... + def __getattribute__(self, attr): + "Returns the given attribute." + try: + # Returns a generic attribute + return object.__getattribute__(self,attr) + except AttributeError: + # OK, so attr must be a field name + pass + # Get the list of fields ...... + _names = self.dtype.names + if attr in _names: + _data = self._data + _mask = self._fieldmask +# obj = masked_array(_data.__getattribute__(attr), copy=False, +# mask=_mask.__getattribute__(attr)) + # Use a view in order to avoid the copy of the mask in MaskedArray.__new__ + obj = narray(_data.__getattribute__(attr), copy=False).view(MaskedArray) + obj._mask = _mask.__getattribute__(attr) + if not obj.ndim and obj._mask: + return masked + return obj + raise AttributeError,"No attribute '%s' !" % attr + + def __setattr__(self, attr, val): + "Sets the attribute attr to the value val." + newattr = attr not in self.__dict__ + try: + # Is attr a generic attribute ? + ret = object.__setattr__(self, attr, val) + except: + # Not a generic attribute: exit if it's not a valid field + fielddict = self.dtype.names or {} + if attr not in fielddict: + exctype, value = sys.exc_info()[:2] + raise exctype, value + else: + # Get the list of names ...... + _names = self.dtype.names + if _names is None: + _names = [] + else: + _names = list(_names) + _names += ['_mask','mask'] + # Check the attribute + if attr not in _names: + return ret + if newattr: # We just added this one + try: # or this setattr worked on an internal + # attribute. + object.__delattr__(self, attr) + except: + return ret + # Case #1.: Basic field ............ + base_fmask = self._fieldmask + _names = self.dtype.names + if attr in _names: + if val is masked: + fval = self.fill_value[attr] + mval = True + else: + fval = filled(val) + mval = getmaskarray(val) + if self._hardmask: + mval = mask_or(mval, base_fmask.__getattr__(attr)) + self._data.__setattr__(attr, fval) + base_fmask.__setattr__(attr, mval) + return + elif attr == '_mask': + #FIXME: We should check for self._harmask over there. +# if self._hardmask: +# val = make_mask(val) +# if val is not nomask: +## mval = getmaskarray(val) +# for k in _names: +# m = mask_or(val, base_fmask.__getattr__(k)) +# base_fmask.__setattr__(k, m) + self.__setmask__(val) + return + #............................................ + def __getitem__(self, indx): + """Returns all the fields sharing the same fieldname base. +The fieldname base is either `_data` or `_mask`.""" + _localdict = self.__dict__ + _fieldmask = _localdict['_fieldmask'] + _data = self._data + # We want a field ........ + if isinstance(indx, basestring): + obj = _data[indx].view(MaskedArray) + obj._set_mask(_fieldmask[indx]) + # Force to nomask if the mask is empty + if not obj._mask.any(): + obj._mask = nomask + # Force to masked if the mask is True + if not obj.ndim and obj._mask: + return masked + return obj + # We want some elements .. + # First, the data ........ + obj = narray(_data[indx], copy=False).view(mrecarray) + obj._fieldmask = narray(_fieldmask[indx], copy=False).view(recarray) + return obj + #.... + def __setitem__(self, indx, value): + "Sets the given record to value." + MaskedArray.__setitem__(self, indx, value) + #............................................ + def __setslice__(self, i, j, value): + "Sets the slice described by [i,j] to `value`." + _localdict = self.__dict__ + d = self._data + m = _localdict['_fieldmask'] + names = self.dtype.names + if value is masked: + for n in names: + m[i:j][n] = True + elif not self._hardmask: + fval = filled(value) + mval = getmaskarray(value) + for n in names: + d[n][i:j] = fval + m[n][i:j] = mval + else: + mindx = getmaskarray(self)[i:j] + dval = numeric.asarray(value) + valmask = getmask(value) + if valmask is nomask: + for n in names: + mval = mask_or(m[n][i:j], valmask) + d[n][i:j][~mval] = value + elif valmask.size > 1: + for n in names: + mval = mask_or(m[n][i:j], valmask) + d[n][i:j][~mval] = dval[~mval] + m[n][i:j] = mask_or(m[n][i:j], mval) + self._fieldmask = m + #...................................................... + def __str__(self): + "Calculates the string representation." + if self.size > 1: + mstr = ["(%s)" % ",".join([str(i) for i in s]) + for s in zip(*[getattr(self,f) for f in self.dtype.names])] + return "[%s]" % ", ".join(mstr) + else: + mstr = ["%s" % ",".join([str(i) for i in s]) + for s in zip([getattr(self,f) for f in self.dtype.names])] + return "(%s)" % ", ".join(mstr) + # + def __repr__(self): + "Calculates the repr representation." + _names = self.dtype.names + fmt = "%%%is : %%s" % (max([len(n) for n in _names])+4,) + reprstr = [fmt % (f,getattr(self,f)) for f in self.dtype.names] + reprstr.insert(0,'masked_records(') + reprstr.extend([fmt % (' fill_value', self.fill_value), + ' )']) + return str("\n".join(reprstr)) + #...................................................... + def view(self, obj): + """Returns a view of the mrecarray.""" + try: + if issubclass(obj, ndarray): + return ndarray.view(self, obj) + except TypeError: + pass + dtype = numeric.dtype(obj) + if dtype.fields is None: + return self.__array__().view(dtype) + return ndarray.view(self, obj) + #...................................................... + def filled(self, fill_value=None): + """Returns an array of the same class as the _data part, where masked + values are filled with fill_value. + If fill_value is None, self.fill_value is used instead. + + Subclassing is preserved. + + """ + _localdict = self.__dict__ + d = self._data + fm = _localdict['_fieldmask'] + if not numeric.asarray(fm, dtype=bool_).any(): + return d + # + if fill_value is None: + value = _check_fill_value(_localdict['_fill_value'],self.dtype) + else: + value = fill_value + if numeric.size(value) == 1: + value = [value,] * len(self.dtype) + # + if self is masked: + result = numeric.asanyarray(value) + else: + result = d.copy() + for (n, v) in zip(d.dtype.names, value): + numpy.putmask(numeric.asarray(result[n]), + numeric.asarray(fm[n]), v) + return result + #...................................................... + def harden_mask(self): + "Forces the mask to hard" + self._hardmask = True + def soften_mask(self): + "Forces the mask to soft" + self._hardmask = False + #...................................................... + def copy(self): + """Returns a copy of the masked record.""" + _localdict = self.__dict__ + copied = self._data.copy().view(type(self)) + copied._fieldmask = self._fieldmask.copy() + return copied + #...................................................... + def tolist(self, fill_value=None): + """Copy the data portion of the array to a hierarchical python + list and returns that list. + + Data items are converted to the nearest compatible Python + type. Masked values are converted to fill_value. If + fill_value is None, the corresponding entries in the output + list will be ``None``. + + """ + if fill_value is not None: + return self.filled(fill_value).tolist() + result = narray(self.filled().tolist(), dtype=object) + mask = narray(self._fieldmask.tolist()) + result[mask] = None + return [tuple(r) for r in result] + #-------------------------------------------- + # Pickling + def __getstate__(self): + """Return the internal state of the masked array, for pickling purposes. + + """ + state = (1, + self.shape, + self.dtype, + self.flags.fnc, + self._data.tostring(), + self._fieldmask.tostring(), + self._fill_value, + ) + return state + # + def __setstate__(self, state): + """Restore the internal state of the masked array, for pickling purposes. + ``state`` is typically the output of the ``__getstate__`` output, and is a + 5-tuple: + + - class name + - a tuple giving the shape of the data + - a typecode for the data + - a binary string for the data + - a binary string for the mask. + + """ + (ver, shp, typ, isf, raw, msk, flv) = state + ndarray.__setstate__(self, (shp, typ, isf, raw)) + mdtype = dtype([(k,bool_) for (k,_) in self.dtype.descr]) + self._fieldmask.__setstate__((shp, mdtype, isf, msk)) + self.fill_value = flv + # + def __reduce__(self): + """Return a 3-tuple for pickling a MaskedArray. + + """ + return (_mrreconstruct, + (self.__class__, self._baseclass, (0,), 'b', ), + self.__getstate__()) + +def _mrreconstruct(subtype, baseclass, baseshape, basetype,): + """Internal function that builds a new MaskedArray from the + information stored in a pickle. + + """ + _data = ndarray.__new__(baseclass, baseshape, basetype).view(subtype) +# _data._mask = ndarray.__new__(ndarray, baseshape, 'b1') +# return _data + _mask = ndarray.__new__(ndarray, baseshape, 'b1') + return subtype.__new__(subtype, _data, mask=_mask, dtype=basetype,) + + +mrecarray = MaskedRecords + +#####--------------------------------------------------------------------------- +#---- --- Constructors --- +#####--------------------------------------------------------------------------- + +def fromarrays(arraylist, dtype=None, shape=None, formats=None, + names=None, titles=None, aligned=False, byteorder=None, + fill_value=None): + """Creates a mrecarray from a (flat) list of masked arrays. + + Parameters + ---------- + arraylist : sequence + A list of (masked) arrays. Each element of the sequence is first converted + to a masked array if needed. If a 2D array is passed as argument, it is + processed line by line + dtype : numeric.dtype + Data type descriptor. + shape : integer + Number of records. If None, shape is defined from the shape of the + first array in the list. + formats : sequence + Sequence of formats for each individual field. If None, the formats will + be autodetected by inspecting the fields and selecting the highest dtype + possible. + names : sequence + Sequence of the names of each field. + titles : sequence + (Description to write) + aligned : boolean + (Description to write, not used anyway) + byteorder: boolean + (Description to write, not used anyway) + fill_value : sequence + Sequence of data to be used as filling values. + + Notes + ----- + Lists of tuples should be preferred over lists of lists for faster processing. + """ + datalist = [getdata(x) for x in arraylist] + masklist = [getmaskarray(x) for x in arraylist] + _array = recfromarrays(datalist, + dtype=dtype, shape=shape, formats=formats, + names=names, titles=titles, aligned=aligned, + byteorder=byteorder).view(mrecarray) + _array._fieldmask[:] = zip(*masklist) + if fill_value is not None: + _array.fill_value = fill_value + return _array + + +#.............................................................................. +def fromrecords(reclist, dtype=None, shape=None, formats=None, names=None, + titles=None, aligned=False, byteorder=None, + fill_value=None, mask=nomask): + """Creates a MaskedRecords from a list of records. + + Parameters + ---------- + arraylist : sequence + A list of (masked) arrays. Each element of the sequence is first converted + to a masked array if needed. If a 2D array is passed as argument, it is + processed line by line + dtype : numeric.dtype + Data type descriptor. + shape : integer + Number of records. If None, ``shape`` is defined from the shape of the + first array in the list. + formats : sequence + Sequence of formats for each individual field. If None, the formats will + be autodetected by inspecting the fields and selecting the highest dtype + possible. + names : sequence + Sequence of the names of each field. + titles : sequence + (Description to write) + aligned : boolean + (Description to write, not used anyway) + byteorder: boolean + (Description to write, not used anyway) + fill_value : sequence + Sequence of data to be used as filling values. + mask : sequence or boolean. + External mask to apply on the data. + +*Notes*: + Lists of tuples should be preferred over lists of lists for faster processing. + """ + # Grab the initial _fieldmask, if needed: + _fieldmask = getattr(reclist, '_fieldmask', None) + # Get the list of records..... + nfields = len(reclist[0]) + if isinstance(reclist, ndarray): + # Make sure we don't have some hidden mask + if isinstance(reclist,MaskedArray): + reclist = reclist.filled().view(ndarray) + # Grab the initial dtype, just in case + if dtype is None: + dtype = reclist.dtype + reclist = reclist.tolist() + mrec = recfromrecords(reclist, dtype=dtype, shape=shape, formats=formats, + names=names, titles=titles, + aligned=aligned, byteorder=byteorder).view(mrecarray) + # Set the fill_value if needed + if fill_value is not None: + mrec.fill_value = fill_value + # Now, let's deal w/ the mask + if mask is not nomask: + mask = narray(mask, copy=False) + maskrecordlength = len(mask.dtype) + if maskrecordlength: + mrec._fieldmask.flat = mask + elif len(mask.shape) == 2: + mrec._fieldmask.flat = [tuple(m) for m in mask] + else: + mrec._mask = mask + if _fieldmask is not None: + mrec._fieldmask[:] = _fieldmask + return mrec + +def _guessvartypes(arr): + """Tries to guess the dtypes of the str_ ndarray `arr`, by testing element-wise +conversion. Returns a list of dtypes. +The array is first converted to ndarray. If the array is 2D, the test is performed +on the first line. An exception is raised if the file is 3D or more. + """ + vartypes = [] + arr = numeric.asarray(arr) + if len(arr.shape) == 2 : + arr = arr[0] + elif len(arr.shape) > 2: + raise ValueError, "The array should be 2D at most!" + # Start the conversion loop ....... + for f in arr: + try: + val = int(f) + except ValueError: + try: + val = float(f) + except ValueError: + try: + val = complex(f) + except ValueError: + vartypes.append(arr.dtype) + else: + vartypes.append(complex_) + else: + vartypes.append(float_) + else: + vartypes.append(int_) + return vartypes + +def openfile(fname): + "Opens the file handle of file `fname`" + # A file handle ................... + if hasattr(fname, 'readline'): + return fname + # Try to open the file and guess its type + try: + f = open(fname) + except IOError: + raise IOError, "No such file: '%s'" % fname + if f.readline()[:2] != "\\x": + f.seek(0,0) + return f + raise NotImplementedError, "Wow, binary file" + + +def fromtextfile(fname, delimitor=None, commentchar='#', missingchar='', + varnames=None, vartypes=None): + """Creates a mrecarray from data stored in the file `filename`. + +*Parameters* : + filename : {file name/handle} + Handle of an opened file. + delimitor : {string} + Alphanumeric character used to separate columns in the file. + If None, any (group of) white spacestring(s) will be used. + commentchar : {string} + Alphanumeric character used to mark the start of a comment. + missingchar` : {string} + String indicating missing data, and used to create the masks. + varnames : {sequence} + Sequence of the variable names. If None, a list will be created from + the first non empty line of the file. + vartypes : {sequence} + Sequence of the variables dtypes. If None, it will be estimated from + the first non-commented line. + + + Ultra simple: the varnames are in the header, one line""" + # Try to open the file ...................... + f = openfile(fname) + # Get the first non-empty line as the varnames + while True: + line = f.readline() + firstline = line[:line.find(commentchar)].strip() + _varnames = firstline.split(delimitor) + if len(_varnames) > 1: + break + if varnames is None: + varnames = _varnames + # Get the data .............................. + _variables = masked_array([line.strip().split(delimitor) for line in f + if line[0] != commentchar and len(line) > 1]) + (_, nfields) = _variables.shape + # Try to guess the dtype .................... + if vartypes is None: + vartypes = _guessvartypes(_variables[0]) + else: + vartypes = [numeric.dtype(v) for v in vartypes] + if len(vartypes) != nfields: + msg = "Attempting to %i dtypes for %i fields!" + msg += " Reverting to default." + warnings.warn(msg % (len(vartypes), nfields)) + vartypes = _guessvartypes(_variables[0]) + # Construct the descriptor .................. + mdescr = [(n,f) for (n,f) in zip(varnames, vartypes)] + # Get the data and the mask ................. + # We just need a list of masked_arrays. It's easier to create it like that: + _mask = (_variables.T == missingchar) + _datalist = [masked_array(a,mask=m,dtype=t) + for (a,m,t) in zip(_variables.T, _mask, vartypes)] + return fromarrays(_datalist, dtype=mdescr) + +#.................................................................... +def addfield(mrecord, newfield, newfieldname=None): + """Adds a new field to the masked record array, using `newfield` as data +and `newfieldname` as name. If `newfieldname` is None, the new field name is +set to 'fi', where `i` is the number of existing fields. + """ + _data = mrecord._data + _mask = mrecord._fieldmask + if newfieldname is None or newfieldname in reserved_fields: + newfieldname = 'f%i' % len(_data.dtype) + newfield = masked_array(newfield) + # Get the new data ............ + # Create a new empty recarray + newdtype = numeric.dtype(_data.dtype.descr + \ + [(newfieldname, newfield.dtype)]) + newdata = recarray(_data.shape, newdtype) + # Add the exisintg field + [newdata.setfield(_data.getfield(*f),*f) + for f in _data.dtype.fields.values()] + # Add the new field + newdata.setfield(newfield._data, *newdata.dtype.fields[newfieldname]) + newdata = newdata.view(MaskedRecords) + # Get the new mask ............. + # Create a new empty recarray + newmdtype = numeric.dtype([(n,bool_) for n in newdtype.names]) + newmask = recarray(_data.shape, newmdtype) + # Add the old masks + [newmask.setfield(_mask.getfield(*f),*f) + for f in _mask.dtype.fields.values()] + # Add the mask of the new field + newmask.setfield(getmaskarray(newfield), + *newmask.dtype.fields[newfieldname]) + newdata._fieldmask = newmask + return newdata + +############################################################################### + + diff --git a/numpy/ma/mstats.py b/numpy/ma/mstats.py new file mode 100644 index 000000000..8daa49c4b --- /dev/null +++ b/numpy/ma/mstats.py @@ -0,0 +1,433 @@ +""" +Generic statistics functions, with support to MA. + +:author: Pierre GF Gerard-Marchant +:contact: pierregm_at_uga_edu +:date: $Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $ +:version: $Id: mstats.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + + +import numpy +from numpy import bool_, float_, int_, \ + sqrt +from numpy import array as narray +import numpy.core.numeric as numeric +from numpy.core.numeric import concatenate + +import numpy.ma +from numpy.ma.core import masked, nomask, MaskedArray, masked_array +from numpy.ma.extras import apply_along_axis, dot + +__all__ = ['cov','meppf','plotting_positions','meppf','mmedian','mquantiles', + 'stde_median','trim_tail','trim_both','trimmed_mean','trimmed_stde', + 'winsorize'] + +#####-------------------------------------------------------------------------- +#---- -- Trimming --- +#####-------------------------------------------------------------------------- + +def winsorize(data, alpha=0.2): + """Returns a Winsorized version of the input array. + +The (alpha/2.) lowest values are set to the (alpha/2.)th percentile, and +the (alpha/2.) highest values are set to the (1-alpha/2.)th percentile +Masked values are skipped. + +*Parameters*: + data : {ndarray} + Input data to Winsorize. The data is first flattened. + alpha : {float}, optional + Percentage of total Winsorization : alpha/2. on the left, alpha/2. on the right + """ + data = masked_array(data, copy=False).ravel() + idxsort = data.argsort() + (nsize, ncounts) = (data.size, data.count()) + ntrim = int(alpha * ncounts) + (xmin,xmax) = data[idxsort[[ntrim, ncounts-nsize-ntrim-1]]] + return masked_array(numpy.clip(data, xmin, xmax), mask=data._mask) + +#.............................................................................. +def trim_both(data, proportiontocut=0.2, axis=None): + """Trims the data by masking the int(trim*n) smallest and int(trim*n) largest +values of data along the given axis, where n is the number of unmasked values. + +*Parameters*: + data : {ndarray} + Data to trim. + proportiontocut : {float} + Percentage of trimming. If n is the number of unmasked values before trimming, + the number of values after trimming is (1-2*trim)*n. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + #................... + def _trim_1D(data, trim): + "Private function: return a trimmed 1D array." + nsize = data.size + ncounts = data.count() + ntrim = int(trim * ncounts) + idxsort = data.argsort() + data[idxsort[:ntrim]] = masked + data[idxsort[ncounts-nsize-ntrim:]] = masked + return data + #................... + data = masked_array(data, copy=False, subok=True) + data.unshare_mask() + if (axis is None): + return _trim_1D(data.ravel(), proportiontocut) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_trim_1D, axis, data, proportiontocut) + +#.............................................................................. +def trim_tail(data, proportiontocut=0.2, tail='left', axis=None): + """Trims the data by masking int(trim*n) values from ONE tail of the data +along the given axis, where n is the number of unmasked values. + +*Parameters*: + data : {ndarray} + Data to trim. + proportiontocut : {float} + Percentage of trimming. If n is the number of unmasked values before trimming, + the number of values after trimming is (1-trim)*n. + tail : {string} + Trimming direction, in ('left', 'right'). If left, the proportiontocut + lowest values are set to the corresponding percentile. If right, the + proportiontocut highest values are used instead. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + #................... + def _trim_1D(data, trim, left): + "Private function: return a trimmed 1D array." + nsize = data.size + ncounts = data.count() + ntrim = int(trim * ncounts) + idxsort = data.argsort() + if left: + data[idxsort[:ntrim]] = masked + else: + data[idxsort[ncounts-nsize-ntrim:]] = masked + return data + #................... + data = masked_array(data, copy=False, subok=True) + data.unshare_mask() + # + if not isinstance(tail, str): + raise TypeError("The tail argument should be in ('left','right')") + tail = tail.lower()[0] + if tail == 'l': + left = True + elif tail == 'r': + left=False + else: + raise ValueError("The tail argument should be in ('left','right')") + # + if (axis is None): + return _trim_1D(data.ravel(), proportiontocut, left) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_trim_1D, axis, data, proportiontocut, left) + +#.............................................................................. +def trimmed_mean(data, proportiontocut=0.2, axis=None): + """Returns the trimmed mean of the data along the given axis. Trimming is +performed on both ends of the distribution. + +*Parameters*: + data : {ndarray} + Data to trim. + proportiontocut : {float} + Proportion of the data to cut from each side of the data . + As a result, (2*proportiontocut*n) values are actually trimmed. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + return trim_both(data, proportiontocut=proportiontocut, axis=axis).mean(axis=axis) + +#.............................................................................. +def trimmed_stde(data, proportiontocut=0.2, axis=None): + """Returns the standard error of the trimmed mean for the input data, +along the given axis. Trimming is performed on both ends of the distribution. + +*Parameters*: + data : {ndarray} + Data to trim. + proportiontocut : {float} + Proportion of the data to cut from each side of the data . + As a result, (2*proportiontocut*n) values are actually trimmed. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + #........................ + def _trimmed_stde_1D(data, trim=0.2): + "Returns the standard error of the trimmed mean for a 1D input data." + winsorized = winsorize(data) + nsize = winsorized.count() + winstd = winsorized.stdu() + return winstd / ((1-2*trim) * numpy.sqrt(nsize)) + #........................ + data = masked_array(data, copy=False, subok=True) + data.unshare_mask() + if (axis is None): + return _trimmed_stde_1D(data.ravel(), proportiontocut) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_trimmed_stde_1D, axis, data, proportiontocut) + +#............................................................................. +def stde_median(data, axis=None): + """Returns the McKean-Schrader estimate of the standard error of the sample +median along the given axis. + + +*Parameters*: + data : {ndarray} + Data to trim. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + def _stdemed_1D(data): + sorted = numpy.sort(data.compressed()) + n = len(sorted) + z = 2.5758293035489004 + k = int(round((n+1)/2. - z * sqrt(n/4.),0)) + return ((sorted[n-k] - sorted[k-1])/(2.*z)) + # + data = masked_array(data, copy=False, subok=True) + if (axis is None): + return _stdemed_1D(data) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_stdemed_1D, axis, data) + + +#####-------------------------------------------------------------------------- +#---- --- Quantiles --- +#####-------------------------------------------------------------------------- + + +def mquantiles(data, prob=list([.25,.5,.75]), alphap=.4, betap=.4, axis=None): + """Computes empirical quantiles for a *1xN* data array. +Samples quantile are defined by: +*Q(p) = (1-g).x[i] +g.x[i+1]* +where *x[j]* is the jth order statistic, +with *i = (floor(n*p+m))*, *m=alpha+p*(1-alpha-beta)* and *g = n*p + m - i)*. + +Typical values of (alpha,beta) are: + + - (0,1) : *p(k) = k/n* : linear interpolation of cdf (R, type 4) + - (.5,.5) : *p(k) = (k+1/2.)/n* : piecewise linear function (R, type 5) + - (0,0) : *p(k) = k/(n+1)* : (R type 6) + - (1,1) : *p(k) = (k-1)/(n-1)*. In this case, p(k) = mode[F(x[k])]. + That's R default (R type 7) + - (1/3,1/3): *p(k) = (k-1/3)/(n+1/3)*. Then p(k) ~ median[F(x[k])]. + The resulting quantile estimates are approximately median-unbiased + regardless of the distribution of x. (R type 8) + - (3/8,3/8): *p(k) = (k-3/8)/(n+1/4)*. Blom. + The resulting quantile estimates are approximately unbiased + if x is normally distributed (R type 9) + - (.4,.4) : approximately quantile unbiased (Cunnane) + - (.35,.35): APL, used with PWM + +*Parameters*: + x : {sequence} + Input data, as a sequence or array of dimension at most 2. + prob : {sequence} + List of quantiles to compute. + alpha : {float} + Plotting positions parameter. + beta : {float} + Plotting positions parameter. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + def _quantiles1D(data,m,p): + x = numpy.sort(data.compressed()) + n = len(x) + if n == 0: + return masked_array(numpy.empty(len(p), dtype=float_), mask=True) + elif n == 1: + return masked_array(numpy.resize(x, p.shape), mask=nomask) + aleph = (n*p + m) + k = numpy.floor(aleph.clip(1, n-1)).astype(int_) + gamma = (aleph-k).clip(0,1) + return (1.-gamma)*x[(k-1).tolist()] + gamma*x[k.tolist()] + + # Initialization & checks --------- + data = masked_array(data, copy=False) + p = narray(prob, copy=False, ndmin=1) + m = alphap + p*(1.-alphap-betap) + # Computes quantiles along axis (or globally) + if (axis is None): + return _quantiles1D(data, m, p) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_quantiles1D, axis, data, m, p) + + +def plotting_positions(data, alpha=0.4, beta=0.4): + """Returns the plotting positions (or empirical percentile points) for the + data. + Plotting positions are defined as (i-alpha)/(n-alpha-beta), where: + - i is the rank order statistics + - n is the number of unmasked values along the given axis + - alpha and beta are two parameters. + + Typical values for alpha and beta are: + - (0,1) : *p(k) = k/n* : linear interpolation of cdf (R, type 4) + - (.5,.5) : *p(k) = (k-1/2.)/n* : piecewise linear function (R, type 5) + - (0,0) : *p(k) = k/(n+1)* : Weibull (R type 6) + - (1,1) : *p(k) = (k-1)/(n-1)*. In this case, p(k) = mode[F(x[k])]. + That's R default (R type 7) + - (1/3,1/3): *p(k) = (k-1/3)/(n+1/3)*. Then p(k) ~ median[F(x[k])]. + The resulting quantile estimates are approximately median-unbiased + regardless of the distribution of x. (R type 8) + - (3/8,3/8): *p(k) = (k-3/8)/(n+1/4)*. Blom. + The resulting quantile estimates are approximately unbiased + if x is normally distributed (R type 9) + - (.4,.4) : approximately quantile unbiased (Cunnane) + - (.35,.35): APL, used with PWM + """ + data = masked_array(data, copy=False).reshape(1,-1) + n = data.count() + plpos = numpy.empty(data.size, dtype=float_) + plpos[n:] = 0 + plpos[data.argsort()[:n]] = (numpy.arange(1,n+1) - alpha)/(n+1-alpha-beta) + return masked_array(plpos, mask=data._mask) + +meppf = plotting_positions + + +def mmedian(data, axis=None): + """Returns the median of data along the given axis. Missing data are discarded.""" + def _median1D(data): + x = numpy.sort(data.compressed()) + if x.size == 0: + return masked + return numpy.median(x) + data = masked_array(data, subok=True, copy=True) + if axis is None: + return _median1D(data) + else: + return apply_along_axis(_median1D, axis, data) + + +def cov(x, y=None, rowvar=True, bias=False, strict=False): + """Estimates the covariance matrix. + + +Normalization is by (N-1) where N is the number of observations (unbiased +estimate). If bias is True then normalization is by N. + +*Parameters*: + x : {ndarray} + Input data. If x is a 1D array, returns the variance. If x is a 2D array, + returns the covariance matrix. + y : {ndarray}, optional + Optional set of variables. + rowvar : {boolean} + If rowvar is true, then each row is a variable with obersvations in columns. + If rowvar is False, each column is a variable and the observations are in + the rows. + bias : {boolean} + Whether to use a biased or unbiased estimate of the covariance. + If bias is True, then the normalization is by N, the number of observations. + Otherwise, the normalization is by (N-1) + strict : {boolean} + If strict is True, masked values are propagated: if a masked value appears in + a row or column, the whole row or column is considered masked. + """ + X = narray(x, ndmin=2, subok=True, dtype=float) + if X.shape[0] == 1: + rowvar = True + if rowvar: + axis = 0 + tup = (slice(None),None) + else: + axis = 1 + tup = (None, slice(None)) + # + if y is not None: + y = narray(y, copy=False, ndmin=2, subok=True, dtype=float) + X = concatenate((X,y),axis) + # + X -= X.mean(axis=1-axis)[tup] + n = X.count(1-axis) + # + if bias: + fact = n*1.0 + else: + fact = n-1.0 + # + if not rowvar: + return (dot(X.T, X.conj(), strict=False) / fact).squeeze() + else: + return (dot(X, X.T.conj(), strict=False) / fact).squeeze() + + +def idealfourths(data, axis=None): + """Returns an estimate of the interquartile range of the data along the given +axis, as computed with the ideal fourths. + """ + def _idf(data): + x = numpy.sort(data.compressed()) + n = len(x) + (j,h) = divmod(n/4. + 5/12.,1) + qlo = (1-h)*x[j] + h*x[j+1] + k = n - j + qup = (1-h)*x[k] + h*x[k-1] + return qup - qlo + data = masked_array(data, copy=False) + if (axis is None): + return _idf(data) + else: + return apply_along_axis(_idf, axis, data) + + +def rsh(data, points=None): + """Evalutates Rosenblatt's shifted histogram estimators for each point +on the dataset 'data'. + +*Parameters* : + data : {sequence} + Input data. Masked values are ignored. + points : {sequence} + Sequence of points where to evaluate Rosenblatt shifted histogram. + If None, use the data. + """ + data = masked_array(data, copy=False) + if points is None: + points = data + else: + points = numpy.array(points, copy=False, ndmin=1) + if data.ndim != 1: + raise AttributeError("The input array should be 1D only !") + n = data.count() + h = 1.2 * idealfourths(data) / n**(1./5) + nhi = (data[:,None] <= points[None,:] + h).sum(0) + nlo = (data[:,None] < points[None,:] - h).sum(0) + return (nhi-nlo) / (2.*n*h) + +################################################################################ +if __name__ == '__main__': + from numpy.ma.testutils import assert_almost_equal + if 1: + a = numpy.ma.arange(1,101) + a[1::2] = masked + b = numpy.ma.resize(a, (100,100)) + assert_almost_equal(mquantiles(b), [25., 50., 75.]) + assert_almost_equal(mquantiles(b, axis=0), numpy.ma.resize(a,(3,100))) + assert_almost_equal(mquantiles(b, axis=1), + numpy.ma.resize([24.9, 50., 75.1], (100,3))) diff --git a/numpy/ma/setup.py b/numpy/ma/setup.py new file mode 100644 index 000000000..024746655 --- /dev/null +++ b/numpy/ma/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import os + +def configuration(parent_package='',top_path=None): + from numpy.distutils.misc_util import Configuration + config = Configuration('ma',parent_package,top_path) + config.add_data_dir('tests') + return config + +if __name__ == "__main__": + from numpy.distutils.core import setup + config = configuration(top_path='').todict() + setup(**config) diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py new file mode 100644 index 000000000..347372b9b --- /dev/null +++ b/numpy/ma/tests/test_core.py @@ -0,0 +1,1449 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for MaskedArray & subclassing. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +""" +__author__ = "Pierre GF Gerard-Marchant" + +import types +import warnings + +import numpy +import numpy.core.fromnumeric as fromnumeric +from numpy.testing import NumpyTest, NumpyTestCase +from numpy.testing import set_local_path, restore_path +from numpy.testing.utils import build_err_msg +from numpy import array as narray + +import numpy.ma.testutils +from numpy.ma.testutils import * + +import numpy.ma.core as coremodule +from numpy.ma.core import * + +pi = numpy.pi + +set_local_path() +from test_old_ma import * +restore_path() + +#.............................................................................. +class TestMA(NumpyTestCase): + "Base test class for MaskedArrays." + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + self.setUp() + + def setUp (self): + "Base data definition." + x = narray([1.,1.,1.,-2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.]) + y = narray([5.,0.,3., 2., -1., -4., 0., -10., 10., 1., 0., 3.]) + a10 = 10. + m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + m2 = [0, 0, 1, 0, 0, 1, 1, 0, 0, 0 ,0, 1] + xm = masked_array(x, mask=m1) + ym = masked_array(y, mask=m2) + z = narray([-.5, 0., .5, .8]) + zm = masked_array(z, mask=[0,1,0,0]) + xf = numpy.where(m1, 1.e+20, x) + xm.set_fill_value(1.e+20) + self.d = (x, y, a10, m1, m2, xm, ym, z, zm, xf) + #........................ + def test_basic1d(self): + "Test of basic array creation and properties in 1 dimension." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + assert(not isMaskedArray(x)) + assert(isMaskedArray(xm)) + assert((xm-ym).filled(0).any()) + fail_if_equal(xm.mask.astype(int_), ym.mask.astype(int_)) + s = x.shape + assert_equal(numpy.shape(xm), s) + assert_equal(xm.shape, s) + assert_equal(xm.dtype, x.dtype) + assert_equal(zm.dtype, z.dtype) + assert_equal(xm.size , reduce(lambda x,y:x*y, s)) + assert_equal(count(xm) , len(m1) - reduce(lambda x,y:x+y, m1)) + assert_array_equal(xm, xf) + assert_array_equal(filled(xm, 1.e20), xf) + assert_array_equal(x, xm) + #........................ + def test_basic2d(self): + "Test of basic array creation and properties in 2 dimensions." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + for s in [(4,3), (6,2)]: + x.shape = s + y.shape = s + xm.shape = s + ym.shape = s + xf.shape = s + + assert(not isMaskedArray(x)) + assert(isMaskedArray(xm)) + assert_equal(shape(xm), s) + assert_equal(xm.shape, s) + assert_equal( xm.size , reduce(lambda x,y:x*y, s)) + assert_equal( count(xm) , len(m1) - reduce(lambda x,y:x+y, m1)) + assert_equal(xm, xf) + assert_equal(filled(xm, 1.e20), xf) + assert_equal(x, xm) + #........................ + def test_basic_arithmetic (self): + "Test of basic arithmetic." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + a2d = array([[1,2],[0,4]]) + a2dm = masked_array(a2d, [[0,0],[1,0]]) + assert_equal(a2d * a2d, a2d * a2dm) + assert_equal(a2d + a2d, a2d + a2dm) + assert_equal(a2d - a2d, a2d - a2dm) + for s in [(12,), (4,3), (2,6)]: + x = x.reshape(s) + y = y.reshape(s) + xm = xm.reshape(s) + ym = ym.reshape(s) + xf = xf.reshape(s) + assert_equal(-x, -xm) + assert_equal(x + y, xm + ym) + assert_equal(x - y, xm - ym) + assert_equal(x * y, xm * ym) + assert_equal(x / y, xm / ym) + assert_equal(a10 + y, a10 + ym) + assert_equal(a10 - y, a10 - ym) + assert_equal(a10 * y, a10 * ym) + assert_equal(a10 / y, a10 / ym) + assert_equal(x + a10, xm + a10) + assert_equal(x - a10, xm - a10) + assert_equal(x * a10, xm * a10) + assert_equal(x / a10, xm / a10) + assert_equal(x**2, xm**2) + assert_equal(abs(x)**2.5, abs(xm) **2.5) + assert_equal(x**y, xm**ym) + assert_equal(numpy.add(x,y), add(xm, ym)) + assert_equal(numpy.subtract(x,y), subtract(xm, ym)) + assert_equal(numpy.multiply(x,y), multiply(xm, ym)) + assert_equal(numpy.divide(x,y), divide(xm, ym)) + #........................ + def test_mixed_arithmetic(self): + "Tests mixed arithmetics." + na = narray([1]) + ma = array([1]) + self.failUnless(isinstance(na + ma, MaskedArray)) + self.failUnless(isinstance(ma + na, MaskedArray)) + #........................ + def test_inplace_arithmetic(self): + """Test of inplace operations and rich comparisons""" + # addition + x = arange(10) + y = arange(10) + xm = arange(10) + xm[2] = masked + x += 1 + assert_equal(x, y+1) + xm += 1 + assert_equal(xm, y+1) + # subtraction + x = arange(10) + xm = arange(10) + xm[2] = masked + x -= 1 + assert_equal(x, y-1) + xm -= 1 + assert_equal(xm, y-1) + # multiplication + x = arange(10)*1.0 + xm = arange(10)*1.0 + xm[2] = masked + x *= 2.0 + assert_equal(x, y*2) + xm *= 2.0 + assert_equal(xm, y*2) + # division + x = arange(10)*2 + xm = arange(10)*2 + xm[2] = masked + x /= 2 + assert_equal(x, y) + xm /= 2 + assert_equal(xm, y) + # division, pt 2 + x = arange(10)*1.0 + xm = arange(10)*1.0 + xm[2] = masked + x /= 2.0 + assert_equal(x, y/2.0) + xm /= arange(10) + assert_equal(xm, ones((10,))) + + warnings.simplefilter('ignore', DeprecationWarning) + x = arange(10).astype(float_) + xm = arange(10) + xm[2] = masked + id1 = x.raw_data().ctypes.data + x += 1. + assert (id1 == x.raw_data().ctypes.data) + assert_equal(x, y+1.) + warnings.simplefilter('default', DeprecationWarning) + + # addition w/ array + x = arange(10, dtype=float_) + xm = arange(10, dtype=float_) + xm[2] = masked + m = xm.mask + a = arange(10, dtype=float_) + a[-1] = masked + x += a + xm += a + assert_equal(x,y+a) + assert_equal(xm,y+a) + assert_equal(xm.mask, mask_or(m,a.mask)) + # subtraction w/ array + x = arange(10, dtype=float_) + xm = arange(10, dtype=float_) + xm[2] = masked + m = xm.mask + a = arange(10, dtype=float_) + a[-1] = masked + x -= a + xm -= a + assert_equal(x,y-a) + assert_equal(xm,y-a) + assert_equal(xm.mask, mask_or(m,a.mask)) + # multiplication w/ array + x = arange(10, dtype=float_) + xm = arange(10, dtype=float_) + xm[2] = masked + m = xm.mask + a = arange(10, dtype=float_) + a[-1] = masked + x *= a + xm *= a + assert_equal(x,y*a) + assert_equal(xm,y*a) + assert_equal(xm.mask, mask_or(m,a.mask)) + # division w/ array + x = arange(10, dtype=float_) + xm = arange(10, dtype=float_) + xm[2] = masked + m = xm.mask + a = arange(10, dtype=float_) + a[-1] = masked + x /= a + xm /= a + assert_equal(x,y/a) + assert_equal(xm,y/a) + assert_equal(xm.mask, mask_or(mask_or(m,a.mask), (a==0))) + # + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + z = xm/ym + assert_equal(z._mask, [1,1,1,0,0,1,1,0,0,0,1,1]) + assert_equal(z._data, [0.2,1.,1./3.,-1.,-pi/2.,-1.,5.,1.,1.,1.,2.,1.]) + xm = xm.copy() + xm /= ym + assert_equal(xm._mask, [1,1,1,0,0,1,1,0,0,0,1,1]) + assert_equal(xm._data, [1/5.,1.,1./3.,-1.,-pi/2.,-1.,5.,1.,1.,1.,2.,1.]) + + + #.......................... + def test_scalararithmetic(self): + "Tests some scalar arithmetics on MaskedArrays." + xm = array(0, mask=1) + assert((1/array(0)).mask) + assert((1 + xm).mask) + assert((-xm).mask) + assert((-xm).mask) + assert(maximum(xm, xm).mask) + assert(minimum(xm, xm).mask) + assert(xm.filled().dtype is xm.data.dtype) + x = array(0, mask=0) + assert_equal(x.filled().ctypes.data, x.ctypes.data) + assert_equal(str(xm), str(masked_print_option)) + #......................... + def test_basic_ufuncs (self): + "Test various functions such as sin, cos." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + assert_equal(numpy.cos(x), cos(xm)) + assert_equal(numpy.cosh(x), cosh(xm)) + assert_equal(numpy.sin(x), sin(xm)) + assert_equal(numpy.sinh(x), sinh(xm)) + assert_equal(numpy.tan(x), tan(xm)) + assert_equal(numpy.tanh(x), tanh(xm)) + assert_equal(numpy.sqrt(abs(x)), sqrt(xm)) + assert_equal(numpy.log(abs(x)), log(xm)) + assert_equal(numpy.log10(abs(x)), log10(xm)) + assert_equal(numpy.exp(x), exp(xm)) + assert_equal(numpy.arcsin(z), arcsin(zm)) + assert_equal(numpy.arccos(z), arccos(zm)) + assert_equal(numpy.arctan(z), arctan(zm)) + assert_equal(numpy.arctan2(x, y), arctan2(xm, ym)) + assert_equal(numpy.absolute(x), absolute(xm)) + assert_equal(numpy.equal(x,y), equal(xm, ym)) + assert_equal(numpy.not_equal(x,y), not_equal(xm, ym)) + assert_equal(numpy.less(x,y), less(xm, ym)) + assert_equal(numpy.greater(x,y), greater(xm, ym)) + assert_equal(numpy.less_equal(x,y), less_equal(xm, ym)) + assert_equal(numpy.greater_equal(x,y), greater_equal(xm, ym)) + assert_equal(numpy.conjugate(x), conjugate(xm)) + #........................ + def test_count_func (self): + "Tests count" + ott = array([0.,1.,2.,3.], mask=[1,0,0,0]) + assert( isinstance(count(ott), int)) + assert_equal(3, count(ott)) + assert_equal(1, count(1)) + assert_equal(0, array(1,mask=[1])) + ott = ott.reshape((2,2)) + assert isMaskedArray(count(ott,0)) + assert isinstance(count(ott), types.IntType) + assert_equal(3, count(ott)) + assert getmask(count(ott,0)) is nomask + assert_equal([1,2],count(ott,0)) + #........................ + def test_minmax_func (self): + "Tests minimum and maximum." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + xr = numpy.ravel(x) #max doesn't work if shaped + xmr = ravel(xm) + assert_equal(max(xr), maximum(xmr)) #true because of careful selection of data + assert_equal(min(xr), minimum(xmr)) #true because of careful selection of data + # + assert_equal(minimum([1,2,3],[4,0,9]), [1,0,3]) + assert_equal(maximum([1,2,3],[4,0,9]), [4,2,9]) + x = arange(5) + y = arange(5) - 2 + x[3] = masked + y[0] = masked + assert_equal(minimum(x,y), where(less(x,y), x, y)) + assert_equal(maximum(x,y), where(greater(x,y), x, y)) + assert minimum(x) == 0 + assert maximum(x) == 4 + # + x = arange(4).reshape(2,2) + x[-1,-1] = masked + assert_equal(maximum(x), 2) + + def test_minmax_methods(self): + "Additional tests on max/min" + (_, _, _, _, _, xm, _, _, _, _) = self.d + xm.shape = (xm.size,) + assert_equal(xm.max(), 10) + assert(xm[0].max() is masked) + assert(xm[0].max(0) is masked) + assert(xm[0].max(-1) is masked) + assert_equal(xm.min(), -10.) + assert(xm[0].min() is masked) + assert(xm[0].min(0) is masked) + assert(xm[0].min(-1) is masked) + assert_equal(xm.ptp(), 20.) + assert(xm[0].ptp() is masked) + assert(xm[0].ptp(0) is masked) + assert(xm[0].ptp(-1) is masked) + # + x = array([1,2,3], mask=True) + assert(x.min() is masked) + assert(x.max() is masked) + assert(x.ptp() is masked) + #........................ + def test_addsumprod (self): + "Tests add, sum, product." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + assert_equal(numpy.add.reduce(x), add.reduce(x)) + assert_equal(numpy.add.accumulate(x), add.accumulate(x)) + assert_equal(4, sum(array(4),axis=0)) + assert_equal(4, sum(array(4), axis=0)) + assert_equal(numpy.sum(x,axis=0), sum(x,axis=0)) + assert_equal(numpy.sum(filled(xm,0),axis=0), sum(xm,axis=0)) + assert_equal(numpy.sum(x,0), sum(x,0)) + assert_equal(numpy.product(x,axis=0), product(x,axis=0)) + assert_equal(numpy.product(x,0), product(x,0)) + assert_equal(numpy.product(filled(xm,1),axis=0), product(xm,axis=0)) + s = (3,4) + x.shape = y.shape = xm.shape = ym.shape = s + if len(s) > 1: + assert_equal(numpy.concatenate((x,y),1), concatenate((xm,ym),1)) + assert_equal(numpy.add.reduce(x,1), add.reduce(x,1)) + assert_equal(numpy.sum(x,1), sum(x,1)) + assert_equal(numpy.product(x,1), product(x,1)) + #......................... + def test_concat(self): + "Tests concatenations." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + # basic concatenation + assert_equal(numpy.concatenate((x,y)), concatenate((xm,ym))) + assert_equal(numpy.concatenate((x,y)), concatenate((x,y))) + assert_equal(numpy.concatenate((x,y)), concatenate((xm,y))) + assert_equal(numpy.concatenate((x,y,x)), concatenate((x,ym,x))) + # Concatenation along an axis + s = (3,4) + x.shape = y.shape = xm.shape = ym.shape = s + assert_equal(xm.mask, numpy.reshape(m1, s)) + assert_equal(ym.mask, numpy.reshape(m2, s)) + xmym = concatenate((xm,ym),1) + assert_equal(numpy.concatenate((x,y),1), xmym) + assert_equal(numpy.concatenate((xm.mask,ym.mask),1), xmym._mask) + # + x=zeros(2) + y=array(ones(2),mask=[False,True]) + z = concatenate((x,y)) + assert_array_equal(z,[0,0,1,1]) + assert_array_equal(z.mask,[False,False,False,True]) + z = concatenate((y,x)) + assert_array_equal(z,[1,1,0,0]) + assert_array_equal(z.mask,[False,True,False,False]) + + #........................ + def test_indexing(self): + "Tests conversions and indexing" + x1 = numpy.array([1,2,4,3]) + x2 = array(x1, mask=[1,0,0,0]) + x3 = array(x1, mask=[0,1,0,1]) + x4 = array(x1) + # test conversion to strings + junk, garbage = str(x2), repr(x2) + assert_equal(numpy.sort(x1),sort(x2,endwith=False)) + # tests of indexing + assert type(x2[1]) is type(x1[1]) + assert x1[1] == x2[1] + assert x2[0] is masked + assert_equal(x1[2],x2[2]) + assert_equal(x1[2:5],x2[2:5]) + assert_equal(x1[:],x2[:]) + assert_equal(x1[1:], x3[1:]) + x1[2] = 9 + x2[2] = 9 + assert_equal(x1,x2) + x1[1:3] = 99 + x2[1:3] = 99 + assert_equal(x1,x2) + x2[1] = masked + assert_equal(x1,x2) + x2[1:3] = masked + assert_equal(x1,x2) + x2[:] = x1 + x2[1] = masked + assert allequal(getmask(x2),array([0,1,0,0])) + x3[:] = masked_array([1,2,3,4],[0,1,1,0]) + assert allequal(getmask(x3), array([0,1,1,0])) + x4[:] = masked_array([1,2,3,4],[0,1,1,0]) + assert allequal(getmask(x4), array([0,1,1,0])) + assert allequal(x4, array([1,2,3,4])) + x1 = numpy.arange(5)*1.0 + x2 = masked_values(x1, 3.0) + assert_equal(x1,x2) + assert allequal(array([0,0,0,1,0],MaskType), x2.mask) +#FIXME: Well, eh, fill_value is now a property assert_equal(3.0, x2.fill_value()) + assert_equal(3.0, x2.fill_value) + x1 = array([1,'hello',2,3],object) + x2 = numpy.array([1,'hello',2,3],object) + s1 = x1[1] + s2 = x2[1] + assert_equal(type(s2), str) + assert_equal(type(s1), str) + assert_equal(s1, s2) + assert x1[1:1].shape == (0,) + #........................ + def test_copy(self): + "Tests of some subtle points of copying and sizing." + n = [0,0,1,0,0] + m = make_mask(n) + m2 = make_mask(m) + assert(m is m2) + m3 = make_mask(m, copy=1) + assert(m is not m3) + + warnings.simplefilter('ignore', DeprecationWarning) + x1 = numpy.arange(5) + y1 = array(x1, mask=m) + #assert( y1._data is x1) + assert_equal(y1._data.__array_interface__, x1.__array_interface__) + assert( allequal(x1,y1.raw_data())) + #assert( y1.mask is m) + assert_equal(y1._mask.__array_interface__, m.__array_interface__) + warnings.simplefilter('default', DeprecationWarning) + + y1a = array(y1) + #assert( y1a.raw_data() is y1.raw_data()) + assert( y1a._data.__array_interface__ == y1._data.__array_interface__) + assert( y1a.mask is y1.mask) + + y2 = array(x1, mask=m) + #assert( y2.raw_data() is x1) + assert (y2._data.__array_interface__ == x1.__array_interface__) + #assert( y2.mask is m) + assert (y2._mask.__array_interface__ == m.__array_interface__) + assert( y2[2] is masked) + y2[2] = 9 + assert( y2[2] is not masked) + #assert( y2.mask is not m) + assert (y2._mask.__array_interface__ != m.__array_interface__) + assert( allequal(y2.mask, 0)) + + y3 = array(x1*1.0, mask=m) + assert(filled(y3).dtype is (x1*1.0).dtype) + + x4 = arange(4) + x4[2] = masked + y4 = resize(x4, (8,)) + assert_equal(concatenate([x4,x4]), y4) + assert_equal(getmask(y4),[0,0,1,0,0,0,1,0]) + y5 = repeat(x4, (2,2,2,2), axis=0) + assert_equal(y5, [0,0,1,1,2,2,3,3]) + y6 = repeat(x4, 2, axis=0) + assert_equal(y5, y6) + y7 = x4.repeat((2,2,2,2), axis=0) + assert_equal(y5,y7) + y8 = x4.repeat(2,0) + assert_equal(y5,y8) + + y9 = x4.copy() + assert_equal(y9._data, x4._data) + assert_equal(y9._mask, x4._mask) + # + x = masked_array([1,2,3], mask=[0,1,0]) + # Copy is False by default + y = masked_array(x) + assert_equal(y._data.ctypes.data, x._data.ctypes.data) + assert_equal(y._mask.ctypes.data, x._mask.ctypes.data) + y = masked_array(x, copy=True) + assert_not_equal(y._data.ctypes.data, x._data.ctypes.data) + assert_not_equal(y._mask.ctypes.data, x._mask.ctypes.data) + #........................ + def test_where(self): + "Test the where function" + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + d = where(xm>2,xm,-9) + assert_equal(d, [-9.,-9.,-9.,-9., -9., 4., -9., -9., 10., -9., -9., 3.]) + assert_equal(d._mask, xm._mask) + d = where(xm>2,-9,ym) + assert_equal(d, [5.,0.,3., 2., -1.,-9.,-9., -10., -9., 1., 0., -9.]) + assert_equal(d._mask, [1,0,1,0,0,0,1,0,0,0,0,0]) + d = where(xm>2, xm, masked) + assert_equal(d, [-9.,-9.,-9.,-9., -9., 4., -9., -9., 10., -9., -9., 3.]) + tmp = xm._mask.copy() + tmp[(xm<=2).filled(True)] = True + assert_equal(d._mask, tmp) + # + ixm = xm.astype(int_) + d = where(ixm>2, ixm, masked) + assert_equal(d, [-9,-9,-9,-9, -9, 4, -9, -9, 10, -9, -9, 3]) + assert_equal(d.dtype, ixm.dtype) + # + x = arange(10) + x[3] = masked + c = x >= 8 + z = where(c , x, masked) + assert z.dtype is x.dtype + assert z[3] is masked + assert z[4] is masked + assert z[7] is masked + assert z[8] is not masked + assert z[9] is not masked + assert_equal(x,z) + # + z = where(c , masked, x) + assert z.dtype is x.dtype + assert z[3] is masked + assert z[4] is not masked + assert z[7] is not masked + assert z[8] is masked + assert z[9] is masked + + #........................ + def test_oddfeatures_1(self): + "Test of other odd features" + x = arange(20) + x = x.reshape(4,5) + x.flat[5] = 12 + assert x[1,0] == 12 + z = x + 10j * x + assert_equal(z.real, x) + assert_equal(z.imag, 10*x) + assert_equal((z*conjugate(z)).real, 101*x*x) + z.imag[...] = 0.0 + + x = arange(10) + x[3] = masked + assert str(x[3]) == str(masked) + c = x >= 8 + assert count(where(c,masked,masked)) == 0 + assert shape(where(c,masked,masked)) == c.shape + # + z = masked_where(c, x) + assert z.dtype is x.dtype + assert z[3] is masked + assert z[4] is not masked + assert z[7] is not masked + assert z[8] is masked + assert z[9] is masked + assert_equal(x,z) + # + #........................ + def test_oddfeatures_2(self): + "Tests some more features." + x = array([1.,2.,3.,4.,5.]) + c = array([1,1,1,0,0]) + x[2] = masked + z = where(c, x, -x) + assert_equal(z, [1.,2.,0., -4., -5]) + c[0] = masked + z = where(c, x, -x) + assert_equal(z, [1.,2.,0., -4., -5]) + assert z[0] is masked + assert z[1] is not masked + assert z[2] is masked + # + x = arange(6) + x[5] = masked + y = arange(6)*10 + y[2] = masked + c = array([1,1,1,0,0,0], mask=[1,0,0,0,0,0]) + cm = c.filled(1) + z = where(c,x,y) + zm = where(cm,x,y) + assert_equal(z, zm) + assert getmask(zm) is nomask + assert_equal(zm, [0,1,2,30,40,50]) + z = where(c, masked, 1) + assert_equal(z, [99,99,99,1,1,1]) + z = where(c, 1, masked) + assert_equal(z, [99, 1, 1, 99, 99, 99]) + #........................ + def test_oddfeatures_3(self): + """Tests some generic features.""" + atest = ones((10,10,10), dtype=float_) + btest = zeros(atest.shape, MaskType) + ctest = masked_where(btest,atest) + assert_equal(atest,ctest) + #........................ + def test_maskingfunctions(self): + "Tests masking functions." + x = array([1.,2.,3.,4.,5.]) + x[2] = masked + assert_equal(masked_where(greater(x, 2), x), masked_greater(x,2)) + assert_equal(masked_where(greater_equal(x, 2), x), masked_greater_equal(x,2)) + assert_equal(masked_where(less(x, 2), x), masked_less(x,2)) + assert_equal(masked_where(less_equal(x, 2), x), masked_less_equal(x,2)) + assert_equal(masked_where(not_equal(x, 2), x), masked_not_equal(x,2)) + assert_equal(masked_where(equal(x, 2), x), masked_equal(x,2)) + assert_equal(masked_where(not_equal(x,2), x), masked_not_equal(x,2)) + assert_equal(masked_inside(range(5), 1, 3), [0, 199, 199, 199, 4]) + assert_equal(masked_outside(range(5), 1, 3),[199,1,2,3,199]) + assert_equal(masked_inside(array(range(5), mask=[1,0,0,0,0]), 1, 3).mask, [1,1,1,1,0]) + assert_equal(masked_outside(array(range(5), mask=[0,1,0,0,0]), 1, 3).mask, [1,1,0,0,1]) + assert_equal(masked_equal(array(range(5), mask=[1,0,0,0,0]), 2).mask, [1,0,1,0,0]) + assert_equal(masked_not_equal(array([2,2,1,2,1], mask=[1,0,0,0,0]), 2).mask, [1,0,1,0,1]) + assert_equal(masked_where([1,1,0,0,0], [1,2,3,4,5]), [99,99,3,4,5]) + #........................ + def test_TakeTransposeInnerOuter(self): + "Test of take, transpose, inner, outer products" + x = arange(24) + y = numpy.arange(24) + x[5:6] = masked + x = x.reshape(2,3,4) + y = y.reshape(2,3,4) + assert_equal(numpy.transpose(y,(2,0,1)), transpose(x,(2,0,1))) + assert_equal(numpy.take(y, (2,0,1), 1), take(x, (2,0,1), 1)) + assert_equal(numpy.inner(filled(x,0),filled(y,0)), + inner(x, y)) + assert_equal(numpy.outer(filled(x,0),filled(y,0)), + outer(x, y)) + y = array(['abc', 1, 'def', 2, 3], object) + y[2] = masked + t = take(y,[0,3,4]) + assert t[0] == 'abc' + assert t[1] == 2 + assert t[2] == 3 + #....................... + def test_maskedelement(self): + "Test of masked element" + x = arange(6) + x[1] = masked + assert(str(masked) == '--') + assert(x[1] is masked) + assert_equal(filled(x[1], 0), 0) + # don't know why these should raise an exception... + #self.failUnlessRaises(Exception, lambda x,y: x+y, masked, masked) + #self.failUnlessRaises(Exception, lambda x,y: x+y, masked, 2) + #self.failUnlessRaises(Exception, lambda x,y: x+y, masked, xx) + #self.failUnlessRaises(Exception, lambda x,y: x+y, xx, masked) + #........................ + def test_scalar(self): + "Checks masking a scalar" + x = masked_array(0) + assert_equal(str(x), '0') + x = masked_array(0,mask=True) + assert_equal(str(x), str(masked_print_option)) + x = masked_array(0, mask=False) + assert_equal(str(x), '0') + #........................ + def test_usingmasked(self): + "Checks that there's no collapsing to masked" + x = masked_array([1,2]) + y = x * masked + assert_equal(y.shape, x.shape) + assert_equal(y._mask, [True, True]) + y = x[0] * masked + assert y is masked + y = x + masked + assert_equal(y.shape, x.shape) + assert_equal(y._mask, [True, True]) + + #........................ + def test_topython(self): + "Tests some communication issues with Python." + assert_equal(1, int(array(1))) + assert_equal(1.0, float(array(1))) + assert_equal(1, int(array([[[1]]]))) + assert_equal(1.0, float(array([[1]]))) + self.failUnlessRaises(ValueError, float, array([1,1])) + + warnings.simplefilter('ignore',UserWarning) + assert numpy.isnan(float(array([1],mask=[1]))) + warnings.simplefilter('default',UserWarning) +#TODO: Check how bool works... +#TODO: self.failUnless(bool(array([0,1]))) +#TODO: self.failUnless(bool(array([0,0],mask=[0,1]))) +#TODO: self.failIf(bool(array([0,0]))) +#TODO: self.failIf(bool(array([0,0],mask=[0,0]))) + #........................ + def test_arraymethods(self): + "Tests some MaskedArray methods." + a = array([1,3,2]) + b = array([1,3,2], mask=[1,0,1]) + assert_equal(a.any(), a.data.any()) + assert_equal(a.all(), a.data.all()) + assert_equal(a.argmax(), a.data.argmax()) + assert_equal(a.argmin(), a.data.argmin()) + assert_equal(a.choose(0,1,2,3,4), a.data.choose(0,1,2,3,4)) + assert_equal(a.compress([1,0,1]), a.data.compress([1,0,1])) + assert_equal(a.conj(), a.data.conj()) + assert_equal(a.conjugate(), a.data.conjugate()) + # + m = array([[1,2],[3,4]]) + assert_equal(m.diagonal(), m.data.diagonal()) + assert_equal(a.sum(), a.data.sum()) + assert_equal(a.take([1,2]), a.data.take([1,2])) + assert_equal(m.transpose(), m.data.transpose()) + #........................ + def test_basicattributes(self): + "Tests some basic array attributes." + a = array([1,3,2]) + b = array([1,3,2], mask=[1,0,1]) + assert_equal(a.ndim, 1) + assert_equal(b.ndim, 1) + assert_equal(a.size, 3) + assert_equal(b.size, 3) + assert_equal(a.shape, (3,)) + assert_equal(b.shape, (3,)) + #........................ + def test_single_element_subscript(self): + "Tests single element subscripts of Maskedarrays." + a = array([1,3,2]) + b = array([1,3,2], mask=[1,0,1]) + assert_equal(a[0].shape, ()) + assert_equal(b[0].shape, ()) + assert_equal(b[1].shape, ()) + #........................ + def test_maskcreation(self): + "Tests how masks are initialized at the creation of Maskedarrays." + data = arange(24, dtype=float_) + data[[3,6,15]] = masked + dma_1 = MaskedArray(data) + assert_equal(dma_1.mask, data.mask) + dma_2 = MaskedArray(dma_1) + assert_equal(dma_2.mask, dma_1.mask) + dma_3 = MaskedArray(dma_1, mask=[1,0,0,0]*6) + fail_if_equal(dma_3.mask, dma_1.mask) + + def test_pickling(self): + "Tests pickling" + import cPickle + a = arange(10) + a[::3] = masked + a.fill_value = 999 + a_pickled = cPickle.loads(a.dumps()) + assert_equal(a_pickled._mask, a._mask) + assert_equal(a_pickled._data, a._data) + assert_equal(a_pickled.fill_value, 999) + # + a = array(numpy.matrix(range(10)), mask=[1,0,1,0,0]*2) + a_pickled = cPickle.loads(a.dumps()) + assert_equal(a_pickled._mask, a._mask) + assert_equal(a_pickled, a) + assert(isinstance(a_pickled._data,numpy.matrix)) + # + def test_fillvalue(self): + "Having fun with the fill_value" + data = masked_array([1,2,3],fill_value=-999) + series = data[[0,2,1]] + assert_equal(series._fill_value, data._fill_value) + # + mtype = [('f',float_),('s','|S3')] + x = array([(1,'a'),(2,'b'),(numpy.pi,'pi')], dtype=mtype) + x.fill_value=999 + assert_equal(x.fill_value,[999.,'999']) + assert_equal(x['f'].fill_value, 999) + assert_equal(x['s'].fill_value, '999') + # + x.fill_value=(9,'???') + assert_equal(x.fill_value, (9,'???')) + assert_equal(x['f'].fill_value, 9) + assert_equal(x['s'].fill_value, '???') + # + x = array([1,2,3.1]) + x.fill_value = 999 + assert_equal(numpy.asarray(x.fill_value).dtype, float_) + assert_equal(x.fill_value, 999.) + # + def test_asarray(self): + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + xmm = asarray(xm) + assert_equal(xmm._data, xm._data) + assert_equal(xmm._mask, xm._mask) + # + def test_fix_invalid(self): + "Checks fix_invalid." + data = masked_array(numpy.sqrt([-1., 0., 1.]), mask=[0,0,1]) + data_fixed = fix_invalid(data) + assert_equal(data_fixed._data, [data.fill_value, 0., 1.]) + assert_equal(data_fixed._mask, [1., 0., 1.]) + # + def test_imag_real(self): + "Check complex" + xx = array([1+10j,20+2j], mask=[1,0]) + assert_equal(xx.imag,[10,2]) + assert_equal(xx.imag.filled(), [1e+20,2]) + assert_equal(xx.imag.dtype, xx._data.imag.dtype) + assert_equal(xx.real,[1,20]) + assert_equal(xx.real.filled(), [1e+20,20]) + assert_equal(xx.real.dtype, xx._data.real.dtype) + # + def test_ndmin(self): + "Check the use of ndmin" + x = array([1,2,3],mask=[1,0,0], ndmin=2) + assert_equal(x.shape,(1,3)) + assert_equal(x._data,[[1,2,3]]) + assert_equal(x._mask,[[1,0,0]]) + # + def test_record(self): + "Check record access" + mtype = [('f',float_),('s','|S3')] + x = array([(1,'a'),(2,'b'),(numpy.pi,'pi')], dtype=mtype) + x[1] = masked + # + (xf, xs) = (x['f'], x['s']) + assert_equal(xf.data, [1,2,numpy.pi]) + assert_equal(xf.mask, [0,1,0]) + assert_equal(xf.dtype, float_) + assert_equal(xs.data, ['a', 'b', 'pi']) + assert_equal(xs.mask, [0,1,0]) + assert_equal(xs.dtype, '|S3') + # + + +#............................................................................... + +class TestUfuncs(NumpyTestCase): + "Test class for the application of ufuncs on MaskedArrays." + def setUp(self): + "Base data definition." + self.d = (array([1.0, 0, -1, pi/2]*2, mask=[0,1]+[0]*6), + array([1.0, 0, -1, pi/2]*2, mask=[1,0]+[0]*6),) + + def test_testUfuncRegression(self): + "Tests new ufuncs on MaskedArrays." + for f in ['sqrt', 'log', 'log10', 'exp', 'conjugate', + 'sin', 'cos', 'tan', + 'arcsin', 'arccos', 'arctan', + 'sinh', 'cosh', 'tanh', + 'arcsinh', + 'arccosh', + 'arctanh', + 'absolute', 'fabs', 'negative', + # 'nonzero', 'around', + 'floor', 'ceil', + # 'sometrue', 'alltrue', + 'logical_not', + 'add', 'subtract', 'multiply', + 'divide', 'true_divide', 'floor_divide', + 'remainder', 'fmod', 'hypot', 'arctan2', + 'equal', 'not_equal', 'less_equal', 'greater_equal', + 'less', 'greater', + 'logical_and', 'logical_or', 'logical_xor', + ]: + #print f + try: + uf = getattr(umath, f) + except AttributeError: + uf = getattr(fromnumeric, f) + mf = getattr(coremodule, f) + args = self.d[:uf.nin] + ur = uf(*args) + mr = mf(*args) + assert_equal(ur.filled(0), mr.filled(0), f) + assert_mask_equal(ur.mask, mr.mask) + #........................ + def test_reduce(self): + "Tests reduce on MaskedArrays." + a = self.d[0] + assert(not alltrue(a,axis=0)) + assert(sometrue(a,axis=0)) + assert_equal(sum(a[:3],axis=0), 0) + assert_equal(product(a,axis=0), 0) + assert_equal(add.reduce(a), pi) + #........................ + def test_minmax(self): + "Tests extrema on MaskedArrays." + a = arange(1,13).reshape(3,4) + amask = masked_where(a < 5,a) + assert_equal(amask.max(), a.max()) + assert_equal(amask.min(), 5) + assert_equal(amask.max(0), a.max(0)) + assert_equal(amask.min(0), [5,6,7,8]) + assert(amask.max(1)[0].mask) + assert(amask.min(1)[0].mask) + +#............................................................................... + +class TestArrayMethods(NumpyTestCase): + "Test class for miscellaneous MaskedArrays methods." + def setUp(self): + "Base data definition." + x = numpy.array([ 8.375, 7.545, 8.828, 8.5 , 1.757, 5.928, + 8.43 , 7.78 , 9.865, 5.878, 8.979, 4.732, + 3.012, 6.022, 5.095, 3.116, 5.238, 3.957, + 6.04 , 9.63 , 7.712, 3.382, 4.489, 6.479, + 7.189, 9.645, 5.395, 4.961, 9.894, 2.893, + 7.357, 9.828, 6.272, 3.758, 6.693, 0.993]) + X = x.reshape(6,6) + XX = x.reshape(3,2,2,3) + + m = numpy.array([0, 1, 0, 1, 0, 0, + 1, 0, 1, 1, 0, 1, + 0, 0, 0, 1, 0, 1, + 0, 0, 0, 1, 1, 1, + 1, 0, 0, 1, 0, 0, + 0, 0, 1, 0, 1, 0]) + mx = array(data=x,mask=m) + mX = array(data=X,mask=m.reshape(X.shape)) + mXX = array(data=XX,mask=m.reshape(XX.shape)) + + m2 = numpy.array([1, 1, 0, 1, 0, 0, + 1, 1, 1, 1, 0, 1, + 0, 0, 1, 1, 0, 1, + 0, 0, 0, 1, 1, 1, + 1, 0, 0, 1, 1, 0, + 0, 0, 1, 0, 1, 1]) + m2x = array(data=x,mask=m2) + m2X = array(data=X,mask=m2.reshape(X.shape)) + m2XX = array(data=XX,mask=m2.reshape(XX.shape)) + self.d = (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) + + #------------------------------------------------------ + def test_trace(self): + "Tests trace on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + mXdiag = mX.diagonal() + assert_equal(mX.trace(), mX.diagonal().compressed().sum()) + assert_almost_equal(mX.trace(), + X.trace() - sum(mXdiag.mask*X.diagonal(),axis=0)) + + def test_clip(self): + "Tests clip on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + clipped = mx.clip(2,8) + assert_equal(clipped.mask,mx.mask) + assert_equal(clipped.data,x.clip(2,8)) + assert_equal(clipped.data,mx.data.clip(2,8)) + + def test_ptp(self): + "Tests ptp on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + (n,m) = X.shape + assert_equal(mx.ptp(),mx.compressed().ptp()) + rows = numpy.zeros(n,numpy.float_) + cols = numpy.zeros(m,numpy.float_) + for k in range(m): + cols[k] = mX[:,k].compressed().ptp() + for k in range(n): + rows[k] = mX[k].compressed().ptp() + assert_equal(mX.ptp(0),cols) + assert_equal(mX.ptp(1),rows) + + def test_swapaxes(self): + "Tests swapaxes on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + mXswapped = mX.swapaxes(0,1) + assert_equal(mXswapped[-1],mX[:,-1]) + mXXswapped = mXX.swapaxes(0,2) + assert_equal(mXXswapped.shape,(2,2,3,3)) + + def test_cumsumprod(self): + "Tests cumsum & cumprod on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + mXcp = mX.cumsum(0) + assert_equal(mXcp.data,mX.filled(0).cumsum(0)) + mXcp = mX.cumsum(1) + assert_equal(mXcp.data,mX.filled(0).cumsum(1)) + # + mXcp = mX.cumprod(0) + assert_equal(mXcp.data,mX.filled(1).cumprod(0)) + mXcp = mX.cumprod(1) + assert_equal(mXcp.data,mX.filled(1).cumprod(1)) + + def test_varstd(self): + "Tests var & std on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + assert_almost_equal(mX.var(axis=None),mX.compressed().var()) + assert_almost_equal(mX.std(axis=None),mX.compressed().std()) + assert_equal(mXX.var(axis=3).shape,XX.var(axis=3).shape) + assert_equal(mX.var().shape,X.var().shape) + (mXvar0,mXvar1) = (mX.var(axis=0), mX.var(axis=1)) + for k in range(6): + assert_almost_equal(mXvar1[k],mX[k].compressed().var()) + assert_almost_equal(mXvar0[k],mX[:,k].compressed().var()) + assert_almost_equal(numpy.sqrt(mXvar0[k]), mX[:,k].compressed().std()) + + def test_argmin(self): + "Tests argmin & argmax on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + # + assert_equal(mx.argmin(),35) + assert_equal(mX.argmin(),35) + assert_equal(m2x.argmin(),4) + assert_equal(m2X.argmin(),4) + assert_equal(mx.argmax(),28) + assert_equal(mX.argmax(),28) + assert_equal(m2x.argmax(),31) + assert_equal(m2X.argmax(),31) + # + assert_equal(mX.argmin(0), [2,2,2,5,0,5]) + assert_equal(m2X.argmin(0), [2,2,4,5,0,4]) + assert_equal(mX.argmax(0), [0,5,0,5,4,0]) + assert_equal(m2X.argmax(0), [5,5,0,5,1,0]) + # + assert_equal(mX.argmin(1), [4,1,0,0,5,5,]) + assert_equal(m2X.argmin(1), [4,4,0,0,5,3]) + assert_equal(mX.argmax(1), [2,4,1,1,4,1]) + assert_equal(m2X.argmax(1), [2,4,1,1,1,1]) + + def test_put(self): + "Tests put." + d = arange(5) + n = [0,0,0,1,1] + m = make_mask(n) + x = array(d, mask = m) + assert( x[3] is masked) + assert( x[4] is masked) + x[[1,4]] = [10,40] +# assert( x.mask is not m) + assert( x[3] is masked) + assert( x[4] is not masked) + assert_equal(x, [0,10,2,-1,40]) + # + x = masked_array(arange(10), mask=[1,0,0,0,0]*2) + i = [0,2,4,6] + x.put(i, [6,4,2,0]) + assert_equal(x, asarray([6,1,4,3,2,5,0,7,8,9,])) + assert_equal(x.mask, [0,0,0,0,0,1,0,0,0,0]) + x.put(i, masked_array([0,2,4,6],[1,0,1,0])) + assert_array_equal(x, [0,1,2,3,4,5,6,7,8,9,]) + assert_equal(x.mask, [1,0,0,0,1,1,0,0,0,0]) + # + x = masked_array(arange(10), mask=[1,0,0,0,0]*2) + put(x, i, [6,4,2,0]) + assert_equal(x, asarray([6,1,4,3,2,5,0,7,8,9,])) + assert_equal(x.mask, [0,0,0,0,0,1,0,0,0,0]) + put(x, i, masked_array([0,2,4,6],[1,0,1,0])) + assert_array_equal(x, [0,1,2,3,4,5,6,7,8,9,]) + assert_equal(x.mask, [1,0,0,0,1,1,0,0,0,0]) + + def test_put_hardmask(self): + "Tests put on hardmask" + d = arange(5) + n = [0,0,0,1,1] + m = make_mask(n) + xh = array(d+1, mask = m, hard_mask=True, copy=True) + xh.put([4,2,0,1,3],[1,2,3,4,5]) + assert_equal(xh._data, [3,4,2,4,5]) + + def test_take(self): + "Tests take" + x = masked_array([10,20,30,40],[0,1,0,1]) + assert_equal(x.take([0,0,3]), masked_array([10, 10, 40], [0,0,1]) ) + assert_equal(x.take([0,0,3]), x[[0,0,3]]) + assert_equal(x.take([[0,1],[0,1]]), + masked_array([[10,20],[10,20]], [[0,1],[0,1]]) ) + # + x = array([[10,20,30],[40,50,60]], mask=[[0,0,1],[1,0,0,]]) + assert_equal(x.take([0,2], axis=1), + array([[10,30],[40,60]], mask=[[0,1],[1,0]])) + assert_equal(take(x, [0,2], axis=1), + array([[10,30],[40,60]], mask=[[0,1],[1,0]])) + #........................ + def test_anyall(self): + """Checks the any/all methods/functions.""" + x = numpy.array([[ 0.13, 0.26, 0.90], + [ 0.28, 0.33, 0.63], + [ 0.31, 0.87, 0.70]]) + m = numpy.array([[ True, False, False], + [False, False, False], + [True, True, False]], dtype=numpy.bool_) + mx = masked_array(x, mask=m) + xbig = numpy.array([[False, False, True], + [False, False, True], + [False, True, True]], dtype=numpy.bool_) + mxbig = (mx > 0.5) + mxsmall = (mx < 0.5) + # + assert (mxbig.all()==False) + assert (mxbig.any()==True) + assert_equal(mxbig.all(0),[False, False, True]) + assert_equal(mxbig.all(1), [False, False, True]) + assert_equal(mxbig.any(0),[False, False, True]) + assert_equal(mxbig.any(1), [True, True, True]) + # + assert (mxsmall.all()==False) + assert (mxsmall.any()==True) + assert_equal(mxsmall.all(0), [True, True, False]) + assert_equal(mxsmall.all(1), [False, False, False]) + assert_equal(mxsmall.any(0), [True, True, False]) + assert_equal(mxsmall.any(1), [True, True, False]) + # + X = numpy.matrix(x) + mX = masked_array(X, mask=m) + mXbig = (mX > 0.5) + mXsmall = (mX < 0.5) + # + assert (mXbig.all()==False) + assert (mXbig.any()==True) + assert_equal(mXbig.all(0), numpy.matrix([False, False, True])) + assert_equal(mXbig.all(1), numpy.matrix([False, False, True]).T) + assert_equal(mXbig.any(0), numpy.matrix([False, False, True])) + assert_equal(mXbig.any(1), numpy.matrix([ True, True, True]).T) + # + assert (mXsmall.all()==False) + assert (mXsmall.any()==True) + assert_equal(mXsmall.all(0), numpy.matrix([True, True, False])) + assert_equal(mXsmall.all(1), numpy.matrix([False, False, False]).T) + assert_equal(mXsmall.any(0), numpy.matrix([True, True, False])) + assert_equal(mXsmall.any(1), numpy.matrix([True, True, False]).T) + + def test_keepmask(self): + "Tests the keep mask flag" + x = masked_array([1,2,3], mask=[1,0,0]) + mx = masked_array(x) + assert_equal(mx.mask, x.mask) + mx = masked_array(x, mask=[0,1,0], keep_mask=False) + assert_equal(mx.mask, [0,1,0]) + mx = masked_array(x, mask=[0,1,0], keep_mask=True) + assert_equal(mx.mask, [1,1,0]) + # We default to true + mx = masked_array(x, mask=[0,1,0]) + assert_equal(mx.mask, [1,1,0]) + + def test_hardmask(self): + "Test hard_mask" + d = arange(5) + n = [0,0,0,1,1] + m = make_mask(n) + xh = array(d, mask = m, hard_mask=True) + # We need to copy, to avoid updating d in xh! + xs = array(d, mask = m, hard_mask=False, copy=True) + xh[[1,4]] = [10,40] + xs[[1,4]] = [10,40] + assert_equal(xh._data, [0,10,2,3,4]) + assert_equal(xs._data, [0,10,2,3,40]) + #assert_equal(xh.mask.ctypes.data, m.ctypes.data) + assert_equal(xs.mask, [0,0,0,1,0]) + assert(xh._hardmask) + assert(not xs._hardmask) + xh[1:4] = [10,20,30] + xs[1:4] = [10,20,30] + assert_equal(xh._data, [0,10,20,3,4]) + assert_equal(xs._data, [0,10,20,30,40]) + #assert_equal(xh.mask.ctypes.data, m.ctypes.data) + assert_equal(xs.mask, nomask) + xh[0] = masked + xs[0] = masked + assert_equal(xh.mask, [1,0,0,1,1]) + assert_equal(xs.mask, [1,0,0,0,0]) + xh[:] = 1 + xs[:] = 1 + assert_equal(xh._data, [0,1,1,3,4]) + assert_equal(xs._data, [1,1,1,1,1]) + assert_equal(xh.mask, [1,0,0,1,1]) + assert_equal(xs.mask, nomask) + # Switch to soft mask + xh.soften_mask() + xh[:] = arange(5) + assert_equal(xh._data, [0,1,2,3,4]) + assert_equal(xh.mask, nomask) + # Switch back to hard mask + xh.harden_mask() + xh[xh<3] = masked + assert_equal(xh._data, [0,1,2,3,4]) + assert_equal(xh._mask, [1,1,1,0,0]) + xh[filled(xh>1,False)] = 5 + assert_equal(xh._data, [0,1,2,5,5]) + assert_equal(xh._mask, [1,1,1,0,0]) + # + xh = array([[1,2],[3,4]], mask = [[1,0],[0,0]], hard_mask=True) + xh[0] = 0 + assert_equal(xh._data, [[1,0],[3,4]]) + assert_equal(xh._mask, [[1,0],[0,0]]) + xh[-1,-1] = 5 + assert_equal(xh._data, [[1,0],[3,5]]) + assert_equal(xh._mask, [[1,0],[0,0]]) + xh[filled(xh<5,False)] = 2 + assert_equal(xh._data, [[1,2],[2,5]]) + assert_equal(xh._mask, [[1,0],[0,0]]) + # + "Another test of hardmask" + d = arange(5) + n = [0,0,0,1,1] + m = make_mask(n) + xh = array(d, mask = m, hard_mask=True) + xh[4:5] = 999 + #assert_equal(xh.mask.ctypes.data, m.ctypes.data) + xh[0:1] = 999 + assert_equal(xh._data,[999,1,2,3,4]) + + def test_smallmask(self): + "Checks the behaviour of _smallmask" + a = arange(10) + a[1] = masked + a[1] = 1 + assert_equal(a._mask, nomask) + a = arange(10) + a._smallmask = False + a[1] = masked + a[1] = 1 + assert_equal(a._mask, zeros(10)) + + + def test_sort(self): + "Test sort" + x = array([1,4,2,3],mask=[0,1,0,0],dtype=numpy.uint8) + # + sortedx = sort(x) + assert_equal(sortedx._data,[1,2,3,4]) + assert_equal(sortedx._mask,[0,0,0,1]) + # + sortedx = sort(x, endwith=False) + assert_equal(sortedx._data, [4,1,2,3]) + assert_equal(sortedx._mask, [1,0,0,0]) + # + x.sort() + assert_equal(x._data,[1,2,3,4]) + assert_equal(x._mask,[0,0,0,1]) + # + x = array([1,4,2,3],mask=[0,1,0,0],dtype=numpy.uint8) + x.sort(endwith=False) + assert_equal(x._data, [4,1,2,3]) + assert_equal(x._mask, [1,0,0,0]) + # + x = [1,4,2,3] + sortedx = sort(x) + assert(not isinstance(sorted, MaskedArray)) + # + x = array([0,1,-1,-2,2], mask=nomask, dtype=numpy.int8) + sortedx = sort(x, endwith=False) + assert_equal(sortedx._data, [-2,-1,0,1,2]) + x = array([0,1,-1,-2,2], mask=[0,1,0,0,1], dtype=numpy.int8) + sortedx = sort(x, endwith=False) + assert_equal(sortedx._data, [1,2,-2,-1,0]) + assert_equal(sortedx._mask, [1,1,0,0,0]) + + def test_sort_2d(self): + "Check sort of 2D array." + # 2D array w/o mask + a = masked_array([[8,4,1],[2,0,9]]) + a.sort(0) + assert_equal(a, [[2,0,1],[8,4,9]]) + a = masked_array([[8,4,1],[2,0,9]]) + a.sort(1) + assert_equal(a, [[1,4,8],[0,2,9]]) + # 2D array w/mask + a = masked_array([[8,4,1],[2,0,9]], mask=[[1,0,0],[0,0,1]]) + a.sort(0) + assert_equal(a, [[2,0,1],[8,4,9]]) + assert_equal(a._mask, [[0,0,0],[1,0,1]]) + a = masked_array([[8,4,1],[2,0,9]], mask=[[1,0,0],[0,0,1]]) + a.sort(1) + assert_equal(a, [[1,4,8],[0,2,9]]) + assert_equal(a._mask, [[0,0,1],[0,0,1]]) + # 3D + a = masked_array([[[7, 8, 9],[4, 5, 6],[1, 2, 3]], + [[1, 2, 3],[7, 8, 9],[4, 5, 6]], + [[7, 8, 9],[1, 2, 3],[4, 5, 6]], + [[4, 5, 6],[1, 2, 3],[7, 8, 9]]]) + a[a%4==0] = masked + am = a.copy() + an = a.filled(99) + am.sort(0) + an.sort(0) + assert_equal(am, an) + am = a.copy() + an = a.filled(99) + am.sort(1) + an.sort(1) + assert_equal(am, an) + am = a.copy() + an = a.filled(99) + am.sort(2) + an.sort(2) + assert_equal(am, an) + + + def test_ravel(self): + "Tests ravel" + a = array([[1,2,3,4,5]], mask=[[0,1,0,0,0]]) + aravel = a.ravel() + assert_equal(a._mask.shape, a.shape) + a = array([0,0], mask=[1,1]) + aravel = a.ravel() + assert_equal(a._mask.shape, a.shape) + a = array(numpy.matrix([1,2,3,4,5]), mask=[[0,1,0,0,0]]) + aravel = a.ravel() + assert_equal(a.shape,(1,5)) + assert_equal(a._mask.shape, a.shape) + # Checs that small_mask is preserved + a = array([1,2,3,4],mask=[0,0,0,0],shrink=False) + assert_equal(a.ravel()._mask, [0,0,0,0]) + + def test_reshape(self): + "Tests reshape" + x = arange(4) + x[0] = masked + y = x.reshape(2,2) + assert_equal(y.shape, (2,2,)) + assert_equal(y._mask.shape, (2,2,)) + assert_equal(x.shape, (4,)) + assert_equal(x._mask.shape, (4,)) + + def test_compressed(self): + "Tests compressed" + a = array([1,2,3,4],mask=[0,0,0,0]) + b = a.compressed() + assert_equal(b, a) + assert_equal(b._mask, nomask) + a[0] = masked + b = a.compressed() + assert_equal(b._data, [2,3,4]) + assert_equal(b._mask, nomask) + + def test_tolist(self): + "Tests to list" + x = array(numpy.arange(12)) + x[[1,-2]] = masked + xlist = x.tolist() + assert(xlist[1] is None) + assert(xlist[-2] is None) + # + x.shape = (3,4) + xlist = x.tolist() + # + assert_equal(xlist[0],[0,None,2,3]) + assert_equal(xlist[1],[4,5,6,7]) + assert_equal(xlist[2],[8,9,None,11]) + # Make sure a masked record is output as a tuple of None + x = array(zip([1,2,3], + [1.1,2.2,3.3], + ['one','two','thr']), + dtype=[('a',int_),('b',float_),('c','|S8')]) + x[-1] = masked + assert_equal(x.tolist(), [(1,1.1,'one'),(2,2.2,'two'),(None,None,None)]) + + + def test_squeeze(self): + "Check squeeze" + data = masked_array([[1,2,3]]) + assert_equal(data.squeeze(), [1,2,3]) + data = masked_array([[1,2,3]], mask=[[1,1,1]]) + assert_equal(data.squeeze(), [1,2,3]) + assert_equal(data.squeeze()._mask, [1,1,1]) + data = masked_array([[1]], mask=True) + assert(data.squeeze() is masked) + + def test_putmask(self): + x = arange(6)+1 + mx = array(x, mask=[0,0,0,1,1,1]) + mask = [0,0,1,0,0,1] + # w/o mask, w/o masked values + xx = x.copy() + putmask(xx, mask, 99) + assert_equal(xx, [1,2,99,4,5,99]) + # w/ mask, w/o masked values + mxx = mx.copy() + putmask(mxx, mask, 99) + assert_equal(mxx._data, [1,2,99,4,5,99]) + assert_equal(mxx._mask, [0,0,0,1,1,0]) + # w/o mask, w/ masked values + values = array([10,20,30,40,50,60],mask=[1,1,1,0,0,0]) + xx = x.copy() + putmask(xx, mask, values) + assert_equal(xx._data, [1,2,30,4,5,60]) + assert_equal(xx._mask, [0,0,1,0,0,0]) + # w/ mask, w/ masked values + mxx = mx.copy() + putmask(mxx, mask, values) + assert_equal(mxx._data, [1,2,30,4,5,60]) + assert_equal(mxx._mask, [0,0,1,1,1,0]) + # w/ mask, w/ masked values + hardmask + mxx = mx.copy() + mxx.harden_mask() + putmask(mxx, mask, values) + assert_equal(mxx, [1,2,30,4,5,60]) + + def test_compress(self): + "test compress" + a = masked_array([1., 2., 3., 4., 5.], fill_value=9999) + condition = (a > 1.5) & (a < 3.5) + assert_equal(a.compress(condition),[2.,3.]) + # + a[[2,3]] = masked + b = a.compress(condition) + assert_equal(b._data,[2.,3.]) + assert_equal(b._mask,[0,1]) + assert_equal(b.fill_value,9999) + assert_equal(b,a[condition]) + # + condition = (a<4.) + b = a.compress(condition) + assert_equal(b._data,[1.,2.,3.]) + assert_equal(b._mask,[0,0,1]) + assert_equal(b.fill_value,9999) + assert_equal(b,a[condition]) + # + a = masked_array([[10,20,30],[40,50,60]], mask=[[0,0,1],[1,0,0]]) + b = a.compress(a.ravel() >= 22) + assert_equal(b._data, [30, 40, 50, 60]) + assert_equal(b._mask, [1,1,0,0]) + # + x = numpy.array([3,1,2]) + b = a.compress(x >= 2, axis=1) + assert_equal(b._data, [[10,30],[40,60]]) + assert_equal(b._mask, [[0,1],[1,0]]) + # + def test_empty(self): + "Tests empty/like" + datatype = [('a',int_),('b',float_),('c','|S8')] + a = masked_array([(1,1.1,'1.1'),(2,2.2,'2.2'),(3,3.3,'3.3')], + dtype=datatype) + assert_equal(len(a.fill_value), len(datatype)) + # + b = empty_like(a) + assert_equal(b.shape, a.shape) + assert_equal(b.fill_value, a.fill_value) + # + b = empty(len(a), dtype=datatype) + assert_equal(b.shape, a.shape) + assert_equal(b.fill_value, a.fill_value) + + +#.............................................................................. + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/ma/tests/test_extras.py b/numpy/ma/tests/test_extras.py new file mode 100644 index 000000000..5a52aeeee --- /dev/null +++ b/numpy/ma/tests/test_extras.py @@ -0,0 +1,331 @@ +# pylint: disable-msg=W0611, W0612, W0511 +"""Tests suite for MaskedArray. +Adapted from the original test_ma by Pierre Gerard-Marchant + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_extras.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import numpy as N +from numpy.testing import NumpyTest, NumpyTestCase +from numpy.testing.utils import build_err_msg + +import numpy.ma.testutils +from numpy.ma.testutils import * + +import numpy.ma.core +from numpy.ma.core import * +import numpy.ma.extras +from numpy.ma.extras import * + +class TestAverage(NumpyTestCase): + "Several tests of average. Why so many ? Good point..." + def check_testAverage1(self): + "Test of average." + ott = array([0.,1.,2.,3.], mask=[1,0,0,0]) + assert_equal(2.0, average(ott,axis=0)) + assert_equal(2.0, average(ott, weights=[1., 1., 2., 1.])) + result, wts = average(ott, weights=[1.,1.,2.,1.], returned=1) + assert_equal(2.0, result) + assert(wts == 4.0) + ott[:] = masked + assert_equal(average(ott,axis=0).mask, [True]) + ott = array([0.,1.,2.,3.], mask=[1,0,0,0]) + ott = ott.reshape(2,2) + ott[:,1] = masked + assert_equal(average(ott,axis=0), [2.0, 0.0]) + assert_equal(average(ott,axis=1).mask[0], [True]) + assert_equal([2.,0.], average(ott, axis=0)) + result, wts = average(ott, axis=0, returned=1) + assert_equal(wts, [1., 0.]) + + def check_testAverage2(self): + "More tests of average." + w1 = [0,1,1,1,1,0] + w2 = [[0,1,1,1,1,0],[1,0,0,0,0,1]] + x = arange(6, dtype=float_) + assert_equal(average(x, axis=0), 2.5) + assert_equal(average(x, axis=0, weights=w1), 2.5) + y = array([arange(6, dtype=float_), 2.0*arange(6)]) + assert_equal(average(y, None), N.add.reduce(N.arange(6))*3./12.) + assert_equal(average(y, axis=0), N.arange(6) * 3./2.) + assert_equal(average(y, axis=1), [average(x,axis=0), average(x,axis=0) * 2.0]) + assert_equal(average(y, None, weights=w2), 20./6.) + assert_equal(average(y, axis=0, weights=w2), [0.,1.,2.,3.,4.,10.]) + assert_equal(average(y, axis=1), [average(x,axis=0), average(x,axis=0) * 2.0]) + m1 = zeros(6) + m2 = [0,0,1,1,0,0] + m3 = [[0,0,1,1,0,0],[0,1,1,1,1,0]] + m4 = ones(6) + m5 = [0, 1, 1, 1, 1, 1] + assert_equal(average(masked_array(x, m1),axis=0), 2.5) + assert_equal(average(masked_array(x, m2),axis=0), 2.5) + assert_equal(average(masked_array(x, m4),axis=0).mask, [True]) + assert_equal(average(masked_array(x, m5),axis=0), 0.0) + assert_equal(count(average(masked_array(x, m4),axis=0)), 0) + z = masked_array(y, m3) + assert_equal(average(z, None), 20./6.) + assert_equal(average(z, axis=0), [0.,1.,99.,99.,4.0, 7.5]) + assert_equal(average(z, axis=1), [2.5, 5.0]) + assert_equal(average(z,axis=0, weights=w2), [0.,1., 99., 99., 4.0, 10.0]) + + def check_testAverage3(self): + "Yet more tests of average!" + a = arange(6) + b = arange(6) * 3 + r1, w1 = average([[a,b],[b,a]], axis=1, returned=1) + assert_equal(shape(r1) , shape(w1)) + assert_equal(r1.shape , w1.shape) + r2, w2 = average(ones((2,2,3)), axis=0, weights=[3,1], returned=1) + assert_equal(shape(w2) , shape(r2)) + r2, w2 = average(ones((2,2,3)), returned=1) + assert_equal(shape(w2) , shape(r2)) + r2, w2 = average(ones((2,2,3)), weights=ones((2,2,3)), returned=1) + assert_equal(shape(w2), shape(r2)) + a2d = array([[1,2],[0,4]], float) + a2dm = masked_array(a2d, [[0,0],[1,0]]) + a2da = average(a2d, axis=0) + assert_equal(a2da, [0.5, 3.0]) + a2dma = average(a2dm, axis=0) + assert_equal(a2dma, [1.0, 3.0]) + a2dma = average(a2dm, axis=None) + assert_equal(a2dma, 7./3.) + a2dma = average(a2dm, axis=1) + assert_equal(a2dma, [1.5, 4.0]) + +class TestConcatenator(NumpyTestCase): + "Tests for mr_, the equivalent of r_ for masked arrays." + def check_1d(self): + "Tests mr_ on 1D arrays." + assert_array_equal(mr_[1,2,3,4,5,6],array([1,2,3,4,5,6])) + b = ones(5) + m = [1,0,0,0,0] + d = masked_array(b,mask=m) + c = mr_[d,0,0,d] + assert(isinstance(c,MaskedArray) or isinstance(c,core.MaskedArray)) + assert_array_equal(c,[1,1,1,1,1,0,0,1,1,1,1,1]) + assert_array_equal(c.mask, mr_[m,0,0,m]) + + def check_2d(self): + "Tests mr_ on 2D arrays." + a_1 = rand(5,5) + a_2 = rand(5,5) + m_1 = N.round_(rand(5,5),0) + m_2 = N.round_(rand(5,5),0) + b_1 = masked_array(a_1,mask=m_1) + b_2 = masked_array(a_2,mask=m_2) + d = mr_['1',b_1,b_2] # append columns + assert(d.shape == (5,10)) + assert_array_equal(d[:,:5],b_1) + assert_array_equal(d[:,5:],b_2) + assert_array_equal(d.mask, N.r_['1',m_1,m_2]) + d = mr_[b_1,b_2] + assert(d.shape == (10,5)) + assert_array_equal(d[:5,:],b_1) + assert_array_equal(d[5:,:],b_2) + assert_array_equal(d.mask, N.r_[m_1,m_2]) + +class TestNotMasked(NumpyTestCase): + "Tests notmasked_edges and notmasked_contiguous." + def check_edges(self): + "Tests unmasked_edges" + a = masked_array(N.arange(24).reshape(3,8), + mask=[[0,0,0,0,1,1,1,0], + [1,1,1,1,1,1,1,1], + [0,0,0,0,0,0,1,0],]) + # + assert_equal(notmasked_edges(a, None), [0,23]) + # + tmp = notmasked_edges(a, 0) + assert_equal(tmp[0], (array([0,0,0,0,2,2,0]), array([0,1,2,3,4,5,7]))) + assert_equal(tmp[1], (array([2,2,2,2,2,2,2]), array([0,1,2,3,4,5,7]))) + # + tmp = notmasked_edges(a, 1) + assert_equal(tmp[0], (array([0,2,]), array([0,0]))) + assert_equal(tmp[1], (array([0,2,]), array([7,7]))) + + def check_contiguous(self): + "Tests notmasked_contiguous" + a = masked_array(N.arange(24).reshape(3,8), + mask=[[0,0,0,0,1,1,1,1], + [1,1,1,1,1,1,1,1], + [0,0,0,0,0,0,1,0],]) + tmp = notmasked_contiguous(a, None) + assert_equal(tmp[-1], slice(23,23,None)) + assert_equal(tmp[-2], slice(16,21,None)) + assert_equal(tmp[-3], slice(0,3,None)) + # + tmp = notmasked_contiguous(a, 0) + assert(len(tmp[-1]) == 1) + assert(tmp[-2] is None) + assert_equal(tmp[-3],tmp[-1]) + assert(len(tmp[0]) == 2) + # + tmp = notmasked_contiguous(a, 1) + assert_equal(tmp[0][-1], slice(0,3,None)) + assert(tmp[1] is None) + assert_equal(tmp[2][-1], slice(7,7,None)) + assert_equal(tmp[2][-2], slice(0,5,None)) + +class Test2DFunctions(NumpyTestCase): + "Tests 2D functions" + def check_compress2d(self): + "Tests compress2d" + x = array(N.arange(9).reshape(3,3), mask=[[1,0,0],[0,0,0],[0,0,0]]) + assert_equal(compress_rowcols(x), [[4,5],[7,8]] ) + assert_equal(compress_rowcols(x,0), [[3,4,5],[6,7,8]] ) + assert_equal(compress_rowcols(x,1), [[1,2],[4,5],[7,8]] ) + x = array(x._data, mask=[[0,0,0],[0,1,0],[0,0,0]]) + assert_equal(compress_rowcols(x), [[0,2],[6,8]] ) + assert_equal(compress_rowcols(x,0), [[0,1,2],[6,7,8]] ) + assert_equal(compress_rowcols(x,1), [[0,2],[3,5],[6,8]] ) + x = array(x._data, mask=[[1,0,0],[0,1,0],[0,0,0]]) + assert_equal(compress_rowcols(x), [[8]] ) + assert_equal(compress_rowcols(x,0), [[6,7,8]] ) + assert_equal(compress_rowcols(x,1,), [[2],[5],[8]] ) + x = array(x._data, mask=[[1,0,0],[0,1,0],[0,0,1]]) + assert_equal(compress_rowcols(x).size, 0 ) + assert_equal(compress_rowcols(x,0).size, 0 ) + assert_equal(compress_rowcols(x,1).size, 0 ) + # + def check_mask_rowcols(self): + "Tests mask_rowcols." + x = array(N.arange(9).reshape(3,3), mask=[[1,0,0],[0,0,0],[0,0,0]]) + assert_equal(mask_rowcols(x).mask, [[1,1,1],[1,0,0],[1,0,0]] ) + assert_equal(mask_rowcols(x,0).mask, [[1,1,1],[0,0,0],[0,0,0]] ) + assert_equal(mask_rowcols(x,1).mask, [[1,0,0],[1,0,0],[1,0,0]] ) + x = array(x._data, mask=[[0,0,0],[0,1,0],[0,0,0]]) + assert_equal(mask_rowcols(x).mask, [[0,1,0],[1,1,1],[0,1,0]] ) + assert_equal(mask_rowcols(x,0).mask, [[0,0,0],[1,1,1],[0,0,0]] ) + assert_equal(mask_rowcols(x,1).mask, [[0,1,0],[0,1,0],[0,1,0]] ) + x = array(x._data, mask=[[1,0,0],[0,1,0],[0,0,0]]) + assert_equal(mask_rowcols(x).mask, [[1,1,1],[1,1,1],[1,1,0]] ) + assert_equal(mask_rowcols(x,0).mask, [[1,1,1],[1,1,1],[0,0,0]] ) + assert_equal(mask_rowcols(x,1,).mask, [[1,1,0],[1,1,0],[1,1,0]] ) + x = array(x._data, mask=[[1,0,0],[0,1,0],[0,0,1]]) + assert(mask_rowcols(x).all()) + assert(mask_rowcols(x,0).all()) + assert(mask_rowcols(x,1).all()) + # + def test_dot(self): + "Tests dot product" + n = N.arange(1,7) + # + m = [1,0,0,0,0,0] + a = masked_array(n, mask=m).reshape(2,3) + b = masked_array(n, mask=m).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask, [[1,1],[1,0]]) + c = dot(b,a,True) + assert_equal(c.mask, [[1,1,1],[1,0,0],[1,0,0]]) + c = dot(a,b,False) + assert_equal(c, N.dot(a.filled(0), b.filled(0))) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0), a.filled(0))) + # + m = [0,0,0,0,0,1] + a = masked_array(n, mask=m).reshape(2,3) + b = masked_array(n, mask=m).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask,[[0,1],[1,1]]) + c = dot(b,a,True) + assert_equal(c.mask, [[0,0,1],[0,0,1],[1,1,1]]) + c = dot(a,b,False) + assert_equal(c, N.dot(a.filled(0), b.filled(0))) + assert_equal(c, dot(a,b)) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0), a.filled(0))) + # + m = [0,0,0,0,0,0] + a = masked_array(n, mask=m).reshape(2,3) + b = masked_array(n, mask=m).reshape(3,2) + c = dot(a,b) + assert_equal(c.mask,nomask) + c = dot(b,a) + assert_equal(c.mask,nomask) + # + a = masked_array(n, mask=[1,0,0,0,0,0]).reshape(2,3) + b = masked_array(n, mask=[0,0,0,0,0,0]).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask,[[1,1],[0,0]]) + c = dot(a,b,False) + assert_equal(c, N.dot(a.filled(0),b.filled(0))) + c = dot(b,a,True) + assert_equal(c.mask,[[1,0,0],[1,0,0],[1,0,0]]) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0),a.filled(0))) + # + a = masked_array(n, mask=[0,0,0,0,0,1]).reshape(2,3) + b = masked_array(n, mask=[0,0,0,0,0,0]).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask,[[0,0],[1,1]]) + c = dot(a,b) + assert_equal(c, N.dot(a.filled(0),b.filled(0))) + c = dot(b,a,True) + assert_equal(c.mask,[[0,0,1],[0,0,1],[0,0,1]]) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0), a.filled(0))) + # + a = masked_array(n, mask=[0,0,0,0,0,1]).reshape(2,3) + b = masked_array(n, mask=[0,0,1,0,0,0]).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask,[[1,0],[1,1]]) + c = dot(a,b,False) + assert_equal(c, N.dot(a.filled(0),b.filled(0))) + c = dot(b,a,True) + assert_equal(c.mask,[[0,0,1],[1,1,1],[0,0,1]]) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0),a.filled(0))) + + def test_mediff1d(self): + "Tests mediff1d" + x = masked_array(N.arange(5), mask=[1,0,0,0,1]) + difx_d = (x._data[1:]-x._data[:-1]) + difx_m = (x._mask[1:]-x._mask[:-1]) + dx = mediff1d(x) + assert_equal(dx._data, difx_d) + assert_equal(dx._mask, difx_m) + # + dx = mediff1d(x, to_begin=masked) + assert_equal(dx._data, N.r_[0,difx_d]) + assert_equal(dx._mask, N.r_[1,difx_m]) + dx = mediff1d(x, to_begin=[1,2,3]) + assert_equal(dx._data, N.r_[[1,2,3],difx_d]) + assert_equal(dx._mask, N.r_[[0,0,0],difx_m]) + # + dx = mediff1d(x, to_end=masked) + assert_equal(dx._data, N.r_[difx_d,0]) + assert_equal(dx._mask, N.r_[difx_m,1]) + dx = mediff1d(x, to_end=[1,2,3]) + assert_equal(dx._data, N.r_[difx_d,[1,2,3]]) + assert_equal(dx._mask, N.r_[difx_m,[0,0,0]]) + # + dx = mediff1d(x, to_end=masked, to_begin=masked) + assert_equal(dx._data, N.r_[0,difx_d,0]) + assert_equal(dx._mask, N.r_[1,difx_m,1]) + dx = mediff1d(x, to_end=[1,2,3], to_begin=masked) + assert_equal(dx._data, N.r_[0,difx_d,[1,2,3]]) + assert_equal(dx._mask, N.r_[1,difx_m,[0,0,0]]) + # + dx = mediff1d(x._data, to_end=masked, to_begin=masked) + assert_equal(dx._data, N.r_[0,difx_d,0]) + assert_equal(dx._mask, N.r_[1,0,0,0,0,1]) + +class TestApplyAlongAxis(NumpyTestCase): + "Tests 2D functions" + def check_3d(self): + a = arange(12.).reshape(2,2,3) + def myfunc(b): + return b[1] + xa = apply_along_axis(myfunc,2,a) + assert_equal(xa,[[1,4],[7,10]]) + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/ma/tests/test_morestats.py b/numpy/ma/tests/test_morestats.py new file mode 100644 index 000000000..933e974da --- /dev/null +++ b/numpy/ma/tests/test_morestats.py @@ -0,0 +1,114 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for maskedArray statistics. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_morestats.py 317 2007-10-04 19:31:14Z backtopop $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: backtopop $)" +__version__ = '1.0' +__revision__ = "$Revision: 317 $" +__date__ = '$Date: 2007-10-04 15:31:14 -0400 (Thu, 04 Oct 2007) $' + +import numpy + +import numpy.ma +from numpy.ma import masked, masked_array + +import numpy.ma.mstats +from numpy.ma.mstats import * +import numpy.ma.morestats +from numpy.ma.morestats import * + +import numpy.ma.testutils +from numpy.ma.testutils import * + + +class TestMisc(NumpyTestCase): + # + def __init__(self, *args, **kwargs): + NumpyTestCase.__init__(self, *args, **kwargs) + # + def test_mjci(self): + "Tests the Marits-Jarrett estimator" + data = masked_array([ 77, 87, 88,114,151,210,219,246,253,262, + 296,299,306,376,428,515,666,1310,2611]) + assert_almost_equal(mjci(data),[55.76819,45.84028,198.8788],5) + # + def test_trimmedmeanci(self): + "Tests the confidence intervals of the trimmed mean." + data = masked_array([545,555,558,572,575,576,578,580, + 594,605,635,651,653,661,666]) + assert_almost_equal(trimmed_mean(data,0.2), 596.2, 1) + assert_equal(numpy.round(trimmed_mean_ci(data,0.2),1), [561.8, 630.6]) + +#.............................................................................. +class TestRanking(NumpyTestCase): + # + def __init__(self, *args, **kwargs): + NumpyTestCase.__init__(self, *args, **kwargs) + # + def test_ranking(self): + x = masked_array([0,1,1,1,2,3,4,5,5,6,]) + assert_almost_equal(rank_data(x),[1,3,3,3,5,6,7,8.5,8.5,10]) + x[[3,4]] = masked + assert_almost_equal(rank_data(x),[1,2.5,2.5,0,0,4,5,6.5,6.5,8]) + assert_almost_equal(rank_data(x,use_missing=True), + [1,2.5,2.5,4.5,4.5,4,5,6.5,6.5,8]) + x = masked_array([0,1,5,1,2,4,3,5,1,6,]) + assert_almost_equal(rank_data(x),[1,3,8.5,3,5,7,6,8.5,3,10]) + x = masked_array([[0,1,1,1,2], [3,4,5,5,6,]]) + assert_almost_equal(rank_data(x),[[1,3,3,3,5],[6,7,8.5,8.5,10]]) + assert_almost_equal(rank_data(x,axis=1),[[1,3,3,3,5],[1,2,3.5,3.5,5]]) + assert_almost_equal(rank_data(x,axis=0),[[1,1,1,1,1],[2,2,2,2,2,]]) + +#.............................................................................. +class TestQuantiles(NumpyTestCase): + # + def __init__(self, *args, **kwargs): + NumpyTestCase.__init__(self, *args, **kwargs) + # + def test_hdquantiles(self): + data = [0.706560797,0.727229578,0.990399276,0.927065621,0.158953014, + 0.887764025,0.239407086,0.349638551,0.972791145,0.149789972, + 0.936947700,0.132359948,0.046041972,0.641675031,0.945530547, + 0.224218684,0.771450991,0.820257774,0.336458052,0.589113496, + 0.509736129,0.696838829,0.491323573,0.622767425,0.775189248, + 0.641461450,0.118455200,0.773029450,0.319280007,0.752229111, + 0.047841438,0.466295911,0.583850781,0.840581845,0.550086491, + 0.466470062,0.504765074,0.226855960,0.362641207,0.891620942, + 0.127898691,0.490094097,0.044882048,0.041441695,0.317976349, + 0.504135618,0.567353033,0.434617473,0.636243375,0.231803616, + 0.230154113,0.160011327,0.819464108,0.854706985,0.438809221, + 0.487427267,0.786907310,0.408367937,0.405534192,0.250444460, + 0.995309248,0.144389588,0.739947527,0.953543606,0.680051621, + 0.388382017,0.863530727,0.006514031,0.118007779,0.924024803, + 0.384236354,0.893687694,0.626534881,0.473051932,0.750134705, + 0.241843555,0.432947602,0.689538104,0.136934797,0.150206859, + 0.474335206,0.907775349,0.525869295,0.189184225,0.854284286, + 0.831089744,0.251637345,0.587038213,0.254475554,0.237781276, + 0.827928620,0.480283781,0.594514455,0.213641488,0.024194386, + 0.536668589,0.699497811,0.892804071,0.093835427,0.731107772] + # + assert_almost_equal(hdquantiles(data,[0., 1.]), + [0.006514031, 0.995309248]) + hdq = hdquantiles(data,[0.25, 0.5, 0.75]) + assert_almost_equal(hdq, [0.253210762, 0.512847491, 0.762232442,]) + hdq = hdquantiles_sd(data,[0.25, 0.5, 0.75]) + assert_almost_equal(hdq, [0.03786954, 0.03805389, 0.03800152,], 4) + # + data = numpy.array(data).reshape(10,10) + hdq = hdquantiles(data,[0.25,0.5,0.75],axis=0) + assert_almost_equal(hdq[:,0], hdquantiles(data[:,0],[0.25,0.5,0.75])) + assert_almost_equal(hdq[:,-1], hdquantiles(data[:,-1],[0.25,0.5,0.75])) + hdq = hdquantiles(data,[0.25,0.5,0.75],axis=0,var=True) + assert_almost_equal(hdq[...,0], + hdquantiles(data[:,0],[0.25,0.5,0.75],var=True)) + assert_almost_equal(hdq[...,-1], + hdquantiles(data[:,-1],[0.25,0.5,0.75], var=True)) + + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/ma/tests/test_mrecords.py b/numpy/ma/tests/test_mrecords.py new file mode 100644 index 000000000..df12121cd --- /dev/null +++ b/numpy/ma/tests/test_mrecords.py @@ -0,0 +1,348 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for mrecords. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import types + +import numpy as N +from numpy import recarray, bool_, int_, float_ +from numpy import array as narray +from numpy.core.records import fromrecords as recfromrecords +from numpy.core.records import fromarrays as recfromarrays +import numpy.core.fromnumeric as fromnumeric_ +from numpy.testing import NumpyTest, NumpyTestCase +from numpy.testing.utils import build_err_msg + +import numpy.ma.testutils +from numpy.ma.testutils import assert_equal, assert_equal_records + +import numpy.ma +from numpy.ma import masked_array, masked, nomask, getdata, getmaskarray + +#import numpy.ma.mrecords +#from numpy.ma.mrecords import mrecarray, fromarrays, fromtextfile, fromrecords + +import numpy.ma.mrecords +from numpy.ma.mrecords import MaskedRecords, mrecarray,\ + fromarrays, fromtextfile, fromrecords, addfield + +#.............................................................................. +class TestMRecords(NumpyTestCase): + "Base test class for MaskedArrays." + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + self.setup() + + def setup(self): + "Generic setup" + ilist = [1,2,3,4,5] + flist = [1.1,2.2,3.3,4.4,5.5] + slist = ['one','two','three','four','five'] + ddtype = [('a',int_),('b',float_),('c','|S8')] + mask = [0,1,0,0,1] + self.base = masked_array(zip(ilist,flist,slist), mask=mask, dtype=ddtype) + + def test_byview(self): + "Test creation by view" + base = self.base + mbase = base.view(mrecarray) + assert_equal(mbase._mask, base._mask) + assert isinstance(mbase._data, recarray) + assert_equal_records(mbase._data, base._data.view(recarray)) + for field in ('a','b','c'): + assert_equal(base[field], mbase[field]) + assert_equal_records(mbase.view(mrecarray), mbase) + + def test_get(self): + "Tests fields retrieval" + base = self.base.copy() + mbase = base.view(mrecarray) + # As fields.......... + for field in ('a','b','c'): + assert_equal(getattr(mbase,field), mbase[field]) + assert_equal(base[field], mbase[field]) + # as elements ....... + mbase_first = mbase[0] + assert isinstance(mbase_first, mrecarray) + assert_equal(mbase_first.dtype, mbase.dtype) + assert_equal(mbase_first.tolist(), (1,1.1,'one')) + assert_equal(mbase_first.mask, nomask) + assert_equal(mbase_first._fieldmask.item(), (False, False, False)) + assert_equal(mbase_first['a'], mbase['a'][0]) + mbase_last = mbase[-1] + assert isinstance(mbase_last, mrecarray) + assert_equal(mbase_last.dtype, mbase.dtype) + assert_equal(mbase_last.tolist(), (None,None,None)) + assert_equal(mbase_last.mask, True) + assert_equal(mbase_last._fieldmask.item(), (True, True, True)) + assert_equal(mbase_last['a'], mbase['a'][-1]) + assert (mbase_last['a'] is masked) + # as slice .......... + mbase_sl = mbase[:2] + assert isinstance(mbase_sl, mrecarray) + assert_equal(mbase_sl.dtype, mbase.dtype) + assert_equal(mbase_sl._mask, [0,1]) + assert_equal_records(mbase_sl, base[:2].view(mrecarray)) + for field in ('a','b','c'): + assert_equal(getattr(mbase_sl,field), base[:2][field]) + + def test_set_fields(self): + "Tests setting fields." + base = self.base.copy() + mbase = base.view(mrecarray) + mbase = mbase.copy() + mbase.fill_value = (999999,1e20,'N/A') + # Change the data, the mask should be conserved + mbase.a._data[:] = 5 + assert_equal(mbase['a']._data, [5,5,5,5,5]) + assert_equal(mbase['a']._mask, [0,1,0,0,1]) + # Change the elements, and the mask will follow + mbase.a = 1 + assert_equal(mbase['a']._data, [1]*5) + assert_equal(getmaskarray(mbase['a']), [0]*5) + assert_equal(mbase._mask, [False]*5) + assert_equal(mbase._fieldmask.tolist(), + narray([(0,0,0),(0,1,1),(0,0,0),(0,0,0),(0,1,1)], + dtype=bool_)) + # Set a field to mask ........................ + mbase.c = masked + assert_equal(mbase.c.mask, [1]*5) + assert_equal(getmaskarray(mbase['c']), [1]*5) + assert_equal(getdata(mbase['c']), ['N/A']*5) + assert_equal(mbase._fieldmask.tolist(), + narray([(0,0,1),(0,1,1),(0,0,1),(0,0,1),(0,1,1)], + dtype=bool_)) + # Set fields by slices ....................... + mbase = base.view(mrecarray).copy() + mbase.a[3:] = 5 + assert_equal(mbase.a, [1,2,3,5,5]) + assert_equal(mbase.a._mask, [0,1,0,0,0]) + mbase.b[3:] = masked + assert_equal(mbase.b, base['b']) + assert_equal(mbase.b._mask, [0,1,0,1,1]) + # + def test_set_mask(self): + base = self.base.copy() + mbase = base.view(mrecarray) + # Set the mask to True ....................... + mbase._mask = masked + assert_equal(getmaskarray(mbase['b']), [1]*5) + assert_equal(mbase['a']._mask, mbase['b']._mask) + assert_equal(mbase['a']._mask, mbase['c']._mask) + assert_equal(mbase._fieldmask.tolist(), + narray([(1,1,1)]*5, + dtype=bool_)) + # Delete the mask ............................ + mbase._mask = nomask + assert_equal(getmaskarray(mbase['c']), [0]*5) + assert_equal(mbase._fieldmask.tolist(), + narray([(0,0,0)]*5, + dtype=bool_)) + # + def test_set_elements(self): + base = self.base.copy() + mbase = base.view(mrecarray) + # Set an element to mask ..................... + mbase[-2] = masked + assert_equal(mbase._fieldmask.tolist(), + narray([(0,0,0),(1,1,1),(0,0,0),(1,1,1),(1,1,1)], + dtype=bool_)) + assert_equal(mbase._mask, [0,1,0,1,1]) + # Set slices ................................. + mbase = base.view(mrecarray).copy() + mbase[:2] = 5 + assert_equal(mbase.a._data, [5,5,3,4,5]) + assert_equal(mbase.a._mask, [0,0,0,0,1]) + assert_equal(mbase.b._data, [5.,5.,3.3,4.4,5.5]) + assert_equal(mbase.b._mask, [0,0,0,0,1]) + assert_equal(mbase.c._data, ['5','5','three','four','five']) + assert_equal(mbase.b._mask, [0,0,0,0,1]) + # + mbase = base.view(mrecarray).copy() + mbase[:2] = masked + assert_equal(mbase.a._data, [1,2,3,4,5]) + assert_equal(mbase.a._mask, [1,1,0,0,1]) + assert_equal(mbase.b._data, [1.1,2.2,3.3,4.4,5.5]) + assert_equal(mbase.b._mask, [1,1,0,0,1]) + assert_equal(mbase.c._data, ['one','two','three','four','five']) + assert_equal(mbase.b._mask, [1,1,0,0,1]) + # + def test_setslices_hardmask(self): + "Tests setting slices w/ hardmask." + base = self.base.copy() + mbase = base.view(mrecarray) + mbase.harden_mask() + mbase[-2:] = 5 + assert_equal(mbase.a._data, [1,2,3,5,5]) + assert_equal(mbase.b._data, [1.1,2.2,3.3,5,5.5]) + assert_equal(mbase.c._data, ['one','two','three','5','five']) + assert_equal(mbase.a._mask, [0,1,0,0,1]) + assert_equal(mbase.b._mask, mbase.a._mask) + assert_equal(mbase.b._mask, mbase.c._mask) + + def test_hardmask(self): + "Test hardmask" + base = self.base.copy() + mbase = base.view(mrecarray) + mbase.harden_mask() + assert(mbase._hardmask) + mbase._mask = nomask + assert_equal(mbase._mask, [0,1,0,0,1]) + mbase.soften_mask() + assert(not mbase._hardmask) + mbase._mask = nomask + assert(mbase['b']._mask is nomask) + assert_equal(mbase['a']._mask,mbase['b']._mask) + # + def test_pickling(self): + "Test pickling" + import cPickle + base = self.base.copy() + mrec = base.view(mrecarray) + _ = cPickle.dumps(mrec) + mrec_ = cPickle.loads(_) + assert_equal(mrec_.dtype, mrec.dtype) + assert_equal_records(mrec_._data, mrec._data) + assert_equal(mrec_._mask, mrec._mask) + assert_equal_records(mrec_._fieldmask, mrec._fieldmask) + # + def test_filled(self): + "Test filling the array" + _a = masked_array([1,2,3],mask=[0,0,1],dtype=int_) + _b = masked_array([1.1,2.2,3.3],mask=[0,0,1],dtype=float_) + _c = masked_array(['one','two','three'],mask=[0,0,1],dtype='|S8') + ddtype = [('a',int_),('b',float_),('c','|S8')] + mrec = fromarrays([_a,_b,_c], dtype=ddtype, + fill_value=(99999,99999.,'N/A')) + mrecfilled = mrec.filled() + assert_equal(mrecfilled['a'], narray((1,2,99999), dtype=int_)) + assert_equal(mrecfilled['b'], narray((1.1,2.2,99999.), dtype=float_)) + assert_equal(mrecfilled['c'], narray(('one','two','N/A'), dtype='|S8')) + # + def test_tolist(self): + "Test tolist." + _a = masked_array([1,2,3],mask=[0,0,1],dtype=int_) + _b = masked_array([1.1,2.2,3.3],mask=[0,0,1],dtype=float_) + _c = masked_array(['one','two','three'],mask=[1,0,0],dtype='|S8') + ddtype = [('a',int_),('b',float_),('c','|S8')] + mrec = fromarrays([_a,_b,_c], dtype=ddtype, + fill_value=(99999,99999.,'N/A')) + # + assert_equal(mrec.tolist(), + [(1,1.1,None),(2,2.2,'two'),(None,None,'three')]) + +################################################################################ +class TestMRecordsImport(NumpyTestCase): + "Base test class for MaskedArrays." + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + self.setup() + + def setup(self): + "Generic setup" + _a = masked_array([1,2,3],mask=[0,0,1],dtype=int_) + _b = masked_array([1.1,2.2,3.3],mask=[0,0,1],dtype=float_) + _c = masked_array(['one','two','three'],mask=[0,0,1],dtype='|S8') + ddtype = [('a',int_),('b',float_),('c','|S8')] + mrec = fromarrays([_a,_b,_c], dtype=ddtype, + fill_value=(99999,99999.,'N/A')) + nrec = recfromarrays((_a.data,_b.data,_c.data), dtype=ddtype) + self.data = (mrec, nrec, ddtype) + + def test_fromarrays(self): + _a = masked_array([1,2,3],mask=[0,0,1],dtype=int_) + _b = masked_array([1.1,2.2,3.3],mask=[0,0,1],dtype=float_) + _c = masked_array(['one','two','three'],mask=[0,0,1],dtype='|S8') + (mrec, nrec, _) = self.data + for (f,l) in zip(('a','b','c'),(_a,_b,_c)): + assert_equal(getattr(mrec,f)._mask, l._mask) + + + def test_fromrecords(self): + "Test construction from records." + (mrec, nrec, ddtype) = self.data + #...... + palist = [(1, 'abc', 3.7000002861022949, 0), + (2, 'xy', 6.6999998092651367, 1), + (0, ' ', 0.40000000596046448, 0)] + pa = recfromrecords(palist, names='c1, c2, c3, c4') + mpa = fromrecords(palist, names='c1, c2, c3, c4') + assert_equal_records(pa,mpa) + #..... + _mrec = fromrecords(nrec) + assert_equal(_mrec.dtype, mrec.dtype) + for field in _mrec.dtype.names: + assert_equal(getattr(_mrec, field), getattr(mrec._data, field)) + # + _mrec = fromrecords(nrec.tolist(), names='c1,c2,c3') + assert_equal(_mrec.dtype, [('c1',int_),('c2',float_),('c3','|S5')]) + for (f,n) in zip(('c1','c2','c3'), ('a','b','c')): + assert_equal(getattr(_mrec,f), getattr(mrec._data, n)) + # + _mrec = fromrecords(mrec) + assert_equal(_mrec.dtype, mrec.dtype) + assert_equal_records(_mrec._data, mrec.filled()) + assert_equal_records(_mrec._fieldmask, mrec._fieldmask) + + def test_fromrecords_wmask(self): + "Tests construction from records w/ mask." + (mrec, nrec, ddtype) = self.data + # + _mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=[0,1,0,]) + assert_equal_records(_mrec._data, mrec._data) + assert_equal(_mrec._fieldmask.tolist(), [(0,0,0),(1,1,1),(0,0,0)]) + # + _mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=True) + assert_equal_records(_mrec._data, mrec._data) + assert_equal(_mrec._fieldmask.tolist(), [(1,1,1),(1,1,1),(1,1,1)]) + # + _mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=mrec._fieldmask) + assert_equal_records(_mrec._data, mrec._data) + assert_equal(_mrec._fieldmask.tolist(), mrec._fieldmask.tolist()) + # + _mrec = fromrecords(nrec.tolist(), dtype=ddtype, + mask=mrec._fieldmask.tolist()) + assert_equal_records(_mrec._data, mrec._data) + assert_equal(_mrec._fieldmask.tolist(), mrec._fieldmask.tolist()) + + def test_fromtextfile(self): + "Tests reading from a text file." + fcontent = """# +'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)' +'strings',1,1.0,'mixed column',,1 +'with embedded "double quotes"',2,2.0,1.0,,1 +'strings',3,3.0E5,3,,1 +'strings',4,-1e-10,,,1 +""" + import os + from datetime import datetime + fname = 'tmp%s' % datetime.now().strftime("%y%m%d%H%M%S%s") + f = open(fname, 'w') + f.write(fcontent) + f.close() + mrectxt = fromtextfile(fname,delimitor=',',varnames='ABCDEFG') + os.unlink(fname) + # + assert(isinstance(mrectxt, MaskedRecords)) + assert_equal(mrectxt.F, [1,1,1,1]) + assert_equal(mrectxt.E._mask, [1,1,1,1]) + assert_equal(mrectxt.C, [1,2,3.e+5,-1e-10]) + + def test_addfield(self): + "Tests addfield" + (mrec, nrec, ddtype) = self.data + (d,m) = ([100,200,300], [1,0,0]) + mrec = addfield(mrec, masked_array(d, mask=m)) + assert_equal(mrec.f3, d) + assert_equal(mrec.f3._mask, m) + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/ma/tests/test_mstats.py b/numpy/ma/tests/test_mstats.py new file mode 100644 index 000000000..e4657a58f --- /dev/null +++ b/numpy/ma/tests/test_mstats.py @@ -0,0 +1,174 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for maskedArray statistics. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_mstats.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import numpy + +import numpy.ma +from numpy.ma import masked, masked_array + +import numpy.ma.testutils +from numpy.ma.testutils import * + +from numpy.ma.mstats import * + +#.............................................................................. +class TestQuantiles(NumpyTestCase): + "Base test class for MaskedArrays." + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + self.a = numpy.ma.arange(1,101) + # + def test_1d_nomask(self): + "Test quantiles 1D - w/o mask." + a = self.a + assert_almost_equal(mquantiles(a, alphap=1., betap=1.), + [25.75, 50.5, 75.25]) + assert_almost_equal(mquantiles(a, alphap=0, betap=1.), + [25., 50., 75.]) + assert_almost_equal(mquantiles(a, alphap=0.5, betap=0.5), + [25.5, 50.5, 75.5]) + assert_almost_equal(mquantiles(a, alphap=0., betap=0.), + [25.25, 50.5, 75.75]) + assert_almost_equal(mquantiles(a, alphap=1./3, betap=1./3), + [25.41666667, 50.5, 75.5833333]) + assert_almost_equal(mquantiles(a, alphap=3./8, betap=3./8), + [25.4375, 50.5, 75.5625]) + assert_almost_equal(mquantiles(a), [25.45, 50.5, 75.55])# + # + def test_1d_mask(self): + "Test quantiles 1D - w/ mask." + a = self.a + a[1::2] = masked + assert_almost_equal(mquantiles(a, alphap=1., betap=1.), + [25.5, 50.0, 74.5]) + assert_almost_equal(mquantiles(a, alphap=0, betap=1.), + [24., 49., 74.]) + assert_almost_equal(mquantiles(a, alphap=0.5, betap=0.5), + [25., 50., 75.]) + assert_almost_equal(mquantiles(a, alphap=0., betap=0.), + [24.5, 50.0, 75.5]) + assert_almost_equal(mquantiles(a, alphap=1./3, betap=1./3), + [24.833333, 50.0, 75.166666]) + assert_almost_equal(mquantiles(a, alphap=3./8, betap=3./8), + [24.875, 50., 75.125]) + assert_almost_equal(mquantiles(a), [24.9, 50., 75.1]) + # + def test_2d_nomask(self): + "Test quantiles 2D - w/o mask." + a = self.a + b = numpy.ma.resize(a, (100,100)) + assert_almost_equal(mquantiles(b), [25.45, 50.5, 75.55]) + assert_almost_equal(mquantiles(b, axis=0), numpy.ma.resize(a,(3,100))) + assert_almost_equal(mquantiles(b, axis=1), + numpy.ma.resize([25.45, 50.5, 75.55], (100,3))) + # + def test_2d_mask(self): + "Test quantiles 2D - w/ mask." + a = self.a + a[1::2] = masked + b = numpy.ma.resize(a, (100,100)) + assert_almost_equal(mquantiles(b), [25., 50., 75.]) + assert_almost_equal(mquantiles(b, axis=0), numpy.ma.resize(a,(3,100))) + assert_almost_equal(mquantiles(b, axis=1), + numpy.ma.resize([24.9, 50., 75.1], (100,3))) + +class TestMedian(NumpyTestCase): + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + + def test_2d(self): + "Tests median w/ 2D" + (n,p) = (101,30) + x = masked_array(numpy.linspace(-1.,1.,n),) + x[:10] = x[-10:] = masked + z = masked_array(numpy.empty((n,p), dtype=numpy.float_)) + z[:,0] = x[:] + idx = numpy.arange(len(x)) + for i in range(1,p): + numpy.random.shuffle(idx) + z[:,i] = x[idx] + assert_equal(mmedian(z[:,0]), 0) + assert_equal(mmedian(z), numpy.zeros((p,))) + + def test_3d(self): + "Tests median w/ 3D" + x = numpy.ma.arange(24).reshape(3,4,2) + x[x%3==0] = masked + assert_equal(mmedian(x,0), [[12,9],[6,15],[12,9],[18,15]]) + x.shape = (4,3,2) + assert_equal(mmedian(x,0),[[99,10],[11,99],[13,14]]) + x = numpy.ma.arange(24).reshape(4,3,2) + x[x%5==0] = masked + assert_equal(mmedian(x,0), [[12,10],[8,9],[16,17]]) + +#.............................................................................. +class TestTrimming(NumpyTestCase): + # + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + # + def test_trim(self): + "Tests trimming." + x = numpy.ma.arange(100) + assert_equal(trim_both(x).count(), 60) + assert_equal(trim_tail(x,tail='r').count(), 80) + x[50:70] = masked + trimx = trim_both(x) + assert_equal(trimx.count(), 48) + assert_equal(trimx._mask, [1]*16 + [0]*34 + [1]*20 + [0]*14 + [1]*16) + x._mask = nomask + x.shape = (10,10) + assert_equal(trim_both(x).count(), 60) + assert_equal(trim_tail(x).count(), 80) + # + def test_trimmedmean(self): + "Tests the trimmed mean." + data = masked_array([ 77, 87, 88,114,151,210,219,246,253,262, + 296,299,306,376,428,515,666,1310,2611]) + assert_almost_equal(trimmed_mean(data,0.1), 343, 0) + assert_almost_equal(trimmed_mean(data,0.2), 283, 0) + # + def test_trimmed_stde(self): + "Tests the trimmed mean standard error." + data = masked_array([ 77, 87, 88,114,151,210,219,246,253,262, + 296,299,306,376,428,515,666,1310,2611]) + assert_almost_equal(trimmed_stde(data,0.2), 56.1, 1) + # + def test_winsorization(self): + "Tests the Winsorization of the data." + data = masked_array([ 77, 87, 88,114,151,210,219,246,253,262, + 296,299,306,376,428,515,666,1310,2611]) + assert_almost_equal(winsorize(data).varu(), 21551.4, 1) + data[5] = masked + winsorized = winsorize(data) + assert_equal(winsorized.mask, data.mask) +#.............................................................................. + +class TestMisc(NumpyTestCase): + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + + def check_cov(self): + "Tests the cov function." + x = masked_array([[1,2,3],[4,5,6]], mask=[[1,0,0],[0,0,0]]) + c = cov(x[0]) + assert_equal(c, (x[0].anom()**2).sum()) + c = cov(x[1]) + assert_equal(c, (x[1].anom()**2).sum()/2.) + c = cov(x) + assert_equal(c[1,0], (x[0].anom()*x[1].anom()).sum()) + + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/core/tests/test_ma.py b/numpy/ma/tests/test_old_ma.py index ed7cb2a79..d4725b25a 100644 --- a/numpy/core/tests/test_ma.py +++ b/numpy/ma/tests/test_old_ma.py @@ -1,6 +1,6 @@ import numpy import types, time -from numpy.core.ma import * +from numpy.ma import * from numpy.core.numerictypes import float32 from numpy.testing import NumpyTestCase, NumpyTest pi = numpy.pi @@ -229,7 +229,7 @@ class TestMa(NumpyTestCase): x2 = masked_values(x1, 3.0) assert eq(x1,x2) assert allequal(array([0,0,0,1,0],MaskType), x2.mask) - assert eq(3.0, x2.fill_value()) + assert eq(3.0, x2.fill_value) x1 = array([1,'hello',2,3],object) x2 = numpy.array([1,'hello',2,3],object) s1 = x1[1] @@ -250,16 +250,14 @@ class TestMa(NumpyTestCase): x1 = numpy.arange(5) y1 = array(x1, mask=m) - self.failUnless( y1.raw_data() is not x1) - self.failUnless( allequal(x1,y1.raw_data())) + self.failUnless( y1.data is not x1) + self.failUnless( allequal(x1,y1.data)) self.failUnless( y1.mask is m) y1a = array(y1, copy=0) - self.failUnless( y1a.raw_data() is y1.raw_data()) self.failUnless( y1a.mask is y1.mask) y2 = array(x1, mask=m, copy=0) - self.failUnless( y2.raw_data() is x1) self.failUnless( y2.mask is m) self.failUnless( y2[2] is masked) y2[2]=9 @@ -295,30 +293,17 @@ class TestMa(NumpyTestCase): self.failUnless( eq(x, [0,10,2,-1,40])) x = array(d, mask = m) - x.put([-1,100,200]) + x.put([0,1,2],[-1,100,200]) self.failUnless( eq(x, [-1,100,200,0,0])) self.failUnless( x[3] is masked) self.failUnless( x[4] is masked) - x = array(d, mask = m) - x.putmask([30,40]) - self.failUnless( eq(x, [0,1,2,30,40])) - self.failUnless( x.mask is nomask) - - x = array(d, mask = m) - y = x.compressed() - z = array(x, mask = m) - z.put(y) - assert eq (x, z) - def check_testMaPut(self): (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d m = [1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1] i = numpy.nonzero(m)[0] - putmask(xm, m, z) - assert take(xm, i,axis=0) == z put(ym, i, zm) - assert take(ym, i,axis=0) == zm + assert all(take(ym, i, axis=0) == zm) def check_testOddFeatures(self): "Test of other odd features" @@ -490,9 +475,9 @@ class TestMa(NumpyTestCase): x = arange(10).astype(float32) xm = arange(10) xm[2] = masked - id1 = id(x.raw_data()) + id1 = id(x.data) x += 1. - assert id1 == id(x.raw_data()) + assert id1 == id(x.data) assert eq(x, y+1.) def check_testPickle(self): @@ -595,11 +580,8 @@ class TestMa(NumpyTestCase): self.assertEqual(1, int(array([[[1]]]))) self.assertEqual(1.0, float(array([[1]]))) self.failUnlessRaises(ValueError, float, array([1,1])) - self.failUnlessRaises(MAError, float, array([1],mask=[1])) - self.failUnless(bool(array([0,1]))) - self.failUnless(bool(array([0,0],mask=[0,1]))) - self.failIf(bool(array([0,0]))) - self.failIf(bool(array([0,0],mask=[0,0]))) + self.failUnlessRaises(ValueError, bool, array([0,1])) + self.failUnlessRaises(ValueError, bool, array([0,0],mask=[0,1])) def check_testScalarArithmetic(self): xm = array(0, mask=1) @@ -638,7 +620,7 @@ class TestMa(NumpyTestCase): def check_testAPI(self): self.failIf([m for m in dir(numpy.ndarray) - if m not in dir(array) and not m.startswith('_')]) + if m not in dir(MaskedArray) and not m.startswith('_')]) def check_testSingleElementSubscript(self): a = array([1,3,2]) @@ -869,5 +851,5 @@ def testinplace(x): testinplace.test_name = 'Inplace operations' if __name__ == "__main__": - NumpyTest('numpy.core.ma').run() + NumpyTest('numpy.ma').run() #timingTest() diff --git a/numpy/ma/tests/test_subclassing.py b/numpy/ma/tests/test_subclassing.py new file mode 100644 index 000000000..331ef887d --- /dev/null +++ b/numpy/ma/tests/test_subclassing.py @@ -0,0 +1,183 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for MaskedArray & subclassing. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_subclassing.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import numpy as N +import numpy.core.numeric as numeric + +from numpy.testing import NumpyTest, NumpyTestCase + +import numpy.ma.testutils +from numpy.ma.testutils import * + +import numpy.ma.core as coremodule +from numpy.ma.core import * + + +class SubArray(N.ndarray): + """Defines a generic N.ndarray subclass, that stores some metadata + in the dictionary `info`.""" + def __new__(cls,arr,info={}): + x = N.asanyarray(arr).view(cls) + x.info = info + return x + def __array_finalize__(self, obj): + self.info = getattr(obj,'info',{}) + return + def __add__(self, other): + result = N.ndarray.__add__(self, other) + result.info.update({'added':result.info.pop('added',0)+1}) + return result +subarray = SubArray + +class MSubArray(SubArray,MaskedArray): + def __new__(cls, data, info=None, mask=nomask): + subarr = SubArray(data, info) + _data = MaskedArray.__new__(cls, data=subarr, mask=mask) + _data.info = subarr.info + return _data + def __array_finalize__(self,obj): + MaskedArray.__array_finalize__(self,obj) + SubArray.__array_finalize__(self, obj) + return + def _get_series(self): + _view = self.view(MaskedArray) + _view._sharedmask = False + return _view + _series = property(fget=_get_series) +msubarray = MSubArray + +class MMatrix(MaskedArray, N.matrix,): + def __new__(cls, data, mask=nomask): + mat = N.matrix(data) + _data = MaskedArray.__new__(cls, data=mat, mask=mask) + return _data + def __array_finalize__(self,obj): + N.matrix.__array_finalize__(self, obj) + MaskedArray.__array_finalize__(self,obj) + return + def _get_series(self): + _view = self.view(MaskedArray) + _view._sharedmask = False + return _view + _series = property(fget=_get_series) +mmatrix = MMatrix + + + +class TestSubclassing(NumpyTestCase): + """Test suite for masked subclasses of ndarray.""" + + def check_data_subclassing(self): + "Tests whether the subclass is kept." + x = N.arange(5) + m = [0,0,1,0,0] + xsub = SubArray(x) + xmsub = masked_array(xsub, mask=m) + assert isinstance(xmsub, MaskedArray) + assert_equal(xmsub._data, xsub) + assert isinstance(xmsub._data, SubArray) + + def check_maskedarray_subclassing(self): + "Tests subclassing MaskedArray" + x = N.arange(5) + mx = mmatrix(x,mask=[0,1,0,0,0]) + assert isinstance(mx._data, N.matrix) + "Tests masked_unary_operation" + assert isinstance(add(mx,mx), mmatrix) + assert isinstance(add(mx,x), mmatrix) + assert_equal(add(mx,x), mx+x) + assert isinstance(add(mx,mx)._data, N.matrix) + assert isinstance(add.outer(mx,mx), mmatrix) + "Tests masked_binary_operation" + assert isinstance(hypot(mx,mx), mmatrix) + assert isinstance(hypot(mx,x), mmatrix) + + def check_attributepropagation(self): + x = array(arange(5), mask=[0]+[1]*4) + my = masked_array(subarray(x)) + ym = msubarray(x) + # + z = (my+1) + assert isinstance(z,MaskedArray) + assert not isinstance(z, MSubArray) + assert isinstance(z._data, SubArray) + assert_equal(z._data.info, {}) + # + z = (ym+1) + assert isinstance(z, MaskedArray) + assert isinstance(z, MSubArray) + assert isinstance(z._data, SubArray) + assert z._data.info['added'] > 0 + # + ym._set_mask([1,0,0,0,1]) + assert_equal(ym._mask, [1,0,0,0,1]) + ym._series._set_mask([0,0,0,0,1]) + assert_equal(ym._mask, [0,0,0,0,1]) + # + xsub = subarray(x, info={'name':'x'}) + mxsub = masked_array(xsub) + assert hasattr(mxsub, 'info') + assert_equal(mxsub.info, xsub.info) + + def check_subclasspreservation(self): + "Checks that masked_array(...,subok=True) preserves the class." + x = N.arange(5) + m = [0,0,1,0,0] + xinfo = [(i,j) for (i,j) in zip(x,m)] + xsub = MSubArray(x, mask=m, info={'xsub':xinfo}) + # + mxsub = masked_array(xsub, subok=False) + assert not isinstance(mxsub, MSubArray) + assert isinstance(mxsub, MaskedArray) + assert_equal(mxsub._mask, m) + # + mxsub = asarray(xsub) + assert not isinstance(mxsub, MSubArray) + assert isinstance(mxsub, MaskedArray) + assert_equal(mxsub._mask, m) + # + mxsub = masked_array(xsub, subok=True) + assert isinstance(mxsub, MSubArray) + assert_equal(mxsub.info, xsub.info) + assert_equal(mxsub._mask, xsub._mask) + # + mxsub = asanyarray(xsub) + assert isinstance(mxsub, MSubArray) + assert_equal(mxsub.info, xsub.info) + assert_equal(mxsub._mask, m) + + +################################################################################ +if __name__ == '__main__': + NumpyTest().run() + # + if 0: + x = array(arange(5), mask=[0]+[1]*4) + my = masked_array(subarray(x)) + ym = msubarray(x) + # + z = (my+1) + assert isinstance(z,MaskedArray) + assert not isinstance(z, MSubArray) + assert isinstance(z._data, SubArray) + assert_equal(z._data.info, {}) + # + z = (ym+1) + assert isinstance(z, MaskedArray) + assert isinstance(z, MSubArray) + assert isinstance(z._data, SubArray) + assert z._data.info['added'] > 0 + # + ym._set_mask([1,0,0,0,1]) + assert_equal(ym._mask, [1,0,0,0,1]) + ym._series._set_mask([0,0,0,0,1]) + assert_equal(ym._mask, [0,0,0,0,1]) diff --git a/numpy/ma/testutils.py b/numpy/ma/testutils.py new file mode 100644 index 000000000..4ea49379f --- /dev/null +++ b/numpy/ma/testutils.py @@ -0,0 +1,219 @@ +"""Miscellaneous functions for testing masked arrays and subclasses + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: testutils.py 3529 2007-11-13 08:01:14Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = "1.0" +__revision__ = "$Revision: 3529 $" +__date__ = "$Date: 2007-11-13 10:01:14 +0200 (Tue, 13 Nov 2007) $" + + +import numpy as N +from numpy.core import ndarray +from numpy.core.numerictypes import float_ +import numpy.core.umath as umath +from numpy.testing import NumpyTest, NumpyTestCase +from numpy.testing.utils import build_err_msg, rand + +import core +from core import mask_or, getmask, getmaskarray, masked_array, nomask, masked +from core import filled, equal, less + +#------------------------------------------------------------------------------ +def approx (a, b, fill_value=True, rtol=1.e-5, atol=1.e-8): + """Returns true if all components of a and b are equal subject to given tolerances. + +If fill_value is True, masked values considered equal. Otherwise, masked values +are considered unequal. +The relative error rtol should be positive and << 1.0 +The absolute error atol comes into play for those elements of b that are very +small or zero; it says how small a must be also. + """ + m = mask_or(getmask(a), getmask(b)) + d1 = filled(a) + d2 = filled(b) + if d1.dtype.char == "O" or d2.dtype.char == "O": + return N.equal(d1,d2).ravel() + x = filled(masked_array(d1, copy=False, mask=m), fill_value).astype(float_) + y = filled(masked_array(d2, copy=False, mask=m), 1).astype(float_) + d = N.less_equal(umath.absolute(x-y), atol + rtol * umath.absolute(y)) + return d.ravel() +#................................................ +def _assert_equal_on_sequences(actual, desired, err_msg=''): + "Asserts the equality of two non-array sequences." + assert_equal(len(actual),len(desired),err_msg) + for k in range(len(desired)): + assert_equal(actual[k], desired[k], 'item=%r\n%s' % (k,err_msg)) + return + +def assert_equal_records(a,b): + """Asserts that two records are equal. Pretty crude for now.""" + assert_equal(a.dtype, b.dtype) + for f in a.dtype.names: + (af, bf) = (getattr(a,f), getattr(b,f)) + if not (af is masked) and not (bf is masked): + assert_equal(getattr(a,f), getattr(b,f)) + return + +def assert_equal(actual,desired,err_msg=''): + """Asserts that two items are equal. + """ + # Case #1: dictionary ..... + if isinstance(desired, dict): + assert isinstance(actual, dict), repr(type(actual)) + assert_equal(len(actual),len(desired),err_msg) + for k,i in desired.items(): + assert k in actual, repr(k) + assert_equal(actual[k], desired[k], 'key=%r\n%s' % (k,err_msg)) + return + # Case #2: lists ..... + if isinstance(desired, (list,tuple)) and isinstance(actual, (list,tuple)): + return _assert_equal_on_sequences(actual, desired, err_msg='') + if not (isinstance(actual, ndarray) or isinstance(desired, ndarray)): + msg = build_err_msg([actual, desired], err_msg,) + assert desired == actual, msg + return + # Case #4. arrays or equivalent + if ((actual is masked) and not (desired is masked)) or \ + ((desired is masked) and not (actual is masked)): + msg = build_err_msg([actual, desired], err_msg, header='', names=('x', 'y')) + raise ValueError(msg) + actual = N.array(actual, copy=False, subok=True) + desired = N.array(desired, copy=False, subok=True) + if actual.dtype.char in "OS" and desired.dtype.char in "OS": + return _assert_equal_on_sequences(actual.tolist(), + desired.tolist(), + err_msg='') + return assert_array_equal(actual, desired, err_msg) +#............................. +def fail_if_equal(actual,desired,err_msg='',): + """Raises an assertion error if two items are equal. + """ + if isinstance(desired, dict): + assert isinstance(actual, dict), repr(type(actual)) + fail_if_equal(len(actual),len(desired),err_msg) + for k,i in desired.items(): + assert k in actual, repr(k) + fail_if_equal(actual[k], desired[k], 'key=%r\n%s' % (k,err_msg)) + return + if isinstance(desired, (list,tuple)) and isinstance(actual, (list,tuple)): + fail_if_equal(len(actual),len(desired),err_msg) + for k in range(len(desired)): + fail_if_equal(actual[k], desired[k], 'item=%r\n%s' % (k,err_msg)) + return + if isinstance(actual, N.ndarray) or isinstance(desired, N.ndarray): + return fail_if_array_equal(actual, desired, err_msg) + msg = build_err_msg([actual, desired], err_msg) + assert desired != actual, msg +assert_not_equal = fail_if_equal +#............................ +def assert_almost_equal(actual,desired,decimal=7,err_msg=''): + """Asserts that two items are almost equal. + The test is equivalent to abs(desired-actual) < 0.5 * 10**(-decimal) + """ + if isinstance(actual, N.ndarray) or isinstance(desired, N.ndarray): + return assert_array_almost_equal(actual, desired, decimal, err_msg) + msg = build_err_msg([actual, desired], err_msg) + assert round(abs(desired - actual),decimal) == 0, msg +#............................ +def assert_array_compare(comparison, x, y, err_msg='', header='', + fill_value=True): + """Asserts that a comparison relation between two masked arrays is satisfied + elementwise.""" + xf = filled(x) + yf = filled(y) + m = mask_or(getmask(x), getmask(y)) + + x = masked_array(xf, copy=False, subok=False, mask=m).filled(fill_value) + y = masked_array(yf, copy=False, subok=False, mask=m).filled(fill_value) + + if ((x is masked) and not (y is masked)) or \ + ((y is masked) and not (x is masked)): + msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y')) + raise ValueError(msg) + + if (x.dtype.char != "O") and (x.dtype.char != "S"): + x = x.astype(float_) + if isinstance(x, N.ndarray) and x.size > 1: + x[N.isnan(x)] = 0 + elif N.isnan(x): + x = 0 + if (y.dtype.char != "O") and (y.dtype.char != "S"): + y = y.astype(float_) + if isinstance(y, N.ndarray) and y.size > 1: + y[N.isnan(y)] = 0 + elif N.isnan(y): + y = 0 + try: + cond = (x.shape==() or y.shape==()) or x.shape == y.shape + if not cond: + msg = build_err_msg([x, y], + err_msg + + '\n(shapes %s, %s mismatch)' % (x.shape, + y.shape), + header=header, + names=('x', 'y')) + assert cond, msg + val = comparison(x,y) + if m is not nomask and fill_value: + val = masked_array(val, mask=m, copy=False) + if isinstance(val, bool): + cond = val + reduced = [0] + else: + reduced = val.ravel() + cond = reduced.all() + reduced = reduced.tolist() + if not cond: + match = 100-100.0*reduced.count(1)/len(reduced) + msg = build_err_msg([x, y], + err_msg + + '\n(mismatch %s%%)' % (match,), + header=header, + names=('x', 'y')) + assert cond, msg + except ValueError: + msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y')) + raise ValueError(msg) +#............................ +def assert_array_equal(x, y, err_msg=''): + """Checks the elementwise equality of two masked arrays.""" + assert_array_compare(equal, x, y, err_msg=err_msg, + header='Arrays are not equal') +##............................ +def fail_if_array_equal(x, y, err_msg=''): + "Raises an assertion error if two masked arrays are not equal (elementwise)." + def compare(x,y): + + return (not N.alltrue(approx(x, y))) + assert_array_compare(compare, x, y, err_msg=err_msg, + header='Arrays are not equal') +#............................ +def assert_array_almost_equal(x, y, decimal=6, err_msg=''): + """Checks the elementwise equality of two masked arrays, up to a given + number of decimals.""" + def compare(x, y): + "Returns the result of the loose comparison between x and y)." + return approx(x,y, rtol=10.**-decimal) + assert_array_compare(compare, x, y, err_msg=err_msg, + header='Arrays are not almost equal') +#............................ +def assert_array_less(x, y, err_msg=''): + "Checks that x is smaller than y elementwise." + assert_array_compare(less, x, y, err_msg=err_msg, + header='Arrays are not less-ordered') +#............................ +assert_close = assert_almost_equal +#............................ +def assert_mask_equal(m1, m2): + """Asserts the equality of two masks.""" + if m1 is nomask: + assert(m2 is nomask) + if m2 is nomask: + assert(m1 is nomask) + assert_array_equal(m1, m2) + +if __name__ == '__main__': + pass
\ No newline at end of file diff --git a/numpy/ma/timer_comparison.py b/numpy/ma/timer_comparison.py new file mode 100644 index 000000000..45f0b22a6 --- /dev/null +++ b/numpy/ma/timer_comparison.py @@ -0,0 +1,457 @@ + +import timeit + +import numpy +from numpy import int_, float_, bool_ +import numpy.core.fromnumeric as fromnumeric + +from numpy.testing.utils import build_err_msg, rand + + +numpy.seterr(all='ignore') + +pi = numpy.pi + +class moduletester: + #----------------------------------- + def __init__(self, module): + self.module = module + self.allequal = module.allequal + self.arange = module.arange + self.array = module.array +# self.average = module.average + self.concatenate = module.concatenate + self.count = module.count + self.equal = module.equal + self.filled = module.filled + self.getmask = module.getmask + self.getmaskarray = module.getmaskarray + self.id = id + self.inner = module.inner + self.make_mask = module.make_mask + self.masked = module.masked + self.masked_array = module.masked_array + self.masked_values = module.masked_values + self.mask_or = module.mask_or + self.nomask = module.nomask + self.ones = module.ones + self.outer = module.outer + self.repeat = module.repeat + self.resize = module.resize + self.sort = module.sort + self.take = module.take + self.transpose = module.transpose + self.zeros = module.zeros + self.MaskType = module.MaskType + try: + self.umath = module.umath + except AttributeError: + self.umath = module.core.umath + self.testnames = [] + #........................ + def assert_array_compare(self, comparison, x, y, err_msg='', header='', + fill_value=True): + """Asserts that a comparison relation between two masked arrays is satisfied + elementwise.""" + xf = self.filled(x) + yf = self.filled(y) + m = self.mask_or(self.getmask(x), self.getmask(y)) + + x = self.filled(self.masked_array(xf, mask=m), fill_value) + y = self.filled(self.masked_array(yf, mask=m), fill_value) + if (x.dtype.char != "O"): + x = x.astype(float_) + if isinstance(x, numpy.ndarray) and x.size > 1: + x[numpy.isnan(x)] = 0 + elif numpy.isnan(x): + x = 0 + if (y.dtype.char != "O"): + y = y.astype(float_) + if isinstance(y, numpy.ndarray) and y.size > 1: + y[numpy.isnan(y)] = 0 + elif numpy.isnan(y): + y = 0 + try: + cond = (x.shape==() or y.shape==()) or x.shape == y.shape + if not cond: + msg = build_err_msg([x, y], + err_msg + + '\n(shapes %s, %s mismatch)' % (x.shape, + y.shape), + header=header, + names=('x', 'y')) + assert cond, msg + val = comparison(x,y) + if m is not self.nomask and fill_value: + val = self.masked_array(val, mask=m) + if isinstance(val, bool): + cond = val + reduced = [0] + else: + reduced = val.ravel() + cond = reduced.all() + reduced = reduced.tolist() + if not cond: + match = 100-100.0*reduced.count(1)/len(reduced) + msg = build_err_msg([x, y], + err_msg + + '\n(mismatch %s%%)' % (match,), + header=header, + names=('x', 'y')) + assert cond, msg + except ValueError: + msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y')) + raise ValueError(msg) + #............................ + def assert_array_equal(self, x, y, err_msg=''): + """Checks the elementwise equality of two masked arrays.""" + self.assert_array_compare(self.equal, x, y, err_msg=err_msg, + header='Arrays are not equal') + #---------------------------------- + def test_0(self): + "Tests creation" + x = numpy.array([1.,1.,1.,-2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.]) + m = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + xm = self.masked_array(x, mask=m) + xm[0] + #---------------------------------- + def test_1(self): + "Tests creation" + x = numpy.array([1.,1.,1.,-2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.]) + y = numpy.array([5.,0.,3., 2., -1., -4., 0., -10., 10., 1., 0., 3.]) + a10 = 10. + m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + m2 = [0, 0, 1, 0, 0, 1, 1, 0, 0, 0 ,0, 1] + xm = self.masked_array(x, mask=m1) + ym = self.masked_array(y, mask=m2) + z = numpy.array([-.5, 0., .5, .8]) + zm = self.masked_array(z, mask=[0,1,0,0]) + xf = numpy.where(m1, 1.e+20, x) + xm.set_fill_value(1.e+20) + #..... + assert((xm-ym).filled(0).any()) + #fail_if_equal(xm.mask.astype(int_), ym.mask.astype(int_)) + s = x.shape + assert(xm.size == reduce(lambda x,y:x*y, s)) + assert(self.count(xm) == len(m1) - reduce(lambda x,y:x+y, m1)) + #..... + for s in [(4,3), (6,2)]: + x.shape = s + y.shape = s + xm.shape = s + ym.shape = s + xf.shape = s + + assert(self.count(xm) == len(m1) - reduce(lambda x,y:x+y, m1)) + #---------------------------------- + def test_2(self): + "Tests conversions and indexing" + x1 = numpy.array([1,2,4,3]) + x2 = self.array(x1, mask=[1,0,0,0]) + x3 = self.array(x1, mask=[0,1,0,1]) + x4 = self.array(x1) + # test conversion to strings + junk, garbage = str(x2), repr(x2) +# assert_equal(numpy.sort(x1), self.sort(x2, fill_value=0)) + # tests of indexing + assert type(x2[1]) is type(x1[1]) + assert x1[1] == x2[1] +# assert self.allequal(x1[2],x2[2]) +# assert self.allequal(x1[2:5],x2[2:5]) +# assert self.allequal(x1[:],x2[:]) +# assert self.allequal(x1[1:], x3[1:]) + x1[2] = 9 + x2[2] = 9 + self.assert_array_equal(x1,x2) + x1[1:3] = 99 + x2[1:3] = 99 +# assert self.allequal(x1,x2) + x2[1] = self.masked +# assert self.allequal(x1,x2) + x2[1:3] = self.masked +# assert self.allequal(x1,x2) + x2[:] = x1 + x2[1] = self.masked +# assert self.allequal(self.getmask(x2),self.array([0,1,0,0])) + x3[:] = self.masked_array([1,2,3,4],[0,1,1,0]) +# assert self.allequal(self.getmask(x3), self.array([0,1,1,0])) + x4[:] = self.masked_array([1,2,3,4],[0,1,1,0]) +# assert self.allequal(self.getmask(x4), self.array([0,1,1,0])) +# assert self.allequal(x4, self.array([1,2,3,4])) + x1 = numpy.arange(5)*1.0 + x2 = self.masked_values(x1, 3.0) +# assert self.allequal(x1,x2) +# assert self.allequal(self.array([0,0,0,1,0], self.MaskType), x2.mask) + x1 = self.array([1,'hello',2,3],object) + x2 = numpy.array([1,'hello',2,3],object) + s1 = x1[1] + s2 = x2[1] + assert x1[1:1].shape == (0,) + # Tests copy-size + n = [0,0,1,0,0] + m = self.make_mask(n) + m2 = self.make_mask(m) + assert(m is m2) + m3 = self.make_mask(m, copy=1) + assert(m is not m3) + + #---------------------------------- + def test_3(self): + "Tests resize/repeat" + x4 = self.arange(4) + x4[2] = self.masked + y4 = self.resize(x4, (8,)) + assert self.allequal(self.concatenate([x4,x4]), y4) + assert self.allequal(self.getmask(y4),[0,0,1,0,0,0,1,0]) + y5 = self.repeat(x4, (2,2,2,2), axis=0) + self.assert_array_equal(y5, [0,0,1,1,2,2,3,3]) + y6 = self.repeat(x4, 2, axis=0) + assert self.allequal(y5, y6) + y7 = x4.repeat((2,2,2,2), axis=0) + assert self.allequal(y5,y7) + y8 = x4.repeat(2,0) + assert self.allequal(y5,y8) + + #---------------------------------- + def test_4(self): + "Test of take, transpose, inner, outer products" + x = self.arange(24) + y = numpy.arange(24) + x[5:6] = self.masked + x = x.reshape(2,3,4) + y = y.reshape(2,3,4) + assert self.allequal(numpy.transpose(y,(2,0,1)), self.transpose(x,(2,0,1))) + assert self.allequal(numpy.take(y, (2,0,1), 1), self.take(x, (2,0,1), 1)) + assert self.allequal(numpy.inner(self.filled(x,0), self.filled(y,0)), + self.inner(x, y)) + assert self.allequal(numpy.outer(self.filled(x,0), self.filled(y,0)), + self.outer(x, y)) + y = self.array(['abc', 1, 'def', 2, 3], object) + y[2] = self.masked + t = self.take(y,[0,3,4]) + assert t[0] == 'abc' + assert t[1] == 2 + assert t[2] == 3 + #---------------------------------- + def test_5(self): + "Tests inplace w/ scalar" + + x = self.arange(10) + y = self.arange(10) + xm = self.arange(10) + xm[2] = self.masked + x += 1 + assert self.allequal(x, y+1) + xm += 1 + assert self.allequal(xm, y+1) + + x = self.arange(10) + xm = self.arange(10) + xm[2] = self.masked + x -= 1 + assert self.allequal(x, y-1) + xm -= 1 + assert self.allequal(xm, y-1) + + x = self.arange(10)*1.0 + xm = self.arange(10)*1.0 + xm[2] = self.masked + x *= 2.0 + assert self.allequal(x, y*2) + xm *= 2.0 + assert self.allequal(xm, y*2) + + x = self.arange(10)*2 + xm = self.arange(10)*2 + xm[2] = self.masked + x /= 2 + assert self.allequal(x, y) + xm /= 2 + assert self.allequal(xm, y) + + x = self.arange(10)*1.0 + xm = self.arange(10)*1.0 + xm[2] = self.masked + x /= 2.0 + assert self.allequal(x, y/2.0) + xm /= self.arange(10) + self.assert_array_equal(xm, self.ones((10,))) + + x = self.arange(10).astype(float_) + xm = self.arange(10) + xm[2] = self.masked + id1 = self.id(x.raw_data()) + x += 1. + #assert id1 == self.id(x.raw_data()) + assert self.allequal(x, y+1.) + + + def test_6(self): + "Tests inplace w/ array" + + x = self.arange(10, dtype=float_) + y = self.arange(10) + xm = self.arange(10, dtype=float_) + xm[2] = self.masked + m = xm.mask + a = self.arange(10, dtype=float_) + a[-1] = self.masked + x += a + xm += a + assert self.allequal(x,y+a) + assert self.allequal(xm,y+a) + assert self.allequal(xm.mask, self.mask_or(m,a.mask)) + + x = self.arange(10, dtype=float_) + xm = self.arange(10, dtype=float_) + xm[2] = self.masked + m = xm.mask + a = self.arange(10, dtype=float_) + a[-1] = self.masked + x -= a + xm -= a + assert self.allequal(x,y-a) + assert self.allequal(xm,y-a) + assert self.allequal(xm.mask, self.mask_or(m,a.mask)) + + x = self.arange(10, dtype=float_) + xm = self.arange(10, dtype=float_) + xm[2] = self.masked + m = xm.mask + a = self.arange(10, dtype=float_) + a[-1] = self.masked + x *= a + xm *= a + assert self.allequal(x,y*a) + assert self.allequal(xm,y*a) + assert self.allequal(xm.mask, self.mask_or(m,a.mask)) + + x = self.arange(10, dtype=float_) + xm = self.arange(10, dtype=float_) + xm[2] = self.masked + m = xm.mask + a = self.arange(10, dtype=float_) + a[-1] = self.masked + x /= a + xm /= a + + #---------------------------------- + def test_7(self): + "Tests ufunc" + d = (self.array([1.0, 0, -1, pi/2]*2, mask=[0,1]+[0]*6), + self.array([1.0, 0, -1, pi/2]*2, mask=[1,0]+[0]*6),) + for f in ['sqrt', 'log', 'log10', 'exp', 'conjugate', +# 'sin', 'cos', 'tan', +# 'arcsin', 'arccos', 'arctan', +# 'sinh', 'cosh', 'tanh', +# 'arcsinh', +# 'arccosh', +# 'arctanh', +# 'absolute', 'fabs', 'negative', +# # 'nonzero', 'around', +# 'floor', 'ceil', +# # 'sometrue', 'alltrue', +# 'logical_not', +# 'add', 'subtract', 'multiply', +# 'divide', 'true_divide', 'floor_divide', +# 'remainder', 'fmod', 'hypot', 'arctan2', +# 'equal', 'not_equal', 'less_equal', 'greater_equal', +# 'less', 'greater', +# 'logical_and', 'logical_or', 'logical_xor', + ]: + #print f + try: + uf = getattr(self.umath, f) + except AttributeError: + uf = getattr(fromnumeric, f) + mf = getattr(self.module, f) + args = d[:uf.nin] + ur = uf(*args) + mr = mf(*args) + self.assert_array_equal(ur.filled(0), mr.filled(0), f) + self.assert_array_equal(ur._mask, mr._mask) + + #---------------------------------- + def test_99(self): + # test average + ott = self.array([0.,1.,2.,3.], mask=[1,0,0,0]) + self.assert_array_equal(2.0, self.average(ott,axis=0)) + self.assert_array_equal(2.0, self.average(ott, weights=[1., 1., 2., 1.])) + result, wts = self.average(ott, weights=[1.,1.,2.,1.], returned=1) + self.assert_array_equal(2.0, result) + assert(wts == 4.0) + ott[:] = self.masked + assert(self.average(ott,axis=0) is self.masked) + ott = self.array([0.,1.,2.,3.], mask=[1,0,0,0]) + ott = ott.reshape(2,2) + ott[:,1] = self.masked + self.assert_array_equal(self.average(ott,axis=0), [2.0, 0.0]) + assert(self.average(ott,axis=1)[0] is self.masked) + self.assert_array_equal([2.,0.], self.average(ott, axis=0)) + result, wts = self.average(ott, axis=0, returned=1) + self.assert_array_equal(wts, [1., 0.]) + w1 = [0,1,1,1,1,0] + w2 = [[0,1,1,1,1,0],[1,0,0,0,0,1]] + x = self.arange(6) + self.assert_array_equal(self.average(x, axis=0), 2.5) + self.assert_array_equal(self.average(x, axis=0, weights=w1), 2.5) + y = self.array([self.arange(6), 2.0*self.arange(6)]) + self.assert_array_equal(self.average(y, None), numpy.add.reduce(numpy.arange(6))*3./12.) + self.assert_array_equal(self.average(y, axis=0), numpy.arange(6) * 3./2.) + self.assert_array_equal(self.average(y, axis=1), [self.average(x,axis=0), self.average(x,axis=0) * 2.0]) + self.assert_array_equal(self.average(y, None, weights=w2), 20./6.) + self.assert_array_equal(self.average(y, axis=0, weights=w2), [0.,1.,2.,3.,4.,10.]) + self.assert_array_equal(self.average(y, axis=1), [self.average(x,axis=0), self.average(x,axis=0) * 2.0]) + m1 = self.zeros(6) + m2 = [0,0,1,1,0,0] + m3 = [[0,0,1,1,0,0],[0,1,1,1,1,0]] + m4 = self.ones(6) + m5 = [0, 1, 1, 1, 1, 1] + self.assert_array_equal(self.average(self.masked_array(x, m1),axis=0), 2.5) + self.assert_array_equal(self.average(self.masked_array(x, m2),axis=0), 2.5) + # assert(self.average(masked_array(x, m4),axis=0) is masked) + self.assert_array_equal(self.average(self.masked_array(x, m5),axis=0), 0.0) + self.assert_array_equal(self.count(self.average(self.masked_array(x, m4),axis=0)), 0) + z = self.masked_array(y, m3) + self.assert_array_equal(self.average(z, None), 20./6.) + self.assert_array_equal(self.average(z, axis=0), [0.,1.,99.,99.,4.0, 7.5]) + self.assert_array_equal(self.average(z, axis=1), [2.5, 5.0]) + self.assert_array_equal(self.average(z,axis=0, weights=w2), [0.,1., 99., 99., 4.0, 10.0]) + #------------------------ + def test_A(self): + x = self.arange(24) + y = numpy.arange(24) + x[5:6] = self.masked + x = x.reshape(2,3,4) + + +################################################################################ +if __name__ == '__main__': + + setup_base = "from __main__ import moduletester \n"\ + "import numpy\n" \ + "tester = moduletester(module)\n" +# setup_new = "import numpy.ma.core_ini as module\n"+setup_base + setup_cur = "import numpy.ma.core as module\n"+setup_base +# setup_alt = "import numpy.ma.core_alt as module\n"+setup_base +# setup_tmp = "import numpy.ma.core_tmp as module\n"+setup_base + + (nrepeat, nloop) = (10, 10) + + if 1: + for i in range(1,8): + func = 'tester.test_%i()' % i +# new = timeit.Timer(func, setup_new).repeat(nrepeat, nloop*10) + cur = timeit.Timer(func, setup_cur).repeat(nrepeat, nloop*10) +# alt = timeit.Timer(func, setup_alt).repeat(nrepeat, nloop*10) +# tmp = timeit.Timer(func, setup_tmp).repeat(nrepeat, nloop*10) +# new = numpy.sort(new) + cur = numpy.sort(cur) +# alt = numpy.sort(alt) +# tmp = numpy.sort(tmp) + print "#%i" % i +50*'.' + print eval("moduletester.test_%i.__doc__" % i) +# print "core_ini : %.3f - %.3f" % (new[0], new[1]) + print "core_current : %.3f - %.3f" % (cur[0], cur[1]) +# print "core_alt : %.3f - %.3f" % (alt[0], alt[1]) +# print "core_tmp : %.3f - %.3f" % (tmp[0], tmp[1]) diff --git a/numpy/ma/version.py b/numpy/ma/version.py new file mode 100644 index 000000000..7a925f1a8 --- /dev/null +++ b/numpy/ma/version.py @@ -0,0 +1,11 @@ +"""Version number""" + +version = '1.00' +release = False + +if not release: + import core + import extras + revision = [core.__revision__.split(':')[-1][:-1].strip(), + extras.__revision__.split(':')[-1][:-1].strip(),] + version += '.dev%04i' % max([int(rev) for rev in revision]) diff --git a/numpy/setup.py b/numpy/setup.py index 9ecf1087b..40375651f 100644 --- a/numpy/setup.py +++ b/numpy/setup.py @@ -13,6 +13,7 @@ def configuration(parent_package='',top_path=None): config.add_subpackage('fft') config.add_subpackage('linalg') config.add_subpackage('random') + config.add_subpackage('ma') config.add_data_dir('doc') config.add_data_dir('tests') config.make_config_py() # installs __config__.py |