From 9c7e6e30315ab4541775d9a78630e88423c752c1 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Fri, 4 Oct 2013 13:49:33 -0600 Subject: BUG: Refactor nanfunctions to behave as agreed on for 1.9. Deal with subclasses of ndarray, like pandas.Series and matrix. Subclasses may not define the new keyword keepdims or deal gracefully with ufuncs in all their forms. This is solved by throwing the problem onto the np.sum, np.any, etc. functions that have ugly hacks to deal with the problem. Settle handling of all-nan slices. nanmax, nanmin -- Raise warning, return NaN for slice. nanargmax, nanargmin -- Raise ValueError. nansum -- Return 0 for slice. nanmean, nanvar, nanstd -- Raise warning, return NaN for slice. Make NaN functions work for scalar arguments. This may seem silly, but it removes a check for special cases. Update tests Deal with new all-nan handling. Test with matrix class as example of subclass without keepdims. Test with scalar arguments. Fix nanvar issue reported in #3860. Closes #3860 #3850 --- numpy/lib/tests/test_nanfunctions.py | 237 +++++++++++++++++++++++++---------- 1 file changed, 171 insertions(+), 66 deletions(-) (limited to 'numpy/lib/tests/test_nanfunctions.py') diff --git a/numpy/lib/tests/test_nanfunctions.py b/numpy/lib/tests/test_nanfunctions.py index 70e7865db..af01a7167 100644 --- a/numpy/lib/tests/test_nanfunctions.py +++ b/numpy/lib/tests/test_nanfunctions.py @@ -7,31 +7,25 @@ from numpy.testing import ( run_module_suite, TestCase, assert_, assert_equal, assert_almost_equal, assert_raises ) -from numpy.lib import ( - nansum, nanmax, nanargmax, nanargmin, nanmin, nanmean, nanvar, nanstd, - NanWarning - ) -_ndat = np.array( - [[0.6244, np.nan, 0.2692, 0.0116, np.nan, 0.1170], - [0.5351, 0.9403, np.nan, 0.2100, 0.4759, 0.2833], - [np.nan, np.nan, np.nan, 0.1042, np.nan, 0.5954], - [0.161, np.nan, np.nan, 0.1859, 0.3146, np.nan]] -) +# Test data +_ndat = np.array([[0.6244, np.nan, 0.2692, 0.0116, np.nan, 0.1170], + [0.5351, 0.9403, np.nan, 0.2100, 0.4759, 0.2833], + [np.nan, np.nan, np.nan, 0.1042, np.nan, 0.5954], + [0.1610, np.nan, np.nan, 0.1859, 0.3146, np.nan]]) + -# rows of _ndat with nans removed -_rdat = [ - np.array([0.6244, 0.2692, 0.0116, 0.1170]), - np.array([0.5351, 0.9403, 0.2100, 0.4759, 0.2833]), - np.array([0.1042, 0.5954]), - np.array([0.1610, 0.1859, 0.3146]) -] +# Rows of _ndat with nans removed +_rdat = [np.array([ 0.6244, 0.2692, 0.0116, 0.1170]), + np.array([ 0.5351, 0.9403, 0.2100, 0.4759, 0.2833]), + np.array([ 0.1042, 0.5954]), + np.array([ 0.1610, 0.1859, 0.3146])] class TestNanFunctions_MinMax(TestCase): - nanfuncs = [nanmin, nanmax] + nanfuncs = [np.nanmin, np.nanmax] stdfuncs = [np.min, np.max] def test_mutation(self): @@ -81,22 +75,50 @@ class TestNanFunctions_MinMax(TestCase): mat = np.array([np.nan]*9).reshape(3, 3) for f in self.nanfuncs: for axis in [None, 0, 1]: - assert_(np.isnan(f(mat, axis=axis)).all()) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + assert_(np.isnan(f(mat, axis=axis)).all()) + assert_(len(w) == 1, 'no warning raised') + assert_(issubclass(w[0].category, RuntimeWarning)) + # Check scalars + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + assert_(np.isnan(f(np.nan))) + assert_(len(w) == 1, 'no warning raised') + assert_(issubclass(w[0].category, RuntimeWarning)) def test_masked(self): mat = np.ma.fix_invalid(_ndat) msk = mat._mask.copy() - for f in [nanmin]: + for f in [np.nanmin]: res = f(mat, axis=1) tgt = f(_ndat, axis=1) assert_equal(res, tgt) assert_equal(mat._mask, msk) assert_(not np.isinf(mat).any()) + def test_scalar(self): + for f in self.nanfuncs: + assert_(f(0.) == 0.) + + def test_matrices(self): + # Check that it works and that type and + # shape are preserved + mat = np.matrix(np.eye(3)) + for f in self.nanfuncs: + res = f(mat, axis=0) + assert_(isinstance(res, np.matrix)) + assert_(res.shape == (1, 3)) + res = f(mat, axis=1) + assert_(isinstance(res, np.matrix)) + assert_(res.shape == (3, 1)) + res = f(mat) + assert_(np.isscalar(res)) + class TestNanFunctions_ArgminArgmax(TestCase): - nanfuncs = [nanargmin, nanargmax] + nanfuncs = [np.nanargmin, np.nanargmax] def test_mutation(self): # Check that passed array is not modified. @@ -120,15 +142,10 @@ class TestNanFunctions_ArgminArgmax(TestCase): def test_allnans(self): mat = np.array([np.nan]*9).reshape(3, 3) - tgt = np.iinfo(np.intp).min for f in self.nanfuncs: for axis in [None, 0, 1]: - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - res = f(mat, axis=axis) - assert_((res == tgt).all()) - assert_(len(w) == 1) - assert_(issubclass(w[0].category, NanWarning)) + assert_raises(ValueError, f, mat, axis=axis) + assert_raises(ValueError, f, np.nan) def test_empty(self): mat = np.zeros((0, 3)) @@ -139,39 +156,83 @@ class TestNanFunctions_ArgminArgmax(TestCase): res = f(mat, axis=axis) assert_equal(res, np.zeros(0)) + def test_scalar(self): + for f in self.nanfuncs: + assert_(f(0.) == 0.) + + def test_matrices(self): + # Check that it works and that type and + # shape are preserved + mat = np.matrix(np.eye(3)) + for f in self.nanfuncs: + res = f(mat, axis=0) + assert_(isinstance(res, np.matrix)) + assert_(res.shape == (1, 3)) + res = f(mat, axis=1) + assert_(isinstance(res, np.matrix)) + assert_(res.shape == (3, 1)) + res = f(mat) + assert_(np.isscalar(res)) + class TestNanFunctions_IntTypes(TestCase): - int_types = ( - np.int8, np.int16, np.int32, np.int64, np.uint8, - np.uint16, np.uint32, np.uint64) + int_types = (np.int8, np.int16, np.int32, np.int64, np.uint8, + np.uint16, np.uint32, np.uint64) - def setUp(self, *args, **kwargs): - self.mat = np.array([127, 39, 93, 87, 46]) + mat = np.array([127, 39, 93, 87, 46]) def integer_arrays(self): for dtype in self.int_types: yield self.mat.astype(dtype) def test_nanmin(self): - min_value = min(self.mat) + tgt = np.min(self.mat) for mat in self.integer_arrays(): - assert_equal(nanmin(mat), min_value) + assert_equal(np.nanmin(mat), tgt) def test_nanmax(self): - max_value = max(self.mat) + tgt = np.max(self.mat) for mat in self.integer_arrays(): - assert_equal(nanmax(mat), max_value) + assert_equal(np.nanmax(mat), tgt) def test_nanargmin(self): - min_arg = np.argmin(self.mat) + tgt = np.argmin(self.mat) for mat in self.integer_arrays(): - assert_equal(nanargmin(mat), min_arg) + assert_equal(np.nanargmin(mat), tgt) def test_nanargmax(self): - max_arg = np.argmax(self.mat) + tgt = np.argmax(self.mat) for mat in self.integer_arrays(): - assert_equal(nanargmax(mat), max_arg) + assert_equal(np.nanargmax(mat), tgt) + + def test_nansum(self): + tgt = np.sum(self.mat) + for mat in self.integer_arrays(): + assert_equal(np.nansum(mat), tgt) + + def test_nanmean(self): + tgt = np.mean(self.mat) + for mat in self.integer_arrays(): + assert_equal(np.nanmean(mat), tgt) + + def test_nanvar(self): + tgt = np.var(self.mat) + for mat in self.integer_arrays(): + assert_equal(np.nanvar(mat), tgt) + + tgt = np.var(mat, ddof=1) + for mat in self.integer_arrays(): + assert_equal(np.nanvar(mat, ddof=1), tgt) + + def test_nanstd(self): + tgt = np.std(self.mat) + for mat in self.integer_arrays(): + assert_equal(np.nanstd(mat), tgt) + + tgt = np.std(self.mat, ddof=1) + for mat in self.integer_arrays(): + assert_equal(np.nanstd(mat, ddof=1), tgt) class TestNanFunctions_Sum(TestCase): @@ -179,21 +240,21 @@ class TestNanFunctions_Sum(TestCase): def test_mutation(self): # Check that passed array is not modified. ndat = _ndat.copy() - nansum(ndat) + np.nansum(ndat) assert_equal(ndat, _ndat) def test_keepdims(self): mat = np.eye(3) for axis in [None, 0, 1]: tgt = np.sum(mat, axis=axis, keepdims=True) - res = nansum(mat, axis=axis, keepdims=True) + res = np.nansum(mat, axis=axis, keepdims=True) assert_(res.ndim == tgt.ndim) def test_out(self): mat = np.eye(3) resout = np.zeros(3) tgt = np.sum(mat, axis=1) - res = nansum(mat, axis=1, out=resout) + res = np.nansum(mat, axis=1, out=resout) assert_almost_equal(res, resout) assert_almost_equal(res, tgt) @@ -202,11 +263,11 @@ class TestNanFunctions_Sum(TestCase): codes = 'efdgFDG' for c in codes: tgt = np.sum(mat, dtype=np.dtype(c), axis=1).dtype.type - res = nansum(mat, dtype=np.dtype(c), axis=1).dtype.type + res = np.nansum(mat, dtype=np.dtype(c), axis=1).dtype.type assert_(res is tgt) # scalar case tgt = np.sum(mat, dtype=np.dtype(c), axis=None).dtype.type - res = nansum(mat, dtype=np.dtype(c), axis=None).dtype.type + res = np.nansum(mat, dtype=np.dtype(c), axis=None).dtype.type assert_(res is tgt) def test_dtype_from_char(self): @@ -214,11 +275,11 @@ class TestNanFunctions_Sum(TestCase): codes = 'efdgFDG' for c in codes: tgt = np.sum(mat, dtype=c, axis=1).dtype.type - res = nansum(mat, dtype=c, axis=1).dtype.type + res = np.nansum(mat, dtype=c, axis=1).dtype.type assert_(res is tgt) # scalar case tgt = np.sum(mat, dtype=c, axis=None).dtype.type - res = nansum(mat, dtype=c, axis=None).dtype.type + res = np.nansum(mat, dtype=c, axis=None).dtype.type assert_(res is tgt) def test_dtype_from_input(self): @@ -226,43 +287,65 @@ class TestNanFunctions_Sum(TestCase): for c in codes: mat = np.eye(3, dtype=c) tgt = np.sum(mat, axis=1).dtype.type - res = nansum(mat, axis=1).dtype.type + res = np.nansum(mat, axis=1).dtype.type assert_(res is tgt) # scalar case tgt = np.sum(mat, axis=None).dtype.type - res = nansum(mat, axis=None).dtype.type + res = np.nansum(mat, axis=None).dtype.type assert_(res is tgt) def test_result_values(self): tgt = [np.sum(d) for d in _rdat] - res = nansum(_ndat, axis=1) + res = np.nansum(_ndat, axis=1) assert_almost_equal(res, tgt) def test_allnans(self): - # Check for FutureWarning and later change of return from - # NaN to zero. + # Check for FutureWarning with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - res = nansum([np.nan]*3, axis=None) + res = np.nansum([np.nan]*3, axis=None) assert_(res == 0, 'result is not 0') assert_(len(w) == 0, 'warning raised') + # Check scalar + res = np.nansum(np.nan) + assert_(res == 0, 'result is not 0') + assert_(len(w) == 0, 'warning raised') + # Check there is no warning for not all-nan + np.nansum([0]*3, axis=None) + assert_(len(w) == 0, 'unwanted warning raised') def test_empty(self): mat = np.zeros((0, 3)) tgt = [0]*3 - res = nansum(mat, axis=0) + res = np.nansum(mat, axis=0) assert_equal(res, tgt) tgt = [] - res = nansum(mat, axis=1) + res = np.nansum(mat, axis=1) assert_equal(res, tgt) tgt = 0 - res = nansum(mat, axis=None) + res = np.nansum(mat, axis=None) assert_equal(res, tgt) + def test_scalar(self): + assert_(np.nansum(0.) == 0.) + + def test_matrices(self): + # Check that it works and that type and + # shape are preserved + mat = np.matrix(np.eye(3)) + res = np.nansum(mat, axis=0) + assert_(isinstance(res, np.matrix)) + assert_(res.shape == (1, 3)) + res = np.nansum(mat, axis=1) + assert_(isinstance(res, np.matrix)) + assert_(res.shape == (3, 1)) + res = np.nansum(mat) + assert_(np.isscalar(res)) + class TestNanFunctions_MeanVarStd(TestCase): - nanfuncs = [nanmean, nanvar, nanstd] + nanfuncs = [np.nanmean, np.nanvar, np.nanstd] stdfuncs = [np.mean, np.var, np.std] def test_mutation(self): @@ -275,13 +358,13 @@ class TestNanFunctions_MeanVarStd(TestCase): def test_dtype_error(self): for f in self.nanfuncs: for dtype in [np.bool_, np.int_, np.object]: - assert_raises(TypeError, f, _ndat, axis=1, dtype=np.int) + assert_raises( TypeError, f, _ndat, axis=1, dtype=np.int) def test_out_dtype_error(self): for f in self.nanfuncs: for dtype in [np.bool_, np.int_, np.object]: out = np.empty(_ndat.shape[0], dtype=dtype) - assert_raises(TypeError, f, _ndat, axis=1, out=out) + assert_raises( TypeError, f, _ndat, axis=1, out=out) def test_keepdims(self): mat = np.eye(3) @@ -333,14 +416,14 @@ class TestNanFunctions_MeanVarStd(TestCase): mat = np.eye(3, dtype=c) tgt = rf(mat, axis=1).dtype.type res = nf(mat, axis=1).dtype.type - assert_(res is tgt) + assert_(res is tgt, "res %s, tgt %s" % (res, tgt)) # scalar case tgt = rf(mat, axis=None).dtype.type res = nf(mat, axis=None).dtype.type assert_(res is tgt) def test_ddof(self): - nanfuncs = [nanvar, nanstd] + nanfuncs = [np.nanvar, np.nanstd] stdfuncs = [np.var, np.std] for nf, rf in zip(nanfuncs, stdfuncs): for ddof in [0, 1]: @@ -349,7 +432,7 @@ class TestNanFunctions_MeanVarStd(TestCase): assert_almost_equal(res, tgt) def test_ddof_too_big(self): - nanfuncs = [nanvar, nanstd] + nanfuncs = [np.nanvar, np.nanstd] stdfuncs = [np.var, np.std] dsize = [len(d) for d in _rdat] for nf, rf in zip(nanfuncs, stdfuncs): @@ -361,7 +444,7 @@ class TestNanFunctions_MeanVarStd(TestCase): assert_equal(np.isnan(res), tgt) if any(tgt): assert_(len(w) == 1) - assert_(issubclass(w[0].category, NanWarning)) + assert_(issubclass(w[0].category, RuntimeWarning)) else: assert_(len(w) == 0) @@ -379,7 +462,11 @@ class TestNanFunctions_MeanVarStd(TestCase): warnings.simplefilter('always') assert_(np.isnan(f(mat, axis=axis)).all()) assert_(len(w) == 1) - assert_(issubclass(w[0].category, NanWarning)) + assert_(issubclass(w[0].category, RuntimeWarning)) + # Check scalar + assert_(np.isnan(f(np.nan))) + assert_(len(w) == 2) + assert_(issubclass(w[0].category, RuntimeWarning)) def test_empty(self): mat = np.zeros((0, 3)) @@ -389,13 +476,31 @@ class TestNanFunctions_MeanVarStd(TestCase): warnings.simplefilter('always') assert_(np.isnan(f(mat, axis=axis)).all()) assert_(len(w) == 1) - assert_(issubclass(w[0].category, NanWarning)) + assert_(issubclass(w[0].category, RuntimeWarning)) for axis in [1]: with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') assert_equal(f(mat, axis=axis), np.zeros([])) assert_(len(w) == 0) + def test_scalar(self): + for f in self.nanfuncs: + assert_(f(0.) == 0.) + + def test_matrices(self): + # Check that it works and that type and + # shape are preserved + mat = np.matrix(np.eye(3)) + for f in self.nanfuncs: + res = f(mat, axis=0) + assert_(isinstance(res, np.matrix)) + assert_(res.shape == (1, 3)) + res = f(mat, axis=1) + assert_(isinstance(res, np.matrix)) + assert_(res.shape == (3, 1)) + res = f(mat) + assert_(np.isscalar(res)) + if __name__ == "__main__": run_module_suite() -- cgit v1.2.1