summaryrefslogtreecommitdiff
path: root/numpy/lib
diff options
context:
space:
mode:
Diffstat (limited to 'numpy/lib')
-rw-r--r--numpy/lib/function_base.py44
-rw-r--r--numpy/lib/npyio.py43
-rw-r--r--numpy/lib/tests/test__datasource.py8
-rw-r--r--numpy/lib/tests/test_arraypad.py58
-rw-r--r--numpy/lib/tests/test_format.py8
-rw-r--r--numpy/lib/tests/test_function_base.py52
-rw-r--r--numpy/lib/tests/test_io.py12
7 files changed, 182 insertions, 43 deletions
diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py
index 2992e92bb..e2a8f4bc2 100644
--- a/numpy/lib/function_base.py
+++ b/numpy/lib/function_base.py
@@ -9,24 +9,23 @@ except ImportError:
import re
import sys
import warnings
-import operator
import numpy as np
import numpy.core.numeric as _nx
-from numpy.core import linspace, atleast_1d, atleast_2d, transpose
+from numpy.core import atleast_1d, transpose
from numpy.core.numeric import (
ones, zeros, arange, concatenate, array, asarray, asanyarray, empty,
empty_like, ndarray, around, floor, ceil, take, dot, where, intp,
- integer, isscalar, absolute, AxisError
+ integer, isscalar, absolute
)
from numpy.core.umath import (
- pi, multiply, add, arctan2, frompyfunc, cos, less_equal, sqrt, sin,
- mod, exp, log10, not_equal, subtract
+ pi, add, arctan2, frompyfunc, cos, less_equal, sqrt, sin,
+ mod, exp, not_equal, subtract
)
from numpy.core.fromnumeric import (
- ravel, nonzero, sort, partition, mean, any, sum
+ ravel, nonzero, partition, mean, any, sum
)
-from numpy.core.numerictypes import typecodes, number
+from numpy.core.numerictypes import typecodes
from numpy.core.function_base import add_newdoc
from numpy.lib.twodim_base import diag
from .utils import deprecate
@@ -36,7 +35,6 @@ from numpy.core.multiarray import (
)
from numpy.core.umath import _add_newdoc_ufunc as add_newdoc_ufunc
from numpy.compat import long
-from numpy.compat.py3k import basestring
if sys.version_info[0] < 3:
# Force range to be a generator, for np.delete's usage.
@@ -1090,7 +1088,7 @@ def gradient(f, *varargs, **kwargs):
return outvals
-def diff(a, n=1, axis=-1):
+def diff(a, n=1, axis=-1, prepend=np._NoValue, append=np._NoValue):
"""
Calculate the n-th discrete difference along the given axis.
@@ -1108,6 +1106,12 @@ def diff(a, n=1, axis=-1):
axis : int, optional
The axis along which the difference is taken, default is the
last axis.
+ prepend, append : array_like, optional
+ Values to prepend or append to "a" along axis prior to
+ performing the difference. Scalar values are expanded to
+ arrays with length 1 in the direction of axis and the shape
+ of the input array in along all other axes. Otherwise the
+ dimension and shape must match "a" except along axis.
Returns
-------
@@ -1176,6 +1180,28 @@ def diff(a, n=1, axis=-1):
nd = a.ndim
axis = normalize_axis_index(axis, nd)
+ combined = []
+ if prepend is not np._NoValue:
+ prepend = np.asanyarray(prepend)
+ if prepend.ndim == 0:
+ shape = list(a.shape)
+ shape[axis] = 1
+ prepend = np.broadcast_to(prepend, tuple(shape))
+ combined.append(prepend)
+
+ combined.append(a)
+
+ if append is not np._NoValue:
+ append = np.asanyarray(append)
+ if append.ndim == 0:
+ shape = list(a.shape)
+ shape[axis] = 1
+ append = np.broadcast_to(append, tuple(shape))
+ combined.append(append)
+
+ if len(combined) > 1:
+ a = np.concatenate(combined, axis)
+
slice1 = [slice(None)] * nd
slice2 = [slice(None)] * nd
slice1[axis] = slice(1, None)
diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py
index c4fa6b10d..77e007d23 100644
--- a/numpy/lib/npyio.py
+++ b/numpy/lib/npyio.py
@@ -13,6 +13,7 @@ import numpy as np
from . import format
from ._datasource import DataSource
from numpy.core.multiarray import packbits, unpackbits
+from numpy.core._internal import recursive
from ._iotools import (
LineSplitter, NameValidator, StringConverter, ConverterError,
ConverterLockError, ConversionWarning, _is_string_like,
@@ -379,16 +380,6 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True,
memmap([4, 5, 6])
"""
- own_fid = False
- if isinstance(file, basestring):
- fid = open(file, "rb")
- own_fid = True
- elif is_pathlib_path(file):
- fid = file.open("rb")
- own_fid = True
- else:
- fid = file
-
if encoding not in ('ASCII', 'latin1', 'bytes'):
# The 'encoding' value for pickle also affects what encoding
# the serialized binary data of NumPy arrays is loaded
@@ -409,6 +400,17 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True,
# Nothing to do on Python 2
pickle_kwargs = {}
+ # TODO: Use contextlib.ExitStack once we drop Python 2
+ if isinstance(file, basestring):
+ fid = open(file, "rb")
+ own_fid = True
+ elif is_pathlib_path(file):
+ fid = file.open("rb")
+ own_fid = True
+ else:
+ fid = file
+ own_fid = False
+
try:
# Code to distinguish from NumPy binary files and pickles.
_ZIP_PREFIX = b'PK\x03\x04'
@@ -421,10 +423,10 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True,
if magic.startswith(_ZIP_PREFIX) or magic.startswith(_ZIP_SUFFIX):
# zip-file (assume .npz)
# Transfer file ownership to NpzFile
- tmp = own_fid
+ ret = NpzFile(fid, own_fid=own_fid, allow_pickle=allow_pickle,
+ pickle_kwargs=pickle_kwargs)
own_fid = False
- return NpzFile(fid, own_fid=tmp, allow_pickle=allow_pickle,
- pickle_kwargs=pickle_kwargs)
+ return ret
elif magic == format.MAGIC_PREFIX:
# .npy file
if mmap_mode:
@@ -949,7 +951,8 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None,
fencoding = locale.getpreferredencoding()
# not to be confused with the flatten_dtype we import...
- def flatten_dtype_internal(dt):
+ @recursive
+ def flatten_dtype_internal(self, dt):
"""Unpack a structured data-type, and produce re-packing info."""
if dt.names is None:
# If the dtype is flattened, return.
@@ -969,7 +972,7 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None,
packing = []
for field in dt.names:
tp, bytes = dt.fields[field]
- flat_dt, flat_packing = flatten_dtype_internal(tp)
+ flat_dt, flat_packing = self(tp)
types.extend(flat_dt)
# Avoid extra nesting for subarrays
if tp.ndim > 0:
@@ -978,7 +981,8 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None,
packing.append((len(flat_dt), flat_packing))
return (types, packing)
- def pack_items(items, packing):
+ @recursive
+ def pack_items(self, items, packing):
"""Pack items into nested lists based on re-packing info."""
if packing is None:
return items[0]
@@ -990,7 +994,7 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None,
start = 0
ret = []
for length, subpacking in packing:
- ret.append(pack_items(items[start:start+length], subpacking))
+ ret.append(self(items[start:start+length], subpacking))
start += length
return tuple(ret)
@@ -1118,11 +1122,6 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None,
finally:
if fown:
fh.close()
- # recursive closures have a cyclic reference to themselves, which
- # requires gc to collect (gh-10620). To avoid this problem, for
- # performance and PyPy friendliness, we break the cycle:
- flatten_dtype_internal = None
- pack_items = None
if X is None:
X = np.array([], dtype)
diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py
index 85788941c..1df8bebf6 100644
--- a/numpy/lib/tests/test__datasource.py
+++ b/numpy/lib/tests/test__datasource.py
@@ -8,7 +8,7 @@ from shutil import rmtree
import numpy.lib._datasource as datasource
from numpy.testing import (
- assert_, assert_equal, assert_raises, assert_warns, SkipTest
+ assert_, assert_equal, assert_raises, assert_warns
)
if sys.version_info[0] >= 3:
@@ -137,7 +137,7 @@ class TestDataSourceOpen(object):
import gzip
except ImportError:
# We don't have the gzip capabilities to test.
- raise SkipTest
+ pytest.skip()
# Test datasource's internal file_opener for Gzip files.
filepath = os.path.join(self.tmpdir, 'foobar.txt.gz')
fp = gzip.open(filepath, 'w')
@@ -153,7 +153,7 @@ class TestDataSourceOpen(object):
import bz2
except ImportError:
# We don't have the bz2 capabilities to test.
- raise SkipTest
+ pytest.skip()
# Test datasource's internal file_opener for BZip2 files.
filepath = os.path.join(self.tmpdir, 'foobar.txt.bz2')
fp = bz2.BZ2File(filepath, 'w')
@@ -170,7 +170,7 @@ class TestDataSourceOpen(object):
import bz2
except ImportError:
# We don't have the bz2 capabilities to test.
- raise SkipTest
+ pytest.skip()
# Test datasource's internal file_opener for BZip2 files.
filepath = os.path.join(self.tmpdir, 'foobar.txt.bz2')
fp = bz2.BZ2File(filepath, 'w')
diff --git a/numpy/lib/tests/test_arraypad.py b/numpy/lib/tests/test_arraypad.py
index 45d624781..e62fccaa0 100644
--- a/numpy/lib/tests/test_arraypad.py
+++ b/numpy/lib/tests/test_arraypad.py
@@ -3,8 +3,11 @@
"""
from __future__ import division, absolute_import, print_function
+import pytest
+
import numpy as np
-from numpy.testing import (assert_array_equal, assert_raises, assert_allclose,)
+from numpy.testing import (assert_array_equal, assert_raises, assert_allclose,
+ assert_equal)
from numpy.lib import pad
@@ -344,6 +347,20 @@ class TestStatistic(object):
)
assert_array_equal(a, b)
+ @pytest.mark.parametrize("mode", [
+ pytest.param("mean", marks=pytest.mark.xfail(reason="gh-11216")),
+ "median",
+ "minimum",
+ "maximum"
+ ])
+ def test_same_prepend_append(self, mode):
+ """ Test that appended and prepended values are equal """
+ # This test is constructed to trigger floating point rounding errors in
+ # a way that caused gh-11216 for mode=='mean'
+ a = np.array([-1, 2, -1]) + np.array([0, 1e-12, 0], dtype=np.float64)
+ a = np.pad(a, (1, 1), mode)
+ assert_equal(a[0], a[-1])
+
class TestConstant(object):
def test_check_constant(self):
@@ -502,6 +519,21 @@ class TestConstant(object):
expected = np.full(7, int64_max, dtype=np.int64)
assert_array_equal(test, expected)
+ def test_check_object_array(self):
+ arr = np.empty(1, dtype=object)
+ obj_a = object()
+ arr[0] = obj_a
+ obj_b = object()
+ obj_c = object()
+ arr = np.pad(arr, pad_width=1, mode='constant',
+ constant_values=(obj_b, obj_c))
+
+ expected = np.empty((3,), dtype=object)
+ expected[0] = obj_b
+ expected[1] = obj_a
+ expected[2] = obj_c
+
+ assert_array_equal(arr, expected)
class TestLinearRamp(object):
def test_check_simple(self):
@@ -542,6 +574,25 @@ class TestLinearRamp(object):
[0., 0., 0., 0., 0., 0., 0., 0., 0.]])
assert_allclose(test, expected)
+ @pytest.mark.xfail(exceptions=(AssertionError,))
+ def test_object_array(self):
+ from fractions import Fraction
+ arr = np.array([Fraction(1, 2), Fraction(-1, 2)])
+ actual = np.pad(arr, (2, 3), mode='linear_ramp', end_values=0)
+
+ # deliberately chosen to have a non-power-of-2 denominator such that
+ # rounding to floats causes a failure.
+ expected = np.array([
+ Fraction( 0, 12),
+ Fraction( 3, 12),
+ Fraction( 6, 12),
+ Fraction(-6, 12),
+ Fraction(-4, 12),
+ Fraction(-2, 12),
+ Fraction(-0, 12),
+ ])
+ assert_equal(actual, expected)
+
class TestReflect(object):
def test_check_simple(self):
@@ -887,6 +938,11 @@ class TestWrap(object):
b = np.array([3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1])
assert_array_equal(a, b)
+ def test_pad_with_zero(self):
+ a = np.ones((3, 5))
+ b = np.pad(a, (0, 5), mode="wrap")
+ assert_array_equal(a, b[:-5, :-5])
+
class TestStatLen(object):
def test_check_simple(self):
diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py
index af64dd043..3185e32ac 100644
--- a/numpy/lib/tests/test_format.py
+++ b/numpy/lib/tests/test_format.py
@@ -287,7 +287,7 @@ from io import BytesIO
import numpy as np
from numpy.testing import (
assert_, assert_array_equal, assert_raises, assert_raises_regex,
- raises, SkipTest
+ raises
)
from numpy.lib import format
@@ -807,7 +807,7 @@ def test_bad_header():
def test_large_file_support():
if (sys.platform == 'win32' or sys.platform == 'cygwin'):
- raise SkipTest("Unknown if Windows has sparse filesystems")
+ pytest.skip("Unknown if Windows has sparse filesystems")
# try creating a large sparse file
tf_name = os.path.join(tempdir, 'sparse_file')
try:
@@ -817,7 +817,7 @@ def test_large_file_support():
import subprocess as sp
sp.check_call(["truncate", "-s", "5368709120", tf_name])
except Exception:
- raise SkipTest("Could not create 5GB large file")
+ pytest.skip("Could not create 5GB large file")
# write a small array to the end
with open(tf_name, "wb") as f:
f.seek(5368709120)
@@ -839,7 +839,7 @@ def test_large_archive():
try:
a = np.empty((2**30, 2), dtype=np.uint8)
except MemoryError:
- raise SkipTest("Could not create large file")
+ pytest.skip("Could not create large file")
fname = os.path.join(tempdir, "large_archive")
diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py
index d5faed6ae..40cca1dbb 100644
--- a/numpy/lib/tests/test_function_base.py
+++ b/numpy/lib/tests/test_function_base.py
@@ -734,6 +734,58 @@ class TestDiff(object):
assert_array_equal(out3.mask, [[], [], [], [], []])
assert_(type(out3) is type(x))
+ def test_prepend(self):
+ x = np.arange(5) + 1
+ assert_array_equal(diff(x, prepend=0), np.ones(5))
+ assert_array_equal(diff(x, prepend=[0]), np.ones(5))
+ assert_array_equal(np.cumsum(np.diff(x, prepend=0)), x)
+ assert_array_equal(diff(x, prepend=[-1, 0]), np.ones(6))
+
+ x = np.arange(4).reshape(2, 2)
+ result = np.diff(x, axis=1, prepend=0)
+ expected = [[0, 1], [2, 1]]
+ assert_array_equal(result, expected)
+ result = np.diff(x, axis=1, prepend=[[0], [0]])
+ assert_array_equal(result, expected)
+
+ result = np.diff(x, axis=0, prepend=0)
+ expected = [[0, 1], [2, 2]]
+ assert_array_equal(result, expected)
+ result = np.diff(x, axis=0, prepend=[[0, 0]])
+ assert_array_equal(result, expected)
+
+ assert_raises(ValueError, np.diff, x, prepend=np.zeros((3,3)))
+
+ assert_raises(np.AxisError, diff, x, prepend=0, axis=3)
+
+ def test_append(self):
+ x = np.arange(5)
+ result = diff(x, append=0)
+ expected = [1, 1, 1, 1, -4]
+ assert_array_equal(result, expected)
+ result = diff(x, append=[0])
+ assert_array_equal(result, expected)
+ result = diff(x, append=[0, 2])
+ expected = expected + [2]
+ assert_array_equal(result, expected)
+
+ x = np.arange(4).reshape(2, 2)
+ result = np.diff(x, axis=1, append=0)
+ expected = [[1, -1], [1, -3]]
+ assert_array_equal(result, expected)
+ result = np.diff(x, axis=1, append=[[0], [0]])
+ assert_array_equal(result, expected)
+
+ result = np.diff(x, axis=0, append=0)
+ expected = [[2, 2], [-2, -3]]
+ assert_array_equal(result, expected)
+ result = np.diff(x, axis=0, append=[[0, 0]])
+ assert_array_equal(result, expected)
+
+ assert_raises(ValueError, np.diff, x, append=np.zeros((3,3)))
+
+ assert_raises(np.AxisError, diff, x, append=0, axis=3)
+
class TestDelete(object):
diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py
index e31115978..08800ff97 100644
--- a/numpy/lib/tests/test_io.py
+++ b/numpy/lib/tests/test_io.py
@@ -21,7 +21,7 @@ from numpy.lib._iotools import ConverterError, ConversionWarning
from numpy.compat import asbytes, bytes, unicode, Path
from numpy.ma.testutils import assert_equal
from numpy.testing import (
- assert_warns, assert_, SkipTest, assert_raises_regex, assert_raises,
+ assert_warns, assert_, assert_raises_regex, assert_raises,
assert_allclose, assert_array_equal, temppath, tempdir, IS_PYPY,
HAS_REFCOUNT, suppress_warnings, assert_no_gc_cycles,
)
@@ -2082,8 +2082,8 @@ M 33 21.99
encoding = locale.getpreferredencoding()
utf8.encode(encoding)
except (UnicodeError, ImportError):
- raise SkipTest('Skipping test_utf8_file_nodtype_unicode, '
- 'unable to encode utf8 in preferred encoding')
+ pytest.skip('Skipping test_utf8_file_nodtype_unicode, '
+ 'unable to encode utf8 in preferred encoding')
with temppath() as path:
with io.open(path, "wt") as f:
@@ -2461,3 +2461,9 @@ def test_load_refcount():
with assert_no_gc_cycles():
np.load(f)
+
+ f.seek(0)
+ dt = [("a", 'u1', 2), ("b", 'u1', 2)]
+ with assert_no_gc_cycles():
+ x = np.loadtxt(TextIO("0 1 2 3"), dtype=dt)
+ assert_equal(x, np.array([((0, 1), (2, 3))], dtype=dt))
View Lorry Controller Status