summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--benchmarks/asv.conf.json5
-rw-r--r--doc/example.py4
-rw-r--r--doc/neps/nep-0034.rst4
-rw-r--r--doc/release/upcoming_changes/13698.deprecation.rst5
-rw-r--r--doc/release/upcoming_changes/14718.compatibility.rst8
-rw-r--r--doc/source/dev/development_environment.rst2
-rw-r--r--doc/source/reference/arrays.datetime.rst6
-rw-r--r--doc/source/reference/ufuncs.rst6
-rw-r--r--doc/source/release/1.11.0-notes.rst4
-rw-r--r--doc/source/user/basics.io.genfromtxt.rst20
-rw-r--r--numpy/core/src/multiarray/convert_datatype.c8
-rw-r--r--numpy/core/src/multiarray/scalartypes.c.src7
-rw-r--r--numpy/core/tests/test_datetime.py9
-rw-r--r--numpy/core/tests/test_scalarmath.py12
-rw-r--r--numpy/distutils/misc_util.py7
-rw-r--r--numpy/lib/type_check.py10
-rw-r--r--numpy/ma/core.py41
-rw-r--r--numpy/random/tests/test_extending.py6
-rw-r--r--test_requirements.txt2
-rw-r--r--tools/refguide_check.py249
20 files changed, 293 insertions, 122 deletions
diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json
index 45da9533f..b3c7f9f20 100644
--- a/benchmarks/asv.conf.json
+++ b/benchmarks/asv.conf.json
@@ -35,7 +35,7 @@
// The Pythons you'd like to test against. If not provided, defaults
// to the current version of Python used to run `asv`.
- "pythons": ["3.6"],
+ "pythons": ["3.7"],
// The matrix of dependencies to test. Each key is the name of a
// package (in PyPI) and the values are version numbers. An empty
@@ -43,6 +43,7 @@
// version.
"matrix": {
"six": [],
+ "Cython": [],
},
// The directory (relative to the current directory) that benchmarks are
@@ -68,7 +69,7 @@
// `asv` will cache wheels of the recent builds in each
// environment, making them faster to install next time. This is
// number of builds to keep, per environment.
- "wheel_cache_size": 2,
+ "build_cache_size": 8,
// The commits after which the regression search in `asv publish`
// should start looking for regressions. Dictionary whose keys are
diff --git a/doc/example.py b/doc/example.py
index 560775038..8a5f9948f 100644
--- a/doc/example.py
+++ b/doc/example.py
@@ -112,9 +112,9 @@ def foo(var1, var2, long_var_name='hi'):
use the function.
>>> a = [1, 2, 3]
- >>> print [x + 3 for x in a]
+ >>> print([x + 3 for x in a])
[4, 5, 6]
- >>> print "a\n\nb"
+ >>> print("a\n\nb")
a
b
diff --git a/doc/neps/nep-0034.rst b/doc/neps/nep-0034.rst
index d9a9c62f2..4863bad86 100644
--- a/doc/neps/nep-0034.rst
+++ b/doc/neps/nep-0034.rst
@@ -3,10 +3,10 @@ NEP 34 — Disallow inferring ``dtype=object`` from sequences
===========================================================
:Author: Matti Picus
-:Status: Draft
+:Status: Accepted
:Type: Standards Track
:Created: 2019-10-10
-
+:Resolution: https://mail.python.org/pipermail/numpy-discussion/2019-October/080200.html
Abstract
--------
diff --git a/doc/release/upcoming_changes/13698.deprecation.rst b/doc/release/upcoming_changes/13698.deprecation.rst
new file mode 100644
index 000000000..923957058
--- /dev/null
+++ b/doc/release/upcoming_changes/13698.deprecation.rst
@@ -0,0 +1,5 @@
+Deprecate non-scalar arrays as fill values in ``ma.fill_value``
+---------------------------------------------------------------
+Setting a ``MaskedArray.fill_value`` to a non-scalar array is deprecated
+since the logic to broadcast the fill value to the array is fragile,
+especially when slicing.
diff --git a/doc/release/upcoming_changes/14718.compatibility.rst b/doc/release/upcoming_changes/14718.compatibility.rst
new file mode 100644
index 000000000..3d6e71ead
--- /dev/null
+++ b/doc/release/upcoming_changes/14718.compatibility.rst
@@ -0,0 +1,8 @@
+``np.can_cast(np.uint64, np.timedelta64, casting='safe')`` is now ``False``
+---------------------------------------------------------------------------
+
+Previously this was ``True`` - however, this was inconsistent with ``uint64``
+not being safely castable to ``int64``, and resulting in strange type
+resolution.
+
+If this impacts your code, cast ``uint64`` to ``int64`` first.
diff --git a/doc/source/dev/development_environment.rst b/doc/source/dev/development_environment.rst
index 297502b31..c73fb3858 100644
--- a/doc/source/dev/development_environment.rst
+++ b/doc/source/dev/development_environment.rst
@@ -145,7 +145,7 @@ Running tests
Besides using ``runtests.py``, there are various ways to run the tests. Inside
the interpreter, tests can be run like this::
- >>> np.test()
+ >>> np.test() # doctest: +SKIPBLOCK
>>> np.test('full') # Also run tests marked as slow
>>> np.test('full', verbose=2) # Additionally print test name/file
diff --git a/doc/source/reference/arrays.datetime.rst b/doc/source/reference/arrays.datetime.rst
index 2225eedb3..9c45e04c7 100644
--- a/doc/source/reference/arrays.datetime.rst
+++ b/doc/source/reference/arrays.datetime.rst
@@ -368,7 +368,7 @@ times in UTC. By default, creating a datetime64 object from a string or
printing it would convert from or to local time::
# old behavior
- >>>> np.datetime64('2000-01-01T00:00:00')
+ >>> np.datetime64('2000-01-01T00:00:00')
numpy.datetime64('2000-01-01T00:00:00-0800') # note the timezone offset -08:00
A consensus of datetime64 users agreed that this behavior is undesirable
@@ -378,7 +378,7 @@ most use cases, a timezone naive datetime type is preferred, similar to the
datetime64 no longer assumes that input is in local time, nor does it print
local times::
- >>>> np.datetime64('2000-01-01T00:00:00')
+ >>> np.datetime64('2000-01-01T00:00:00')
numpy.datetime64('2000-01-01T00:00:00')
For backwards compatibility, datetime64 still parses timezone offsets, which
@@ -393,4 +393,4 @@ As a corollary to this change, we no longer prohibit casting between datetimes
with date units and datetimes with timeunits. With timezone naive datetimes,
the rule for casting from dates to times is no longer ambiguous.
-.. _pandas: http://pandas.pydata.org \ No newline at end of file
+.. _pandas: http://pandas.pydata.org
diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst
index 0416d6efc..361cf11b9 100644
--- a/doc/source/reference/ufuncs.rst
+++ b/doc/source/reference/ufuncs.rst
@@ -253,9 +253,9 @@ can generate this table for your system with the code given in the Figure.
B - - Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y
H - - - Y Y Y Y - Y Y Y Y Y - Y Y Y Y Y Y Y Y Y Y - Y
I - - - - Y Y Y - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - Y
- L - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - Y
- Q - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - Y
- P - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - Y
+ L - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - -
+ Q - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - -
+ P - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - -
e - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y Y - -
f - - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y - -
d - - - - - - - - - - - - - - - Y Y - Y Y Y Y Y Y - -
diff --git a/doc/source/release/1.11.0-notes.rst b/doc/source/release/1.11.0-notes.rst
index 1a179657b..36cd1d65a 100644
--- a/doc/source/release/1.11.0-notes.rst
+++ b/doc/source/release/1.11.0-notes.rst
@@ -85,7 +85,7 @@ times in UTC. By default, creating a datetime64 object from a string or
printing it would convert from or to local time::
# old behavior
- >>>> np.datetime64('2000-01-01T00:00:00')
+ >>> np.datetime64('2000-01-01T00:00:00')
numpy.datetime64('2000-01-01T00:00:00-0800') # note the timezone offset -08:00
@@ -96,7 +96,7 @@ type is preferred, similar to the ``datetime.datetime`` type in the Python
standard library. Accordingly, datetime64 no longer assumes that input is in
local time, nor does it print local times::
- >>>> np.datetime64('2000-01-01T00:00:00')
+ >>> np.datetime64('2000-01-01T00:00:00')
numpy.datetime64('2000-01-01T00:00:00')
For backwards compatibility, datetime64 still parses timezone offsets, which
diff --git a/doc/source/user/basics.io.genfromtxt.rst b/doc/source/user/basics.io.genfromtxt.rst
index 19e37eabc..3fce6a8aa 100644
--- a/doc/source/user/basics.io.genfromtxt.rst
+++ b/doc/source/user/basics.io.genfromtxt.rst
@@ -98,13 +98,11 @@ This behavior can be overwritten by setting the optional argument
>>> # Without autostrip
>>> np.genfromtxt(StringIO(data), delimiter=",", dtype="|U5")
array([['1', ' abc ', ' 2'],
- ['3', ' xxx', ' 4']],
- dtype='|U5')
+ ['3', ' xxx', ' 4']], dtype='<U5')
>>> # With autostrip
>>> np.genfromtxt(StringIO(data), delimiter=",", dtype="|U5", autostrip=True)
array([['1', 'abc', '2'],
- ['3', 'xxx', '4']],
- dtype='|U5')
+ ['3', 'xxx', '4']], dtype='<U5')
The ``comments`` argument
@@ -127,11 +125,11 @@ marker(s) is simply ignored::
... 9, 0
... """
>>> np.genfromtxt(StringIO(data), comments="#", delimiter=",")
- [[ 1. 2.]
- [ 3. 4.]
- [ 5. 6.]
- [ 7. 8.]
- [ 9. 0.]]
+ array([[1., 2.],
+ [3., 4.],
+ [5., 6.],
+ [7., 8.],
+ [9., 0.]])
.. versionadded:: 1.7.0
@@ -376,12 +374,12 @@ single element of the wanted type.
In the following example, the second column is converted from as string
representing a percentage to a float between 0 and 1::
- >>> convertfunc = lambda x: float(x.strip("%"))/100.
+ >>> convertfunc = lambda x: float(x.strip(b"%"))/100.
>>> data = u"1, 2.3%, 45.\n6, 78.9%, 0"
>>> names = ("i", "p", "n")
>>> # General case .....
>>> np.genfromtxt(StringIO(data), delimiter=",", names=names)
- array([(1.0, nan, 45.0), (6.0, nan, 0.0)],
+ array([(1., nan, 45.), (6., nan, 0.)],
dtype=[('i', '<f8'), ('p', '<f8'), ('n', '<f8')])
We need to keep in mind that by default, ``dtype=float``. A float is
diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c
index 025c66013..4326448dc 100644
--- a/numpy/core/src/multiarray/convert_datatype.c
+++ b/numpy/core/src/multiarray/convert_datatype.c
@@ -877,7 +877,13 @@ PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to,
from_order = dtype_kind_to_ordering(from->kind);
to_order = dtype_kind_to_ordering(to->kind);
- return from_order != -1 && from_order <= to_order;
+ if (to->kind == 'm') {
+ /* both types being timedelta is already handled before. */
+ int integer_order = dtype_kind_to_ordering('i');
+ return (from_order != -1) && (from_order <= integer_order);
+ }
+
+ return (from_order != -1) && (from_order <= to_order);
}
else {
return 0;
diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src
index 32d712e0c..5da7f7738 100644
--- a/numpy/core/src/multiarray/scalartypes.c.src
+++ b/numpy/core/src/multiarray/scalartypes.c.src
@@ -4060,8 +4060,11 @@ initialize_casting_tables(void)
_npy_can_cast_safely_table[_FROM_NUM][NPY_STRING] = 1;
_npy_can_cast_safely_table[_FROM_NUM][NPY_UNICODE] = 1;
- /* Allow casts from any integer to the TIMEDELTA type */
-#if @from_isint@ || @from_isuint@
+#if @from_isint@ && NPY_SIZEOF_TIMEDELTA >= _FROM_BSIZE
+ /* Allow casts from smaller or equal signed integers to the TIMEDELTA type */
+ _npy_can_cast_safely_table[_FROM_NUM][NPY_TIMEDELTA] = 1;
+#elif @from_isuint@ && NPY_SIZEOF_TIMEDELTA > _FROM_BSIZE
+ /* Allow casts from smaller unsigned integers to the TIMEDELTA type */
_npy_can_cast_safely_table[_FROM_NUM][NPY_TIMEDELTA] = 1;
#endif
diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py
index d1208e8ec..41b84a69f 100644
--- a/numpy/core/tests/test_datetime.py
+++ b/numpy/core/tests/test_datetime.py
@@ -75,6 +75,15 @@ class TestDateTime(object):
# Can cast safely/same_kind from integer to timedelta
assert_(np.can_cast('i8', 'm8', casting='same_kind'))
assert_(np.can_cast('i8', 'm8', casting='safe'))
+ assert_(np.can_cast('i4', 'm8', casting='same_kind'))
+ assert_(np.can_cast('i4', 'm8', casting='safe'))
+ assert_(np.can_cast('u4', 'm8', casting='same_kind'))
+ assert_(np.can_cast('u4', 'm8', casting='safe'))
+
+ # Cannot cast safely from unsigned integer of the same size, which
+ # could overflow
+ assert_(np.can_cast('u8', 'm8', casting='same_kind'))
+ assert_(not np.can_cast('u8', 'm8', casting='safe'))
# Cannot cast safely/same_kind from float to timedelta
assert_(not np.can_cast('f4', 'm8', casting='same_kind'))
diff --git a/numpy/core/tests/test_scalarmath.py b/numpy/core/tests/test_scalarmath.py
index 854df5590..c84380cd9 100644
--- a/numpy/core/tests/test_scalarmath.py
+++ b/numpy/core/tests/test_scalarmath.py
@@ -11,7 +11,7 @@ import numpy as np
from numpy.testing import (
assert_, assert_equal, assert_raises, assert_almost_equal,
assert_array_equal, IS_PYPY, suppress_warnings, _gen_alignment_data,
- assert_warns
+ assert_warns, assert_raises_regex,
)
types = [np.bool_, np.byte, np.ubyte, np.short, np.ushort, np.intc, np.uintc,
@@ -293,6 +293,16 @@ class TestModulus(object):
rem = operator.mod(finf, fone)
assert_(np.isnan(rem), 'dt: %s' % dt)
+ def test_inplace_floordiv_handling(self):
+ # issue gh-12927
+ # this only applies to in-place floordiv //=, because the output type
+ # promotes to float which does not fit
+ a = np.array([1, 2], np.int64)
+ b = np.array([1, 2], np.uint64)
+ pattern = 'could not be coerced to provided output parameter'
+ with assert_raises_regex(TypeError, pattern):
+ a //= b
+
class TestComplexDivision(object):
def test_zero_division(self):
diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py
index 7ba8ad862..d46ff8981 100644
--- a/numpy/distutils/misc_util.py
+++ b/numpy/distutils/misc_util.py
@@ -2329,8 +2329,11 @@ def generate_config_py(target):
extra_dll_dir = os.path.join(os.path.dirname(__file__), '.libs')
if sys.platform == 'win32' and os.path.isdir(extra_dll_dir):
- os.environ.setdefault('PATH', '')
- os.environ['PATH'] += os.pathsep + extra_dll_dir
+ if sys.version_info >= (3, 8):
+ os.add_dll_directory(extra_dll_dir)
+ else:
+ os.environ.setdefault('PATH', '')
+ os.environ['PATH'] += os.pathsep + extra_dll_dir
"""))
diff --git a/numpy/lib/type_check.py b/numpy/lib/type_check.py
index 586824743..8b9e4b8ab 100644
--- a/numpy/lib/type_check.py
+++ b/numpy/lib/type_check.py
@@ -68,16 +68,14 @@ def mintypecode(typechars, typeset='GDFgdf', default='d'):
'G'
"""
- typecodes = [(isinstance(t, str) and t) or asarray(t).dtype.char
- for t in typechars]
- intersection = [t for t in typecodes if t in typeset]
+ typecodes = ((isinstance(t, str) and t) or asarray(t).dtype.char
+ for t in typechars)
+ intersection = set(t for t in typecodes if t in typeset)
if not intersection:
return default
if 'F' in intersection and 'd' in intersection:
return 'D'
- l = [(_typecodes_by_elsize.index(t), t) for t in intersection]
- l.sort()
- return l[0][1]
+ return min((_typecodes_by_elsize.index(t), t) for t in intersection)[1]
def _asfarray_dispatcher(a, dtype=None):
diff --git a/numpy/ma/core.py b/numpy/ma/core.py
index bb0d8d412..2baf547a4 100644
--- a/numpy/ma/core.py
+++ b/numpy/ma/core.py
@@ -602,8 +602,10 @@ def filled(a, fill_value=None):
----------
a : MaskedArray or array_like
An input object.
- fill_value : scalar, optional
- Filling value. Default is None.
+ fill_value : array_like, optional.
+ Can be scalar or non-scalar. If non-scalar, the
+ resulting filled array should be broadcastable
+ over input array. Default is None.
Returns
-------
@@ -623,10 +625,19 @@ def filled(a, fill_value=None):
array([[999999, 1, 2],
[999999, 4, 5],
[ 6, 7, 8]])
+ >>> x.filled(fill_value=333)
+ array([[333, 1, 2],
+ [333, 4, 5],
+ [ 6, 7, 8]])
+ >>> x.filled(fill_value=np.arange(3))
+ array([[0, 1, 2],
+ [0, 4, 5],
+ [6, 7, 8]])
"""
if hasattr(a, 'filled'):
return a.filled(fill_value)
+
elif isinstance(a, ndarray):
# Should we check for contiguity ? and a.flags['CONTIGUOUS']:
return a
@@ -3653,6 +3664,14 @@ class MaskedArray(ndarray):
@fill_value.setter
def fill_value(self, value=None):
target = _check_fill_value(value, self.dtype)
+ if not target.ndim == 0:
+ # 2019-11-12, 1.18.0
+ warnings.warn(
+ "Non-scalar arrays for the fill value are deprecated. Use "
+ "arrays with scalar values instead. The filled function "
+ "still supports any array as `fill_value`.",
+ DeprecationWarning, stacklevel=2)
+
_fill_value = self._fill_value
if _fill_value is None:
# Create the attribute if it was undefined
@@ -3673,9 +3692,11 @@ class MaskedArray(ndarray):
Parameters
----------
- fill_value : scalar, optional
- The value to use for invalid entries (None by default).
- If None, the `fill_value` attribute of the array is used instead.
+ fill_value : array_like, optional
+ The value to use for invalid entries. Can be scalar or non-scalar.
+ If non-scalar, the resulting ndarray must be broadcastable over
+ input array. Default is None, in which case, the `fill_value`
+ attribute of the array is used instead.
Returns
-------
@@ -3694,6 +3715,8 @@ class MaskedArray(ndarray):
>>> x = np.ma.array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999)
>>> x.filled()
array([ 1, 2, -999, 4, -999])
+ >>> x.filled(fill_value=1000)
+ array([ 1, 2, 1000, 4, 1000])
>>> type(x.filled())
<class 'numpy.ndarray'>
@@ -6220,9 +6243,11 @@ class mvoid(MaskedArray):
Parameters
----------
- fill_value : scalar, optional
- The value to use for invalid entries (None by default).
- If None, the `fill_value` attribute is used instead.
+ fill_value : array_like, optional
+ The value to use for invalid entries. Can be scalar or
+ non-scalar. If latter is the case, the filled array should
+ be broadcastable over input array. Default is None, in
+ which case the `fill_value` attribute is used instead.
Returns
-------
diff --git a/numpy/random/tests/test_extending.py b/numpy/random/tests/test_extending.py
index 6f0f7a462..efd922ff5 100644
--- a/numpy/random/tests/test_extending.py
+++ b/numpy/random/tests/test_extending.py
@@ -11,6 +11,12 @@ try:
except ImportError:
numba = None
+try:
+ import cython
+except ImportError:
+ cython = None
+
+@pytest.mark.skipif(cython is None, reason="requires cython")
def test_cython():
curdir = os.getcwd()
argv = sys.argv
diff --git a/test_requirements.txt b/test_requirements.txt
index 8b4807607..65b0c240b 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -1,5 +1,5 @@
cython==0.29.14
-pytest==5.2.4
+pytest==5.3.0
pytz==2019.3
pytest-cov==2.8.1
pickle5; python_version == '3.7'
diff --git a/tools/refguide_check.py b/tools/refguide_check.py
index c20807267..a71279173 100644
--- a/tools/refguide_check.py
+++ b/tools/refguide_check.py
@@ -2,8 +2,10 @@
"""
refguide_check.py [OPTIONS] [-- ARGS]
-Check for a NumPy submodule whether the objects in its __all__ dict
-correspond to the objects included in the reference guide.
+- Check for a NumPy submodule whether the objects in its __all__ dict
+ correspond to the objects included in the reference guide.
+- Check docstring examples
+- Check example blocks in RST files
Example of usage::
@@ -15,12 +17,13 @@ objects are left out of the refguide for a good reason (it's an alias of
another function, or deprecated, or ...)
Another use of this helper script is to check validity of code samples
-in docstrings. This is different from doctesting [we do not aim to have
-numpy docstrings doctestable!], this is just to make sure that code in
-docstrings is valid python::
+in docstrings::
- $ python refguide_check.py --doctests optimize
+ $ python refguide_check.py --doctests ma
+or in RST-based documentations::
+
+ $ python refguide_check.py --rst docs
"""
from __future__ import print_function
@@ -47,9 +50,11 @@ import numpy as np
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'doc', 'sphinxext'))
from numpydoc.docscrape_sphinx import get_doc_object
+SKIPBLOCK = doctest.register_optionflag('SKIPBLOCK')
+
if parse_version(sphinx.__version__) >= parse_version('1.5'):
# Enable specific Sphinx directives
- from sphinx.directives import SeeAlso, Only
+ from sphinx.directives.other import SeeAlso, Only
directives.register_directive('seealso', SeeAlso)
directives.register_directive('only', Only)
else:
@@ -102,6 +107,21 @@ DOCTEST_SKIPLIST = set([
'numpy.lib.Repository',
])
+# Skip non-numpy RST files, historical release notes
+# Any single-directory exact match will skip the directory and all subdirs.
+# Any exact match (like 'doc/release') will scan subdirs but skip files in
+# the matched directory.
+# Any filename will skip that file
+RST_SKIPLIST = [
+ 'scipy-sphinx-theme',
+ 'sphinxext',
+ 'neps',
+ 'changelog',
+ 'doc/release',
+ 'doc/source/release',
+ 'c-info.ufunc-tutorial.rst',
+]
+
# these names are not required to be present in ALL despite being in
# autosummary:: listing
REFGUIDE_ALL_SKIPLIST = [
@@ -243,6 +263,7 @@ def compare(all_dict, others, names, module_name):
return only_all, only_ref, missing
+
def is_deprecated(f):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("error")
@@ -254,7 +275,12 @@ def is_deprecated(f):
pass
return False
+
def check_items(all_dict, names, deprecated, others, module_name, dots=True):
+ """
+ Check that `all_dict` is consistent with the `names` in `module_name`
+ For instance, that there are no deprecated or extra objects.
+ """
num_all = len(all_dict)
num_ref = len(names)
@@ -450,6 +476,7 @@ DEFAULT_NAMESPACE = {'np': np}
# the namespace to do checks in
CHECK_NAMESPACE = {
'np': np,
+ 'numpy': np,
'assert_allclose': np.testing.assert_allclose,
'assert_equal': np.testing.assert_equal,
# recognize numpy repr's
@@ -465,7 +492,9 @@ CHECK_NAMESPACE = {
'nan': np.nan,
'NaN': np.nan,
'inf': np.inf,
- 'Inf': np.inf,}
+ 'Inf': np.inf,
+ 'StringIO': io.StringIO,
+}
class DTRunner(doctest.DocTestRunner):
@@ -517,7 +546,7 @@ class Checker(doctest.OutputChecker):
self.parse_namedtuples = parse_namedtuples
self.atol, self.rtol = atol, rtol
if ns is None:
- self.ns = dict(CHECK_NAMESPACE)
+ self.ns = CHECK_NAMESPACE
else:
self.ns = ns
@@ -581,9 +610,9 @@ class Checker(doctest.OutputChecker):
# and then compare the tuples.
try:
num = len(a_want)
- regex = ('[\w\d_]+\(' +
- ', '.join(['[\w\d_]+=(.+)']*num) +
- '\)')
+ regex = (r'[\w\d_]+\(' +
+ ', '.join([r'[\w\d_]+=(.+)']*num) +
+ r'\)')
grp = re.findall(regex, got.replace('\n', ' '))
if len(grp) > 1: # no more than one for now
return False
@@ -655,11 +684,21 @@ def _run_doctests(tests, full_name, verbose, doctest_warnings):
# try to ensure random seed is NOT reproducible
np.random.seed(None)
+ ns = {}
for t in tests:
+ # We broke the tests up into chunks to try to avoid PSEUDOCODE
+ # This has the unfortunate side effect of restarting the global
+ # namespace for each test chunk, so variables will be "lost" after
+ # a chunk. Chain the globals to avoid this
+ t.globs.update(ns)
t.filename = short_path(t.filename, cwd)
- fails, successes = runner.run(t, out=out)
+ # Process our options
+ if any([SKIPBLOCK in ex.options for ex in t.examples]):
+ continue
+ fails, successes = runner.run(t, out=out, clear_globs=False)
if fails > 0:
success = False
+ ns = t.globs
finally:
sys.stderr = old_stderr
os.chdir(cwd)
@@ -757,10 +796,9 @@ def check_doctests_testfile(fname, verbose, ns=None,
5
"""
- results = []
-
if ns is None:
- ns = dict(DEFAULT_NAMESPACE)
+ ns = CHECK_NAMESPACE
+ results = []
_, short_name = os.path.split(fname)
if short_name in DOCTEST_SKIPLIST:
@@ -782,20 +820,32 @@ def check_doctests_testfile(fname, verbose, ns=None,
# split the text into "blocks" and try to detect and omit pseudocode blocks.
parser = doctest.DocTestParser()
good_parts = []
+ base_line_no = 0
for part in text.split('\n\n'):
- tests = parser.get_doctest(part, ns, fname, fname, 0)
+ try:
+ tests = parser.get_doctest(part, ns, fname, fname, base_line_no)
+ except ValueError as e:
+ if e.args[0].startswith('line '):
+ # fix line number since `parser.get_doctest` does not increment
+ # the reported line number by base_line_no in the error message
+ parts = e.args[0].split()
+ parts[1] = str(int(parts[1]) + base_line_no)
+ e.args = (' '.join(parts),) + e.args[1:]
+ raise
if any(word in ex.source for word in PSEUDOCODE
for ex in tests.examples):
# omit it
pass
else:
# `part` looks like a good code, let's doctest it
- good_parts += [part]
+ good_parts.append((part, base_line_no))
+ base_line_no += part.count('\n') + 2
# Reassemble the good bits and doctest them:
- good_text = '\n\n'.join(good_parts)
- tests = parser.get_doctest(good_text, ns, fname, fname, 0)
- success, output = _run_doctests([tests], full_name, verbose,
+ tests = []
+ for good_text, line_no in good_parts:
+ tests.append(parser.get_doctest(good_text, ns, fname, fname, line_no))
+ success, output = _run_doctests(tests, full_name, verbose,
doctest_warnings)
if dots:
@@ -810,6 +860,59 @@ def check_doctests_testfile(fname, verbose, ns=None,
return results
+def iter_included_files(base_path, verbose=0, suffixes=('.rst',)):
+ """
+ Generator function to walk `base_path` and its subdirectories, skipping
+ files or directories in RST_SKIPLIST, and yield each file with a suffix in
+ `suffixes`
+ """
+ if os.path.exists(base_path) and os.path.isfile(base_path):
+ yield base_path
+ for dir_name, subdirs, files in os.walk(base_path, topdown=True):
+ if dir_name in RST_SKIPLIST:
+ if verbose > 0:
+ sys.stderr.write('skipping files in %s' % dir_name)
+ files = []
+ for p in RST_SKIPLIST:
+ if p in subdirs:
+ if verbose > 0:
+ sys.stderr.write('skipping %s and subdirs' % p)
+ subdirs.remove(p)
+ for f in files:
+ if (os.path.splitext(f)[1] in suffixes and
+ f not in RST_SKIPLIST):
+ yield os.path.join(dir_name, f)
+
+
+def check_documentation(base_path, results, args, dots):
+ """
+ Check examples in any *.rst located inside `base_path`.
+ Add the output to `results`.
+
+ See Also
+ --------
+ check_doctests_testfile
+ """
+ for filename in iter_included_files(base_path, args.verbose):
+ if dots:
+ sys.stderr.write(filename + ' ')
+ sys.stderr.flush()
+
+ tut_results = check_doctests_testfile(
+ filename,
+ (args.verbose >= 2), dots=dots,
+ doctest_warnings=args.doctest_warnings)
+
+ # stub out a "module" which is needed when reporting the result
+ def scratch():
+ pass
+ scratch.__name__ = filename
+ results.append((scratch, tut_results))
+ if dots:
+ sys.stderr.write('\n')
+ sys.stderr.flush()
+
+
def init_matplotlib():
global HAVE_MATPLOTLIB
@@ -825,20 +928,21 @@ def main(argv):
parser = ArgumentParser(usage=__doc__.lstrip())
parser.add_argument("module_names", metavar="SUBMODULES", default=[],
nargs='*', help="Submodules to check (default: all public)")
- parser.add_argument("--doctests", action="store_true", help="Run also doctests")
+ parser.add_argument("--doctests", action="store_true",
+ help="Run also doctests on ")
parser.add_argument("-v", "--verbose", action="count", default=0)
parser.add_argument("--doctest-warnings", action="store_true",
help="Enforce warning checking for doctests")
- parser.add_argument("--skip-tutorial", action="store_true",
- help="Skip running doctests in the tutorial.")
+ parser.add_argument("--rst", nargs='?', const='doc', default=None,
+ help=("Run also examples from *rst files"
+ "dicovered walking the directory(s) specified, "
+ "defaults to 'doc'"))
args = parser.parse_args(argv)
modules = []
names_dict = {}
- if args.module_names:
- args.skip_tutorial = True
- else:
+ if not args.module_names:
args.module_names = list(PUBLIC_SUBMODULES)
os.environ['SCIPY_PIL_IMAGE_VIEWER'] = 'true'
@@ -850,6 +954,15 @@ def main(argv):
if name not in module_names:
module_names.append(name)
+ dots = True
+ success = True
+ results = []
+ errormsgs = []
+
+
+ if args.doctests or args.rst:
+ init_matplotlib()
+
for submodule_name in module_names:
module_name = BASE_MODULE + '.' + submodule_name
__import__(module_name)
@@ -861,69 +974,55 @@ def main(argv):
if submodule_name in args.module_names:
modules.append(module)
- dots = True
- success = True
- results = []
-
- print("Running checks for %d modules:" % (len(modules),))
- if args.doctests or not args.skip_tutorial:
- init_matplotlib()
-
- for module in modules:
- if dots:
- if module is not modules[0]:
- sys.stderr.write(' ')
- sys.stderr.write(module.__name__ + ' ')
- sys.stderr.flush()
+ if args.doctests or not args.rst:
+ print("Running checks for %d modules:" % (len(modules),))
+ for module in modules:
+ if dots:
+ sys.stderr.write(module.__name__ + ' ')
+ sys.stderr.flush()
- all_dict, deprecated, others = get_all_dict(module)
- names = names_dict.get(module.__name__, set())
+ all_dict, deprecated, others = get_all_dict(module)
+ names = names_dict.get(module.__name__, set())
- mod_results = []
- mod_results += check_items(all_dict, names, deprecated, others, module.__name__)
- mod_results += check_rest(module, set(names).difference(deprecated),
- dots=dots)
- if args.doctests:
- mod_results += check_doctests(module, (args.verbose >= 2), dots=dots,
- doctest_warnings=args.doctest_warnings)
+ mod_results = []
+ mod_results += check_items(all_dict, names, deprecated, others,
+ module.__name__)
+ mod_results += check_rest(module, set(names).difference(deprecated),
+ dots=dots)
+ if args.doctests:
+ mod_results += check_doctests(module, (args.verbose >= 2), dots=dots,
+ doctest_warnings=args.doctest_warnings)
- for v in mod_results:
- assert isinstance(v, tuple), v
+ for v in mod_results:
+ assert isinstance(v, tuple), v
- results.append((module, mod_results))
+ results.append((module, mod_results))
- if dots:
- sys.stderr.write("\n")
- sys.stderr.flush()
-
- if not args.skip_tutorial:
- base_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')
- tut_path = os.path.join(base_dir, 'doc', 'source', 'tutorial', '*.rst')
- print('\nChecking tutorial files at %s:' % os.path.relpath(tut_path, os.getcwd()))
- for filename in sorted(glob.glob(tut_path)):
if dots:
sys.stderr.write('\n')
- sys.stderr.write(os.path.split(filename)[1] + ' ')
sys.stderr.flush()
- tut_results = check_doctests_testfile(filename, (args.verbose >= 2),
- dots=dots, doctest_warnings=args.doctest_warnings)
-
- def scratch(): pass # stub out a "module", see below
- scratch.__name__ = filename
- results.append((scratch, tut_results))
+ all_dict, deprecated, others = get_all_dict(module)
+ if args.rst:
+ base_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')
+ rst_path = os.path.relpath(os.path.join(base_dir, args.rst))
+ if os.path.exists(rst_path):
+ print('\nChecking files in %s:' % rst_path)
+ check_documentation(rst_path, results, args, dots)
+ else:
+ sys.stderr.write(f'\ninvalid --rst argument "{args.rst}"')
+ errormsgs.append('invalid directory argument to --rst')
if dots:
sys.stderr.write("\n")
sys.stderr.flush()
# Report results
- all_success = True
-
for module, mod_results in results:
success = all(x[1] for x in mod_results)
- all_success = all_success and success
+ if not success:
+ errormsgs.append(f'failed checking {module.__name__}')
if success and args.verbose == 0:
continue
@@ -946,11 +1045,11 @@ def main(argv):
print(output.strip())
print("")
- if all_success:
- print("\nOK: refguide and doctests checks passed!")
+ if len(errormsgs) == 0:
+ print("\nOK: all checks passed!")
sys.exit(0)
else:
- print("\nERROR: refguide or doctests have errors")
+ print('\nERROR: ', '\n '.join(errormsgs))
sys.exit(1)
View Lorry Controller Status