From 8e8a8f15970822a6242dc2ecba2ba4947204b4bb Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 9 Jun 2020 21:21:40 -0700 Subject: MAINT: make typing module available at runtime Closes https://github.com/numpy/numpy/issues/16550. This makes `np.typing.ArrayLike` and `np.typing.DtypeLike` available at runtime in addition to typing time. Some things to consider: - `ArrayLike` uses protocols, which are only in the standard library in 3.8+, but are backported in `typing_extensions`. This conditionally imports `Protocol` and sets `_SupportsArray` to `Any` at runtime if the module is not available to prevent NumPy from having a hard dependency on `typing_extensions`. Since e.g. mypy already includes `typing_extensions` as a dependency, anybody actually doing type checking will have it set correctly. - We are starting to hit the edges of "the fiction of the stubs". In particular, they could just cram everything into `__init__.pyi` and ignore the real structure of NumPy. But now that typing is available a runtime, we have to e.g. carefully import `ndarray` from `numpy` in the typing module and not from `..core.multiarray`, because otherwise mypy will think you are talking about a different ndarray. We will probably need to do some shuffling the stubs into more fitting locations to mitigate weirdness like this. --- numpy/setup.py | 1 + numpy/tests/test_public_api.py | 3 +- numpy/tests/typing/fail/array_like.py | 8 +---- numpy/tests/typing/pass/array_like.py | 10 ++---- numpy/typing.pyi | 64 ----------------------------------- numpy/typing/__init__.py | 3 ++ numpy/typing/_array_like.py | 27 +++++++++++++++ numpy/typing/_dtype_like.py | 46 +++++++++++++++++++++++++ numpy/typing/_shape.py | 6 ++++ 9 files changed, 88 insertions(+), 80 deletions(-) delete mode 100644 numpy/typing.pyi create mode 100644 numpy/typing/__init__.py create mode 100644 numpy/typing/_array_like.py create mode 100644 numpy/typing/_dtype_like.py create mode 100644 numpy/typing/_shape.py (limited to 'numpy') diff --git a/numpy/setup.py b/numpy/setup.py index c6498d101..cbf633504 100644 --- a/numpy/setup.py +++ b/numpy/setup.py @@ -17,6 +17,7 @@ def configuration(parent_package='',top_path=None): config.add_subpackage('polynomial') config.add_subpackage('random') config.add_subpackage('testing') + config.add_subpackage('typing') config.add_data_dir('doc') config.add_data_files('py.typed') config.add_data_files('*.pyi') diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 7ce74bc43..beaf38e5a 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -98,7 +98,7 @@ def test_dir_testing(): """Assert that output of dir has only one "testing/tester" attribute without duplicate""" assert len(dir(np)) == len(set(dir(np))) - + def test_numpy_linalg(): bad_results = check_dir(np.linalg) @@ -176,6 +176,7 @@ PUBLIC_MODULES = ['numpy.' + s for s in [ "polynomial.polyutils", "random", "testing", + "typing", "version", ]] diff --git a/numpy/tests/typing/fail/array_like.py b/numpy/tests/typing/fail/array_like.py index a5ef5795f..a97e72dc7 100644 --- a/numpy/tests/typing/fail/array_like.py +++ b/numpy/tests/typing/fail/array_like.py @@ -1,11 +1,5 @@ -from typing import Any, TYPE_CHECKING - import numpy as np - -if TYPE_CHECKING: - from numpy.typing import ArrayLike -else: - ArrayLike = Any +from numpy.typing import ArrayLike class A: diff --git a/numpy/tests/typing/pass/array_like.py b/numpy/tests/typing/pass/array_like.py index 098149c4b..e668b4963 100644 --- a/numpy/tests/typing/pass/array_like.py +++ b/numpy/tests/typing/pass/array_like.py @@ -1,13 +1,7 @@ -from typing import Any, List, Optional, TYPE_CHECKING +from typing import Any, List, Optional import numpy as np - -if TYPE_CHECKING: - from numpy.typing import ArrayLike, DtypeLike, _SupportsArray -else: - ArrayLike = Any - DtypeLike = Any - _SupportsArray = Any +from numpy.typing import ArrayLike, DtypeLike, _SupportsArray x1: ArrayLike = True x2: ArrayLike = 5 diff --git a/numpy/typing.pyi b/numpy/typing.pyi deleted file mode 100644 index f5705192a..000000000 --- a/numpy/typing.pyi +++ /dev/null @@ -1,64 +0,0 @@ -import sys -from typing import Any, Dict, List, overload, Sequence, Text, Tuple, Union - -from numpy import dtype, ndarray - -if sys.version_info >= (3, 8): - from typing import Protocol -else: - from typing_extensions import Protocol - -_Shape = Tuple[int, ...] - -# Anything that can be coerced to a shape tuple -_ShapeLike = Union[int, Sequence[int]] - -_DtypeLikeNested = Any # TODO: wait for support for recursive types - -# Anything that can be coerced into numpy.dtype. -# Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html -DtypeLike = Union[ - dtype, - # default data type (float64) - None, - # array-scalar types and generic types - type, # TODO: enumerate these when we add type hints for numpy scalars - # TODO: add a protocol for anything with a dtype attribute - # character codes, type strings or comma-separated fields, e.g., 'float64' - str, - # (flexible_dtype, itemsize) - Tuple[_DtypeLikeNested, int], - # (fixed_dtype, shape) - Tuple[_DtypeLikeNested, _ShapeLike], - # [(field_name, field_dtype, field_shape), ...] - # - # The type here is quite broad because NumPy accepts quite a wide - # range of inputs inside the list; see the tests for some - # examples. - List[Any], - # {'names': ..., 'formats': ..., 'offsets': ..., 'titles': ..., - # 'itemsize': ...} - # TODO: use TypedDict when/if it's officially supported - Dict[ - str, - Union[ - Sequence[str], # names - Sequence[_DtypeLikeNested], # formats - Sequence[int], # offsets - Sequence[Union[bytes, Text, None]], # titles - int, # itemsize - ], - ], - # {'field1': ..., 'field2': ..., ...} - Dict[str, Tuple[_DtypeLikeNested, int]], - # (base_dtype, new_dtype) - Tuple[_DtypeLikeNested, _DtypeLikeNested], -] - -class _SupportsArray(Protocol): - @overload - def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... - @overload - def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... - -ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence] diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py new file mode 100644 index 000000000..94f76a91f --- /dev/null +++ b/numpy/typing/__init__.py @@ -0,0 +1,3 @@ +from ._array_like import _SupportsArray, ArrayLike +from ._shape import _Shape, _ShapeLike +from ._dtype_like import DtypeLike diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py new file mode 100644 index 000000000..54a612fb4 --- /dev/null +++ b/numpy/typing/_array_like.py @@ -0,0 +1,27 @@ +import sys +from typing import Any, overload, Sequence, Tuple, Union + +from numpy import ndarray +from ._dtype_like import DtypeLike + +if sys.version_info >= (3, 8): + from typing import Protocol + HAVE_PROTOCOL = True +else: + try: + from typing_extensions import Protocol + except ImportError: + HAVE_PROTOCOL = False + else: + HAVE_PROTOCOL = True + +if HAVE_PROTOCOL: + class _SupportsArray(Protocol): + @overload + def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... + @overload + def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... +else: + _SupportsArray = Any + +ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence] diff --git a/numpy/typing/_dtype_like.py b/numpy/typing/_dtype_like.py new file mode 100644 index 000000000..b9df0af04 --- /dev/null +++ b/numpy/typing/_dtype_like.py @@ -0,0 +1,46 @@ +from typing import Any, Dict, List, Sequence, Tuple, Union + +from numpy import dtype +from ._shape import _ShapeLike + +_DtypeLikeNested = Any # TODO: wait for support for recursive types + +# Anything that can be coerced into numpy.dtype. +# Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html +DtypeLike = Union[ + dtype, + # default data type (float64) + None, + # array-scalar types and generic types + type, # TODO: enumerate these when we add type hints for numpy scalars + # TODO: add a protocol for anything with a dtype attribute + # character codes, type strings or comma-separated fields, e.g., 'float64' + str, + # (flexible_dtype, itemsize) + Tuple[_DtypeLikeNested, int], + # (fixed_dtype, shape) + Tuple[_DtypeLikeNested, _ShapeLike], + # [(field_name, field_dtype, field_shape), ...] + # + # The type here is quite broad because NumPy accepts quite a wide + # range of inputs inside the list; see the tests for some + # examples. + List[Any], + # {'names': ..., 'formats': ..., 'offsets': ..., 'titles': ..., + # 'itemsize': ...} + # TODO: use TypedDict when/if it's officially supported + Dict[ + str, + Union[ + Sequence[str], # names + Sequence[_DtypeLikeNested], # formats + Sequence[int], # offsets + Sequence[Union[bytes, str, None]], # titles + int, # itemsize + ], + ], + # {'field1': ..., 'field2': ..., ...} + Dict[str, Tuple[_DtypeLikeNested, int]], + # (base_dtype, new_dtype) + Tuple[_DtypeLikeNested, _DtypeLikeNested], +] diff --git a/numpy/typing/_shape.py b/numpy/typing/_shape.py new file mode 100644 index 000000000..4629046ea --- /dev/null +++ b/numpy/typing/_shape.py @@ -0,0 +1,6 @@ +from typing import Sequence, Tuple, Union + +_Shape = Tuple[int, ...] + +# Anything that can be coerced to a shape tuple +_ShapeLike = Union[int, Sequence[int]] -- cgit v1.2.1 From 70130f848b7c526862fa6ff9667f078a628d86a1 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 10 Jun 2020 20:48:36 -0700 Subject: DOC: add documentation for the numpy.typing module --- numpy/typing/__init__.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) (limited to 'numpy') diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index 94f76a91f..bc3442a15 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -1,3 +1,56 @@ +""" +============================ +Typing (:mod:`numpy.typing`) +============================ + +Large parts of the NumPy API have PEP-484-style type annotations. In +addition, the following type aliases are available for users. + +- ``typing.ArrayLike``: objects that can be converted to arrays +- ``typing.DtypeLike``: objects that can be converted to dtypes + +Roughly speaking, ``typing.ArrayLike`` is "objects that can be used as +inputs to ``np.array``" and ``typing.DtypeLike`` is "objects that can +be used as inputs to ``np.dtype``". + +Differences from the runtime NumPy API +-------------------------------------- + +NumPy is very flexible. Trying to describe the full range of +possibilities statically would result in types that are not very +helpful. For that reason, the typed NumPy API is often stricter than +the runtime NumPy API. This section describes some notable +differences. + +ArrayLike +~~~~~~~~~ + +The ``ArrayLike`` type tries to avoid creating object arrays. For +example, + +.. code-block:: python + + np.array(x**2 for x in range(10)) + +is valid NumPy code which will create an object array. The types will +complain about this usage however. + +ndarray +~~~~~~~ + +It's possible to mutate the dtype of an array at runtime. For example, +the following code is valid: + +.. code-block:: python + + x = np.array([1, 2]) + x.dtype = np.bool_ + +This sort of mutation is not allowed by the types. Users who want to +write statically typed code should insted use the `numpy.ndarray.view` +method to create a view of the array with a different dtype. + +""" from ._array_like import _SupportsArray, ArrayLike from ._shape import _Shape, _ShapeLike from ._dtype_like import DtypeLike -- cgit v1.2.1 From 4a120f0d073506329b564bfe9e69db2f6b612964 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 11 Jun 2020 08:42:55 -0700 Subject: DOC: add warning about typing-extensions module to numpy.typing docs Typing `ArrayLike` correctly relies on `Protocol`, so warn users that they should be on 3.8+ or install `typing-extensions` if they want everything to work as expected. --- numpy/typing/__init__.py | 9 +++++++++ numpy/typing/_array_like.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'numpy') diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index bc3442a15..1d377f8c4 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -3,6 +3,13 @@ Typing (:mod:`numpy.typing`) ============================ +.. warning:: + + Some of the types in this module rely on features only present in + the standard library in Python 3.8 and greater. If you want to use + these types in earlier versions of Python, you should install the + typing-extensions_ package. + Large parts of the NumPy API have PEP-484-style type annotations. In addition, the following type aliases are available for users. @@ -13,6 +20,8 @@ Roughly speaking, ``typing.ArrayLike`` is "objects that can be used as inputs to ``np.array``" and ``typing.DtypeLike`` is "objects that can be used as inputs to ``np.dtype``". +.. _typing-extensions: https://pypi.org/project/typing-extensions/ + Differences from the runtime NumPy API -------------------------------------- diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index 54a612fb4..b73585cf0 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -1,5 +1,5 @@ import sys -from typing import Any, overload, Sequence, Tuple, Union +from typing import Any, overload, Sequence, TYPE_CHECKING, Union from numpy import ndarray from ._dtype_like import DtypeLike @@ -15,7 +15,7 @@ else: else: HAVE_PROTOCOL = True -if HAVE_PROTOCOL: +if TYPE_CHECKING or HAVE_PROTOCOL: class _SupportsArray(Protocol): @overload def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... -- cgit v1.2.1 From c63f2333288772defcd84627986b035b6e7018ef Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 15 Jun 2020 20:56:57 -0700 Subject: DOC: clarify `ArrayLike` example in typing docs --- numpy/typing/__init__.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'numpy') diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index 1d377f8c4..f2000823f 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -39,10 +39,26 @@ example, .. code-block:: python - np.array(x**2 for x in range(10)) + >>> np.array(x**2 for x in range(10)) + array( at 0x10c004cd0>, dtype=object) -is valid NumPy code which will create an object array. The types will -complain about this usage however. +is valid NumPy code which will create a 0-dimensional object +array. Type checkers will complain about the above example when using +the NumPy types however. If you really intended to do the above, then +you can either use a ``# type: ignore`` comment: + +.. code-block:: python + + >>> np.array(x**2 for x in range(10)) # type: ignore + +or explicitly type the array like object as ``Any``: + +.. code-block:: python + + >>> from typing import Any + >>> array_like: Any = (x**2 for x in range(10)) + >>> np.array(array_like) + array( at 0x1192741d0>, dtype=object) ndarray ~~~~~~~ @@ -52,8 +68,8 @@ the following code is valid: .. code-block:: python - x = np.array([1, 2]) - x.dtype = np.bool_ + x = np.array([1, 2]) + x.dtype = np.bool_ This sort of mutation is not allowed by the types. Users who want to write statically typed code should insted use the `numpy.ndarray.view` -- cgit v1.2.1 From 347a368cd937b73c74ec8f684dcbaacb233ec84a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 15 Jun 2020 20:57:24 -0700 Subject: DOC: add note about supporting buffer protocols in `ArrayLike` --- numpy/typing/_array_like.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'numpy') diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index b73585cf0..a4c526778 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -24,4 +24,9 @@ if TYPE_CHECKING or HAVE_PROTOCOL: else: _SupportsArray = Any +# TODO: support buffer protocols once +# +# https://github.com/python/typing/issues/593 +# +# is resolved. ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence] -- cgit v1.2.1 From d985e8ca2fca154d3770c842a2da1ba6dc3aaf1c Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 16 Jun 2020 21:32:29 -0700 Subject: DOC: add reference to Python issue about buffer protocols --- numpy/typing/_array_like.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'numpy') diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index a4c526778..76c0c839c 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -26,7 +26,9 @@ else: # TODO: support buffer protocols once # -# https://github.com/python/typing/issues/593 +# https://bugs.python.org/issue27501 +# +# is resolved. See also the mypy issue: # -# is resolved. +# https://github.com/python/typing/issues/593 ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence] -- cgit v1.2.1