diff options
| -rw-r--r-- | doc/source/_static/versions.json | 62 | ||||
| -rw-r--r-- | doc/source/conf.py | 12 | ||||
| -rw-r--r-- | doc/source/f2py/advanced.rst | 10 | ||||
| -rw-r--r-- | doc/source/f2py/index.rst | 8 | ||||
| -rw-r--r-- | doc/source/f2py/windows/conda.rst | 34 | ||||
| -rw-r--r-- | doc/source/f2py/windows/index.rst | 208 | ||||
| -rw-r--r-- | doc/source/f2py/windows/intel.rst | 57 | ||||
| -rw-r--r-- | doc/source/f2py/windows/msys2.rst | 19 | ||||
| -rw-r--r-- | doc/source/f2py/windows/pgi.rst | 28 | ||||
| -rw-r--r-- | numpy/lib/tests/test_format.py | 9 | ||||
| -rw-r--r-- | numpy/lib/tests/test_io.py | 1 |
11 files changed, 442 insertions, 6 deletions
diff --git a/doc/source/_static/versions.json b/doc/source/_static/versions.json new file mode 100644 index 000000000..104a30de7 --- /dev/null +++ b/doc/source/_static/versions.json @@ -0,0 +1,62 @@ +[ + { + "name": "dev", + "version": "devdocs", + "url": "https://numpy.org/devdocs/" + }, + { + "name": "1.22 (stable)", + "version": "stable", + "url": "https://numpy.org/doc/stable/" + }, + { + "name": "1.22", + "version": "1.22", + "url": "https://numpy.org/doc/1.22/" + }, + { + "name": "1.21", + "version": "1.21", + "url": "https://numpy.org/doc/1.21/" + }, + { + "name": "1.20", + "version": "1.20", + "url": "https://numpy.org/doc/1.20/" + }, + { + "name": "1.19", + "version": "1.19", + "url": "https://numpy.org/doc/1.19/" + }, + { + "name": "1.18", + "version": "1.18", + "url": "https://numpy.org/doc/1.18/" + }, + { + "name": "1.17", + "version": "1.17", + "url": "https://numpy.org/doc/1.17/" + }, + { + "name": "1.16", + "version": "1.16", + "url": "https://numpy.org/doc/1.16/" + }, + { + "name": "1.15", + "version": "1.15", + "url": "https://numpy.org/doc/1.15/" + }, + { + "name": "1.14", + "version": "1.14", + "url": "https://numpy.org/doc/1.14/" + }, + { + "name": "1.13", + "version": "1.13", + "url": "https://numpy.org/doc/1.13/" + } +] diff --git a/doc/source/conf.py b/doc/source/conf.py index 4301fe553..1a201a3c5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -171,6 +171,12 @@ html_logo = '_static/numpylogo.svg' html_favicon = '_static/favicon/favicon.ico' +# Set up the version switcher. The versions.json is stored in the devdocs. +if ".dev" in version: + switcher_version = "devdocs" +else: + switcher_version = f"doc/{version}" + html_theme_options = { "logo_link": "index", "github_url": "https://github.com/numpy/numpy", @@ -179,6 +185,12 @@ html_theme_options = { "external_links": [ {"name": "Learn", "url": "https://numpy.org/numpy-tutorials/"} ], + # Add light/dark mode and documentation version switcher: + "navbar_end": ["version-switcher", "navbar-icon-links"], + "switcher": { + "version_match": switcher_version, + "json_url": "https://numpy.org/devdocs/_static/versions.json", + }, } html_title = "%s v%s Manual" % (project, version) diff --git a/doc/source/f2py/advanced.rst b/doc/source/f2py/advanced.rst index c8efbaadb..cf9984380 100644 --- a/doc/source/f2py/advanced.rst +++ b/doc/source/f2py/advanced.rst @@ -79,14 +79,18 @@ that defines mapping between Fortran type:: and the corresponding <C type>. The <C type> can be one of the following:: + double + float + long_double char signed_char + unsigned_char short + unsigned_short int + long long_long - float - double - long_double + unsigned complex_float complex_double complex_long_double diff --git a/doc/source/f2py/index.rst b/doc/source/f2py/index.rst index 913eacbb1..dedfe3f64 100644 --- a/doc/source/f2py/index.rst +++ b/doc/source/f2py/index.rst @@ -34,9 +34,15 @@ replace all calls to ``f2py`` mentioned in this guide with the longer version. .. toctree:: :maxdepth: 3 + f2py.getting-started f2py-user f2py-reference - + usage + python-usage + signature-file + buildtools/index + advanced + windows/index .. _Python: https://www.python.org/ .. _NumPy: https://www.numpy.org/ diff --git a/doc/source/f2py/windows/conda.rst b/doc/source/f2py/windows/conda.rst new file mode 100644 index 000000000..b16402bbc --- /dev/null +++ b/doc/source/f2py/windows/conda.rst @@ -0,0 +1,34 @@ +.. _f2py-win-conda: + +========================= +F2PY and Conda on Windows +========================= + +As a convienience measure, we will additionally assume the +existence of ``scoop``, which can be used to install tools without +administrative access. + +.. code-block:: powershell + + Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh') + +Now we will setup a ``conda`` environment. + +.. code-block:: powershell + + scoop install miniconda3 + # For conda activate / deactivate in powershell + conda install -n root -c pscondaenvs pscondaenvs + Powershell -c Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + conda init powershell + # Open a new shell for the rest + +``conda`` pulls packages from ``msys2``, however, the UX is sufficiently different enough to warrant a separate discussion. + +.. warning:: + + As of 30-01-2022, the `MSYS2 binaries`_ shipped with ``conda`` are **outdated** and this approach is **not preferred**. + + + +.. _MSYS2 binaries: https://github.com/conda-forge/conda-forge.github.io/issues/1044 diff --git a/doc/source/f2py/windows/index.rst b/doc/source/f2py/windows/index.rst new file mode 100644 index 000000000..aee96cb13 --- /dev/null +++ b/doc/source/f2py/windows/index.rst @@ -0,0 +1,208 @@ +.. _f2py-windows: + +================= +F2PY and Windows +================= + +.. warning:: + + F2PY support for Windows is not at par with Linux support, and + OS specific flags can be seen via ``python -m numpy.f2py`` + +Broadly speaking, there are two issues working with F2PY on Windows: + +- the lack of actively developed FOSS Fortran compilers, and, +- the linking issues related to the C runtime library for building Python-C extensions. + +The focus of this section is to establish a guideline for developing and +extending Fortran modules for Python natively, via F2PY on Windows. + +Overview +======== +From a user perspective, the most UNIX compatible Windows +development environment is through emulation, either via the Windows Subsystem +on Linux, or facilitated by Docker. In a similar vein, traditional +virtualization methods like VirtualBox are also reasonable methods to develop +UNIX tools on Windows. + +Native Windows support is typically stunted beyond the usage of commercial compilers. +However, as of 2022, most commercial compilers have free plans which are sufficient for +general use. Additionally, the Fortran language features supported by ``f2py`` +(partial coverage of Fortran 2003), means that newer toolchains are often not +required. Briefly, then, for an end user, in order of use: + +Classic Intel Compilers (commercial) + These are maintained actively, though licensing restrictions may apply as + further detailed in :ref:`f2py-win-intel`. + + Suitable for general use for those building native Windows programs by + building off of MSVC. + +MSYS2 (FOSS) + In conjunction with the ``mingw-w64`` project, ``gfortran`` and ``gcc`` + toolchains can be used to natively build Windows programs. + +Windows Subsystem for Linux + Assuming the usage of ``gfortran``, this can be used for cross-compiling + Windows applications, but is significantly more complicated. + +Conda + Windows support for compilers in ``conda`` is facilitated by pulling MSYS2 + binaries, however these `are outdated`_, and therefore not recommended (as of 30-01-2022). + +PGI Compilers (commercial) + Unmaintained but sufficient if an existing license is present. Works + natively, but has been superseded by the Nvidia HPC SDK, with no `native + Windows support`_. + +Cygwin (FOSS) + Can also be used for ``gfortran``. Howeve, the POSIX API compatibility layer provided by + Cygwin is meant to compile UNIX software on Windows, instead of building + native Windows programs. This means cross compilation is required. + +The compilation suites described so far are compatible with the `now +deprecated`_ ``np.distutils`` build backend which is exposed by the F2PY CLI. +Additional build system usage (``meson``, ``cmake``) as described in +:ref:`f2py-bldsys` allows for a more flexible set of compiler +backends including: + +Intel oneAPI + The newer Intel compilers (``ifx``, ``icx``) are based on LLVM and can be + used for native compilation. Licensing requirements can be onerous. + +Classic Flang (FOSS) + The backbone of the PGI compilers were cannibalized to form the "classic" or + `legacy version of Flang`_. This may be compiled from source and used + natively. `LLVM Flang`_ does not support Windows yet (30-01-2022). + +LFortran (FOSS) + One of two LLVM based compilers. Not all of F2PY supported Fortran can be + compiled yet (30-01-2022) but uses MSVC for native linking. + + +Baseline +======== + +For this document we will asume the following basic tools: + +- The IDE being considered is the community supported `Microsoft Visual Studio Code`_ +- The terminal being used is the `Windows Terminal`_ +- The shell environment is assumed to be `Powershell 7.x`_ +- Python 3.10 from `the Microsoft Store`_ and this can be tested with + ``Get-Command python.exe`` resolving to + ``C:\Users\$USERNAME\AppData\Local\Microsoft\WindowsApps\python.exe`` +- The Microsoft Visual C++ (MSVC) toolset + +With this baseline configuration, we will further consider a configuration +matrix as follows: + +.. _table-f2py-winsup-mat: + +.. table:: Support matrix, exe implies a Windows installer + + +----------------------+--------------------+-------------------+ + | **Fortran Compiler** | **C/C++ Compiler** | **Source** | + +======================+====================+===================+ + | Intel Fortran | MSVC / ICC | exe | + +----------------------+--------------------+-------------------+ + | GFortran | MSVC | MSYS2/exe | + +----------------------+--------------------+-------------------+ + | GFortran | GCC | WSL | + +----------------------+--------------------+-------------------+ + | Classic Flang | MSVC | Source / Conda | + +----------------------+--------------------+-------------------+ + | Anaconda GFortran | Anaconda GCC | exe | + +----------------------+--------------------+-------------------+ + +For an understanding of the key issues motivating the need for such a matrix +`Pauli Virtanen's in-depth post on wheels with Fortran for Windows`_ is an +excellent resource. An entertaining explanation of an application binary +interface (ABI) can be found in this post by `JeanHeyd Meneide`_. + +Powershell and MSVC +==================== + +MSVC is installed either via the Visual Studio Bundle or the lighter (preferred) +`Build Tools for Visual Studio`_ with the ``Desktop development with C++`` +setting. + +.. note:: + + This can take a significant amount of time as it includes a download of around + 2GB and requires a restart. + +It is possible to use the resulting environment from a `standard command +prompt`_. However, it is more pleasant to use a `developer powershell`_, +with a `profile in Windows Terminal`_. This can be achieved by adding the +following block to the ``profiles->list`` section of the JSON file used to +configure Windows Terminal (see ``Settings->Open JSON file``): + +.. code-block:: json + + { + "name": "Developer PowerShell for VS 2019", + "commandline": "powershell.exe -noe -c \"$vsPath = (Join-Path ${env:ProgramFiles(x86)} -ChildPath 'Microsoft Visual Studio\\2019\\BuildTools'); Import-Module (Join-Path $vsPath 'Common7\\Tools\\Microsoft.VisualStudio.DevShell.dll'); Enter-VsDevShell -VsInstallPath $vsPath -SkipAutomaticLocation\"", + "icon": "ms-appx:///ProfileIcons/{61c54bbd-c2c6-5271-96e7-009a87ff44bf}.png" + } + +Now, testing the compiler toolchain could look like: + +.. code-block:: powershell + + # New Windows Developer Powershell instance / tab + # or + $vsPath = (Join-Path ${env:ProgramFiles(x86)} -ChildPath 'Microsoft Visual Studio\\2019\\BuildTools'); + Import-Module (Join-Path $vsPath 'Common7\\Tools\\Microsoft.VisualStudio.DevShell.dll'); + Enter-VsDevShell -VsInstallPath $vsPath -SkipAutomaticLocation + ********************************************************************** + ** Visual Studio 2019 Developer PowerShell v16.11.9 + ** Copyright (c) 2021 Microsoft Corporation + ********************************************************************** + cd $HOME + echo "#include<stdio.h>" > blah.cpp; echo 'int main(){printf("Hi");return 1;}' >> blah.cpp + cl blah.cpp + .\blah.exe + # Hi + rm blah.cpp + +It is also possible to check that the environment has been updated correctly +with ``$ENV:PATH``. + + +Windows Store Python Paths +========================== + +The MS Windows version of Python discussed here installs to a non-deterministic +path using a hash. This needs to be added to the ``PATH`` variable. + +.. code-block:: powershell + + $Env:Path += ";$env:LOCALAPPDATA\packages\pythonsoftwarefoundation.python.3.10_qbz5n2kfra8p0\localcache\local-packages\python310\scripts" + +.. toctree:: + :maxdepth: 2 + + intel + msys2 + conda + pgi + + +.. _the Microsoft Store: https://www.microsoft.com/en-us/p/python-310/9pjpw5ldxlz5 +.. _Microsoft Visual Studio Code: https://code.visualstudio.com/Download +.. _more complete POSIX environment: https://www.cygwin.com/ +.. _This MSYS2 document: https://www.msys2.org/wiki/How-does-MSYS2-differ-from-Cygwin/ +.. _Build Tools for Visual Studio: https://visualstudio.microsoft.com/downloads/#build-tools-for-visual-studio-2019 +.. _Windows Terminal: https://www.microsoft.com/en-us/p/windows-terminal/9n0dx20hk701?activetab=pivot:overviewtab +.. _Powershell 7.x: https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.1 +.. _standard command prompt: https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-160#developer_command_file_locations +.. _developer powershell: https://docs.microsoft.com/en-us/visualstudio/ide/reference/command-prompt-powershell?view=vs-2019 +.. _profile in Windows Terminal: https://techcommunity.microsoft.com/t5/microsoft-365-pnp-blog/add-developer-powershell-and-developer-command-prompt-for-visual/ba-p/2243078 +.. _Pauli Virtanen's in-depth post on wheels with Fortran for Windows: https://pav.iki.fi/blog/2017-10-08/pywingfortran.html#building-python-wheels-with-fortran-for-windows +.. _Nvidia HPC SDK: https://www.pgroup.com/index.html +.. _JeanHeyd Meneide: https://thephd.dev/binary-banshees-digital-demons-abi-c-c++-help-me-god-please +.. _legacy version of Flang: https://github.com/flang-compiler/flang +.. _native Windows support: https://developer.nvidia.com/nvidia-hpc-sdk-downloads#collapseFour +.. _are outdated: https://github.com/conda-forge/conda-forge.github.io/issues/1044 +.. _now deprecated: https://github.com/numpy/numpy/pull/20875 +.. _LLVM Flang: https://releases.llvm.org/11.0.0/tools/flang/docs/ReleaseNotes.html diff --git a/doc/source/f2py/windows/intel.rst b/doc/source/f2py/windows/intel.rst new file mode 100644 index 000000000..ab0cea219 --- /dev/null +++ b/doc/source/f2py/windows/intel.rst @@ -0,0 +1,57 @@ +.. _f2py-win-intel: + +============================== +F2PY and Windows Intel Fortran +============================== + +As of NumPy 1.23, only the classic Intel compilers (``ifort``) are supported. + +.. note:: + + The licensing restrictions for beta software `have been relaxed`_ during + the transition to the LLVM backed ``ifx/icc`` family of compilers. + However this document does not endorse the usage of Intel in downstream + projects due to the issues pertaining to `disassembly of components and + liability`_. + + Neither the Python Intel installation nor the `Classic Intel C/C++ + Compiler` are required. + +- The `Intel Fortran Compilers`_ come in a combined installer providing both + Classic and Beta versions; these also take around a gigabyte and a half or so. + +We will consider the classic example of the generation of Fibonnaci numbers, +``fib1.f``, given by: + +.. literalinclude:: ../code/fib1.f + :language: fortran + +For ``cmd.exe`` fans, using the Intel oneAPI command prompt is the easiest approach, as +it loads the required environment for both ``ifort`` and ``msvc``. Helper batch +scripts are also provided. + +.. code-block:: bat + + # cmd.exe + "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + python -m numpy.f2py -c fib1.f -m fib1 + python -c "import fib1; import numpy as np; a=np.zeros(8); fib1.fib(a); print(a)" + +Powershell usage is a little less pleasant, and this configuration now works with MSVC as: + +.. code-block:: powershell + + # Powershell + python -m numpy.f2py -c fib1.f -m fib1 --f77exec='C:\Program Files (x86)\Intel\oneAPI\compiler\latest\windows\bin\intel64\ifort.exe' --f90exec='C:\Program Files (x86)\Intel\oneAPI\compiler\latest\windows\bin\intel64\ifort.exe' -L'C:\Program Files (x86)\Intel\oneAPI\compiler\latest\windows\compiler\lib\ia32' + python -c "import fib1; import numpy as np; a=np.zeros(8); fib1.fib(a); print(a)" + # Alternatively, set environment and reload Powershell in one line + cmd.exe /k '"C:\Program Files (x86)\Intel\oneAPI\setvars.bat" && powershell' + python -m numpy.f2py -c fib1.f -m fib1 + python -c "import fib1; import numpy as np; a=np.zeros(8); fib1.fib(a); print(a)" + +Note that the actual path to your local installation of `ifort` may vary, and the command above will need to be updated accordingly. + +.. _have been relaxed: https://www.intel.com/content/www/us/en/developer/articles/release-notes/oneapi-fortran-compiler-release-notes.html +.. _disassembly of components and liability: https://software.sintel.com/content/www/us/en/develop/articles/end-user-license-agreement.html +.. _Intel Fortran Compilers: https://www.intel.com/content/www/us/en/developer/articles/tool/oneapi-standalone-components.html#inpage-nav-6-1 +.. _Classic Intel C/C++ Compiler: https://www.intel.com/content/www/us/en/developer/articles/tool/oneapi-standalone-components.html#inpage-nav-6-undefined
\ No newline at end of file diff --git a/doc/source/f2py/windows/msys2.rst b/doc/source/f2py/windows/msys2.rst new file mode 100644 index 000000000..68c435311 --- /dev/null +++ b/doc/source/f2py/windows/msys2.rst @@ -0,0 +1,19 @@ +.. _f2py-win-msys2: + +=========================== +F2PY and Windows with MSYS2 +=========================== + +Follow the standard `installation instructions`_. Then, to grab the requisite Fortran compiler with ``MVSC``: + +.. code-block:: bash + + # Assuming a fresh install + pacman -Syu # Restart the terminal + pacman -Su # Update packages + # Get the toolchains + pacman -S --needed base-devel gcc-fortran + pacman -S mingw-w64-x86_64-toolchain + + +.. _`installation instructions`: https://www.msys2.org/ diff --git a/doc/source/f2py/windows/pgi.rst b/doc/source/f2py/windows/pgi.rst new file mode 100644 index 000000000..3139d9c5b --- /dev/null +++ b/doc/source/f2py/windows/pgi.rst @@ -0,0 +1,28 @@ +.. _f2py-win-pgi: + +=============================== +F2PY and PGI Fortran on Windows +=============================== + +A variant of these are part of the so called "classic" Flang, however, +as classic Flang requires a custom LLVM and compilation from sources. + +.. warning:: + + Since the proprietary compilers are no longer available for + usage they are not recommended and will not be ported to the + new ``f2py`` CLI. + + + +.. note:: + + **As of November 2021** + + As of 29-01-2022, `PGI compiler toolchains`_ have been superceeded by the Nvidia + HPC SDK, with no `native Windows support`_. + +However, + +.. _PGI compiler toolchains: https://www.pgroup.com/index.html +.. _native Windows support: https://developer.nvidia.com/nvidia-hpc-sdk-downloads#collapseFour
\ No newline at end of file diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 78e67a89b..cf35e1563 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -285,6 +285,7 @@ from numpy.testing import ( assert_, assert_array_equal, assert_raises, assert_raises_regex, assert_warns, ) +from numpy.testing._private.utils import requires_memory from numpy.lib import format @@ -879,11 +880,13 @@ def test_large_file_support(tmpdir): @pytest.mark.skipif(np.dtype(np.intp).itemsize < 8, reason="test requires 64-bit system") @pytest.mark.slow +@requires_memory(free_bytes=2 * 2**30) def test_large_archive(tmpdir): # Regression test for product of saving arrays with dimensions of array # having a product that doesn't fit in int32. See gh-7598 for details. + shape = (2**30, 2) try: - a = np.empty((2**30, 2), dtype=np.uint8) + a = np.empty(shape, dtype=np.uint8) except MemoryError: pytest.skip("Could not create large file") @@ -892,10 +895,12 @@ def test_large_archive(tmpdir): with open(fname, "wb") as f: np.savez(f, arr=a) + del a + with open(fname, "rb") as f: new_a = np.load(f)["arr"] - assert_(a.shape == new_a.shape) + assert new_a.shape == shape def test_empty_npz(tmpdir): diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index a2758123b..38a751d11 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -203,6 +203,7 @@ class TestSavezLoad(RoundtripTest): self.arr_reloaded.fid.close() os.remove(self.arr_reloaded.fid.name) + @pytest.mark.skipif(IS_PYPY, reason="Hangs on PyPy") @pytest.mark.skipif(not IS_64BIT, reason="Needs 64bit platform") @pytest.mark.slow def test_big_arrays(self): |