diff options
Diffstat (limited to 'doc/source')
152 files changed, 11654 insertions, 1245 deletions
diff --git a/doc/source/_static/numpy_logo.png b/doc/source/_static/numpy_logo.png Binary files differnew file mode 100644 index 000000000..af8cbe323 --- /dev/null +++ b/doc/source/_static/numpy_logo.png diff --git a/doc/source/_templates/autosummary/base.rst b/doc/source/_templates/autosummary/base.rst new file mode 100644 index 000000000..0331154a7 --- /dev/null +++ b/doc/source/_templates/autosummary/base.rst @@ -0,0 +1,14 @@ +{% if objtype == 'property' %} +:orphan: +{% endif %} + +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +{% if objtype == 'property' %} +property +{% endif %} + +.. auto{{ objtype }}:: {{ objname }} + diff --git a/doc/source/_templates/indexcontent.html b/doc/source/_templates/indexcontent.html index 008eaaa7c..294d39233 100644 --- a/doc/source/_templates/indexcontent.html +++ b/doc/source/_templates/indexcontent.html @@ -7,6 +7,8 @@ <span class="linkdescr">start here</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("reference/index") }}">NumPy Reference</a><br/> <span class="linkdescr">reference documentation</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("benchmarking") }}">Benchmarking</a><br/> + <span class="linkdescr">benchmarking NumPy</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("f2py/index") }}">F2Py Guide</a><br/> <span class="linkdescr">f2py documentation</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("dev/index") }}">NumPy Developer Guide</a><br/> diff --git a/doc/source/_templates/indexsidebar.html b/doc/source/_templates/indexsidebar.html index 51e7c4308..4707fc0e8 100644 --- a/doc/source/_templates/indexsidebar.html +++ b/doc/source/_templates/indexsidebar.html @@ -1,4 +1,5 @@ <h3>Resources</h3> <ul> + <li><a href="https://numpy.org/">NumPy.org website</a></li> <li><a href="https://scipy.org/">Scipy.org website</a></li> </ul> diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html index 77da54a00..beaa297db 100644 --- a/doc/source/_templates/layout.html +++ b/doc/source/_templates/layout.html @@ -1,5 +1,15 @@ {% extends "!layout.html" %} +{%- block header %} +<div class="container"> + <div class="top-scipy-org-logo-header" style="background-color: #a2bae8;"> + <a href="{{ pathto('index') }}"> + <img border=0 alt="NumPy" src="{{ pathto('_static/numpy_logo.png', 1) }}"></a> + </div> + </div> +</div> + +{% endblock %} {% block rootrellink %} {% if pagename != 'index' %} <li class="active"><a href="{{ pathto('index') }}">{{ shorttitle|e }}</a></li> diff --git a/doc/source/about.rst b/doc/source/about.rst index 5ac4facbb..3e83833d1 100644 --- a/doc/source/about.rst +++ b/doc/source/about.rst @@ -8,7 +8,7 @@ needed for scientific computing with Python. This package contains: - sophisticated :ref:`(broadcasting) functions <ufuncs>` - basic :ref:`linear algebra functions <routines.linalg>` - basic :ref:`Fourier transforms <routines.fft>` -- sophisticated :ref:`random number capabilities <routines.random>` +- sophisticated :ref:`random number capabilities <numpyrandom>` - tools for integrating Fortran code - tools for integrating C/C++ code diff --git a/doc/source/benchmarking.rst b/doc/source/benchmarking.rst new file mode 100644 index 000000000..9f0eeb03a --- /dev/null +++ b/doc/source/benchmarking.rst @@ -0,0 +1 @@ +.. include:: ../../benchmarks/README.rst diff --git a/doc/source/conf.py b/doc/source/conf.py index 455e9748b..83cecc917 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -3,12 +3,8 @@ from __future__ import division, absolute_import, print_function import sys, os, re -# Check Sphinx version -import sphinx -if sphinx.__version__ < "1.2.1": - raise RuntimeError("Sphinx 1.2.1 or newer required") - -needs_sphinx = '1.0' +# Minimum version, enforced by sphinx +needs_sphinx = '2.2.0' # ----------------------------------------------------------------------------- # General configuration @@ -19,17 +15,22 @@ needs_sphinx = '1.0' sys.path.insert(0, os.path.abspath('../sphinxext')) -extensions = ['sphinx.ext.autodoc', 'numpydoc', - 'sphinx.ext.intersphinx', 'sphinx.ext.coverage', - 'sphinx.ext.doctest', 'sphinx.ext.autosummary', - 'sphinx.ext.graphviz', 'sphinx.ext.ifconfig', - 'matplotlib.sphinxext.plot_directive'] +extensions = [ + 'sphinx.ext.autodoc', + 'numpydoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.autosummary', + 'sphinx.ext.graphviz', + 'sphinx.ext.ifconfig', + 'matplotlib.sphinxext.plot_directive', + 'IPython.sphinxext.ipython_console_highlighting', + 'IPython.sphinxext.ipython_directive', + 'sphinx.ext.imgmath', +] -if sphinx.__version__ >= "1.4": - extensions.append('sphinx.ext.imgmath') - imgmath_image_format = 'svg' -else: - extensions.append('sphinx.ext.pngmath') +imgmath_image_format = 'svg' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -37,9 +38,11 @@ templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' +master_doc = 'contents' + # General substitutions. project = 'NumPy' -copyright = '2008-2018, The SciPy community' +copyright = '2008-2019, The SciPy community' # The default replacements for |version| and |release|, also used in various # other places throughout the built documents. @@ -85,6 +88,7 @@ pygments_style = 'sphinx' def setup(app): # add a config value for `ifconfig` directives app.add_config_value('python_version_major', str(sys.version_info.major), 'env') + app.add_lexer('NumPyC', NumPyLexer(stripnl=False)) # ----------------------------------------------------------------------------- # HTML output @@ -113,7 +117,9 @@ else: "edit_link": False, "sidebar": "left", "scipy_org_logo": False, - "rootlinks": [] + "rootlinks": [("https://numpy.org/", "NumPy.org"), + ("https://numpy.org/doc", "Docs"), + ] } html_sidebars = {'index': ['indexsidebar.html', 'searchbox.html']} @@ -167,6 +173,10 @@ latex_documents = [ # not chapters. #latex_use_parts = False +latex_elements = { + 'fontenc': r'\usepackage[LGR,T1]{fontenc}' +} + # Additional stuff for the LaTeX preamble. latex_preamble = r''' \usepackage{amsmath} @@ -234,7 +244,7 @@ numpydoc_use_plots = True # ----------------------------------------------------------------------------- import glob -autosummary_generate = glob.glob("reference/*.rst") +autosummary_generate = True # ----------------------------------------------------------------------------- # Coverage checker @@ -355,3 +365,18 @@ def linkcode_resolve(domain, info): else: return "https://github.com/numpy/numpy/blob/v%s/numpy/%s%s" % ( numpy.__version__, fn, linespec) + +from pygments.lexers import CLexer +from pygments import token +import copy + +class NumPyLexer(CLexer): + name = 'NUMPYLEXER' + + tokens = copy.deepcopy(CLexer.tokens) + # Extend the regex for valid identifiers with @ + for k, val in tokens.items(): + for i, v in enumerate(val): + if isinstance(v, tuple): + if isinstance(v[0], str): + val[i] = (v[0].replace('a-zA-Z', 'a-zA-Z@'),) + v[1:] diff --git a/doc/source/dev/conduct/code_of_conduct.rst b/doc/source/dev/conduct/code_of_conduct.rst index 604f14662..aca39d8a7 100644 --- a/doc/source/dev/conduct/code_of_conduct.rst +++ b/doc/source/dev/conduct/code_of_conduct.rst @@ -121,8 +121,7 @@ a conflict of interest in handling it, then they will recuse themselves from considering your report. Alternatively, if for any reason you feel uncomfortable making a report to the committee, then you can also contact: -- NumFOCUS Executive Director: Leah Silen -- NumFOCUS President: Andy Terrel +- Senior `NumFOCUS staff <https://numfocus.org/code-of-conduct#persons-responsible>`__: conduct@numfocus.org Incident reporting resolution & Code of Conduct enforcement diff --git a/doc/source/dev/conduct/report_handling_manual.rst b/doc/source/dev/conduct/report_handling_manual.rst index 5f5e32f13..d39b615bb 100644 --- a/doc/source/dev/conduct/report_handling_manual.rst +++ b/doc/source/dev/conduct/report_handling_manual.rst @@ -1,3 +1,5 @@ +:orphan: + .. _CoC_reporting_manual: NumPy Code of Conduct - How to follow up on a report diff --git a/doc/source/dev/development_environment.rst b/doc/source/dev/development_environment.rst index aa4326f63..9d618cc9f 100644 --- a/doc/source/dev/development_environment.rst +++ b/doc/source/dev/development_environment.rst @@ -3,18 +3,27 @@ Setting up and using your development environment ================================================= +.. _recommended-development-setup: + Recommended development setup ----------------------------- Since NumPy contains parts written in C and Cython that need to be compiled before use, make sure you have the necessary compilers and Python -development headers installed - see :ref:`building-from-source`. +development headers installed - see :ref:`building-from-source`. Building +NumPy as of version ``1.17`` requires a C99 compliant compiler. For +some older compilers this may require ``export CFLAGS='-std=c99'``. Having compiled code also means that importing NumPy from the development sources needs some additional steps, which are explained below. For the rest of this chapter we assume that you have set up your git repo as described in :ref:`using-git`. +.. _testing-builds: + +Testing builds +-------------- + To build the development version of NumPy and run tests, spawn interactive shells with the Python import paths properly set up etc., do one of:: @@ -43,6 +52,10 @@ When using pytest as a target (the default), you can $ python runtests.py -v -t numpy/core/tests/test_multiarray.py -- -k "MatMul and not vector" +.. note:: + + Remember that all tests of NumPy should pass before commiting your changes. + Using ``runtests.py`` is the recommended approach to running tests. There are also a number of alternatives to it, for example in-place build or installing to a virtualenv. See the FAQ below for details. @@ -83,19 +96,31 @@ installs a ``.egg-link`` file into your site-packages as well as adjusts the Other build options ------------------- +Build options can be discovered by running any of:: + + $ python setup.py --help + $ python setup.py --help-commands + It's possible to do a parallel build with ``numpy.distutils`` with the ``-j`` option; see :ref:`parallel-builds` for more details. -In order to install the development version of NumPy in ``site-packages``, use -``python setup.py install --user``. - A similar approach to in-place builds and use of ``PYTHONPATH`` but outside the source tree is to use:: - $ python setup.py install --prefix /some/owned/folder + $ pip install . --prefix /some/owned/folder $ export PYTHONPATH=/some/owned/folder/lib/python3.4/site-packages +NumPy uses a series of tests to probe the compiler and libc libraries for +funtions. The results are stored in ``_numpyconfig.h`` and ``config.h`` files +using ``HAVE_XXX`` definitions. These tests are run during the ``build_src`` +phase of the ``_multiarray_umath`` module in the ``generate_config_h`` and +``generate_numpyconfig_h`` functions. Since the output of these calls includes +many compiler warnings and errors, by default it is run quietly. If you wish +to see this output, you can run the ``build_src`` stage verbosely:: + + $ python build build_src -v + Using virtualenvs ----------------- @@ -125,6 +150,9 @@ the interpreter, tests can be run like this:: >>> np.test('full') # Also run tests marked as slow >>> np.test('full', verbose=2) # Additionally print test name/file + An example of a successful test : + ``4686 passed, 362 skipped, 9 xfailed, 5 warnings in 213.99 seconds`` + Or a similar way from the command line:: $ python -c "import numpy as np; np.test()" @@ -142,9 +170,9 @@ That also takes extra arguments, like ``--pdb`` which drops you into the Python debugger when a test fails or an exception is raised. Running tests with `tox`_ is also supported. For example, to build NumPy and -run the test suite with Python 3.4, use:: +run the test suite with Python 3.7, use:: - $ tox -e py34 + $ tox -e py37 For more extensive information, see :ref:`testing-guidelines` diff --git a/doc/source/dev/gitwash/development_workflow.rst b/doc/source/dev/development_workflow.rst index 9561e25f7..900431374 100644 --- a/doc/source/dev/gitwash/development_workflow.rst +++ b/doc/source/dev/development_workflow.rst @@ -28,7 +28,7 @@ In short: - *Core developers* If you want to push changes without further review, see the notes :ref:`below <pushing-to-main>`. - + This way of working helps to keep work well organized and the history as clear as possible. @@ -69,7 +69,7 @@ Overview git status # Optional git diff # Optional git add modified_file - git commit + git commit # push the branch to your own Github repo git push origin my-new-feature @@ -112,38 +112,38 @@ In more detail properly formatted and sufficiently detailed commit message. After saving your message and closing the editor, your commit will be saved. For trivial commits, a short commit message can be passed in through the command line - using the ``-m`` flag. For example, ``git commit -am "ENH: Some message"``. - + using the ``-m`` flag. For example, ``git commit -am "ENH: Some message"``. + In some cases, you will see this form of the commit command: ``git commit -a``. The extra ``-a`` flag automatically commits all modified files and removes all deleted files. This can save you some typing of numerous ``git add`` commands; however, it can add unwanted changes to a commit if you're not careful. For more information, see `why the -a flag?`_ - and the - helpful use-case description in the `tangled working copy problem`_. + helpful use-case description in the `tangled working copy problem`_. #. Push the changes to your forked repo on github_:: git push origin my-new-feature For more information, see `git push`_. - + .. note:: - + Assuming you have followed the instructions in these pages, git will create a default link to your github_ repo called ``origin``. In git >= 1.7 you can ensure that the link to origin is permanently set by using the ``--set-upstream`` option:: - + git push --set-upstream origin my-new-feature - + From now on git_ will know that ``my-new-feature`` is related to the ``my-new-feature`` branch in your own github_ repo. Subsequent push calls are then simplified to the following:: git push - + You have to use ``--set-upstream`` for each new branch that you create. - + It may be the case that while you were working on your edits, new commits have been added to ``upstream`` that affect your work. In this case, follow the @@ -194,12 +194,18 @@ Asking for your changes to be merged with the main repo ======================================================= When you feel your work is finished, you can create a pull request (PR). Github -has a nice help page that outlines the process for `filing pull requests`_. +has a nice help page that outlines the process for `filing pull requests`_. If your changes involve modifications to the API or addition/modification of a -function, you should initiate a code review. This involves sending an email to -the `NumPy mailing list`_ with a link to your PR along with a description of -and a motivation for your changes. +function, you should + +- send an email to the `NumPy mailing list`_ with a link to your PR along with + a description of and a motivation for your changes. This may generate + changes and feedback. It might be prudent to start with this step if your + change may be controversial. +- add a release note to the ``doc/release/upcoming_changes/`` directory, + following the instructions and format in the + ``doc/release/upcoming_changes/README.rst`` file. .. _rebasing-on-master: @@ -500,11 +506,11 @@ them to ``upstream`` as follows: git push upstream my-feature-branch:master -.. note:: +.. note:: It's usually a good idea to use the ``-n`` flag to ``git push`` to check first that you're about to push the changes you want to the place you want. -.. include:: git_links.inc +.. include:: gitwash/git_links.inc diff --git a/doc/source/dev/gitwash/development_setup.rst b/doc/source/dev/gitwash/development_setup.rst index 1ebd4b486..9027dda64 100644 --- a/doc/source/dev/gitwash/development_setup.rst +++ b/doc/source/dev/gitwash/development_setup.rst @@ -25,6 +25,8 @@ to the instructions at http://help.github.com/forking/ - please see that page for more detail. We're repeating some of it here just to give the specifics for the NumPy_ project, and to suggest some default names. +.. _set-up-and-configure-a-github-account: + Set up and configure a github_ account ====================================== diff --git a/doc/source/dev/gitwash/following_latest.rst b/doc/source/dev/gitwash/following_latest.rst index ad497bf9a..0e98b4ec4 100644 --- a/doc/source/dev/gitwash/following_latest.rst +++ b/doc/source/dev/gitwash/following_latest.rst @@ -1,9 +1,5 @@ .. _following-latest: -============================= - Following the latest source -============================= - These are the instructions if you just want to follow the latest *NumPy* source, but you don't need to do any development for now. If you do want to contribute a patch (excellent!) or do more extensive diff --git a/doc/source/dev/gitwash/git_development.rst b/doc/source/dev/gitwash/git_development.rst deleted file mode 100644 index 5d7d47f89..000000000 --- a/doc/source/dev/gitwash/git_development.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _git-development: - -===================== - Git for development -===================== - -Contents: - -.. toctree:: - :maxdepth: 2 - - development_setup - configure_git - dot2_dot3 diff --git a/doc/source/dev/gitwash/git_intro.rst b/doc/source/dev/gitwash/git_intro.rst index 3ce322f8f..9d596d4d4 100644 --- a/doc/source/dev/gitwash/git_intro.rst +++ b/doc/source/dev/gitwash/git_intro.rst @@ -1,42 +1,8 @@ -============ -Introduction -============ - -These pages describe a git_ and github_ workflow for the NumPy_ -project. - -There are several different workflows here, for different ways of -working with *NumPy*. - -This is not a comprehensive git_ reference, it's just a workflow for our -own project. It's tailored to the github_ hosting service. You may well -find better or quicker ways of getting stuff done with git_, but these -should get you started. - -For general resources for learning git_ see :ref:`git-resources`. - -.. _install-git: - Install git =========== -Overview --------- - -================ ============= -Debian / Ubuntu ``sudo apt-get install git-core`` -Fedora ``sudo yum install git-core`` -Windows Download and install msysGit_ -OS X Use the git-osx-installer_ -================ ============= - -In detail ---------- - -See the git_ page for the most recent information. - -Have a look at the github_ install help pages available from `github help`_ - -There are good instructions here: http://book.git-scm.com/2_installing_git.html +Developing with git can be done entirely without github. Git is a distributed +version control system. In order to use git on your machine you must `install +it`_. .. include:: git_links.inc diff --git a/doc/source/dev/gitwash/git_links.inc b/doc/source/dev/gitwash/git_links.inc index cebbb3a67..f69a3cf62 100644 --- a/doc/source/dev/gitwash/git_links.inc +++ b/doc/source/dev/gitwash/git_links.inc @@ -10,10 +10,9 @@ .. git stuff .. _git: https://git-scm.com/ -.. _github: https://github.com +.. _github: https://github.com/numpy/numpy .. _github help: https://help.github.com -.. _msysgit: https://code.google.com/p/msysgit/downloads/list -.. _git-osx-installer: https://code.google.com/p/git-osx-installer/downloads/list +.. _`install it`: https://git-scm.com/downloads .. _subversion: http://subversion.tigris.org/ .. _git cheat sheet: http://cheat.errtheblog.com/s/git .. _pro git book: https://git-scm.com/book/ diff --git a/doc/source/dev/gitwash/index.rst b/doc/source/dev/gitwash/index.rst index b867bbd97..afbb5e019 100644 --- a/doc/source/dev/gitwash/index.rst +++ b/doc/source/dev/gitwash/index.rst @@ -1,7 +1,22 @@ .. _using-git: +.. _git-development: + +===================== + Git for development +===================== + +These pages describe a general git_ and github_ workflow. + +This is not a comprehensive git_ reference. It's tailored to the github_ +hosting service. You may well find better or quicker ways of getting stuff done +with git_, but these should get you started. + +For general resources for learning git_ see :ref:`git-resources`. + +Have a look at the github_ install help pages available from `github help`_ + +.. _install-git: -Working with *NumPy* source code -================================ Contents: @@ -10,6 +25,9 @@ Contents: git_intro following_latest - git_development - development_workflow + development_setup + configure_git + dot2_dot3 git_resources + +.. include:: git_links.inc diff --git a/doc/source/dev/governance/people.rst b/doc/source/dev/governance/people.rst index 7b8d3cab0..10af7f221 100644 --- a/doc/source/dev/governance/people.rst +++ b/doc/source/dev/governance/people.rst @@ -48,7 +48,7 @@ NumFOCUS Subcommittee * Jaime Fernández del Río -* Nathaniel Smith +* Sebastian Berg * External member: Thomas Caswell @@ -56,10 +56,7 @@ NumFOCUS Subcommittee Institutional Partners ---------------------- -* UC Berkeley (Stefan van der Walt) +* UC Berkeley (Stefan van der Walt, Matti Picus, Tyler Reddy, Sebastian Berg) +* Quansight (Ralf Gommers, Hameer Abbasi) -Document history ----------------- - -https://github.com/numpy/numpy/commits/master/doc/source/dev/governance/governance.rst diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index 9ce04cc1b..306c15069 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -2,13 +2,263 @@ Contributing to NumPy ##################### +Not a coder? Not a problem! NumPy is multi-faceted, and we can use a lot of help. +These are all activities we'd like to get help with (they're all important, so +we list them in alphabetical order): + +- Code maintenance and development +- Community coordination +- DevOps +- Developing educational content & narrative documentation +- Writing technical documentation +- Fundraising +- Project management +- Marketing +- Translating content +- Website design and development + +The rest of this document discusses working on the NumPy code base and documentation. +We're in the process of updating our descriptions of other activities and roles. +If you are interested in these other activities, please contact us! +You can do this via +the `numpy-discussion mailing list <https://scipy.org/scipylib/mailing-lists.html>`__, +or on GitHub (open an issue or comment on a relevant issue). These are our preferred +communication channels (open source is open by nature!), however if you prefer +to discuss in private first, please reach out to our community coordinators +at `numpy-team@googlegroups.com` or `numpy-team.slack.com` (send an email to +`numpy-team@googlegroups.com` for an invite the first time). + + +Development process - summary +============================= + +Here's the short summary, complete TOC links are below: + +1. If you are a first-time contributor: + + * Go to `https://github.com/numpy/numpy + <https://github.com/numpy/numpy>`_ and click the + "fork" button to create your own copy of the project. + + * Clone the project to your local computer:: + + git clone https://github.com/your-username/numpy.git + + * Change the directory:: + + cd numpy + + * Add the upstream repository:: + + git remote add upstream https://github.com/numpy/numpy.git + + * Now, `git remote -v` will show two remote repositories named: + + - ``upstream``, which refers to the ``numpy`` repository + - ``origin``, which refers to your personal fork + +2. Develop your contribution: + + * Pull the latest changes from upstream:: + + git checkout master + git pull upstream master + + * Create a branch for the feature you want to work on. Since the + branch name will appear in the merge message, use a sensible name + such as 'linspace-speedups':: + + git checkout -b linspace-speedups + + * Commit locally as you progress (``git add`` and ``git commit``) + Use a :ref:`properly formatted<writing-the-commit-message>` commit message, + write tests that fail before your change and pass afterward, run all the + :ref:`tests locally<development-environment>`. Be sure to document any + changed behavior in docstrings, keeping to the NumPy docstring + :ref:`standard<howto-document>`. + +3. To submit your contribution: + + * Push your changes back to your fork on GitHub:: + + git push origin linspace-speedups + + * Enter your GitHub username and password (repeat contributors or advanced + users can remove this step by connecting to GitHub with + :ref:`SSH<set-up-and-configure-a-github-account>` . + + * Go to GitHub. The new branch will show up with a green Pull Request + button. Make sure the title and message are clear, concise, and self- + explanatory. Then click the button to submit it. + + * If your commit introduces a new feature or changes functionality, post on + the `mailing list`_ to explain your changes. For bug fixes, documentation + updates, etc., this is generally not necessary, though if you do not get + any reaction, do feel free to ask for review. + +4. Review process: + + * Reviewers (the other developers and interested community members) will + write inline and/or general comments on your Pull Request (PR) to help + you improve its implementation, documentation and style. Every single + developer working on the project has their code reviewed, and we've come + to see it as friendly conversation from which we all learn and the + overall code quality benefits. Therefore, please don't let the review + discourage you from contributing: its only aim is to improve the quality + of project, not to criticize (we are, after all, very grateful for the + time you're donating!). + + * To update your PR, make your changes on your local repository, commit, + **run tests, and only if they succeed** push to your fork. As soon as + those changes are pushed up (to the same branch as before) the PR will + update automatically. If you have no idea how to fix the test failures, + you may push your changes anyway and ask for help in a PR comment. + + * Various continuous integration (CI) services are triggered after each PR + update to build the code, run unit tests, measure code coverage and check + coding style of your branch. The CI tests must pass before your PR can be + merged. If CI fails, you can find out why by clicking on the "failed" + icon (red cross) and inspecting the build and test log. To avoid overuse + and waste of this resource, + :ref:`test your work<recommended-development-setup>` locally before + committing. + + * A PR must be **approved** by at least one core team member before merging. + Approval means the core team member has carefully reviewed the changes, + and the PR is ready for merging. + +5. Document changes + + Beyond changes to a functions docstring and possible description in the + general documentation, if your change introduces any user-facing + modifications they may need to be mentioned in the release notes. + To add your change to the release notes, you need to create a short file + with a summary and place it in ``doc/release/upcoming_changes``. + The file ``doc/release/upcoming_changes/README.rst`` details the format and + filename conventions. + + If your change introduces a deprecation, make sure to discuss this first on + GitHub or the mailing list first. If agreement on the deprecation is + reached, follow `NEP 23 deprecation policy <http://www.numpy.org/neps/ + nep-0023-backwards-compatibility.html>`_ to add the deprecation. + +6. Cross referencing issues + + If the PR relates to any issues, you can add the text ``xref gh-xxxx`` where + ``xxxx`` is the number of the issue to github comments. Likewise, if the PR + solves an issue, replace the ``xref`` with ``closes``, ``fixes`` or any of + the other flavors `github accepts <https://help.github.com/en/articles/ + closing-issues-using-keywords>`_. + + In the source code, be sure to preface any issue or PR reference with + ``gh-xxxx``. + +For a more detailed discussion, read on and follow the links at the bottom of +this page. + +Divergence between ``upstream/master`` and your feature branch +-------------------------------------------------------------- + +If GitHub indicates that the branch of your Pull Request can no longer +be merged automatically, you have to incorporate changes that have been made +since you started into your branch. Our recommended way to do this is to +:ref:`rebase on master<rebasing-on-master>`. + +Guidelines +---------- + +* All code should have tests (see `test coverage`_ below for more details). +* All code should be `documented <https://numpydoc.readthedocs.io/ + en/latest/format.html#docstring-standard>`_. +* No changes are ever committed without review and approval by a core + team member.Please ask politely on the PR or on the `mailing list`_ if you + get no response to your pull request within a week. + +Stylistic Guidelines +-------------------- + +* Set up your editor to follow `PEP 8 <https://www.python.org/dev/peps/ + pep-0008/>`_ (remove trailing white space, no tabs, etc.). Check code with + pyflakes / flake8. + +* Use numpy data types instead of strings (``np.uint8`` instead of + ``"uint8"``). + +* Use the following import conventions:: + + import numpy as np + +* For C code, see the :ref:`numpy-c-style-guide<style_guide>` + + +Test coverage +------------- + +Pull requests (PRs) that modify code should either have new tests, or modify existing +tests to fail before the PR and pass afterwards. You should :ref:`run the tests +<development-environment>` before pushing a PR. + +Tests for a module should ideally cover all code in that module, +i.e., statement coverage should be at 100%. + +To measure the test coverage, install +`pytest-cov <https://pytest-cov.readthedocs.io/en/latest/>`__ +and then run:: + + $ python runtests.py --coverage + +This will create a report in ``build/coverage``, which can be viewed with:: + + $ firefox build/coverage/index.html + +Building docs +------------- + +To build docs, run ``make`` from the ``doc`` directory. ``make help`` lists +all targets. For example, to build the HTML documentation, you can run: + +.. code:: sh + + make html + +Then, all the HTML files will be generated in ``doc/build/html/``. +Since the documentation is based on docstrings, the appropriate version of +numpy must be installed in the host python used to run sphinx. + +Requirements +~~~~~~~~~~~~ + +`Sphinx <http://www.sphinx-doc.org/en/stable/>`__ is needed to build +the documentation. Matplotlib, SciPy, and IPython are also required. + +Fixing Warnings +~~~~~~~~~~~~~~~ + +- "citation not found: R###" There is probably an underscore after a + reference in the first line of a docstring (e.g. [1]\_). Use this + method to find the source file: $ cd doc/build; grep -rin R#### + +- "Duplicate citation R###, other instance in..."" There is probably a + [2] without a [1] in one of the docstrings + +Development process - details +============================= + +The rest of the story + .. toctree:: - :maxdepth: 3 + :maxdepth: 2 conduct/code_of_conduct - gitwash/index + Git Basics <gitwash/index> development_environment + development_workflow + ../benchmarking + style_guide releasing governance/index -For core developers: see :ref:`development-workflow`. +NumPy-specific workflow is in :ref:`numpy-development-workflow +<development-workflow>`. + +.. _`mailing list`: https://mail.python.org/mailman/listinfo/numpy-devel diff --git a/doc/source/dev/gitwash/pull_button.png b/doc/source/dev/pull_button.png Binary files differindex e5031681b..e5031681b 100644 --- a/doc/source/dev/gitwash/pull_button.png +++ b/doc/source/dev/pull_button.png diff --git a/doc/source/dev/style_guide.rst b/doc/source/dev/style_guide.rst new file mode 100644 index 000000000..bede3424d --- /dev/null +++ b/doc/source/dev/style_guide.rst @@ -0,0 +1,8 @@ +.. _style_guide: + +=================== +NumPy C Style Guide +=================== + +.. include:: ../../C_STYLE_GUIDE.rst.txt + :start-line: 4 diff --git a/doc/source/docs/howto_build_docs.rst b/doc/source/docs/howto_build_docs.rst index cdf490c37..6deacda5c 100644 --- a/doc/source/docs/howto_build_docs.rst +++ b/doc/source/docs/howto_build_docs.rst @@ -5,7 +5,7 @@ Building the NumPy API and reference docs ========================================= We currently use Sphinx_ for generating the API and reference -documentation for NumPy. You will need Sphinx 1.0.1 or newer. +documentation for NumPy. You will need Sphinx 1.8.3 <= 1.8.5. If you only want to get the documentation, note that pre-built versions can be found at @@ -30,11 +30,9 @@ In addition, building the documentation requires the Sphinx extension `plot_directive`, which is shipped with Matplotlib_. This Sphinx extension can be installed by installing Matplotlib. You will also need python3.6. -Since large parts of the main documentation are stored in -docstrings, you will need to first build NumPy, and install it so -that the correct version is imported by - - >>> import numpy +Since large parts of the main documentation are obtained from numpy via +``import numpy`` and examining the docstrings, you will need to first build +NumPy, and install it so that the correct version is imported. Note that you can eg. install NumPy to a temporary location and set the PYTHONPATH environment variable appropriately. @@ -46,8 +44,11 @@ generate the docs, so write:: make html in the ``doc/`` directory. If all goes well, this will generate a -``build/html`` subdirectory containing the built documentation. Note -that building the documentation on Windows is currently not actively +``build/html`` subdirectory containing the built documentation. If you get +a message about ``installed numpy != current repo git version``, you must +either override the check by setting ``GITVER`` or re-install NumPy. + +Note that building the documentation on Windows is currently not actively supported, though it should be possible. (See Sphinx_ documentation for more information.) diff --git a/doc/source/f2py/distutils.rst b/doc/source/f2py/distutils.rst index fdcd38468..71f6eab5a 100644 --- a/doc/source/f2py/distutils.rst +++ b/doc/source/f2py/distutils.rst @@ -26,7 +26,7 @@ sources, call F2PY to construct extension modules, etc. :mod:`numpy.distutils` extends ``distutils`` with the following features: -* ``Extension`` class argument ``sources`` may contain Fortran source +* :class:`Extension` class argument ``sources`` may contain Fortran source files. In addition, the list ``sources`` may contain at most one F2PY signature file, and then the name of an Extension module must match with the ``<modulename>`` used in signature file. It is @@ -37,7 +37,7 @@ sources, call F2PY to construct extension modules, etc. to scan Fortran source files for routine signatures to construct the wrappers to Fortran codes. - Additional options to F2PY process can be given using ``Extension`` + Additional options to F2PY process can be given using :class:`Extension` class argument ``f2py_options``. * The following new ``distutils`` commands are defined: diff --git a/doc/source/f2py/run_main_session.dat b/doc/source/f2py/run_main_session.dat index b9a7e1b0d..be6cacd22 100644 --- a/doc/source/f2py/run_main_session.dat +++ b/doc/source/f2py/run_main_session.dat @@ -8,7 +8,7 @@ Post-processing... Building modules... Building module "scalar"... Wrote C/API module "scalar" to file "./scalarmodule.c" ->>> printr(r) +>>> print(r) {'scalar': {'h': ['/home/users/pearu/src_cvs/f2py/src/fortranobject.h'], 'csrc': ['./scalarmodule.c', '/home/users/pearu/src_cvs/f2py/src/fortranobject.c']}} diff --git a/doc/source/f2py/usage.rst b/doc/source/f2py/usage.rst index 0f5068e0e..5043ec430 100644 --- a/doc/source/f2py/usage.rst +++ b/doc/source/f2py/usage.rst @@ -214,32 +214,7 @@ Python module ``numpy.f2py`` The current Python interface to the ``f2py`` module is not mature and may change in the future. -The following functions are provided by the ``numpy.f2py`` module: -``run_main(<list>)`` - Equivalent to running:: +.. automodule:: numpy.f2py + :members: - f2py <args> - - where ``<args>=string.join(<list>,' ')``, but in Python. Unless - ``-h`` is used, this function returns a dictionary containing - information on generated modules and their dependencies on source - files. For example, the command ``f2py -m scalar scalar.f`` can be - executed from Python as follows - - .. include:: run_main_session.dat - :literal: - - You cannot build extension modules with this function, that is, - using ``-c`` is not allowed. Use ``compile`` command instead, see - below. - -``compile(source, modulename='untitled', extra_args='', verbose=1, source_fn=None)`` - Build extension module from Fortran 77 source string ``source``. - Return 0 if successful. - Note that this function actually calls ``f2py -c ..`` from shell to - ensure safety of the current Python process. - For example, - - .. include:: compile_session.dat - :literal: diff --git a/doc/source/dev/alignment.rst b/doc/source/reference/alignment.rst index f067f0d03..ebc8f353c 100644 --- a/doc/source/dev/alignment.rst +++ b/doc/source/reference/alignment.rst @@ -1,8 +1,10 @@ .. _alignment: +Memory Alignment +================ Numpy Alignment Goals -===================== +--------------------- There are three use-cases related to memory alignment in numpy (as of 1.14): @@ -32,8 +34,16 @@ datatype is implemented as ``struct { float real, imag; }``. This has "true" alignment of 4 and "uint" alignment of 8 (equal to the true alignment of ``uint64``). +Some cases where uint and true alignment are different (default gcc linux): + arch type true-aln uint-aln + ---- ---- -------- -------- + x86_64 complex64 4 8 + x86_64 float128 16 8 + x86 float96 4 - + + Variables in Numpy which control and describe alignment -======================================================= +------------------------------------------------------- There are 4 relevant uses of the word ``align`` used in numpy: @@ -60,7 +70,7 @@ There are 4 relevant uses of the word ``align`` used in numpy: an analagous way to how ``IsAligned`` checks for true-alignment. Consequences of alignment -========================= +------------------------- Here is how the variables above are used: @@ -80,17 +90,15 @@ Here is how the variables above are used: appropriate N. Otherwise numpy copies by doing ``memcpy(dst, src, N)``. 5. Nditer code: Since this often calls the strided copy code, it must check for "uint alignment". - 6. Cast code: if the array is "uint aligned" this will essentially do - ``*dst = CASTFUNC(*src)``. If not, it does + 6. Cast code: This checks for "true" alignment, as it does + ``*dst = CASTFUNC(*src)`` if aligned. Otherwise, it does ``memmove(srcval, src); dstval = CASTFUNC(srcval); memmove(dst, dstval)`` where dstval/srcval are aligned. -Note that in principle, only "true alignment" is required for casting code. -However, because the casting code and copy code are deeply intertwined they -both use "uint" alignment. This should be safe assuming uint alignment is -always larger than true alignment, though it can cause unnecessary buffering if -an array is "true aligned" but not "uint aligned". If there is ever a big -rewrite of this code it would be good to allow them to use different -alignments. +Note that the strided-copy and strided-cast code are deeply intertwined and so +any arrays being processed by them must be both uint and true aligned, even +though the copy-code only needs uint alignment and the cast code only true +alignment. If there is ever a big rewrite of this code it would be good to +allow them to use different alignments. diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index f17cb932a..39410b2a4 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -6,8 +6,15 @@ Standard array subclasses .. currentmodule:: numpy -The :class:`ndarray` in NumPy is a "new-style" Python -built-in-type. Therefore, it can be inherited from (in Python or in C) +.. note:: + + Subclassing a ``numpy.ndarray`` is possible but if your goal is to create + an array with *modified* behavior, as do dask arrays for distributed + computation and cupy arrays for GPU-based computation, subclassing is + discouraged. Instead, using numpy's + :ref:`dispatch mechanism <basics.dispatch>` is recommended. + +The :class:`ndarray` can be inherited from (in Python or in C) if desired. Therefore, it can form a foundation for many useful classes. Often whether to sub-class the array object or to simply use the core array component as an internal part of a new class is a @@ -43,10 +50,6 @@ NumPy provides several hooks that classes can customize: .. versionadded:: 1.13 - .. note:: The API is `provisional - <https://docs.python.org/3/glossary.html#term-provisional-api>`_, - i.e., we do not yet guarantee backward compatibility. - Any class, ndarray subclass or not, can define this method or set it to :obj:`None` in order to override the behavior of NumPy's ufuncs. This works quite similarly to Python's ``__mul__`` and other binary operation routines. @@ -79,7 +82,7 @@ NumPy provides several hooks that classes can customize: :func:`~numpy.matmul`, which currently is not a Ufunc, but could be relatively easily be rewritten as a (set of) generalized Ufuncs. The same may happen with functions such as :func:`~numpy.median`, - :func:`~numpy.min`, and :func:`~numpy.argsort`. + :func:`~numpy.amin`, and :func:`~numpy.argsort`. Like with some other special methods in python, such as ``__hash__`` and ``__iter__``, it is possible to indicate that your class does *not* @@ -151,6 +154,121 @@ NumPy provides several hooks that classes can customize: :func:`__array_prepare__`, :data:`__array_priority__` mechanism described below for ufuncs (which may eventually be deprecated). +.. py:method:: class.__array_function__(func, types, args, kwargs) + + .. versionadded:: 1.16 + + .. note:: + + - In NumPy 1.17, the protocol is enabled by default, but can be disabled + with ``NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=0``. + - In NumPy 1.16, you need to set the environment variable + ``NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=1`` before importing NumPy to use + NumPy function overrides. + - Eventually, expect to ``__array_function__`` to always be enabled. + + - ``func`` is an arbitrary callable exposed by NumPy's public API, + which was called in the form ``func(*args, **kwargs)``. + - ``types`` is a `collection <collections.abc.Collection>`_ + of unique argument types from the original NumPy function call that + implement ``__array_function__``. + - The tuple ``args`` and dict ``kwargs`` are directly passed on from the + original call. + + As a convenience for ``__array_function__`` implementors, ``types`` + provides all argument types with an ``'__array_function__'`` attribute. + This allows implementors to quickly identify cases where they should defer + to ``__array_function__`` implementations on other arguments. + Implementations should not rely on the iteration order of ``types``. + + Most implementations of ``__array_function__`` will start with two + checks: + + 1. Is the given function something that we know how to overload? + 2. Are all arguments of a type that we know how to handle? + + If these conditions hold, ``__array_function__`` should return the result + from calling its implementation for ``func(*args, **kwargs)``. Otherwise, + it should return the sentinel value ``NotImplemented``, indicating that the + function is not implemented by these types. + + There are no general requirements on the return value from + ``__array_function__``, although most sensible implementations should + probably return array(s) with the same type as one of the function's + arguments. + + It may also be convenient to define a custom decorators (``implements`` + below) for registering ``__array_function__`` implementations. + + .. code:: python + + HANDLED_FUNCTIONS = {} + + class MyArray: + def __array_function__(self, func, types, args, kwargs): + if func not in HANDLED_FUNCTIONS: + return NotImplemented + # Note: this allows subclasses that don't override + # __array_function__ to handle MyArray objects + if not all(issubclass(t, MyArray) for t in types): + return NotImplemented + return HANDLED_FUNCTIONS[func](*args, **kwargs) + + def implements(numpy_function): + """Register an __array_function__ implementation for MyArray objects.""" + def decorator(func): + HANDLED_FUNCTIONS[numpy_function] = func + return func + return decorator + + @implements(np.concatenate) + def concatenate(arrays, axis=0, out=None): + ... # implementation of concatenate for MyArray objects + + @implements(np.broadcast_to) + def broadcast_to(array, shape): + ... # implementation of broadcast_to for MyArray objects + + Note that it is not required for ``__array_function__`` implementations to + include *all* of the corresponding NumPy function's optional arguments + (e.g., ``broadcast_to`` above omits the irrelevant ``subok`` argument). + Optional arguments are only passed in to ``__array_function__`` if they + were explicitly used in the NumPy function call. + + Just like the case for builtin special methods like ``__add__``, properly + written ``__array_function__`` methods should always return + ``NotImplemented`` when an unknown type is encountered. Otherwise, it will + be impossible to correctly override NumPy functions from another object + if the operation also includes one of your objects. + + For the most part, the rules for dispatch with ``__array_function__`` + match those for ``__array_ufunc__``. In particular: + + - NumPy will gather implementations of ``__array_function__`` from all + specified inputs and call them in order: subclasses before + superclasses, and otherwise left to right. Note that in some edge cases + involving subclasses, this differs slightly from the + `current behavior <https://bugs.python.org/issue30140>`_ of Python. + - Implementations of ``__array_function__`` indicate that they can + handle the operation by returning any value other than + ``NotImplemented``. + - If all ``__array_function__`` methods return ``NotImplemented``, + NumPy will raise ``TypeError``. + + If no ``__array_function__`` methods exists, NumPy will default to calling + its own implementation, intended for use on NumPy arrays. This case arises, + for example, when all array-like arguments are Python numbers or lists. + (NumPy arrays do have a ``__array_function__`` method, given below, but it + always returns ``NotImplemented`` if any argument other than a NumPy array + subclass implements ``__array_function__``.) + + One deviation from the current behavior of ``__array_ufunc__`` is that + NumPy will only call ``__array_function__`` on the *first* argument of each + unique type. This matches Python's `rule for calling reflected methods + <https://docs.python.org/3/reference/datamodel.html#object.__ror__>`_, and + this ensures that checking overloads has acceptable performance even when + there are a large number of overloaded arguments. + .. py:method:: class.__array_finalize__(obj) This method is called whenever the system internally allocates a @@ -452,7 +570,7 @@ object, then the Python code:: some code involving val ... -calls ``val = myiter.next()`` repeatedly until :exc:`StopIteration` is +calls ``val = next(myiter)`` repeatedly until :exc:`StopIteration` is raised by the iterator. There are several ways to iterate over an array that may be useful: default iteration, flat iteration, and :math:`N`-dimensional enumeration. diff --git a/doc/source/reference/arrays.datetime.rst b/doc/source/reference/arrays.datetime.rst index e64d0c17e..2225eedb3 100644 --- a/doc/source/reference/arrays.datetime.rst +++ b/doc/source/reference/arrays.datetime.rst @@ -26,7 +26,9 @@ be either a :ref:`date unit <arrays.dtypes.dateunits>` or a :ref:`time unit <arrays.dtypes.timeunits>`. The date units are years ('Y'), months ('M'), weeks ('W'), and days ('D'), while the time units are hours ('h'), minutes ('m'), seconds ('s'), milliseconds ('ms'), and -some additional SI-prefix seconds-based units. +some additional SI-prefix seconds-based units. The datetime64 data type +also accepts the string "NAT", in any combination of lowercase/uppercase +letters, for a "Not A Time" value. .. admonition:: Example @@ -50,6 +52,11 @@ some additional SI-prefix seconds-based units. >>> np.datetime64('2005-02-25T03:30') numpy.datetime64('2005-02-25T03:30') + NAT (not a time): + + >>> numpy.datetime64('nat') + numpy.datetime64('NaT') + When creating an array of datetimes from a string, it is still possible to automatically select the unit from the inputs, by using the datetime type with generic units. @@ -100,7 +107,21 @@ Datetime and Timedelta Arithmetic NumPy allows the subtraction of two Datetime values, an operation which produces a number with a time unit. Because NumPy doesn't have a physical quantities system in its core, the timedelta64 data type was created -to complement datetime64. +to complement datetime64. The arguments for timedelta64 are a number, +to represent the number of units, and a date/time unit, such as +(D)ay, (M)onth, (Y)ear, (h)ours, (m)inutes, or (s)econds. The timedelta64 +data type also accepts the string "NAT" in place of the number for a "Not A Time" value. + +.. admonition:: Example + + >>> numpy.timedelta64(1, 'D') + numpy.timedelta64(1,'D') + + >>> numpy.timedelta64(4, 'h') + numpy.timedelta64(4,'h') + + >>> numpy.timedelta64('nAt') + numpy.timedelta64('NaT') Datetimes and Timedeltas work together to provide ways for simple datetime calculations. @@ -119,6 +140,15 @@ simple datetime calculations. >>> np.timedelta64(1,'W') / np.timedelta64(1,'D') 7.0 + >>> np.timedelta64(1,'W') % np.timedelta64(10,'D') + numpy.timedelta64(7,'D') + + >>> numpy.datetime64('nat') - numpy.datetime64('2009-01-01') + numpy.timedelta64('NaT','D') + + >>> numpy.datetime64('2009-01-01') + numpy.timedelta64('nat') + numpy.datetime64('NaT') + There are two Timedelta units ('Y', years and 'M', months) which are treated specially, because how much time they represent changes depending on when they are used. While a timedelta day unit is equivalent to @@ -363,132 +393,4 @@ As a corollary to this change, we no longer prohibit casting between datetimes with date units and datetimes with timeunits. With timezone naive datetimes, the rule for casting from dates to times is no longer ambiguous. -.. _pandas: http://pandas.pydata.org - - -Differences Between 1.6 and 1.7 Datetimes -========================================= - -The NumPy 1.6 release includes a more primitive datetime data type -than 1.7. This section documents many of the changes that have taken -place. - -String Parsing -`````````````` - -The datetime string parser in NumPy 1.6 is very liberal in what it accepts, -and silently allows invalid input without raising errors. The parser in -NumPy 1.7 is quite strict about only accepting ISO 8601 dates, with a few -convenience extensions. 1.6 always creates microsecond (us) units by -default, whereas 1.7 detects a unit based on the format of the string. -Here is a comparison.:: - - # NumPy 1.6.1 - >>> np.datetime64('1979-03-22') - 1979-03-22 00:00:00 - # NumPy 1.7.0 - >>> np.datetime64('1979-03-22') - numpy.datetime64('1979-03-22') - - # NumPy 1.6.1, unit default microseconds - >>> np.datetime64('1979-03-22').dtype - dtype('datetime64[us]') - # NumPy 1.7.0, unit of days detected from string - >>> np.datetime64('1979-03-22').dtype - dtype('<M8[D]') - - # NumPy 1.6.1, ignores invalid part of string - >>> np.datetime64('1979-03-2corruptedstring') - 1979-03-02 00:00:00 - # NumPy 1.7.0, raises error for invalid input - >>> np.datetime64('1979-03-2corruptedstring') - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - ValueError: Error parsing datetime string "1979-03-2corruptedstring" at position 8 - - # NumPy 1.6.1, 'nat' produces today's date - >>> np.datetime64('nat') - 2012-04-30 00:00:00 - # NumPy 1.7.0, 'nat' produces not-a-time - >>> np.datetime64('nat') - numpy.datetime64('NaT') - - # NumPy 1.6.1, 'garbage' produces today's date - >>> np.datetime64('garbage') - 2012-04-30 00:00:00 - # NumPy 1.7.0, 'garbage' raises an exception - >>> np.datetime64('garbage') - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - ValueError: Error parsing datetime string "garbage" at position 0 - - # NumPy 1.6.1, can't specify unit in scalar constructor - >>> np.datetime64('1979-03-22T19:00', 'h') - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: function takes at most 1 argument (2 given) - # NumPy 1.7.0, unit in scalar constructor - >>> np.datetime64('1979-03-22T19:00', 'h') - numpy.datetime64('1979-03-22T19:00-0500','h') - - # NumPy 1.6.1, reads ISO 8601 strings w/o TZ as UTC - >>> np.array(['1979-03-22T19:00'], dtype='M8[h]') - array([1979-03-22 19:00:00], dtype=datetime64[h]) - # NumPy 1.7.0, reads ISO 8601 strings w/o TZ as local (ISO specifies this) - >>> np.array(['1979-03-22T19:00'], dtype='M8[h]') - array(['1979-03-22T19-0500'], dtype='datetime64[h]') - - # NumPy 1.6.1, doesn't parse all ISO 8601 strings correctly - >>> np.array(['1979-03-22T12'], dtype='M8[h]') - array([1979-03-22 00:00:00], dtype=datetime64[h]) - >>> np.array(['1979-03-22T12:00'], dtype='M8[h]') - array([1979-03-22 12:00:00], dtype=datetime64[h]) - # NumPy 1.7.0, handles this case correctly - >>> np.array(['1979-03-22T12'], dtype='M8[h]') - array(['1979-03-22T12-0500'], dtype='datetime64[h]') - >>> np.array(['1979-03-22T12:00'], dtype='M8[h]') - array(['1979-03-22T12-0500'], dtype='datetime64[h]') - -Unit Conversion -``````````````` - -The 1.6 implementation of datetime does not convert between units correctly.:: - - # NumPy 1.6.1, the representation value is untouched - >>> np.array(['1979-03-22'], dtype='M8[D]') - array([1979-03-22 00:00:00], dtype=datetime64[D]) - >>> np.array(['1979-03-22'], dtype='M8[D]').astype('M8[M]') - array([2250-08-01 00:00:00], dtype=datetime64[M]) - # NumPy 1.7.0, the representation is scaled accordingly - >>> np.array(['1979-03-22'], dtype='M8[D]') - array(['1979-03-22'], dtype='datetime64[D]') - >>> np.array(['1979-03-22'], dtype='M8[D]').astype('M8[M]') - array(['1979-03'], dtype='datetime64[M]') - -Datetime Arithmetic -``````````````````` - -The 1.6 implementation of datetime only works correctly for a small subset of -arithmetic operations. Here we show some simple cases.:: - - # NumPy 1.6.1, produces invalid results if units are incompatible - >>> a = np.array(['1979-03-22T12'], dtype='M8[h]') - >>> b = np.array([3*60], dtype='m8[m]') - >>> a + b - array([1970-01-01 00:00:00.080988], dtype=datetime64[us]) - # NumPy 1.7.0, promotes to higher-resolution unit - >>> a = np.array(['1979-03-22T12'], dtype='M8[h]') - >>> b = np.array([3*60], dtype='m8[m]') - >>> a + b - array(['1979-03-22T15:00-0500'], dtype='datetime64[m]') - - # NumPy 1.6.1, arithmetic works if everything is microseconds - >>> a = np.array(['1979-03-22T12:00'], dtype='M8[us]') - >>> b = np.array([3*60*60*1000000], dtype='m8[us]') - >>> a + b - array([1979-03-22 15:00:00], dtype=datetime64[us]) - # NumPy 1.7.0 - >>> a = np.array(['1979-03-22T12:00'], dtype='M8[us]') - >>> b = np.array([3*60*60*1000000], dtype='m8[us]') - >>> a + b - array(['1979-03-22T15:00:00.000000-0500'], dtype='datetime64[us]') +.. _pandas: http://pandas.pydata.org
\ No newline at end of file diff --git a/doc/source/reference/arrays.dtypes.rst b/doc/source/reference/arrays.dtypes.rst index f2072263f..ab743a8ee 100644 --- a/doc/source/reference/arrays.dtypes.rst +++ b/doc/source/reference/arrays.dtypes.rst @@ -14,7 +14,7 @@ following aspects of the data: 1. Type of the data (integer, float, Python object, etc.) 2. Size of the data (how many bytes is in *e.g.* the integer) 3. Byte order of the data (:term:`little-endian` or :term:`big-endian`) -4. If the data type is :term:`structured`, an aggregate of other +4. If the data type is :term:`structured data type`, an aggregate of other data types, (*e.g.*, describing an array item consisting of an integer and a float), @@ -42,7 +42,7 @@ needed in NumPy. pair: dtype; field Structured data types are formed by creating a data type whose -:term:`fields` contain other data types. Each field has a name by +:term:`field` contain other data types. Each field has a name by which it can be :ref:`accessed <arrays.indexing.fields>`. The parent data type should be of sufficient size to contain all its fields; the parent is nearly always based on the :class:`void` type which allows @@ -145,7 +145,7 @@ Array-scalar types This is true for their sub-classes as well. Note that not all data-type information can be supplied with a - type-object: for example, :term:`flexible` data-types have + type-object: for example, `flexible` data-types have a default *itemsize* of 0, and require an explicitly given size to be useful. @@ -511,7 +511,7 @@ Endianness of this data: dtype.byteorder -Information about sub-data-types in a :term:`structured` data type: +Information about sub-data-types in a :term:`structured data type`: .. autosummary:: :toctree: generated/ @@ -538,6 +538,7 @@ Attributes providing additional information: dtype.isnative dtype.descr dtype.alignment + dtype.base Methods diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index 62d36e28c..8ec8d8330 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -3,6 +3,10 @@ Indexing ======== +.. seealso:: + + :ref:`Indexing basics <basics.indexing>` + .. sectionauthor:: adapted from "Guide to NumPy" by Travis E. Oliphant .. currentmodule:: numpy @@ -57,6 +61,17 @@ interpreted as counting from the end of the array (*i.e.*, if All arrays generated by basic slicing are always :term:`views <view>` of the original array. +.. note:: + + NumPy slicing creates a :term:`view` instead of a copy as in the case of + builtin Python sequences such as string, tuple and list. + Care must be taken when extracting + a small portion from a large array which becomes useless after the + extraction, because the small portion extracted contains a reference + to the large original array whose memory will not be released until + all arrays derived from it are garbage-collected. In such cases an + explicit ``copy()`` is recommended. + The standard rules of sequence slicing apply to basic slicing on a per-dimension basis (including using a step index). Some useful concepts to remember include: @@ -111,9 +126,10 @@ concepts to remember include: [5], [6]]]) -- :const:`Ellipsis` expand to the number of ``:`` objects needed to - make a selection tuple of the same length as ``x.ndim``. There may - only be a single ellipsis present. +- :const:`Ellipsis` expands to the number of ``:`` objects needed for the + selection tuple to index all dimensions. In most cases, this means that + length of the expanded selection tuple is ``x.ndim``. There may only be a + single ellipsis present. .. admonition:: Example @@ -513,14 +529,10 @@ only the part of the data in the specified field. Also :ref:`record array <arrays.classes.rec>` scalars can be "indexed" this way. Indexing into a structured array can also be done with a list of field names, -*e.g.* ``x[['field-name1','field-name2']]``. Currently this returns a new -array containing a copy of the values in the fields specified in the list. -As of NumPy 1.7, returning a copy is being deprecated in favor of returning -a view. A copy will continue to be returned for now, but a FutureWarning -will be issued when writing to the copy. If you depend on the current -behavior, then we suggest copying the returned array explicitly, i.e. use -x[['field-name1','field-name2']].copy(). This will work with both past and -future versions of NumPy. +*e.g.* ``x[['field-name1','field-name2']]``. As of NumPy 1.16 this returns a +view containing only those fields. In older versions of numpy it returned a +copy. See the user guide section on :ref:`structured_arrays` for more +information on multifield indexing. If the accessed field is a sub-array, the dimensions of the sub-array are appended to the shape of the result. diff --git a/doc/source/reference/arrays.ndarray.rst b/doc/source/reference/arrays.ndarray.rst index 4c8bbf66d..8f431bc9c 100644 --- a/doc/source/reference/arrays.ndarray.rst +++ b/doc/source/reference/arrays.ndarray.rst @@ -9,7 +9,7 @@ The N-dimensional array (:class:`ndarray`) An :class:`ndarray` is a (usually fixed-size) multidimensional container of items of the same type and size. The number of dimensions and items in an array is defined by its :attr:`shape <ndarray.shape>`, -which is a :class:`tuple` of *N* positive integers that specify the +which is a :class:`tuple` of *N* non-negative integers that specify the sizes of each dimension. The type of items in the array is specified by a separate :ref:`data-type object (dtype) <arrays.dtypes>`, one of which is associated with each ndarray. @@ -82,10 +82,12 @@ Indexing arrays Arrays can be indexed using an extended Python slicing syntax, ``array[selection]``. Similar syntax is also used for accessing -fields in a :ref:`structured array <arrays.dtypes.field>`. +fields in a :term:`structured data type`. .. seealso:: :ref:`Array Indexing <arrays.indexing>`. +.. _memory-layout: + Internal memory layout of an ndarray ==================================== @@ -127,7 +129,7 @@ strided scheme, and correspond to memory that can be *addressed* by the strides: where :math:`d_j` `= self.shape[j]`. Both the C and Fortran orders are :term:`contiguous`, *i.e.,* -:term:`single-segment`, memory layouts, in which every part of the +single-segment, memory layouts, in which every part of the memory block can be accessed by some combination of the indices. While a C-style and Fortran-style contiguous array, which has the corresponding @@ -143,14 +145,15 @@ different. This can happen in two cases: considered C-style and Fortran-style contiguous. Point 1. means that ``self`` and ``self.squeeze()`` always have the same -contiguity and :term:`aligned` flags value. This also means that even a high -dimensional array could be C-style and Fortran-style contiguous at the same -time. +contiguity and ``aligned`` flags value. This also means +that even a high dimensional array could be C-style and Fortran-style +contiguous at the same time. .. index:: aligned An array is considered aligned if the memory offsets for all elements and the -base offset itself is a multiple of `self.itemsize`. +base offset itself is a multiple of `self.itemsize`. Understanding +`memory-alignment` leads to better performance on most hardware. .. note:: @@ -409,6 +412,7 @@ be performed. .. autosummary:: :toctree: generated/ + ndarray.max ndarray.argmax ndarray.min ndarray.argmin @@ -440,7 +444,7 @@ Each of the arithmetic operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``, ``divmod()``, ``**`` or ``pow()``, ``<<``, ``>>``, ``&``, ``^``, ``|``, ``~``) and the comparisons (``==``, ``<``, ``>``, ``<=``, ``>=``, ``!=``) is equivalent to the corresponding -:term:`universal function` (or :term:`ufunc` for short) in NumPy. For +universal function (or :term:`ufunc` for short) in NumPy. For more information, see the section on :ref:`Universal Functions <ufuncs>`. @@ -461,12 +465,12 @@ Truth value of an array (:func:`bool()`): .. autosummary:: :toctree: generated/ - ndarray.__nonzero__ + ndarray.__bool__ .. note:: Truth-value testing of an array invokes - :meth:`ndarray.__nonzero__`, which raises an error if the number of + :meth:`ndarray.__bool__`, which raises an error if the number of elements in the array is larger than 1, because the truth value of such arrays is ambiguous. Use :meth:`.any() <ndarray.any>` and :meth:`.all() <ndarray.all>` instead to be clear about what is meant @@ -492,7 +496,6 @@ Arithmetic: ndarray.__add__ ndarray.__sub__ ndarray.__mul__ - ndarray.__div__ ndarray.__truediv__ ndarray.__floordiv__ ndarray.__mod__ @@ -517,7 +520,7 @@ Arithmetic: ``__r{op}__`` special methods are not directly defined. - The functions called to implement many arithmetic special methods - for arrays can be modified using :func:`set_numeric_ops`. + for arrays can be modified using :class:`__array_ufunc__ <numpy.class.__array_ufunc__>`. Arithmetic, in-place: @@ -527,7 +530,6 @@ Arithmetic, in-place: ndarray.__iadd__ ndarray.__isub__ ndarray.__imul__ - ndarray.__idiv__ ndarray.__itruediv__ ndarray.__ifloordiv__ ndarray.__imod__ @@ -597,19 +599,17 @@ Container customization: (see :ref:`Indexing <arrays.indexing>`) ndarray.__setitem__ ndarray.__contains__ -Conversion; the operations :func:`complex()`, :func:`int()`, -:func:`long()`, :func:`float()`, :func:`oct()`, and -:func:`hex()`. They work only on arrays that have one element in them +Conversion; the operations :func:`int()`, :func:`float()` and +:func:`complex()`. +. They work only on arrays that have one element in them and return the appropriate scalar. .. autosummary:: :toctree: generated/ ndarray.__int__ - ndarray.__long__ ndarray.__float__ - ndarray.__oct__ - ndarray.__hex__ + ndarray.__complex__ String representations: diff --git a/doc/source/reference/arrays.nditer.rst b/doc/source/reference/arrays.nditer.rst index fa8183f75..7dab09a71 100644 --- a/doc/source/reference/arrays.nditer.rst +++ b/doc/source/reference/arrays.nditer.rst @@ -115,13 +115,18 @@ context is exited. array([[ 0, 2, 4], [ 6, 8, 10]]) +If you are writing code that needs to support older versions of numpy, +note that prior to 1.15, :class:`nditer` was not a context manager and +did not have a `close` method. Instead it relied on the destructor to +initiate the writeback of the buffer. + Using an External Loop ---------------------- In all the examples so far, the elements of `a` are provided by the iterator one at a time, because all the looping logic is internal to the -iterator. While this is simple and convenient, it is not very efficient. A -better approach is to move the one-dimensional innermost loop into your +iterator. While this is simple and convenient, it is not very efficient. +A better approach is to move the one-dimensional innermost loop into your code, external to the iterator. This way, NumPy's vectorized operations can be used on larger chunks of the elements being visited. @@ -156,41 +161,29 @@ element in a computation. For example, you may want to visit the elements of an array in memory order, but use a C-order, Fortran-order, or multidimensional index to look up values in a different array. -The Python iterator protocol doesn't have a natural way to query these -additional values from the iterator, so we introduce an alternate syntax -for iterating with an :class:`nditer`. This syntax explicitly works -with the iterator object itself, so its properties are readily accessible -during iteration. With this looping construct, the current value is -accessible by indexing into the iterator, and the index being tracked -is the property `index` or `multi_index` depending on what was requested. - -The Python interactive interpreter unfortunately prints out the -values of expressions inside the while loop during each iteration of the -loop. We have modified the output in the examples using this looping -construct in order to be more readable. +The index is tracked by the iterator object itself, and accessible +through the `index` or `multi_index` properties, depending on what was +requested. The examples below show printouts demonstrating the +progression of the index: .. admonition:: Example >>> a = np.arange(6).reshape(2,3) >>> it = np.nditer(a, flags=['f_index']) - >>> while not it.finished: - ... print("%d <%d>" % (it[0], it.index), end=' ') - ... it.iternext() + >>> for x in it: + ... print("%d <%d>" % (x, it.index), end=' ') ... 0 <0> 1 <2> 2 <4> 3 <1> 4 <3> 5 <5> >>> it = np.nditer(a, flags=['multi_index']) - >>> while not it.finished: - ... print("%d <%s>" % (it[0], it.multi_index), end=' ') - ... it.iternext() + >>> for x in it: + ... print("%d <%s>" % (x, it.multi_index), end=' ') ... 0 <(0, 0)> 1 <(0, 1)> 2 <(0, 2)> 3 <(1, 0)> 4 <(1, 1)> 5 <(1, 2)> - >>> it = np.nditer(a, flags=['multi_index'], op_flags=['writeonly']) - >>> with it: - .... while not it.finished: - ... it[0] = it.multi_index[1] - it.multi_index[0] - ... it.iternext() + >>> with np.nditer(a, flags=['multi_index'], op_flags=['writeonly']) as it: + ... for x in it: + ... x[...] = it.multi_index[1] - it.multi_index[0] ... >>> a array([[ 0, 1, 2], @@ -199,7 +192,7 @@ construct in order to be more readable. Tracking an index or multi-index is incompatible with using an external loop, because it requires a different index value per element. If you try to combine these flags, the :class:`nditer` object will -raise an exception +raise an exception. .. admonition:: Example @@ -209,6 +202,42 @@ raise an exception File "<stdin>", line 1, in <module> ValueError: Iterator flag EXTERNAL_LOOP cannot be used if an index or multi-index is being tracked +Alternative Looping and Element Access +-------------------------------------- + +To make its properties more readily accessible during iteration, +:class:`nditer` has an alternative syntax for iterating, which works +explicitly with the iterator object itself. With this looping construct, +the current value is accessible by indexing into the iterator. Other +properties, such as tracked indices remain as before. The examples below +produce identical results to the ones in the previous section. + +.. admonition:: Example + + >>> a = np.arange(6).reshape(2,3) + >>> it = np.nditer(a, flags=['f_index']) + >>> while not it.finished: + ... print("%d <%d>" % (it[0], it.index), end=' ') + ... it.iternext() + ... + 0 <0> 1 <2> 2 <4> 3 <1> 4 <3> 5 <5> + + >>> it = np.nditer(a, flags=['multi_index']) + >>> while not it.finished: + ... print("%d <%s>" % (it[0], it.multi_index), end=' ') + ... it.iternext() + ... + 0 <(0, 0)> 1 <(0, 1)> 2 <(0, 2)> 3 <(1, 0)> 4 <(1, 1)> 5 <(1, 2)> + + >>> with np.nditer(a, flags=['multi_index'], op_flags=['writeonly']) as it: + ... while not it.finished: + ... it[0] = it.multi_index[1] - it.multi_index[0] + ... it.iternext() + ... + >>> a + array([[ 0, 1, 2], + [-1, 0, 1]]) + Buffering the Array Elements ---------------------------- diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index 9c4f05f75..d27d61e2c 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -177,7 +177,7 @@ Any Python object: .. note:: - The data actually stored in :term:`object arrays <object array>` + The data actually stored in object arrays (*i.e.*, arrays having dtype :class:`object_`) are references to Python objects, not the objects themselves. Hence, object arrays behave more like usual Python :class:`lists <list>`, in the sense @@ -188,8 +188,10 @@ Any Python object: on item access, but instead returns the actual object that the array item refers to. -The following data types are :term:`flexible`. They have no predefined -size: the data they describe can be of different length in different +.. index:: flexible + +The following data types are **flexible**: they have no predefined +size and the data they describe can be of different length in different arrays. (In the character codes ``#`` is an integer denoting how many elements the data type consists of.) diff --git a/doc/source/reference/c-api.array.rst b/doc/source/reference/c-api/array.rst index 9265b1a97..08bf06b00 100644 --- a/doc/source/reference/c-api.array.rst +++ b/doc/source/reference/c-api/array.rst @@ -20,27 +20,44 @@ Array API Array structure and data access ------------------------------- -These macros all access the :c:type:`PyArrayObject` structure members. The input -argument, arr, can be any :c:type:`PyObject *<PyObject>` that is directly interpretable -as a :c:type:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` and its -sub-types). +These macros access the :c:type:`PyArrayObject` structure members and are +defined in ``ndarraytypes.h``. The input argument, *arr*, can be any +:c:type:`PyObject *<PyObject>` that is directly interpretable as a +:c:type:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` +and itssub-types). .. c:function:: int PyArray_NDIM(PyArrayObject *arr) The number of dimensions in the array. -.. c:function:: npy_intp *PyArray_DIMS(PyArrayObject *arr) +.. c:function:: int PyArray_FLAGS(PyArrayObject* arr) - Returns a pointer to the dimensions/shape of the array. The - number of elements matches the number of dimensions - of the array. + Returns an integer representing the :ref:`array-flags<array-flags>`. -.. c:function:: npy_intp *PyArray_SHAPE(PyArrayObject *arr) +.. c:function:: int PyArray_TYPE(PyArrayObject* arr) + + Return the (builtin) typenumber for the elements of this array. + +.. c:function:: int PyArray_SETITEM( \ + PyArrayObject* arr, void* itemptr, PyObject* obj) + + Convert obj and place it in the ndarray, *arr*, at the place + pointed to by itemptr. Return -1 if an error occurs or 0 on + success. + +.. c:function:: void PyArray_ENABLEFLAGS(PyArrayObject* arr, int flags) .. versionadded:: 1.7 - A synonym for PyArray_DIMS, named to be consistent with the - 'shape' usage within Python. + Enables the specified array flags. This function does no validation, + and assumes that you know what you're doing. + +.. c:function:: void PyArray_CLEARFLAGS(PyArrayObject* arr, int flags) + + .. versionadded:: 1.7 + + Clears the specified array flags. This function does no validation, + and assumes that you know what you're doing. .. c:function:: void *PyArray_DATA(PyArrayObject *arr) @@ -53,6 +70,19 @@ sub-types). array then be sure you understand how to access the data in the array to avoid memory and/or alignment problems. +.. c:function:: npy_intp *PyArray_DIMS(PyArrayObject *arr) + + Returns a pointer to the dimensions/shape of the array. The + number of elements matches the number of dimensions + of the array. Can return ``NULL`` for 0-dimensional arrays. + +.. c:function:: npy_intp *PyArray_SHAPE(PyArrayObject *arr) + + .. versionadded:: 1.7 + + A synonym for :c:func:`PyArray_DIMS`, named to be consistent with the + `shape <numpy.ndarray.shape>` usage within Python. + .. c:function:: npy_intp *PyArray_STRIDES(PyArrayObject* arr) Returns a pointer to the strides of the array. The @@ -67,6 +97,27 @@ sub-types). Return the stride in the *n* :math:`^{\textrm{th}}` dimension. +.. c:function:: npy_intp PyArray_ITEMSIZE(PyArrayObject* arr) + + Return the itemsize for the elements of this array. + + Note that, in the old API that was deprecated in version 1.7, this function + had the return type ``int``. + +.. c:function:: npy_intp PyArray_SIZE(PyArrayObject* arr) + + Returns the total size (in number of elements) of the array. + +.. c:function:: npy_intp PyArray_Size(PyArrayObject* obj) + + Returns 0 if *obj* is not a sub-class of ndarray. Otherwise, + returns the total number of elements in the array. Safer version + of :c:func:`PyArray_SIZE` (*obj*). + +.. c:function:: npy_intp PyArray_NBYTES(PyArrayObject* arr) + + Returns the total number of bytes consumed by the array. + .. c:function:: PyObject *PyArray_BASE(PyArrayObject* arr) This returns the base object of the array. In most cases, this @@ -93,60 +144,12 @@ sub-types). A synonym for PyArray_DESCR, named to be consistent with the 'dtype' usage within Python. -.. c:function:: void PyArray_ENABLEFLAGS(PyArrayObject* arr, int flags) - - .. versionadded:: 1.7 - - Enables the specified array flags. This function does no validation, - and assumes that you know what you're doing. - -.. c:function:: void PyArray_CLEARFLAGS(PyArrayObject* arr, int flags) - - .. versionadded:: 1.7 - - Clears the specified array flags. This function does no validation, - and assumes that you know what you're doing. - -.. c:function:: int PyArray_FLAGS(PyArrayObject* arr) - -.. c:function:: npy_intp PyArray_ITEMSIZE(PyArrayObject* arr) - - Return the itemsize for the elements of this array. - - Note that, in the old API that was deprecated in version 1.7, this function - had the return type ``int``. - -.. c:function:: int PyArray_TYPE(PyArrayObject* arr) - - Return the (builtin) typenumber for the elements of this array. - .. c:function:: PyObject *PyArray_GETITEM(PyArrayObject* arr, void* itemptr) - Get a Python object of a builtin type from the ndarray, *arr*, + Get a Python object of a builtin type from the ndarray, *arr*, at the location pointed to by itemptr. Return ``NULL`` on failure. - - `numpy.ndarray.item` is identical to PyArray_GETITEM. - -.. c:function:: int PyArray_SETITEM( \ - PyArrayObject* arr, void* itemptr, PyObject* obj) - - Convert obj and place it in the ndarray, *arr*, at the place - pointed to by itemptr. Return -1 if an error occurs or 0 on - success. - -.. c:function:: npy_intp PyArray_SIZE(PyArrayObject* arr) - - Returns the total size (in number of elements) of the array. - -.. c:function:: npy_intp PyArray_Size(PyArrayObject* obj) - Returns 0 if *obj* is not a sub-class of ndarray. Otherwise, - returns the total number of elements in the array. Safer version - of :c:func:`PyArray_SIZE` (*obj*). - -.. c:function:: npy_intp PyArray_NBYTES(PyArrayObject* arr) - - Returns the total number of bytes consumed by the array. + `numpy.ndarray.item` is identical to PyArray_GETITEM. Data access @@ -199,10 +202,11 @@ From scratch ^^^^^^^^^^^^ .. c:function:: PyObject* PyArray_NewFromDescr( \ - PyTypeObject* subtype, PyArray_Descr* descr, int nd, npy_intp* dims, \ - npy_intp* strides, void* data, int flags, PyObject* obj) + PyTypeObject* subtype, PyArray_Descr* descr, int nd, npy_intp const* dims, \ + npy_intp const* strides, void* data, int flags, PyObject* obj) - This function steals a reference to *descr*. + This function steals a reference to *descr*. The easiest way to get one + is using :c:func:`PyArray_DescrFromType`. This is the main array creation function. Most new arrays are created with this flexible function. @@ -216,11 +220,13 @@ From scratch :c:data:`&PyArray_Type<PyArray_Type>`, then *obj* is the object to pass to the :obj:`~numpy.class.__array_finalize__` method of the subclass. - If *data* is ``NULL``, then new memory will be allocated and *flags* - can be non-zero to indicate a Fortran-style contiguous array. If - *data* is not ``NULL``, then it is assumed to point to the memory + If *data* is ``NULL``, then new unitinialized memory will be allocated and + *flags* can be non-zero to indicate a Fortran-style contiguous array. Use + :c:func:`PyArray_FILLWBYTE` to initialize the memory. + + If *data* is not ``NULL``, then it is assumed to point to the memory to be used for the array and the *flags* argument is used as the - new flags for the array (except the state of :c:data:`NPY_OWNDATA`, + new flags for the array (except the state of :c:data:`NPY_ARRAY_OWNDATA`, :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` and :c:data:`NPY_ARRAY_UPDATEIFCOPY` flags of the new array will be reset). @@ -232,6 +238,12 @@ From scratch provided *dims* and *strides* are copied into newly allocated dimension and strides arrays for the new array object. + :c:func:`PyArray_CheckStrides` can help verify non- ``NULL`` stride + information. + + If ``data`` is provided, it must stay alive for the life of the array. One + way to manage this is through :c:func:`PyArray_SetBaseObject` + .. c:function:: PyObject* PyArray_NewLikeArray( \ PyArrayObject* prototype, NPY_ORDER order, PyArray_Descr* descr, \ int subok) @@ -257,8 +269,9 @@ From scratch base-class array. .. c:function:: PyObject* PyArray_New( \ - PyTypeObject* subtype, int nd, npy_intp* dims, int type_num, \ - npy_intp* strides, void* data, int itemsize, int flags, PyObject* obj) + PyTypeObject* subtype, int nd, npy_intp const* dims, int type_num, \ + npy_intp const* strides, void* data, int itemsize, int flags, \ + PyObject* obj) This is similar to :c:func:`PyArray_NewFromDescr` (...) except you specify the data-type descriptor with *type_num* and *itemsize*, @@ -279,29 +292,40 @@ From scratch are passed in they must be consistent with the dimensions, the itemsize, and the data of the array. -.. c:function:: PyObject* PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) +.. c:function:: PyObject* PyArray_SimpleNew(int nd, npy_intp const* dims, int typenum) Create a new uninitialized array of type, *typenum*, whose size in - each of *nd* dimensions is given by the integer array, *dims*. - This function cannot be used to create a flexible-type array (no - itemsize given). + each of *nd* dimensions is given by the integer array, *dims*.The memory + for the array is uninitialized (unless typenum is :c:data:`NPY_OBJECT` + in which case each element in the array is set to NULL). The + *typenum* argument allows specification of any of the builtin + data-types such as :c:data:`NPY_FLOAT` or :c:data:`NPY_LONG`. The + memory for the array can be set to zero if desired using + :c:func:`PyArray_FILLWBYTE` (return_object, 0).This function cannot be + used to create a flexible-type array (no itemsize given). .. c:function:: PyObject* PyArray_SimpleNewFromData( \ - int nd, npy_intp* dims, int typenum, void* data) + int nd, npy_intp const* dims, int typenum, void* data) Create an array wrapper around *data* pointed to by the given pointer. The array flags will have a default that the data area is well-behaved and C-style contiguous. The shape of the array is given by the *dims* c-array of length *nd*. The data-type of the - array is indicated by *typenum*. + array is indicated by *typenum*. If data comes from another + reference-counted Python object, the reference count on this object + should be increased after the pointer is passed in, and the base member + of the returned ndarray should point to the Python object that owns + the data. This will ensure that the provided memory is not + freed while the returned array is in existence. To free memory as soon + as the ndarray is deallocated, set the OWNDATA flag on the returned ndarray. .. c:function:: PyObject* PyArray_SimpleNewFromDescr( \ - int nd, npy_intp* dims, PyArray_Descr* descr) + int nd, npy_int const* dims, PyArray_Descr* descr) - This function steals a reference to *descr* if it is not NULL. + This function steals a reference to *descr*. - Create a new array with the provided data-type descriptor, *descr* - , of the shape determined by *nd* and *dims*. + Create a new array with the provided data-type descriptor, *descr*, + of the shape determined by *nd* and *dims*. .. c:function:: PyArray_FILLWBYTE(PyObject* obj, int val) @@ -310,7 +334,7 @@ From scratch This macro calls memset, so obj must be contiguous. .. c:function:: PyObject* PyArray_Zeros( \ - int nd, npy_intp* dims, PyArray_Descr* dtype, int fortran) + int nd, npy_intp const* dims, PyArray_Descr* dtype, int fortran) Construct a new *nd* -dimensional array with shape given by *dims* and data type given by *dtype*. If *fortran* is non-zero, then a @@ -319,13 +343,13 @@ From scratch corresponds to :c:type:`NPY_OBJECT` ). .. c:function:: PyObject* PyArray_ZEROS( \ - int nd, npy_intp* dims, int type_num, int fortran) + int nd, npy_intp const* dims, int type_num, int fortran) Macro form of :c:func:`PyArray_Zeros` which takes a type-number instead of a data-type object. .. c:function:: PyObject* PyArray_Empty( \ - int nd, npy_intp* dims, PyArray_Descr* dtype, int fortran) + int nd, npy_intp const* dims, PyArray_Descr* dtype, int fortran) Construct a new *nd* -dimensional array with shape given by *dims* and data type given by *dtype*. If *fortran* is non-zero, then a @@ -335,7 +359,7 @@ From scratch filled with :c:data:`Py_None`. .. c:function:: PyObject* PyArray_EMPTY( \ - int nd, npy_intp* dims, int typenum, int fortran) + int nd, npy_intp const* dims, int typenum, int fortran) Macro form of :c:func:`PyArray_Empty` which takes a type-number, *typenum*, instead of a data-type object. @@ -407,10 +431,6 @@ From other objects the array is constructed that way. Almost always this parameter is ``NULL``. - In versions 1.6 and earlier of NumPy, the following flags - did not have the ``_ARRAY_`` macro namespace in them. That form - of the constant names is deprecated in 1.7. - .. c:var:: NPY_ARRAY_C_CONTIGUOUS Make sure the returned array is C-style contiguous @@ -505,6 +525,11 @@ From other objects :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| :c:data:`NPY_ARRAY_WRITEABLE` \| :c:data:`NPY_ARRAY_ALIGNED` + .. c:var:: NPY_ARRAY_OUT_ARRAY + + :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| :c:data:`NPY_ARRAY_ALIGNED` \| + :c:data:`NPY_ARRAY_WRITEABLE` + .. c:var:: NPY_ARRAY_OUT_FARRAY :c:data:`NPY_ARRAY_F_CONTIGUOUS` \| :c:data:`NPY_ARRAY_WRITEABLE` \| @@ -568,8 +593,9 @@ From other objects return NULL; } if (arr == NULL) { + /* ... validate/change dtype, validate flags, ndim, etc ... - // Could make custom strides here too + Could make custom strides here too */ arr = PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, dims, NULL, fortran ? NPY_ARRAY_F_CONTIGUOUS : 0, @@ -583,10 +609,14 @@ From other objects } } else { + /* ... in this case the other parameters weren't filled, just validate and possibly copy arr itself ... + */ } + /* ... use arr ... + */ .. c:function:: PyObject* PyArray_CheckFromAny( \ PyObject* op, PyArray_Descr* dtype, int min_depth, int max_depth, \ @@ -779,7 +809,7 @@ From other objects PyObject* obj, int typenum, int requirements) Combination of :c:func:`PyArray_FROM_OF` and :c:func:`PyArray_FROM_OT` - allowing both a *typenum* and a *flags* argument to be provided.. + allowing both a *typenum* and a *flags* argument to be provided. .. c:function:: PyObject* PyArray_FROMANY( \ PyObject* obj, int typenum, int min, int max, int requirements) @@ -811,17 +841,17 @@ Dealing with types General check of Python Type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. c:function:: PyArray_Check(op) +.. c:function:: PyArray_Check(PyObject *op) Evaluates true if *op* is a Python object whose type is a sub-type of :c:data:`PyArray_Type`. -.. c:function:: PyArray_CheckExact(op) +.. c:function:: PyArray_CheckExact(PyObject *op) Evaluates true if *op* is a Python object with type :c:data:`PyArray_Type`. -.. c:function:: PyArray_HasArrayInterface(op, out) +.. c:function:: PyArray_HasArrayInterface(PyObject *op, PyObject *out) If ``op`` implements any part of the array interface, then ``out`` will contain a new reference to the newly created ndarray using @@ -967,6 +997,10 @@ argument must be a :c:type:`PyObject *<PyObject>` that can be directly interpret called on flexible dtypes. Types that are attached to an array will always be sized, hence the array form of this macro not existing. + .. versionchanged:: 1.18 + + For structured datatypes with no fields this function now returns False. + .. c:function:: PyTypeNum_ISUSERDEF(num) .. c:function:: PyDataType_ISUSERDEF(descr) @@ -1370,6 +1404,7 @@ Special functions for NPY_OBJECT Returns 0 for success, -1 for failure. +.. _array-flags: Array flags ----------- @@ -1547,7 +1582,7 @@ Flag checking ^^^^^^^^^^^^^ For all of these macros *arr* must be an instance of a (subclass of) -:c:data:`PyArray_Type`, but no checking is done. +:c:data:`PyArray_Type`. .. c:function:: PyArray_CHKFLAGS(arr, flags) @@ -1645,11 +1680,13 @@ Conversion .. c:function:: PyObject* PyArray_GetField( \ PyArrayObject* self, PyArray_Descr* dtype, int offset) - Equivalent to :meth:`ndarray.getfield<numpy.ndarray.getfield>` (*self*, *dtype*, *offset*). Return - a new array of the given *dtype* using the data in the current - array at a specified *offset* in bytes. The *offset* plus the - itemsize of the new array type must be less than *self* - ->descr->elsize or an error is raised. The same shape and strides + Equivalent to :meth:`ndarray.getfield<numpy.ndarray.getfield>` + (*self*, *dtype*, *offset*). This function `steals a reference + <https://docs.python.org/3/c-api/intro.html?reference-count-details>`_ + to `PyArray_Descr` and returns a new array of the given `dtype` using + the data in the current array at a specified `offset` in bytes. The + `offset` plus the itemsize of the new array type must be less than ``self + ->descr->elsize`` or an error is raised. The same shape and strides as the original array are used. Therefore, this function has the effect of returning a field from a structured array. But, it can also be used to select specific bytes or groups of bytes from any array @@ -1899,22 +1936,23 @@ Item selection and manipulation all values are clipped to the region [0, len(*op*) ). -.. c:function:: PyObject* PyArray_Sort(PyArrayObject* self, int axis) +.. c:function:: PyObject* PyArray_Sort(PyArrayObject* self, int axis, NPY_SORTKIND kind) - Equivalent to :meth:`ndarray.sort<numpy.ndarray.sort>` (*self*, *axis*). Return an array with - the items of *self* sorted along *axis*. + Equivalent to :meth:`ndarray.sort<numpy.ndarray.sort>` (*self*, *axis*, *kind*). + Return an array with the items of *self* sorted along *axis*. The array + is sorted using the algorithm denoted by *kind* , which is an integer/enum pointing + to the type of sorting algorithms used. .. c:function:: PyObject* PyArray_ArgSort(PyArrayObject* self, int axis) - Equivalent to :meth:`ndarray.argsort<numpy.ndarray.argsort>` (*self*, *axis*). Return an array of - indices such that selection of these indices along the given - ``axis`` would return a sorted version of *self*. If *self* - ->descr is a data-type with fields defined, then - self->descr->names is used to determine the sort order. A - comparison where the first field is equal will use the second - field and so on. To alter the sort order of a structured array, create - a new data-type with a different order of names and construct a - view of the array with that new data-type. + Equivalent to :meth:`ndarray.argsort<numpy.ndarray.argsort>` (*self*, *axis*). + Return an array of indices such that selection of these indices + along the given ``axis`` would return a sorted version of *self*. If *self* ->descr + is a data-type with fields defined, then self->descr->names is used + to determine the sort order. A comparison where the first field is equal + will use the second field and so on. To alter the sort order of a + structured array, create a new data-type with a different order of names + and construct a view of the array with that new data-type. .. c:function:: PyObject* PyArray_LexSort(PyObject* sort_keys, int axis) @@ -2018,6 +2056,17 @@ Calculation effect that is obtained by passing in *axis* = :const:`None` in Python (treating the array as a 1-d array). + +.. note:: + + The out argument specifies where to place the result. If out is + NULL, then the output array is created, otherwise the output is + placed in out which must be the correct size and type. A new + reference to the output array is always returned even when out + is not NULL. The caller of the routine has the responsibility + to ``Py_DECREF`` out if not NULL or a memory-leak will occur. + + .. c:function:: PyObject* PyArray_ArgMax( \ PyArrayObject* self, int axis, PyArrayObject* out) @@ -2030,18 +2079,6 @@ Calculation Equivalent to :meth:`ndarray.argmin<numpy.ndarray.argmin>` (*self*, *axis*). Return the index of the smallest element of *self* along *axis*. - - - -.. note:: - - The out argument specifies where to place the result. If out is - NULL, then the output array is created, otherwise the output is - placed in out which must be the correct size and type. A new - reference to the output array is always returned even when out - is not NULL. The caller of the routine has the responsibility - to ``DECREF`` out if not NULL or a memory-leak will occur. - .. c:function:: PyObject* PyArray_Max( \ PyArrayObject* self, int axis, PyArrayObject* out) @@ -2328,8 +2365,8 @@ Other functions ^^^^^^^^^^^^^^^ .. c:function:: Bool PyArray_CheckStrides( \ - int elsize, int nd, npy_intp numbytes, npy_intp* dims, \ - npy_intp* newstrides) + int elsize, int nd, npy_intp numbytes, npy_intp const* dims, \ + npy_intp const* newstrides) Determine if *newstrides* is a strides array consistent with the memory of an *nd* -dimensional array with shape ``dims`` and @@ -2341,14 +2378,14 @@ Other functions *elsize* refer to a single-segment array. Return :c:data:`NPY_TRUE` if *newstrides* is acceptable, otherwise return :c:data:`NPY_FALSE`. -.. c:function:: npy_intp PyArray_MultiplyList(npy_intp* seq, int n) +.. c:function:: npy_intp PyArray_MultiplyList(npy_intp const* seq, int n) -.. c:function:: int PyArray_MultiplyIntList(int* seq, int n) +.. c:function:: int PyArray_MultiplyIntList(int const* seq, int n) Both of these routines multiply an *n* -length array, *seq*, of integers and return the result. No overflow checking is performed. -.. c:function:: int PyArray_CompareLists(npy_intp* l1, npy_intp* l2, int n) +.. c:function:: int PyArray_CompareLists(npy_intp const* l1, npy_intp const* l2, int n) Given two *n* -length arrays of integers, *l1*, and *l2*, return 1 if the lists are identical; otherwise, return 0. @@ -2621,22 +2658,24 @@ cost of a slight overhead. The mode should be one of: - * NPY_NEIGHBORHOOD_ITER_ZERO_PADDING: zero padding. Outside bounds values - will be 0. - * NPY_NEIGHBORHOOD_ITER_ONE_PADDING: one padding, Outside bounds values - will be 1. - * NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING: constant padding. Outside bounds - values will be the same as the first item in fill_value. - * NPY_NEIGHBORHOOD_ITER_MIRROR_PADDING: mirror padding. Outside bounds - values will be as if the array items were mirrored. For example, for the - array [1, 2, 3, 4], x[-2] will be 2, x[-2] will be 1, x[4] will be 4, - x[5] will be 1, etc... - * NPY_NEIGHBORHOOD_ITER_CIRCULAR_PADDING: circular padding. Outside bounds - values will be as if the array was repeated. For example, for the - array [1, 2, 3, 4], x[-2] will be 3, x[-2] will be 4, x[4] will be 1, - x[5] will be 2, etc... - - If the mode is constant filling (NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING), + .. c:macro:: NPY_NEIGHBORHOOD_ITER_ZERO_PADDING + Zero padding. Outside bounds values will be 0. + .. c:macro:: NPY_NEIGHBORHOOD_ITER_ONE_PADDING + One padding, Outside bounds values will be 1. + .. c:macro:: NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING + Constant padding. Outside bounds values will be the + same as the first item in fill_value. + .. c:macro:: NPY_NEIGHBORHOOD_ITER_MIRROR_PADDING + Mirror padding. Outside bounds values will be as if the + array items were mirrored. For example, for the array [1, 2, 3, 4], + x[-2] will be 2, x[-2] will be 1, x[4] will be 4, x[5] will be 1, + etc... + .. c:macro:: NPY_NEIGHBORHOOD_ITER_CIRCULAR_PADDING + Circular padding. Outside bounds values will be as if the array + was repeated. For example, for the array [1, 2, 3, 4], x[-2] will + be 3, x[-2] will be 4, x[4] will be 1, x[5] will be 2, etc... + + If the mode is constant filling (`NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING`), fill_value should point to an array object which holds the filling value (the first item will be the filling value if the array contains more than one item). For other cases, fill_value may be NULL. @@ -2654,22 +2693,22 @@ cost of a slight overhead. .. code-block:: c - PyArrayIterObject \*iter; - PyArrayNeighborhoodIterObject \*neigh_iter; + PyArrayIterObject *iter; + PyArrayNeighborhoodIterObject *neigh_iter; iter = PyArray_IterNew(x); - //For a 3x3 kernel + /*For a 3x3 kernel */ bounds = {-1, 1, -1, 1}; neigh_iter = (PyArrayNeighborhoodIterObject*)PyArrayNeighborhoodIter_New( iter, bounds, NPY_NEIGHBORHOOD_ITER_ZERO_PADDING, NULL); for(i = 0; i < iter->size; ++i) { for (j = 0; j < neigh_iter->size; ++j) { - // Walk around the item currently pointed by iter->dataptr + /* Walk around the item currently pointed by iter->dataptr */ PyArrayNeighborhoodIter_Next(neigh_iter); } - // Move to the next point of iter + /* Move to the next point of iter */ PyArrayIter_Next(iter); PyArrayNeighborhoodIter_Reset(neigh_iter); } @@ -2758,10 +2797,7 @@ Array Scalars *arr* is not ``NULL`` and the first element is negative then :c:data:`NPY_INTNEG_SCALAR` is returned, otherwise :c:data:`NPY_INTPOS_SCALAR` is returned. The possible return values - are :c:data:`NPY_{kind}_SCALAR` where ``{kind}`` can be **INTPOS**, - **INTNEG**, **FLOAT**, **COMPLEX**, **BOOL**, or **OBJECT**. - :c:data:`NPY_NOSCALAR` is also an enumerated value - :c:type:`NPY_SCALARKIND` variables can take on. + are the enumerated values in :c:type:`NPY_SCALARKIND`. .. c:function:: int PyArray_CanCoerceScalar( \ char thistype, char neededtype, NPY_SCALARKIND scalar) @@ -2849,7 +2885,10 @@ Data-type descriptors Returns a data-type object corresponding to *typenum*. The *typenum* can be one of the enumerated types, a character code for - one of the enumerated types, or a user-defined type. + one of the enumerated types, or a user-defined type. If you want to use a + flexible size array, then you need to ``flexible typenum`` and set the + results ``elsize`` parameter to the desired size. The typenum is one of the + :c:data:`NPY_TYPES`. .. c:function:: int PyArray_DescrConverter(PyObject* obj, PyArray_Descr** dtype) @@ -2980,8 +3019,11 @@ to. .. c:function:: int PyArray_SortkindConverter(PyObject* obj, NPY_SORTKIND* sort) Convert Python strings into one of :c:data:`NPY_QUICKSORT` (starts - with 'q' or 'Q') , :c:data:`NPY_HEAPSORT` (starts with 'h' or 'H'), - or :c:data:`NPY_MERGESORT` (starts with 'm' or 'M'). + with 'q' or 'Q'), :c:data:`NPY_HEAPSORT` (starts with 'h' or 'H'), + :c:data:`NPY_MERGESORT` (starts with 'm' or 'M') or :c:data:`NPY_STABLESORT` + (starts with 't' or 'T'). :c:data:`NPY_MERGESORT` and :c:data:`NPY_STABLESORT` + are aliased to each other for backwards compatibility and may refer to one + of several stable sorting algorithms depending on the data type. .. c:function:: int PyArray_SearchsideConverter( \ PyObject* obj, NPY_SEARCHSIDE* side) @@ -3209,12 +3251,16 @@ Internal Flexibility setting a Python Error) if one of the objects being assigned is not callable. + .. deprecated:: 1.16 + .. c:function:: PyObject* PyArray_GetNumericOps(void) Return a Python dictionary containing the callable Python objects stored in the internal arithmetic operation table. The keys of this dictionary are given in the explanation for :c:func:`PyArray_SetNumericOps`. + .. deprecated:: 1.16 + .. c:function:: void PyArray_SetStringFunction(PyObject* op, int repr) This function allows you to alter the tp_str and tp_repr methods @@ -3240,19 +3286,19 @@ Memory management Macros to allocate, free, and reallocate memory. These macros are used internally to create arrays. -.. c:function:: npy_intp* PyDimMem_NEW(nd) +.. c:function:: npy_intp* PyDimMem_NEW(int nd) -.. c:function:: PyDimMem_FREE(npy_intp* ptr) +.. c:function:: PyDimMem_FREE(char* ptr) -.. c:function:: npy_intp* PyDimMem_RENEW(npy_intp* ptr, npy_intp newnd) +.. c:function:: npy_intp* PyDimMem_RENEW(void* ptr, size_t newnd) Macros to allocate, free, and reallocate dimension and strides memory. -.. c:function:: PyArray_malloc(nbytes) +.. c:function:: void* PyArray_malloc(size_t nbytes) -.. c:function:: PyArray_free(ptr) +.. c:function:: PyArray_free(void* ptr) -.. c:function:: PyArray_realloc(ptr, nbytes) +.. c:function:: void* PyArray_realloc(npy_intp* ptr, size_t nbytes) These macros use different memory allocators, depending on the constant :c:data:`NPY_USE_PYMEM`. The system malloc is used when @@ -3332,7 +3378,7 @@ Group 1 Useful to release the GIL only if *dtype* does not contain arbitrary Python objects which may need the Python interpreter - during execution of the loop. Equivalent to + during execution of the loop. .. c:function:: NPY_END_THREADS_DESCR(PyArray_Descr *dtype) @@ -3426,6 +3472,10 @@ Other constants The maximum number of dimensions allowed in arrays. +.. c:var:: NPY_MAXARGS + + The maximum number of array arguments that can be used in functions. + .. c:var:: NPY_VERSION The current version of the ndarray object (check to see if this @@ -3454,31 +3504,31 @@ Other constants Miscellaneous Macros ^^^^^^^^^^^^^^^^^^^^ -.. c:function:: PyArray_SAMESHAPE(a1, a2) +.. c:function:: PyArray_SAMESHAPE(PyArrayObject *a1, PyArrayObject *a2) Evaluates as True if arrays *a1* and *a2* have the same shape. -.. c:function:: PyArray_MAX(a,b) +.. c:macro:: PyArray_MAX(a,b) Returns the maximum of *a* and *b*. If (*a*) or (*b*) are expressions they are evaluated twice. -.. c:function:: PyArray_MIN(a,b) +.. c:macro:: PyArray_MIN(a,b) Returns the minimum of *a* and *b*. If (*a*) or (*b*) are expressions they are evaluated twice. -.. c:function:: PyArray_CLT(a,b) +.. c:macro:: PyArray_CLT(a,b) -.. c:function:: PyArray_CGT(a,b) +.. c:macro:: PyArray_CGT(a,b) -.. c:function:: PyArray_CLE(a,b) +.. c:macro:: PyArray_CLE(a,b) -.. c:function:: PyArray_CGE(a,b) +.. c:macro:: PyArray_CGE(a,b) -.. c:function:: PyArray_CEQ(a,b) +.. c:macro:: PyArray_CEQ(a,b) -.. c:function:: PyArray_CNE(a,b) +.. c:macro:: PyArray_CNE(a,b) Implements the complex comparisons between two complex numbers (structures with a real and imag member) using NumPy's definition @@ -3518,24 +3568,46 @@ Enumerated Types .. c:type:: NPY_SORTKIND - A special variable-type which can take on the values :c:data:`NPY_{KIND}` - where ``{KIND}`` is + A special variable-type which can take on different values to indicate + the sorting algorithm being used. + + .. c:var:: NPY_QUICKSORT + + .. c:var:: NPY_HEAPSORT - **QUICKSORT**, **HEAPSORT**, **MERGESORT** + .. c:var:: NPY_MERGESORT + + .. c:var:: NPY_STABLESORT + + Used as an alias of :c:data:`NPY_MERGESORT` and vica versa. .. c:var:: NPY_NSORTS - Defined to be the number of sorts. + Defined to be the number of sorts. It is fixed at three by the need for + backwards compatibility, and consequently :c:data:`NPY_MERGESORT` and + :c:data:`NPY_STABLESORT` are aliased to each other and may refer to one + of several stable sorting algorithms depending on the data type. + .. c:type:: NPY_SCALARKIND A special variable type indicating the number of "kinds" of scalars distinguished in determining scalar-coercion rules. This - variable can take on the values :c:data:`NPY_{KIND}` where ``{KIND}`` can be + variable can take on the values: + + .. c:var:: NPY_NOSCALAR + + .. c:var:: NPY_BOOL_SCALAR + + .. c:var:: NPY_INTPOS_SCALAR + + .. c:var:: NPY_INTNEG_SCALAR + + .. c:var:: NPY_FLOAT_SCALAR + + .. c:var:: NPY_COMPLEX_SCALAR - **NOSCALAR**, **BOOL_SCALAR**, **INTPOS_SCALAR**, - **INTNEG_SCALAR**, **FLOAT_SCALAR**, **COMPLEX_SCALAR**, - **OBJECT_SCALAR** + .. c:var:: NPY_OBJECT_SCALAR .. c:var:: NPY_NSCALARKINDS diff --git a/doc/source/reference/c-api.config.rst b/doc/source/reference/c-api/config.rst index 60bf61a32..05e6fe44d 100644 --- a/doc/source/reference/c-api.config.rst +++ b/doc/source/reference/c-api/config.rst @@ -101,3 +101,22 @@ Platform information Returns the endianness of the current platform. One of :c:data:`NPY_CPU_BIG`, :c:data:`NPY_CPU_LITTLE`, or :c:data:`NPY_CPU_UNKNOWN_ENDIAN`. + + +Compiler directives +------------------- + +.. c:var:: NPY_LIKELY +.. c:var:: NPY_UNLIKELY +.. c:var:: NPY_UNUSED + + +Interrupt Handling +------------------ + +.. c:var:: NPY_INTERRUPT_H +.. c:var:: NPY_SIGSETJMP +.. c:var:: NPY_SIGLONGJMP +.. c:var:: NPY_SIGJMP_BUF +.. c:var:: NPY_SIGINT_ON +.. c:var:: NPY_SIGINT_OFF diff --git a/doc/source/reference/c-api.coremath.rst b/doc/source/reference/c-api/coremath.rst index 691f73287..7e00322f9 100644 --- a/doc/source/reference/c-api.coremath.rst +++ b/doc/source/reference/c-api/coremath.rst @@ -80,8 +80,9 @@ Floating point classification Useful math constants ~~~~~~~~~~~~~~~~~~~~~ -The following math constants are available in npy_math.h. Single and extended -precision are also available by adding the F and L suffixes respectively. +The following math constants are available in ``npy_math.h``. Single +and extended precision are also available by adding the ``f`` and +``l`` suffixes respectively. .. c:var:: NPY_E @@ -184,7 +185,7 @@ Those can be useful for precise floating point comparison. * NPY_FPE_INVALID Note that :c:func:`npy_get_floatstatus_barrier` is preferable as it prevents - agressive compiler optimizations reordering the call relative to + aggressive compiler optimizations reordering the call relative to the code setting the status, which could lead to incorrect results. .. versionadded:: 1.9.0 @@ -192,7 +193,7 @@ Those can be useful for precise floating point comparison. .. c:function:: int npy_get_floatstatus_barrier(char*) Get floating point status. A pointer to a local variable is passed in to - prevent aggresive compiler optimizations from reodering this function call + prevent aggressive compiler optimizations from reodering this function call relative to the code setting the status, which could lead to incorrect results. @@ -210,7 +211,7 @@ Those can be useful for precise floating point comparison. Clears the floating point status. Returns the previous status mask. Note that :c:func:`npy_clear_floatstatus_barrier` is preferable as it - prevents agressive compiler optimizations reordering the call relative to + prevents aggressive compiler optimizations reordering the call relative to the code setting the status, which could lead to incorrect results. .. versionadded:: 1.9.0 @@ -218,7 +219,7 @@ Those can be useful for precise floating point comparison. .. c:function:: int npy_clear_floatstatus_barrier(char*) Clears the floating point status. A pointer to a local variable is passed in to - prevent aggresive compiler optimizations from reodering this function call. + prevent aggressive compiler optimizations from reodering this function call. Returns the previous status mask. .. versionadded:: 1.15.0 @@ -258,7 +259,7 @@ and co. Half-precision functions ~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 2.0.0 +.. versionadded:: 1.6.0 The header file <numpy/halffloat.h> provides functions to work with IEEE 754-2008 16-bit floating point values. While this format is diff --git a/doc/source/reference/c-api.deprecations.rst b/doc/source/reference/c-api/deprecations.rst index a382017a2..a382017a2 100644 --- a/doc/source/reference/c-api.deprecations.rst +++ b/doc/source/reference/c-api/deprecations.rst diff --git a/doc/source/reference/c-api.dtype.rst b/doc/source/reference/c-api/dtype.rst index 8af3a9080..72e908861 100644 --- a/doc/source/reference/c-api.dtype.rst +++ b/doc/source/reference/c-api/dtype.rst @@ -25,6 +25,8 @@ select the precision desired. Enumerated Types ---------------- +.. c:var:: NPY_TYPES + There is a list of enumerated types defined providing the basic 24 data types plus some useful generic names. Whenever the code requires a type number, one of these enumerated types is requested. The types @@ -306,13 +308,45 @@ to the front of the integer name. (unsigned) char -.. c:type:: npy_(u)short +.. c:type:: npy_short + + short + +.. c:type:: npy_ushort + + unsigned short + +.. c:type:: npy_uint + + unsigned int + +.. c:type:: npy_int + + int + +.. c:type:: npy_int16 + + 16-bit integer + +.. c:type:: npy_uint16 - (unsigned) short + 16-bit unsigned integer -.. c:type:: npy_(u)int +.. c:type:: npy_int32 - (unsigned) int + 32-bit integer + +.. c:type:: npy_uint32 + + 32-bit unsigned integer + +.. c:type:: npy_int64 + + 64-bit integer + +.. c:type:: npy_uint64 + + 64-bit unsigned integer .. c:type:: npy_(u)long @@ -322,22 +356,31 @@ to the front of the integer name. (unsigned long long int) -.. c:type:: npy_(u)intp +.. c:type:: npy_intp - (unsigned) Py_intptr_t (an integer that is the size of a pointer on + Py_intptr_t (an integer that is the size of a pointer on + the platform). + +.. c:type:: npy_uintp + + unsigned Py_intptr_t (an integer that is the size of a pointer on the platform). (Complex) Floating point ^^^^^^^^^^^^^^^^^^^^^^^^ +.. c:type:: npy_half + + 16-bit float + .. c:type:: npy_(c)float - float + 32-bit float .. c:type:: npy_(c)double - double + 64-bit double .. c:type:: npy_(c)longdouble diff --git a/doc/source/reference/c-api.generalized-ufuncs.rst b/doc/source/reference/c-api/generalized-ufuncs.rst index 2c631531f..b59f077ad 100644 --- a/doc/source/reference/c-api.generalized-ufuncs.rst +++ b/doc/source/reference/c-api/generalized-ufuncs.rst @@ -127,38 +127,56 @@ The formal syntax of signatures is as follows:: <Output arguments> ::= <Argument list> <Argument list> ::= nil | <Argument> | <Argument> "," <Argument list> <Argument> ::= "(" <Core dimension list> ")" - <Core dimension list> ::= nil | <Core dimension name> | - <Core dimension name> "," <Core dimension list> - <Core dimension name> ::= valid Python variable name - + <Core dimension list> ::= nil | <Core dimension> | + <Core dimension> "," <Core dimension list> + <Core dimension> ::= <Dimension name> <Dimension modifier> + <Dimension name> ::= valid Python variable name | valid integer + <Dimension modifier> ::= nil | "?" Notes: #. All quotes are for clarity. -#. Core dimensions that share the same name must have the exact same size. +#. Unmodified core dimensions that share the same name must have the same size. Each dimension name typically corresponds to one level of looping in the elementary function's implementation. #. White spaces are ignored. +#. An integer as a dimension name freezes that dimension to the value. +#. If the name is suffixed with the "?" modifier, the dimension is a core + dimension only if it exists on all inputs and outputs that share it; + otherwise it is ignored (and replaced by a dimension of size 1 for the + elementary function). Here are some examples of signatures: -+-------------+------------------------+-----------------------------------+ -| add | ``(),()->()`` | | -+-------------+------------------------+-----------------------------------+ -| sum1d | ``(i)->()`` | | -+-------------+------------------------+-----------------------------------+ -| inner1d | ``(i),(i)->()`` | | -+-------------+------------------------+-----------------------------------+ -| matmat | ``(m,n),(n,p)->(m,p)`` | matrix multiplication | -+-------------+------------------------+-----------------------------------+ -| vecmat | ``(n),(n,p)->(p)`` | vector-matrix multiplication | -+-------------+------------------------+-----------------------------------+ -| matvec | ``(m,n),(n)->(m)`` | matrix-vector multiplication | -+-------------+------------------------+-----------------------------------+ -| outer_inner | ``(i,t),(j,t)->(i,j)`` | inner over the last dimension, | -| | | outer over the second to last, | -| | | and loop/broadcast over the rest. | -+-------------+------------------------+-----------------------------------+ ++-------------+----------------------------+-----------------------------------+ +| name | signature | common usage | ++=============+============================+===================================+ +| add | ``(),()->()`` | binary ufunc | ++-------------+----------------------------+-----------------------------------+ +| sum1d | ``(i)->()`` | reduction | ++-------------+----------------------------+-----------------------------------+ +| inner1d | ``(i),(i)->()`` | vector-vector multiplication | ++-------------+----------------------------+-----------------------------------+ +| matmat | ``(m,n),(n,p)->(m,p)`` | matrix multiplication | ++-------------+----------------------------+-----------------------------------+ +| vecmat | ``(n),(n,p)->(p)`` | vector-matrix multiplication | ++-------------+----------------------------+-----------------------------------+ +| matvec | ``(m,n),(n)->(m)`` | matrix-vector multiplication | ++-------------+----------------------------+-----------------------------------+ +| matmul | ``(m?,n),(n,p?)->(m?,p?)`` | combination of the four above | ++-------------+----------------------------+-----------------------------------+ +| outer_inner | ``(i,t),(j,t)->(i,j)`` | inner over the last dimension, | +| | | outer over the second to last, | +| | | and loop/broadcast over the rest. | ++-------------+----------------------------+-----------------------------------+ +| cross1d | ``(3),(3)->(3)`` | cross product where the last | +| | | dimension is frozen and must be 3 | ++-------------+----------------------------+-----------------------------------+ + +.. _frozen: + +The last is an instance of freezing a core dimension and can be used to +improve ufunc performance C-API for implementing Elementary Functions ------------------------------------------- diff --git a/doc/source/reference/c-api.rst b/doc/source/reference/c-api/index.rst index b8cbe97b2..56fe8e473 100644 --- a/doc/source/reference/c-api.rst +++ b/doc/source/reference/c-api/index.rst @@ -40,12 +40,12 @@ code. .. toctree:: :maxdepth: 2 - c-api.types-and-structures - c-api.config - c-api.dtype - c-api.array - c-api.iterator - c-api.ufunc - c-api.generalized-ufuncs - c-api.coremath - c-api.deprecations + types-and-structures + config + dtype + array + iterator + ufunc + generalized-ufuncs + coremath + deprecations diff --git a/doc/source/reference/c-api.iterator.rst b/doc/source/reference/c-api/iterator.rst index 940452d3c..b77d029cc 100644 --- a/doc/source/reference/c-api.iterator.rst +++ b/doc/source/reference/c-api/iterator.rst @@ -593,25 +593,23 @@ Construction and Destruction code doing iteration can write to this operand to control which elements will be untouched and which ones will be modified. This is useful when the mask should be a combination - of input masks, for example. Mask values can be created - with the :c:func:`NpyMask_Create` function. + of input masks. .. c:var:: NPY_ITER_WRITEMASKED .. versionadded:: 1.7 - Indicates that only elements which the operand with - the ARRAYMASK flag indicates are intended to be modified - by the iteration. In general, the iterator does not enforce - this, it is up to the code doing the iteration to follow - that promise. Code can use the :c:func:`NpyMask_IsExposed` - inline function to test whether the mask at a particular - element allows writing. + This array is the mask for all `writemasked <numpy.nditer>` + operands. Code uses the ``writemasked`` flag which indicates + that only elements where the chosen ARRAYMASK operand is True + will be written to. In general, the iterator does not enforce + this, it is up to the code doing the iteration to follow that + promise. - When this flag is used, and this operand is buffered, this - changes how data is copied from the buffer into the array. + When ``writemasked`` flag is used, and this operand is buffered, + this changes how data is copied from the buffer into the array. A masked copying routine is used, which only copies the - elements in the buffer for which :c:func:`NpyMask_IsExposed` + elements in the buffer for which ``writemasked`` returns true from the corresponding element in the ARRAYMASK operand. @@ -630,7 +628,7 @@ Construction and Destruction .. c:function:: NpyIter* NpyIter_AdvancedNew( \ npy_intp nop, PyArrayObject** op, npy_uint32 flags, NPY_ORDER order, \ NPY_CASTING casting, npy_uint32* op_flags, PyArray_Descr** op_dtypes, \ - int oa_ndim, int** op_axes, npy_intp* itershape, npy_intp buffersize) + int oa_ndim, int** op_axes, npy_intp const* itershape, npy_intp buffersize) Extends :c:func:`NpyIter_MultiNew` with several advanced options providing more control over broadcasting and buffering. @@ -867,7 +865,7 @@ Construction and Destruction } while (iternext2(iter2)); } while (iternext1(iter1)); -.. c:function:: int NpyIter_GotoMultiIndex(NpyIter* iter, npy_intp* multi_index) +.. c:function:: int NpyIter_GotoMultiIndex(NpyIter* iter, npy_intp const* multi_index) Adjusts the iterator to point to the ``ndim`` indices pointed to by ``multi_index``. Returns an error if a multi-index @@ -974,19 +972,6 @@ Construction and Destruction Returns the number of operands in the iterator. - When :c:data:`NPY_ITER_USE_MASKNA` is used on an operand, a new - operand is added to the end of the operand list in the iterator - to track that operand's NA mask. Thus, this equals the number - of construction operands plus the number of operands for - which the flag :c:data:`NPY_ITER_USE_MASKNA` was specified. - -.. c:function:: int NpyIter_GetFirstMaskNAOp(NpyIter* iter) - - .. versionadded:: 1.7 - - Returns the index of the first NA mask operand in the array. This - value is equal to the number of operands passed into the constructor. - .. c:function:: npy_intp* NpyIter_GetAxisStrideArray(NpyIter* iter, int axis) Gets the array of strides for the specified axis. Requires that @@ -1023,16 +1008,6 @@ Construction and Destruction that are being iterated. The result points into ``iter``, so the caller does not gain any references to the PyObjects. -.. c:function:: npy_int8* NpyIter_GetMaskNAIndexArray(NpyIter* iter) - - .. versionadded:: 1.7 - - This gives back a pointer to the ``nop`` indices which map - construction operands with :c:data:`NPY_ITER_USE_MASKNA` flagged - to their corresponding NA mask operands and vice versa. For - operands which were not flagged with :c:data:`NPY_ITER_USE_MASKNA`, - this array contains negative values. - .. c:function:: PyObject* NpyIter_GetIterView(NpyIter* iter, npy_intp i) This gives back a reference to a new ndarray view, which is a view diff --git a/doc/source/reference/c-api.types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 095693c5b..336dff211 100644 --- a/doc/source/reference/c-api.types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -1,3 +1,4 @@ + ***************************** Python Types and C-Structures ***************************** @@ -57,8 +58,8 @@ types are place holders that allow the array scalars to fit into a hierarchy of actual Python types. -PyArray_Type ------------- +PyArray_Type and PyArrayObject +------------------------------ .. c:var:: PyArray_Type @@ -74,8 +75,9 @@ PyArray_Type subclasses) will have this structure. For future compatibility, these structure members should normally be accessed using the provided macros. If you need a shorter name, then you can make use - of :c:type:`NPY_AO` which is defined to be equivalent to - :c:type:`PyArrayObject`. + of :c:type:`NPY_AO` (deprecated) which is defined to be equivalent to + :c:type:`PyArrayObject`. Direct access to the struct fields are + deprecated. Use the `PyArray_*(arr)` form instead. .. code-block:: c @@ -91,7 +93,7 @@ PyArray_Type PyObject *weakreflist; } PyArrayObject; -.. c:macro: PyArrayObject.PyObject_HEAD +.. c:macro:: PyArrayObject.PyObject_HEAD This is needed by all Python objects. It consists of (at least) a reference count member ( ``ob_refcnt`` ) and a pointer to the @@ -103,7 +105,8 @@ PyArray_Type .. c:member:: char *PyArrayObject.data - A pointer to the first element of the array. This pointer can + Accessible via :c:data:`PyArray_DATA`, this data member is a + pointer to the first element of the array. This pointer can (and normally should) be recast to the data type of the array. .. c:member:: int PyArrayObject.nd @@ -111,33 +114,38 @@ PyArray_Type An integer providing the number of dimensions for this array. When nd is 0, the array is sometimes called a rank-0 array. Such arrays have undefined dimensions and strides and - cannot be accessed. :c:data:`NPY_MAXDIMS` is the largest number of - dimensions for any array. + cannot be accessed. Macro :c:data:`PyArray_NDIM` defined in + ``ndarraytypes.h`` points to this data member. :c:data:`NPY_MAXDIMS` + is the largest number of dimensions for any array. .. c:member:: npy_intp PyArrayObject.dimensions An array of integers providing the shape in each dimension as long as nd :math:`\geq` 1. The integer is always large enough to hold a pointer on the platform, so the dimension size is - only limited by memory. + only limited by memory. :c:data:`PyArray_DIMS` is the macro + associated with this data member. .. c:member:: npy_intp *PyArrayObject.strides An array of integers providing for each dimension the number of bytes that must be skipped to get to the next element in that - dimension. + dimension. Associated with macro :c:data:`PyArray_STRIDES`. .. c:member:: PyObject *PyArrayObject.base - This member is used to hold a pointer to another Python object that - is related to this array. There are two use cases: 1) If this array - does not own its own memory, then base points to the Python object - that owns it (perhaps another array object), 2) If this array has - the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or - :c:data:NPY_ARRAY_WRITEBACKIFCOPY`: flag set, then this array is - a working copy of a "misbehaved" array. When - ``PyArray_ResolveWritebackIfCopy`` is called, the array pointed to by base - will be updated with the contents of this array. + Pointed to by :c:data:`PyArray_BASE`, this member is used to hold a + pointer to another Python object that is related to this array. + There are two use cases: + + - If this array does not own its own memory, then base points to the + Python object that owns it (perhaps another array object) + - If this array has the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag set, then this array is a working + copy of a "misbehaved" array. + + When ``PyArray_ResolveWritebackIfCopy`` is called, the array pointed to + by base will be updated with the contents of this array. .. c:member:: PyArray_Descr *PyArrayObject.descr @@ -147,11 +155,13 @@ PyArray_Type descriptor structure for each data type supported. This descriptor structure contains useful information about the type as well as a pointer to a table of function pointers to - implement specific functionality. + implement specific functionality. As the name suggests, it is + associated with the macro :c:data:`PyArray_DESCR`. .. c:member:: int PyArrayObject.flags - Flags indicating how the memory pointed to by data is to be + Pointed to by the macro :c:data:`PyArray_FLAGS`, this data member represents + the flags indicating how the memory pointed to by data is to be interpreted. Possible flags are :c:data:`NPY_ARRAY_C_CONTIGUOUS`, :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_OWNDATA`, :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, @@ -163,8 +173,8 @@ PyArray_Type weakref module). -PyArrayDescr_Type ------------------ +PyArrayDescr_Type and PyArray_Descr +----------------------------------- .. c:var:: PyArrayDescr_Type @@ -182,8 +192,18 @@ PyArrayDescr_Type .. c:type:: PyArray_Descr - The format of the :c:type:`PyArray_Descr` structure that lies at the - heart of the :c:data:`PyArrayDescr_Type` is + The :c:type:`PyArray_Descr` structure lies at the heart of the + :c:data:`PyArrayDescr_Type`. While it is described here for + completeness, it should be considered internal to NumPy and manipulated via + ``PyArrayDescr_*`` or ``PyDataType*`` functions and macros. The size of this + structure is subject to change across versions of NumPy. To ensure + compatibility: + + - Never declare a non-pointer instance of the struct + - Never perform pointer arithmatic + - Never use ``sizof(PyArray_Descr)`` + + It has the following structure: .. code-block:: c @@ -193,14 +213,17 @@ PyArrayDescr_Type char kind; char type; char byteorder; - char unused; - int flags; + char flags; int type_num; int elsize; int alignment; PyArray_ArrayDescr *subarray; PyObject *fields; + PyObject *names; PyArray_ArrFuncs *f; + PyObject *metadata; + NpyAuxData *c_metadata; + npy_hash_t hash; } PyArray_Descr; .. c:member:: PyTypeObject *PyArray_Descr.typeobj @@ -232,7 +255,7 @@ PyArrayDescr_Type endian), '=' (native), '\|' (irrelevant, ignore). All builtin data- types have byteorder '='. -.. c:member:: int PyArray_Descr.flags +.. c:member:: char PyArray_Descr.flags A data-type bit-flag that determines if the data-type exhibits object- array like behavior. Each bit in this member is a flag which are named @@ -240,11 +263,13 @@ PyArrayDescr_Type .. c:var:: NPY_ITEM_REFCOUNT - .. c:var:: NPY_ITEM_HASOBJECT - Indicates that items of this data-type must be reference counted (using :c:func:`Py_INCREF` and :c:func:`Py_DECREF` ). + .. c:var:: NPY_ITEM_HASOBJECT + + Same as :c:data:`NPY_ITEM_REFCOUNT`. + .. c:var:: NPY_LIST_PICKLE Indicates arrays of this data-type must be converted to a list @@ -367,6 +392,11 @@ PyArrayDescr_Type normally a Python string. These tuples are placed in this dictionary keyed by name (and also title if given). +.. c:member:: PyObject *PyArray_Descr.names + + An ordered tuple of field names. It is NULL if no field is + defined. + .. c:member:: PyArray_ArrFuncs *PyArray_Descr.f A pointer to a structure containing functions that the type needs @@ -374,6 +404,20 @@ PyArrayDescr_Type thing as the universal functions (ufuncs) described later. Their signatures can vary arbitrarily. +.. c:member:: PyObject *PyArray_Descr.metadata + + Metadata about this dtype. + +.. c:member:: NpyAuxData *PyArray_Descr.c_metadata + + Metadata specific to the C implementation + of the particular dtype. Added for NumPy 1.7.0. + +.. c:member:: Npy_hash_t *PyArray_Descr.hash + + Currently unused. Reserved for future use in caching + hash values. + .. c:type:: PyArray_ArrFuncs Functions implementing internal features. Not all of these @@ -498,20 +542,19 @@ PyArrayDescr_Type and ``is2`` *bytes*, respectively. This function requires behaved (though not necessarily contiguous) memory. - .. c:member:: int scanfunc(FILE* fd, void* ip , void* sep , void* arr) + .. c:member:: int scanfunc(FILE* fd, void* ip, void* arr) A pointer to a function that scans (scanf style) one element of the corresponding type from the file descriptor ``fd`` into the array memory pointed to by ``ip``. The array is assumed - to be behaved. If ``sep`` is not NULL, then a separator string - is also scanned from the file before returning. The last - argument ``arr`` is the array to be scanned into. A 0 is - returned if the scan is successful. A negative number - indicates something went wrong: -1 means the end of file was - reached before the separator string could be scanned, -4 means - that the end of file was reached before the element could be - scanned, and -3 means that the element could not be - interpreted from the format string. Requires a behaved array. + to be behaved. + The last argument ``arr`` is the array to be scanned into. + Returns number of receiving arguments successfully assigned (which + may be zero in case a matching failure occurred before the first + receiving argument was assigned), or EOF if input failure occurs + before the first receiving argument was assigned. + This function should be called without holding the Python GIL, and + has to grab it for error reporting. .. c:member:: int fromstr(char* str, void* ip, char** endptr, void* arr) @@ -522,6 +565,8 @@ PyArrayDescr_Type string. The last argument ``arr`` is the array into which ip points (needed for variable-size data- types). Returns 0 on success or -1 on failure. Requires a behaved array. + This function should be called without holding the Python GIL, and + has to grab it for error reporting. .. c:member:: Bool nonzero(void* data, void* arr) @@ -643,25 +688,28 @@ PyArrayDescr_Type The :c:data:`PyArray_Type` typeobject implements many of the features of -Python objects including the tp_as_number, tp_as_sequence, -tp_as_mapping, and tp_as_buffer interfaces. The rich comparison -(tp_richcompare) is also used along with new-style attribute lookup -for methods (tp_methods) and properties (tp_getset). The -:c:data:`PyArray_Type` can also be sub-typed. +:c:type:`Python objects <PyTypeObject>` including the :c:member:`tp_as_number +<PyTypeObject.tp_as_number>`, :c:member:`tp_as_sequence +<PyTypeObject.tp_as_sequence>`, :c:member:`tp_as_mapping +<PyTypeObject.tp_as_mapping>`, and :c:member:`tp_as_buffer +<PyTypeObject.tp_as_buffer>` interfaces. The :c:type:`rich comparison +<richcmpfunc>`) is also used along with new-style attribute lookup for +member (:c:member:`tp_members <PyTypeObject.tp_members>`) and properties +(:c:member:`tp_getset <PyTypeObject.tp_getset>`). +The :c:data:`PyArray_Type` can also be sub-typed. .. tip:: - The tp_as_number methods use a generic approach to call whatever - function has been registered for handling the operation. The - function PyNumeric_SetOps(..) can be used to register functions to - handle particular mathematical operations (for all arrays). When - the umath module is imported, it sets the numeric operations for - all arrays to the corresponding ufuncs. The tp_str and tp_repr - methods can also be altered using PyString_SetStringFunction(...). + The ``tp_as_number`` methods use a generic approach to call whatever + function has been registered for handling the operation. When the + ``_multiarray_umath module`` is imported, it sets the numeric operations + for all arrays to the corresponding ufuncs. This choice can be changed with + :c:func:`PyUFunc_ReplaceLoopBySignature` The ``tp_str`` and ``tp_repr`` + methods can also be altered using :c:func:`PyArray_SetStringFunction`. -PyUFunc_Type ------------- +PyUFunc_Type and PyUFuncObject +------------------------------ .. c:var:: PyUFunc_Type @@ -685,7 +733,14 @@ PyUFunc_Type the information needed to call the underlying C-code loops that perform the actual work. While it is described here for completeness, it should be considered internal to NumPy and manipulated via ``PyUFunc_*`` - functions. It has the following structure: + functions. The size of this structure is subject to change across versions + of NumPy. To ensure compatibility: + + - Never declare a non-pointer instance of the struct + - Never perform pointer arithmetic + - Never use ``sizeof(PyUFuncObject)`` + + It has the following structure: .. code-block:: c @@ -713,10 +768,13 @@ PyUFunc_Type char *core_signature; PyUFunc_TypeResolutionFunc *type_resolver; PyUFunc_LegacyInnerLoopSelectionFunc *legacy_inner_loop_selector; - void *reserved2; PyUFunc_MaskedInnerLoopSelectionFunc *masked_inner_loop_selector; npy_uint32 *op_flags; npy_uint32 *iter_flags; + /* new in API version 0x0000000D */ + npy_intp *core_dim_sizes; + npy_intp *core_dim_flags; + } PyUFuncObject; .. c:macro: PyUFuncObject.PyObject_HEAD @@ -743,8 +801,8 @@ PyUFunc_Type the identity for this operation. It is only used for a reduce-like call on an empty array. - .. c:member:: void PyUFuncObject.functions(char** args, npy_intp* dims, - npy_intp* steps, void* extradata) + .. c:member:: void PyUFuncObject.functions( \ + char** args, npy_intp* dims, npy_intp* steps, void* extradata) An array of function pointers --- one for each data type supported by the ufunc. This is the vector loop that is called @@ -776,6 +834,10 @@ PyUFunc_Type specifies how many different 1-d loops (of the builtin data types) are available. + .. c:member:: int PyUFuncObject.reserved1 + + Unused. + .. c:member:: char *PyUFuncObject.name A string name for the ufunc. This is used dynamically to build @@ -870,8 +932,23 @@ PyUFunc_Type Override the default nditer flags for the ufunc. -PyArrayIter_Type ----------------- + Added in API version 0x0000000D + + .. c:member:: npy_intp *PyUFuncObject.core_dim_sizes + + For each distinct core dimension, the possible + :ref:`frozen <frozen>` size if :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` is 0 + + .. c:member:: npy_uint32 *PyUFuncObject.core_dim_flags + + For each distinct core dimension, a set of ``UFUNC_CORE_DIM*`` flags + + - :c:data:`UFUNC_CORE_DIM_CAN_IGNORE` if the dim name ends in ``?`` + - :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` if the dim size will be + determined from the operands and not from a :ref:`frozen <frozen>` signature + +PyArrayIter_Type and PyArrayIterObject +-------------------------------------- .. c:var:: PyArrayIter_Type @@ -980,8 +1057,8 @@ with it through the use of the macros :c:func:`PyArray_ITER_NEXT` (it), :c:type:`PyArrayIterObject *`. -PyArrayMultiIter_Type ---------------------- +PyArrayMultiIter_Type and PyArrayMultiIterObject +------------------------------------------------ .. c:var:: PyArrayMultiIter_Type @@ -1042,8 +1119,8 @@ PyArrayMultiIter_Type arrays to be broadcast together. On return, the iterators are adjusted for broadcasting. -PyArrayNeighborhoodIter_Type ----------------------------- +PyArrayNeighborhoodIter_Type and PyArrayNeighborhoodIterObject +-------------------------------------------------------------- .. c:var:: PyArrayNeighborhoodIter_Type @@ -1056,8 +1133,33 @@ PyArrayNeighborhoodIter_Type :c:data:`PyArrayNeighborhoodIter_Type` is the :c:type:`PyArrayNeighborhoodIterObject`. -PyArrayFlags_Type ------------------ + .. code-block:: c + + typedef struct { + PyObject_HEAD + int nd_m1; + npy_intp index, size; + npy_intp coordinates[NPY_MAXDIMS] + npy_intp dims_m1[NPY_MAXDIMS]; + npy_intp strides[NPY_MAXDIMS]; + npy_intp backstrides[NPY_MAXDIMS]; + npy_intp factors[NPY_MAXDIMS]; + PyArrayObject *ao; + char *dataptr; + npy_bool contiguous; + npy_intp bounds[NPY_MAXDIMS][2]; + npy_intp limits[NPY_MAXDIMS][2]; + npy_intp limits_sizes[NPY_MAXDIMS]; + npy_iter_get_dataptr_t translate; + npy_intp nd; + npy_intp dimensions[NPY_MAXDIMS]; + PyArrayIterObject* _internal_iter; + char* constant; + int mode; + } PyArrayNeighborhoodIterObject; + +PyArrayFlags_Type and PyArrayFlagsObject +---------------------------------------- .. c:var:: PyArrayFlags_Type @@ -1067,6 +1169,16 @@ PyArrayFlags_Type attributes or by accessing them as if the object were a dictionary with the flag names as entries. +.. c:type:: PyArrayFlagsObject + + .. code-block:: c + + typedef struct PyArrayFlagsObject { + PyObject_HEAD + PyObject *arr; + int flags; + } PyArrayFlagsObject; + ScalarArrayTypes ---------------- diff --git a/doc/source/reference/c-api.ufunc.rst b/doc/source/reference/c-api/ufunc.rst index 07c7b0c80..c9cc60141 100644 --- a/doc/source/reference/c-api.ufunc.rst +++ b/doc/source/reference/c-api/ufunc.rst @@ -21,7 +21,17 @@ Constants .. c:var:: PyUFunc_{VALUE} - ``{VALUE}`` can be **One** (1), **Zero** (0), or **None** (-1) + .. c:var:: PyUFunc_One + + .. c:var:: PyUFunc_Zero + + .. c:var:: PyUFunc_MinusOne + + .. c:var:: PyUFunc_ReorderableNone + + .. c:var:: PyUFunc_None + + .. c:var:: PyUFunc_IdentityValue Macros @@ -39,28 +49,6 @@ Macros Used in universal function code to re-acquire the Python GIL if it was released (because loop->obj was not true). -.. c:function:: UFUNC_CHECK_ERROR(loop) - - A macro used internally to check for errors and goto fail if - found. This macro requires a fail label in the current code - block. The *loop* variable must have at least members (obj, - errormask, and errorobj). If *loop* ->obj is nonzero, then - :c:func:`PyErr_Occurred` () is called (meaning the GIL must be held). If - *loop* ->obj is zero, then if *loop* ->errormask is nonzero, - :c:func:`PyUFunc_checkfperr` is called with arguments *loop* ->errormask - and *loop* ->errobj. If the result of this check of the IEEE - floating point registers is true then the code redirects to the - fail label which must be defined. - -.. c:function:: UFUNC_CHECK_STATUS(ret) - - Deprecated: use npy_clear_floatstatus from npy_math.h instead. - - A macro that expands to platform-dependent code. The *ret* - variable can be any integer. The :c:data:`UFUNC_FPE_{ERR}` bits are - set in *ret* according to the status of the corresponding error - flags of the floating point processor. - Functions --------- @@ -169,8 +157,12 @@ Functions :param identity: Either :c:data:`PyUFunc_One`, :c:data:`PyUFunc_Zero`, - :c:data:`PyUFunc_None`. This specifies what should be returned when + :c:data:`PyUFunc_MinusOne`, or :c:data:`PyUFunc_None`. + This specifies what should be returned when an empty array is passed to the reduce method of the ufunc. + The special value :c:data:`PyUFunc_IdentityValue` may only be used with + the :c:func:`PyUFunc_FromFuncAndDataAndSignatureAndIdentity` method, to + allow an arbitrary python object to be used as the identity. :param name: The name for the ufunc as a ``NULL`` terminated string. Specifying @@ -206,6 +198,21 @@ Functions to calling PyUFunc_FromFuncAndData. A copy of the string is made, so the passed in buffer can be freed. +.. c:function:: PyObject* PyUFunc_FromFuncAndDataAndSignatureAndIdentity( \ + PyUFuncGenericFunction *func, void **data, char *types, int ntypes, \ + int nin, int nout, int identity, char *name, char *doc, int unused, \ + char *signature, PyObject *identity_value) + + This function is very similar to `PyUFunc_FromFuncAndDataAndSignature` above, + but has an extra *identity_value* argument, to define an arbitrary identity + for the ufunc when ``identity`` is passed as ``PyUFunc_IdentityValue``. + + :param identity_value: + The identity for the new gufunc. Must be passed as ``NULL`` unless the + ``identity`` argument is ``PyUFunc_IdentityValue``. Setting it to NULL + is equivalent to calling PyUFunc_FromFuncAndDataAndSignature. + + .. c:function:: int PyUFunc_RegisterLoopForType( \ PyUFuncObject* ufunc, int usertype, PyUFuncGenericFunction function, \ int* arg_types, void* data) diff --git a/doc/source/reference/distutils.rst b/doc/source/reference/distutils.rst index 289822909..a22db3e8e 100644 --- a/doc/source/reference/distutils.rst +++ b/doc/source/reference/distutils.rst @@ -13,8 +13,7 @@ distutils, use the :func:`setup <core.setup>` command from :mod:`numpy.distutils.misc_util` that can make it easier to construct keyword arguments to pass to the setup function (by passing the dictionary obtained from the todict() method of the class). More -information is available in the NumPy Distutils Users Guide in -``<site-packages>/numpy/doc/DISTUTILS.txt``. +information is available in the :ref:`distutils-user-guide`. .. index:: @@ -23,39 +22,31 @@ information is available in the NumPy Distutils Users Guide in Modules in :mod:`numpy.distutils` ================================= +.. toctree:: + :maxdepth: 2 -misc_util ---------- + distutils/misc_util -.. module:: numpy.distutils.misc_util + +.. currentmodule:: numpy.distutils .. autosummary:: :toctree: generated/ - get_numpy_include_dirs - dict_append - appendpath - allpath - dot_join - generate_config_py - get_cmd - terminal_has_colors - red_text - green_text - yellow_text - blue_text - cyan_text - cyg2win32 - all_strings - has_f_sources - has_cxx_sources - filter_sources - get_dependencies - is_local_src_dir - get_ext_source_files - get_script_files + ccompiler + cpuinfo.cpu + core.Extension + exec_command + log.set_verbosity + system_info.get_info + system_info.get_standard_file +Configuration class +=================== + +.. currentmodule:: numpy.distutils.misc_util + .. class:: Configuration(package_name=None, parent_name=None, top_path=None, package_path=None, **attrs) Construct a configuration instance for the given package name. If @@ -110,20 +101,6 @@ misc_util .. automethod:: get_info -Other modules -------------- - -.. currentmodule:: numpy.distutils - -.. autosummary:: - :toctree: generated/ - - system_info.get_info - system_info.get_standard_file - cpuinfo.cpu - log.set_verbosity - exec_command - Building Installable C libraries ================================ @@ -215,102 +192,4 @@ template and placed in the build directory to be used instead. Two forms of template conversion are supported. The first form occurs for files named <file>.ext.src where ext is a recognized Fortran extension (f, f90, f95, f77, for, ftn, pyf). The second form is used -for all other cases. - -.. index:: - single: code generation - -Fortran files -------------- - -This template converter will replicate all **function** and -**subroutine** blocks in the file with names that contain '<...>' -according to the rules in '<...>'. The number of comma-separated words -in '<...>' determines the number of times the block is repeated. What -these words are indicates what that repeat rule, '<...>', should be -replaced with in each block. All of the repeat rules in a block must -contain the same number of comma-separated words indicating the number -of times that block should be repeated. If the word in the repeat rule -needs a comma, leftarrow, or rightarrow, then prepend it with a -backslash ' \'. If a word in the repeat rule matches ' \\<index>' then -it will be replaced with the <index>-th word in the same repeat -specification. There are two forms for the repeat rule: named and -short. - - -Named repeat rule -^^^^^^^^^^^^^^^^^ - -A named repeat rule is useful when the same set of repeats must be -used several times in a block. It is specified using <rule1=item1, -item2, item3,..., itemN>, where N is the number of times the block -should be repeated. On each repeat of the block, the entire -expression, '<...>' will be replaced first with item1, and then with -item2, and so forth until N repeats are accomplished. Once a named -repeat specification has been introduced, the same repeat rule may be -used **in the current block** by referring only to the name -(i.e. <rule1>. - - -Short repeat rule -^^^^^^^^^^^^^^^^^ - -A short repeat rule looks like <item1, item2, item3, ..., itemN>. The -rule specifies that the entire expression, '<...>' should be replaced -first with item1, and then with item2, and so forth until N repeats -are accomplished. - - -Pre-defined names -^^^^^^^^^^^^^^^^^ - -The following predefined named repeat rules are available: - -- <prefix=s,d,c,z> - -- <_c=s,d,c,z> - -- <_t=real, double precision, complex, double complex> - -- <ftype=real, double precision, complex, double complex> - -- <ctype=float, double, complex_float, complex_double> - -- <ftypereal=float, double precision, \\0, \\1> - -- <ctypereal=float, double, \\0, \\1> - - -Other files ------------ - -Non-Fortran files use a separate syntax for defining template blocks -that should be repeated using a variable expansion similar to the -named repeat rules of the Fortran-specific repeats. The template rules -for these files are: - -1. "/\**begin repeat "on a line by itself marks the beginning of - a segment that should be repeated. - -2. Named variable expansions are defined using #name=item1, item2, item3, - ..., itemN# and placed on successive lines. These variables are - replaced in each repeat block with corresponding word. All named - variables in the same repeat block must define the same number of - words. - -3. In specifying the repeat rule for a named variable, item*N is short- - hand for item, item, ..., item repeated N times. In addition, - parenthesis in combination with \*N can be used for grouping several - items that should be repeated. Thus, #name=(item1, item2)*4# is - equivalent to #name=item1, item2, item1, item2, item1, item2, item1, - item2# - -4. "\*/ "on a line by itself marks the end of the variable expansion - naming. The next line is the first line that will be repeated using - the named rules. - -5. Inside the block to be repeated, the variables that should be expanded - are specified as @name@. - -6. "/\**end repeat**/ "on a line by itself marks the previous line - as the last line of the block to be repeated. +for all other cases. See :ref:`templating`. diff --git a/doc/source/reference/distutils/misc_util.rst b/doc/source/reference/distutils/misc_util.rst new file mode 100644 index 000000000..bbb83a5ab --- /dev/null +++ b/doc/source/reference/distutils/misc_util.rst @@ -0,0 +1,7 @@ +distutils.misc_util +=================== + +.. automodule:: numpy.distutils.misc_util + :members: + :undoc-members: + :exclude-members: Configuration diff --git a/doc/source/reference/distutils_guide.rst b/doc/source/reference/distutils_guide.rst new file mode 100644 index 000000000..081719d16 --- /dev/null +++ b/doc/source/reference/distutils_guide.rst @@ -0,0 +1,7 @@ +.. _distutils-user-guide: + +NumPy Distutils - Users Guide +============================= + +.. include:: ../../DISTUTILS.rst.txt + :start-line: 6 diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index 2140c57f7..6742d605a 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -23,7 +23,8 @@ For learning how to use NumPy, see also :ref:`user`. ufuncs routines distutils - c-api + distutils_guide + c-api/index internals swig @@ -35,4 +36,4 @@ Large parts of this manual originate from Travis E. Oliphant's book `Guide to NumPy <https://archive.org/details/NumPyBook>`__ (which generously entered Public Domain in August 2008). The reference documentation for many of the functions are written by numerous contributors and developers of -NumPy.
\ No newline at end of file +NumPy. diff --git a/doc/source/reference/internals.rst b/doc/source/reference/internals.rst index e1d6644a6..03d081bf9 100644 --- a/doc/source/reference/internals.rst +++ b/doc/source/reference/internals.rst @@ -5,5 +5,6 @@ NumPy internals .. toctree:: internals.code-explanations + alignment .. automodule:: numpy.doc.internals diff --git a/doc/source/reference/maskedarray.baseclass.rst b/doc/source/reference/maskedarray.baseclass.rst index 427ad1536..204ebfe08 100644 --- a/doc/source/reference/maskedarray.baseclass.rst +++ b/doc/source/reference/maskedarray.baseclass.rst @@ -49,11 +49,11 @@ The :class:`MaskedArray` class .. class:: MaskedArray - A subclass of :class:`~numpy.ndarray` designed to manipulate numerical arrays with missing data. +A subclass of :class:`~numpy.ndarray` designed to manipulate numerical arrays with missing data. - An instance of :class:`MaskedArray` can be thought as the combination of several elements: +An instance of :class:`MaskedArray` can be thought as the combination of several elements: * The :attr:`~MaskedArray.data`, as a regular :class:`numpy.ndarray` of any shape or datatype (the data). * A boolean :attr:`~numpy.ma.MaskedArray.mask` with the same shape as the data, where a ``True`` value indicates that the corresponding element of the data is invalid. @@ -62,89 +62,26 @@ The :class:`MaskedArray` class +.. _ma-attributes: + Attributes and properties of masked arrays ------------------------------------------ .. seealso:: :ref:`Array Attributes <arrays.ndarray.attributes>` +.. autoattribute:: MaskedArray.data -.. attribute:: MaskedArray.data - - Returns the underlying data, as a view of the masked array. - If the underlying data is a subclass of :class:`numpy.ndarray`, it is - returned as such. - - >>> x = ma.array(np.matrix([[1, 2], [3, 4]]), mask=[[0, 1], [1, 0]]) - >>> x.data - matrix([[1, 2], - [3, 4]]) - - The type of the data can be accessed through the :attr:`baseclass` - attribute. - -.. attribute:: MaskedArray.mask - - Returns the underlying mask, as an array with the same shape and structure - as the data, but where all fields are atomically booleans. - A value of ``True`` indicates an invalid entry. - - -.. attribute:: MaskedArray.recordmask - - Returns the mask of the array if it has no named fields. For structured - arrays, returns a ndarray of booleans where entries are ``True`` if **all** - the fields are masked, ``False`` otherwise:: - - >>> x = ma.array([(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)], - ... mask=[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)], - ... dtype=[('a', int), ('b', int)]) - >>> x.recordmask - array([False, False, True, False, False]) - - -.. attribute:: MaskedArray.fill_value - - Returns the value used to fill the invalid entries of a masked array. - The value is either a scalar (if the masked array has no named fields), - or a 0-D ndarray with the same :attr:`dtype` as the masked array if it has - named fields. - - The default filling value depends on the datatype of the array: - - ======== ======== - datatype default - ======== ======== - bool True - int 999999 - float 1.e20 - complex 1.e20+0j - object '?' - string 'N/A' - ======== ======== - - - -.. attribute:: MaskedArray.baseclass - - Returns the class of the underlying data. - - >>> x = ma.array(np.matrix([[1, 2], [3, 4]]), mask=[[0, 0], [1, 0]]) - >>> x.baseclass - <class 'numpy.matrixlib.defmatrix.matrix'> - - -.. attribute:: MaskedArray.sharedmask +.. autoattribute:: MaskedArray.mask - Returns whether the mask of the array is shared between several masked arrays. - If this is the case, any modification to the mask of one array will be - propagated to the others. +.. autoattribute:: MaskedArray.recordmask +.. autoattribute:: MaskedArray.fill_value -.. attribute:: MaskedArray.hardmask +.. autoattribute:: MaskedArray.baseclass - Returns whether the mask is hard (``True``) or soft (``False``). - When the mask is hard, masked entries cannot be unmasked. +.. autoattribute:: MaskedArray.sharedmask +.. autoattribute:: MaskedArray.hardmask As :class:`MaskedArray` is a subclass of :class:`~numpy.ndarray`, a masked array also inherits all the attributes and properties of a :class:`~numpy.ndarray` instance. @@ -184,10 +121,8 @@ Conversion :toctree: generated/ MaskedArray.__float__ - MaskedArray.__hex__ MaskedArray.__int__ MaskedArray.__long__ - MaskedArray.__oct__ MaskedArray.view MaskedArray.astype @@ -311,7 +246,7 @@ Truth value of an array (:func:`bool()`): .. autosummary:: :toctree: generated/ - MaskedArray.__nonzero__ + MaskedArray.__bool__ Arithmetic: @@ -328,7 +263,6 @@ Arithmetic: MaskedArray.__mul__ MaskedArray.__rmul__ MaskedArray.__div__ - MaskedArray.__rdiv__ MaskedArray.__truediv__ MaskedArray.__rtruediv__ MaskedArray.__floordiv__ diff --git a/doc/source/reference/maskedarray.generic.rst b/doc/source/reference/maskedarray.generic.rst index 07ad6c292..7375d60fb 100644 --- a/doc/source/reference/maskedarray.generic.rst +++ b/doc/source/reference/maskedarray.generic.rst @@ -2,7 +2,7 @@ .. _maskedarray.generic: - +.. module:: numpy.ma The :mod:`numpy.ma` module ========================== diff --git a/doc/source/reference/random/bit_generators/bitgenerators.rst b/doc/source/reference/random/bit_generators/bitgenerators.rst new file mode 100644 index 000000000..1474f7dac --- /dev/null +++ b/doc/source/reference/random/bit_generators/bitgenerators.rst @@ -0,0 +1,11 @@ +:orphan: + +BitGenerator +------------ + +.. currentmodule:: numpy.random.bit_generator + +.. autosummary:: + :toctree: generated/ + + BitGenerator diff --git a/doc/source/reference/random/bit_generators/index.rst b/doc/source/reference/random/bit_generators/index.rst new file mode 100644 index 000000000..35d9e5d09 --- /dev/null +++ b/doc/source/reference/random/bit_generators/index.rst @@ -0,0 +1,112 @@ +.. _bit_generator: + +.. currentmodule:: numpy.random + +Bit Generators +-------------- + +The random values produced by :class:`~Generator` +orignate in a BitGenerator. The BitGenerators do not directly provide +random numbers and only contains methods used for seeding, getting or +setting the state, jumping or advancing the state, and for accessing +low-level wrappers for consumption by code that can efficiently +access the functions provided, e.g., `numba <https://numba.pydata.org>`_. + +Supported BitGenerators +======================= + +The included BitGenerators are: + +* PCG-64 - The default. A fast generator that supports many parallel streams + and can be advanced by an arbitrary amount. See the documentation for + :meth:`~.PCG64.advance`. PCG-64 has a period of :math:`2^{128}`. See the `PCG + author's page`_ for more details about this class of PRNG. +* MT19937 - The standard Python BitGenerator. Adds a `~mt19937.MT19937.jumped` + function that returns a new generator with state as-if :math:`2^{128}` draws have + been made. +* Philox - A counter-based generator capable of being advanced an + arbitrary number of steps or generating independent streams. See the + `Random123`_ page for more details about this class of bit generators. +* SFC64 - A fast generator based on random invertible mappings. Usually the + fastest generator of the four. See the `SFC author's page`_ for (a little) + more detail. + +.. _`PCG author's page`: http://www.pcg-random.org/ +.. _`Random123`: https://www.deshawresearch.com/resources_random123.html +.. _`SFC author's page`: http://pracrand.sourceforge.net/RNG_engines.txt + +.. toctree:: + :maxdepth: 1 + + BitGenerator <bitgenerators> + MT19937 <mt19937> + PCG64 <pcg64> + Philox <philox> + SFC64 <sfc64> + +Seeding and Entropy +------------------- + +A BitGenerator provides a stream of random values. In order to generate +reproducible streams, BitGenerators support setting their initial state via a +seed. All of the provided BitGenerators will take an arbitrary-sized +non-negative integer, or a list of such integers, as a seed. BitGenerators +need to take those inputs and process them into a high-quality internal state +for the BitGenerator. All of the BitGenerators in numpy delegate that task to +`~SeedSequence`, which uses hashing techniques to ensure that even low-quality +seeds generate high-quality initial states. + +.. code-block:: python + + from numpy.random import PCG64 + + bg = PCG64(12345678903141592653589793) + +.. end_block + +`~SeedSequence` is designed to be convenient for implementing best practices. +We recommend that a stochastic program defaults to using entropy from the OS so +that each run is different. The program should print out or log that entropy. +In order to reproduce a past value, the program should allow the user to +provide that value through some mechanism, a command-line argument is common, +so that the user can then re-enter that entropy to reproduce the result. +`~SeedSequence` can take care of everything except for communicating with the +user, which is up to you. + +.. code-block:: python + + from numpy.random import PCG64, SeedSequence + + # Get the user's seed somehow, maybe through `argparse`. + # If the user did not provide a seed, it should return `None`. + seed = get_user_seed() + ss = SeedSequence(seed) + print('seed = {}'.format(ss.entropy)) + bg = PCG64(ss) + +.. end_block + +We default to using a 128-bit integer using entropy gathered from the OS. This +is a good amount of entropy to initialize all of the generators that we have in +numpy. We do not recommend using small seeds below 32 bits for general use. +Using just a small set of seeds to instantiate larger state spaces means that +there are some initial states that are impossible to reach. This creates some +biases if everyone uses such values. + +There will not be anything *wrong* with the results, per se; even a seed of +0 is perfectly fine thanks to the processing that `~SeedSequence` does. If you +just need *some* fixed value for unit tests or debugging, feel free to use +whatever seed you like. But if you want to make inferences from the results or +publish them, drawing from a larger set of seeds is good practice. + +If you need to generate a good seed "offline", then ``SeedSequence().entropy`` +or using ``secrets.randbits(128)`` from the standard library are both +convenient ways. + +.. autosummary:: + :toctree: generated/ + + SeedSequence + bit_generator.ISeedSequence + bit_generator.ISpawnableSeedSequence + bit_generator.SeedlessSeedSequence diff --git a/doc/source/reference/random/bit_generators/mt19937.rst b/doc/source/reference/random/bit_generators/mt19937.rst new file mode 100644 index 000000000..71875db4e --- /dev/null +++ b/doc/source/reference/random/bit_generators/mt19937.rst @@ -0,0 +1,32 @@ +Mersenne Twister (MT19937) +-------------------------- + +.. currentmodule:: numpy.random + +.. autoclass:: MT19937 + :exclude-members: + +State +===== + +.. autosummary:: + :toctree: generated/ + + ~MT19937.state + +Parallel generation +=================== +.. autosummary:: + :toctree: generated/ + + ~MT19937.jumped + +Extending +========= +.. autosummary:: + :toctree: generated/ + + ~MT19937.cffi + ~MT19937.ctypes + + diff --git a/doc/source/reference/random/bit_generators/pcg64.rst b/doc/source/reference/random/bit_generators/pcg64.rst new file mode 100644 index 000000000..5881b7008 --- /dev/null +++ b/doc/source/reference/random/bit_generators/pcg64.rst @@ -0,0 +1,31 @@ +Parallel Congruent Generator (64-bit, PCG64) +-------------------------------------------- + +.. currentmodule:: numpy.random + +.. autoclass:: PCG64 + :exclude-members: + +State +===== + +.. autosummary:: + :toctree: generated/ + + ~PCG64.state + +Parallel generation +=================== +.. autosummary:: + :toctree: generated/ + + ~PCG64.advance + ~PCG64.jumped + +Extending +========= +.. autosummary:: + :toctree: generated/ + + ~PCG64.cffi + ~PCG64.ctypes diff --git a/doc/source/reference/random/bit_generators/philox.rst b/doc/source/reference/random/bit_generators/philox.rst new file mode 100644 index 000000000..8eba2d351 --- /dev/null +++ b/doc/source/reference/random/bit_generators/philox.rst @@ -0,0 +1,33 @@ +Philox Counter-based RNG +------------------------ + +.. currentmodule:: numpy.random + +.. autoclass:: Philox + :exclude-members: + +State +===== + +.. autosummary:: + :toctree: generated/ + + ~Philox.state + +Parallel generation +=================== +.. autosummary:: + :toctree: generated/ + + ~Philox.advance + ~Philox.jumped + +Extending +========= +.. autosummary:: + :toctree: generated/ + + ~Philox.cffi + ~Philox.ctypes + + diff --git a/doc/source/reference/random/bit_generators/sfc64.rst b/doc/source/reference/random/bit_generators/sfc64.rst new file mode 100644 index 000000000..d34124a33 --- /dev/null +++ b/doc/source/reference/random/bit_generators/sfc64.rst @@ -0,0 +1,26 @@ +SFC64 Small Fast Chaotic PRNG +----------------------------- + +.. currentmodule:: numpy.random + +.. autoclass:: SFC64 + :exclude-members: + +State +===== + +.. autosummary:: + :toctree: generated/ + + ~SFC64.state + +Extending +========= +.. autosummary:: + :toctree: generated/ + + ~SFC64.cffi + ~SFC64.ctypes + + + diff --git a/doc/source/reference/random/extending.rst b/doc/source/reference/random/extending.rst new file mode 100644 index 000000000..22f9cb7e4 --- /dev/null +++ b/doc/source/reference/random/extending.rst @@ -0,0 +1,165 @@ +.. currentmodule:: numpy.random + +Extending +--------- +The BitGenerators have been designed to be extendable using standard tools for +high-performance Python -- numba and Cython. The `~Generator` object can also +be used with user-provided BitGenerators as long as these export a small set of +required functions. + +Numba +===== +Numba can be used with either CTypes or CFFI. The current iteration of the +BitGenerators all export a small set of functions through both interfaces. + +This example shows how numba can be used to produce Box-Muller normals using +a pure Python implementation which is then compiled. The random numbers are +provided by ``ctypes.next_double``. + +.. code-block:: python + + from numpy.random import PCG64 + import numpy as np + import numba as nb + + x = PCG64() + f = x.ctypes.next_double + s = x.ctypes.state + state_addr = x.ctypes.state_address + + def normals(n, state): + out = np.empty(n) + for i in range((n+1)//2): + x1 = 2.0*f(state) - 1.0 + x2 = 2.0*f(state) - 1.0 + r2 = x1*x1 + x2*x2 + while r2 >= 1.0 or r2 == 0.0: + x1 = 2.0*f(state) - 1.0 + x2 = 2.0*f(state) - 1.0 + r2 = x1*x1 + x2*x2 + g = np.sqrt(-2.0*np.log(r2)/r2) + out[2*i] = g*x1 + if 2*i+1 < n: + out[2*i+1] = g*x2 + return out + + # Compile using Numba + print(normals(10, s).var()) + # Warm up + normalsj = nb.jit(normals, nopython=True) + # Must use state address not state with numba + normalsj(1, state_addr) + %timeit normalsj(1000000, state_addr) + print('1,000,000 Box-Muller (numba/PCG64) randoms') + %timeit np.random.standard_normal(1000000) + print('1,000,000 Box-Muller (NumPy) randoms') + + +Both CTypes and CFFI allow the more complicated distributions to be used +directly in Numba after compiling the file distributions.c into a DLL or so. +An example showing the use of a more complicated distribution is in the +examples folder. + +.. _randomgen_cython: + +Cython +====== + +Cython can be used to unpack the ``PyCapsule`` provided by a BitGenerator. +This example uses `~pcg64.PCG64` and +``random_gauss_zig``, the Ziggurat-based generator for normals, to fill an +array. The usual caveats for writing high-performance code using Cython -- +removing bounds checks and wrap around, providing array alignment information +-- still apply. + +.. code-block:: cython + + import numpy as np + cimport numpy as np + cimport cython + from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer + from numpy.random.common cimport * + from numpy.random.distributions cimport random_gauss_zig + from numpy.random import PCG64 + + + @cython.boundscheck(False) + @cython.wraparound(False) + def normals_zig(Py_ssize_t n): + cdef Py_ssize_t i + cdef bitgen_t *rng + cdef const char *capsule_name = "BitGenerator" + cdef double[::1] random_values + + x = PCG64() + capsule = x.capsule + if not PyCapsule_IsValid(capsule, capsule_name): + raise ValueError("Invalid pointer to anon_func_state") + rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name) + random_values = np.empty(n) + # Best practice is to release GIL and acquire the lock + with x.lock, nogil: + for i in range(n): + random_values[i] = random_gauss_zig(rng) + randoms = np.asarray(random_values) + return randoms + +The BitGenerator can also be directly accessed using the members of the basic +RNG structure. + +.. code-block:: cython + + @cython.boundscheck(False) + @cython.wraparound(False) + def uniforms(Py_ssize_t n): + cdef Py_ssize_t i + cdef bitgen_t *rng + cdef const char *capsule_name = "BitGenerator" + cdef double[::1] random_values + + x = PCG64() + capsule = x.capsule + # Optional check that the capsule if from a BitGenerator + if not PyCapsule_IsValid(capsule, capsule_name): + raise ValueError("Invalid pointer to anon_func_state") + # Cast the pointer + rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name) + random_values = np.empty(n) + with x.lock, nogil: + for i in range(n): + # Call the function + random_values[i] = rng.next_double(rng.state) + randoms = np.asarray(random_values) + return randoms + +These functions along with a minimal setup file are included in the +examples folder. + +New Basic RNGs +============== +`~Generator` can be used with other user-provided BitGenerators. The simplest +way to write a new BitGenerator is to examine the pyx file of one of the +existing BitGenerators. The key structure that must be provided is the +``capsule`` which contains a ``PyCapsule`` to a struct pointer of type +``bitgen_t``, + +.. code-block:: c + + typedef struct bitgen { + void *state; + uint64_t (*next_uint64)(void *st); + uint32_t (*next_uint32)(void *st); + double (*next_double)(void *st); + uint64_t (*next_raw)(void *st); + } bitgen_t; + +which provides 5 pointers. The first is an opaque pointer to the data structure +used by the BitGenerators. The next three are function pointers which return +the next 64- and 32-bit unsigned integers, the next random double and the next +raw value. This final function is used for testing and so can be set to +the next 64-bit unsigned integer function if not needed. Functions inside +``Generator`` use this structure as in + +.. code-block:: c + + bitgen_state->next_uint64(bitgen_state->state) diff --git a/doc/source/reference/random/generator.rst b/doc/source/reference/random/generator.rst new file mode 100644 index 000000000..068143270 --- /dev/null +++ b/doc/source/reference/random/generator.rst @@ -0,0 +1,84 @@ +.. currentmodule:: numpy.random + +Random Generator +---------------- +The `~Generator` provides access to +a wide range of distributions, and served as a replacement for +:class:`~numpy.random.RandomState`. The main difference between +the two is that ``Generator`` relies on an additional BitGenerator to +manage state and generate the random bits, which are then transformed into +random values from useful distributions. The default BitGenerator used by +``Generator`` is `~PCG64`. The BitGenerator +can be changed by passing an instantized BitGenerator to ``Generator``. + + +.. autofunction:: default_rng + +.. autoclass:: Generator + :exclude-members: + +Accessing the BitGenerator +========================== +.. autosummary:: + :toctree: generated/ + + ~numpy.random.Generator.bit_generator + +Simple random data +================== +.. autosummary:: + :toctree: generated/ + + ~numpy.random.Generator.integers + ~numpy.random.Generator.random + ~numpy.random.Generator.choice + ~numpy.random.Generator.bytes + +Permutations +============ +.. autosummary:: + :toctree: generated/ + + ~numpy.random.Generator.shuffle + ~numpy.random.Generator.permutation + +Distributions +============= +.. autosummary:: + :toctree: generated/ + + ~numpy.random.Generator.beta + ~numpy.random.Generator.binomial + ~numpy.random.Generator.chisquare + ~numpy.random.Generator.dirichlet + ~numpy.random.Generator.exponential + ~numpy.random.Generator.f + ~numpy.random.Generator.gamma + ~numpy.random.Generator.geometric + ~numpy.random.Generator.gumbel + ~numpy.random.Generator.hypergeometric + ~numpy.random.Generator.laplace + ~numpy.random.Generator.logistic + ~numpy.random.Generator.lognormal + ~numpy.random.Generator.logseries + ~numpy.random.Generator.multinomial + ~numpy.random.Generator.multivariate_normal + ~numpy.random.Generator.negative_binomial + ~numpy.random.Generator.noncentral_chisquare + ~numpy.random.Generator.noncentral_f + ~numpy.random.Generator.normal + ~numpy.random.Generator.pareto + ~numpy.random.Generator.poisson + ~numpy.random.Generator.power + ~numpy.random.Generator.rayleigh + ~numpy.random.Generator.standard_cauchy + ~numpy.random.Generator.standard_exponential + ~numpy.random.Generator.standard_gamma + ~numpy.random.Generator.standard_normal + ~numpy.random.Generator.standard_t + ~numpy.random.Generator.triangular + ~numpy.random.Generator.uniform + ~numpy.random.Generator.vonmises + ~numpy.random.Generator.wald + ~numpy.random.Generator.weibull + ~numpy.random.Generator.zipf diff --git a/doc/source/reference/random/index.rst b/doc/source/reference/random/index.rst new file mode 100644 index 000000000..b0283f3a7 --- /dev/null +++ b/doc/source/reference/random/index.rst @@ -0,0 +1,208 @@ +.. _numpyrandom: + +.. py:module:: numpy.random + +.. currentmodule:: numpy.random + +Random sampling (:mod:`numpy.random`) +===================================== + +Numpy's random number routines produce pseudo random numbers using +combinations of a `BitGenerator` to create sequences and a `Generator` +to use those sequences to sample from different statistical distributions: + +* BitGenerators: Objects that generate random numbers. These are typically + unsigned integer words filled with sequences of either 32 or 64 random bits. +* Generators: Objects that transform sequences of random bits from a + BitGenerator into sequences of numbers that follow a specific probability + distribution (such as uniform, Normal or Binomial) within a specified + interval. + +Since Numpy version 1.17.0 the Generator can be initialized with a +number of different BitGenerators. It exposes many different probability +distributions. See `NEP 19 <https://www.numpy.org/neps/ +nep-0019-rng-policy.html>`_ for context on the updated random Numpy number +routines. The legacy `.RandomState` random number routines are still +available, but limited to a single BitGenerator. + +For convenience and backward compatibility, a single `~.RandomState` +instance's methods are imported into the numpy.random namespace, see +:ref:`legacy` for the complete list. + +Quick Start +----------- + +By default, `~Generator` uses bits provided by `~pcg64.PCG64` which +has better statistical properties than the legacy mt19937 random +number generator in `~.RandomState`. + +.. code-block:: python + + # Uses the old numpy.random.RandomState + from numpy import random + random.standard_normal() + +`~Generator` can be used as a replacement for `~.RandomState`. Both class +instances now hold a internal `BitGenerator` instance to provide the bit +stream, it is accessible as ``gen.bit_generator``. Some long-overdue API +cleanup means that legacy and compatibility methods have been removed from +`~.Generator` + +=================== ============== ============ +`~.RandomState` `~.Generator` Notes +------------------- -------------- ------------ +``random_sample``, ``random`` Compatible with `random.random` +``rand`` +------------------- -------------- ------------ +``randint``, ``integers`` Add an ``endpoint`` kwarg +``random_integers`` +------------------- -------------- ------------ +``tomaxint`` removed Use ``integers(0, np.iinfo(np.int).max,`` + ``endpoint=False)`` +------------------- -------------- ------------ +``seed`` removed Use `~.SeedSequence.spawn` +=================== ============== ============ + +See `new-or-different` for more information + +.. code-block:: python + + # As replacement for RandomState(); default_rng() instantiates Generator with + # the default PCG64 BitGenerator. + from numpy.random import default_rng + rg = default_rng() + rg.standard_normal() + rg.bit_generator + +Something like the following code can be used to support both ``RandomState`` +and ``Generator``, with the understanding that the interfaces are slightly +different + +.. code-block:: python + + try: + rg_integers = rg.integers + except AttributeError: + rg_integers = rg.randint + a = rg_integers(1000) + +Seeds can be passed to any of the BitGenerators. The provided value is mixed +via `~.SeedSequence` to spread a possible sequence of seeds across a wider +range of initialization states for the BitGenerator. Here `~.PCG64` is used and +is wrapped with a `~.Generator`. + +.. code-block:: python + + from numpy.random import Generator, PCG64 + rg = Generator(PCG64(12345)) + rg.standard_normal() + +Introduction +------------ +The new infrastructure takes a different approach to producing random numbers +from the `~.RandomState` object. Random number generation is separated into +two components, a bit generator and a random generator. + +The `BitGenerator` has a limited set of responsibilities. It manages state +and provides functions to produce random doubles and random unsigned 32- and +64-bit values. + +The `random generator <Generator>` takes the +bit generator-provided stream and transforms them into more useful +distributions, e.g., simulated normal random values. This structure allows +alternative bit generators to be used with little code duplication. + +The `Generator` is the user-facing object that is nearly identical to +`.RandomState`. The canonical method to initialize a generator passes a +`~.PCG64` bit generator as the sole argument. + +.. code-block:: python + + from numpy.random import default_rng + rg = default_rng(12345) + rg.random() + +One can also instantiate `Generator` directly with a `BitGenerator` instance. +To use the older `~mt19937.MT19937` algorithm, one can instantiate it directly +and pass it to `Generator`. + +.. code-block:: python + + from numpy.random import Generator, MT19937 + rg = Generator(MT19937(12345)) + rg.random() + +What's New or Different +~~~~~~~~~~~~~~~~~~~~~~~ +.. warning:: + + The Box-Muller method used to produce NumPy's normals is no longer available + in `Generator`. It is not possible to reproduce the exact random + values using Generator for the normal distribution or any other + distribution that relies on the normal such as the `.RandomState.gamma` or + `.RandomState.standard_t`. If you require bitwise backward compatible + streams, use `.RandomState`. + +* The Generator's normal, exponential and gamma functions use 256-step Ziggurat + methods which are 2-10 times faster than NumPy's Box-Muller or inverse CDF + implementations. +* Optional ``dtype`` argument that accepts ``np.float32`` or ``np.float64`` + to produce either single or double prevision uniform random variables for + select distributions +* Optional ``out`` argument that allows existing arrays to be filled for + select distributions +* All BitGenerators can produce doubles, uint64s and uint32s via CTypes + (`~.PCG64.ctypes`) and CFFI (`~.PCG64.cffi`). This allows the bit generators + to be used in numba. +* The bit generators can be used in downstream projects via + :ref:`Cython <randomgen_cython>`. +* `~.Generator.integers` is now the canonical way to generate integer + random numbers from a discrete uniform distribution. The ``rand`` and + ``randn`` methods are only available through the legacy `~.RandomState`. + The ``endpoint`` keyword can be used to specify open or closed intervals. + This replaces both ``randint`` and the deprecated ``random_integers``. +* `~.Generator.random` is now the canonical way to generate floating-point + random numbers, which replaces `.RandomState.random_sample`, + `.RandomState.sample`, and `.RandomState.ranf`. This is consistent with + Python's `random.random`. +* All BitGenerators in numpy use `~SeedSequence` to convert seeds into + initialized states. + +See :ref:`new-or-different` for a complete list of improvements and +differences from the traditional ``Randomstate``. + +Parallel Generation +~~~~~~~~~~~~~~~~~~~ + +The included generators can be used in parallel, distributed applications in +one of three ways: + +* :ref:`seedsequence-spawn` +* :ref:`independent-streams` +* :ref:`parallel-jumped` + +Concepts +-------- +.. toctree:: + :maxdepth: 1 + + generator + Legacy Generator (RandomState) <legacy> + BitGenerators, SeedSequences <bit_generators/index> + +Features +-------- +.. toctree:: + :maxdepth: 2 + + Parallel Applications <parallel> + Multithreaded Generation <multithreading> + new-or-different + Comparing Performance <performance> + extending + +Original Source +~~~~~~~~~~~~~~~ + +This package was developed independently of NumPy and was integrated in version +1.17.0. The original repo is at https://github.com/bashtage/randomgen. diff --git a/doc/source/reference/random/legacy.rst b/doc/source/reference/random/legacy.rst new file mode 100644 index 000000000..413a42727 --- /dev/null +++ b/doc/source/reference/random/legacy.rst @@ -0,0 +1,123 @@ +.. currentmodule:: numpy.random + +.. _legacy: + +Legacy Random Generation +------------------------ +The `RandomState` provides access to +legacy generators. This generator is considered frozen and will have +no further improvements. It is guaranteed to produce the same values +as the final point release of NumPy v1.16. These all depend on Box-Muller +normals or inverse CDF exponentials or gammas. This class should only be used +if it is essential to have randoms that are identical to what +would have been produced by previous versions of NumPy. + +`RandomState` adds additional information +to the state which is required when using Box-Muller normals since these +are produced in pairs. It is important to use +`RandomState.get_state`, and not the underlying bit generators +`state`, when accessing the state so that these extra values are saved. + +Although we provide the `MT19937` BitGenerator for use independent of +`RandomState`, note that its default seeding uses `SeedSequence` +rather than the legacy seeding algorithm. `RandomState` will use the +legacy seeding algorithm. The methods to use the legacy seeding algorithm are +currently private as the main reason to use them is just to implement +`RandomState`. However, one can reset the state of `MT19937` +using the state of the `RandomState`: + +.. code-block:: python + + from numpy.random import MT19937 + from numpy.random import RandomState + + rs = RandomState(12345) + mt19937 = MT19937() + mt19937.state = rs.get_state() + rs2 = RandomState(mt19937) + + # Same output + rs.standard_normal() + rs2.standard_normal() + + rs.random() + rs2.random() + + rs.standard_exponential() + rs2.standard_exponential() + + +.. autoclass:: RandomState + :exclude-members: + +Seeding and State +================= + +.. autosummary:: + :toctree: generated/ + + ~RandomState.get_state + ~RandomState.set_state + ~RandomState.seed + +Simple random data +================== +.. autosummary:: + :toctree: generated/ + + ~RandomState.rand + ~RandomState.randn + ~RandomState.randint + ~RandomState.random_integers + ~RandomState.random_sample + ~RandomState.choice + ~RandomState.bytes + +Permutations +============ +.. autosummary:: + :toctree: generated/ + + ~RandomState.shuffle + ~RandomState.permutation + +Distributions +============= +.. autosummary:: + :toctree: generated/ + + ~RandomState.beta + ~RandomState.binomial + ~RandomState.chisquare + ~RandomState.dirichlet + ~RandomState.exponential + ~RandomState.f + ~RandomState.gamma + ~RandomState.geometric + ~RandomState.gumbel + ~RandomState.hypergeometric + ~RandomState.laplace + ~RandomState.logistic + ~RandomState.lognormal + ~RandomState.logseries + ~RandomState.multinomial + ~RandomState.multivariate_normal + ~RandomState.negative_binomial + ~RandomState.noncentral_chisquare + ~RandomState.noncentral_f + ~RandomState.normal + ~RandomState.pareto + ~RandomState.poisson + ~RandomState.power + ~RandomState.rayleigh + ~RandomState.standard_cauchy + ~RandomState.standard_exponential + ~RandomState.standard_gamma + ~RandomState.standard_normal + ~RandomState.standard_t + ~RandomState.triangular + ~RandomState.uniform + ~RandomState.vonmises + ~RandomState.wald + ~RandomState.weibull + ~RandomState.zipf diff --git a/doc/source/reference/random/multithreading.rst b/doc/source/reference/random/multithreading.rst new file mode 100644 index 000000000..6883d3672 --- /dev/null +++ b/doc/source/reference/random/multithreading.rst @@ -0,0 +1,108 @@ +Multithreaded Generation +======================== + +The four core distributions (:meth:`~.Generator.random`, +:meth:`~.Generator.standard_normal`, :meth:`~.Generator.standard_exponential`, +and :meth:`~.Generator.standard_gamma`) all allow existing arrays to be filled +using the ``out`` keyword argument. Existing arrays need to be contiguous and +well-behaved (writable and aligned). Under normal circumstances, arrays +created using the common constructors such as :meth:`numpy.empty` will satisfy +these requirements. + +This example makes use of Python 3 :mod:`concurrent.futures` to fill an array +using multiple threads. Threads are long-lived so that repeated calls do not +require any additional overheads from thread creation. The underlying +BitGenerator is `PCG64` which is fast, has a long period and supports +using `PCG64.jumped` to return a new generator while advancing the +state. The random numbers generated are reproducible in the sense that the same +seed will produce the same outputs. + +.. code-block:: ipython + + from numpy.random import Generator, PCG64 + import multiprocessing + import concurrent.futures + import numpy as np + + class MultithreadedRNG(object): + def __init__(self, n, seed=None, threads=None): + rg = PCG64(seed) + if threads is None: + threads = multiprocessing.cpu_count() + self.threads = threads + + self._random_generators = [rg] + last_rg = rg + for _ in range(0, threads-1): + new_rg = last_rg.jumped() + self._random_generators.append(new_rg) + last_rg = new_rg + + self.n = n + self.executor = concurrent.futures.ThreadPoolExecutor(threads) + self.values = np.empty(n) + self.step = np.ceil(n / threads).astype(np.int) + + def fill(self): + def _fill(random_state, out, first, last): + random_state.standard_normal(out=out[first:last]) + + futures = {} + for i in range(self.threads): + args = (_fill, + self._random_generators[i], + self.values, + i * self.step, + (i + 1) * self.step) + futures[self.executor.submit(*args)] = i + concurrent.futures.wait(futures) + + def __del__(self): + self.executor.shutdown(False) + + +The multithreaded random number generator can be used to fill an array. +The ``values`` attributes shows the zero-value before the fill and the +random value after. + +.. code-block:: ipython + + In [2]: mrng = MultithreadedRNG(10000000, seed=0) + ...: print(mrng.values[-1]) + 0.0 + + In [3]: mrng.fill() + ...: print(mrng.values[-1]) + 3.296046120254392 + +The time required to produce using multiple threads can be compared to +the time required to generate using a single thread. + +.. code-block:: ipython + + In [4]: print(mrng.threads) + ...: %timeit mrng.fill() + + 4 + 32.8 ms ± 2.71 ms per loop (mean ± std. dev. of 7 runs, 10 loops each) + +The single threaded call directly uses the BitGenerator. + +.. code-block:: ipython + + In [5]: values = np.empty(10000000) + ...: rg = Generator(PCG64()) + ...: %timeit rg.standard_normal(out=values) + + 99.6 ms ± 222 µs per loop (mean ± std. dev. of 7 runs, 10 loops each) + +The gains are substantial and the scaling is reasonable even for large that +are only moderately large. The gains are even larger when compared to a call +that does not use an existing array due to array creation overhead. + +.. code-block:: ipython + + In [6]: rg = Generator(PCG64()) + ...: %timeit rg.standard_normal(10000000) + + 125 ms ± 309 µs per loop (mean ± std. dev. of 7 runs, 10 loops each) diff --git a/doc/source/reference/random/new-or-different.rst b/doc/source/reference/random/new-or-different.rst new file mode 100644 index 000000000..c8815f98f --- /dev/null +++ b/doc/source/reference/random/new-or-different.rst @@ -0,0 +1,115 @@ +.. _new-or-different: + +.. currentmodule:: numpy.random + +What's New or Different +----------------------- + +.. warning:: + + The Box-Muller method used to produce NumPy's normals is no longer available + in `Generator`. It is not possible to reproduce the exact random + values using ``Generator`` for the normal distribution or any other + distribution that relies on the normal such as the `gamma` or + `standard_t`. If you require bitwise backward compatible + streams, use `RandomState`. + +Quick comparison of legacy `mtrand <legacy>`_ to the new `Generator` + +================== ==================== ============= +Feature Older Equivalent Notes +------------------ -------------------- ------------- +`~.Generator` `~.RandomState` ``Generator`` requires a stream + source, called a `BitGenerator + <bit_generators>` A number of these + are provided. ``RandomState`` uses + the Mersenne Twister `~.MT19937` by + default, but can also be instantiated + with any BitGenerator. +------------------ -------------------- ------------- +``random`` ``random_sample``, Access the values in a BitGenerator, + ``rand`` convert them to ``float64`` in the + interval ``[0.0.,`` `` 1.0)``. + In addition to the ``size`` kwarg, now + supports ``dtype='d'`` or ``dtype='f'``, + and an ``out`` kwarg to fill a user- + supplied array. + + Many other distributions are also + supported. +------------------ -------------------- ------------- +``integers`` ``randint``, Use the ``endpoint`` kwarg to adjust + ``random_integers`` the inclusion or exclution of the + ``high`` interval endpoint +================== ==================== ============= + +And in more detail: + +* Simulate from the complex normal distribution + (`~.Generator.complex_normal`) +* The normal, exponential and gamma generators use 256-step Ziggurat + methods which are 2-10 times faster than NumPy's default implementation in + `~.Generator.standard_normal`, `~.Generator.standard_exponential` or + `~.Generator.standard_gamma`. +* `~.Generator.integers` is now the canonical way to generate integer + random numbers from a discrete uniform distribution. The ``rand`` and + ``randn`` methods are only available through the legacy `~.RandomState`. + This replaces both ``randint`` and the deprecated ``random_integers``. +* The Box-Muller method used to produce NumPy's normals is no longer available. +* All bit generators can produce doubles, uint64s and + uint32s via CTypes (`~PCG64.ctypes`) and CFFI (`~PCG64.cffi`). + This allows these bit generators to be used in numba. +* The bit generators can be used in downstream projects via + Cython. + + +.. ipython:: python + + from numpy.random import Generator, PCG64 + import numpy.random + rg = Generator(PCG64()) + %timeit rg.standard_normal(100000) + %timeit numpy.random.standard_normal(100000) + +.. ipython:: python + + %timeit rg.standard_exponential(100000) + %timeit numpy.random.standard_exponential(100000) + +.. ipython:: python + + %timeit rg.standard_gamma(3.0, 100000) + %timeit numpy.random.standard_gamma(3.0, 100000) + +* Optional ``dtype`` argument that accepts ``np.float32`` or ``np.float64`` + to produce either single or double prevision uniform random variables for + select distributions + + * Uniforms (`~.Generator.random` and `~.Generator.integers`) + * Normals (`~.Generator.standard_normal`) + * Standard Gammas (`~.Generator.standard_gamma`) + * Standard Exponentials (`~.Generator.standard_exponential`) + +.. ipython:: python + + rg = Generator(PCG64(0)) + rg.random(3, dtype='d') + rg.random(3, dtype='f') + +* Optional ``out`` argument that allows existing arrays to be filled for + select distributions + + * Uniforms (`~.Generator.random`) + * Normals (`~.Generator.standard_normal`) + * Standard Gammas (`~.Generator.standard_gamma`) + * Standard Exponentials (`~.Generator.standard_exponential`) + + This allows multithreading to fill large arrays in chunks using suitable + BitGenerators in parallel. + +.. ipython:: python + + existing = np.zeros(4) + rg.random(out=existing[:2]) + print(existing) + diff --git a/doc/source/reference/random/parallel.rst b/doc/source/reference/random/parallel.rst new file mode 100644 index 000000000..2f79f22d8 --- /dev/null +++ b/doc/source/reference/random/parallel.rst @@ -0,0 +1,193 @@ +Parallel Random Number Generation +================================= + +There are three strategies implemented that can be used to produce +repeatable pseudo-random numbers across multiple processes (local +or distributed). + +.. currentmodule:: numpy.random + +.. _seedsequence-spawn: + +`~SeedSequence` spawning +------------------------ + +`~SeedSequence` `implements an algorithm`_ to process a user-provided seed, +typically as an integer of some size, and to convert it into an initial state for +a `~BitGenerator`. It uses hashing techniques to ensure that low-quality seeds +are turned into high quality initial states (at least, with very high +probability). + +For example, `~mt19937.MT19937` has a state consisting of 624 +`uint32` integers. A naive way to take a 32-bit integer seed would be to just set +the last element of the state to the 32-bit seed and leave the rest 0s. This is +a valid state for `~mt19937.MT19937`, but not a good one. The Mersenne Twister +algorithm `suffers if there are too many 0s`_. Similarly, two adjacent 32-bit +integer seeds (i.e. ``12345`` and ``12346``) would produce very similar +streams. + +`~SeedSequence` avoids these problems by using successions of integer hashes +with good `avalanche properties`_ to ensure that flipping any bit in the input +input has about a 50% chance of flipping any bit in the output. Two input seeds +that are very close to each other will produce initial states that are very far +from each other (with very high probability). It is also constructed in such +a way that you can provide arbitrary-sized integers or lists of integers. +`~SeedSequence` will take all of the bits that you provide and mix them +together to produce however many bits the consuming `~BitGenerator` needs to +initialize itself. + +These properties together mean that we can safely mix together the usual +user-provided seed with simple incrementing counters to get `~BitGenerator` +states that are (to very high probability) independent of each other. We can +wrap this together into an API that is easy to use and difficult to misuse. + +.. code-block:: python + + from numpy.random import SeedSequence, default_rng + + ss = SeedSequence(12345) + + # Spawn off 10 child SeedSequences to pass to child processes. + child_seeds = ss.spawn(10) + streams = [default_rng(s) for s in child_seeds] + +.. end_block + +Child `~SeedSequence` objects can also spawn to make grandchildren, and so on. +Each `~SeedSequence` has its position in the tree of spawned `~SeedSequence` +objects mixed in with the user-provided seed to generate independent (with very +high probability) streams. + +.. code-block:: python + + grandchildren = child_seeds[0].spawn(4) + grand_streams = [default_rng(s) for s in grandchildren] + +.. end_block + +This feature lets you make local decisions about when and how to split up +streams without coordination between processes. You do not have to preallocate +space to avoid overlapping or request streams from a common global service. This +general "tree-hashing" scheme is `not unique to numpy`_ but not yet widespread. +Python has increasingly-flexible mechanisms for parallelization available, and +this scheme fits in very well with that kind of use. + +Using this scheme, an upper bound on the probability of a collision can be +estimated if one knows the number of streams that you derive. `~SeedSequence` +hashes its inputs, both the seed and the spawn-tree-path, down to a 128-bit +pool by default. The probability that there is a collision in +that pool, pessimistically-estimated ([1]_), will be about :math:`n^2*2^{-128}` where +`n` is the number of streams spawned. If a program uses an aggressive million +streams, about :math:`2^{20}`, then the probability that at least one pair of +them are identical is about :math:`2^{-88}`, which is in solidly-ignorable +territory ([2]_). + +.. [1] The algorithm is carefully designed to eliminate a number of possible + ways to collide. For example, if one only does one level of spawning, it + is guaranteed that all states will be unique. But it's easier to + estimate the naive upper bound on a napkin and take comfort knowing + that the probability is actually lower. + +.. [2] In this calculation, we can ignore the amount of numbers drawn from each + stream. Each of the PRNGs we provide has some extra protection built in + that avoids overlaps if the `~SeedSequence` pools differ in the + slightest bit. `~pcg64.PCG64` has :math:`2^{127}` separate cycles + determined by the seed in addition to the position in the + :math:`2^{128}` long period for each cycle, so one has to both get on or + near the same cycle *and* seed a nearby position in the cycle. + `~philox.Philox` has completely independent cycles determined by the seed. + `~sfc64.SFC64` incorporates a 64-bit counter so every unique seed is at + least :math:`2^{64}` iterations away from any other seed. And + finally, `~mt19937.MT19937` has just an unimaginably huge period. Getting + a collision internal to `~SeedSequence` is the way a failure would be + observed. + +.. _`implements an algorithm`: http://www.pcg-random.org/posts/developing-a-seed_seq-alternative.html +.. _`suffers if there are too many 0s`: http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html +.. _`avalanche properties`: https://en.wikipedia.org/wiki/Avalanche_effect +.. _`not unique to numpy`: https://www.iro.umontreal.ca/~lecuyer/myftp/papers/parallel-rng-imacs.pdf + + +.. _independent-streams: + +Independent Streams +------------------- + +:class:`~philox.Philox` is a counter-based RNG based which generates values by +encrypting an incrementing counter using weak cryptographic primitives. The +seed determines the key that is used for the encryption. Unique keys create +unique, independent streams. :class:`~philox.Philox` lets you bypass the +seeding algorithm to directly set the 128-bit key. Similar, but different, keys +will still create independent streams. + +.. code-block:: python + + import secrets + from numpy.random import Philox + + # 128-bit number as a seed + root_seed = secrets.getrandbits(128) + streams = [Philox(key=root_seed + stream_id) for stream_id in range(10)] + +.. end_block + +This scheme does require that you avoid reusing stream IDs. This may require +coordination between the parallel processes. + + +.. _parallel-jumped: + +Jumping the BitGenerator state +------------------------------ + +``jumped`` advances the state of the BitGenerator *as-if* a large number of +random numbers have been drawn, and returns a new instance with this state. +The specific number of draws varies by BitGenerator, and ranges from +:math:`2^{64}` to :math:`2^{128}`. Additionally, the *as-if* draws also depend +on the size of the default random number produced by the specific BitGenerator. +The BitGenerators that support ``jumped``, along with the period of the +BitGenerator, the size of the jump and the bits in the default unsigned random +are listed below. + ++-----------------+-------------------------+-------------------------+-------------------------+ +| BitGenerator | Period | Jump Size | Bits | ++=================+=========================+=========================+=========================+ +| MT19937 | :math:`2^{19937}` | :math:`2^{128}` | 32 | ++-----------------+-------------------------+-------------------------+-------------------------+ +| PCG64 | :math:`2^{128}` | :math:`~2^{127}` ([3]_) | 64 | ++-----------------+-------------------------+-------------------------+-------------------------+ +| Philox | :math:`2^{256}` | :math:`2^{128}` | 64 | ++-----------------+-------------------------+-------------------------+-------------------------+ + +.. [3] The jump size is :math:`(\phi-1)*2^{128}` where :math:`\phi` is the + golden ratio. As the jumps wrap around the period, the actual distances + between neighboring streams will slowly grow smaller than the jump size, + but using the golden ratio this way is a classic method of constructing + a low-discrepancy sequence that spreads out the states around the period + optimally. You will not be able to jump enough to make those distances + small enough to overlap in your lifetime. + +``jumped`` can be used to produce long blocks which should be long enough to not +overlap. + +.. code-block:: python + + import secrets + from numpy.random import PCG64 + + seed = secrets.getrandbits(128) + blocked_rng = [] + rng = PCG64(seed) + for i in range(10): + blocked_rng.append(rng.jumped(i)) + +.. end_block + +When using ``jumped``, one does have to take care not to jump to a stream that +was already used. In the above example, one could not later use +``blocked_rng[0].jumped()`` as it would overlap with ``blocked_rng[1]``. Like +with the independent streams, if the main process here wants to split off 10 +more streams by jumping, then it needs to start with ``range(10, 20)``, +otherwise it would recreate the same streams. On the other hand, if you +carefully construct the streams, then you are guaranteed to have streams that +do not overlap. diff --git a/doc/source/reference/random/performance.py b/doc/source/reference/random/performance.py new file mode 100644 index 000000000..28a42eb0d --- /dev/null +++ b/doc/source/reference/random/performance.py @@ -0,0 +1,87 @@ +from collections import OrderedDict +from timeit import repeat + +import pandas as pd + +import numpy as np +from numpy.random import MT19937, PCG64, Philox, SFC64 + +PRNGS = [MT19937, PCG64, Philox, SFC64] + +funcs = OrderedDict() +integers = 'integers(0, 2**{bits},size=1000000, dtype="uint{bits}")' +funcs['32-bit Unsigned Ints'] = integers.format(bits=32) +funcs['64-bit Unsigned Ints'] = integers.format(bits=64) +funcs['Uniforms'] = 'random(size=1000000)' +funcs['Normals'] = 'standard_normal(size=1000000)' +funcs['Exponentials'] = 'standard_exponential(size=1000000)' +funcs['Gammas'] = 'standard_gamma(3.0,size=1000000)' +funcs['Binomials'] = 'binomial(9, .1, size=1000000)' +funcs['Laplaces'] = 'laplace(size=1000000)' +funcs['Poissons'] = 'poisson(3.0, size=1000000)' + +setup = """ +from numpy.random import {prng}, Generator +rg = Generator({prng}()) +""" + +test = "rg.{func}" +table = OrderedDict() +for prng in PRNGS: + print(prng) + col = OrderedDict() + for key in funcs: + t = repeat(test.format(func=funcs[key]), + setup.format(prng=prng().__class__.__name__), + number=1, repeat=3) + col[key] = 1000 * min(t) + col = pd.Series(col) + table[prng().__class__.__name__] = col + +npfuncs = OrderedDict() +npfuncs.update(funcs) +npfuncs['32-bit Unsigned Ints'] = 'randint(2**32,dtype="uint32",size=1000000)' +npfuncs['64-bit Unsigned Ints'] = 'randint(2**64,dtype="uint64",size=1000000)' +setup = """ +from numpy.random import RandomState +rg = RandomState() +""" +col = {} +for key in npfuncs: + t = repeat(test.format(func=npfuncs[key]), + setup.format(prng=prng().__class__.__name__), + number=1, repeat=3) + col[key] = 1000 * min(t) +table['RandomState'] = pd.Series(col) + +columns = ['MT19937','PCG64','Philox','SFC64', 'RandomState'] +table = pd.DataFrame(table) +order = np.log(table).mean().sort_values().index +table = table.T +table = table.reindex(columns) +table = table.T +table = table.reindex([k for k in funcs], axis=0) +print(table.to_csv(float_format='%0.1f')) + + +rel = table.loc[:, ['RandomState']].values @ np.ones( + (1, table.shape[1])) / table +rel.pop('RandomState') +rel = rel.T +rel['Overall'] = np.exp(np.log(rel).mean(1)) +rel *= 100 +rel = np.round(rel) +rel = rel.T +print(rel.to_csv(float_format='%0d')) + +# Cross-platform table +rows = ['32-bit Unsigned Ints','64-bit Unsigned Ints','Uniforms','Normals','Exponentials'] +xplat = rel.reindex(rows, axis=0) +xplat = 100 * (xplat / xplat.MT19937.values[:,None]) +overall = np.exp(np.log(xplat).mean(0)) +xplat = xplat.T.copy() +xplat['Overall']=overall +print(xplat.T.round(1)) + + + diff --git a/doc/source/reference/random/performance.rst b/doc/source/reference/random/performance.rst new file mode 100644 index 000000000..2d5fca496 --- /dev/null +++ b/doc/source/reference/random/performance.rst @@ -0,0 +1,153 @@ +Performance +----------- + +.. currentmodule:: numpy.random + +Recommendation +************** +The recommended generator for general use is :class:`~pcg64.PCG64`. It is +statistically high quality, full-featured, and fast on most platforms, but +somewhat slow when compiled for 32-bit processes. + +:class:`~philox.Philox` is fairly slow, but its statistical properties have +very high quality, and it is easy to get assuredly-independent stream by using +unique keys. If that is the style you wish to use for parallel streams, or you +are porting from another system that uses that style, then +:class:`~philox.Philox` is your choice. + +:class:`~sfc64.SFC64` is statistically high quality and very fast. However, it +lacks jumpability. If you are not using that capability and want lots of speed, +even on 32-bit processes, this is your choice. + +:class:`~mt19937.MT19937` `fails some statistical tests`_ and is not especially +fast compared to modern PRNGs. For these reasons, we mostly do not recommend +using it on its own, only through the legacy `~.RandomState` for +reproducing old results. That said, it has a very long history as a default in +many systems. + +.. _`fails some statistical tests`: https://www.iro.umontreal.ca/~lecuyer/myftp/papers/testu01.pdf + +Timings +******* + +The timings below are the time in ns to produce 1 random value from a +specific distribution. The original :class:`~mt19937.MT19937` generator is +much slower since it requires 2 32-bit values to equal the output of the +faster generators. + +Integer performance has a similar ordering. + +The pattern is similar for other, more complex generators. The normal +performance of the legacy :class:`~.RandomState` generator is much +lower than the other since it uses the Box-Muller transformation rather +than the Ziggurat generator. The performance gap for Exponentials is also +large due to the cost of computing the log function to invert the CDF. +The column labeled MT19973 is used the same 32-bit generator as +:class:`~.RandomState` but produces random values using +:class:`~Generator`. + +.. csv-table:: + :header: ,MT19937,PCG64,Philox,SFC64,RandomState + :widths: 14,14,14,14,14,14 + + 32-bit Unsigned Ints,3.2,2.7,4.9,2.7,3.2 + 64-bit Unsigned Ints,5.6,3.7,6.3,2.9,5.7 + Uniforms,7.3,4.1,8.1,3.1,7.3 + Normals,13.1,10.2,13.5,7.8,34.6 + Exponentials,7.9,5.4,8.5,4.1,40.3 + Gammas,34.8,28.0,34.7,25.1,58.1 + Binomials,25.0,21.4,26.1,19.5,25.2 + Laplaces,45.1,40.7,45.5,38.1,45.6 + Poissons,67.6,52.4,69.2,46.4,78.1 + +The next table presents the performance in percentage relative to values +generated by the legacy generator, `RandomState(MT19937())`. The overall +performance was computed using a geometric mean. + +.. csv-table:: + :header: ,MT19937,PCG64,Philox,SFC64 + :widths: 14,14,14,14,14 + + 32-bit Unsigned Ints,101,121,67,121 + 64-bit Unsigned Ints,102,156,91,199 + Uniforms,100,179,90,235 + Normals,263,338,257,443 + Exponentials,507,752,474,985 + Gammas,167,207,167,231 + Binomials,101,118,96,129 + Laplaces,101,112,100,120 + Poissons,116,149,113,168 + Overall,144,192,132,225 + +.. note:: + + All timings were taken using Linux on a i5-3570 processor. + +Performance on different Operating Systems +****************************************** +Performance differs across platforms due to compiler and hardware availability +(e.g., register width) differences. The default bit generator has been chosen +to perform well on 64-bit platforms. Performance on 32-bit operating systems +is very different. + +The values reported are normalized relative to the speed of MT19937 in +each table. A value of 100 indicates that the performance matches the MT19937. +Higher values indicate improved performance. These values cannot be compared +across tables. + +64-bit Linux +~~~~~~~~~~~~ + +=================== ========= ======= ======== ======= +Distribution MT19937 PCG64 Philox SFC64 +=================== ========= ======= ======== ======= +32-bit Unsigned Int 100 119.8 67.7 120.2 +64-bit Unsigned Int 100 152.9 90.8 213.3 +Uniforms 100 179.0 87.0 232.0 +Normals 100 128.5 99.2 167.8 +Exponentials 100 148.3 93.0 189.3 +**Overall** 100 144.3 86.8 180.0 +=================== ========= ======= ======== ======= + + +64-bit Windows +~~~~~~~~~~~~~~ +The relative performance on 64-bit Linux and 64-bit Windows is broadly similar. + + +=================== ========= ======= ======== ======= +Distribution MT19937 PCG64 Philox SFC64 +=================== ========= ======= ======== ======= +32-bit Unsigned Int 100 129.1 35.0 135.0 +64-bit Unsigned Int 100 146.9 35.7 176.5 +Uniforms 100 165.0 37.0 192.0 +Normals 100 128.5 48.5 158.0 +Exponentials 100 151.6 39.0 172.8 +**Overall** 100 143.6 38.7 165.7 +=================== ========= ======= ======== ======= + + +32-bit Windows +~~~~~~~~~~~~~~ + +The performance of 64-bit generators on 32-bit Windows is much lower than on 64-bit +operating systems due to register width. MT19937, the generator that has been +in NumPy since 2005, operates on 32-bit integers. + +=================== ========= ======= ======== ======= +Distribution MT19937 PCG64 Philox SFC64 +=================== ========= ======= ======== ======= +32-bit Unsigned Int 100 30.5 21.1 77.9 +64-bit Unsigned Int 100 26.3 19.2 97.0 +Uniforms 100 28.0 23.0 106.0 +Normals 100 40.1 31.3 112.6 +Exponentials 100 33.7 26.3 109.8 +**Overall** 100 31.4 23.8 99.8 +=================== ========= ======= ======== ======= + + +.. note:: + + Linux timings used Ubuntu 18.04 and GCC 7.4. Windows timings were made on + Windows 10 using Microsoft C/C++ Optimizing Compiler Version 19 (Visual + Studio 2015). All timings were produced on a i5-3570 processor. diff --git a/doc/source/reference/routines.char.rst b/doc/source/reference/routines.char.rst index 7413e3615..ed8393855 100644 --- a/doc/source/reference/routines.char.rst +++ b/doc/source/reference/routines.char.rst @@ -1,11 +1,13 @@ String operations ***************** -.. currentmodule:: numpy.core.defchararray +.. currentmodule:: numpy.char -This module provides a set of vectorized string operations for arrays -of type `numpy.string_` or `numpy.unicode_`. All of them are based on -the string methods in the Python standard library. +.. module:: numpy.char + +The `numpy.char` module provides a set of vectorized string +operations for arrays of type `numpy.string_` or `numpy.unicode_`. +All of them are based on the string methods in the Python standard library. String operations ----------------- @@ -20,6 +22,7 @@ String operations center decode encode + expandtabs join ljust lower @@ -55,6 +58,7 @@ comparison. less_equal greater less + compare_chararrays String information ------------------ @@ -63,9 +67,11 @@ String information :toctree: generated/ count + endswith find index isalpha + isalnum isdecimal isdigit islower @@ -76,6 +82,7 @@ String information rfind rindex startswith + str_len Convenience class ----------------- @@ -83,4 +90,6 @@ Convenience class .. autosummary:: :toctree: generated/ + array + asarray chararray diff --git a/doc/source/reference/routines.ctypeslib.rst b/doc/source/reference/routines.ctypeslib.rst index b04713b61..562638e9c 100644 --- a/doc/source/reference/routines.ctypeslib.rst +++ b/doc/source/reference/routines.ctypeslib.rst @@ -1,3 +1,5 @@ +.. module:: numpy.ctypeslib + *********************************************************** C-Types Foreign Function Interface (:mod:`numpy.ctypeslib`) *********************************************************** @@ -6,6 +8,7 @@ C-Types Foreign Function Interface (:mod:`numpy.ctypeslib`) .. autofunction:: as_array .. autofunction:: as_ctypes +.. autofunction:: as_ctypes_type .. autofunction:: ctypes_load_library .. autofunction:: load_library .. autofunction:: ndpointer diff --git a/doc/source/reference/routines.dtype.rst b/doc/source/reference/routines.dtype.rst index ec8d2981d..e9189ca07 100644 --- a/doc/source/reference/routines.dtype.rst +++ b/doc/source/reference/routines.dtype.rst @@ -17,11 +17,9 @@ Data type routines Creating data types ------------------- - .. autosummary:: :toctree: generated/ - dtype format_parser @@ -53,3 +51,4 @@ Miscellaneous typename sctype2char mintypecode + maximum_sctype diff --git a/doc/source/reference/routines.linalg.rst b/doc/source/reference/routines.linalg.rst index 0520df413..d42e77ad8 100644 --- a/doc/source/reference/routines.linalg.rst +++ b/doc/source/reference/routines.linalg.rst @@ -1,8 +1,23 @@ .. _routines.linalg: +.. module:: numpy.linalg + Linear algebra (:mod:`numpy.linalg`) ************************************ +The NumPy linear algebra functions rely on BLAS and LAPACK to provide efficient +low level implementations of standard linear algebra algorithms. Those +libraries may be provided by NumPy itself using C versions of a subset of their +reference implementations but, when possible, highly optimized libraries that +take advantage of specialized processor functionality are preferred. Examples +of such libraries are OpenBLAS_, MKL (TM), and ATLAS. Because those libraries +are multithreaded and processor dependent, environmental variables and external +packages such as threadpoolctl_ may be needed to control the number of threads +or specify the processor architecture. + +.. _OpenBLAS: https://www.openblas.net/ +.. _threadpoolctl: https://github.com/joblib/threadpoolctl + .. currentmodule:: numpy Matrix and vector products diff --git a/doc/source/reference/routines.ma.rst b/doc/source/reference/routines.ma.rst index 15f2ba0a4..5b2098c7a 100644 --- a/doc/source/reference/routines.ma.rst +++ b/doc/source/reference/routines.ma.rst @@ -68,9 +68,6 @@ Inspecting the array ma.is_masked ma.is_mask - ma.MaskedArray.data - ma.MaskedArray.mask - ma.MaskedArray.recordmask ma.MaskedArray.all ma.MaskedArray.any @@ -80,6 +77,12 @@ Inspecting the array ma.size +.. autosummary:: + + ma.MaskedArray.data + ma.MaskedArray.mask + ma.MaskedArray.recordmask + _____ Manipulating a MaskedArray @@ -261,17 +264,6 @@ Conversion operations ma.MaskedArray.tobytes -Pickling and unpickling -~~~~~~~~~~~~~~~~~~~~~~~ -.. autosummary:: - :toctree: generated/ - - ma.dump - ma.dumps - ma.load - ma.loads - - Filling a masked array ~~~~~~~~~~~~~~~~~~~~~~ .. autosummary:: @@ -285,8 +277,10 @@ Filling a masked array ma.MaskedArray.get_fill_value ma.MaskedArray.set_fill_value - ma.MaskedArray.fill_value +.. autosummary:: + + ma.MaskedArray.fill_value _____ diff --git a/doc/source/reference/routines.math.rst b/doc/source/reference/routines.math.rst index 821363987..3c2f96830 100644 --- a/doc/source/reference/routines.math.rst +++ b/doc/source/reference/routines.math.rst @@ -141,6 +141,7 @@ Handling complex numbers real imag conj + conjugate Miscellaneous diff --git a/doc/source/reference/routines.matlib.rst b/doc/source/reference/routines.matlib.rst index a35eaec78..c7f675425 100644 --- a/doc/source/reference/routines.matlib.rst +++ b/doc/source/reference/routines.matlib.rst @@ -1,3 +1,5 @@ +.. module:: numpy.matlib + Matrix library (:mod:`numpy.matlib`) ************************************ diff --git a/doc/source/reference/routines.other.rst b/doc/source/reference/routines.other.rst index 45b9ac3d9..28c9a1ad1 100644 --- a/doc/source/reference/routines.other.rst +++ b/doc/source/reference/routines.other.rst @@ -5,14 +5,6 @@ Miscellaneous routines .. currentmodule:: numpy -Buffer objects --------------- -.. autosummary:: - :toctree: generated/ - - getbuffer - newbuffer - Performance tuning ------------------ .. autosummary:: @@ -29,6 +21,7 @@ Memory ranges shares_memory may_share_memory + byte_bounds Array mixins ------------ @@ -43,3 +36,21 @@ NumPy version comparison :toctree: generated/ lib.NumpyVersion + +Utility +------- + +.. autosummary:: + :toctree: generated/ + + get_include + deprecate + deprecate_with_doc + +Matlab-like Functions +--------------------- +.. autosummary:: + :toctree: generated/ + + who + disp
\ No newline at end of file diff --git a/doc/source/reference/routines.polynomials.package.rst b/doc/source/reference/routines.polynomials.package.rst index 61cb57fbb..7e40d9f00 100644 --- a/doc/source/reference/routines.polynomials.package.rst +++ b/doc/source/reference/routines.polynomials.package.rst @@ -1,3 +1,5 @@ +.. module:: numpy.polynomial + Polynomial Package ================== diff --git a/doc/source/reference/routines.polynomials.polynomial.rst b/doc/source/reference/routines.polynomials.polynomial.rst index 8194ca867..365c8da98 100644 --- a/doc/source/reference/routines.polynomials.polynomial.rst +++ b/doc/source/reference/routines.polynomials.polynomial.rst @@ -1,3 +1,5 @@ +.. module:: numpy.polynomial.polynomial + Polynomial Module (:mod:`numpy.polynomial.polynomial`) ====================================================== diff --git a/doc/source/reference/routines.random.rst b/doc/source/reference/routines.random.rst deleted file mode 100644 index c8b097d7d..000000000 --- a/doc/source/reference/routines.random.rst +++ /dev/null @@ -1,81 +0,0 @@ -.. _routines.random: - -Random sampling (:mod:`numpy.random`) -************************************* - -.. currentmodule:: numpy.random - -Simple random data -================== -.. autosummary:: - :toctree: generated/ - - rand - randn - randint - random_integers - random_sample - random - ranf - sample - choice - bytes - -Permutations -============ -.. autosummary:: - :toctree: generated/ - - shuffle - permutation - -Distributions -============= -.. autosummary:: - :toctree: generated/ - - beta - binomial - chisquare - dirichlet - exponential - f - gamma - geometric - gumbel - hypergeometric - laplace - logistic - lognormal - logseries - multinomial - multivariate_normal - negative_binomial - noncentral_chisquare - noncentral_f - normal - pareto - poisson - power - rayleigh - standard_cauchy - standard_exponential - standard_gamma - standard_normal - standard_t - triangular - uniform - vonmises - wald - weibull - zipf - -Random generator -================ -.. autosummary:: - :toctree: generated/ - - RandomState - seed - get_state - set_state diff --git a/doc/source/reference/routines.rst b/doc/source/reference/routines.rst index a9e80480b..7a9b97d77 100644 --- a/doc/source/reference/routines.rst +++ b/doc/source/reference/routines.rst @@ -41,7 +41,7 @@ indentation. routines.other routines.padding routines.polynomials - routines.random + random/index routines.set routines.sort routines.statistics diff --git a/doc/source/reference/routines.testing.rst b/doc/source/reference/routines.testing.rst index 5a52a40d6..98ce3f377 100644 --- a/doc/source/reference/routines.testing.rst +++ b/doc/source/reference/routines.testing.rst @@ -1,4 +1,4 @@ -.. _numpy-testing: +.. module:: numpy.testing Test Support (:mod:`numpy.testing`) =================================== @@ -37,11 +37,11 @@ Decorators .. autosummary:: :toctree: generated/ - decorators.deprecated - decorators.knownfailureif - decorators.setastest - decorators.skipif - decorators.slow + dec.deprecated + dec.knownfailureif + dec.setastest + dec.skipif + dec.slow decorate_methods Test Running diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst index 3cc956887..3a3b67632 100644 --- a/doc/source/reference/ufuncs.rst +++ b/doc/source/reference/ufuncs.rst @@ -16,7 +16,7 @@ A universal function (or :term:`ufunc` for short) is a function that operates on :class:`ndarrays <ndarray>` in an element-by-element fashion, supporting :ref:`array broadcasting <ufuncs.broadcasting>`, :ref:`type casting <ufuncs.casting>`, and several other standard features. That -is, a ufunc is a ":term:`vectorized`" wrapper for a function that +is, a ufunc is a ":term:`vectorized <vectorization>`" wrapper for a function that takes a fixed number of specific inputs and produces a fixed number of specific outputs. @@ -59,7 +59,7 @@ understood by four rules: entry in that dimension will be used for all calculations along that dimension. In other words, the stepping machinery of the :term:`ufunc` will simply not step along that dimension (the - :term:`stride` will be 0 for that dimension). + :ref:`stride <memory-layout>` will be 0 for that dimension). Broadcasting is used throughout NumPy to decide how to handle disparately shaped arrays; for example, all arithmetic operations (``+``, @@ -70,7 +70,7 @@ arrays before operation. .. index:: broadcastable -A set of arrays is called ":term:`broadcastable`" to the same shape if +A set of arrays is called "broadcastable" to the same shape if the above rules produce a valid result, *i.e.*, one of the following is true: @@ -118,7 +118,7 @@ all output arrays will be passed to the :obj:`~class.__array_prepare__` and the highest :obj:`~class.__array_priority__` of any other input to the universal function. The default :obj:`~class.__array_priority__` of the ndarray is 0.0, and the default :obj:`~class.__array_priority__` of a subtype -is 1.0. Matrices have :obj:`~class.__array_priority__` equal to 10.0. +is 0.0. Matrices have :obj:`~class.__array_priority__` equal to 10.0. All ufuncs can also take output arguments. If necessary, output will be cast to the data-type(s) of the provided output array(s). If a class @@ -228,46 +228,47 @@ can generate this table for your system with the code given in the Figure. .. admonition:: Figure - Code segment showing the "can cast safely" table for a 32-bit system. + Code segment showing the "can cast safely" table for a 64-bit system. + Generally the output depends on the system; your system might result in + a different table. + >>> mark = {False: ' -', True: ' Y'} >>> def print_table(ntypes): - ... print 'X', - ... for char in ntypes: print char, - ... print + ... print('X ' + ' '.join(ntypes)) ... for row in ntypes: - ... print row, + ... print(row, end='') ... for col in ntypes: - ... print int(np.can_cast(row, col)), - ... print + ... print(mark[np.can_cast(row, col)], end='') + ... print() + ... >>> print_table(np.typecodes['All']) X ? b h i l q p B H I L Q P e f d g F D G S U V O M m - ? 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 - b 0 1 1 1 1 1 1 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 - h 0 0 1 1 1 1 1 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 0 0 - i 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - l 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - q 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - p 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - B 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 - H 0 0 0 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 1 1 1 1 1 0 0 - I 0 0 0 0 1 1 1 0 0 1 1 1 1 0 0 1 1 0 1 1 1 1 1 1 0 0 - L 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 1 1 0 1 1 1 1 1 1 0 0 - Q 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 1 1 0 1 1 1 1 1 1 0 0 - P 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 1 1 0 1 1 1 1 1 1 0 0 - e 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 - f 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 0 0 - d 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - g 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 1 1 1 1 0 0 - F 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 - D 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 - G 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 0 0 - S 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 - U 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 - V 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 - O 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 - M 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 - m 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 - + ? Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y + b - Y Y Y Y Y Y - - - - - - Y Y Y Y Y Y Y Y Y Y Y - Y + h - - Y Y Y Y Y - - - - - - - Y Y Y Y Y Y Y Y Y Y - Y + i - - - Y Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y + l - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y + q - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y + p - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y + B - - Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y + H - - - Y Y Y Y - Y Y Y Y Y - Y Y Y Y Y Y Y Y Y Y - Y + I - - - - Y Y Y - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - Y + L - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - Y + Q - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - Y + P - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - Y + e - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y Y - - + f - - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y - - + d - - - - - - - - - - - - - - - Y Y - Y Y Y Y Y Y - - + g - - - - - - - - - - - - - - - - Y - - Y Y Y Y Y - - + F - - - - - - - - - - - - - - - - - Y Y Y Y Y Y Y - - + D - - - - - - - - - - - - - - - - - - Y Y Y Y Y Y - - + G - - - - - - - - - - - - - - - - - - - Y Y Y Y Y - - + S - - - - - - - - - - - - - - - - - - - - Y Y Y Y - - + U - - - - - - - - - - - - - - - - - - - - - Y Y Y - - + V - - - - - - - - - - - - - - - - - - - - - - Y Y - - + O - - - - - - - - - - - - - - - - - - - - - - Y Y - - + M - - - - - - - - - - - - - - - - - - - - - - Y Y Y - + m - - - - - - - - - - - - - - - - - - - - - - Y Y - Y You should note that, while included in the table for completeness, the 'S', 'U', and 'V' types cannot be operated on by ufuncs. Also, @@ -586,6 +587,7 @@ Math operations sign heaviside conj + conjugate exp exp2 log diff --git a/doc/source/release.rst b/doc/source/release.rst index 7f0197f3f..fb4e2b14d 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -2,44 +2,58 @@ Release Notes ************* -.. include:: ../release/1.16.0-notes.rst -.. include:: ../release/1.15.2-notes.rst -.. include:: ../release/1.15.1-notes.rst -.. include:: ../release/1.15.0-notes.rst -.. include:: ../release/1.14.6-notes.rst -.. include:: ../release/1.14.5-notes.rst -.. include:: ../release/1.14.4-notes.rst -.. include:: ../release/1.14.3-notes.rst -.. include:: ../release/1.14.2-notes.rst -.. include:: ../release/1.14.1-notes.rst -.. include:: ../release/1.14.0-notes.rst -.. include:: ../release/1.13.3-notes.rst -.. include:: ../release/1.13.2-notes.rst -.. include:: ../release/1.13.1-notes.rst -.. include:: ../release/1.13.0-notes.rst -.. include:: ../release/1.12.1-notes.rst -.. include:: ../release/1.12.0-notes.rst -.. include:: ../release/1.11.3-notes.rst -.. include:: ../release/1.11.2-notes.rst -.. include:: ../release/1.11.1-notes.rst -.. include:: ../release/1.11.0-notes.rst -.. include:: ../release/1.10.4-notes.rst -.. include:: ../release/1.10.3-notes.rst -.. include:: ../release/1.10.2-notes.rst -.. include:: ../release/1.10.1-notes.rst -.. include:: ../release/1.10.0-notes.rst -.. include:: ../release/1.9.2-notes.rst -.. include:: ../release/1.9.1-notes.rst -.. include:: ../release/1.9.0-notes.rst -.. include:: ../release/1.8.2-notes.rst -.. include:: ../release/1.8.1-notes.rst -.. include:: ../release/1.8.0-notes.rst -.. include:: ../release/1.7.2-notes.rst -.. include:: ../release/1.7.1-notes.rst -.. include:: ../release/1.7.0-notes.rst -.. include:: ../release/1.6.2-notes.rst -.. include:: ../release/1.6.1-notes.rst -.. include:: ../release/1.6.0-notes.rst -.. include:: ../release/1.5.0-notes.rst -.. include:: ../release/1.4.0-notes.rst -.. include:: ../release/1.3.0-notes.rst +.. toctree:: + :maxdepth: 3 + + 1.18.0 <release/1.18.0-notes> + 1.17.1 <release/1.17.2-notes> + 1.17.1 <release/1.17.1-notes> + 1.17.0 <release/1.17.0-notes> + 1.16.5 <release/1.16.5-notes> + 1.16.4 <release/1.16.4-notes> + 1.16.3 <release/1.16.3-notes> + 1.16.2 <release/1.16.2-notes> + 1.16.1 <release/1.16.1-notes> + 1.16.0 <release/1.16.0-notes> + 1.15.4 <release/1.15.4-notes> + 1.15.3 <release/1.15.3-notes> + 1.15.2 <release/1.15.2-notes> + 1.15.1 <release/1.15.1-notes> + 1.15.0 <release/1.15.0-notes> + 1.14.6 <release/1.14.6-notes> + 1.14.5 <release/1.14.5-notes> + 1.14.4 <release/1.14.4-notes> + 1.14.3 <release/1.14.3-notes> + 1.14.2 <release/1.14.2-notes> + 1.14.1 <release/1.14.1-notes> + 1.14.0 <release/1.14.0-notes> + 1.13.3 <release/1.13.3-notes> + 1.13.2 <release/1.13.2-notes> + 1.13.1 <release/1.13.1-notes> + 1.13.0 <release/1.13.0-notes> + 1.12.1 <release/1.12.1-notes> + 1.12.0 <release/1.12.0-notes> + 1.11.3 <release/1.11.3-notes> + 1.11.2 <release/1.11.2-notes> + 1.11.1 <release/1.11.1-notes> + 1.11.0 <release/1.11.0-notes> + 1.10.4 <release/1.10.4-notes> + 1.10.3 <release/1.10.3-notes> + 1.10.2 <release/1.10.2-notes> + 1.10.1 <release/1.10.1-notes> + 1.10.0 <release/1.10.0-notes> + 1.9.2 <release/1.9.2-notes> + 1.9.1 <release/1.9.1-notes> + 1.9.0 <release/1.9.0-notes> + 1.8.2 <release/1.8.2-notes> + 1.8.1 <release/1.8.1-notes> + 1.8.0 <release/1.8.0-notes> + 1.7.2 <release/1.7.2-notes> + 1.7.1 <release/1.7.1-notes> + 1.7.0 <release/1.7.0-notes> + 1.6.2 <release/1.6.2-notes> + 1.6.1 <release/1.6.1-notes> + 1.6.0 <release/1.6.0-notes> + 1.5.0 <release/1.5.0-notes> + 1.4.0 <release/1.4.0-notes> + 1.3.0 <release/1.3.0-notes> diff --git a/doc/source/release/1.10.0-notes.rst b/doc/source/release/1.10.0-notes.rst new file mode 100644 index 000000000..88062e463 --- /dev/null +++ b/doc/source/release/1.10.0-notes.rst @@ -0,0 +1,464 @@ +========================== +NumPy 1.10.0 Release Notes +========================== + +This release supports Python 2.6 - 2.7 and 3.2 - 3.5. + + +Highlights +========== +* numpy.distutils now supports parallel compilation via the --parallel/-j + argument passed to setup.py build +* numpy.distutils now supports additional customization via site.cfg to + control compilation parameters, i.e. runtime libraries, extra + linking/compilation flags. +* Addition of *np.linalg.multi_dot*: compute the dot product of two or more + arrays in a single function call, while automatically selecting the fastest + evaluation order. +* The new function `np.stack` provides a general interface for joining a + sequence of arrays along a new axis, complementing `np.concatenate` for + joining along an existing axis. +* Addition of `nanprod` to the set of nanfunctions. +* Support for the '@' operator in Python 3.5. + +Dropped Support +=============== + +* The _dotblas module has been removed. CBLAS Support is now in + Multiarray. +* The testcalcs.py file has been removed. +* The polytemplate.py file has been removed. +* npy_PyFile_Dup and npy_PyFile_DupClose have been removed from + npy_3kcompat.h. +* splitcmdline has been removed from numpy/distutils/exec_command.py. +* try_run and get_output have been removed from + numpy/distutils/command/config.py +* The a._format attribute is no longer supported for array printing. +* Keywords ``skiprows`` and ``missing`` removed from np.genfromtxt. +* Keyword ``old_behavior`` removed from np.correlate. + +Future Changes +============== + +* In array comparisons like ``arr1 == arr2``, many corner cases + involving strings or structured dtypes that used to return scalars + now issue ``FutureWarning`` or ``DeprecationWarning``, and in the + future will be change to either perform elementwise comparisons or + raise an error. +* In ``np.lib.split`` an empty array in the result always had dimension + ``(0,)`` no matter the dimensions of the array being split. In Numpy 1.11 + that behavior will be changed so that the dimensions will be preserved. A + ``FutureWarning`` for this change has been in place since Numpy 1.9 but, + due to a bug, sometimes no warning was raised and the dimensions were + already preserved. +* The SafeEval class will be removed in Numpy 1.11. +* The alterdot and restoredot functions will be removed in Numpy 1.11. + +See below for more details on these changes. + +Compatibility notes +=================== + +Default casting rule change +--------------------------- +Default casting for inplace operations has changed to ``'same_kind'``. For +instance, if n is an array of integers, and f is an array of floats, then +``n += f`` will result in a ``TypeError``, whereas in previous Numpy +versions the floats would be silently cast to ints. In the unlikely case +that the example code is not an actual bug, it can be updated in a backward +compatible way by rewriting it as ``np.add(n, f, out=n, casting='unsafe')``. +The old ``'unsafe'`` default has been deprecated since Numpy 1.7. + +numpy version string +-------------------- +The numpy version string for development builds has been changed from +``x.y.z.dev-githash`` to ``x.y.z.dev0+githash`` (note the +) in order to comply +with PEP 440. + +relaxed stride checking +----------------------- +NPY_RELAXED_STRIDE_CHECKING is now true by default. + +UPDATE: In 1.10.2 the default value of NPY_RELAXED_STRIDE_CHECKING was +changed to false for back compatibility reasons. More time is needed before +it can be made the default. As part of the roadmap a deprecation of +dimension changing views of f_contiguous not c_contiguous arrays was also +added. + +Concatenation of 1d arrays along any but ``axis=0`` raises ``IndexError`` +------------------------------------------------------------------------- +Using axis != 0 has raised a DeprecationWarning since NumPy 1.7, it now +raises an error. + +*np.ravel*, *np.diagonal* and *np.diag* now preserve subtypes +------------------------------------------------------------- +There was inconsistent behavior between *x.ravel()* and *np.ravel(x)*, as +well as between *x.diagonal()* and *np.diagonal(x)*, with the methods +preserving subtypes while the functions did not. This has been fixed and +the functions now behave like the methods, preserving subtypes except in +the case of matrices. Matrices are special cased for backward +compatibility and still return 1-D arrays as before. If you need to +preserve the matrix subtype, use the methods instead of the functions. + +*rollaxis* and *swapaxes* always return a view +---------------------------------------------- +Previously, a view was returned except when no change was made in the order +of the axes, in which case the input array was returned. A view is now +returned in all cases. + +*nonzero* now returns base ndarrays +----------------------------------- +Previously, an inconsistency existed between 1-D inputs (returning a +base ndarray) and higher dimensional ones (which preserved subclasses). +Behavior has been unified, and the return will now be a base ndarray. +Subclasses can still override this behavior by providing their own +*nonzero* method. + +C API +----- +The changes to *swapaxes* also apply to the *PyArray_SwapAxes* C function, +which now returns a view in all cases. + +The changes to *nonzero* also apply to the *PyArray_Nonzero* C function, +which now returns a base ndarray in all cases. + +The dtype structure (PyArray_Descr) has a new member at the end to cache +its hash value. This shouldn't affect any well-written applications. + +The change to the concatenation function DeprecationWarning also affects +PyArray_ConcatenateArrays, + +recarray field return types +--------------------------- +Previously the returned types for recarray fields accessed by attribute and by +index were inconsistent, and fields of string type were returned as chararrays. +Now, fields accessed by either attribute or indexing will return an ndarray for +fields of non-structured type, and a recarray for fields of structured type. +Notably, this affect recarrays containing strings with whitespace, as trailing +whitespace is trimmed from chararrays but kept in ndarrays of string type. +Also, the dtype.type of nested structured fields is now inherited. + +recarray views +-------------- +Viewing an ndarray as a recarray now automatically converts the dtype to +np.record. See new record array documentation. Additionally, viewing a recarray +with a non-structured dtype no longer converts the result's type to ndarray - +the result will remain a recarray. + +'out' keyword argument of ufuncs now accepts tuples of arrays +------------------------------------------------------------- +When using the 'out' keyword argument of a ufunc, a tuple of arrays, one per +ufunc output, can be provided. For ufuncs with a single output a single array +is also a valid 'out' keyword argument. Previously a single array could be +provided in the 'out' keyword argument, and it would be used as the first +output for ufuncs with multiple outputs, is deprecated, and will result in a +`DeprecationWarning` now and an error in the future. + +byte-array indices now raises an IndexError +------------------------------------------- +Indexing an ndarray using a byte-string in Python 3 now raises an IndexError +instead of a ValueError. + +Masked arrays containing objects with arrays +-------------------------------------------- +For such (rare) masked arrays, getting a single masked item no longer returns a +corrupted masked array, but a fully masked version of the item. + +Median warns and returns nan when invalid values are encountered +---------------------------------------------------------------- +Similar to mean, median and percentile now emits a Runtime warning and +returns `NaN` in slices where a `NaN` is present. +To compute the median or percentile while ignoring invalid values use the +new `nanmedian` or `nanpercentile` functions. + +Functions available from numpy.ma.testutils have changed +-------------------------------------------------------- +All functions from numpy.testing were once available from +numpy.ma.testutils but not all of them were redefined to work with masked +arrays. Most of those functions have now been removed from +numpy.ma.testutils with a small subset retained in order to preserve +backward compatibility. In the long run this should help avoid mistaken use +of the wrong functions, but it may cause import problems for some. + + +New Features +============ + +Reading extra flags from site.cfg +--------------------------------- +Previously customization of compilation of dependency libraries and numpy +itself was only accomblishable via code changes in the distutils package. +Now numpy.distutils reads in the following extra flags from each group of the +*site.cfg*: + +* ``runtime_library_dirs/rpath``, sets runtime library directories to override + ``LD_LIBRARY_PATH`` +* ``extra_compile_args``, add extra flags to the compilation of sources +* ``extra_link_args``, add extra flags when linking libraries + +This should, at least partially, complete user customization. + +*np.cbrt* to compute cube root for real floats +---------------------------------------------- +*np.cbrt* wraps the C99 cube root function *cbrt*. +Compared to *np.power(x, 1./3.)* it is well defined for negative real floats +and a bit faster. + +numpy.distutils now allows parallel compilation +----------------------------------------------- +By passing *--parallel=n* or *-j n* to *setup.py build* the compilation of +extensions is now performed in *n* parallel processes. +The parallelization is limited to files within one extension so projects using +Cython will not profit because it builds extensions from single files. + +*genfromtxt* has a new ``max_rows`` argument +-------------------------------------------- +A ``max_rows`` argument has been added to *genfromtxt* to limit the +number of rows read in a single call. Using this functionality, it is +possible to read in multiple arrays stored in a single file by making +repeated calls to the function. + +New function *np.broadcast_to* for invoking array broadcasting +-------------------------------------------------------------- +*np.broadcast_to* manually broadcasts an array to a given shape according to +numpy's broadcasting rules. The functionality is similar to broadcast_arrays, +which in fact has been rewritten to use broadcast_to internally, but only a +single array is necessary. + +New context manager *clear_and_catch_warnings* for testing warnings +------------------------------------------------------------------- +When Python emits a warning, it records that this warning has been emitted in +the module that caused the warning, in a module attribute +``__warningregistry__``. Once this has happened, it is not possible to emit +the warning again, unless you clear the relevant entry in +``__warningregistry__``. This makes is hard and fragile to test warnings, +because if your test comes after another that has already caused the warning, +you will not be able to emit the warning or test it. The context manager +``clear_and_catch_warnings`` clears warnings from the module registry on entry +and resets them on exit, meaning that warnings can be re-raised. + +*cov* has new ``fweights`` and ``aweights`` arguments +----------------------------------------------------- +The ``fweights`` and ``aweights`` arguments add new functionality to +covariance calculations by applying two types of weighting to observation +vectors. An array of ``fweights`` indicates the number of repeats of each +observation vector, and an array of ``aweights`` provides their relative +importance or probability. + +Support for the '@' operator in Python 3.5+ +------------------------------------------- +Python 3.5 adds support for a matrix multiplication operator '@' proposed +in PEP465. Preliminary support for that has been implemented, and an +equivalent function ``matmul`` has also been added for testing purposes and +use in earlier Python versions. The function is preliminary and the order +and number of its optional arguments can be expected to change. + +New argument ``norm`` to fft functions +-------------------------------------- +The default normalization has the direct transforms unscaled and the inverse +transforms are scaled by :math:`1/n`. It is possible to obtain unitary +transforms by setting the keyword argument ``norm`` to ``"ortho"`` (default is +`None`) so that both direct and inverse transforms will be scaled by +:math:`1/\\sqrt{n}`. + + +Improvements +============ + +*np.digitize* using binary search +--------------------------------- +*np.digitize* is now implemented in terms of *np.searchsorted*. This means +that a binary search is used to bin the values, which scales much better +for larger number of bins than the previous linear search. It also removes +the requirement for the input array to be 1-dimensional. + +*np.poly* now casts integer inputs to float +------------------------------------------- +*np.poly* will now cast 1-dimensional input arrays of integer type to double +precision floating point, to prevent integer overflow when computing the monic +polynomial. It is still possible to obtain higher precision results by +passing in an array of object type, filled e.g. with Python ints. + +*np.interp* can now be used with periodic functions +--------------------------------------------------- +*np.interp* now has a new parameter *period* that supplies the period of the +input data *xp*. In such case, the input data is properly normalized to the +given period and one end point is added to each extremity of *xp* in order to +close the previous and the next period cycles, resulting in the correct +interpolation behavior. + +*np.pad* supports more input types for ``pad_width`` and ``constant_values`` +---------------------------------------------------------------------------- +``constant_values`` parameters now accepts NumPy arrays and float values. +NumPy arrays are supported as input for ``pad_width``, and an exception is +raised if its values are not of integral type. + +*np.argmax* and *np.argmin* now support an ``out`` argument +----------------------------------------------------------- +The ``out`` parameter was added to *np.argmax* and *np.argmin* for consistency +with *ndarray.argmax* and *ndarray.argmin*. The new parameter behaves exactly +as it does in those methods. + +More system C99 complex functions detected and used +--------------------------------------------------- +All of the functions ``in complex.h`` are now detected. There are new +fallback implementations of the following functions. + +* npy_ctan, +* npy_cacos, npy_casin, npy_catan +* npy_ccosh, npy_csinh, npy_ctanh, +* npy_cacosh, npy_casinh, npy_catanh + +As a result of these improvements, there will be some small changes in +returned values, especially for corner cases. + +*np.loadtxt* support for the strings produced by the ``float.hex`` method +------------------------------------------------------------------------- +The strings produced by ``float.hex`` look like ``0x1.921fb54442d18p+1``, +so this is not the hex used to represent unsigned integer types. + +*np.isclose* properly handles minimal values of integer dtypes +-------------------------------------------------------------- +In order to properly handle minimal values of integer types, *np.isclose* will +now cast to the float dtype during comparisons. This aligns its behavior with +what was provided by *np.allclose*. + +*np.allclose* uses *np.isclose* internally. +------------------------------------------- +*np.allclose* now uses *np.isclose* internally and inherits the ability to +compare NaNs as equal by setting ``equal_nan=True``. Subclasses, such as +*np.ma.MaskedArray*, are also preserved now. + +*np.genfromtxt* now handles large integers correctly +---------------------------------------------------- +*np.genfromtxt* now correctly handles integers larger than ``2**31-1`` on +32-bit systems and larger than ``2**63-1`` on 64-bit systems (it previously +crashed with an ``OverflowError`` in these cases). Integers larger than +``2**63-1`` are converted to floating-point values. + +*np.load*, *np.save* have pickle backward compatibility flags +------------------------------------------------------------- + +The functions *np.load* and *np.save* have additional keyword +arguments for controlling backward compatibility of pickled Python +objects. This enables Numpy on Python 3 to load npy files containing +object arrays that were generated on Python 2. + +MaskedArray support for more complicated base classes +----------------------------------------------------- +Built-in assumptions that the baseclass behaved like a plain array are being +removed. In particular, setting and getting elements and ranges will respect +baseclass overrides of ``__setitem__`` and ``__getitem__``, and arithmetic +will respect overrides of ``__add__``, ``__sub__``, etc. + +Changes +======= + +dotblas functionality moved to multiarray +----------------------------------------- +The cblas versions of dot, inner, and vdot have been integrated into +the multiarray module. In particular, vdot is now a multiarray function, +which it was not before. + +stricter check of gufunc signature compliance +--------------------------------------------- +Inputs to generalized universal functions are now more strictly checked +against the function's signature: all core dimensions are now required to +be present in input arrays; core dimensions with the same label must have +the exact same size; and output core dimension's must be specified, either +by a same label input core dimension or by a passed-in output array. + +views returned from *np.einsum* are writeable +--------------------------------------------- +Views returned by *np.einsum* will now be writeable whenever the input +array is writeable. + +*np.argmin* skips NaT values +---------------------------- + +*np.argmin* now skips NaT values in datetime64 and timedelta64 arrays, +making it consistent with *np.min*, *np.argmax* and *np.max*. + + +Deprecations +============ + +Array comparisons involving strings or structured dtypes +-------------------------------------------------------- + +Normally, comparison operations on arrays perform elementwise +comparisons and return arrays of booleans. But in some corner cases, +especially involving strings are structured dtypes, NumPy has +historically returned a scalar instead. For example:: + + ### Current behaviour + + np.arange(2) == "foo" + # -> False + + np.arange(2) < "foo" + # -> True on Python 2, error on Python 3 + + np.ones(2, dtype="i4,i4") == np.ones(2, dtype="i4,i4,i4") + # -> False + +Continuing work started in 1.9, in 1.10 these comparisons will now +raise ``FutureWarning`` or ``DeprecationWarning``, and in the future +they will be modified to behave more consistently with other +comparison operations, e.g.:: + + ### Future behaviour + + np.arange(2) == "foo" + # -> array([False, False]) + + np.arange(2) < "foo" + # -> error, strings and numbers are not orderable + + np.ones(2, dtype="i4,i4") == np.ones(2, dtype="i4,i4,i4") + # -> [False, False] + +SafeEval +-------- +The SafeEval class in numpy/lib/utils.py is deprecated and will be removed +in the next release. + +alterdot, restoredot +-------------------- +The alterdot and restoredot functions no longer do anything, and are +deprecated. + +pkgload, PackageLoader +---------------------- +These ways of loading packages are now deprecated. + +bias, ddof arguments to corrcoef +-------------------------------- + +The values for the ``bias`` and ``ddof`` arguments to the ``corrcoef`` +function canceled in the division implied by the correlation coefficient and +so had no effect on the returned values. + +We now deprecate these arguments to ``corrcoef`` and the masked array version +``ma.corrcoef``. + +Because we are deprecating the ``bias`` argument to ``ma.corrcoef``, we also +deprecate the use of the ``allow_masked`` argument as a positional argument, +as its position will change with the removal of ``bias``. ``allow_masked`` +will in due course become a keyword-only argument. + +dtype string representation changes +----------------------------------- +Since 1.6, creating a dtype object from its string representation, e.g. +``'f4'``, would issue a deprecation warning if the size did not correspond +to an existing type, and default to creating a dtype of the default size +for the type. Starting with this release, this will now raise a ``TypeError``. + +The only exception is object dtypes, where both ``'O4'`` and ``'O8'`` will +still issue a deprecation warning. This platform-dependent representation +will raise an error in the next release. + +In preparation for this upcoming change, the string representation of an +object dtype, i.e. ``np.dtype(object).str``, no longer includes the item +size, i.e. will return ``'|O'`` instead of ``'|O4'`` or ``'|O8'`` as +before. diff --git a/doc/source/release/1.10.1-notes.rst b/doc/source/release/1.10.1-notes.rst new file mode 100644 index 000000000..4e541d279 --- /dev/null +++ b/doc/source/release/1.10.1-notes.rst @@ -0,0 +1,42 @@ +========================== +NumPy 1.10.1 Release Notes +========================== + +This release deals with a few build problems that showed up in 1.10.0. Most +users would not have seen these problems. The differences are: + +* Compiling with msvc9 or msvc10 for 32 bit Windows now requires SSE2. + This was the easiest fix for what looked to be some miscompiled code when + SSE2 was not used. If you need to compile for 32 bit Windows systems + without SSE2 support, mingw32 should still work. + +* Make compiling with VS2008 python2.7 SDK easier + +* Change Intel compiler options so that code will also be generated to + support systems without SSE4.2. + +* Some _config test functions needed an explicit integer return in + order to avoid the openSUSE rpmlinter erring out. + +* We ran into a problem with pipy not allowing reuse of filenames and a + resulting proliferation of *.*.*.postN releases. Not only were the names + getting out of hand, some packages were unable to work with the postN + suffix. + + +Numpy 1.10.1 supports Python 2.6 - 2.7 and 3.2 - 3.5. + + +Commits: + +45a3d84 DEP: Remove warning for `full` when dtype is set. +0c1a5df BLD: import setuptools to allow compile with VS2008 python2.7 sdk +04211c6 BUG: mask nan to 1 in ordered compare +826716f DOC: Document the reason msvc requires SSE2 on 32 bit platforms. +49fa187 BLD: enable SSE2 for 32-bit msvc 9 and 10 compilers +dcbc4cc MAINT: remove Wreturn-type warnings from config checks +d6564cb BLD: do not build exclusively for SSE4.2 processors +15cb66f BLD: do not build exclusively for SSE4.2 processors +c38bc08 DOC: fix var. reference in percentile docstring +78497f4 DOC: Sync 1.10.0-notes.rst in 1.10.x branch with master. + diff --git a/doc/source/release/1.10.2-notes.rst b/doc/source/release/1.10.2-notes.rst new file mode 100644 index 000000000..8c26b463c --- /dev/null +++ b/doc/source/release/1.10.2-notes.rst @@ -0,0 +1,158 @@ +========================== +NumPy 1.10.2 Release Notes +========================== + +This release deals with a number of bugs that turned up in 1.10.1 and +adds various build and release improvements. + +Numpy 1.10.1 supports Python 2.6 - 2.7 and 3.2 - 3.5. + + +Compatibility notes +=================== + +Relaxed stride checking is no longer the default +------------------------------------------------ +There were back compatibility problems involving views changing the dtype of +multidimensional Fortran arrays that need to be dealt with over a longer +timeframe. + +Fix swig bug in ``numpy.i`` +--------------------------- +Relaxed stride checking revealed a bug in ``array_is_fortran(a)``, that was +using PyArray_ISFORTRAN to check for Fortran contiguity instead of +PyArray_IS_F_CONTIGUOUS. You may want to regenerate swigged files using the +updated numpy.i + +Deprecate views changing dimensions in fortran order +---------------------------------------------------- +This deprecates assignment of a new descriptor to the dtype attribute of +a non-C-contiguous array if it result in changing the shape. This +effectively bars viewing a multidimensional Fortran array using a dtype +that changes the element size along the first axis. + +The reason for the deprecation is that, when relaxed strides checking is +enabled, arrays that are both C and Fortran contiguous are always treated +as C contiguous which breaks some code that depended the two being mutually +exclusive for non-scalar arrays of ndim > 1. This deprecation prepares the +way to always enable relaxed stride checking. + + +Issues Fixed +============ + +* gh-6019 Masked array repr fails for structured array with multi-dimensional column. +* gh-6462 Median of empty array produces IndexError. +* gh-6467 Performance regression for record array access. +* gh-6468 numpy.interp uses 'left' value even when x[0]==xp[0]. +* gh-6475 np.allclose returns a memmap when one of its arguments is a memmap. +* gh-6491 Error in broadcasting stride_tricks array. +* gh-6495 Unrecognized command line option '-ffpe-summary' in gfortran. +* gh-6497 Failure of reduce operation on recarrays. +* gh-6498 Mention change in default casting rule in 1.10 release notes. +* gh-6530 The partition function errors out on empty input. +* gh-6532 numpy.inner return wrong inaccurate value sometimes. +* gh-6563 Intent(out) broken in recent versions of f2py. +* gh-6569 Cannot run tests after 'python setup.py build_ext -i' +* gh-6572 Error in broadcasting stride_tricks array component. +* gh-6575 BUG: Split produces empty arrays with wrong number of dimensions +* gh-6590 Fortran Array problem in numpy 1.10. +* gh-6602 Random __all__ missing choice and dirichlet. +* gh-6611 ma.dot no longer always returns a masked array in 1.10. +* gh-6618 NPY_FORTRANORDER in make_fortran() in numpy.i +* gh-6636 Memory leak in nested dtypes in numpy.recarray +* gh-6641 Subsetting recarray by fields yields a structured array. +* gh-6667 ma.make_mask handles ma.nomask input incorrectly. +* gh-6675 Optimized blas detection broken in master and 1.10. +* gh-6678 Getting unexpected error from: X.dtype = complex (or Y = X.view(complex)) +* gh-6718 f2py test fail in pip installed numpy-1.10.1 in virtualenv. +* gh-6719 Error compiling Cython file: Pythonic division not allowed without gil. +* gh-6771 Numpy.rec.fromarrays losing dtype metadata between versions 1.9.2 and 1.10.1 +* gh-6781 The travis-ci script in maintenance/1.10.x needs fixing. +* gh-6807 Windows testing errors for 1.10.2 + + +Merged PRs +========== + +The following PRs have been merged into 1.10.2. When the PR is a backport, +the PR number for the original PR against master is listed. + +* gh-5773 MAINT: Hide testing helper tracebacks when using them with pytest. +* gh-6094 BUG: Fixed a bug with string representation of masked structured arrays. +* gh-6208 MAINT: Speedup field access by removing unneeded safety checks. +* gh-6460 BUG: Replacing the os.environ.clear by less invasive procedure. +* gh-6470 BUG: Fix AttributeError in numpy distutils. +* gh-6472 MAINT: Use Python 3.5 instead of 3.5-dev for travis 3.5 testing. +* gh-6474 REL: Update Paver script for sdist and auto-switch test warnings. +* gh-6478 BUG: Fix Intel compiler flags for OS X build. +* gh-6481 MAINT: LIBPATH with spaces is now supported Python 2.7+ and Win32. +* gh-6487 BUG: Allow nested use of parameters in definition of arrays in f2py. +* gh-6488 BUG: Extend common blocks rather than overwriting in f2py. +* gh-6499 DOC: Mention that default casting for inplace operations has changed. +* gh-6500 BUG: Recarrays viewed as subarrays don't convert to np.record type. +* gh-6501 REL: Add "make upload" command for built docs, update "make dist". +* gh-6526 BUG: Fix use of __doc__ in setup.py for -OO mode. +* gh-6527 BUG: Fix the IndexError when taking the median of an empty array. +* gh-6537 BUG: Make ma.atleast_* with scalar argument return arrays. +* gh-6538 BUG: Fix ma.masked_values does not shrink mask if requested. +* gh-6546 BUG: Fix inner product regression for non-contiguous arrays. +* gh-6553 BUG: Fix partition and argpartition error for empty input. +* gh-6556 BUG: Error in broadcast_arrays with as_strided array. +* gh-6558 MAINT: Minor update to "make upload" doc build command. +* gh-6562 BUG: Disable view safety checks in recarray. +* gh-6567 BUG: Revert some import * fixes in f2py. +* gh-6574 DOC: Release notes for Numpy 1.10.2. +* gh-6577 BUG: Fix for #6569, allowing build_ext --inplace +* gh-6579 MAINT: Fix mistake in doc upload rule. +* gh-6596 BUG: Fix swig for relaxed stride checking. +* gh-6606 DOC: Update 1.10.2 release notes. +* gh-6614 BUG: Add choice and dirichlet to numpy.random.__all__. +* gh-6621 BUG: Fix swig make_fortran function. +* gh-6628 BUG: Make allclose return python bool. +* gh-6642 BUG: Fix memleak in _convert_from_dict. +* gh-6643 ENH: make recarray.getitem return a recarray. +* gh-6653 BUG: Fix ma dot to always return masked array. +* gh-6668 BUG: ma.make_mask should always return nomask for nomask argument. +* gh-6686 BUG: Fix a bug in assert_string_equal. +* gh-6695 BUG: Fix removing tempdirs created during build. +* gh-6697 MAINT: Fix spurious semicolon in macro definition of PyArray_FROM_OT. +* gh-6698 TST: test np.rint bug for large integers. +* gh-6717 BUG: Readd fallback CBLAS detection on linux. +* gh-6721 BUG: Fix for #6719. +* gh-6726 BUG: Fix bugs exposed by relaxed stride rollback. +* gh-6757 BUG: link cblas library if cblas is detected. +* gh-6756 TST: only test f2py, not f2py2.7 etc, fixes #6718. +* gh-6747 DEP: Deprecate changing shape of non-C-contiguous array via descr. +* gh-6775 MAINT: Include from __future__ boilerplate in some files missing it. +* gh-6780 BUG: metadata is not copied to base_dtype. +* gh-6783 BUG: Fix travis ci testing for new google infrastructure. +* gh-6785 BUG: Quick and dirty fix for interp. +* gh-6813 TST,BUG: Make test_mvoid_multidim_print work for 32 bit systems. +* gh-6817 BUG: Disable 32-bit msvc9 compiler optimizations for npy_rint. +* gh-6819 TST: Fix test_mvoid_multidim_print failures on Python 2.x for Windows. + +Initial support for mingwpy was reverted as it was causing problems for +non-windows builds. + +* gh-6536 BUG: Revert gh-5614 to fix non-windows build problems + +A fix for np.lib.split was reverted because it resulted in "fixing" +behavior that will be present in the Numpy 1.11 and that was already +present in Numpy 1.9. See the discussion of the issue at gh-6575 for +clarification. + +* gh-6576 BUG: Revert gh-6376 to fix split behavior for empty arrays. + +Relaxed stride checking was reverted. There were back compatibility +problems involving views changing the dtype of multidimensional Fortran +arrays that need to be dealt with over a longer timeframe. + +* gh-6735 MAINT: Make no relaxed stride checking the default for 1.10. + + +Notes +===== +A bug in the Numpy 1.10.1 release resulted in exceptions being raised for +``RuntimeWarning`` and ``DeprecationWarning`` in projects depending on Numpy. +That has been fixed. diff --git a/doc/source/release/1.10.3-notes.rst b/doc/source/release/1.10.3-notes.rst new file mode 100644 index 000000000..0d4df4ce6 --- /dev/null +++ b/doc/source/release/1.10.3-notes.rst @@ -0,0 +1,5 @@ +========================== +NumPy 1.10.3 Release Notes +========================== + +N/A this release did not happen due to various screwups involving PyPi. diff --git a/doc/source/release/1.10.4-notes.rst b/doc/source/release/1.10.4-notes.rst new file mode 100644 index 000000000..481928ca7 --- /dev/null +++ b/doc/source/release/1.10.4-notes.rst @@ -0,0 +1,40 @@ +========================== +NumPy 1.10.4 Release Notes +========================== + +This release is a bugfix source release motivated by a segfault regression. +No windows binaries are provided for this release, as there appear to be +bugs in the toolchain we use to generate those files. Hopefully that +problem will be fixed for the next release. In the meantime, we suggest +using one of the providers of windows binaries. + +Compatibility notes +=================== + +* The trace function now calls the trace method on subclasses of ndarray, + except for matrix, for which the current behavior is preserved. This is + to help with the units package of AstroPy and hopefully will not cause + problems. + +Issues Fixed +============ + +* gh-6922 BUG: numpy.recarray.sort segfaults on Windows. +* gh-6937 BUG: busday_offset does the wrong thing with modifiedpreceding roll. +* gh-6949 BUG: Type is lost when slicing a subclass of recarray. + +Merged PRs +========== + +The following PRs have been merged into 1.10.4. When the PR is a backport, +the PR number for the original PR against master is listed. + +* gh-6840 TST: Update travis testing script in 1.10.x +* gh-6843 BUG: Fix use of python 3 only FileNotFoundError in test_f2py. +* gh-6884 REL: Update pavement.py and setup.py to reflect current version. +* gh-6916 BUG: Fix test_f2py so it runs correctly in runtests.py. +* gh-6924 BUG: Fix segfault gh-6922. +* gh-6942 Fix datetime roll='modifiedpreceding' bug. +* gh-6943 DOC,BUG: Fix some latex generation problems. +* gh-6950 BUG trace is not subclass aware, np.trace(ma) != ma.trace(). +* gh-6952 BUG recarray slices should preserve subclass. diff --git a/doc/source/release/1.11.0-notes.rst b/doc/source/release/1.11.0-notes.rst new file mode 100644 index 000000000..166502ac5 --- /dev/null +++ b/doc/source/release/1.11.0-notes.rst @@ -0,0 +1,393 @@ +========================== +NumPy 1.11.0 Release Notes +========================== + +This release supports Python 2.6 - 2.7 and 3.2 - 3.5 and contains a number +of enhancements and improvements. Note also the build system changes listed +below as they may have subtle effects. + +No Windows (TM) binaries are provided for this release due to a broken +toolchain. One of the providers of Python packages for Windows (TM) is your +best bet. + + +Highlights +========== + +Details of these improvements can be found below. + +* The datetime64 type is now timezone naive. +* A dtype parameter has been added to ``randint``. +* Improved detection of two arrays possibly sharing memory. +* Automatic bin size estimation for ``np.histogram``. +* Speed optimization of A @ A.T and dot(A, A.T). +* New function ``np.moveaxis`` for reordering array axes. + + +Build System Changes +==================== + +* Numpy now uses ``setuptools`` for its builds instead of plain distutils. + This fixes usage of ``install_requires='numpy'`` in the ``setup.py`` files of + projects that depend on Numpy (see gh-6551). It potentially affects the way + that build/install methods for Numpy itself behave though. Please report any + unexpected behavior on the Numpy issue tracker. +* Bento build support and related files have been removed. +* Single file build support and related files have been removed. + + +Future Changes +============== + +The following changes are scheduled for Numpy 1.12.0. + +* Support for Python 2.6, 3.2, and 3.3 will be dropped. +* Relaxed stride checking will become the default. See the 1.8.0 release + notes for a more extended discussion of what this change implies. +* The behavior of the datetime64 "not a time" (NaT) value will be changed + to match that of floating point "not a number" (NaN) values: all + comparisons involving NaT will return False, except for NaT != NaT which + will return True. +* Indexing with floats will raise IndexError, + e.g., a[0, 0.0]. +* Indexing with non-integer array_like will raise ``IndexError``, + e.g., ``a['1', '2']`` +* Indexing with multiple ellipsis will raise ``IndexError``, + e.g., ``a[..., ...]``. +* Non-integers used as index values will raise ``TypeError``, + e.g., in ``reshape``, ``take``, and specifying reduce axis. + + +In a future release the following changes will be made. + +* The ``rand`` function exposed in ``numpy.testing`` will be removed. That + function is left over from early Numpy and was implemented using the + Python random module. The random number generators from ``numpy.random`` + should be used instead. +* The ``ndarray.view`` method will only allow c_contiguous arrays to be + viewed using a dtype of different size causing the last dimension to + change. That differs from the current behavior where arrays that are + f_contiguous but not c_contiguous can be viewed as a dtype type of + different size causing the first dimension to change. +* Slicing a ``MaskedArray`` will return views of both data **and** mask. + Currently the mask is copy-on-write and changes to the mask in the slice do + not propagate to the original mask. See the FutureWarnings section below for + details. + + +Compatibility notes +=================== + +datetime64 changes +------------------ +In prior versions of NumPy the experimental datetime64 type always stored +times in UTC. By default, creating a datetime64 object from a string or +printing it would convert from or to local time:: + + # old behavior + >>>> np.datetime64('2000-01-01T00:00:00') + numpy.datetime64('2000-01-01T00:00:00-0800') # note the timezone offset -08:00 + + +A consensus of datetime64 users agreed that this behavior is undesirable +and at odds with how datetime64 is usually used (e.g., by `pandas +<http://pandas.pydata.org>`__). For most use cases, a timezone naive datetime +type is preferred, similar to the ``datetime.datetime`` type in the Python +standard library. Accordingly, datetime64 no longer assumes that input is in +local time, nor does it print local times:: + + >>>> np.datetime64('2000-01-01T00:00:00') + numpy.datetime64('2000-01-01T00:00:00') + +For backwards compatibility, datetime64 still parses timezone offsets, which +it handles by converting to UTC. However, the resulting datetime is timezone +naive:: + + >>> np.datetime64('2000-01-01T00:00:00-08') + DeprecationWarning: parsing timezone aware datetimes is deprecated; + this will raise an error in the future + numpy.datetime64('2000-01-01T08:00:00') + +As a corollary to this change, we no longer prohibit casting between datetimes +with date units and datetimes with time units. With timezone naive datetimes, +the rule for casting from dates to times is no longer ambiguous. + +``linalg.norm`` return type changes +----------------------------------- +The return type of the ``linalg.norm`` function is now floating point without +exception. Some of the norm types previously returned integers. + +polynomial fit changes +---------------------- +The various fit functions in the numpy polynomial package no longer accept +non-integers for degree specification. + +*np.dot* now raises ``TypeError`` instead of ``ValueError`` +----------------------------------------------------------- +This behaviour mimics that of other functions such as ``np.inner``. If the two +arguments cannot be cast to a common type, it could have raised a ``TypeError`` +or ``ValueError`` depending on their order. Now, ``np.dot`` will now always +raise a ``TypeError``. + +FutureWarning to changed behavior +--------------------------------- + +* In ``np.lib.split`` an empty array in the result always had dimension + ``(0,)`` no matter the dimensions of the array being split. This + has been changed so that the dimensions will be preserved. A + ``FutureWarning`` for this change has been in place since Numpy 1.9 but, + due to a bug, sometimes no warning was raised and the dimensions were + already preserved. + +``%`` and ``//`` operators +-------------------------- +These operators are implemented with the ``remainder`` and ``floor_divide`` +functions respectively. Those functions are now based around ``fmod`` and are +computed together so as to be compatible with each other and with the Python +versions for float types. The results should be marginally more accurate or +outright bug fixes compared to the previous results, but they may +differ significantly in cases where roundoff makes a difference in the integer +returned by ``floor_divide``. Some corner cases also change, for instance, NaN +is always returned for both functions when the divisor is zero, +``divmod(1.0, inf)`` returns ``(0.0, 1.0)`` except on MSVC 2008, and +``divmod(-1.0, inf)`` returns ``(-1.0, inf)``. + +C API +----- + +Removed the ``check_return`` and ``inner_loop_selector`` members of +the ``PyUFuncObject`` struct (replacing them with ``reserved`` slots +to preserve struct layout). These were never used for anything, so +it's unlikely that any third-party code is using them either, but we +mention it here for completeness. + + +object dtype detection for old-style classes +-------------------------------------------- + +In python 2, objects which are instances of old-style user-defined classes no +longer automatically count as 'object' type in the dtype-detection handler. +Instead, as in python 3, they may potentially count as sequences, but only if +they define both a `__len__` and a `__getitem__` method. This fixes a segfault +and inconsistency between python 2 and 3. + +New Features +============ + +* ``np.histogram`` now provides plugin estimators for automatically + estimating the optimal number of bins. Passing one of ['auto', 'fd', + 'scott', 'rice', 'sturges'] as the argument to 'bins' results in the + corresponding estimator being used. + +* A benchmark suite using `Airspeed Velocity + <https://asv.readthedocs.io/>`__ has been added, converting the + previous vbench-based one. You can run the suite locally via ``python + runtests.py --bench``. For more details, see ``benchmarks/README.rst``. + +* A new function ``np.shares_memory`` that can check exactly whether two + arrays have memory overlap is added. ``np.may_share_memory`` also now has + an option to spend more effort to reduce false positives. + +* ``SkipTest`` and ``KnownFailureException`` exception classes are exposed + in the ``numpy.testing`` namespace. Raise them in a test function to mark + the test to be skipped or mark it as a known failure, respectively. + +* ``f2py.compile`` has a new ``extension`` keyword parameter that allows the + fortran extension to be specified for generated temp files. For instance, + the files can be specifies to be ``*.f90``. The ``verbose`` argument is + also activated, it was previously ignored. + +* A ``dtype`` parameter has been added to ``np.random.randint`` + Random ndarrays of the following types can now be generated: + + - ``np.bool``, + - ``np.int8``, ``np.uint8``, + - ``np.int16``, ``np.uint16``, + - ``np.int32``, ``np.uint32``, + - ``np.int64``, ``np.uint64``, + - ``np.int_ ``, ``np.intp`` + + The specification is by precision rather than by C type. Hence, on some + platforms ``np.int64`` may be a ``long`` instead of ``long long`` even if + the specified dtype is ``long long`` because the two may have the same + precision. The resulting type depends on which C type numpy uses for the + given precision. The byteorder specification is also ignored, the + generated arrays are always in native byte order. + +* A new ``np.moveaxis`` function allows for moving one or more array axes + to a new position by explicitly providing source and destination axes. + This function should be easier to use than the current ``rollaxis`` + function as well as providing more functionality. + +* The ``deg`` parameter of the various ``numpy.polynomial`` fits has been + extended to accept a list of the degrees of the terms to be included in + the fit, the coefficients of all other terms being constrained to zero. + The change is backward compatible, passing a scalar ``deg`` will behave + as before. + +* A divmod function for float types modeled after the Python version has + been added to the npy_math library. + + +Improvements +============ + +``np.gradient`` now supports an ``axis`` argument +------------------------------------------------- +The ``axis`` parameter was added to ``np.gradient`` for consistency. It +allows to specify over which axes the gradient is calculated. + +``np.lexsort`` now supports arrays with object data-type +-------------------------------------------------------- +The function now internally calls the generic ``npy_amergesort`` when the +type does not implement a merge-sort kind of ``argsort`` method. + +``np.ma.core.MaskedArray`` now supports an ``order`` argument +------------------------------------------------------------- +When constructing a new ``MaskedArray`` instance, it can be configured with +an ``order`` argument analogous to the one when calling ``np.ndarray``. The +addition of this argument allows for the proper processing of an ``order`` +argument in several MaskedArray-related utility functions such as +``np.ma.core.array`` and ``np.ma.core.asarray``. + +Memory and speed improvements for masked arrays +----------------------------------------------- +Creating a masked array with ``mask=True`` (resp. ``mask=False``) now uses +``np.ones`` (resp. ``np.zeros``) to create the mask, which is faster and +avoid a big memory peak. Another optimization was done to avoid a memory +peak and useless computations when printing a masked array. + +``ndarray.tofile`` now uses fallocate on linux +---------------------------------------------- +The function now uses the fallocate system call to reserve sufficient +disk space on file systems that support it. + +Optimizations for operations of the form ``A.T @ A`` and ``A @ A.T`` +-------------------------------------------------------------------- +Previously, ``gemm`` BLAS operations were used for all matrix products. Now, +if the matrix product is between a matrix and its transpose, it will use +``syrk`` BLAS operations for a performance boost. This optimization has been +extended to ``@``, ``numpy.dot``, ``numpy.inner``, and ``numpy.matmul``. + +**Note:** Requires the transposed and non-transposed matrices to share data. + +``np.testing.assert_warns`` can now be used as a context manager +---------------------------------------------------------------- +This matches the behavior of ``assert_raises``. + +Speed improvement for np.random.shuffle +--------------------------------------- +``np.random.shuffle`` is now much faster for 1d ndarrays. + + +Changes +======= + +Pyrex support was removed from ``numpy.distutils`` +-------------------------------------------------- +The method ``build_src.generate_a_pyrex_source`` will remain available; it +has been monkeypatched by users to support Cython instead of Pyrex. It's +recommended to switch to a better supported method of build Cython +extensions though. + +``np.broadcast`` can now be called with a single argument +--------------------------------------------------------- +The resulting object in that case will simply mimic iteration over +a single array. This change obsoletes distinctions like + + if len(x) == 1: + shape = x[0].shape + else: + shape = np.broadcast(\*x).shape + +Instead, ``np.broadcast`` can be used in all cases. + +``np.trace`` now respects array subclasses +------------------------------------------ +This behaviour mimics that of other functions such as ``np.diagonal`` and +ensures, e.g., that for masked arrays ``np.trace(ma)`` and ``ma.trace()`` give +the same result. + +``np.dot`` now raises ``TypeError`` instead of ``ValueError`` +------------------------------------------------------------- +This behaviour mimics that of other functions such as ``np.inner``. If the two +arguments cannot be cast to a common type, it could have raised a ``TypeError`` +or ``ValueError`` depending on their order. Now, ``np.dot`` will now always +raise a ``TypeError``. + +``linalg.norm`` return type changes +----------------------------------- +The ``linalg.norm`` function now does all its computations in floating point +and returns floating results. This change fixes bugs due to integer overflow +and the failure of abs with signed integers of minimum value, e.g., int8(-128). +For consistency, floats are used even where an integer might work. + + +Deprecations +============ + +Views of arrays in Fortran order +-------------------------------- +The F_CONTIGUOUS flag was used to signal that views using a dtype that +changed the element size would change the first index. This was always +problematical for arrays that were both F_CONTIGUOUS and C_CONTIGUOUS +because C_CONTIGUOUS took precedence. Relaxed stride checking results in +more such dual contiguous arrays and breaks some existing code as a result. +Note that this also affects changing the dtype by assigning to the dtype +attribute of an array. The aim of this deprecation is to restrict views to +C_CONTIGUOUS arrays at some future time. A work around that is backward +compatible is to use ``a.T.view(...).T`` instead. A parameter may also be +added to the view method to explicitly ask for Fortran order views, but +that will not be backward compatible. + +Invalid arguments for array ordering +------------------------------------ +It is currently possible to pass in arguments for the ``order`` +parameter in methods like ``array.flatten`` or ``array.ravel`` +that were not one of the following: 'C', 'F', 'A', 'K' (note that +all of these possible values are both unicode and case insensitive). +Such behavior will not be allowed in future releases. + +Random number generator in the ``testing`` namespace +---------------------------------------------------- +The Python standard library random number generator was previously exposed +in the ``testing`` namespace as ``testing.rand``. Using this generator is +not recommended and it will be removed in a future release. Use generators +from ``numpy.random`` namespace instead. + +Random integer generation on a closed interval +---------------------------------------------- +In accordance with the Python C API, which gives preference to the half-open +interval over the closed one, ``np.random.random_integers`` is being +deprecated in favor of calling ``np.random.randint``, which has been +enhanced with the ``dtype`` parameter as described under "New Features". +However, ``np.random.random_integers`` will not be removed anytime soon. + + +FutureWarnings +============== + +Assigning to slices/views of ``MaskedArray`` +-------------------------------------------- +Currently a slice of a masked array contains a view of the original data and a +copy-on-write view of the mask. Consequently, any changes to the slice's mask +will result in a copy of the original mask being made and that new mask being +changed rather than the original. For example, if we make a slice of the +original like so, ``view = original[:]``, then modifications to the data in one +array will affect the data of the other but, because the mask will be copied +during assignment operations, changes to the mask will remain local. A similar +situation occurs when explicitly constructing a masked array using +``MaskedArray(data, mask)``, the returned array will contain a view of ``data`` +but the mask will be a copy-on-write view of ``mask``. + +In the future, these cases will be normalized so that the data and mask arrays +are treated the same way and modifications to either will propagate between +views. In 1.11, numpy will issue a ``MaskedArrayFutureWarning`` warning +whenever user code modifies the mask of a view that in the future may cause +values to propagate back to the original. To silence these warnings and make +your code robust against the upcoming changes, you have two options: if you +want to keep the current behavior, call ``masked_view.unshare_mask()`` before +modifying the mask. If you want to get the future behavior early, use +``masked_view._sharedmask = False``. However, note that setting the +``_sharedmask`` attribute will break following explicit calls to +``masked_view.unshare_mask()``. diff --git a/doc/source/release/1.11.1-notes.rst b/doc/source/release/1.11.1-notes.rst new file mode 100644 index 000000000..6303c32f0 --- /dev/null +++ b/doc/source/release/1.11.1-notes.rst @@ -0,0 +1,32 @@ +========================== +NumPy 1.11.1 Release Notes +========================== + +Numpy 1.11.1 supports Python 2.6 - 2.7 and 3.2 - 3.5. It fixes bugs and +regressions found in Numpy 1.11.0 and includes several build related +improvements. Wheels for Linux, Windows, and OSX can be found on pypi. + +Fixes Merged +============ + +- #7506 BUG: Make sure numpy imports on python 2.6 when nose is unavailable. +- #7530 BUG: Floating exception with invalid axis in np.lexsort. +- #7535 BUG: Extend glibc complex trig functions blacklist to glibc < 2.18. +- #7551 BUG: Allow graceful recovery for no compiler. +- #7558 BUG: Constant padding expected wrong type in constant_values. +- #7578 BUG: Fix OverflowError in Python 3.x. in swig interface. +- #7590 BLD: Fix configparser.InterpolationSyntaxError. +- #7597 BUG: Make np.ma.take work on scalars. +- #7608 BUG: linalg.norm(): Don't convert object arrays to float. +- #7638 BLD: Correct C compiler customization in system_info.py. +- #7654 BUG: ma.median of 1d array should return a scalar. +- #7656 BLD: Remove hardcoded Intel compiler flag -xSSE4.2. +- #7660 BUG: Temporary fix for str(mvoid) for object field types. +- #7665 BUG: Fix incorrect printing of 1D masked arrays. +- #7670 BUG: Correct initial index estimate in histogram. +- #7671 BUG: Boolean assignment no GIL release when transfer needs API. +- #7676 BUG: Fix handling of right edge of final histogram bin. +- #7680 BUG: Fix np.clip bug NaN handling for Visual Studio 2015. +- #7724 BUG: Fix segfaults in np.random.shuffle. +- #7731 MAINT: Change mkl_info.dir_env_var from MKL to MKLROOT. +- #7737 BUG: Fix issue on OS X with Python 3.x, npymath.ini not installed. diff --git a/doc/source/release/1.11.2-notes.rst b/doc/source/release/1.11.2-notes.rst new file mode 100644 index 000000000..c954089d5 --- /dev/null +++ b/doc/source/release/1.11.2-notes.rst @@ -0,0 +1,40 @@ +========================== +NumPy 1.11.2 Release Notes +========================== + +Numpy 1.11.2 supports Python 2.6 - 2.7 and 3.2 - 3.5. It fixes bugs and +regressions found in Numpy 1.11.1 and includes several build related +improvements. Wheels for Linux, Windows, and OS X can be found on PyPI. + +Pull Requests Merged +==================== + +Fixes overridden by later merges and release notes updates are omitted. + +- #7736 BUG: Many functions silently drop 'keepdims' kwarg. +- #7738 ENH: Add extra kwargs and update doc of many MA methods. +- #7778 DOC: Update Numpy 1.11.1 release notes. +- #7793 BUG: MaskedArray.count treats negative axes incorrectly. +- #7816 BUG: Fix array too big error for wide dtypes. +- #7821 BUG: Make sure npy_mul_with_overflow_<type> detects overflow. +- #7824 MAINT: Allocate fewer bytes for empty arrays. +- #7847 MAINT,DOC: Fix some imp module uses and update f2py.compile docstring. +- #7849 MAINT: Fix remaining uses of deprecated Python imp module. +- #7851 BLD: Fix ATLAS version detection. +- #7896 BUG: Construct ma.array from np.array which contains padding. +- #7904 BUG: Fix float16 type not being called due to wrong ordering. +- #7917 BUG: Production install of numpy should not require nose. +- #7919 BLD: Fixed MKL detection for recent versions of this library. +- #7920 BUG: Fix for issue #7835 (ma.median of 1d). +- #7932 BUG: Monkey-patch _msvccompile.gen_lib_option like other compilers. +- #7939 BUG: Check for HAVE_LDOUBLE_DOUBLE_DOUBLE_LE in npy_math_complex. +- #7953 BUG: Guard against buggy comparisons in generic quicksort. +- #7954 BUG: Use keyword arguments to initialize Extension base class. +- #7955 BUG: Make sure numpy globals keep identity after reload. +- #7972 BUG: MSVCCompiler grows 'lib' & 'include' env strings exponentially. +- #8005 BLD: Remove __NUMPY_SETUP__ from builtins at end of setup.py. +- #8010 MAINT: Remove leftover imp module imports. +- #8020 BUG: Fix return of np.ma.count if keepdims is True and axis is None. +- #8024 BUG: Fix numpy.ma.median. +- #8031 BUG: Fix np.ma.median with only one non-masked value. +- #8044 BUG: Fix bug in NpyIter buffering with discontinuous arrays. diff --git a/doc/source/release/1.11.3-notes.rst b/doc/source/release/1.11.3-notes.rst new file mode 100644 index 000000000..8381a97f7 --- /dev/null +++ b/doc/source/release/1.11.3-notes.rst @@ -0,0 +1,25 @@ +========================== +NumPy 1.11.3 Release Notes +========================== + +Numpy 1.11.3 fixes a bug that leads to file corruption when very large files +opened in append mode are used in ``ndarray.tofile``. It supports Python +versions 2.6 - 2.7 and 3.2 - 3.5. Wheels for Linux, Windows, and OS X can be +found on PyPI. + + +Contributors to maintenance/1.11.3 +================================== + +A total of 2 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +- Charles Harris +- Pavel Potocek + + +Pull Requests Merged +==================== + +- `#8341 <https://github.com/numpy/numpy/pull/8341>`__: BUG: Fix ndarray.tofile large file corruption in append mode. +- `#8346 <https://github.com/numpy/numpy/pull/8346>`__: TST: Fix tests in PR #8341 for NumPy 1.11.x + diff --git a/doc/source/release/1.12.0-notes.rst b/doc/source/release/1.12.0-notes.rst new file mode 100644 index 000000000..711055d16 --- /dev/null +++ b/doc/source/release/1.12.0-notes.rst @@ -0,0 +1,495 @@ +========================== +NumPy 1.12.0 Release Notes +========================== + +This release supports Python 2.7 and 3.4 - 3.6. + +Highlights +========== +The NumPy 1.12.0 release contains a large number of fixes and improvements, but +few that stand out above all others. That makes picking out the highlights +somewhat arbitrary but the following may be of particular interest or indicate +areas likely to have future consequences. + +* Order of operations in ``np.einsum`` can now be optimized for large speed improvements. +* New ``signature`` argument to ``np.vectorize`` for vectorizing with core dimensions. +* The ``keepdims`` argument was added to many functions. +* New context manager for testing warnings +* Support for BLIS in numpy.distutils +* Much improved support for PyPy (not yet finished) + +Dropped Support +=============== + +* Support for Python 2.6, 3.2, and 3.3 has been dropped. + + +Added Support +============= + +* Support for PyPy 2.7 v5.6.0 has been added. While not complete (nditer + ``updateifcopy`` is not supported yet), this is a milestone for PyPy's + C-API compatibility layer. + + +Build System Changes +==================== + +* Library order is preserved, instead of being reordered to match that of + the directories. + + +Deprecations +============ + +Assignment of ndarray object's ``data`` attribute +------------------------------------------------- +Assigning the 'data' attribute is an inherently unsafe operation as pointed +out in gh-7083. Such a capability will be removed in the future. + +Unsafe int casting of the num attribute in ``linspace`` +------------------------------------------------------- +``np.linspace`` now raises DeprecationWarning when num cannot be safely +interpreted as an integer. + +Insufficient bit width parameter to ``binary_repr`` +--------------------------------------------------- +If a 'width' parameter is passed into ``binary_repr`` that is insufficient to +represent the number in base 2 (positive) or 2's complement (negative) form, +the function used to silently ignore the parameter and return a representation +using the minimal number of bits needed for the form in question. Such behavior +is now considered unsafe from a user perspective and will raise an error in the +future. + + +Future Changes +============== + +* In 1.13 NAT will always compare False except for ``NAT != NAT``, + which will be True. In short, NAT will behave like NaN +* In 1.13 ``np.average`` will preserve subclasses, to match the behavior of most + other numpy functions such as np.mean. In particular, this means calls which + returned a scalar may return a 0-d subclass object instead. + +Multiple-field manipulation of structured arrays +------------------------------------------------ +In 1.13 the behavior of structured arrays involving multiple fields will change +in two ways: + +First, indexing a structured array with multiple fields (eg, +``arr[['f1', 'f3']]``) will return a view into the original array in 1.13, +instead of a copy. Note the returned view will have extra padding bytes +corresponding to intervening fields in the original array, unlike the copy in +1.12, which will affect code such as ``arr[['f1', 'f3']].view(newdtype)``. + +Second, for numpy versions 1.6 to 1.12 assignment between structured arrays +occurs "by field name": Fields in the destination array are set to the +identically-named field in the source array or to 0 if the source does not have +a field:: + + >>> a = np.array([(1,2),(3,4)], dtype=[('x', 'i4'), ('y', 'i4')]) + >>> b = np.ones(2, dtype=[('z', 'i4'), ('y', 'i4'), ('x', 'i4')]) + >>> b[:] = a + >>> b + array([(0, 2, 1), (0, 4, 3)], + dtype=[('z', '<i4'), ('y', '<i4'), ('x', '<i4')]) + +In 1.13 assignment will instead occur "by position": The Nth field of the +destination will be set to the Nth field of the source regardless of field +name. The old behavior can be obtained by using indexing to reorder the fields +before +assignment, e.g., ``b[['x', 'y']] = a[['y', 'x']]``. + + +Compatibility notes +=================== + +DeprecationWarning to error +--------------------------- + +* Indexing with floats raises ``IndexError``, + e.g., a[0, 0.0]. +* Indexing with non-integer array_like raises ``IndexError``, + e.g., ``a['1', '2']`` +* Indexing with multiple ellipsis raises ``IndexError``, + e.g., ``a[..., ...]``. +* Non-integers used as index values raise ``TypeError``, + e.g., in ``reshape``, ``take``, and specifying reduce axis. + +FutureWarning to changed behavior +--------------------------------- + +* ``np.full`` now returns an array of the fill-value's dtype if no dtype is + given, instead of defaulting to float. +* ``np.average`` will emit a warning if the argument is a subclass of ndarray, + as the subclass will be preserved starting in 1.13. (see Future Changes) + +``power`` and ``**`` raise errors for integer to negative integer powers +------------------------------------------------------------------------ +The previous behavior depended on whether numpy scalar integers or numpy +integer arrays were involved. + +For arrays + +* Zero to negative integer powers returned least integral value. +* Both 1, -1 to negative integer powers returned correct values. +* The remaining integers returned zero when raised to negative integer powers. + +For scalars + +* Zero to negative integer powers returned least integral value. +* Both 1, -1 to negative integer powers returned correct values. +* The remaining integers sometimes returned zero, sometimes the + correct float depending on the integer type combination. + +All of these cases now raise a ``ValueError`` except for those integer +combinations whose common type is float, for instance uint64 and int8. It was +felt that a simple rule was the best way to go rather than have special +exceptions for the integer units. If you need negative powers, use an inexact +type. + +Relaxed stride checking is the default +-------------------------------------- +This will have some impact on code that assumed that ``F_CONTIGUOUS`` and +``C_CONTIGUOUS`` were mutually exclusive and could be set to determine the +default order for arrays that are now both. + +The ``np.percentile`` 'midpoint' interpolation method fixed for exact indices +----------------------------------------------------------------------------- +The 'midpoint' interpolator now gives the same result as 'lower' and 'higher' when +the two coincide. Previous behavior of 'lower' + 0.5 is fixed. + +``keepdims`` kwarg is passed through to user-class methods +---------------------------------------------------------- +numpy functions that take a ``keepdims`` kwarg now pass the value +through to the corresponding methods on ndarray sub-classes. Previously the +``keepdims`` keyword would be silently dropped. These functions now have +the following behavior: + +1. If user does not provide ``keepdims``, no keyword is passed to the underlying + method. +2. Any user-provided value of ``keepdims`` is passed through as a keyword + argument to the method. + +This will raise in the case where the method does not support a +``keepdims`` kwarg and the user explicitly passes in ``keepdims``. + +The following functions are changed: ``sum``, ``product``, +``sometrue``, ``alltrue``, ``any``, ``all``, ``amax``, ``amin``, +``prod``, ``mean``, ``std``, ``var``, ``nanmin``, ``nanmax``, +``nansum``, ``nanprod``, ``nanmean``, ``nanmedian``, ``nanvar``, +``nanstd`` + +``bitwise_and`` identity changed +-------------------------------- +The previous identity was 1, it is now -1. See entry in Improvements for +more explanation. + +ma.median warns and returns nan when unmasked invalid values are encountered +---------------------------------------------------------------------------- +Similar to unmasked median the masked median `ma.median` now emits a Runtime +warning and returns `NaN` in slices where an unmasked `NaN` is present. + +Greater consistency in ``assert_almost_equal`` +---------------------------------------------- +The precision check for scalars has been changed to match that for arrays. It +is now:: + + abs(actual - desired) < 1.5 * 10**(-decimal) + +Note that this is looser than previously documented, but agrees with the +previous implementation used in ``assert_array_almost_equal``. Due to the +change in implementation some very delicate tests may fail that did not +fail before. + +``NoseTester`` behaviour of warnings during testing +--------------------------------------------------- +When ``raise_warnings="develop"`` is given, all uncaught warnings will now +be considered a test failure. Previously only selected ones were raised. +Warnings which are not caught or raised (mostly when in release mode) +will be shown once during the test cycle similar to the default python +settings. + +``assert_warns`` and ``deprecated`` decorator more specific +----------------------------------------------------------- +The ``assert_warns`` function and context manager are now more specific +to the given warning category. This increased specificity leads to them +being handled according to the outer warning settings. This means that +no warning may be raised in cases where a wrong category warning is given +and ignored outside the context. Alternatively the increased specificity +may mean that warnings that were incorrectly ignored will now be shown +or raised. See also the new ``suppress_warnings`` context manager. +The same is true for the ``deprecated`` decorator. + +C API +----- +No changes. + + +New Features +============ + +Writeable keyword argument for ``as_strided`` +--------------------------------------------- +``np.lib.stride_tricks.as_strided`` now has a ``writeable`` +keyword argument. It can be set to False when no write operation +to the returned array is expected to avoid accidental +unpredictable writes. + +``axes`` keyword argument for ``rot90`` +--------------------------------------- +The ``axes`` keyword argument in ``rot90`` determines the plane in which the +array is rotated. It defaults to ``axes=(0,1)`` as in the original function. + +Generalized ``flip`` +-------------------- +``flipud`` and ``fliplr`` reverse the elements of an array along axis=0 and +axis=1 respectively. The newly added ``flip`` function reverses the elements of +an array along any given axis. + +* ``np.count_nonzero`` now has an ``axis`` parameter, allowing + non-zero counts to be generated on more than just a flattened + array object. + +BLIS support in ``numpy.distutils`` +----------------------------------- +Building against the BLAS implementation provided by the BLIS library is now +supported. See the ``[blis]`` section in ``site.cfg.example`` (in the root of +the numpy repo or source distribution). + +Hook in ``numpy/__init__.py`` to run distribution-specific checks +----------------------------------------------------------------- +Binary distributions of numpy may need to run specific hardware checks or load +specific libraries during numpy initialization. For example, if we are +distributing numpy with a BLAS library that requires SSE2 instructions, we +would like to check the machine on which numpy is running does have SSE2 in +order to give an informative error. + +Add a hook in ``numpy/__init__.py`` to import a ``numpy/_distributor_init.py`` +file that will remain empty (bar a docstring) in the standard numpy source, +but that can be overwritten by people making binary distributions of numpy. + +New nanfunctions ``nancumsum`` and ``nancumprod`` added +------------------------------------------------------- +Nan-functions ``nancumsum`` and ``nancumprod`` have been added to +compute ``cumsum`` and ``cumprod`` by ignoring nans. + +``np.interp`` can now interpolate complex values +------------------------------------------------ +``np.lib.interp(x, xp, fp)`` now allows the interpolated array ``fp`` +to be complex and will interpolate at ``complex128`` precision. + +New polynomial evaluation function ``polyvalfromroots`` added +------------------------------------------------------------- +The new function ``polyvalfromroots`` evaluates a polynomial at given points +from the roots of the polynomial. This is useful for higher order polynomials, +where expansion into polynomial coefficients is inaccurate at machine +precision. + +New array creation function ``geomspace`` added +----------------------------------------------- +The new function ``geomspace`` generates a geometric sequence. It is similar +to ``logspace``, but with start and stop specified directly: +``geomspace(start, stop)`` behaves the same as +``logspace(log10(start), log10(stop))``. + +New context manager for testing warnings +---------------------------------------- +A new context manager ``suppress_warnings`` has been added to the testing +utils. This context manager is designed to help reliably test warnings. +Specifically to reliably filter/ignore warnings. Ignoring warnings +by using an "ignore" filter in Python versions before 3.4.x can quickly +result in these (or similar) warnings not being tested reliably. + +The context manager allows to filter (as well as record) warnings similar +to the ``catch_warnings`` context, but allows for easier specificity. +Also printing warnings that have not been filtered or nesting the +context manager will work as expected. Additionally, it is possible +to use the context manager as a decorator which can be useful when +multiple tests give need to hide the same warning. + +New masked array functions ``ma.convolve`` and ``ma.correlate`` added +--------------------------------------------------------------------- +These functions wrapped the non-masked versions, but propagate through masked +values. There are two different propagation modes. The default causes masked +values to contaminate the result with masks, but the other mode only outputs +masks if there is no alternative. + +New ``float_power`` ufunc +------------------------- +The new ``float_power`` ufunc is like the ``power`` function except all +computation is done in a minimum precision of float64. There was a long +discussion on the numpy mailing list of how to treat integers to negative +integer powers and a popular proposal was that the ``__pow__`` operator should +always return results of at least float64 precision. The ``float_power`` +function implements that option. Note that it does not support object arrays. + +``np.loadtxt`` now supports a single integer as ``usecol`` argument +------------------------------------------------------------------- +Instead of using ``usecol=(n,)`` to read the nth column of a file +it is now allowed to use ``usecol=n``. Also the error message is +more user friendly when a non-integer is passed as a column index. + +Improved automated bin estimators for ``histogram`` +--------------------------------------------------- +Added 'doane' and 'sqrt' estimators to ``histogram`` via the ``bins`` +argument. Added support for range-restricted histograms with automated +bin estimation. + +``np.roll`` can now roll multiple axes at the same time +------------------------------------------------------- +The ``shift`` and ``axis`` arguments to ``roll`` are now broadcast against each +other, and each specified axis is shifted accordingly. + +The ``__complex__`` method has been implemented for the ndarrays +---------------------------------------------------------------- +Calling ``complex()`` on a size 1 array will now cast to a python +complex. + +``pathlib.Path`` objects now supported +-------------------------------------- +The standard ``np.load``, ``np.save``, ``np.loadtxt``, ``np.savez``, and similar +functions can now take ``pathlib.Path`` objects as an argument instead of a +filename or open file object. + +New ``bits`` attribute for ``np.finfo`` +--------------------------------------- +This makes ``np.finfo`` consistent with ``np.iinfo`` which already has that +attribute. + +New ``signature`` argument to ``np.vectorize`` +---------------------------------------------- +This argument allows for vectorizing user defined functions with core +dimensions, in the style of NumPy's +:ref:`generalized universal functions<c-api.generalized-ufuncs>`. This allows +for vectorizing a much broader class of functions. For example, an arbitrary +distance metric that combines two vectors to produce a scalar could be +vectorized with ``signature='(n),(n)->()'``. See ``np.vectorize`` for full +details. + +Emit py3kwarnings for division of integer arrays +------------------------------------------------ +To help people migrate their code bases from Python 2 to Python 3, the +python interpreter has a handy option -3, which issues warnings at runtime. +One of its warnings is for integer division:: + + $ python -3 -c "2/3" + + -c:1: DeprecationWarning: classic int division + +In Python 3, the new integer division semantics also apply to numpy arrays. +With this version, numpy will emit a similar warning:: + + $ python -3 -c "import numpy as np; np.array(2)/np.array(3)" + + -c:1: DeprecationWarning: numpy: classic int division + +numpy.sctypes now includes bytes on Python3 too +----------------------------------------------- +Previously, it included str (bytes) and unicode on Python2, but only str +(unicode) on Python3. + + +Improvements +============ + +``bitwise_and`` identity changed +-------------------------------- +The previous identity was 1 with the result that all bits except the LSB were +masked out when the reduce method was used. The new identity is -1, which +should work properly on twos complement machines as all bits will be set to +one. + +Generalized Ufuncs will now unlock the GIL +------------------------------------------ +Generalized Ufuncs, including most of the linalg module, will now unlock +the Python global interpreter lock. + +Caches in `np.fft` are now bounded in total size and item count +--------------------------------------------------------------- +The caches in `np.fft` that speed up successive FFTs of the same length can no +longer grow without bounds. They have been replaced with LRU (least recently +used) caches that automatically evict no longer needed items if either the +memory size or item count limit has been reached. + +Improved handling of zero-width string/unicode dtypes +----------------------------------------------------- +Fixed several interfaces that explicitly disallowed arrays with zero-width +string dtypes (i.e. ``dtype('S0')`` or ``dtype('U0')``, and fixed several +bugs where such dtypes were not handled properly. In particular, changed +``ndarray.__new__`` to not implicitly convert ``dtype('S0')`` to +``dtype('S1')`` (and likewise for unicode) when creating new arrays. + +Integer ufuncs vectorized with AVX2 +----------------------------------- +If the cpu supports it at runtime the basic integer ufuncs now use AVX2 +instructions. This feature is currently only available when compiled with GCC. + +Order of operations optimization in ``np.einsum`` +-------------------------------------------------- +``np.einsum`` now supports the ``optimize`` argument which will optimize the +order of contraction. For example, ``np.einsum`` would complete the chain dot +example ``np.einsum(‘ij,jk,kl->il’, a, b, c)`` in a single pass which would +scale like ``N^4``; however, when ``optimize=True`` ``np.einsum`` will create +an intermediate array to reduce this scaling to ``N^3`` or effectively +``np.dot(a, b).dot(c)``. Usage of intermediate tensors to reduce scaling has +been applied to the general einsum summation notation. See ``np.einsum_path`` +for more details. + +quicksort has been changed to an introsort +------------------------------------------ +The quicksort kind of ``np.sort`` and ``np.argsort`` is now an introsort which +is regular quicksort but changing to a heapsort when not enough progress is +made. This retains the good quicksort performance while changing the worst case +runtime from ``O(N^2)`` to ``O(N*log(N))``. + +``ediff1d`` improved performance and subclass handling +------------------------------------------------------ +The ediff1d function uses an array instead on a flat iterator for the +subtraction. When to_begin or to_end is not None, the subtraction is performed +in place to eliminate a copy operation. A side effect is that certain +subclasses are handled better, namely astropy.Quantity, since the complete +array is created, wrapped, and then begin and end values are set, instead of +using concatenate. + +Improved precision of ``ndarray.mean`` for float16 arrays +--------------------------------------------------------- +The computation of the mean of float16 arrays is now carried out in float32 for +improved precision. This should be useful in packages such as Theano +where the precision of float16 is adequate and its smaller footprint is +desirable. + + +Changes +======= + +All array-like methods are now called with keyword arguments in fromnumeric.py +------------------------------------------------------------------------------ +Internally, many array-like methods in fromnumeric.py were being called with +positional arguments instead of keyword arguments as their external signatures +were doing. This caused a complication in the downstream 'pandas' library +that encountered an issue with 'numpy' compatibility. Now, all array-like +methods in this module are called with keyword arguments instead. + +Operations on np.memmap objects return numpy arrays in most cases +----------------------------------------------------------------- +Previously operations on a memmap object would misleadingly return a memmap +instance even if the result was actually not memmapped. For example, +``arr + 1`` or ``arr + arr`` would return memmap instances, although no memory +from the output array is memmapped. Version 1.12 returns ordinary numpy arrays +from these operations. + +Also, reduction of a memmap (e.g. ``.sum(axis=None``) now returns a numpy +scalar instead of a 0d memmap. + +stacklevel of warnings increased +-------------------------------- +The stacklevel for python based warnings was increased so that most warnings +will report the offending line of the user code instead of the line the +warning itself is given. Passing of stacklevel is now tested to ensure that +new warnings will receive the ``stacklevel`` argument. + +This causes warnings with the "default" or "module" filter to be shown once +for every offending user code line or user module instead of only once. On +python versions before 3.4, this can cause warnings to appear that were falsely +ignored before, which may be surprising especially in test suits. diff --git a/doc/source/release/1.12.1-notes.rst b/doc/source/release/1.12.1-notes.rst new file mode 100644 index 000000000..f67dab108 --- /dev/null +++ b/doc/source/release/1.12.1-notes.rst @@ -0,0 +1,26 @@ +========================== +NumPy 1.12.1 Release Notes +========================== + +NumPy 1.12.1 supports Python 2.7 and 3.4 - 3.6 and fixes bugs and regressions +found in NumPy 1.12.0. In particular, the regression in f2py constant parsing +is fixed. Wheels for Linux, Windows, and OSX can be found on pypi, + +Bugs Fixed +========== + +* BUG: Fix wrong future nat warning and equiv type logic error... +* BUG: Fix wrong masked median for some special cases +* DOC: Place np.average in inline code +* TST: Work around isfinite inconsistency on i386 +* BUG: Guard against replacing constants without '_' spec in f2py. +* BUG: Fix mean for float 16 non-array inputs for 1.12 +* BUG: Fix calling python api with error set and minor leaks for... +* BUG: Make iscomplexobj compatible with custom dtypes again +* BUG: Fix undefined behaviour induced by bad __array_wrap__ +* BUG: Fix MaskedArray.__setitem__ +* BUG: PPC64el machines are POWER for Fortran in f2py +* BUG: Look up methods on MaskedArray in `_frommethod` +* BUG: Remove extra digit in binary_repr at limit +* BUG: Fix deepcopy regression for empty arrays. +* BUG: Fix ma.median for empty ndarrays diff --git a/doc/source/release/1.13.0-notes.rst b/doc/source/release/1.13.0-notes.rst new file mode 100644 index 000000000..3b719db09 --- /dev/null +++ b/doc/source/release/1.13.0-notes.rst @@ -0,0 +1,556 @@ +========================== +NumPy 1.13.0 Release Notes +========================== + +This release supports Python 2.7 and 3.4 - 3.6. + + +Highlights +========== + + * Operations like ``a + b + c`` will reuse temporaries on some platforms, + resulting in less memory use and faster execution. + * Inplace operations check if inputs overlap outputs and create temporaries + to avoid problems. + * New ``__array_ufunc__`` attribute provides improved ability for classes to + override default ufunc behavior. + * New ``np.block`` function for creating blocked arrays. + + +New functions +============= + +* New ``np.positive`` ufunc. +* New ``np.divmod`` ufunc provides more efficient divmod. +* New ``np.isnat`` ufunc tests for NaT special values. +* New ``np.heaviside`` ufunc computes the Heaviside function. +* New ``np.isin`` function, improves on ``in1d``. +* New ``np.block`` function for creating blocked arrays. +* New ``PyArray_MapIterArrayCopyIfOverlap`` added to NumPy C-API. + +See below for details. + + +Deprecations +============ + +* Calling ``np.fix``, ``np.isposinf``, and ``np.isneginf`` with ``f(x, y=out)`` + is deprecated - the argument should be passed as ``f(x, out=out)``, which + matches other ufunc-like interfaces. +* Use of the C-API ``NPY_CHAR`` type number deprecated since version 1.7 will + now raise deprecation warnings at runtime. Extensions built with older f2py + versions need to be recompiled to remove the warning. +* ``np.ma.argsort``, ``np.ma.minimum.reduce``, and ``np.ma.maximum.reduce`` + should be called with an explicit `axis` argument when applied to arrays with + more than 2 dimensions, as the default value of this argument (``None``) is + inconsistent with the rest of numpy (``-1``, ``0``, and ``0``, respectively). +* ``np.ma.MaskedArray.mini`` is deprecated, as it almost duplicates the + functionality of ``np.MaskedArray.min``. Exactly equivalent behaviour + can be obtained with ``np.ma.minimum.reduce``. +* The single-argument form of ``np.ma.minimum`` and ``np.ma.maximum`` is + deprecated. ``np.maximum``. ``np.ma.minimum(x)`` should now be spelt + ``np.ma.minimum.reduce(x)``, which is consistent with how this would be done + with ``np.minimum``. +* Calling ``ndarray.conjugate`` on non-numeric dtypes is deprecated (it + should match the behavior of ``np.conjugate``, which throws an error). +* Calling ``expand_dims`` when the ``axis`` keyword does not satisfy + ``-a.ndim - 1 <= axis <= a.ndim``, where ``a`` is the array being reshaped, + is deprecated. + + +Future Changes +============== + +* Assignment between structured arrays with different field names will change + in NumPy 1.14. Previously, fields in the dst would be set to the value of the + identically-named field in the src. In numpy 1.14 fields will instead be + assigned 'by position': The n-th field of the dst will be set to the n-th + field of the src array. Note that the ``FutureWarning`` raised in NumPy 1.12 + incorrectly reported this change as scheduled for NumPy 1.13 rather than + NumPy 1.14. + + +Build System Changes +==================== + +* ``numpy.distutils`` now automatically determines C-file dependencies with + GCC compatible compilers. + + +Compatibility notes +=================== + +Error type changes +------------------ + +* ``numpy.hstack()`` now throws ``ValueError`` instead of ``IndexError`` when + input is empty. +* Functions taking an axis argument, when that argument is out of range, now + throw ``np.AxisError`` instead of a mixture of ``IndexError`` and + ``ValueError``. For backwards compatibility, ``AxisError`` subclasses both of + these. + +Tuple object dtypes +------------------- + +Support has been removed for certain obscure dtypes that were unintentionally +allowed, of the form ``(old_dtype, new_dtype)``, where either of the dtypes +is or contains the ``object`` dtype. As an exception, dtypes of the form +``(object, [('name', object)])`` are still supported due to evidence of +existing use. + +DeprecationWarning to error +--------------------------- +See Changes section for more detail. + +* ``partition``, TypeError when non-integer partition index is used. +* ``NpyIter_AdvancedNew``, ValueError when ``oa_ndim == 0`` and ``op_axes`` is NULL +* ``negative(bool_)``, TypeError when negative applied to booleans. +* ``subtract(bool_, bool_)``, TypeError when subtracting boolean from boolean. +* ``np.equal, np.not_equal``, object identity doesn't override failed comparison. +* ``np.equal, np.not_equal``, object identity doesn't override non-boolean comparison. +* Deprecated boolean indexing behavior dropped. See Changes below for details. +* Deprecated ``np.alterdot()`` and ``np.restoredot()`` removed. + +FutureWarning to changed behavior +--------------------------------- +See Changes section for more detail. + +* ``numpy.average`` preserves subclasses +* ``array == None`` and ``array != None`` do element-wise comparison. +* ``np.equal, np.not_equal``, object identity doesn't override comparison result. + +dtypes are now always true +-------------------------- + +Previously ``bool(dtype)`` would fall back to the default python +implementation, which checked if ``len(dtype) > 0``. Since ``dtype`` objects +implement ``__len__`` as the number of record fields, ``bool`` of scalar dtypes +would evaluate to ``False``, which was unintuitive. Now ``bool(dtype) == True`` +for all dtypes. + +``__getslice__`` and ``__setslice__`` are no longer needed in ``ndarray`` subclasses +------------------------------------------------------------------------------------ +When subclassing np.ndarray in Python 2.7, it is no longer _necessary_ to +implement ``__*slice__`` on the derived class, as ``__*item__`` will intercept +these calls correctly. + +Any code that did implement these will work exactly as before. Code that +invokes``ndarray.__getslice__`` (e.g. through ``super(...).__getslice__``) will +now issue a DeprecationWarning - ``.__getitem__(slice(start, end))`` should be +used instead. + +Indexing MaskedArrays/Constants with ``...`` (ellipsis) now returns MaskedArray +------------------------------------------------------------------------------- +This behavior mirrors that of np.ndarray, and accounts for nested arrays in +MaskedArrays of object dtype, and ellipsis combined with other forms of +indexing. + +C API changes +============= + +GUfuncs on empty arrays and NpyIter axis removal +------------------------------------------------ +It is now allowed to remove a zero-sized axis from NpyIter. Which may mean +that code removing axes from NpyIter has to add an additional check when +accessing the removed dimensions later on. + +The largest followup change is that gufuncs are now allowed to have zero-sized +inner dimensions. This means that a gufunc now has to anticipate an empty inner +dimension, while this was never possible and an error raised instead. + +For most gufuncs no change should be necessary. However, it is now possible +for gufuncs with a signature such as ``(..., N, M) -> (..., M)`` to return +a valid result if ``N=0`` without further wrapping code. + +``PyArray_MapIterArrayCopyIfOverlap`` added to NumPy C-API +---------------------------------------------------------- +Similar to ``PyArray_MapIterArray`` but with an additional ``copy_if_overlap`` +argument. If ``copy_if_overlap != 0``, checks if input has memory overlap with +any of the other arrays and make copies as appropriate to avoid problems if the +input is modified during the iteration. See the documentation for more complete +documentation. + + +New Features +============ + +``__array_ufunc__`` added +------------------------- +This is the renamed and redesigned ``__numpy_ufunc__``. Any class, ndarray +subclass or not, can define this method or set it to ``None`` in order to +override the behavior of NumPy's ufuncs. This works quite similarly to Python's +``__mul__`` and other binary operation routines. See the documentation for a +more detailed description of the implementation and behavior of this new +option. The API is provisional, we do not yet guarantee backward compatibility +as modifications may be made pending feedback. See `NEP 13`_ and +documentation_ for more details. + +.. _`NEP 13`: http://www.numpy.org/neps/nep-0013-ufunc-overrides.html +.. _documentation: https://github.com/numpy/numpy/blob/master/doc/source/reference/arrays.classes.rst + +New ``positive`` ufunc +---------------------- +This ufunc corresponds to unary `+`, but unlike `+` on an ndarray it will raise +an error if array values do not support numeric operations. + +New ``divmod`` ufunc +-------------------- +This ufunc corresponds to the Python builtin `divmod`, and is used to implement +`divmod` when called on numpy arrays. ``np.divmod(x, y)`` calculates a result +equivalent to ``(np.floor_divide(x, y), np.remainder(x, y))`` but is +approximately twice as fast as calling the functions separately. + +``np.isnat`` ufunc tests for NaT special datetime and timedelta values +---------------------------------------------------------------------- +The new ufunc ``np.isnat`` finds the positions of special NaT values +within datetime and timedelta arrays. This is analogous to ``np.isnan``. + +``np.heaviside`` ufunc computes the Heaviside function +------------------------------------------------------ +The new function ``np.heaviside(x, h0)`` (a ufunc) computes the Heaviside +function: + +.. code:: + + { 0 if x < 0, + heaviside(x, h0) = { h0 if x == 0, + { 1 if x > 0. + +``np.block`` function for creating blocked arrays +------------------------------------------------- +Add a new ``block`` function to the current stacking functions ``vstack``, +``hstack``, and ``stack``. This allows concatenation across multiple axes +simultaneously, with a similar syntax to array creation, but where elements +can themselves be arrays. For instance:: + + >>> A = np.eye(2) * 2 + >>> B = np.eye(3) * 3 + >>> np.block([ + ... [A, np.zeros((2, 3))], + ... [np.ones((3, 2)), B ] + ... ]) + array([[ 2., 0., 0., 0., 0.], + [ 0., 2., 0., 0., 0.], + [ 1., 1., 3., 0., 0.], + [ 1., 1., 0., 3., 0.], + [ 1., 1., 0., 0., 3.]]) + +While primarily useful for block matrices, this works for arbitrary dimensions +of arrays. + +It is similar to Matlab's square bracket notation for creating block matrices. + +``isin`` function, improving on ``in1d`` +---------------------------------------- +The new function ``isin`` tests whether each element of an N-dimensonal +array is present anywhere within a second array. It is an enhancement +of ``in1d`` that preserves the shape of the first array. + +Temporary elision +----------------- +On platforms providing the ``backtrace`` function NumPy will try to avoid +creating temporaries in expression involving basic numeric types. +For example ``d = a + b + c`` is transformed to ``d = a + b; d += c`` which can +improve performance for large arrays as less memory bandwidth is required to +perform the operation. + +``axes`` argument for ``unique`` +-------------------------------- +In an N-dimensional array, the user can now choose the axis along which to look +for duplicate N-1-dimensional elements using ``numpy.unique``. The original +behaviour is recovered if ``axis=None`` (default). + +``np.gradient`` now supports unevenly spaced data +------------------------------------------------- +Users can now specify a not-constant spacing for data. +In particular ``np.gradient`` can now take: + +1. A single scalar to specify a sample distance for all dimensions. +2. N scalars to specify a constant sample distance for each dimension. + i.e. ``dx``, ``dy``, ``dz``, ... +3. N arrays to specify the coordinates of the values along each dimension of F. + The length of the array must match the size of the corresponding dimension +4. Any combination of N scalars/arrays with the meaning of 2. and 3. + +This means that, e.g., it is now possible to do the following:: + + >>> f = np.array([[1, 2, 6], [3, 4, 5]], dtype=np.float) + >>> dx = 2. + >>> y = [1., 1.5, 3.5] + >>> np.gradient(f, dx, y) + [array([[ 1. , 1. , -0.5], [ 1. , 1. , -0.5]]), + array([[ 2. , 2. , 2. ], [ 2. , 1.7, 0.5]])] + +Support for returning arrays of arbitrary dimensions in ``apply_along_axis`` +---------------------------------------------------------------------------- +Previously, only scalars or 1D arrays could be returned by the function passed +to ``apply_along_axis``. Now, it can return an array of any dimensionality +(including 0D), and the shape of this array replaces the axis of the array +being iterated over. + +``.ndim`` property added to ``dtype`` to complement ``.shape`` +-------------------------------------------------------------- +For consistency with ``ndarray`` and ``broadcast``, ``d.ndim`` is a shorthand +for ``len(d.shape)``. + +Support for tracemalloc in Python 3.6 +------------------------------------- +NumPy now supports memory tracing with tracemalloc_ module of Python 3.6 or +newer. Memory allocations from NumPy are placed into the domain defined by +``numpy.lib.tracemalloc_domain``. +Note that NumPy allocation will not show up in tracemalloc_ of earlier Python +versions. + +.. _tracemalloc: https://docs.python.org/3/library/tracemalloc.html + +NumPy may be built with relaxed stride checking debugging +--------------------------------------------------------- +Setting NPY_RELAXED_STRIDES_DEBUG=1 in the environment when relaxed stride +checking is enabled will cause NumPy to be compiled with the affected strides +set to the maximum value of npy_intp in order to help detect invalid usage of +the strides in downstream projects. When enabled, invalid usage often results +in an error being raised, but the exact type of error depends on the details of +the code. TypeError and OverflowError have been observed in the wild. + +It was previously the case that this option was disabled for releases and +enabled in master and changing between the two required editing the code. It is +now disabled by default but can be enabled for test builds. + + +Improvements +============ + +Ufunc behavior for overlapping inputs +------------------------------------- + +Operations where ufunc input and output operands have memory overlap +produced undefined results in previous NumPy versions, due to data +dependency issues. In NumPy 1.13.0, results from such operations are +now defined to be the same as for equivalent operations where there is +no memory overlap. + +Operations affected now make temporary copies, as needed to eliminate +data dependency. As detecting these cases is computationally +expensive, a heuristic is used, which may in rare cases result to +needless temporary copies. For operations where the data dependency +is simple enough for the heuristic to analyze, temporary copies will +not be made even if the arrays overlap, if it can be deduced copies +are not necessary. As an example,``np.add(a, b, out=a)`` will not +involve copies. + +To illustrate a previously undefined operation:: + + >>> x = np.arange(16).astype(float) + >>> np.add(x[1:], x[:-1], out=x[1:]) + +In NumPy 1.13.0 the last line is guaranteed to be equivalent to:: + + >>> np.add(x[1:].copy(), x[:-1].copy(), out=x[1:]) + +A similar operation with simple non-problematic data dependence is:: + + >>> x = np.arange(16).astype(float) + >>> np.add(x[1:], x[:-1], out=x[:-1]) + +It will continue to produce the same results as in previous NumPy +versions, and will not involve unnecessary temporary copies. + +The change applies also to in-place binary operations, for example:: + + >>> x = np.random.rand(500, 500) + >>> x += x.T + +This statement is now guaranteed to be equivalent to ``x[...] = x + x.T``, +whereas in previous NumPy versions the results were undefined. + +Partial support for 64-bit f2py extensions with MinGW +----------------------------------------------------- +Extensions that incorporate Fortran libraries can now be built using the free +MinGW_ toolset, also under Python 3.5. This works best for extensions that only +do calculations and uses the runtime modestly (reading and writing from files, +for instance). Note that this does not remove the need for Mingwpy; if you make +extensive use of the runtime, you will most likely run into issues_. Instead, +it should be regarded as a band-aid until Mingwpy is fully functional. + +Extensions can also be compiled using the MinGW toolset using the runtime +library from the (moveable) WinPython 3.4 distribution, which can be useful for +programs with a PySide1/Qt4 front-end. + +.. _MinGW: https://sf.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/6.2.0/threads-win32/seh/ + +.. _issues: https://mingwpy.github.io/issues.html + +Performance improvements for ``packbits`` and ``unpackbits`` +------------------------------------------------------------ +The functions ``numpy.packbits`` with boolean input and ``numpy.unpackbits`` have +been optimized to be a significantly faster for contiguous data. + +Fix for PPC long double floating point information +-------------------------------------------------- +In previous versions of NumPy, the ``finfo`` function returned invalid +information about the `double double`_ format of the ``longdouble`` float type +on Power PC (PPC). The invalid values resulted from the failure of the NumPy +algorithm to deal with the variable number of digits in the significand +that are a feature of `PPC long doubles`. This release by-passes the failing +algorithm by using heuristics to detect the presence of the PPC double double +format. A side-effect of using these heuristics is that the ``finfo`` +function is faster than previous releases. + +.. _PPC long doubles: https://www.ibm.com/support/knowledgecenter/en/ssw_aix_71/com.ibm.aix.genprogc/128bit_long_double_floating-point_datatype.htm + +.. _double double: https://en.wikipedia.org/wiki/Quadruple-precision_floating-point_format#Double-double_arithmetic + +Better default repr for ``ndarray`` subclasses +---------------------------------------------- +Subclasses of ndarray with no ``repr`` specialization now correctly indent +their data and type lines. + +More reliable comparisons of masked arrays +------------------------------------------ +Comparisons of masked arrays were buggy for masked scalars and failed for +structured arrays with dimension higher than one. Both problems are now +solved. In the process, it was ensured that in getting the result for a +structured array, masked fields are properly ignored, i.e., the result is equal +if all fields that are non-masked in both are equal, thus making the behaviour +identical to what one gets by comparing an unstructured masked array and then +doing ``.all()`` over some axis. + +np.matrix with booleans elements can now be created using the string syntax +--------------------------------------------------------------------------- +``np.matrix`` failed whenever one attempts to use it with booleans, e.g., +``np.matrix('True')``. Now, this works as expected. + +More ``linalg`` operations now accept empty vectors and matrices +---------------------------------------------------------------- +All of the following functions in ``np.linalg`` now work when given input +arrays with a 0 in the last two dimensions: ``det``, ``slogdet``, ``pinv``, +``eigvals``, ``eigvalsh``, ``eig``, ``eigh``. + +Bundled version of LAPACK is now 3.2.2 +-------------------------------------- +NumPy comes bundled with a minimal implementation of lapack for systems without +a lapack library installed, under the name of ``lapack_lite``. This has been +upgraded from LAPACK 3.0.0 (June 30, 1999) to LAPACK 3.2.2 (June 30, 2010). See +the `LAPACK changelogs`_ for details on the all the changes this entails. + +While no new features are exposed through ``numpy``, this fixes some bugs +regarding "workspace" sizes, and in some places may use faster algorithms. + +.. _`LAPACK changelogs`: http://www.netlib.org/lapack/release_notes.html#_4_history_of_lapack_releases + +``reduce`` of ``np.hypot.reduce`` and ``np.logical_xor`` allowed in more cases +------------------------------------------------------------------------------ +This now works on empty arrays, returning 0, and can reduce over multiple axes. +Previously, a ``ValueError`` was thrown in these cases. + +Better ``repr`` of object arrays +-------------------------------- +Object arrays that contain themselves no longer cause a recursion error. + +Object arrays that contain ``list`` objects are now printed in a way that makes +clear the difference between a 2d object array, and a 1d object array of lists. + +Changes +======= + +``argsort`` on masked arrays takes the same default arguments as ``sort`` +------------------------------------------------------------------------- +By default, ``argsort`` now places the masked values at the end of the sorted +array, in the same way that ``sort`` already did. Additionally, the +``end_with`` argument is added to ``argsort``, for consistency with ``sort``. +Note that this argument is not added at the end, so breaks any code that +passed ``fill_value`` as a positional argument. + +``average`` now preserves subclasses +------------------------------------ +For ndarray subclasses, ``numpy.average`` will now return an instance of the +subclass, matching the behavior of most other NumPy functions such as ``mean``. +As a consequence, also calls that returned a scalar may now return a subclass +array scalar. + +``array == None`` and ``array != None`` do element-wise comparison +------------------------------------------------------------------ +Previously these operations returned scalars ``False`` and ``True`` respectively. + +``np.equal, np.not_equal`` for object arrays ignores object identity +-------------------------------------------------------------------- +Previously, these functions always treated identical objects as equal. This had +the effect of overriding comparison failures, comparison of objects that did +not return booleans, such as np.arrays, and comparison of objects where the +results differed from object identity, such as NaNs. + +Boolean indexing changes +------------------------ +* Boolean array-likes (such as lists of python bools) are always treated as + boolean indexes. + +* Boolean scalars (including python ``True``) are legal boolean indexes and + never treated as integers. + +* Boolean indexes must match the dimension of the axis that they index. + +* Boolean indexes used on the lhs of an assignment must match the dimensions of + the rhs. + +* Boolean indexing into scalar arrays return a new 1-d array. This means that + ``array(1)[array(True)]`` gives ``array([1])`` and not the original array. + +``np.random.multivariate_normal`` behavior with bad covariance matrix +--------------------------------------------------------------------- + +It is now possible to adjust the behavior the function will have when dealing +with the covariance matrix by using two new keyword arguments: + +* ``tol`` can be used to specify a tolerance to use when checking that + the covariance matrix is positive semidefinite. + +* ``check_valid`` can be used to configure what the function will do in the + presence of a matrix that is not positive semidefinite. Valid options are + ``ignore``, ``warn`` and ``raise``. The default value, ``warn`` keeps the + the behavior used on previous releases. + +``assert_array_less`` compares ``np.inf`` and ``-np.inf`` now +------------------------------------------------------------- +Previously, ``np.testing.assert_array_less`` ignored all infinite values. This +is not the expected behavior both according to documentation and intuitively. +Now, -inf < x < inf is considered ``True`` for any real number x and all +other cases fail. + +``assert_array_`` and masked arrays ``assert_equal`` hide less warnings +----------------------------------------------------------------------- +Some warnings that were previously hidden by the ``assert_array_`` +functions are not hidden anymore. In most cases the warnings should be +correct and, should they occur, will require changes to the tests using +these functions. +For the masked array ``assert_equal`` version, warnings may occur when +comparing NaT. The function presently does not handle NaT or NaN +specifically and it may be best to avoid it at this time should a warning +show up due to this change. + +``offset`` attribute value in ``memmap`` objects +------------------------------------------------ +The ``offset`` attribute in a ``memmap`` object is now set to the +offset into the file. This is a behaviour change only for offsets +greater than ``mmap.ALLOCATIONGRANULARITY``. + +``np.real`` and ``np.imag`` return scalars for scalar inputs +------------------------------------------------------------ +Previously, ``np.real`` and ``np.imag`` used to return array objects when +provided a scalar input, which was inconsistent with other functions like +``np.angle`` and ``np.conj``. + +The polynomial convenience classes cannot be passed to ufuncs +------------------------------------------------------------- +The ABCPolyBase class, from which the convenience classes are derived, sets +``__array_ufun__ = None`` in order of opt out of ufuncs. If a polynomial +convenience class instance is passed as an argument to a ufunc, a ``TypeError`` +will now be raised. + +Output arguments to ufuncs can be tuples also for ufunc methods +--------------------------------------------------------------- +For calls to ufuncs, it was already possible, and recommended, to use an +``out`` argument with a tuple for ufuncs with multiple outputs. This has now +been extended to output arguments in the ``reduce``, ``accumulate``, and +``reduceat`` methods. This is mostly for compatibility with ``__array_ufunc``; +there are no ufuncs yet that have more than one output. diff --git a/doc/source/release/1.13.1-notes.rst b/doc/source/release/1.13.1-notes.rst new file mode 100644 index 000000000..88a4bc3dd --- /dev/null +++ b/doc/source/release/1.13.1-notes.rst @@ -0,0 +1,60 @@ +========================== +NumPy 1.13.1 Release Notes +========================== + +This is a bugfix release for problems found in 1.13.0. The major changes are +fixes for the new memory overlap detection and temporary elision as well as +reversion of the removal of the boolean binary ``-`` operator. Users of 1.13.0 +should upgrade. + +Thr Python versions supported are 2.7 and 3.4 - 3.6. Note that the Python 3.6 +wheels available from PIP are built against 3.6.1, hence will not work when +used with 3.6.0 due to Python bug 29943_. NumPy 1.13.2 will be released shortly +after Python 3.6.2 is out to fix that problem. If you are using 3.6.0 the +workaround is to upgrade to 3.6.1 or use an earlier Python version. + +.. _29943: https://bugs.python.org/issue29943 + + +Pull requests merged +==================== +A total of 19 pull requests were merged for this release. + +* #9240 DOC: BLD: fix lots of Sphinx warnings/errors. +* #9255 Revert "DEP: Raise TypeError for subtract(bool, bool)." +* #9261 BUG: don't elide into readonly and updateifcopy temporaries for... +* #9262 BUG: fix missing keyword rename for common block in numpy.f2py +* #9263 BUG: handle resize of 0d array +* #9267 DOC: update f2py front page and some doc build metadata. +* #9299 BUG: Fix Intel compilation on Unix. +* #9317 BUG: fix wrong ndim used in empty where check +* #9319 BUG: Make extensions compilable with MinGW on Py2.7 +* #9339 BUG: Prevent crash if ufunc doc string is null +* #9340 BUG: umath: un-break ufunc where= when no out= is given +* #9371 DOC: Add isnat/positive ufunc to documentation +* #9372 BUG: Fix error in fromstring function from numpy.core.records... +* #9373 BUG: ')' is printed at the end pointer of the buffer in numpy.f2py. +* #9374 DOC: Create NumPy 1.13.1 release notes. +* #9376 BUG: Prevent hang traversing ufunc userloop linked list +* #9377 DOC: Use x1 and x2 in the heaviside docstring. +* #9378 DOC: Add $PARAMS to the isnat docstring +* #9379 DOC: Update the 1.13.1 release notes + + +Contributors +============ +A total of 12 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Andras Deak + +* Bob Eldering + +* Charles Harris +* Daniel Hrisca + +* Eric Wieser +* Joshua Leahy + +* Julian Taylor +* Michael Seifert +* Pauli Virtanen +* Ralf Gommers +* Roland Kaufmann +* Warren Weckesser diff --git a/doc/source/release/1.13.2-notes.rst b/doc/source/release/1.13.2-notes.rst new file mode 100644 index 000000000..f2f9120f5 --- /dev/null +++ b/doc/source/release/1.13.2-notes.rst @@ -0,0 +1,58 @@ +========================== +NumPy 1.13.2 Release Notes +========================== + +This is a bugfix release for some problems found since 1.13.1. The most +important fixes are for CVE-2017-12852 and temporary elision. Users of earlier +versions of 1.13 should upgrade. + +The Python versions supported are 2.7 and 3.4 - 3.6. The Python 3.6 wheels +available from PIP are built with Python 3.6.2 and should be compatible with +all previous versions of Python 3.6. The Windows wheels are now built +with OpenBlas instead ATLAS, which should improve the performance of the linear +algebra functions. + +Contributors +============ + +A total of 12 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Allan Haldane +* Brandon Carter +* Charles Harris +* Eric Wieser +* Iryna Shcherbina + +* James Bourbeau + +* Jonathan Helmus +* Julian Taylor +* Matti Picus +* Michael Lamparski + +* Michael Seifert +* Ralf Gommers + +Pull requests merged +==================== + +A total of 20 pull requests were merged for this release. + +* #9390 BUG: Return the poly1d coefficients array directly +* #9555 BUG: Fix regression in 1.13.x in distutils.mingw32ccompiler. +* #9556 BUG: Fix true_divide when dtype=np.float64 specified. +* #9557 DOC: Fix some rst markup in numpy/doc/basics.py. +* #9558 BLD: Remove -xhost flag from IntelFCompiler. +* #9559 DOC: Removes broken docstring example (source code, png, pdf)... +* #9580 BUG: Add hypot and cabs functions to WIN32 blacklist. +* #9732 BUG: Make scalar function elision check if temp is writeable. +* #9736 BUG: Various fixes to np.gradient +* #9742 BUG: Fix np.pad for CVE-2017-12852 +* #9744 BUG: Check for exception in sort functions, add tests +* #9745 DOC: Add whitespace after "versionadded::" directive so it actually... +* #9746 BUG: Memory leak in np.dot of size 0 +* #9747 BUG: Adjust gfortran version search regex +* #9757 BUG: Cython 0.27 breaks NumPy on Python 3. +* #9764 BUG: Ensure `_npy_scaled_cexp{,f,l}` is defined when needed. +* #9765 BUG: PyArray_CountNonzero does not check for exceptions +* #9766 BUG: Fixes histogram monotonicity check for unsigned bin values +* #9767 BUG: Ensure consistent result dtype of count_nonzero +* #9771 BUG, MAINT: Fix mtrand for Cython 0.27. diff --git a/doc/source/release/1.13.3-notes.rst b/doc/source/release/1.13.3-notes.rst new file mode 100644 index 000000000..7f7170bcc --- /dev/null +++ b/doc/source/release/1.13.3-notes.rst @@ -0,0 +1,64 @@ +========================== +NumPy 1.13.3 Release Notes +========================== + +This is a bugfix release for some problems found since 1.13.1. The most +important fixes are for CVE-2017-12852 and temporary elision. Users of earlier +versions of 1.13 should upgrade. + +The Python versions supported are 2.7 and 3.4 - 3.6. The Python 3.6 wheels +available from PIP are built with Python 3.6.2 and should be compatible with +all previous versions of Python 3.6. It was cythonized with Cython 0.26.1, +which should be free of the bugs found in 0.27 while also being compatible with +Python 3.7-dev. The Windows wheels were built with OpenBlas instead ATLAS, +which should improve the performance of the linear algebra functions. + +The NumPy 1.13.3 release is a re-release of 1.13.2, which suffered from a +bug in Cython 0.27.0. + +Contributors +============ + +A total of 12 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Allan Haldane +* Brandon Carter +* Charles Harris +* Eric Wieser +* Iryna Shcherbina + +* James Bourbeau + +* Jonathan Helmus +* Julian Taylor +* Matti Picus +* Michael Lamparski + +* Michael Seifert +* Ralf Gommers + +Pull requests merged +==================== + +A total of 22 pull requests were merged for this release. + +* #9390 BUG: Return the poly1d coefficients array directly +* #9555 BUG: Fix regression in 1.13.x in distutils.mingw32ccompiler. +* #9556 BUG: Fix true_divide when dtype=np.float64 specified. +* #9557 DOC: Fix some rst markup in numpy/doc/basics.py. +* #9558 BLD: Remove -xhost flag from IntelFCompiler. +* #9559 DOC: Removes broken docstring example (source code, png, pdf)... +* #9580 BUG: Add hypot and cabs functions to WIN32 blacklist. +* #9732 BUG: Make scalar function elision check if temp is writeable. +* #9736 BUG: Various fixes to np.gradient +* #9742 BUG: Fix np.pad for CVE-2017-12852 +* #9744 BUG: Check for exception in sort functions, add tests +* #9745 DOC: Add whitespace after "versionadded::" directive so it actually... +* #9746 BUG: Memory leak in np.dot of size 0 +* #9747 BUG: Adjust gfortran version search regex +* #9757 BUG: Cython 0.27 breaks NumPy on Python 3. +* #9764 BUG: Ensure `_npy_scaled_cexp{,f,l}` is defined when needed. +* #9765 BUG: PyArray_CountNonzero does not check for exceptions +* #9766 BUG: Fixes histogram monotonicity check for unsigned bin values +* #9767 BUG: Ensure consistent result dtype of count_nonzero +* #9771 BUG: MAINT: Fix mtrand for Cython 0.27. +* #9772 DOC: Create the 1.13.2 release notes. +* #9794 DOC: Create 1.13.3 release notes. diff --git a/doc/source/release/1.14.0-notes.rst b/doc/source/release/1.14.0-notes.rst new file mode 100644 index 000000000..462631de6 --- /dev/null +++ b/doc/source/release/1.14.0-notes.rst @@ -0,0 +1,656 @@ +========================== +NumPy 1.14.0 Release Notes +========================== + +Numpy 1.14.0 is the result of seven months of work and contains a large number +of bug fixes and new features, along with several changes with potential +compatibility issues. The major change that users will notice are the +stylistic changes in the way numpy arrays and scalars are printed, a change +that will affect doctests. See below for details on how to preserve the +old style printing when needed. + +A major decision affecting future development concerns the schedule for +dropping Python 2.7 support in the runup to 2020. The decision has been made to +support 2.7 for all releases made in 2018, with the last release being +designated a long term release with support for bug fixes extending through +2019. In 2019 support for 2.7 will be dropped in all new releases. More details +can be found in `NEP 12`_. + +This release supports Python 2.7 and 3.4 - 3.6. + +.. _`NEP 12`: http://www.numpy.org/neps/nep-0014-dropping-python2.7-proposal.html + + +Highlights +========== + +* The `np.einsum` function uses BLAS when possible + +* ``genfromtxt``, ``loadtxt``, ``fromregex`` and ``savetxt`` can now handle + files with arbitrary Python supported encoding. + +* Major improvements to printing of NumPy arrays and scalars. + + +New functions +============= + +* ``parametrize``: decorator added to numpy.testing + +* ``chebinterpolate``: Interpolate function at Chebyshev points. + +* ``format_float_positional`` and ``format_float_scientific`` : format + floating-point scalars unambiguously with control of rounding and padding. + +* ``PyArray_ResolveWritebackIfCopy`` and ``PyArray_SetWritebackIfCopyBase``, + new C-API functions useful in achieving PyPy compatibity. + + +Deprecations +============ + +* Using ``np.bool_`` objects in place of integers is deprecated. Previously + ``operator.index(np.bool_)`` was legal and allowed constructs such as + ``[1, 2, 3][np.True_]``. That was misleading, as it behaved differently from + ``np.array([1, 2, 3])[np.True_]``. + +* Truth testing of an empty array is deprecated. To check if an array is not + empty, use ``array.size > 0``. + +* Calling ``np.bincount`` with ``minlength=None`` is deprecated. + ``minlength=0`` should be used instead. + +* Calling ``np.fromstring`` with the default value of the ``sep`` argument is + deprecated. When that argument is not provided, a broken version of + ``np.frombuffer`` is used that silently accepts unicode strings and -- after + encoding them as either utf-8 (python 3) or the default encoding + (python 2) -- treats them as binary data. If reading binary data is + desired, ``np.frombuffer`` should be used directly. + +* The ``style`` option of array2string is deprecated in non-legacy printing mode. + +* ``PyArray_SetUpdateIfCopyBase`` has been deprecated. For NumPy versions >= 1.14 + use ``PyArray_SetWritebackIfCopyBase`` instead, see `C API changes` below for + more details. + + + +* The use of ``UPDATEIFCOPY`` arrays is deprecated, see `C API changes` below + for details. We will not be dropping support for those arrays, but they are + not compatible with PyPy. + + +Future Changes +============== + +* ``np.issubdtype`` will stop downcasting dtype-like arguments. + It might be expected that ``issubdtype(np.float32, 'float64')`` and + ``issubdtype(np.float32, np.float64)`` mean the same thing - however, there + was an undocumented special case that translated the former into + ``issubdtype(np.float32, np.floating)``, giving the surprising result of True. + + This translation now gives a warning that explains what translation is + occurring. In the future, the translation will be disabled, and the first + example will be made equivalent to the second. + +* ``np.linalg.lstsq`` default for ``rcond`` will be changed. The ``rcond`` + parameter to ``np.linalg.lstsq`` will change its default to machine precision + times the largest of the input array dimensions. A FutureWarning is issued + when ``rcond`` is not passed explicitly. + +* ``a.flat.__array__()`` will return a writeable copy of ``a`` when ``a`` is + non-contiguous. Previously it returned an UPDATEIFCOPY array when ``a`` was + writeable. Currently it returns a non-writeable copy. See gh-7054 for a + discussion of the issue. + +* Unstructured void array's ``.item`` method will return a bytes object. In the + future, calling ``.item()`` on arrays or scalars of ``np.void`` datatype will + return a ``bytes`` object instead of a buffer or int array, the same as + returned by ``bytes(void_scalar)``. This may affect code which assumed the + return value was mutable, which will no longer be the case. A + ``FutureWarning`` is now issued when this would occur. + + +Compatibility notes +=================== + +The mask of a masked array view is also a view rather than a copy +----------------------------------------------------------------- +There was a FutureWarning about this change in NumPy 1.11.x. In short, it is +now the case that, when changing a view of a masked array, changes to the mask +are propagated to the original. That was not previously the case. This change +affects slices in particular. Note that this does not yet work properly if the +mask of the original array is ``nomask`` and the mask of the view is changed. +See gh-5580 for an extended discussion. The original behavior of having a copy +of the mask can be obtained by calling the ``unshare_mask`` method of the view. + +``np.ma.masked`` is no longer writeable +--------------------------------------- +Attempts to mutate the ``masked`` constant now error, as the underlying arrays +are marked readonly. In the past, it was possible to get away with:: + + # emulating a function that sometimes returns np.ma.masked + val = random.choice([np.ma.masked, 10]) + var_arr = np.asarray(val) + val_arr += 1 # now errors, previously changed np.ma.masked.data + +``np.ma`` functions producing ``fill_value`` s have changed +----------------------------------------------------------- +Previously, ``np.ma.default_fill_value`` would return a 0d array, but +``np.ma.minimum_fill_value`` and ``np.ma.maximum_fill_value`` would return a +tuple of the fields. Instead, all three methods return a structured ``np.void`` +object, which is what you would already find in the ``.fill_value`` attribute. + +Additionally, the dtype guessing now matches that of ``np.array`` - so when +passing a python scalar ``x``, ``maximum_fill_value(x)`` is always the same as +``maximum_fill_value(np.array(x))``. Previously ``x = long(1)`` on Python 2 +violated this assumption. + +``a.flat.__array__()`` returns non-writeable arrays when ``a`` is non-contiguous +-------------------------------------------------------------------------------- +The intent is that the UPDATEIFCOPY array previously returned when ``a`` was +non-contiguous will be replaced by a writeable copy in the future. This +temporary measure is aimed to notify folks who expect the underlying array be +modified in this situation that that will no longer be the case. The most +likely places for this to be noticed is when expressions of the form +``np.asarray(a.flat)`` are used, or when ``a.flat`` is passed as the out +parameter to a ufunc. + +``np.tensordot`` now returns zero array when contracting over 0-length dimension +-------------------------------------------------------------------------------- +Previously ``np.tensordot`` raised a ValueError when contracting over 0-length +dimension. Now it returns a zero array, which is consistent with the behaviour +of ``np.dot`` and ``np.einsum``. + +``numpy.testing`` reorganized +----------------------------- +This is not expected to cause problems, but possibly something has been left +out. If you experience an unexpected import problem using ``numpy.testing`` +let us know. + +``np.asfarray`` no longer accepts non-dtypes through the ``dtype`` argument +--------------------------------------------------------------------------- +This previously would accept ``dtype=some_array``, with the implied semantics +of ``dtype=some_array.dtype``. This was undocumented, unique across the numpy +functions, and if used would likely correspond to a typo. + +1D ``np.linalg.norm`` preserves float input types, even for arbitrary orders +---------------------------------------------------------------------------- +Previously, this would promote to ``float64`` when arbitrary orders were +passed, despite not doing so under the simple cases:: + + >>> f32 = np.float32([[1, 2]]) + >>> np.linalg.norm(f32, 2.0, axis=-1).dtype + dtype('float32') + >>> np.linalg.norm(f32, 2.0001, axis=-1).dtype + dtype('float64') # numpy 1.13 + dtype('float32') # numpy 1.14 + +This change affects only ``float32`` and ``float16`` arrays. + +``count_nonzero(arr, axis=())`` now counts over no axes, not all axes +--------------------------------------------------------------------- +Elsewhere, ``axis==()`` is always understood as "no axes", but +`count_nonzero` had a special case to treat this as "all axes". This was +inconsistent and surprising. The correct way to count over all axes has always +been to pass ``axis == None``. + +``__init__.py`` files added to test directories +----------------------------------------------- +This is for pytest compatibility in the case of duplicate test file names in +the different directories. As a result, ``run_module_suite`` no longer works, +i.e., ``python <path-to-test-file>`` results in an error. + +``.astype(bool)`` on unstructured void arrays now calls ``bool`` on each element +-------------------------------------------------------------------------------- +On Python 2, ``void_array.astype(bool)`` would always return an array of +``True``, unless the dtype is ``V0``. On Python 3, this operation would usually +crash. Going forwards, `astype` matches the behavior of ``bool(np.void)``, +considering a buffer of all zeros as false, and anything else as true. +Checks for ``V0`` can still be done with ``arr.dtype.itemsize == 0``. + +``MaskedArray.squeeze`` never returns ``np.ma.masked`` +------------------------------------------------------ +``np.squeeze`` is documented as returning a view, but the masked variant would +sometimes return ``masked``, which is not a view. This has been fixed, so that +the result is always a view on the original masked array. +This breaks any code that used ``masked_arr.squeeze() is np.ma.masked``, but +fixes code that writes to the result of `.squeeze()`. + +Renamed first parameter of ``can_cast`` from ``from`` to ``from_`` +------------------------------------------------------------------ +The previous parameter name ``from`` is a reserved keyword in Python, which made +it difficult to pass the argument by name. This has been fixed by renaming +the parameter to ``from_``. + +``isnat`` raises ``TypeError`` when passed wrong type +------------------------------------------------------ +The ufunc ``isnat`` used to raise a ``ValueError`` when it was not passed +variables of type ``datetime`` or ``timedelta``. This has been changed to +raising a ``TypeError``. + +``dtype.__getitem__`` raises ``TypeError`` when passed wrong type +----------------------------------------------------------------- +When indexed with a float, the dtype object used to raise ``ValueError``. + +User-defined types now need to implement ``__str__`` and ``__repr__`` +--------------------------------------------------------------------- +Previously, user-defined types could fall back to a default implementation of +``__str__`` and ``__repr__`` implemented in numpy, but this has now been +removed. Now user-defined types will fall back to the python default +``object.__str__`` and ``object.__repr__``. + +Many changes to array printing, disableable with the new "legacy" printing mode +------------------------------------------------------------------------------- +The ``str`` and ``repr`` of ndarrays and numpy scalars have been changed in +a variety of ways. These changes are likely to break downstream user's +doctests. + +These new behaviors can be disabled to mostly reproduce numpy 1.13 behavior by +enabling the new 1.13 "legacy" printing mode. This is enabled by calling +``np.set_printoptions(legacy="1.13")``, or using the new ``legacy`` argument to +``np.array2string``, as ``np.array2string(arr, legacy='1.13')``. + +In summary, the major changes are: + +* For floating-point types: + + * The ``repr`` of float arrays often omits a space previously printed + in the sign position. See the new ``sign`` option to ``np.set_printoptions``. + * Floating-point arrays and scalars use a new algorithm for decimal + representations, giving the shortest unique representation. This will + usually shorten ``float16`` fractional output, and sometimes ``float32`` and + ``float128`` output. ``float64`` should be unaffected. See the new + ``floatmode`` option to ``np.set_printoptions``. + * Float arrays printed in scientific notation no longer use fixed-precision, + and now instead show the shortest unique representation. + * The ``str`` of floating-point scalars is no longer truncated in python2. + +* For other data types: + + * Non-finite complex scalars print like ``nanj`` instead of ``nan*j``. + * ``NaT`` values in datetime arrays are now properly aligned. + * Arrays and scalars of ``np.void`` datatype are now printed using hex + notation. + +* For line-wrapping: + + * The "dtype" part of ndarray reprs will now be printed on the next line + if there isn't space on the last line of array output. + * The ``linewidth`` format option is now always respected. + The `repr` or `str` of an array will never exceed this, unless a single + element is too wide. + * The last line of an array string will never have more elements than earlier + lines. + * An extra space is no longer inserted on the first line if the elements are + too wide. + +* For summarization (the use of ``...`` to shorten long arrays): + + * A trailing comma is no longer inserted for ``str``. + Previously, ``str(np.arange(1001))`` gave + ``'[ 0 1 2 ..., 998 999 1000]'``, which has an extra comma. + * For arrays of 2-D and beyond, when ``...`` is printed on its own line in + order to summarize any but the last axis, newlines are now appended to that + line to match its leading newlines and a trailing space character is + removed. + +* ``MaskedArray`` arrays now separate printed elements with commas, always + print the dtype, and correctly wrap the elements of long arrays to multiple + lines. If there is more than 1 dimension, the array attributes are now + printed in a new "left-justified" printing style. +* ``recarray`` arrays no longer print a trailing space before their dtype, and + wrap to the right number of columns. +* 0d arrays no longer have their own idiosyncratic implementations of ``str`` + and ``repr``. The ``style`` argument to ``np.array2string`` is deprecated. +* Arrays of ``bool`` datatype will omit the datatype in the ``repr``. +* User-defined ``dtypes`` (subclasses of ``np.generic``) now need to + implement ``__str__`` and ``__repr__``. + +Some of these changes are described in more detail below. If you need to retain +the previous behavior for doctests or other reasons, you may want to do +something like:: + + # FIXME: We need the str/repr formatting used in Numpy < 1.14. + try: + np.set_printoptions(legacy='1.13') + except TypeError: + pass + + +C API changes +============= + +PyPy compatible alternative to ``UPDATEIFCOPY`` arrays +------------------------------------------------------ +``UPDATEIFCOPY`` arrays are contiguous copies of existing arrays, possibly with +different dimensions, whose contents are copied back to the original array when +their refcount goes to zero and they are deallocated. Because PyPy does not use +refcounts, they do not function correctly with PyPy. NumPy is in the process of +eliminating their use internally and two new C-API functions, + +* ``PyArray_SetWritebackIfCopyBase`` +* ``PyArray_ResolveWritebackIfCopy``, + +have been added together with a complimentary flag, +``NPY_ARRAY_WRITEBACKIFCOPY``. Using the new functionality also requires that +some flags be changed when new arrays are created, to wit: +``NPY_ARRAY_INOUT_ARRAY`` should be replaced by ``NPY_ARRAY_INOUT_ARRAY2`` and +``NPY_ARRAY_INOUT_FARRAY`` should be replaced by ``NPY_ARRAY_INOUT_FARRAY2``. +Arrays created with these new flags will then have the ``WRITEBACKIFCOPY`` +semantics. + +If PyPy compatibility is not a concern, these new functions can be ignored, +although there will be a ``DeprecationWarning``. If you do wish to pursue PyPy +compatibility, more information on these functions and their use may be found +in the c-api_ documentation and the example in how-to-extend_. + +.. _c-api: https://github.com/numpy/numpy/blob/master/doc/source/reference/c-api.array.rst +.. _how-to-extend: https://github.com/numpy/numpy/blob/master/doc/source/user/c-info.how-to-extend.rst + + +New Features +============ + +Encoding argument for text IO functions +--------------------------------------- +``genfromtxt``, ``loadtxt``, ``fromregex`` and ``savetxt`` can now handle files +with arbitrary encoding supported by Python via the encoding argument. +For backward compatibility the argument defaults to the special ``bytes`` value +which continues to treat text as raw byte values and continues to pass latin1 +encoded bytes to custom converters. +Using any other value (including ``None`` for system default) will switch the +functions to real text IO so one receives unicode strings instead of bytes in +the resulting arrays. + +External ``nose`` plugins are usable by ``numpy.testing.Tester`` +---------------------------------------------------------------- +``numpy.testing.Tester`` is now aware of ``nose`` plugins that are outside the +``nose`` built-in ones. This allows using, for example, ``nose-timer`` like +so: ``np.test(extra_argv=['--with-timer', '--timer-top-n', '20'])`` to +obtain the runtime of the 20 slowest tests. An extra keyword ``timer`` was +also added to ``Tester.test``, so ``np.test(timer=20)`` will also report the 20 +slowest tests. + +``parametrize`` decorator added to ``numpy.testing`` +---------------------------------------------------- +A basic ``parametrize`` decorator is now available in ``numpy.testing``. It is +intended to allow rewriting yield based tests that have been deprecated in +pytest so as to facilitate the transition to pytest in the future. The nose +testing framework has not been supported for several years and looks like +abandonware. + +The new ``parametrize`` decorator does not have the full functionality of the +one in pytest. It doesn't work for classes, doesn't support nesting, and does +not substitute variable names. Even so, it should be adequate to rewrite the +NumPy tests. + +``chebinterpolate`` function added to ``numpy.polynomial.chebyshev`` +-------------------------------------------------------------------- +The new ``chebinterpolate`` function interpolates a given function at the +Chebyshev points of the first kind. A new ``Chebyshev.interpolate`` class +method adds support for interpolation over arbitrary intervals using the scaled +and shifted Chebyshev points of the first kind. + +Support for reading lzma compressed text files in Python 3 +---------------------------------------------------------- +With Python versions containing the ``lzma`` module the text IO functions can +now transparently read from files with ``xz`` or ``lzma`` extension. + +``sign`` option added to ``np.setprintoptions`` and ``np.array2string`` +----------------------------------------------------------------------- +This option controls printing of the sign of floating-point types, and may be +one of the characters '-', '+' or ' '. With '+' numpy always prints the sign of +positive values, with ' ' it always prints a space (whitespace character) in +the sign position of positive values, and with '-' it will omit the sign +character for positive values. The new default is '-'. + +This new default changes the float output relative to numpy 1.13. The old +behavior can be obtained in 1.13 "legacy" printing mode, see compatibility +notes above. + +``hermitian`` option added to``np.linalg.matrix_rank`` +------------------------------------------------------ +The new ``hermitian`` option allows choosing between standard SVD based matrix +rank calculation and the more efficient eigenvalue based method for +symmetric/hermitian matrices. + +``threshold`` and ``edgeitems`` options added to ``np.array2string`` +-------------------------------------------------------------------- +These options could previously be controlled using ``np.set_printoptions``, but +now can be changed on a per-call basis as arguments to ``np.array2string``. + +``concatenate`` and ``stack`` gained an ``out`` argument +-------------------------------------------------------- +A preallocated buffer of the desired dtype can now be used for the output of +these functions. + +Support for PGI flang compiler on Windows +----------------------------------------- +The PGI flang compiler is a Fortran front end for LLVM released by NVIDIA under +the Apache 2 license. It can be invoked by :: + + python setup.py config --compiler=clang --fcompiler=flang install + +There is little experience with this new compiler, so any feedback from people +using it will be appreciated. + + +Improvements +============ + +Numerator degrees of freedom in ``random.noncentral_f`` need only be positive. +------------------------------------------------------------------------------ +Prior to NumPy 1.14.0, the numerator degrees of freedom needed to be > 1, but +the distribution is valid for values > 0, which is the new requirement. + +The GIL is released for all ``np.einsum`` variations +---------------------------------------------------- +Some specific loop structures which have an accelerated loop version +did not release the GIL prior to NumPy 1.14.0. This oversight has been +fixed. + +The `np.einsum` function will use BLAS when possible and optimize by default +---------------------------------------------------------------------------- +The ``np.einsum`` function will now call ``np.tensordot`` when appropriate. +Because ``np.tensordot`` uses BLAS when possible, that will speed up execution. +By default, ``np.einsum`` will also attempt optimization as the overhead is +small relative to the potential improvement in speed. + +``f2py`` now handles arrays of dimension 0 +------------------------------------------ +``f2py`` now allows for the allocation of arrays of dimension 0. This allows +for more consistent handling of corner cases downstream. + +``numpy.distutils`` supports using MSVC and mingw64-gfortran together +--------------------------------------------------------------------- +Numpy distutils now supports using Mingw64 gfortran and MSVC compilers +together. This enables the production of Python extension modules on Windows +containing Fortran code while retaining compatibility with the +binaries distributed by Python.org. Not all use cases are supported, +but most common ways to wrap Fortran for Python are functional. + +Compilation in this mode is usually enabled automatically, and can be +selected via the ``--fcompiler`` and ``--compiler`` options to +``setup.py``. Moreover, linking Fortran codes to static OpenBLAS is +supported; by default a gfortran compatible static archive +``openblas.a`` is looked for. + +``np.linalg.pinv`` now works on stacked matrices +------------------------------------------------ +Previously it was limited to a single 2d array. + +``numpy.save`` aligns data to 64 bytes instead of 16 +---------------------------------------------------- +Saving NumPy arrays in the ``npy`` format with ``numpy.save`` inserts +padding before the array data to align it at 64 bytes. Previously +this was only 16 bytes (and sometimes less due to a bug in the code +for version 2). Now the alignment is 64 bytes, which matches the +widest SIMD instruction set commonly available, and is also the most +common cache line size. This makes ``npy`` files easier to use in +programs which open them with ``mmap``, especially on Linux where an +``mmap`` offset must be a multiple of the page size. + +NPZ files now can be written without using temporary files +---------------------------------------------------------- +In Python 3.6+ ``numpy.savez`` and ``numpy.savez_compressed`` now write +directly to a ZIP file, without creating intermediate temporary files. + +Better support for empty structured and string types +---------------------------------------------------- +Structured types can contain zero fields, and string dtypes can contain zero +characters. Zero-length strings still cannot be created directly, and must be +constructed through structured dtypes:: + + str0 = np.empty(10, np.dtype([('v', str, N)]))['v'] + void0 = np.empty(10, np.void) + +It was always possible to work with these, but the following operations are +now supported for these arrays: + + * `arr.sort()` + * `arr.view(bytes)` + * `arr.resize(...)` + * `pickle.dumps(arr)` + +Support for ``decimal.Decimal`` in ``np.lib.financial`` +------------------------------------------------------- +Unless otherwise stated all functions within the ``financial`` package now +support using the ``decimal.Decimal`` built-in type. + +Float printing now uses "dragon4" algorithm for shortest decimal representation +------------------------------------------------------------------------------- +The ``str`` and ``repr`` of floating-point values (16, 32, 64 and 128 bit) are +now printed to give the shortest decimal representation which uniquely +identifies the value from others of the same type. Previously this was only +true for ``float64`` values. The remaining float types will now often be shorter +than in numpy 1.13. Arrays printed in scientific notation now also use the +shortest scientific representation, instead of fixed precision as before. + + Additionally, the `str` of float scalars scalars will no longer be truncated + in python2, unlike python2 `float`s. `np.double` scalars now have a ``str`` + and ``repr`` identical to that of a python3 float. + +New functions ``np.format_float_scientific`` and ``np.format_float_positional`` +are provided to generate these decimal representations. + +A new option ``floatmode`` has been added to ``np.set_printoptions`` and +``np.array2string``, which gives control over uniqueness and rounding of +printed elements in an array. The new default is ``floatmode='maxprec'`` with +``precision=8``, which will print at most 8 fractional digits, or fewer if an +element can be uniquely represented with fewer. A useful new mode is +``floatmode="unique"``, which will output enough digits to specify the array +elements uniquely. + +Numpy complex-floating-scalars with values like ``inf*j`` or ``nan*j`` now +print as ``infj`` and ``nanj``, like the pure-python ``complex`` type. + +The ``FloatFormat`` and ``LongFloatFormat`` classes are deprecated and should +both be replaced by ``FloatingFormat``. Similarly ``ComplexFormat`` and +``LongComplexFormat`` should be replaced by ``ComplexFloatingFormat``. + +``void`` datatype elements are now printed in hex notation +---------------------------------------------------------- +A hex representation compatible with the python ``bytes`` type is now printed +for unstructured ``np.void`` elements, e.g., ``V4`` datatype. Previously, in +python2 the raw void data of the element was printed to stdout, or in python3 +the integer byte values were shown. + +printing style for ``void`` datatypes is now independently customizable +----------------------------------------------------------------------- +The printing style of ``np.void`` arrays is now independently customizable +using the ``formatter`` argument to ``np.set_printoptions``, using the +``'void'`` key, instead of the catch-all ``numpystr`` key as before. + +Reduced memory usage of ``np.loadtxt`` +-------------------------------------- +``np.loadtxt`` now reads files in chunks instead of all at once which decreases +its memory usage significantly for large files. + + +Changes +======= + +Multiple-field indexing/assignment of structured arrays +------------------------------------------------------- +The indexing and assignment of structured arrays with multiple fields has +changed in a number of ways, as warned about in previous releases. + +First, indexing a structured array with multiple fields, e.g., +``arr[['f1', 'f3']]``, returns a view into the original array instead of a +copy. The returned view will have extra padding bytes corresponding to +intervening fields in the original array, unlike the copy in 1.13, which will +affect code such as ``arr[['f1', 'f3']].view(newdtype)``. + +Second, assignment between structured arrays will now occur "by position" +instead of "by field name". The Nth field of the destination will be set to the +Nth field of the source regardless of field name, unlike in numpy versions 1.6 +to 1.13 in which fields in the destination array were set to the +identically-named field in the source array or to 0 if the source did not have +a field. + +Correspondingly, the order of fields in a structured dtypes now matters when +computing dtype equality. For example, with the dtypes :: + + x = dtype({'names': ['A', 'B'], 'formats': ['i4', 'f4'], 'offsets': [0, 4]}) + y = dtype({'names': ['B', 'A'], 'formats': ['f4', 'i4'], 'offsets': [4, 0]}) + +the expression ``x == y`` will now return ``False``, unlike before. +This makes dictionary based dtype specifications like +``dtype({'a': ('i4', 0), 'b': ('f4', 4)})`` dangerous in python < 3.6 +since dict key order is not preserved in those versions. + +Assignment from a structured array to a boolean array now raises a ValueError, +unlike in 1.13, where it always set the destination elements to ``True``. + +Assignment from structured array with more than one field to a non-structured +array now raises a ValueError. In 1.13 this copied just the first field of the +source to the destination. + +Using field "titles" in multiple-field indexing is now disallowed, as is +repeating a field name in a multiple-field index. + +The documentation for structured arrays in the user guide has been +significantly updated to reflect these changes. + +Integer and Void scalars are now unaffected by ``np.set_string_function`` +------------------------------------------------------------------------- +Previously, unlike most other numpy scalars, the ``str`` and ``repr`` of +integer and void scalars could be controlled by ``np.set_string_function``. +This is no longer possible. + +0d array printing changed, ``style`` arg of array2string deprecated +------------------------------------------------------------------- +Previously the ``str`` and ``repr`` of 0d arrays had idiosyncratic +implementations which returned ``str(a.item())`` and ``'array(' + +repr(a.item()) + ')'`` respectively for 0d array ``a``, unlike both numpy +scalars and higher dimension ndarrays. + +Now, the ``str`` of a 0d array acts like a numpy scalar using ``str(a[()])`` +and the ``repr`` acts like higher dimension arrays using ``formatter(a[()])``, +where ``formatter`` can be specified using ``np.set_printoptions``. The +``style`` argument of ``np.array2string`` is deprecated. + +This new behavior is disabled in 1.13 legacy printing mode, see compatibility +notes above. + +Seeding ``RandomState`` using an array requires a 1-d array +----------------------------------------------------------- +``RandomState`` previously would accept empty arrays or arrays with 2 or more +dimensions, which resulted in either a failure to seed (empty arrays) or for +some of the passed values to be ignored when setting the seed. + +``MaskedArray`` objects show a more useful ``repr`` +--------------------------------------------------- +The ``repr`` of a ``MaskedArray`` is now closer to the python code that would +produce it, with arrays now being shown with commas and dtypes. Like the other +formatting changes, this can be disabled with the 1.13 legacy printing mode in +order to help transition doctests. + +The ``repr`` of ``np.polynomial`` classes is more explicit +---------------------------------------------------------- +It now shows the domain and window parameters as keyword arguments to make +them more clear:: + + >>> np.polynomial.Polynomial(range(4)) + Polynomial([0., 1., 2., 3.], domain=[-1, 1], window=[-1, 1]) diff --git a/doc/source/release/1.14.1-notes.rst b/doc/source/release/1.14.1-notes.rst new file mode 100644 index 000000000..7b95c2e28 --- /dev/null +++ b/doc/source/release/1.14.1-notes.rst @@ -0,0 +1,92 @@ +========================== +NumPy 1.14.1 Release Notes +========================== + +This is a bugfix release for some problems reported following the 1.14.0 release. The major +problems fixed are the following. + +* Problems with the new array printing, particularly the printing of complex + values, Please report any additional problems that may turn up. +* Problems with ``np.einsum`` due to the new ``optimized=True`` default. Some + fixes for optimization have been applied and ``optimize=False`` is now the + default. +* The sort order in ``np.unique`` when ``axis=<some-number>`` will now always + be lexicographic in the subarray elements. In previous NumPy versions there + was an optimization that could result in sorting the subarrays as unsigned + byte strings. +* The change in 1.14.0 that multi-field indexing of structured arrays returns a + view instead of a copy has been reverted but remains on track for NumPy 1.15. + Affected users should read the 1.14.1 Numpy User Guide section + "basics/structured arrays/accessing multiple fields" for advice on how to + manage this transition. + +The Python versions supported in this release are 2.7 and 3.4 - 3.6. The Python +3.6 wheels available from PIP are built with Python 3.6.2 and should be +compatible with all previous versions of Python 3.6. The source releases were +cythonized with Cython 0.26.1, which is known to **not** support the upcoming +Python 3.7 release. People who wish to run Python 3.7 should check out the +NumPy repo and try building with the, as yet, unreleased master branch of +Cython. + +Contributors +============ + +A total of 14 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Allan Haldane +* Charles Harris +* Daniel Smith +* Dennis Weyland + +* Eric Larson +* Eric Wieser +* Jarrod Millman +* Kenichi Maehashi + +* Marten van Kerkwijk +* Mathieu Lamarre +* Sebastian Berg +* Simon Conseil +* Simon Gibbons +* xoviat + +Pull requests merged +==================== + +A total of 36 pull requests were merged for this release. + +* `#10339 <https://github.com/numpy/numpy/pull/10339>`__: BUG: restrict the __config__ modifications to win32 +* `#10368 <https://github.com/numpy/numpy/pull/10368>`__: MAINT: Adjust type promotion in linalg.norm +* `#10375 <https://github.com/numpy/numpy/pull/10375>`__: BUG: add missing paren and remove quotes from repr of fieldless... +* `#10395 <https://github.com/numpy/numpy/pull/10395>`__: MAINT: Update download URL in setup.py. +* `#10396 <https://github.com/numpy/numpy/pull/10396>`__: BUG: fix einsum issue with unicode input and py2 +* `#10397 <https://github.com/numpy/numpy/pull/10397>`__: BUG: fix error message not formatted in einsum +* `#10398 <https://github.com/numpy/numpy/pull/10398>`__: DOC: add documentation about how to handle new array printing +* `#10403 <https://github.com/numpy/numpy/pull/10403>`__: BUG: Set einsum optimize parameter default to `False`. +* `#10424 <https://github.com/numpy/numpy/pull/10424>`__: ENH: Fix repr of np.record objects to match np.void types #10412 +* `#10425 <https://github.com/numpy/numpy/pull/10425>`__: MAINT: Update zesty to artful for i386 testing +* `#10431 <https://github.com/numpy/numpy/pull/10431>`__: REL: Add 1.14.1 release notes template +* `#10435 <https://github.com/numpy/numpy/pull/10435>`__: MAINT: Use ValueError for duplicate field names in lookup (backport) +* `#10534 <https://github.com/numpy/numpy/pull/10534>`__: BUG: Provide a better error message for out-of-order fields +* `#10536 <https://github.com/numpy/numpy/pull/10536>`__: BUG: Resize bytes columns in genfromtxt (backport of #10401) +* `#10537 <https://github.com/numpy/numpy/pull/10537>`__: BUG: multifield-indexing adds padding bytes: revert for 1.14.1 +* `#10539 <https://github.com/numpy/numpy/pull/10539>`__: BUG: fix np.save issue with python 2.7.5 +* `#10540 <https://github.com/numpy/numpy/pull/10540>`__: BUG: Add missing DECREF in Py2 int() cast +* `#10541 <https://github.com/numpy/numpy/pull/10541>`__: TST: Add circleci document testing to maintenance/1.14.x +* `#10542 <https://github.com/numpy/numpy/pull/10542>`__: BUG: complex repr has extra spaces, missing + (1.14 backport) +* `#10550 <https://github.com/numpy/numpy/pull/10550>`__: BUG: Set missing exception after malloc +* `#10557 <https://github.com/numpy/numpy/pull/10557>`__: BUG: In numpy.i, clear CARRAY flag if wrapped buffer is not C_CONTIGUOUS. +* `#10558 <https://github.com/numpy/numpy/pull/10558>`__: DEP: Issue FutureWarning when malformed records detected. +* `#10559 <https://github.com/numpy/numpy/pull/10559>`__: BUG: Fix einsum optimize logic for singleton dimensions +* `#10560 <https://github.com/numpy/numpy/pull/10560>`__: BUG: Fix calling ufuncs with a positional output argument. +* `#10561 <https://github.com/numpy/numpy/pull/10561>`__: BUG: Fix various Big-Endian test failures (ppc64) +* `#10562 <https://github.com/numpy/numpy/pull/10562>`__: BUG: Make dtype.descr error for out-of-order fields. +* `#10563 <https://github.com/numpy/numpy/pull/10563>`__: BUG: arrays not being flattened in `union1d` +* `#10607 <https://github.com/numpy/numpy/pull/10607>`__: MAINT: Update sphinxext submodule hash. +* `#10608 <https://github.com/numpy/numpy/pull/10608>`__: BUG: Revert sort optimization in np.unique. +* `#10609 <https://github.com/numpy/numpy/pull/10609>`__: BUG: infinite recursion in str of 0d subclasses +* `#10610 <https://github.com/numpy/numpy/pull/10610>`__: BUG: Align type definition with generated lapack +* `#10612 <https://github.com/numpy/numpy/pull/10612>`__: BUG/ENH: Improve output for structured non-void types +* `#10622 <https://github.com/numpy/numpy/pull/10622>`__: BUG: deallocate recursive closure in arrayprint.py (1.14 backport) +* `#10624 <https://github.com/numpy/numpy/pull/10624>`__: BUG: Correctly identify comma separated dtype strings +* `#10629 <https://github.com/numpy/numpy/pull/10629>`__: BUG: deallocate recursive closure in arrayprint.py (backport... +* `#10630 <https://github.com/numpy/numpy/pull/10630>`__: REL: Prepare for 1.14.1 release. diff --git a/doc/source/release/1.14.2-notes.rst b/doc/source/release/1.14.2-notes.rst new file mode 100644 index 000000000..3f47cb5f5 --- /dev/null +++ b/doc/source/release/1.14.2-notes.rst @@ -0,0 +1,40 @@ +========================== +NumPy 1.14.2 Release Notes +========================== + +This is a bugfix release for some bugs reported following the 1.14.1 release. The major +problems dealt with are as follows. + +* Residual bugs in the new array printing functionality. +* Regression resulting in a relocation problem with shared library. +* Improved PyPy compatibility. + +The Python versions supported in this release are 2.7 and 3.4 - 3.6. The Python +3.6 wheels available from PIP are built with Python 3.6.2 and should be +compatible with all previous versions of Python 3.6. The source releases were +cythonized with Cython 0.26.1, which is known to **not** support the upcoming +Python 3.7 release. People who wish to run Python 3.7 should check out the +NumPy repo and try building with the, as yet, unreleased master branch of +Cython. + +Contributors +============ + +A total of 4 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Allan Haldane +* Charles Harris +* Eric Wieser +* Pauli Virtanen + +Pull requests merged +==================== + +A total of 5 pull requests were merged for this release. + +* `#10674 <https://github.com/numpy/numpy/pull/10674>`__: BUG: Further back-compat fix for subclassed array repr +* `#10725 <https://github.com/numpy/numpy/pull/10725>`__: BUG: dragon4 fractional output mode adds too many trailing zeros +* `#10726 <https://github.com/numpy/numpy/pull/10726>`__: BUG: Fix f2py generated code to work on PyPy +* `#10727 <https://github.com/numpy/numpy/pull/10727>`__: BUG: Fix missing NPY_VISIBILITY_HIDDEN on npy_longdouble_to_PyLong +* `#10729 <https://github.com/numpy/numpy/pull/10729>`__: DOC: Create 1.14.2 notes and changelog. diff --git a/doc/source/release/1.14.3-notes.rst b/doc/source/release/1.14.3-notes.rst new file mode 100644 index 000000000..60b631168 --- /dev/null +++ b/doc/source/release/1.14.3-notes.rst @@ -0,0 +1,41 @@ +========================== +NumPy 1.14.3 Release Notes +========================== + +This is a bugfix release for a few bugs reported following the 1.14.2 release: + +* np.lib.recfunctions.fromrecords accepts a list-of-lists, until 1.15 +* In python2, float types use the new print style when printing to a file +* style arg in "legacy" print mode now works for 0d arrays + +The Python versions supported in this release are 2.7 and 3.4 - 3.6. The Python +3.6 wheels available from PIP are built with Python 3.6.2 and should be +compatible with all previous versions of Python 3.6. The source releases were +cythonized with Cython 0.28.2. + +Contributors +============ + +A total of 6 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Allan Haldane +* Charles Harris +* Jonathan March + +* Malcolm Smith + +* Matti Picus +* Pauli Virtanen + +Pull requests merged +==================== + +A total of 8 pull requests were merged for this release. + +* `#10862 <https://github.com/numpy/numpy/pull/10862>`__: BUG: floating types should override tp_print (1.14 backport) +* `#10905 <https://github.com/numpy/numpy/pull/10905>`__: BUG: for 1.14 back-compat, accept list-of-lists in fromrecords +* `#10947 <https://github.com/numpy/numpy/pull/10947>`__: BUG: 'style' arg to array2string broken in legacy mode (1.14... +* `#10959 <https://github.com/numpy/numpy/pull/10959>`__: BUG: test, fix for missing flags['WRITEBACKIFCOPY'] key +* `#10960 <https://github.com/numpy/numpy/pull/10960>`__: BUG: Add missing underscore to prototype in check_embedded_lapack +* `#10961 <https://github.com/numpy/numpy/pull/10961>`__: BUG: Fix encoding regression in ma/bench.py (Issue #10868) +* `#10962 <https://github.com/numpy/numpy/pull/10962>`__: BUG: core: fix NPY_TITLE_KEY macro on pypy +* `#10974 <https://github.com/numpy/numpy/pull/10974>`__: BUG: test, fix PyArray_DiscardWritebackIfCopy... diff --git a/doc/source/release/1.14.4-notes.rst b/doc/source/release/1.14.4-notes.rst new file mode 100644 index 000000000..3fb94383b --- /dev/null +++ b/doc/source/release/1.14.4-notes.rst @@ -0,0 +1,60 @@ +========================== +NumPy 1.14.4 Release Notes +========================== + +This is a bugfix release for bugs reported following the 1.14.3 release. The +most significant fixes are: + +* fixes for compiler instruction reordering that resulted in NaN's not being + properly propagated in `np.max` and `np.min`, + +* fixes for bus faults on SPARC and older ARM due to incorrect alignment + checks. + +There are also improvements to printing of long doubles on PPC platforms. All +is not yet perfect on that platform, the whitespace padding is still incorrect +and is to be fixed in numpy 1.15, consequently NumPy still fails some +printing-related (and other) unit tests on ppc systems. However, the printed +values are now correct. + +Note that NumPy will error on import if it detects incorrect float32 `dot` +results. This problem has been seen on the Mac when working in the Anaconda +environment and is due to a subtle interaction between MKL and PyQt5. It is not +strictly a NumPy problem, but it is best that users be aware of it. See the +gh-8577 NumPy issue for more information. + +The Python versions supported in this release are 2.7 and 3.4 - 3.6. The Python +3.6 wheels available from PIP are built with Python 3.6.2 and should be +compatible with all previous versions of Python 3.6. The source releases were +cythonized with Cython 0.28.2 and should work for the upcoming Python 3.7. + +Contributors +============ + +A total of 7 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Allan Haldane +* Charles Harris +* Marten van Kerkwijk +* Matti Picus +* Pauli Virtanen +* Ryan Soklaski + +* Sebastian Berg + +Pull requests merged +==================== + +A total of 11 pull requests were merged for this release. + +* `#11104 <https://github.com/numpy/numpy/pull/11104>`__: BUG: str of DOUBLE_DOUBLE format wrong on ppc64 +* `#11170 <https://github.com/numpy/numpy/pull/11170>`__: TST: linalg: add regression test for gh-8577 +* `#11174 <https://github.com/numpy/numpy/pull/11174>`__: MAINT: add sanity-checks to be run at import time +* `#11181 <https://github.com/numpy/numpy/pull/11181>`__: BUG: void dtype setup checked offset not actual pointer for alignment +* `#11194 <https://github.com/numpy/numpy/pull/11194>`__: BUG: Python2 doubles don't print correctly in interactive shell. +* `#11198 <https://github.com/numpy/numpy/pull/11198>`__: BUG: optimizing compilers can reorder call to npy_get_floatstatus +* `#11199 <https://github.com/numpy/numpy/pull/11199>`__: BUG: reduce using SSE only warns if inside SSE loop +* `#11203 <https://github.com/numpy/numpy/pull/11203>`__: BUG: Bytes delimiter/comments in genfromtxt should be decoded +* `#11211 <https://github.com/numpy/numpy/pull/11211>`__: BUG: Fix reference count/memory leak exposed by better testing +* `#11219 <https://github.com/numpy/numpy/pull/11219>`__: BUG: Fixes einsum broadcasting bug when optimize=True +* `#11251 <https://github.com/numpy/numpy/pull/11251>`__: DOC: Document 1.14.4 release. diff --git a/doc/source/release/1.14.5-notes.rst b/doc/source/release/1.14.5-notes.rst new file mode 100644 index 000000000..9a97cc033 --- /dev/null +++ b/doc/source/release/1.14.5-notes.rst @@ -0,0 +1,30 @@ +========================== +NumPy 1.14.5 Release Notes +========================== + +This is a bugfix release for bugs reported following the 1.14.4 release. The +most significant fixes are: + +* fixes for compilation errors on alpine and NetBSD + +The Python versions supported in this release are 2.7 and 3.4 - 3.6. The Python +3.6 wheels available from PIP are built with Python 3.6.2 and should be +compatible with all previous versions of Python 3.6. The source releases were +cythonized with Cython 0.28.2 and should work for the upcoming Python 3.7. + +Contributors +============ + +A total of 1 person contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris + +Pull requests merged +==================== + +A total of 2 pull requests were merged for this release. + +* `#11274 <https://github.com/numpy/numpy/pull/11274>`__: BUG: Correct use of NPY_UNUSED. +* `#11294 <https://github.com/numpy/numpy/pull/11294>`__: BUG: Remove extra trailing parentheses. + diff --git a/doc/source/release/1.14.6-notes.rst b/doc/source/release/1.14.6-notes.rst new file mode 100644 index 000000000..ac6a78272 --- /dev/null +++ b/doc/source/release/1.14.6-notes.rst @@ -0,0 +1,33 @@ +========================== +NumPy 1.14.6 Release Notes +========================== + +This is a bugfix release for bugs reported following the 1.14.5 release. The +most significant fixes are: + +* Fix for behavior change in ``ma.masked_values(shrink=True)`` +* Fix the new cached allocations machinery to be thread safe. + +The Python versions supported in this release are 2.7 and 3.4 - 3.7. The Python +3.6 wheels on PyPI should be compatible with all Python 3.6 versions. + +Contributors +============ + +A total of 4 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Eric Wieser +* Julian Taylor +* Matti Picus + +Pull requests merged +==================== + +A total of 4 pull requests were merged for this release. + +* `#11985 <https://github.com/numpy/numpy/pull/11985>`__: BUG: fix cached allocations without the GIL +* `#11986 <https://github.com/numpy/numpy/pull/11986>`__: BUG: Undo behavior change in ma.masked_values(shrink=True) +* `#11987 <https://github.com/numpy/numpy/pull/11987>`__: BUG: fix refcount leak in PyArray_AdaptFlexibleDType +* `#11995 <https://github.com/numpy/numpy/pull/11995>`__: TST: Add Python 3.7 testing to NumPy 1.14. diff --git a/doc/source/release/1.15.0-notes.rst b/doc/source/release/1.15.0-notes.rst new file mode 100644 index 000000000..7235ca915 --- /dev/null +++ b/doc/source/release/1.15.0-notes.rst @@ -0,0 +1,502 @@ +========================== +NumPy 1.15.0 Release Notes +========================== + +NumPy 1.15.0 is a release with an unusual number of cleanups, many deprecations +of old functions, and improvements to many existing functions. Please read the +detailed descriptions below to see if you are affected. + +For testing, we have switched to pytest as a replacement for the no longer +maintained nose framework. The old nose based interface remains for downstream +projects who may still be using it. + +The Python versions supported by this release are 2.7, 3.4-3.7. The wheels are +linked with OpenBLAS v0.3.0, which should fix some of the linalg problems +reported for NumPy 1.14. + + +Highlights +========== + +* NumPy has switched to pytest for testing. +* A new `numpy.printoptions` context manager. +* Many improvements to the histogram functions. +* Support for unicode field names in python 2.7. +* Improved support for PyPy. +* Fixes and improvements to `numpy.einsum`. + + +New functions +============= + +* `numpy.gcd` and `numpy.lcm`, to compute the greatest common divisor and least + common multiple. + +* `numpy.ma.stack`, the `numpy.stack` array-joining function generalized to + masked arrays. + +* `numpy.quantile` function, an interface to ``percentile`` without factors of + 100 + +* `numpy.nanquantile` function, an interface to ``nanpercentile`` without + factors of 100 + +* `numpy.printoptions`, a context manager that sets print options temporarily + for the scope of the ``with`` block:: + + >>> with np.printoptions(precision=2): + ... print(np.array([2.0]) / 3) + [0.67] + +* `numpy.histogram_bin_edges`, a function to get the edges of the bins used by a + histogram without needing to calculate the histogram. + +* C functions `npy_get_floatstatus_barrier` and `npy_clear_floatstatus_barrier` + have been added to deal with compiler optimization changing the order of + operations. See below for details. + + +Deprecations +============ + +* Aliases of builtin `pickle` functions are deprecated, in favor of their + unaliased ``pickle.<func>`` names: + + * `numpy.loads` + * `numpy.core.numeric.load` + * `numpy.core.numeric.loads` + * `numpy.ma.loads`, `numpy.ma.dumps` + * `numpy.ma.load`, `numpy.ma.dump` - these functions already failed on + python 3 when called with a string. + +* Multidimensional indexing with anything but a tuple is deprecated. This means + that the index list in ``ind = [slice(None), 0]; arr[ind]`` should be changed + to a tuple, e.g., ``ind = [slice(None), 0]; arr[tuple(ind)]`` or + ``arr[(slice(None), 0)]``. That change is necessary to avoid ambiguity in + expressions such as ``arr[[[0, 1], [0, 1]]]``, currently interpreted as + ``arr[array([0, 1]), array([0, 1])]``, that will be interpreted + as ``arr[array([[0, 1], [0, 1]])]`` in the future. + +* Imports from the following sub-modules are deprecated, they will be removed + at some future date. + + * `numpy.testing.utils` + * `numpy.testing.decorators` + * `numpy.testing.nosetester` + * `numpy.testing.noseclasses` + * `numpy.core.umath_tests` + +* Giving a generator to `numpy.sum` is now deprecated. This was undocumented + behavior, but worked. Previously, it would calculate the sum of the generator + expression. In the future, it might return a different result. Use + ``np.sum(np.from_iter(generator))`` or the built-in Python ``sum`` instead. + +* Users of the C-API should call ``PyArrayResolveWriteBackIfCopy`` or + ``PyArray_DiscardWritbackIfCopy`` on any array with the ``WRITEBACKIFCOPY`` + flag set, before deallocating the array. A deprecation warning will be + emitted if those calls are not used when needed. + +* Users of ``nditer`` should use the nditer object as a context manager + anytime one of the iterator operands is writeable, so that numpy can + manage writeback semantics, or should call ``it.close()``. A + `RuntimeWarning` may be emitted otherwise in these cases. + +* The ``normed`` argument of ``np.histogram``, deprecated long ago in 1.6.0, + now emits a ``DeprecationWarning``. + + +Future Changes +============== + +* NumPy 1.16 will drop support for Python 3.4. +* NumPy 1.17 will drop support for Python 2.7. + + +Compatibility notes +=================== + +Compiled testing modules renamed and made private +------------------------------------------------- +The following compiled modules have been renamed and made private: + +* ``umath_tests`` -> ``_umath_tests`` +* ``test_rational`` -> ``_rational_tests`` +* ``multiarray_tests`` -> ``_multiarray_tests`` +* ``struct_ufunc_test`` -> ``_struct_ufunc_tests`` +* ``operand_flag_tests`` -> ``_operand_flag_tests`` + +The ``umath_tests`` module is still available for backwards compatibility, but +will be removed in the future. + +The ``NpzFile`` returned by ``np.savez`` is now a ``collections.abc.Mapping`` +----------------------------------------------------------------------------- +This means it behaves like a readonly dictionary, and has a new ``.values()`` +method and ``len()`` implementation. + +For python 3, this means that ``.iteritems()``, ``.iterkeys()`` have been +deprecated, and ``.keys()`` and ``.items()`` now return views and not lists. +This is consistent with how the builtin ``dict`` type changed between python 2 +and python 3. + +Under certain conditions, ``nditer`` must be used in a context manager +---------------------------------------------------------------------- +When using an `numpy.nditer` with the ``"writeonly"`` or ``"readwrite"`` flags, there +are some circumstances where nditer doesn't actually give you a view of the +writable array. Instead, it gives you a copy, and if you make changes to the +copy, nditer later writes those changes back into your actual array. Currently, +this writeback occurs when the array objects are garbage collected, which makes +this API error-prone on CPython and entirely broken on PyPy. Therefore, +``nditer`` should now be used as a context manager whenever it is used +with writeable arrays, e.g., ``with np.nditer(...) as it: ...``. You may also +explicitly call ``it.close()`` for cases where a context manager is unusable, +for instance in generator expressions. + +Numpy has switched to using pytest instead of nose for testing +-------------------------------------------------------------- +The last nose release was 1.3.7 in June, 2015, and development of that tool has +ended, consequently NumPy has now switched to using pytest. The old decorators +and nose tools that were previously used by some downstream projects remain +available, but will not be maintained. The standard testing utilities, +``assert_almost_equal`` and such, are not be affected by this change except for +the nose specific functions ``import_nose`` and ``raises``. Those functions are +not used in numpy, but are kept for downstream compatibility. + +Numpy no longer monkey-patches ``ctypes`` with ``__array_interface__`` +---------------------------------------------------------------------- +Previously numpy added ``__array_interface__`` attributes to all the integer +types from ``ctypes``. + +``np.ma.notmasked_contiguous`` and ``np.ma.flatnotmasked_contiguous`` always return lists +----------------------------------------------------------------------------------------- +This is the documented behavior, but previously the result could be any of +slice, None, or list. + +All downstream users seem to check for the ``None`` result from +``flatnotmasked_contiguous`` and replace it with ``[]``. Those callers will +continue to work as before. + +``np.squeeze`` restores old behavior of objects that cannot handle an ``axis`` argument +--------------------------------------------------------------------------------------- +Prior to version ``1.7.0``, `numpy.squeeze` did not have an ``axis`` argument and +all empty axes were removed by default. The incorporation of an ``axis`` +argument made it possible to selectively squeeze single or multiple empty axes, +but the old API expectation was not respected because axes could still be +selectively removed (silent success) from an object expecting all empty axes to +be removed. That silent, selective removal of empty axes for objects expecting +the old behavior has been fixed and the old behavior restored. + +unstructured void array's ``.item`` method now returns a bytes object +--------------------------------------------------------------------- +``.item`` now returns a ``bytes`` object instead of a buffer or byte array. +This may affect code which assumed the return value was mutable, which is no +longer the case. + +``copy.copy`` and ``copy.deepcopy`` no longer turn ``masked`` into an array +--------------------------------------------------------------------------- +Since ``np.ma.masked`` is a readonly scalar, copying should be a no-op. These +functions now behave consistently with ``np.copy()``. + +Multifield Indexing of Structured Arrays will still return a copy +----------------------------------------------------------------- +The change that multi-field indexing of structured arrays returns a view +instead of a copy is pushed back to 1.16. A new method +``numpy.lib.recfunctions.repack_fields`` has been introduced to help mitigate +the effects of this change, which can be used to write code compatible with +both numpy 1.15 and 1.16. For more information on how to update code to account +for this future change see the "accessing multiple fields" section of the +`user guide <https://docs.scipy.org/doc/numpy/user/basics.rec.html>`__. + + +C API changes +============= + +New functions ``npy_get_floatstatus_barrier`` and ``npy_clear_floatstatus_barrier`` +----------------------------------------------------------------------------------- +Functions ``npy_get_floatstatus_barrier`` and ``npy_clear_floatstatus_barrier`` +have been added and should be used in place of the ``npy_get_floatstatus``and +``npy_clear_status`` functions. Optimizing compilers like GCC 8.1 and Clang +were rearranging the order of operations when the previous functions were used +in the ufunc SIMD functions, resulting in the floatstatus flags being checked +before the operation whose status we wanted to check was run. See `#10339 +<https://github.com/numpy/numpy/issues/10370>`__. + +Changes to ``PyArray_GetDTypeTransferFunction`` +----------------------------------------------- +``PyArray_GetDTypeTransferFunction`` now defaults to using user-defined +``copyswapn`` / ``copyswap`` for user-defined dtypes. If this causes a +significant performance hit, consider implementing ``copyswapn`` to reflect the +implementation of ``PyArray_GetStridedCopyFn``. See `#10898 +<https://github.com/numpy/numpy/pull/10898>`__. + + +New Features +============ + +``np.gcd`` and ``np.lcm`` ufuncs added for integer and objects types +-------------------------------------------------------------------- +These compute the greatest common divisor, and lowest common multiple, +respectively. These work on all the numpy integer types, as well as the +builtin arbitrary-precision ``Decimal`` and ``long`` types. + +Support for cross-platform builds for iOS +----------------------------------------- +The build system has been modified to add support for the +``_PYTHON_HOST_PLATFORM`` environment variable, used by ``distutils`` when +compiling on one platform for another platform. This makes it possible to +compile NumPy for iOS targets. + +This only enables you to compile NumPy for one specific platform at a time. +Creating a full iOS-compatible NumPy package requires building for the 5 +architectures supported by iOS (i386, x86_64, armv7, armv7s and arm64), and +combining these 5 compiled builds products into a single "fat" binary. + +``return_indices`` keyword added for ``np.intersect1d`` +------------------------------------------------------- +New keyword ``return_indices`` returns the indices of the two input arrays +that correspond to the common elements. + +``np.quantile`` and ``np.nanquantile`` +-------------------------------------- +Like ``np.percentile`` and ``np.nanpercentile``, but takes quantiles in [0, 1] +rather than percentiles in [0, 100]. ``np.percentile`` is now a thin wrapper +around ``np.quantile`` with the extra step of dividing by 100. + + +Build system +------------ +Added experimental support for the 64-bit RISC-V architecture. + + +Improvements +============ + +``np.einsum`` updates +--------------------- +Syncs einsum path optimization tech between `numpy` and `opt_einsum`. In +particular, the `greedy` path has received many enhancements by @jcmgray. A +full list of issues fixed are: + +* Arbitrary memory can be passed into the `greedy` path. Fixes gh-11210. +* The greedy path has been updated to contain more dynamic programming ideas + preventing a large number of duplicate (and expensive) calls that figure out + the actual pair contraction that takes place. Now takes a few seconds on + several hundred input tensors. Useful for matrix product state theories. +* Reworks the broadcasting dot error catching found in gh-11218 gh-10352 to be + a bit earlier in the process. +* Enhances the `can_dot` functionality that previous missed an edge case (part + of gh-11308). + +``np.ufunc.reduce`` and related functions now accept an initial value +--------------------------------------------------------------------- +``np.ufunc.reduce``, ``np.sum``, ``np.prod``, ``np.min`` and ``np.max`` all +now accept an ``initial`` keyword argument that specifies the value to start +the reduction with. + +``np.flip`` can operate over multiple axes +------------------------------------------ +``np.flip`` now accepts None, or tuples of int, in its ``axis`` argument. If +axis is None, it will flip over all the axes. + +``histogram`` and ``histogramdd`` functions have moved to ``np.lib.histograms`` +------------------------------------------------------------------------------- +These were originally found in ``np.lib.function_base``. They are still +available under their un-scoped ``np.histogram(dd)`` names, and +to maintain compatibility, aliased at ``np.lib.function_base.histogram(dd)``. + +Code that does ``from np.lib.function_base import *`` will need to be updated +with the new location, and should consider not using ``import *`` in future. + +``histogram`` will accept NaN values when explicit bins are given +----------------------------------------------------------------- +Previously it would fail when trying to compute a finite range for the data. +Since the range is ignored anyway when the bins are given explicitly, this error +was needless. + +Note that calling ``histogram`` on NaN values continues to raise the +``RuntimeWarning`` s typical of working with nan values, which can be silenced +as usual with ``errstate``. + +``histogram`` works on datetime types, when explicit bin edges are given +------------------------------------------------------------------------ +Dates, times, and timedeltas can now be histogrammed. The bin edges must be +passed explicitly, and are not yet computed automatically. + +``histogram`` "auto" estimator handles limited variance better +-------------------------------------------------------------- +No longer does an IQR of 0 result in ``n_bins=1``, rather the number of bins +chosen is related to the data size in this situation. + +The edges retuned by `histogram`` and ``histogramdd`` now match the data float type +----------------------------------------------------------------------------------- +When passed ``np.float16``, ``np.float32``, or ``np.longdouble`` data, the +returned edges are now of the same dtype. Previously, ``histogram`` would only +return the same type if explicit bins were given, and ``histogram`` would +produce ``float64`` bins no matter what the inputs. + +``histogramdd`` allows explicit ranges to be given in a subset of axes +---------------------------------------------------------------------- +The ``range`` argument of `numpy.histogramdd` can now contain ``None`` values to +indicate that the range for the corresponding axis should be computed from the +data. Previously, this could not be specified on a per-axis basis. + +The normed arguments of ``histogramdd`` and ``histogram2d`` have been renamed +----------------------------------------------------------------------------- +These arguments are now called ``density``, which is consistent with +``histogram``. The old argument continues to work, but the new name should be +preferred. + +``np.r_`` works with 0d arrays, and ``np.ma.mr_`` works with ``np.ma.masked`` +----------------------------------------------------------------------------- +0d arrays passed to the `r_` and `mr_` concatenation helpers are now treated as +though they are arrays of length 1. Previously, passing these was an error. +As a result, `numpy.ma.mr_` now works correctly on the ``masked`` constant. + +``np.ptp`` accepts a ``keepdims`` argument, and extended axis tuples +-------------------------------------------------------------------- +``np.ptp`` (peak-to-peak) can now work over multiple axes, just like ``np.max`` +and ``np.min``. + +``MaskedArray.astype`` now is identical to ``ndarray.astype`` +------------------------------------------------------------- +This means it takes all the same arguments, making more code written for +ndarray work for masked array too. + +Enable AVX2/AVX512 at compile time +---------------------------------- +Change to simd.inc.src to allow use of AVX2 or AVX512 at compile time. Previously +compilation for avx2 (or 512) with -march=native would still use the SSE +code for the simd functions even when the rest of the code got AVX2. + +``nan_to_num`` always returns scalars when receiving scalar or 0d inputs +------------------------------------------------------------------------ +Previously an array was returned for integer scalar inputs, which is +inconsistent with the behavior for float inputs, and that of ufuncs in general. +For all types of scalar or 0d input, the result is now a scalar. + +``np.flatnonzero`` works on numpy-convertible types +--------------------------------------------------- +``np.flatnonzero`` now uses ``np.ravel(a)`` instead of ``a.ravel()``, so it +works for lists, tuples, etc. + +``np.interp`` returns numpy scalars rather than builtin scalars +--------------------------------------------------------------- +Previously ``np.interp(0.5, [0, 1], [10, 20])`` would return a ``float``, but +now it returns a ``np.float64`` object, which more closely matches the behavior +of other functions. + +Additionally, the special case of ``np.interp(object_array_0d, ...)`` is no +longer supported, as ``np.interp(object_array_nd)`` was never supported anyway. + +As a result of this change, the ``period`` argument can now be used on 0d +arrays. + +Allow dtype field names to be unicode in Python 2 +------------------------------------------------- +Previously ``np.dtype([(u'name', float)])`` would raise a ``TypeError`` in +Python 2, as only bytestrings were allowed in field names. Now any unicode +string field names will be encoded with the ``ascii`` codec, raising a +``UnicodeEncodeError`` upon failure. + +This change makes it easier to write Python 2/3 compatible code using +``from __future__ import unicode_literals``, which previously would cause +string literal field names to raise a TypeError in Python 2. + +Comparison ufuncs accept ``dtype=object``, overriding the default ``bool`` +-------------------------------------------------------------------------- +This allows object arrays of symbolic types, which override ``==`` and other +operators to return expressions, to be compared elementwise with +``np.equal(a, b, dtype=object)``. + +``sort`` functions accept ``kind='stable'`` +------------------------------------------- +Up until now, to perform a stable sort on the data, the user must do: + + >>> np.sort([5, 2, 6, 2, 1], kind='mergesort') + [1, 2, 2, 5, 6] + +because merge sort is the only stable sorting algorithm available in +NumPy. However, having kind='mergesort' does not make it explicit that +the user wants to perform a stable sort thus harming the readability. + +This change allows the user to specify kind='stable' thus clarifying +the intent. + +Do not make temporary copies for in-place accumulation +------------------------------------------------------ +When ufuncs perform accumulation they no longer make temporary copies because +of the overlap between input an output, that is, the next element accumulated +is added before the accumulated result is stored in its place, hence the +overlap is safe. Avoiding the copy results in faster execution. + +``linalg.matrix_power`` can now handle stacks of matrices +--------------------------------------------------------- +Like other functions in ``linalg``, ``matrix_power`` can now deal with arrays +of dimension larger than 2, which are treated as stacks of matrices. As part +of the change, to further improve consistency, the name of the first argument +has been changed to ``a`` (from ``M``), and the exceptions for non-square +matrices have been changed to ``LinAlgError`` (from ``ValueError``). + +Increased performance in ``random.permutation`` for multidimensional arrays +--------------------------------------------------------------------------- +``permutation`` uses the fast path in ``random.shuffle`` for all input +array dimensions. Previously the fast path was only used for 1-d arrays. + +Generalized ufuncs now accept ``axes``, ``axis`` and ``keepdims`` arguments +--------------------------------------------------------------------------- +One can control over which axes a generalized ufunc operates by passing in an +``axes`` argument, a list of tuples with indices of particular axes. For +instance, for a signature of ``(i,j),(j,k)->(i,k)`` appropriate for matrix +multiplication, the base elements are two-dimensional matrices and these are +taken to be stored in the two last axes of each argument. The corresponding +axes keyword would be ``[(-2, -1), (-2, -1), (-2, -1)]``. If one wanted to +use leading dimensions instead, one would pass in ``[(0, 1), (0, 1), (0, 1)]``. + +For simplicity, for generalized ufuncs that operate on 1-dimensional arrays +(vectors), a single integer is accepted instead of a single-element tuple, and +for generalized ufuncs for which all outputs are scalars, the (empty) output +tuples can be omitted. Hence, for a signature of ``(i),(i)->()`` appropriate +for an inner product, one could pass in ``axes=[0, 0]`` to indicate that the +vectors are stored in the first dimensions of the two inputs arguments. + +As a short-cut for generalized ufuncs that are similar to reductions, i.e., +that act on a single, shared core dimension such as the inner product example +above, one can pass an ``axis`` argument. This is equivalent to passing in +``axes`` with identical entries for all arguments with that core dimension +(e.g., for the example above, ``axes=[(axis,), (axis,)]``). + +Furthermore, like for reductions, for generalized ufuncs that have inputs that +all have the same number of core dimensions and outputs with no core dimension, +one can pass in ``keepdims`` to leave a dimension with size 1 in the outputs, +thus allowing proper broadcasting against the original inputs. The location of +the extra dimension can be controlled with ``axes``. For instance, for the +inner-product example, ``keepdims=True, axes=[-2, -2, -2]`` would act on the +inner-product example, ``keepdims=True, axis=-2`` would act on the +one-but-last dimension of the input arguments, and leave a size 1 dimension in +that place in the output. + +float128 values now print correctly on ppc systems +-------------------------------------------------- +Previously printing float128 values was buggy on ppc, since the special +double-double floating-point-format on these systems was not accounted for. +float128s now print with correct rounding and uniqueness. + +Warning to ppc users: You should upgrade glibc if it is version <=2.23, +especially if using float128. On ppc, glibc's malloc in these version often +misaligns allocated memory which can crash numpy when using float128 values. + +New ``np.take_along_axis`` and ``np.put_along_axis`` functions +-------------------------------------------------------------- +When used on multidimensional arrays, ``argsort``, ``argmin``, ``argmax``, and +``argpartition`` return arrays that are difficult to use as indices. +``take_along_axis`` provides an easy way to use these indices to lookup values +within an array, so that:: + + np.take_along_axis(a, np.argsort(a, axis=axis), axis=axis) + +is the same as:: + + np.sort(a, axis=axis) + +``np.put_along_axis`` acts as the dual operation for writing to these indices +within an array. + diff --git a/doc/source/release/1.15.1-notes.rst b/doc/source/release/1.15.1-notes.rst new file mode 100644 index 000000000..ddb83303c --- /dev/null +++ b/doc/source/release/1.15.1-notes.rst @@ -0,0 +1,74 @@ +========================== +NumPy 1.15.1 Release Notes +========================== + +This is a bugfix release for bugs and regressions reported following the 1.15.0 +release. + +* The annoying but harmless RuntimeWarning that "numpy.dtype size changed" has + been suppressed. The long standing suppression was lost in the transition to + pytest. +* The update to Cython 0.28.3 exposed a problematic use of a gcc attribute used + to prefer code size over speed in module initialization, possibly resulting in + incorrect compiled code. This has been fixed in latest Cython but has been + disabled here for safety. +* Support for big-endian and ARMv8 architectures has been improved. + +The Python versions supported by this release are 2.7, 3.4-3.7. The wheels are +linked with OpenBLAS v0.3.0, which should fix some of the linalg problems +reported for NumPy 1.14. + + +Compatibility Note +================== + +The NumPy 1.15.x OS X wheels released on PyPI no longer contain 32-bit +binaries. That will also be the case in future releases. See +`#11625 <https://github.com/numpy/numpy/issues/11625>`__ for the related +discussion. Those needing 32-bit support should look elsewhere or build +from source. + + +Contributors +============ + +A total of 7 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Chris Billington +* Elliott Sales de Andrade + +* Eric Wieser +* Jeremy Manning + +* Matti Picus +* Ralf Gommers + +Pull requests merged +==================== + +A total of 24 pull requests were merged for this release. + +* `#11647 <https://github.com/numpy/numpy/pull/11647>`__: MAINT: Filter Cython warnings in ``__init__.py`` +* `#11648 <https://github.com/numpy/numpy/pull/11648>`__: BUG: Fix doc source links to unwrap decorators +* `#11657 <https://github.com/numpy/numpy/pull/11657>`__: BUG: Ensure singleton dimensions are not dropped when converting... +* `#11661 <https://github.com/numpy/numpy/pull/11661>`__: BUG: Warn on Nan in minimum,maximum for scalars +* `#11665 <https://github.com/numpy/numpy/pull/11665>`__: BUG: cython sometimes emits invalid gcc attribute +* `#11682 <https://github.com/numpy/numpy/pull/11682>`__: BUG: Fix regression in void_getitem +* `#11698 <https://github.com/numpy/numpy/pull/11698>`__: BUG: Make matrix_power again work for object arrays. +* `#11700 <https://github.com/numpy/numpy/pull/11700>`__: BUG: Add missing PyErr_NoMemory after failing malloc +* `#11719 <https://github.com/numpy/numpy/pull/11719>`__: BUG: Fix undefined functions on big-endian systems. +* `#11720 <https://github.com/numpy/numpy/pull/11720>`__: MAINT: Make einsum optimize default to False. +* `#11746 <https://github.com/numpy/numpy/pull/11746>`__: BUG: Fix regression in loadtxt for bz2 text files in Python 2. +* `#11757 <https://github.com/numpy/numpy/pull/11757>`__: BUG: Revert use of `console_scripts`. +* `#11758 <https://github.com/numpy/numpy/pull/11758>`__: BUG: Fix Fortran kind detection for aarch64 & s390x. +* `#11759 <https://github.com/numpy/numpy/pull/11759>`__: BUG: Fix printing of longdouble on ppc64le. +* `#11760 <https://github.com/numpy/numpy/pull/11760>`__: BUG: Fixes for unicode field names in Python 2 +* `#11761 <https://github.com/numpy/numpy/pull/11761>`__: BUG: Increase required cython version on python 3.7 +* `#11763 <https://github.com/numpy/numpy/pull/11763>`__: BUG: check return value of _buffer_format_string +* `#11775 <https://github.com/numpy/numpy/pull/11775>`__: MAINT: Make assert_array_compare more generic. +* `#11776 <https://github.com/numpy/numpy/pull/11776>`__: TST: Fix urlopen stubbing. +* `#11777 <https://github.com/numpy/numpy/pull/11777>`__: BUG: Fix regression in intersect1d. +* `#11779 <https://github.com/numpy/numpy/pull/11779>`__: BUG: Fix test sensitive to platform byte order. +* `#11781 <https://github.com/numpy/numpy/pull/11781>`__: BUG: Avoid signed overflow in histogram +* `#11785 <https://github.com/numpy/numpy/pull/11785>`__: BUG: Fix pickle and memoryview for datetime64, timedelta64 scalars +* `#11786 <https://github.com/numpy/numpy/pull/11786>`__: BUG: Deprecation triggers segfault diff --git a/doc/source/release/1.15.2-notes.rst b/doc/source/release/1.15.2-notes.rst new file mode 100644 index 000000000..a3e61fccd --- /dev/null +++ b/doc/source/release/1.15.2-notes.rst @@ -0,0 +1,45 @@ +========================== +NumPy 1.15.2 Release Notes +========================== + +This is a bugfix release for bugs and regressions reported following the 1.15.1 +release. + +* The matrix PendingDeprecationWarning is now suppressed in pytest 3.8. +* The new cached allocations machinery has been fixed to be thread safe. +* The boolean indexing of subclasses now works correctly. +* A small memory leak in PyArray_AdaptFlexibleDType has been fixed. + +The Python versions supported by this release are 2.7, 3.4-3.7. The wheels are +linked with OpenBLAS v0.3.0, which should fix some of the linalg problems +reported for NumPy 1.14. + +Compatibility Note +================== + +The NumPy 1.15.x OS X wheels released on PyPI no longer contain 32-bit +binaries. That will also be the case in future releases. See +`#11625 <https://github.com/numpy/numpy/issues/11625>`__ for the related +discussion. Those needing 32-bit support should look elsewhere or build +from source. + +Contributors +============ + +A total of 4 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Julian Taylor +* Marten van Kerkwijk +* Matti Picus + +Pull requests merged +==================== + +A total of 4 pull requests were merged for this release. + +* `#11902 <https://github.com/numpy/numpy/pull/11902>`__: BUG: Fix matrix PendingDeprecationWarning suppression for pytest... +* `#11981 <https://github.com/numpy/numpy/pull/11981>`__: BUG: fix cached allocations without the GIL for 1.15.x +* `#11982 <https://github.com/numpy/numpy/pull/11982>`__: BUG: fix refcount leak in PyArray_AdaptFlexibleDType +* `#11992 <https://github.com/numpy/numpy/pull/11992>`__: BUG: Ensure boolean indexing of subclasses sets base correctly. diff --git a/doc/source/release/1.15.3-notes.rst b/doc/source/release/1.15.3-notes.rst new file mode 100644 index 000000000..753eecec9 --- /dev/null +++ b/doc/source/release/1.15.3-notes.rst @@ -0,0 +1,49 @@ +========================== +NumPy 1.15.3 Release Notes +========================== + +This is a bugfix release for bugs and regressions reported following the 1.15.2 +release. The Python versions supported by this release are 2.7, 3.4-3.7. The +wheels are linked with OpenBLAS v0.3.0, which should fix some of the linalg +problems reported for NumPy 1.14. + +Compatibility Note +================== + +The NumPy 1.15.x OS X wheels released on PyPI no longer contain 32-bit +binaries. That will also be the case in future releases. See +`#11625 <https://github.com/numpy/numpy/issues/11625>`__ for the related +discussion. Those needing 32-bit support should look elsewhere or build +from source. + +Contributors +============ + +A total of 7 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Allan Haldane +* Charles Harris +* Jeroen Demeyer +* Kevin Sheppard +* Matthew Bowden + +* Matti Picus +* Tyler Reddy + +Pull requests merged +==================== + +A total of 12 pull requests were merged for this release. + +* `#12080 <https://github.com/numpy/numpy/pull/12080>`__: MAINT: Blacklist some MSVC complex functions. +* `#12083 <https://github.com/numpy/numpy/pull/12083>`__: TST: Add azure CI testing to 1.15.x branch. +* `#12084 <https://github.com/numpy/numpy/pull/12084>`__: BUG: test_path() now uses Path.resolve() +* `#12085 <https://github.com/numpy/numpy/pull/12085>`__: TST, MAINT: Fix some failing tests on azure-pipelines mac and... +* `#12187 <https://github.com/numpy/numpy/pull/12187>`__: BUG: Fix memory leak in mapping.c +* `#12188 <https://github.com/numpy/numpy/pull/12188>`__: BUG: Allow boolean subtract in histogram +* `#12189 <https://github.com/numpy/numpy/pull/12189>`__: BUG: Fix in-place permutation +* `#12190 <https://github.com/numpy/numpy/pull/12190>`__: BUG: limit default for get_num_build_jobs() to 8 +* `#12191 <https://github.com/numpy/numpy/pull/12191>`__: BUG: OBJECT_to_* should check for errors +* `#12192 <https://github.com/numpy/numpy/pull/12192>`__: DOC: Prepare for NumPy 1.15.3 release. +* `#12237 <https://github.com/numpy/numpy/pull/12237>`__: BUG: Fix MaskedArray fill_value type conversion. +* `#12238 <https://github.com/numpy/numpy/pull/12238>`__: TST: Backport azure-pipeline testing fixes for Mac diff --git a/doc/source/release/1.15.4-notes.rst b/doc/source/release/1.15.4-notes.rst new file mode 100644 index 000000000..033bd5828 --- /dev/null +++ b/doc/source/release/1.15.4-notes.rst @@ -0,0 +1,38 @@ +========================== +NumPy 1.15.4 Release Notes +========================== + +This is a bugfix release for bugs and regressions reported following the 1.15.3 +release. The Python versions supported by this release are 2.7, 3.4-3.7. The +wheels are linked with OpenBLAS v0.3.0, which should fix some of the linalg +problems reported for NumPy 1.14. + +Compatibility Note +================== + +The NumPy 1.15.x OS X wheels released on PyPI no longer contain 32-bit +binaries. That will also be the case in future releases. See +`#11625 <https://github.com/numpy/numpy/issues/11625>`__ for the related +discussion. Those needing 32-bit support should look elsewhere or build +from source. + +Contributors +============ + +A total of 4 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Matti Picus +* Sebastian Berg +* bbbbbbbbba + + +Pull requests merged +==================== + +A total of 4 pull requests were merged for this release. + +* `#12296 <https://github.com/numpy/numpy/pull/12296>`__: BUG: Dealloc cached buffer info +* `#12297 <https://github.com/numpy/numpy/pull/12297>`__: BUG: Fix fill value in masked array '==' and '!=' ops. +* `#12307 <https://github.com/numpy/numpy/pull/12307>`__: DOC: Correct the default value of `optimize` in `numpy.einsum` +* `#12320 <https://github.com/numpy/numpy/pull/12320>`__: REL: Prepare for the NumPy 1.15.4 release diff --git a/doc/source/release/1.16.0-notes.rst b/doc/source/release/1.16.0-notes.rst new file mode 100644 index 000000000..1034d6e6c --- /dev/null +++ b/doc/source/release/1.16.0-notes.rst @@ -0,0 +1,536 @@ +========================== +NumPy 1.16.0 Release Notes +========================== + +This NumPy release is the last one to support Python 2.7 and will be maintained +as a long term release with bug fixes until 2020. Support for Python 3.4 been +dropped, the supported Python versions are 2.7 and 3.5-3.7. The wheels on PyPI +are linked with OpenBLAS v0.3.4+, which should fix the known threading issues +found in previous OpenBLAS versions. + +Downstream developers building this release should use Cython >= 0.29 and, if +using OpenBLAS, OpenBLAS > v0.3.4. + +This release has seen a lot of refactoring and features many bug fixes, improved +code organization, and better cross platform compatibility. Not all of these +improvements will be visible to users, but they should help make maintenance +easier going forward. + + +Highlights +========== + +* Experimental (opt-in only) support for overriding numpy functions, + see ``__array_function__`` below. + +* The ``matmul`` function is now a ufunc. This provides better + performance and allows overriding with ``__array_ufunc__``. + +* Improved support for the ARM and POWER architectures. + +* Improved support for AIX and PyPy. + +* Improved interop with ctypes. + +* Improved support for PEP 3118. + + + +New functions +============= + +* New functions added to the `numpy.lib.recfuntions` module to ease the + structured assignment changes: + + * ``assign_fields_by_name`` + * ``structured_to_unstructured`` + * ``unstructured_to_structured`` + * ``apply_along_fields`` + * ``require_fields`` + + See the user guide at <https://docs.scipy.org/doc/numpy/user/basics.rec.html> + for more info. + + +New deprecations +================ + +* The type dictionaries `numpy.core.typeNA` and `numpy.core.sctypeNA` are + deprecated. They were buggy and not documented and will be removed in the + 1.18 release. Use`numpy.sctypeDict` instead. + +* The `numpy.asscalar` function is deprecated. It is an alias to the more + powerful `numpy.ndarray.item`, not tested, and fails for scalars. + +* The `numpy.set_array_ops` and `numpy.get_array_ops` functions are deprecated. + As part of `NEP 15`, they have been deprecated along with the C-API functions + :c:func:`PyArray_SetNumericOps` and :c:func:`PyArray_GetNumericOps`. Users + who wish to override the inner loop functions in built-in ufuncs should use + :c:func:`PyUFunc_ReplaceLoopBySignature`. + +* The `numpy.unravel_index` keyword argument ``dims`` is deprecated, use + ``shape`` instead. + +* The `numpy.histogram` ``normed`` argument is deprecated. It was deprecated + previously, but no warning was issued. + +* The ``positive`` operator (``+``) applied to non-numerical arrays is + deprecated. See below for details. + +* Passing an iterator to the stack functions is deprecated + + +Expired deprecations +==================== + +* NaT comparisons now return ``False`` without a warning, finishing a + deprecation cycle begun in NumPy 1.11. + +* ``np.lib.function_base.unique`` was removed, finishing a deprecation cycle + begun in NumPy 1.4. Use `numpy.unique` instead. + +* multi-field indexing now returns views instead of copies, finishing a + deprecation cycle begun in NumPy 1.7. The change was previously attempted in + NumPy 1.14 but reverted until now. + +* ``np.PackageLoader`` and ``np.pkgload`` have been removed. These were + deprecated in 1.10, had no tests, and seem to no longer work in 1.15. + + +Future changes +============== + +* NumPy 1.17 will drop support for Python 2.7. + + +Compatibility notes +=================== + +f2py script on Windows +---------------------- +On Windows, the installed script for running f2py is now an ``.exe`` file +rather than a ``*.py`` file and should be run from the command line as ``f2py`` +whenever the ``Scripts`` directory is in the path. Running ``f2py`` as a module +``python -m numpy.f2py [...]`` will work without path modification in any +version of NumPy. + +NaT comparisons +--------------- +Consistent with the behavior of NaN, all comparisons other than inequality +checks with datetime64 or timedelta64 NaT ("not-a-time") values now always +return ``False``, and inequality checks with NaT now always return ``True``. +This includes comparisons beteween NaT values. For compatibility with the +old behavior, use ``np.isnat`` to explicitly check for NaT or convert +datetime64/timedelta64 arrays with ``.astype(np.int64)`` before making +comparisons. + +complex64/128 alignment has changed +----------------------------------- +The memory alignment of complex types is now the same as a C-struct composed of +two floating point values, while before it was equal to the size of the type. +For many users (for instance on x64/unix/gcc) this means that complex64 is now +4-byte aligned instead of 8-byte aligned. An important consequence is that +aligned structured dtypes may now have a different size. For instance, +``np.dtype('c8,u1', align=True)`` used to have an itemsize of 16 (on x64/gcc) +but now it is 12. + +More in detail, the complex64 type now has the same alignment as a C-struct +``struct {float r, i;}``, according to the compiler used to compile numpy, and +similarly for the complex128 and complex256 types. + +nd_grid __len__ removal +----------------------- +``len(np.mgrid)`` and ``len(np.ogrid)`` are now considered nonsensical +and raise a ``TypeError``. + +``np.unravel_index`` now accepts ``shape`` keyword argument +----------------------------------------------------------- +Previously, only the ``dims`` keyword argument was accepted +for specification of the shape of the array to be used +for unraveling. ``dims`` remains supported, but is now deprecated. + +multi-field views return a view instead of a copy +------------------------------------------------- +Indexing a structured array with multiple fields, e.g., ``arr[['f1', 'f3']]``, +returns a view into the original array instead of a copy. The returned view +will often have extra padding bytes corresponding to intervening fields in the +original array, unlike before, which will affect code such as +``arr[['f1', 'f3']].view('float64')``. This change has been planned since numpy +1.7. Operations hitting this path have emitted ``FutureWarnings`` since then. +Additional ``FutureWarnings`` about this change were added in 1.12. + +To help users update their code to account for these changes, a number of +functions have been added to the ``numpy.lib.recfunctions`` module which +safely allow such operations. For instance, the code above can be replaced +with ``structured_to_unstructured(arr[['f1', 'f3']], dtype='float64')``. +See the "accessing multiple fields" section of the +`user guide <https://docs.scipy.org/doc/numpy/user/basics.rec.html#accessing-multiple-fields>`__. + + +C API changes +============= + +The :c:data:`NPY_API_VERSION` was incremented to 0x0000D, due to the addition +of: + +* :c:member:`PyUFuncObject.core_dim_flags` +* :c:member:`PyUFuncObject.core_dim_sizes` +* :c:member:`PyUFuncObject.identity_value` +* :c:func:`PyUFunc_FromFuncAndDataAndSignatureAndIdentity` + + +New Features +============ + +Integrated squared error (ISE) estimator added to ``histogram`` +--------------------------------------------------------------- +This method (``bins='stone'``) for optimizing the bin number is a +generalization of the Scott's rule. The Scott's rule assumes the distribution +is approximately Normal, while the ISE_ is a non-parametric method based on +cross-validation. + +.. _ISE: https://en.wikipedia.org/wiki/Histogram#Minimizing_cross-validation_estimated_squared_error + +``max_rows`` keyword added for ``np.loadtxt`` +--------------------------------------------- +New keyword ``max_rows`` in `numpy.loadtxt` sets the maximum rows of the +content to be read after ``skiprows``, as in `numpy.genfromtxt`. + +modulus operator support added for ``np.timedelta64`` operands +-------------------------------------------------------------- +The modulus (remainder) operator is now supported for two operands +of type ``np.timedelta64``. The operands may have different units +and the return value will match the type of the operands. + + +Improvements +============ + +no-copy pickling of numpy arrays +-------------------------------- +Up to protocol 4, numpy array pickling created 2 spurious copies of the data +being serialized. With pickle protocol 5, and the ``PickleBuffer`` API, a +large variety of numpy arrays can now be serialized without any copy using +out-of-band buffers, and with one less copy using in-band buffers. This +results, for large arrays, in an up to 66% drop in peak memory usage. + +build shell independence +------------------------ +NumPy builds should no longer interact with the host machine +shell directly. ``exec_command`` has been replaced with +``subprocess.check_output`` where appropriate. + +`np.polynomial.Polynomial` classes render in LaTeX in Jupyter notebooks +----------------------------------------------------------------------- +When used in a front-end that supports it, `Polynomial` instances are now +rendered through LaTeX. The current format is experimental, and is subject to +change. + +``randint`` and ``choice`` now work on empty distributions +---------------------------------------------------------- +Even when no elements needed to be drawn, ``np.random.randint`` and +``np.random.choice`` raised an error when the arguments described an empty +distribution. This has been fixed so that e.g. +``np.random.choice([], 0) == np.array([], dtype=float64)``. + +``linalg.lstsq``, ``linalg.qr``, and ``linalg.svd`` now work with empty arrays +------------------------------------------------------------------------------ +Previously, a ``LinAlgError`` would be raised when an empty matrix/empty +matrices (with zero rows and/or columns) is/are passed in. Now outputs of +appropriate shapes are returned. + +Chain exceptions to give better error messages for invalid PEP3118 format strings +--------------------------------------------------------------------------------- +This should help track down problems. + +Einsum optimization path updates and efficiency improvements +------------------------------------------------------------ +Einsum was synchronized with the current upstream work. + +`numpy.angle` and `numpy.expand_dims` now work on ``ndarray`` subclasses +------------------------------------------------------------------------ +In particular, they now work for masked arrays. + +``NPY_NO_DEPRECATED_API`` compiler warning suppression +------------------------------------------------------ +Setting ``NPY_NO_DEPRECATED_API`` to a value of 0 will suppress the current compiler +warnings when the deprecated numpy API is used. + +``np.diff`` Added kwargs prepend and append +------------------------------------------- +New kwargs ``prepend`` and ``append``, allow for values to be inserted on +either end of the differences. Similar to options for `ediff1d`. Now the +inverse of `cumsum` can be obtained easily via ``prepend=0``. + +ARM support updated +------------------- +Support for ARM CPUs has been updated to accommodate 32 and 64 bit targets, +and also big and little endian byte ordering. AARCH32 memory alignment issues +have been addressed. CI testing has been expanded to include AARCH64 targets +via the services of shippable.com. + +Appending to build flags +------------------------ +`numpy.distutils` has always overridden rather than appended to `LDFLAGS` and +other similar such environment variables for compiling Fortran extensions. +Now, if the `NPY_DISTUTILS_APPEND_FLAGS` environment variable is set to 1, the +behavior will be appending. This applied to: `LDFLAGS`, `F77FLAGS`, +`F90FLAGS`, `FREEFLAGS`, `FOPT`, `FDEBUG`, and `FFLAGS`. See gh-11525 for more +details. + +Generalized ufunc signatures now allow fixed-size dimensions +------------------------------------------------------------ +By using a numerical value in the signature of a generalized ufunc, one can +indicate that the given function requires input or output to have dimensions +with the given size. E.g., the signature of a function that converts a polar +angle to a two-dimensional cartesian unit vector would be ``()->(2)``; that +for one that converts two spherical angles to a three-dimensional unit vector +would be ``(),()->(3)``; and that for the cross product of two +three-dimensional vectors would be ``(3),(3)->(3)``. + +Note that to the elementary function these dimensions are not treated any +differently from variable ones indicated with a name starting with a letter; +the loop still is passed the corresponding size, but it can now count on that +size being equal to the fixed one given in the signature. + +Generalized ufunc signatures now allow flexible dimensions +---------------------------------------------------------- +Some functions, in particular numpy's implementation of ``@`` as ``matmul``, +are very similar to generalized ufuncs in that they operate over core +dimensions, but one could not present them as such because they were able to +deal with inputs in which a dimension is missing. To support this, it is now +allowed to postfix a dimension name with a question mark to indicate that the +dimension does not necessarily have to be present. + +With this addition, the signature for ``matmul`` can be expressed as +``(m?,n),(n,p?)->(m?,p?)``. This indicates that if, e.g., the second operand +has only one dimension, for the purposes of the elementary function it will be +treated as if that input has core shape ``(n, 1)``, and the output has the +corresponding core shape of ``(m, 1)``. The actual output array, however, has +the flexible dimension removed, i.e., it will have shape ``(..., m)``. +Similarly, if both arguments have only a single dimension, the inputs will be +presented as having shapes ``(1, n)`` and ``(n, 1)`` to the elementary +function, and the output as ``(1, 1)``, while the actual output array returned +will have shape ``()``. In this way, the signature allows one to use a +single elementary function for four related but different signatures, +``(m,n),(n,p)->(m,p)``, ``(n),(n,p)->(p)``, ``(m,n),(n)->(m)`` and +``(n),(n)->()``. + +``np.clip`` and the ``clip`` method check for memory overlap +------------------------------------------------------------ +The ``out`` argument to these functions is now always tested for memory overlap +to avoid corrupted results when memory overlap occurs. + +New value ``unscaled`` for option ``cov`` in ``np.polyfit`` +----------------------------------------------------------- +A further possible value has been added to the ``cov`` parameter of the +``np.polyfit`` function. With ``cov='unscaled'`` the scaling of the covariance +matrix is disabled completely (similar to setting ``absolute_sigma=True`` in +``scipy.optimize.curve_fit``). This would be useful in occasions, where the +weights are given by 1/sigma with sigma being the (known) standard errors of +(Gaussian distributed) data points, in which case the unscaled matrix is +already a correct estimate for the covariance matrix. + +Detailed docstrings for scalar numeric types +-------------------------------------------- +The ``help`` function, when applied to numeric types such as `numpy.intc`, +`numpy.int_`, and `numpy.longlong`, now lists all of the aliased names for that +type, distinguishing between platform -dependent and -independent aliases. + +``__module__`` attribute now points to public modules +----------------------------------------------------- +The ``__module__`` attribute on most NumPy functions has been updated to refer +to the preferred public module from which to access a function, rather than +the module in which the function happens to be defined. This produces more +informative displays for functions in tools such as IPython, e.g., instead of +``<function 'numpy.core.fromnumeric.sum'>`` you now see +``<function 'numpy.sum'>``. + +Large allocations marked as suitable for transparent hugepages +-------------------------------------------------------------- +On systems that support transparent hugepages over the madvise system call +numpy now marks that large memory allocations can be backed by hugepages which +reduces page fault overhead and can in some fault heavy cases improve +performance significantly. On Linux the setting for huge pages to be used, +`/sys/kernel/mm/transparent_hugepage/enabled`, must be at least `madvise`. +Systems which already have it set to `always` will not see much difference as +the kernel will automatically use huge pages where appropriate. + +Users of very old Linux kernels (~3.x and older) should make sure that +`/sys/kernel/mm/transparent_hugepage/defrag` is not set to `always` to avoid +performance problems due concurrency issues in the memory defragmentation. + +Alpine Linux (and other musl c library distros) support +------------------------------------------------------- +We now default to use `fenv.h` for floating point status error reporting. +Previously we had a broken default that sometimes would not report underflow, +overflow, and invalid floating point operations. Now we can support non-glibc +distrubutions like Alpine Linux as long as they ship `fenv.h`. + +Speedup ``np.block`` for large arrays +------------------------------------- +Large arrays (greater than ``512 * 512``) now use a blocking algorithm based on +copying the data directly into the appropriate slice of the resulting array. +This results in significant speedups for these large arrays, particularly for +arrays being blocked along more than 2 dimensions. + +``arr.ctypes.data_as(...)`` holds a reference to arr +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Previously the caller was responsible for keeping the array alive for the +lifetime of the pointer. + +Speedup ``np.take`` for read-only arrays +---------------------------------------- +The implementation of ``np.take`` no longer makes an unnecessary copy of the +source array when its ``writeable`` flag is set to ``False``. + +Support path-like objects for more functions +-------------------------------------------- +The ``np.core.records.fromfile`` function now supports ``pathlib.Path`` +and other path-like objects in addition to a file object. Furthermore, the +``np.load`` function now also supports path-like objects when using memory +mapping (``mmap_mode`` keyword argument). + +Better behaviour of ufunc identities during reductions +------------------------------------------------------ +Universal functions have an ``.identity`` which is used when ``.reduce`` is +called on an empty axis. + +As of this release, the logical binary ufuncs, `logical_and`, `logical_or`, +and `logical_xor`, now have ``identity`` s of type `bool`, where previously they +were of type `int`. This restores the 1.14 behavior of getting ``bool`` s when +reducing empty object arrays with these ufuncs, while also keeping the 1.15 +behavior of getting ``int`` s when reducing empty object arrays with arithmetic +ufuncs like ``add`` and ``multiply``. + +Additionally, `logaddexp` now has an identity of ``-inf``, allowing it to be +called on empty sequences, where previously it could not be. + +This is possible thanks to the new +:c:func:`PyUFunc_FromFuncAndDataAndSignatureAndIdentity`, which allows +arbitrary values to be used as identities now. + +Improved conversion from ctypes objects +--------------------------------------- +Numpy has always supported taking a value or type from ``ctypes`` and +converting it into an array or dtype, but only behaved correctly for simpler +types. As of this release, this caveat is lifted - now: + +* The ``_pack_`` attribute of ``ctypes.Structure``, used to emulate C's + ``__attribute__((packed))``, is respected. +* Endianness of all ctypes objects is preserved +* ``ctypes.Union`` is supported +* Non-representable constructs raise exceptions, rather than producing + dangerously incorrect results: + + * Bitfields are no longer interpreted as sub-arrays + * Pointers are no longer replaced with the type that they point to + +A new ``ndpointer.contents`` member +----------------------------------- +This matches the ``.contents`` member of normal ctypes arrays, and can be used +to construct an ``np.array`` around the pointers contents. This replaces +``np.array(some_nd_pointer)``, which stopped working in 1.15. As a side effect +of this change, ``ndpointer`` now supports dtypes with overlapping fields and +padding. + +``matmul`` is now a ``ufunc`` +----------------------------- +`numpy.matmul` is now a ufunc which means that both the function and the +``__matmul__`` operator can now be overridden by ``__array_ufunc__``. Its +implementation has also changed. It uses the same BLAS routines as +`numpy.dot`, ensuring its performance is similar for large matrices. + +Start and stop arrays for ``linspace``, ``logspace`` and ``geomspace`` +---------------------------------------------------------------------- +These functions used to be limited to scalar stop and start values, but can +now take arrays, which will be properly broadcast and result in an output +which has one axis prepended. This can be used, e.g., to obtain linearly +interpolated points between sets of points. + +CI extended with additional services +------------------------------------ +We now use additional free CI services, thanks to the companies that provide: + +* Codecoverage testing via codecov.io +* Arm testing via shippable.com +* Additional test runs on azure pipelines + +These are in addition to our continued use of travis, appveyor (for wheels) and +LGTM + + +Changes +======= + +Comparison ufuncs will now error rather than return NotImplemented +------------------------------------------------------------------ +Previously, comparison ufuncs such as ``np.equal`` would return +`NotImplemented` if their arguments had structured dtypes, to help comparison +operators such as ``__eq__`` deal with those. This is no longer needed, as the +relevant logic has moved to the comparison operators proper (which thus do +continue to return `NotImplemented` as needed). Hence, like all other ufuncs, +the comparison ufuncs will now error on structured dtypes. + +Positive will now raise a deprecation warning for non-numerical arrays +---------------------------------------------------------------------- +Previously, ``+array`` unconditionally returned a copy. Now, it will +raise a ``DeprecationWarning`` if the array is not numerical (i.e., +if ``np.positive(array)`` raises a ``TypeError``. For ``ndarray`` +subclasses that override the default ``__array_ufunc__`` implementation, +the ``TypeError`` is passed on. + +``NDArrayOperatorsMixin`` now implements matrix multiplication +-------------------------------------------------------------- +Previously, ``np.lib.mixins.NDArrayOperatorsMixin`` did not implement the +special methods for Python's matrix multiplication operator (``@``). This has +changed now that ``matmul`` is a ufunc and can be overridden using +``__array_ufunc__``. + +The scaling of the covariance matrix in ``np.polyfit`` is different +------------------------------------------------------------------- +So far, ``np.polyfit`` used a non-standard factor in the scaling of the the +covariance matrix. Namely, rather than using the standard ``chisq/(M-N)``, it +scaled it with ``chisq/(M-N-2)`` where M is the number of data points and N is the +number of parameters. This scaling is inconsistent with other fitting programs +such as e.g. ``scipy.optimize.curve_fit`` and was changed to ``chisq/(M-N)``. + +``maximum`` and ``minimum`` no longer emit warnings +--------------------------------------------------- +As part of code introduced in 1.10, ``float32`` and ``float64`` set invalid +float status when a Nan is encountered in `numpy.maximum` and `numpy.minimum`, +when using SSE2 semantics. This caused a `RuntimeWarning` to sometimes be +emitted. In 1.15 we fixed the inconsistencies which caused the warnings to +become more conspicuous. Now no warnings will be emitted. + +Umath and multiarray c-extension modules merged into a single module +-------------------------------------------------------------------- +The two modules were merged, according to `NEP 15`_. Previously `np.core.umath` +and `np.core.multiarray` were seperate c-extension modules. They are now python +wrappers to the single `np.core/_multiarray_math` c-extension module. + +.. _`NEP 15` : http://www.numpy.org/neps/nep-0015-merge-multiarray-umath.html + +``getfield`` validity checks extended +------------------------------------- +`numpy.ndarray.getfield` now checks the dtype and offset arguments to prevent +accessing invalid memory locations. + +NumPy functions now support overrides with ``__array_function__`` +----------------------------------------------------------------- +NumPy has a new experimental mechanism for overriding the implementation of +almost all NumPy functions on non-NumPy arrays by defining an +``__array_function__`` method, as described in `NEP 18`_. + +This feature is not yet been enabled by default, but has been released to +facilitate experimentation by potential users. See the NEP for details on +setting the appropriate environment variable. We expect the NumPy 1.17 release +will enable overrides by default, which will also be more performant due to a +new implementation written in C. + +.. _`NEP 18` : http://www.numpy.org/neps/nep-0018-array-function-protocol.html + +Arrays based off readonly buffers cannot be set ``writeable`` +------------------------------------------------------------- +We now disallow setting the ``writeable`` flag True on arrays created +from ``fromstring(readonly-buffer)``. diff --git a/doc/source/release/1.16.1-notes.rst b/doc/source/release/1.16.1-notes.rst new file mode 100644 index 000000000..2a190ef91 --- /dev/null +++ b/doc/source/release/1.16.1-notes.rst @@ -0,0 +1,107 @@ +========================== +NumPy 1.16.1 Release Notes +========================== + +The NumPy 1.16.1 release fixes bugs reported against the 1.16.0 release, and +also backports several enhancements from master that seem appropriate for a +release series that is the last to support Python 2.7. The wheels on PyPI are +linked with OpenBLAS v0.3.4+, which should fix the known threading issues +found in previous OpenBLAS versions. + +Downstream developers building this release should use Cython >= 0.29.2 and, if +using OpenBLAS, OpenBLAS > v0.3.4. + +If you are installing using pip, you may encounter a problem with older +installed versions of NumPy that pip did not delete becoming mixed with the +current version, resulting in an ``ImportError``. That problem is particularly +common on Debian derived distributions due to a modified pip. The fix is to +make sure all previous NumPy versions installed by pip have been removed. See +`#12736 <https://github.com/numpy/numpy/issues/12736>`__ for discussion of the +issue. Note that previously this problem resulted in an ``AttributeError``. + + +Contributors +============ + +A total of 16 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Antoine Pitrou +* Arcesio Castaneda Medina + +* Charles Harris +* Chris Markiewicz + +* Christoph Gohlke +* Christopher J. Markiewicz + +* Daniel Hrisca + +* EelcoPeacs + +* Eric Wieser +* Kevin Sheppard +* Matti Picus +* OBATA Akio + +* Ralf Gommers +* Sebastian Berg +* Stephan Hoyer +* Tyler Reddy + + +Enhancements +============ + +* `#12767 <https://github.com/numpy/numpy/pull/12767>`__: ENH: add mm->q floordiv +* `#12768 <https://github.com/numpy/numpy/pull/12768>`__: ENH: port np.core.overrides to C for speed +* `#12769 <https://github.com/numpy/numpy/pull/12769>`__: ENH: Add np.ctypeslib.as_ctypes_type(dtype), improve `np.ctypeslib.as_ctypes` +* `#12773 <https://github.com/numpy/numpy/pull/12773>`__: ENH: add "max difference" messages to np.testing.assert_array_equal... +* `#12820 <https://github.com/numpy/numpy/pull/12820>`__: ENH: Add mm->qm divmod +* `#12890 <https://github.com/numpy/numpy/pull/12890>`__: ENH: add _dtype_ctype to namespace for freeze analysis + + +Compatibility notes +=================== + +* The changed error message emited by array comparison testing functions may + affect doctests. See below for detail. + +* Casting from double and single denormals to float16 has been corrected. In + some rare cases, this may result in results being rounded up instead of down, + changing the last bit (ULP) of the result. + + +New Features +============ + +divmod operation is now supported for two ``timedelta64`` operands +------------------------------------------------------------------ +The divmod operator now handles two ``np.timedelta64`` operands, with +type signature ``mm->qm``. + + +Improvements +============ + +Further improvements to ``ctypes`` support in ``np.ctypeslib`` +-------------------------------------------------------------- +A new `numpy.ctypeslib.as_ctypes_type` function has been added, which can be +used to converts a `dtype` into a best-guess `ctypes` type. Thanks to this +new function, `numpy.ctypeslib.as_ctypes` now supports a much wider range of +array types, including structures, booleans, and integers of non-native +endianness. + +Array comparison assertions include maximum differences +------------------------------------------------------- +Error messages from array comparison tests such as +`np.testing.assert_allclose` now include "max absolute difference" and +"max relative difference," in addition to the previous "mismatch" percentage. +This information makes it easier to update absolute and relative error +tolerances. + + +Changes +======= + +``timedelta64 % 0`` behavior adjusted to return ``NaT`` +------------------------------------------------------- +The modulus operation with two ``np.timedelta64`` operands now returns +``NaT`` in the case of division by zero, rather than returning zero + + + diff --git a/doc/source/release/1.16.2-notes.rst b/doc/source/release/1.16.2-notes.rst new file mode 100644 index 000000000..62b90dc40 --- /dev/null +++ b/doc/source/release/1.16.2-notes.rst @@ -0,0 +1,70 @@ +========================== +NumPy 1.16.2 Release Notes +========================== + +NumPy 1.16.2 is a quick release fixing several problems encountered on Windows. +The Python versions supported are 2.7 and 3.5-3.7. The Windows problems +addressed are: + +- DLL load problems for NumPy wheels on Windows, +- distutils command line parsing on Windows. + +There is also a regression fix correcting signed zeros produced by divmod, see +below for details. + +Downstream developers building this release should use Cython >= 0.29.2 and, if +using OpenBLAS, OpenBLAS > v0.3.4. + +If you are installing using pip, you may encounter a problem with older +installed versions of NumPy that pip did not delete becoming mixed with the +current version, resulting in an ``ImportError``. That problem is particularly +common on Debian derived distributions due to a modified pip. The fix is to +make sure all previous NumPy versions installed by pip have been removed. See +`#12736 <https://github.com/numpy/numpy/issues/12736>`__ for discussion of the +issue. + + +Compatibility notes +=================== + +Signed zero when using divmod +----------------------------- +Starting in version 1.12.0, numpy incorrectly returned a negatively signed zero +when using the ``divmod`` and ``floor_divide`` functions when the result was +zero. For example:: + + >>> np.zeros(10)//1 + array([-0., -0., -0., -0., -0., -0., -0., -0., -0., -0.]) + +With this release, the result is correctly returned as a positively signed +zero:: + + >>> np.zeros(10)//1 + array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) + + +Contributors +============ + +A total of 5 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Eric Wieser +* Matti Picus +* Tyler Reddy +* Tony LaTorre + + + +Pull requests merged +==================== + +A total of 7 pull requests were merged for this release. + +* `#12909 <https://github.com/numpy/numpy/pull/12909>`__: TST: fix vmImage dispatch in Azure +* `#12923 <https://github.com/numpy/numpy/pull/12923>`__: MAINT: remove complicated test of multiarray import failure mode +* `#13020 <https://github.com/numpy/numpy/pull/13020>`__: BUG: fix signed zero behavior in npy_divmod +* `#13026 <https://github.com/numpy/numpy/pull/13026>`__: MAINT: Add functions to parse shell-strings in the platform-native... +* `#13028 <https://github.com/numpy/numpy/pull/13028>`__: BUG: Fix regression in parsing of F90 and F77 environment variables +* `#13038 <https://github.com/numpy/numpy/pull/13038>`__: BUG: parse shell escaping in extra_compile_args and extra_link_args +* `#13041 <https://github.com/numpy/numpy/pull/13041>`__: BLD: Windows absolute path DLL loading diff --git a/doc/source/release/1.16.3-notes.rst b/doc/source/release/1.16.3-notes.rst new file mode 100644 index 000000000..181a7264d --- /dev/null +++ b/doc/source/release/1.16.3-notes.rst @@ -0,0 +1,46 @@ +========================== +NumPy 1.16.3 Release Notes +========================== + +The NumPy 1.16.3 release fixes bugs reported against the 1.16.2 release, and +also backports several enhancements from master that seem appropriate for a +release series that is the last to support Python 2.7. The wheels on PyPI are +linked with OpenBLAS v0.3.4+, which should fix the known threading issues +found in previous OpenBLAS versions. + +Downstream developers building this release should use Cython >= 0.29.2 and, +if using OpenBLAS, OpenBLAS > v0.3.4. + +The most noticeable change in this release is that unpickling object arrays +when loading ``*.npy`` or ``*.npz`` files now requires an explicit opt-in. +This backwards incompatible change was made in response to +`CVE-2019-6446 <https://nvd.nist.gov/vuln/detail/CVE-2019-6446>`_. + + +Compatibility notes +=================== + +Unpickling while loading requires explicit opt-in +------------------------------------------------- +The functions ``np.load``, and ``np.lib.format.read_array`` take an +`allow_pickle` keyword which now defaults to ``False`` in response to +`CVE-2019-6446 <https://nvd.nist.gov/vuln/detail/CVE-2019-6446>`_. + + +Improvements +============ + +Covariance in `random.mvnormal` cast to double +---------------------------------------------- +This should make the tolerance used when checking the singular values of the +covariance matrix more meaningful. + + +Changes +======= + +``__array_interface__`` offset now works as documented +------------------------------------------------------ +The interface may use an ``offset`` value that was previously mistakenly +ignored. + diff --git a/doc/source/release/1.16.4-notes.rst b/doc/source/release/1.16.4-notes.rst new file mode 100644 index 000000000..a236b05c8 --- /dev/null +++ b/doc/source/release/1.16.4-notes.rst @@ -0,0 +1,94 @@ +========================== +NumPy 1.16.4 Release Notes +========================== + +The NumPy 1.16.4 release fixes bugs reported against the 1.16.3 release, and +also backports several enhancements from master that seem appropriate for a +release series that is the last to support Python 2.7. The wheels on PyPI are +linked with OpenBLAS v0.3.7-dev, which should fix issues on Skylake series +cpus. + +Downstream developers building this release should use Cython >= 0.29.2 and, +if using OpenBLAS, OpenBLAS > v0.3.7. The supported Python versions are 2.7 and +3.5-3.7. + + +New deprecations +================ +Writeable flag of C-API wrapped arrays +-------------------------------------- +When an array is created from the C-API to wrap a pointer to data, the only +indication we have of the read-write nature of the data is the ``writeable`` +flag set during creation. It is dangerous to force the flag to writeable. In +the future it will not be possible to switch the writeable flag to ``True`` +from python. This deprecation should not affect many users since arrays +created in such a manner are very rare in practice and only available through +the NumPy C-API. + + +Compatibility notes +=================== + +Potential changes to the random stream +-------------------------------------- +Due to bugs in the application of log to random floating point numbers, +the stream may change when sampling from ``np.random.beta``, ``np.random.binomial``, +``np.random.laplace``, ``np.random.logistic``, ``np.random.logseries`` or +``np.random.multinomial`` if a 0 is generated in the underlying MT19937 random stream. +There is a 1 in :math:`10^{53}` chance of this occurring, and so the probability that +the stream changes for any given seed is extremely small. If a 0 is encountered in the +underlying generator, then the incorrect value produced (either ``np.inf`` +or ``np.nan``) is now dropped. + + +Changes +======= + +`numpy.lib.recfunctions.structured_to_unstructured` does not squeeze single-field views +--------------------------------------------------------------------------------------- +Previously ``structured_to_unstructured(arr[['a']])`` would produce a squeezed +result inconsistent with ``structured_to_unstructured(arr[['a', b']])``. This +was accidental. The old behavior can be retained with +``structured_to_unstructured(arr[['a']]).squeeze(axis=-1)`` or far more simply, +``arr['a']``. + + +Contributors +============ + +A total of 10 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Eric Wieser +* Dennis Zollo + +* Hunter Damron + +* Jingbei Li + +* Kevin Sheppard +* Matti Picus +* Nicola Soranzo + +* Sebastian Berg +* Tyler Reddy + + +Pull requests merged +==================== + +A total of 16 pull requests were merged for this release. + +* `#13392 <https://github.com/numpy/numpy/pull/13392>`__: BUG: Some PyPy versions lack PyStructSequence_InitType2. +* `#13394 <https://github.com/numpy/numpy/pull/13394>`__: MAINT, DEP: Fix deprecated ``assertEquals()`` +* `#13396 <https://github.com/numpy/numpy/pull/13396>`__: BUG: Fix structured_to_unstructured on single-field types (backport) +* `#13549 <https://github.com/numpy/numpy/pull/13549>`__: BLD: Make CI pass again with pytest 4.5 +* `#13552 <https://github.com/numpy/numpy/pull/13552>`__: TST: Register markers in conftest.py. +* `#13559 <https://github.com/numpy/numpy/pull/13559>`__: BUG: Removes ValueError for empty kwargs in arraymultiter_new +* `#13560 <https://github.com/numpy/numpy/pull/13560>`__: BUG: Add TypeError to accepted exceptions in crackfortran. +* `#13561 <https://github.com/numpy/numpy/pull/13561>`__: BUG: Handle subarrays in descr_to_dtype +* `#13562 <https://github.com/numpy/numpy/pull/13562>`__: BUG: Protect generators from log(0.0) +* `#13563 <https://github.com/numpy/numpy/pull/13563>`__: BUG: Always return views from structured_to_unstructured when... +* `#13564 <https://github.com/numpy/numpy/pull/13564>`__: BUG: Catch stderr when checking compiler version +* `#13565 <https://github.com/numpy/numpy/pull/13565>`__: BUG: longdouble(int) does not work +* `#13587 <https://github.com/numpy/numpy/pull/13587>`__: BUG: distutils/system_info.py fix missing subprocess import (#13523) +* `#13620 <https://github.com/numpy/numpy/pull/13620>`__: BUG,DEP: Fix writeable flag setting for arrays without base +* `#13641 <https://github.com/numpy/numpy/pull/13641>`__: MAINT: Prepare for the 1.16.4 release. +* `#13644 <https://github.com/numpy/numpy/pull/13644>`__: BUG: special case object arrays when printing rel-, abs-error diff --git a/doc/source/release/1.16.5-notes.rst b/doc/source/release/1.16.5-notes.rst new file mode 100644 index 000000000..5b6eb585b --- /dev/null +++ b/doc/source/release/1.16.5-notes.rst @@ -0,0 +1,68 @@ +========================== +NumPy 1.16.5 Release Notes +========================== + +The NumPy 1.16.5 release fixes bugs reported against the 1.16.4 release, and +also backports several enhancements from master that seem appropriate for a +release series that is the last to support Python 2.7. The wheels on PyPI are +linked with OpenBLAS v0.3.7-dev, which should fix errors on Skylake series +cpus. + +Downstream developers building this release should use Cython >= 0.29.2 and, if +using OpenBLAS, OpenBLAS >= v0.3.7. The supported Python versions are 2.7 and +3.5-3.7. + + +Contributors +============ + +A total of 18 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Alexander Shadchin +* Allan Haldane +* Bruce Merry + +* Charles Harris +* Colin Snyder + +* Dan Allan + +* Emile + +* Eric Wieser +* Grey Baker + +* Maksim Shabunin + +* Marten van Kerkwijk +* Matti Picus +* Peter Andreas Entschev + +* Ralf Gommers +* Richard Harris + +* Sebastian Berg +* Sergei Lebedev + +* Stephan Hoyer + +Pull requests merged +==================== + +A total of 23 pull requests were merged for this release. + +* `#13742 <https://github.com/numpy/numpy/pull/13742>`__: ENH: Add project URLs to setup.py +* `#13823 <https://github.com/numpy/numpy/pull/13823>`__: TEST, ENH: fix tests and ctypes code for PyPy +* `#13845 <https://github.com/numpy/numpy/pull/13845>`__: BUG: use npy_intp instead of int for indexing array +* `#13867 <https://github.com/numpy/numpy/pull/13867>`__: TST: Ignore DeprecationWarning during nose imports +* `#13905 <https://github.com/numpy/numpy/pull/13905>`__: BUG: Fix use-after-free in boolean indexing +* `#13933 <https://github.com/numpy/numpy/pull/13933>`__: MAINT/BUG/DOC: Fix errors in _add_newdocs +* `#13984 <https://github.com/numpy/numpy/pull/13984>`__: BUG: fix byte order reversal for datetime64[ns] +* `#13994 <https://github.com/numpy/numpy/pull/13994>`__: MAINT,BUG: Use nbytes to also catch empty descr during allocation +* `#14042 <https://github.com/numpy/numpy/pull/14042>`__: BUG: np.array cleared errors occured in PyMemoryView_FromObject +* `#14043 <https://github.com/numpy/numpy/pull/14043>`__: BUG: Fixes for Undefined Behavior Sanitizer (UBSan) errors. +* `#14044 <https://github.com/numpy/numpy/pull/14044>`__: BUG: ensure that casting to/from structured is properly checked. +* `#14045 <https://github.com/numpy/numpy/pull/14045>`__: MAINT: fix histogram*d dispatchers +* `#14046 <https://github.com/numpy/numpy/pull/14046>`__: BUG: further fixup to histogram2d dispatcher. +* `#14052 <https://github.com/numpy/numpy/pull/14052>`__: BUG: Replace contextlib.suppress for Python 2.7 +* `#14056 <https://github.com/numpy/numpy/pull/14056>`__: BUG: fix compilation of 3rd party modules with Py_LIMITED_API... +* `#14057 <https://github.com/numpy/numpy/pull/14057>`__: BUG: Fix memory leak in dtype from dict contructor +* `#14058 <https://github.com/numpy/numpy/pull/14058>`__: DOC: Document array_function at a higher level. +* `#14084 <https://github.com/numpy/numpy/pull/14084>`__: BUG, DOC: add new recfunctions to `__all__` +* `#14162 <https://github.com/numpy/numpy/pull/14162>`__: BUG: Remove stray print that causes a SystemError on python 3.7 +* `#14297 <https://github.com/numpy/numpy/pull/14297>`__: TST: Pin pytest version to 5.0.1. +* `#14322 <https://github.com/numpy/numpy/pull/14322>`__: ENH: Enable huge pages in all Linux builds +* `#14346 <https://github.com/numpy/numpy/pull/14346>`__: BUG: fix behavior of structured_to_unstructured on non-trivial... +* `#14382 <https://github.com/numpy/numpy/pull/14382>`__: REL: Prepare for the NumPy 1.16.5 release. diff --git a/doc/source/release/1.17.0-notes.rst b/doc/source/release/1.17.0-notes.rst new file mode 100644 index 000000000..8d69e36d9 --- /dev/null +++ b/doc/source/release/1.17.0-notes.rst @@ -0,0 +1,562 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.17.0 Release Notes +========================== + +This NumPy release contains a number of new features that should substantially +improve its performance and usefulness, see Highlights below for a summary. The +Python versions supported are 3.5-3.7, note that Python 2.7 has been dropped. +Python 3.8b2 should work with the released source packages, but there are no +future guarantees. + +Downstream developers should use Cython >= 0.29.11 for Python 3.8 support and +OpenBLAS >= 3.7 (not currently out) to avoid problems on the Skylake +architecture. The NumPy wheels on PyPI are built from the OpenBLAS development +branch in order to avoid those problems. + + +Highlights +========== + +* A new extensible `random` module along with four selectable `random number + generators <random.BitGenerators>` and improved seeding designed for use in parallel + processes has been added. The currently available bit generators are `MT19937 + <random.mt19937.MT19937>`, `PCG64 <random.pcg64.PCG64>`, `Philox + <random.philox.Philox>`, and `SFC64 <random.sfc64.SFC64>`. See below under + New Features. + +* NumPy's `FFT <fft>` implementation was changed from fftpack to pocketfft, + resulting in faster, more accurate transforms and better handling of datasets + of prime length. See below under Improvements. + +* New radix sort and timsort sorting methods. It is currently not possible to + choose which will be used. They are hardwired to the datatype and used + when either ``stable`` or ``mergesort`` is passed as the method. See below + under Improvements. + +* Overriding numpy functions is now possible by default, + see ``__array_function__`` below. + + +New functions +============= + +* `numpy.errstate` is now also a function decorator + + +Deprecations +============ + +`numpy.polynomial` functions warn when passed ``float`` in place of ``int`` +--------------------------------------------------------------------------- +Previously functions in this module would accept ``float`` values provided they +were integral (``1.0``, ``2.0``, etc). For consistency with the rest of numpy, +doing so is now deprecated, and in future will raise a ``TypeError``. + +Similarly, passing a float like ``0.5`` in place of an integer will now raise a +``TypeError`` instead of the previous ``ValueError``. + +Deprecate `numpy.distutils.exec_command` and ``temp_file_name`` +--------------------------------------------------------------- +The internal use of these functions has been refactored and there are better +alternatives. Replace ``exec_command`` with `subprocess.Popen` and +`temp_file_name <numpy.distutils.exec_command>` with `tempfile.mkstemp`. + +Writeable flag of C-API wrapped arrays +-------------------------------------- +When an array is created from the C-API to wrap a pointer to data, the only +indication we have of the read-write nature of the data is the ``writeable`` +flag set during creation. It is dangerous to force the flag to writeable. +In the future it will not be possible to switch the writeable flag to ``True`` +from python. +This deprecation should not affect many users since arrays created in such +a manner are very rare in practice and only available through the NumPy C-API. + +`numpy.nonzero` should no longer be called on 0d arrays +------------------------------------------------------- +The behavior of `numpy.nonzero` on 0d arrays was surprising, making uses of it +almost always incorrect. If the old behavior was intended, it can be preserved +without a warning by using ``nonzero(atleast_1d(arr))`` instead of +``nonzero(arr)``. In a future release, it is most likely this will raise a +``ValueError``. + +Writing to the result of `numpy.broadcast_arrays` will warn +----------------------------------------------------------- + +Commonly `numpy.broadcast_arrays` returns a writeable array with internal +overlap, making it unsafe to write to. A future version will set the +``writeable`` flag to ``False``, and require users to manually set it to +``True`` if they are sure that is what they want to do. Now writing to it will +emit a deprecation warning with instructions to set the ``writeable`` flag +``True``. Note that if one were to inspect the flag before setting it, one +would find it would already be ``True``. Explicitly setting it, though, as one +will need to do in future versions, clears an internal flag that is used to +produce the deprecation warning. To help alleviate confusion, an additional +`FutureWarning` will be emitted when accessing the ``writeable`` flag state to +clarify the contradiction. + +Note that for the C-side buffer protocol such an array will return a +readonly buffer immediately unless a writable buffer is requested. If +a writeable buffer is requested a warning will be given. When using +cython, the ``const`` qualifier should be used with such arrays to avoid +the warning (e.g. ``cdef const double[::1] view``). + + +Future Changes +============== + +Shape-1 fields in dtypes won't be collapsed to scalars in a future version +-------------------------------------------------------------------------- + +Currently, a field specified as ``[(name, dtype, 1)]`` or ``"1type"`` is +interpreted as a scalar field (i.e., the same as ``[(name, dtype)]`` or +``[(name, dtype, ()]``). This now raises a FutureWarning; in a future version, +it will be interpreted as a shape-(1,) field, i.e. the same as ``[(name, +dtype, (1,))]`` or ``"(1,)type"`` (consistently with ``[(name, dtype, n)]`` +/ ``"ntype"`` with ``n>1``, which is already equivalent to ``[(name, dtype, +(n,)]`` / ``"(n,)type"``). + + +Compatibility notes +=================== + +``float16`` subnormal rounding +------------------------------ +Casting from a different floating point precision to ``float16`` used incorrect +rounding in some edge cases. This means in rare cases, subnormal results will +now be rounded up instead of down, changing the last bit (ULP) of the result. + +Signed zero when using divmod +----------------------------- +Starting in version `1.12.0`, numpy incorrectly returned a negatively signed zero +when using the ``divmod`` and ``floor_divide`` functions when the result was +zero. For example:: + + >>> np.zeros(10)//1 + array([-0., -0., -0., -0., -0., -0., -0., -0., -0., -0.]) + +With this release, the result is correctly returned as a positively signed +zero:: + + >>> np.zeros(10)//1 + array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) + +``MaskedArray.mask`` now returns a view of the mask, not the mask itself +------------------------------------------------------------------------ +Returning the mask itself was unsafe, as it could be reshaped in place which +would violate expectations of the masked array code. The behavior of `mask +<ma.MaskedArray.mask>` is now consistent with `data <ma.MaskedArray.data>`, +which also returns a view. + +The underlying mask can still be accessed with ``._mask`` if it is needed. +Tests that contain ``assert x.mask is not y.mask`` or similar will need to be +updated. + +Do not lookup ``__buffer__`` attribute in `numpy.frombuffer` +------------------------------------------------------------ +Looking up ``__buffer__`` attribute in `numpy.frombuffer` was undocumented and +non-functional. This code was removed. If needed, use +``frombuffer(memoryview(obj), ...)`` instead. + +``out`` is buffered for memory overlaps in `take`, `choose`, `put` +------------------------------------------------------------------ +If the out argument to these functions is provided and has memory overlap with +the other arguments, it is now buffered to avoid order-dependent behavior. + +Unpickling while loading requires explicit opt-in +------------------------------------------------- +The functions `load`, and ``lib.format.read_array`` take an +``allow_pickle`` keyword which now defaults to ``False`` in response to +`CVE-2019-6446 <https://nvd.nist.gov/vuln/detail/CVE-2019-6446>`_. + + +.. currentmodule:: numpy.random.mtrand + +Potential changes to the random stream in old random module +----------------------------------------------------------- +Due to bugs in the application of ``log`` to random floating point numbers, +the stream may change when sampling from `~RandomState.beta`, `~RandomState.binomial`, +`~RandomState.laplace`, `~RandomState.logistic`, `~RandomState.logseries` or +`~RandomState.multinomial` if a ``0`` is generated in the underlying `MT19937 +<~numpy.random.mt11937.MT19937>` random stream. There is a ``1`` in +:math:`10^{53}` chance of this occurring, so the probability that the stream +changes for any given seed is extremely small. If a ``0`` is encountered in the +underlying generator, then the incorrect value produced (either `numpy.inf` or +`numpy.nan`) is now dropped. + +.. currentmodule:: numpy + +`i0` now always returns a result with the same shape as the input +----------------------------------------------------------------- +Previously, the output was squeezed, such that, e.g., input with just a single +element would lead to an array scalar being returned, and inputs with shapes +such as ``(10, 1)`` would yield results that would not broadcast against the +input. + +Note that we generally recommend the SciPy implementation over the numpy one: +it is a proper ufunc written in C, and more than an order of magnitude faster. + +`can_cast` no longer assumes all unsafe casting is allowed +---------------------------------------------------------- +Previously, `can_cast` returned `True` for almost all inputs for +``casting='unsafe'``, even for cases where casting was not possible, such as +from a structured dtype to a regular one. This has been fixed, making it +more consistent with actual casting using, e.g., the `.astype <ndarray.astype>` +method. + +``ndarray.flags.writeable`` can be switched to true slightly more often +----------------------------------------------------------------------- + +In rare cases, it was not possible to switch an array from not writeable +to writeable, although a base array is writeable. This can happen if an +intermediate `ndarray.base` object is writeable. Previously, only the deepest +base object was considered for this decision. However, in rare cases this +object does not have the necessary information. In that case switching to +writeable was never allowed. This has now been fixed. + + +C API changes +============= + +dimension or stride input arguments are now passed by ``npy_intp const*`` +------------------------------------------------------------------------- +Previously these function arguments were declared as the more strict +``npy_intp*``, which prevented the caller passing constant data. +This change is backwards compatible, but now allows code like:: + + npy_intp const fixed_dims[] = {1, 2, 3}; + // no longer complains that the const-qualifier is discarded + npy_intp size = PyArray_MultiplyList(fixed_dims, 3); + + +New Features +============ + +.. currentmodule:: numpy.random + +New extensible `numpy.random` module with selectable random number generators +----------------------------------------------------------------------------- +A new extensible `numpy.random` module along with four selectable random number +generators and improved seeding designed for use in parallel processes has been +added. The currently available :ref:`Bit Generators <bit_generator>` are +`~mt19937.MT19937`, `~pcg64.PCG64`, `~philox.Philox`, and `~sfc64.SFC64`. +``PCG64`` is the new default while ``MT19937`` is retained for backwards +compatibility. Note that the legacy random module is unchanged and is now +frozen, your current results will not change. More information is available in +the :ref:`API change description <new-or-different>` and in the `top-level view +<numpy.random>` documentation. + +.. currentmodule:: numpy + +libFLAME +-------- +Support for building NumPy with the libFLAME linear algebra package as the LAPACK, +implementation, see +`libFLAME <https://www.cs.utexas.edu/~flame/web/libFLAME.html>`_ for details. + +User-defined BLAS detection order +--------------------------------- +`distutils` now uses an environment variable, comma-separated and case +insensitive, to determine the detection order for BLAS libraries. +By default ``NPY_BLAS_ORDER=mkl,blis,openblas,atlas,accelerate,blas``. +However, to force the use of OpenBLAS simply do:: + + NPY_BLAS_ORDER=openblas python setup.py build + +which forces the use of OpenBLAS. +This may be helpful for users which have a MKL installation but wishes to try +out different implementations. + +User-defined LAPACK detection order +----------------------------------- +``numpy.distutils`` now uses an environment variable, comma-separated and case +insensitive, to determine the detection order for LAPACK libraries. +By default ``NPY_LAPACK_ORDER=mkl,openblas,flame,atlas,accelerate,lapack``. +However, to force the use of OpenBLAS simply do:: + + NPY_LAPACK_ORDER=openblas python setup.py build + +which forces the use of OpenBLAS. +This may be helpful for users which have a MKL installation but wishes to try +out different implementations. + +`ufunc.reduce` and related functions now accept a ``where`` mask +---------------------------------------------------------------- +`ufunc.reduce`, `sum`, `prod`, `min`, `max` all +now accept a ``where`` keyword argument, which can be used to tell which +elements to include in the reduction. For reductions that do not have an +identity, it is necessary to also pass in an initial value (e.g., +``initial=np.inf`` for `min`). For instance, the equivalent of +`nansum` would be ``np.sum(a, where=~np.isnan(a))``. + +Timsort and radix sort have replaced mergesort for stable sorting +----------------------------------------------------------------- +Both radix sort and timsort have been implemented and are now used in place of +mergesort. Due to the need to maintain backward compatibility, the sorting +``kind`` options ``"stable"`` and ``"mergesort"`` have been made aliases of +each other with the actual sort implementation depending on the array type. +Radix sort is used for small integer types of 16 bits or less and timsort for +the remaining types. Timsort features improved performace on data containing +already or nearly sorted data and performs like mergesort on random data and +requires :math:`O(n/2)` working space. Details of the timsort algorithm can be +found at `CPython listsort.txt +<https://github.com/python/cpython/blob/3.7/Objects/listsort.txt>`_. + +`packbits` and `unpackbits` accept an ``order`` keyword +------------------------------------------------------- +The ``order`` keyword defaults to ``big``, and will order the **bits** +accordingly. For ``'order=big'`` 3 will become ``[0, 0, 0, 0, 0, 0, 1, 1]``, +and ``[1, 1, 0, 0, 0, 0, 0, 0]`` for ``order=little`` + +`unpackbits` now accepts a ``count`` parameter +---------------------------------------------- +``count`` allows subsetting the number of bits that will be unpacked up-front, +rather than reshaping and subsetting later, making the `packbits` operation +invertible, and the unpacking less wasteful. Counts larger than the number of +available bits add zero padding. Negative counts trim bits off the end instead +of counting from the beginning. None counts implement the existing behavior of +unpacking everything. + +`linalg.svd` and `linalg.pinv` can be faster on hermitian inputs +---------------------------------------------------------------- +These functions now accept a ``hermitian`` argument, matching the one added +to `linalg.matrix_rank` in 1.14.0. + +divmod operation is now supported for two ``timedelta64`` operands +------------------------------------------------------------------ +The divmod operator now handles two ``timedelta64`` operands, with +type signature ``mm->qm``. + +`fromfile` now takes an ``offset`` argument +------------------------------------------- +This function now takes an ``offset`` keyword argument for binary files, +which specifics the offset (in bytes) from the file's current position. +Defaults to ``0``. + +New mode "empty" for `pad` +-------------------------- +This mode pads an array to a desired shape without initializing the new +entries. + +`empty_like` and related functions now accept a ``shape`` argument +------------------------------------------------------------------ +`empty_like`, `full_like`, `ones_like` and `zeros_like` now accept a ``shape`` +keyword argument, which can be used to create a new array +as the prototype, overriding its shape as well. This is particularly useful +when combined with the ``__array_function__`` protocol, allowing the creation +of new arbitrary-shape arrays from NumPy-like libraries when such an array +is used as the prototype. + +Floating point scalars implement ``as_integer_ratio`` to match the builtin float +-------------------------------------------------------------------------------- +This returns a (numerator, denominator) pair, which can be used to construct a +`fractions.Fraction`. + +Structured ``dtype`` objects can be indexed with multiple fields names +---------------------------------------------------------------------- +``arr.dtype[['a', 'b']]`` now returns a dtype that is equivalent to +``arr[['a', 'b']].dtype``, for consistency with +``arr.dtype['a'] == arr['a'].dtype``. + +Like the dtype of structured arrays indexed with a list of fields, this dtype +has the same ``itemsize`` as the original, but only keeps a subset of the fields. + +This means that ``arr[['a', 'b']]`` and ``arr.view(arr.dtype[['a', 'b']])`` are +equivalent. + +``.npy`` files support unicode field names +------------------------------------------ +A new format version of 3.0 has been introduced, which enables structured types +with non-latin1 field names. This is used automatically when needed. + + +Improvements +============ + +Array comparison assertions include maximum differences +------------------------------------------------------- +Error messages from array comparison tests such as +`testing.assert_allclose` now include "max absolute difference" and +"max relative difference," in addition to the previous "mismatch" percentage. +This information makes it easier to update absolute and relative error +tolerances. + +Replacement of the fftpack based `fft` module by the pocketfft library +---------------------------------------------------------------------- +Both implementations have the same ancestor (Fortran77 FFTPACK by Paul N. +Swarztrauber), but pocketfft contains additional modifications which improve +both accuracy and performance in some circumstances. For FFT lengths containing +large prime factors, pocketfft uses Bluestein's algorithm, which maintains +:math:`O(N log N)` run time complexity instead of deteriorating towards +:math:`O(N*N)` for prime lengths. Also, accuracy for real valued FFTs with near +prime lengths has improved and is on par with complex valued FFTs. + +Further improvements to ``ctypes`` support in `numpy.ctypeslib` +--------------------------------------------------------------- +A new `numpy.ctypeslib.as_ctypes_type` function has been added, which can be +used to converts a `dtype` into a best-guess `ctypes` type. Thanks to this +new function, `numpy.ctypeslib.as_ctypes` now supports a much wider range of +array types, including structures, booleans, and integers of non-native +endianness. + +`numpy.errstate` is now also a function decorator +------------------------------------------------- +Currently, if you have a function like:: + + def foo(): + pass + +and you want to wrap the whole thing in `errstate`, you have to rewrite it +like so:: + + def foo(): + with np.errstate(...): + pass + +but with this change, you can do:: + + @np.errstate(...) + def foo(): + pass + +thereby saving a level of indentation + +`numpy.exp` and `numpy.log` speed up for float32 implementation +--------------------------------------------------------------- +float32 implementation of `exp` and `log` now benefit from AVX2/AVX512 +instruction set which are detected during runtime. `exp` has a max ulp +error of 2.52 and `log` has a max ulp error or 3.83. + +Improve performance of `numpy.pad` +---------------------------------- +The performance of the function has been improved for most cases by filling in +a preallocated array with the desired padded shape instead of using +concatenation. + +`numpy.interp` handles infinities more robustly +----------------------------------------------- +In some cases where `interp` would previously return `nan`, it now +returns an appropriate infinity. + +Pathlib support for `fromfile`, `tofile` and `ndarray.dump` +----------------------------------------------------------- +`fromfile`, `ndarray.ndarray.tofile` and `ndarray.dump` now support +the `pathlib.Path` type for the ``file``/``fid`` parameter. + +Specialized `isnan`, `isinf`, and `isfinite` ufuncs for bool and int types +-------------------------------------------------------------------------- +The boolean and integer types are incapable of storing `nan` and `inf` values, +which allows us to provide specialized ufuncs that are up to 250x faster than +the previous approach. + +`isfinite` supports ``datetime64`` and ``timedelta64`` types +----------------------------------------------------------------- +Previously, `isfinite` used to raise a `TypeError` on being used on these +two types. + +New keywords added to `nan_to_num` +---------------------------------- +`nan_to_num` now accepts keywords ``nan``, ``posinf`` and ``neginf`` +allowing the user to define the value to replace the ``nan``, positive and +negative ``np.inf`` values respectively. + +MemoryErrors caused by allocated overly large arrays are more descriptive +------------------------------------------------------------------------- +Often the cause of a MemoryError is incorrect broadcasting, which results in a +very large and incorrect shape. The message of the error now includes this +shape to help diagnose the cause of failure. + +`floor`, `ceil`, and `trunc` now respect builtin magic methods +-------------------------------------------------------------- +These ufuncs now call the ``__floor__``, ``__ceil__``, and ``__trunc__`` +methods when called on object arrays, making them compatible with +`decimal.Decimal` and `fractions.Fraction` objects. + +`quantile` now works on `fraction.Fraction` and `decimal.Decimal` objects +------------------------------------------------------------------------- +In general, this handles object arrays more gracefully, and avoids floating- +point operations if exact arithmetic types are used. + +Support of object arrays in `matmul` +------------------------------------ +It is now possible to use `matmul` (or the ``@`` operator) with object arrays. +For instance, it is now possible to do:: + + from fractions import Fraction + a = np.array([[Fraction(1, 2), Fraction(1, 3)], [Fraction(1, 3), Fraction(1, 2)]]) + b = a @ a + + +Changes +======= + +`median` and `percentile` family of functions no longer warn about ``nan`` +-------------------------------------------------------------------------- +`numpy.median`, `numpy.percentile`, and `numpy.quantile` used to emit a +``RuntimeWarning`` when encountering an `nan`. Since they return the +``nan`` value, the warning is redundant and has been removed. + +``timedelta64 % 0`` behavior adjusted to return ``NaT`` +------------------------------------------------------- +The modulus operation with two ``np.timedelta64`` operands now returns +``NaT`` in the case of division by zero, rather than returning zero + +NumPy functions now always support overrides with ``__array_function__`` +------------------------------------------------------------------------ +NumPy now always checks the ``__array_function__`` method to implement overrides +of NumPy functions on non-NumPy arrays, as described in `NEP 18`_. The feature +was available for testing with NumPy 1.16 if appropriate environment variables +are set, but is now always enabled. + +.. _`NEP 18` : http://www.numpy.org/neps/nep-0018-array-function-protocol.html + +``lib.recfunctions.structured_to_unstructured`` does not squeeze single-field views +----------------------------------------------------------------------------------- +Previously ``structured_to_unstructured(arr[['a']])`` would produce a squeezed +result inconsistent with ``structured_to_unstructured(arr[['a', b']])``. This +was accidental. The old behavior can be retained with +``structured_to_unstructured(arr[['a']]).squeeze(axis=-1)`` or far more simply, +``arr['a']``. + +`clip` now uses a ufunc under the hood +-------------------------------------- +This means that registering clip functions for custom dtypes in C via +``descr->f->fastclip`` is deprecated - they should use the ufunc registration +mechanism instead, attaching to the ``np.core.umath.clip`` ufunc. + +It also means that ``clip`` accepts ``where`` and ``casting`` arguments, +and can be override with ``__array_ufunc__``. + +A consequence of this change is that some behaviors of the old ``clip`` have +been deprecated: + +* Passing ``nan`` to mean "do not clip" as one or both bounds. This didn't work + in all cases anyway, and can be better handled by passing infinities of the + appropriate sign. +* Using "unsafe" casting by default when an ``out`` argument is passed. Using + ``casting="unsafe"`` explicitly will silence this warning. + +Additionally, there are some corner cases with behavior changes: + +* Padding ``max < min`` has changed to be more consistent across dtypes, but + should not be relied upon. +* Scalar ``min`` and ``max`` take part in promotion rules like they do in all + other ufuncs. + +``__array_interface__`` offset now works as documented +------------------------------------------------------ +The interface may use an ``offset`` value that was mistakenly ignored. + +Pickle protocol in `savez` set to 3 for ``force zip64`` flag +----------------------------------------------------------------- +`savez` was not using the ``force_zip64`` flag, which limited the size of +the archive to 2GB. But using the flag requires us to use pickle protocol 3 to +write ``object`` arrays. The protocol used was bumped to 3, meaning the archive +will be unreadable by Python2. + +Structured arrays indexed with non-existent fields raise ``KeyError`` not ``ValueError`` +---------------------------------------------------------------------------------------- +``arr['bad_field']`` on a structured type raises ``KeyError``, for consistency +with ``dict['bad_field']``. + diff --git a/doc/source/release/1.17.1-notes.rst b/doc/source/release/1.17.1-notes.rst new file mode 100644 index 000000000..bd837ee5b --- /dev/null +++ b/doc/source/release/1.17.1-notes.rst @@ -0,0 +1,73 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.17.1 Release Notes +========================== + +This release contains a number of fixes for bugs reported against NumPy 1.17.0 +along with a few documentation and build improvements. The Python versions +supported are 3.5-3.7, note that Python 2.7 has been dropped. Python 3.8b3 +should work with the released source packages, but there are no future +guarantees. + +Downstream developers should use Cython >= 0.29.13 for Python 3.8 support and +OpenBLAS >= 3.7 to avoid problems on the Skylake architecture. The NumPy wheels +on PyPI are built from the OpenBLAS development branch in order to avoid those +problems. + + +Contributors +============ + +A total of 17 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Alexander Jung + +* Allan Haldane +* Charles Harris +* Eric Wieser +* Giuseppe Cuccu + +* Hiroyuki V. Yamazaki +* Jérémie du Boisberranger +* Kmol Yuan + +* Matti Picus +* Max Bolingbroke + +* Maxwell Aladago + +* Oleksandr Pavlyk +* Peter Andreas Entschev +* Sergei Lebedev +* Seth Troisi + +* Vladimir Pershin + +* Warren Weckesser + + +Pull requests merged +==================== + +A total of 24 pull requests were merged for this release. + +* `#14156 <https://github.com/numpy/numpy/pull/14156>`__: TST: Allow fuss in testing strided/non-strided exp/log loops +* `#14157 <https://github.com/numpy/numpy/pull/14157>`__: BUG: avx2_scalef_ps must be static +* `#14158 <https://github.com/numpy/numpy/pull/14158>`__: BUG: Remove stray print that causes a SystemError on python 3.7. +* `#14159 <https://github.com/numpy/numpy/pull/14159>`__: BUG: Fix DeprecationWarning in python 3.8. +* `#14160 <https://github.com/numpy/numpy/pull/14160>`__: BLD: Add missing gcd/lcm definitions to npy_math.h +* `#14161 <https://github.com/numpy/numpy/pull/14161>`__: DOC, BUILD: cleanups and fix (again) 'build dist' +* `#14166 <https://github.com/numpy/numpy/pull/14166>`__: TST: Add 3.8-dev to travisCI testing. +* `#14194 <https://github.com/numpy/numpy/pull/14194>`__: BUG: Remove the broken clip wrapper (Backport) +* `#14198 <https://github.com/numpy/numpy/pull/14198>`__: DOC: Fix hermitian argument docs in svd. +* `#14199 <https://github.com/numpy/numpy/pull/14199>`__: MAINT: Workaround for Intel compiler bug leading to failing test +* `#14200 <https://github.com/numpy/numpy/pull/14200>`__: TST: Clean up of test_pocketfft.py +* `#14201 <https://github.com/numpy/numpy/pull/14201>`__: BUG: Make advanced indexing result on read-only subclass writeable... +* `#14236 <https://github.com/numpy/numpy/pull/14236>`__: BUG: Fixed default BitGenerator name +* `#14237 <https://github.com/numpy/numpy/pull/14237>`__: ENH: add c-imported modules for freeze analysis in np.random +* `#14296 <https://github.com/numpy/numpy/pull/14296>`__: TST: Pin pytest version to 5.0.1 +* `#14301 <https://github.com/numpy/numpy/pull/14301>`__: BUG: Fix leak in the f2py-generated module init and `PyMem_Del`... +* `#14302 <https://github.com/numpy/numpy/pull/14302>`__: BUG: Fix formatting error in exception message +* `#14307 <https://github.com/numpy/numpy/pull/14307>`__: MAINT: random: Match type of SeedSequence.pool_size to DEFAULT_POOL_SIZE. +* `#14308 <https://github.com/numpy/numpy/pull/14308>`__: BUG: Fix numpy.random bug in platform detection +* `#14309 <https://github.com/numpy/numpy/pull/14309>`__: ENH: Enable huge pages in all Linux builds +* `#14330 <https://github.com/numpy/numpy/pull/14330>`__: BUG: Fix segfault in `random.permutation(x)` when x is a string. +* `#14338 <https://github.com/numpy/numpy/pull/14338>`__: BUG: don't fail when lexsorting some empty arrays (#14228) +* `#14339 <https://github.com/numpy/numpy/pull/14339>`__: BUG: Fix misuse of .names and .fields in various places (backport... +* `#14345 <https://github.com/numpy/numpy/pull/14345>`__: BUG: fix behavior of structured_to_unstructured on non-trivial... +* `#14350 <https://github.com/numpy/numpy/pull/14350>`__: REL: Prepare 1.17.1 release diff --git a/doc/source/release/1.17.2-notes.rst b/doc/source/release/1.17.2-notes.rst new file mode 100644 index 000000000..65cdaf903 --- /dev/null +++ b/doc/source/release/1.17.2-notes.rst @@ -0,0 +1,49 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.17.2 Release Notes +========================== + +This release contains fixes for bugs reported against NumPy 1.17.1 along with a +some documentation improvements. The most important fix is for lexsort when the +keys are of type (u)int8 or (u)int16. If you are currently using 1.17 you +should upgrade. + +The Python versions supported in this release are 3.5-3.7, Python 2.7 has been +dropped. Python 3.8b4 should work with the released source packages, but there +are no future guarantees. + +Downstream developers should use Cython >= 0.29.13 for Python 3.8 support and +OpenBLAS >= 3.7 to avoid errors on the Skylake architecture. The NumPy wheels +on PyPI are built from the OpenBLAS development branch in order to avoid those +errors. + + +Contributors +============ + +A total of 7 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* CakeWithSteak + +* Charles Harris +* Dan Allan +* Hameer Abbasi +* Lars Grueter +* Matti Picus +* Sebastian Berg + + +Pull requests merged +==================== + +A total of 8 pull requests were merged for this release. + +* `#14418 <https://github.com/numpy/numpy/pull/14418>`__: BUG: Fix aradixsort indirect indexing. +* `#14420 <https://github.com/numpy/numpy/pull/14420>`__: DOC: Fix a minor typo in dispatch documentation. +* `#14421 <https://github.com/numpy/numpy/pull/14421>`__: BUG: test, fix regression in converting to ctypes +* `#14430 <https://github.com/numpy/numpy/pull/14430>`__: BUG: Do not show Override module in private error classes. +* `#14432 <https://github.com/numpy/numpy/pull/14432>`__: BUG: Fixed maximum relative error reporting in assert_allclose. +* `#14433 <https://github.com/numpy/numpy/pull/14433>`__: BUG: Fix uint-overflow if padding with linear_ramp and negative... +* `#14436 <https://github.com/numpy/numpy/pull/14436>`__: BUG: Update 1.17.x with 1.18.0-dev pocketfft.py. +* `#14446 <https://github.com/numpy/numpy/pull/14446>`__: REL: Prepare for NumPy 1.17.2 release. diff --git a/doc/source/release/1.18.0-notes.rst b/doc/source/release/1.18.0-notes.rst new file mode 100644 index 000000000..e66540410 --- /dev/null +++ b/doc/source/release/1.18.0-notes.rst @@ -0,0 +1,8 @@ +The NumPy 1.18 release is currently in developement. Please check +the ``numpy/doc/release/upcoming_changes/`` folder for upcoming +release notes. +The ``numpy/doc/release/upcoming_changes/README.txt`` details how +to add new release notes. + +For the work in progress release notes for the current development +version, see the `devdocs <https://numpy.org/devdocs/release.html>`__. diff --git a/doc/source/release/1.3.0-notes.rst b/doc/source/release/1.3.0-notes.rst new file mode 100644 index 000000000..239714246 --- /dev/null +++ b/doc/source/release/1.3.0-notes.rst @@ -0,0 +1,278 @@ +========================= +NumPy 1.3.0 Release Notes +========================= + +This minor includes numerous bug fixes, official python 2.6 support, and +several new features such as generalized ufuncs. + +Highlights +========== + +Python 2.6 support +------------------ + +Python 2.6 is now supported on all previously supported platforms, including +windows. + +https://www.python.org/dev/peps/pep-0361/ + +Generalized ufuncs +------------------ + +There is a general need for looping over not only functions on scalars but also +over functions on vectors (or arrays), as explained on +http://scipy.org/scipy/numpy/wiki/GeneralLoopingFunctions. We propose to +realize this concept by generalizing the universal functions (ufuncs), and +provide a C implementation that adds ~500 lines to the numpy code base. In +current (specialized) ufuncs, the elementary function is limited to +element-by-element operations, whereas the generalized version supports +"sub-array" by "sub-array" operations. The Perl vector library PDL provides a +similar functionality and its terms are re-used in the following. + +Each generalized ufunc has information associated with it that states what the +"core" dimensionality of the inputs is, as well as the corresponding +dimensionality of the outputs (the element-wise ufuncs have zero core +dimensions). The list of the core dimensions for all arguments is called the +"signature" of a ufunc. For example, the ufunc numpy.add has signature +"(),()->()" defining two scalar inputs and one scalar output. + +Another example is (see the GeneralLoopingFunctions page) the function +inner1d(a,b) with a signature of "(i),(i)->()". This applies the inner product +along the last axis of each input, but keeps the remaining indices intact. For +example, where a is of shape (3,5,N) and b is of shape (5,N), this will return +an output of shape (3,5). The underlying elementary function is called 3*5 +times. In the signature, we specify one core dimension "(i)" for each input and +zero core dimensions "()" for the output, since it takes two 1-d arrays and +returns a scalar. By using the same name "i", we specify that the two +corresponding dimensions should be of the same size (or one of them is of size +1 and will be broadcasted). + +The dimensions beyond the core dimensions are called "loop" dimensions. In the +above example, this corresponds to (3,5). + +The usual numpy "broadcasting" rules apply, where the signature determines how +the dimensions of each input/output object are split into core and loop +dimensions: + +While an input array has a smaller dimensionality than the corresponding number +of core dimensions, 1's are pre-pended to its shape. The core dimensions are +removed from all inputs and the remaining dimensions are broadcasted; defining +the loop dimensions. The output is given by the loop dimensions plus the +output core dimensions. + +Experimental Windows 64 bits support +------------------------------------ + +Numpy can now be built on windows 64 bits (amd64 only, not IA64), with both MS +compilers and mingw-w64 compilers: + +This is *highly experimental*: DO NOT USE FOR PRODUCTION USE. See INSTALL.txt, +Windows 64 bits section for more information on limitations and how to build it +by yourself. + +New features +============ + +Formatting issues +----------------- + +Float formatting is now handled by numpy instead of the C runtime: this enables +locale independent formatting, more robust fromstring and related methods. +Special values (inf and nan) are also more consistent across platforms (nan vs +IND/NaN, etc...), and more consistent with recent python formatting work (in +2.6 and later). + +Nan handling in max/min +----------------------- + +The maximum/minimum ufuncs now reliably propagate nans. If one of the +arguments is a nan, then nan is returned. This affects np.min/np.max, amin/amax +and the array methods max/min. New ufuncs fmax and fmin have been added to deal +with non-propagating nans. + +Nan handling in sign +-------------------- + +The ufunc sign now returns nan for the sign of anan. + + +New ufuncs +---------- + +#. fmax - same as maximum for integer types and non-nan floats. Returns the + non-nan argument if one argument is nan and returns nan if both arguments + are nan. +#. fmin - same as minimum for integer types and non-nan floats. Returns the + non-nan argument if one argument is nan and returns nan if both arguments + are nan. +#. deg2rad - converts degrees to radians, same as the radians ufunc. +#. rad2deg - converts radians to degrees, same as the degrees ufunc. +#. log2 - base 2 logarithm. +#. exp2 - base 2 exponential. +#. trunc - truncate floats to nearest integer towards zero. +#. logaddexp - add numbers stored as logarithms and return the logarithm + of the result. +#. logaddexp2 - add numbers stored as base 2 logarithms and return the base 2 + logarithm of the result. + +Masked arrays +------------- + +Several new features and bug fixes, including: + + * structured arrays should now be fully supported by MaskedArray + (r6463, r6324, r6305, r6300, r6294...) + * Minor bug fixes (r6356, r6352, r6335, r6299, r6298) + * Improved support for __iter__ (r6326) + * made baseclass, sharedmask and hardmask accessible to the user (but + read-only) + * doc update + +gfortran support on windows +--------------------------- + +Gfortran can now be used as a fortran compiler for numpy on windows, even when +the C compiler is Visual Studio (VS 2005 and above; VS 2003 will NOT work). +Gfortran + Visual studio does not work on windows 64 bits (but gcc + gfortran +does). It is unclear whether it will be possible to use gfortran and visual +studio at all on x64. + +Arch option for windows binary +------------------------------ + +Automatic arch detection can now be bypassed from the command line for the superpack installed: + + numpy-1.3.0-superpack-win32.exe /arch=nosse + +will install a numpy which works on any x86, even if the running computer +supports SSE set. + +Deprecated features +=================== + +Histogram +--------- + +The semantics of histogram has been modified to fix long-standing issues +with outliers handling. The main changes concern + +#. the definition of the bin edges, now including the rightmost edge, and +#. the handling of upper outliers, now ignored rather than tallied in the + rightmost bin. + +The previous behavior is still accessible using `new=False`, but this is +deprecated, and will be removed entirely in 1.4.0. + +Documentation changes +===================== + +A lot of documentation has been added. Both user guide and references can be +built from sphinx. + +New C API +========= + +Multiarray API +-------------- + +The following functions have been added to the multiarray C API: + + * PyArray_GetEndianness: to get runtime endianness + +Ufunc API +--------- + +The following functions have been added to the ufunc API: + + * PyUFunc_FromFuncAndDataAndSignature: to declare a more general ufunc + (generalized ufunc). + + +New defines +----------- + +New public C defines are available for ARCH specific code through numpy/npy_cpu.h: + + * NPY_CPU_X86: x86 arch (32 bits) + * NPY_CPU_AMD64: amd64 arch (x86_64, NOT Itanium) + * NPY_CPU_PPC: 32 bits ppc + * NPY_CPU_PPC64: 64 bits ppc + * NPY_CPU_SPARC: 32 bits sparc + * NPY_CPU_SPARC64: 64 bits sparc + * NPY_CPU_S390: S390 + * NPY_CPU_IA64: ia64 + * NPY_CPU_PARISC: PARISC + +New macros for CPU endianness has been added as well (see internal changes +below for details): + + * NPY_BYTE_ORDER: integer + * NPY_LITTLE_ENDIAN/NPY_BIG_ENDIAN defines + +Those provide portable alternatives to glibc endian.h macros for platforms +without it. + +Portable NAN, INFINITY, etc... +------------------------------ + +npy_math.h now makes available several portable macro to get NAN, INFINITY: + + * NPY_NAN: equivalent to NAN, which is a GNU extension + * NPY_INFINITY: equivalent to C99 INFINITY + * NPY_PZERO, NPY_NZERO: positive and negative zero respectively + +Corresponding single and extended precision macros are available as well. All +references to NAN, or home-grown computation of NAN on the fly have been +removed for consistency. + +Internal changes +================ + +numpy.core math configuration revamp +------------------------------------ + +This should make the porting to new platforms easier, and more robust. In +particular, the configuration stage does not need to execute any code on the +target platform, which is a first step toward cross-compilation. + +https://www.numpy.org/neps/nep-0003-math_config_clean.html + +umath refactor +-------------- + +A lot of code cleanup for umath/ufunc code (charris). + +Improvements to build warnings +------------------------------ + +Numpy can now build with -W -Wall without warnings + +https://www.numpy.org/neps/nep-0002-warnfix.html + +Separate core math library +-------------------------- + +The core math functions (sin, cos, etc... for basic C types) have been put into +a separate library; it acts as a compatibility layer, to support most C99 maths +functions (real only for now). The library includes platform-specific fixes for +various maths functions, such as using those versions should be more robust +than using your platform functions directly. The API for existing functions is +exactly the same as the C99 math functions API; the only difference is the npy +prefix (npy_cos vs cos). + +The core library will be made available to any extension in 1.4.0. + +CPU arch detection +------------------ + +npy_cpu.h defines numpy specific CPU defines, such as NPY_CPU_X86, etc... +Those are portable across OS and toolchains, and set up when the header is +parsed, so that they can be safely used even in the case of cross-compilation +(the values is not set when numpy is built), or for multi-arch binaries (e.g. +fat binaries on Max OS X). + +npy_endian.h defines numpy specific endianness defines, modeled on the glibc +endian.h. NPY_BYTE_ORDER is equivalent to BYTE_ORDER, and one of +NPY_LITTLE_ENDIAN or NPY_BIG_ENDIAN is defined. As for CPU archs, those are set +when the header is parsed by the compiler, and as such can be used for +cross-compilation and multi-arch binaries. diff --git a/doc/source/release/1.4.0-notes.rst b/doc/source/release/1.4.0-notes.rst new file mode 100644 index 000000000..9480a054e --- /dev/null +++ b/doc/source/release/1.4.0-notes.rst @@ -0,0 +1,238 @@ +========================= +NumPy 1.4.0 Release Notes +========================= + +This minor includes numerous bug fixes, as well as a few new features. It +is backward compatible with 1.3.0 release. + +Highlights +========== + +* New datetime dtype support to deal with dates in arrays + +* Faster import time + +* Extended array wrapping mechanism for ufuncs + +* New Neighborhood iterator (C-level only) + +* C99-like complex functions in npymath + +New features +============ + +Extended array wrapping mechanism for ufuncs +-------------------------------------------- + +An __array_prepare__ method has been added to ndarray to provide subclasses +greater flexibility to interact with ufuncs and ufunc-like functions. ndarray +already provided __array_wrap__, which allowed subclasses to set the array type +for the result and populate metadata on the way out of the ufunc (as seen in +the implementation of MaskedArray). For some applications it is necessary to +provide checks and populate metadata *on the way in*. __array_prepare__ is +therefore called just after the ufunc has initialized the output array but +before computing the results and populating it. This way, checks can be made +and errors raised before operations which may modify data in place. + +Automatic detection of forward incompatibilities +------------------------------------------------ + +Previously, if an extension was built against a version N of NumPy, and used on +a system with NumPy M < N, the import_array was successful, which could cause +crashes because the version M does not have a function in N. Starting from +NumPy 1.4.0, this will cause a failure in import_array, so the error will be +caught early on. + +New iterators +------------- + +A new neighborhood iterator has been added to the C API. It can be used to +iterate over the items in a neighborhood of an array, and can handle boundaries +conditions automatically. Zero and one padding are available, as well as +arbitrary constant value, mirror and circular padding. + +New polynomial support +---------------------- + +New modules chebyshev and polynomial have been added. The new polynomial module +is not compatible with the current polynomial support in numpy, but is much +like the new chebyshev module. The most noticeable difference to most will +be that coefficients are specified from low to high power, that the low +level functions do *not* work with the Chebyshev and Polynomial classes as +arguments, and that the Chebyshev and Polynomial classes include a domain. +Mapping between domains is a linear substitution and the two classes can be +converted one to the other, allowing, for instance, a Chebyshev series in +one domain to be expanded as a polynomial in another domain. The new classes +should generally be used instead of the low level functions, the latter are +provided for those who wish to build their own classes. + +The new modules are not automatically imported into the numpy namespace, +they must be explicitly brought in with an "import numpy.polynomial" +statement. + +New C API +--------- + +The following C functions have been added to the C API: + + #. PyArray_GetNDArrayCFeatureVersion: return the *API* version of the + loaded numpy. + #. PyArray_Correlate2 - like PyArray_Correlate, but implements the usual + definition of correlation. Inputs are not swapped, and conjugate is + taken for complex arrays. + #. PyArray_NeighborhoodIterNew - a new iterator to iterate over a + neighborhood of a point, with automatic boundaries handling. It is + documented in the iterators section of the C-API reference, and you can + find some examples in the multiarray_test.c.src file in numpy.core. + +New ufuncs +---------- + +The following ufuncs have been added to the C API: + + #. copysign - return the value of the first argument with the sign copied + from the second argument. + #. nextafter - return the next representable floating point value of the + first argument toward the second argument. + +New defines +----------- + +The alpha processor is now defined and available in numpy/npy_cpu.h. The +failed detection of the PARISC processor has been fixed. The defines are: + + #. NPY_CPU_HPPA: PARISC + #. NPY_CPU_ALPHA: Alpha + +Testing +------- + + #. deprecated decorator: this decorator may be used to avoid cluttering + testing output while testing DeprecationWarning is effectively raised by + the decorated test. + #. assert_array_almost_equal_nulps: new method to compare two arrays of + floating point values. With this function, two values are considered + close if there are not many representable floating point values in + between, thus being more robust than assert_array_almost_equal when the + values fluctuate a lot. + #. assert_array_max_ulp: raise an assertion if there are more than N + representable numbers between two floating point values. + #. assert_warns: raise an AssertionError if a callable does not generate a + warning of the appropriate class, without altering the warning state. + +Reusing npymath +--------------- + +In 1.3.0, we started putting portable C math routines in npymath library, so +that people can use those to write portable extensions. Unfortunately, it was +not possible to easily link against this library: in 1.4.0, support has been +added to numpy.distutils so that 3rd party can reuse this library. See coremath +documentation for more information. + +Improved set operations +----------------------- + +In previous versions of NumPy some set functions (intersect1d, +setxor1d, setdiff1d and setmember1d) could return incorrect results if +the input arrays contained duplicate items. These now work correctly +for input arrays with duplicates. setmember1d has been renamed to +in1d, as with the change to accept arrays with duplicates it is +no longer a set operation, and is conceptually similar to an +elementwise version of the Python operator 'in'. All of these +functions now accept the boolean keyword assume_unique. This is False +by default, but can be set True if the input arrays are known not +to contain duplicates, which can increase the functions' execution +speed. + +Improvements +============ + + #. numpy import is noticeably faster (from 20 to 30 % depending on the + platform and computer) + + #. The sort functions now sort nans to the end. + + * Real sort order is [R, nan] + * Complex sort order is [R + Rj, R + nanj, nan + Rj, nan + nanj] + + Complex numbers with the same nan placements are sorted according to + the non-nan part if it exists. + #. The type comparison functions have been made consistent with the new + sort order of nans. Searchsorted now works with sorted arrays + containing nan values. + #. Complex division has been made more resistant to overflow. + #. Complex floor division has been made more resistant to overflow. + +Deprecations +============ + +The following functions are deprecated: + + #. correlate: it takes a new keyword argument old_behavior. When True (the + default), it returns the same result as before. When False, compute the + conventional correlation, and take the conjugate for complex arrays. The + old behavior will be removed in NumPy 1.5, and raises a + DeprecationWarning in 1.4. + + #. unique1d: use unique instead. unique1d raises a deprecation + warning in 1.4, and will be removed in 1.5. + + #. intersect1d_nu: use intersect1d instead. intersect1d_nu raises + a deprecation warning in 1.4, and will be removed in 1.5. + + #. setmember1d: use in1d instead. setmember1d raises a deprecation + warning in 1.4, and will be removed in 1.5. + +The following raise errors: + + #. When operating on 0-d arrays, ``numpy.max`` and other functions accept + only ``axis=0``, ``axis=-1`` and ``axis=None``. Using an out-of-bounds + axes is an indication of a bug, so Numpy raises an error for these cases + now. + + #. Specifying ``axis > MAX_DIMS`` is no longer allowed; Numpy raises now an + error instead of behaving similarly as for ``axis=None``. + +Internal changes +================ + +Use C99 complex functions when available +---------------------------------------- + +The numpy complex types are now guaranteed to be ABI compatible with C99 +complex type, if available on the platform. Moreover, the complex ufunc now use +the platform C99 functions instead of our own. + +split multiarray and umath source code +-------------------------------------- + +The source code of multiarray and umath has been split into separate logic +compilation units. This should make the source code more amenable for +newcomers. + +Separate compilation +-------------------- + +By default, every file of multiarray (and umath) is merged into one for +compilation as was the case before, but if NPY_SEPARATE_COMPILATION env +variable is set to a non-negative value, experimental individual compilation of +each file is enabled. This makes the compile/debug cycle much faster when +working on core numpy. + +Separate core math library +-------------------------- + +New functions which have been added: + + * npy_copysign + * npy_nextafter + * npy_cpack + * npy_creal + * npy_cimag + * npy_cabs + * npy_cexp + * npy_clog + * npy_cpow + * npy_csqr + * npy_ccos + * npy_csin diff --git a/doc/source/release/1.5.0-notes.rst b/doc/source/release/1.5.0-notes.rst new file mode 100644 index 000000000..a2184ab13 --- /dev/null +++ b/doc/source/release/1.5.0-notes.rst @@ -0,0 +1,131 @@ +========================= +NumPy 1.5.0 Release Notes +========================= + + +Highlights +========== + +Python 3 compatibility +---------------------- + +This is the first NumPy release which is compatible with Python 3. Support for +Python 3 and Python 2 is done from a single code base. Extensive notes on +changes can be found at +`<http://projects.scipy.org/numpy/browser/trunk/doc/Py3K.txt>`_. + +Note that the Numpy testing framework relies on nose, which does not have a +Python 3 compatible release yet. A working Python 3 branch of nose can be found +at `<http://bitbucket.org/jpellerin/nose3/>`_ however. + +Porting of SciPy to Python 3 is expected to be completed soon. + +:pep:`3118` compatibility +------------------------- + +The new buffer protocol described by PEP 3118 is fully supported in this +version of Numpy. On Python versions >= 2.6 Numpy arrays expose the buffer +interface, and array(), asarray() and other functions accept new-style buffers +as input. + + +New features +============ + +Warning on casting complex to real +---------------------------------- + +Numpy now emits a `numpy.ComplexWarning` when a complex number is cast +into a real number. For example: + + >>> x = np.array([1,2,3]) + >>> x[:2] = np.array([1+2j, 1-2j]) + ComplexWarning: Casting complex values to real discards the imaginary part + +The cast indeed discards the imaginary part, and this may not be the +intended behavior in all cases, hence the warning. This warning can be +turned off in the standard way: + + >>> import warnings + >>> warnings.simplefilter("ignore", np.ComplexWarning) + +Dot method for ndarrays +----------------------- + +Ndarrays now have the dot product also as a method, which allows writing +chains of matrix products as + + >>> a.dot(b).dot(c) + +instead of the longer alternative + + >>> np.dot(a, np.dot(b, c)) + +linalg.slogdet function +----------------------- + +The slogdet function returns the sign and logarithm of the determinant +of a matrix. Because the determinant may involve the product of many +small/large values, the result is often more accurate than that obtained +by simple multiplication. + +new header +---------- + +The new header file ndarraytypes.h contains the symbols from +ndarrayobject.h that do not depend on the PY_ARRAY_UNIQUE_SYMBOL and +NO_IMPORT/_ARRAY macros. Broadly, these symbols are types, typedefs, +and enumerations; the array function calls are left in +ndarrayobject.h. This allows users to include array-related types and +enumerations without needing to concern themselves with the macro +expansions and their side- effects. + + +Changes +======= + +polynomial.polynomial +--------------------- + +* The polyint and polyder functions now check that the specified number + integrations or derivations is a non-negative integer. The number 0 is + a valid value for both functions. +* A degree method has been added to the Polynomial class. +* A trimdeg method has been added to the Polynomial class. It operates like + truncate except that the argument is the desired degree of the result, + not the number of coefficients. +* Polynomial.fit now uses None as the default domain for the fit. The default + Polynomial domain can be specified by using [] as the domain value. +* Weights can be used in both polyfit and Polynomial.fit +* A linspace method has been added to the Polynomial class to ease plotting. +* The polymulx function was added. + +polynomial.chebyshev +-------------------- + +* The chebint and chebder functions now check that the specified number + integrations or derivations is a non-negative integer. The number 0 is + a valid value for both functions. +* A degree method has been added to the Chebyshev class. +* A trimdeg method has been added to the Chebyshev class. It operates like + truncate except that the argument is the desired degree of the result, + not the number of coefficients. +* Chebyshev.fit now uses None as the default domain for the fit. The default + Chebyshev domain can be specified by using [] as the domain value. +* Weights can be used in both chebfit and Chebyshev.fit +* A linspace method has been added to the Chebyshev class to ease plotting. +* The chebmulx function was added. +* Added functions for the Chebyshev points of the first and second kind. + + +histogram +--------- + +After a two years transition period, the old behavior of the histogram function +has been phased out, and the "new" keyword has been removed. + +correlate +--------- + +The old behavior of correlate was deprecated in 1.4.0, the new behavior (the +usual definition for cross-correlation) is now the default. diff --git a/doc/source/release/1.6.0-notes.rst b/doc/source/release/1.6.0-notes.rst new file mode 100644 index 000000000..c5f53a0eb --- /dev/null +++ b/doc/source/release/1.6.0-notes.rst @@ -0,0 +1,177 @@ +========================= +NumPy 1.6.0 Release Notes +========================= + +This release includes several new features as well as numerous bug fixes and +improved documentation. It is backward compatible with the 1.5.0 release, and +supports Python 2.4 - 2.7 and 3.1 - 3.2. + + +Highlights +========== + +* Re-introduction of datetime dtype support to deal with dates in arrays. + +* A new 16-bit floating point type. + +* A new iterator, which improves performance of many functions. + + +New features +============ + +New 16-bit floating point type +------------------------------ + +This release adds support for the IEEE 754-2008 binary16 format, available as +the data type ``numpy.half``. Within Python, the type behaves similarly to +`float` or `double`, and C extensions can add support for it with the exposed +half-float API. + + +New iterator +------------ + +A new iterator has been added, replacing the functionality of the +existing iterator and multi-iterator with a single object and API. +This iterator works well with general memory layouts different from +C or Fortran contiguous, and handles both standard NumPy and +customized broadcasting. The buffering, automatic data type +conversion, and optional output parameters, offered by +ufuncs but difficult to replicate elsewhere, are now exposed by this +iterator. + + +Legendre, Laguerre, Hermite, HermiteE polynomials in ``numpy.polynomial`` +------------------------------------------------------------------------- + +Extend the number of polynomials available in the polynomial package. In +addition, a new ``window`` attribute has been added to the classes in +order to specify the range the ``domain`` maps to. This is mostly useful +for the Laguerre, Hermite, and HermiteE polynomials whose natural domains +are infinite and provides a more intuitive way to get the correct mapping +of values without playing unnatural tricks with the domain. + + +Fortran assumed shape array and size function support in ``numpy.f2py`` +----------------------------------------------------------------------- + +F2py now supports wrapping Fortran 90 routines that use assumed shape +arrays. Before such routines could be called from Python but the +corresponding Fortran routines received assumed shape arrays as zero +length arrays which caused unpredicted results. Thanks to Lorenz +Hüdepohl for pointing out the correct way to interface routines with +assumed shape arrays. + +In addition, f2py supports now automatic wrapping of Fortran routines +that use two argument ``size`` function in dimension specifications. + + +Other new functions +------------------- + +``numpy.ravel_multi_index`` : Converts a multi-index tuple into +an array of flat indices, applying boundary modes to the indices. + +``numpy.einsum`` : Evaluate the Einstein summation convention. Using the +Einstein summation convention, many common multi-dimensional array operations +can be represented in a simple fashion. This function provides a way compute +such summations. + +``numpy.count_nonzero`` : Counts the number of non-zero elements in an array. + +``numpy.result_type`` and ``numpy.min_scalar_type`` : These functions expose +the underlying type promotion used by the ufuncs and other operations to +determine the types of outputs. These improve upon the ``numpy.common_type`` +and ``numpy.mintypecode`` which provide similar functionality but do +not match the ufunc implementation. + + +Changes +======= + +``default error handling`` +-------------------------- + +The default error handling has been change from ``print`` to ``warn`` for +all except for ``underflow``, which remains as ``ignore``. + + +``numpy.distutils`` +------------------- + +Several new compilers are supported for building Numpy: the Portland Group +Fortran compiler on OS X, the PathScale compiler suite and the 64-bit Intel C +compiler on Linux. + + +``numpy.testing`` +----------------- + +The testing framework gained ``numpy.testing.assert_allclose``, which provides +a more convenient way to compare floating point arrays than +`assert_almost_equal`, `assert_approx_equal` and `assert_array_almost_equal`. + + +``C API`` +--------- + +In addition to the APIs for the new iterator and half data type, a number +of other additions have been made to the C API. The type promotion +mechanism used by ufuncs is exposed via ``PyArray_PromoteTypes``, +``PyArray_ResultType``, and ``PyArray_MinScalarType``. A new enumeration +``NPY_CASTING`` has been added which controls what types of casts are +permitted. This is used by the new functions ``PyArray_CanCastArrayTo`` +and ``PyArray_CanCastTypeTo``. A more flexible way to handle +conversion of arbitrary python objects into arrays is exposed by +``PyArray_GetArrayParamsFromObject``. + + +Deprecated features +=================== + +The "normed" keyword in ``numpy.histogram`` is deprecated. Its functionality +will be replaced by the new "density" keyword. + + +Removed features +================ + +``numpy.fft`` +------------- + +The functions `refft`, `refft2`, `refftn`, `irefft`, `irefft2`, `irefftn`, +which were aliases for the same functions without the 'e' in the name, were +removed. + + +``numpy.memmap`` +---------------- + +The `sync()` and `close()` methods of memmap were removed. Use `flush()` and +"del memmap" instead. + + +``numpy.lib`` +------------- + +The deprecated functions ``numpy.unique1d``, ``numpy.setmember1d``, +``numpy.intersect1d_nu`` and ``numpy.lib.ufunclike.log2`` were removed. + + +``numpy.ma`` +------------ + +Several deprecated items were removed from the ``numpy.ma`` module:: + + * ``numpy.ma.MaskedArray`` "raw_data" method + * ``numpy.ma.MaskedArray`` constructor "flag" keyword + * ``numpy.ma.make_mask`` "flag" keyword + * ``numpy.ma.allclose`` "fill_value" keyword + + +``numpy.distutils`` +------------------- + +The ``numpy.get_numpy_include`` function was removed, use ``numpy.get_include`` +instead. diff --git a/doc/source/release/1.6.1-notes.rst b/doc/source/release/1.6.1-notes.rst new file mode 100644 index 000000000..05fcb4ab9 --- /dev/null +++ b/doc/source/release/1.6.1-notes.rst @@ -0,0 +1,21 @@ +========================= +NumPy 1.6.1 Release Notes +========================= + +This is a bugfix only release in the 1.6.x series. + + +Issues Fixed +============ + +* #1834: einsum fails for specific shapes +* #1837: einsum throws nan or freezes python for specific array shapes +* #1838: object <-> structured type arrays regression +* #1851: regression for SWIG based code in 1.6.0 +* #1863: Buggy results when operating on array copied with astype() +* #1870: Fix corner case of object array assignment +* #1843: Py3k: fix error with recarray +* #1885: nditer: Error in detecting double reduction loop +* #1874: f2py: fix --include_paths bug +* #1749: Fix ctypes.load_library() +* #1895/1896: iter: writeonly operands weren't always being buffered correctly diff --git a/doc/source/release/1.6.2-notes.rst b/doc/source/release/1.6.2-notes.rst new file mode 100644 index 000000000..8f0b06f98 --- /dev/null +++ b/doc/source/release/1.6.2-notes.rst @@ -0,0 +1,93 @@ +========================= +NumPy 1.6.2 Release Notes +========================= + +This is a bugfix release in the 1.6.x series. Due to the delay of the NumPy +1.7.0 release, this release contains far more fixes than a regular NumPy bugfix +release. It also includes a number of documentation and build improvements. + +Issues fixed +============ + +``numpy.core`` +-------------- + +* #2063: make unique() return consistent index +* #1138: allow creating arrays from empty buffers or empty slices +* #1446: correct note about correspondence vstack and concatenate +* #1149: make argmin() work for datetime +* #1672: fix allclose() to work for scalar inf +* #1747: make np.median() work for 0-D arrays +* #1776: make complex division by zero to yield inf properly +* #1675: add scalar support for the format() function +* #1905: explicitly check for NaNs in allclose() +* #1952: allow floating ddof in std() and var() +* #1948: fix regression for indexing chararrays with empty list +* #2017: fix type hashing +* #2046: deleting array attributes causes segfault +* #2033: a**2.0 has incorrect type +* #2045: make attribute/iterator_element deletions not segfault +* #2021: fix segfault in searchsorted() +* #2073: fix float16 __array_interface__ bug + + +``numpy.lib`` +------------- + +* #2048: break reference cycle in NpzFile +* #1573: savetxt() now handles complex arrays +* #1387: allow bincount() to accept empty arrays +* #1899: fixed histogramdd() bug with empty inputs +* #1793: fix failing npyio test under py3k +* #1936: fix extra nesting for subarray dtypes +* #1848: make tril/triu return the same dtype as the original array +* #1918: use Py_TYPE to access ob_type, so it works also on Py3 + + +``numpy.distutils`` +------------------- + +* #1261: change compile flag on AIX from -O5 to -O3 +* #1377: update HP compiler flags +* #1383: provide better support for C++ code on HPUX +* #1857: fix build for py3k + pip +* BLD: raise a clearer warning in case of building without cleaning up first +* BLD: follow build_ext coding convention in build_clib +* BLD: fix up detection of Intel CPU on OS X in system_info.py +* BLD: add support for the new X11 directory structure on Ubuntu & co. +* BLD: add ufsparse to the libraries search path. +* BLD: add 'pgfortran' as a valid compiler in the Portland Group +* BLD: update version match regexp for IBM AIX Fortran compilers. + + +``numpy.random`` +---------------- + +* BUG: Use npy_intp instead of long in mtrand + +Changes +======= + +``numpy.f2py`` +-------------- + +* ENH: Introduce new options extra_f77_compiler_args and extra_f90_compiler_args +* BLD: Improve reporting of fcompiler value +* BUG: Fix f2py test_kind.py test + + +``numpy.poly`` +-------------- + +* ENH: Add some tests for polynomial printing +* ENH: Add companion matrix functions +* DOC: Rearrange the polynomial documents +* BUG: Fix up links to classes +* DOC: Add version added to some of the polynomial package modules +* DOC: Document xxxfit functions in the polynomial package modules +* BUG: The polynomial convenience classes let different types interact +* DOC: Document the use of the polynomial convenience classes +* DOC: Improve numpy reference documentation of polynomial classes +* ENH: Improve the computation of polynomials from roots +* STY: Code cleanup in polynomial [*]fromroots functions +* DOC: Remove references to cast and NA, which were added in 1.7 diff --git a/doc/source/release/1.7.0-notes.rst b/doc/source/release/1.7.0-notes.rst new file mode 100644 index 000000000..f111f80dc --- /dev/null +++ b/doc/source/release/1.7.0-notes.rst @@ -0,0 +1,289 @@ +========================= +NumPy 1.7.0 Release Notes +========================= + +This release includes several new features as well as numerous bug fixes and +refactorings. It supports Python 2.4 - 2.7 and 3.1 - 3.3 and is the last +release that supports Python 2.4 - 2.5. + +Highlights +========== + +* ``where=`` parameter to ufuncs (allows the use of boolean arrays to choose + where a computation should be done) +* ``vectorize`` improvements (added 'excluded' and 'cache' keyword, general + cleanup and bug fixes) +* ``numpy.random.choice`` (random sample generating function) + + +Compatibility notes +=================== + +In a future version of numpy, the functions np.diag, np.diagonal, and the +diagonal method of ndarrays will return a view onto the original array, +instead of producing a copy as they do now. This makes a difference if you +write to the array returned by any of these functions. To facilitate this +transition, numpy 1.7 produces a FutureWarning if it detects that you may +be attempting to write to such an array. See the documentation for +np.diagonal for details. + +Similar to np.diagonal above, in a future version of numpy, indexing a +record array by a list of field names will return a view onto the original +array, instead of producing a copy as they do now. As with np.diagonal, +numpy 1.7 produces a FutureWarning if it detects that you may be attempting +to write to such an array. See the documentation for array indexing for +details. + +In a future version of numpy, the default casting rule for UFunc out= +parameters will be changed from 'unsafe' to 'same_kind'. (This also applies +to in-place operations like a += b, which is equivalent to np.add(a, b, +out=a).) Most usages which violate the 'same_kind' rule are likely bugs, so +this change may expose previously undetected errors in projects that depend +on NumPy. In this version of numpy, such usages will continue to succeed, +but will raise a DeprecationWarning. + +Full-array boolean indexing has been optimized to use a different, +optimized code path. This code path should produce the same results, +but any feedback about changes to your code would be appreciated. + +Attempting to write to a read-only array (one with ``arr.flags.writeable`` +set to ``False``) used to raise either a RuntimeError, ValueError, or +TypeError inconsistently, depending on which code path was taken. It now +consistently raises a ValueError. + +The <ufunc>.reduce functions evaluate some reductions in a different order +than in previous versions of NumPy, generally providing higher performance. +Because of the nature of floating-point arithmetic, this may subtly change +some results, just as linking NumPy to a different BLAS implementations +such as MKL can. + +If upgrading from 1.5, then generally in 1.6 and 1.7 there have been +substantial code added and some code paths altered, particularly in the +areas of type resolution and buffered iteration over universal functions. +This might have an impact on your code particularly if you relied on +accidental behavior in the past. + +New features +============ + +Reduction UFuncs Generalize axis= Parameter +------------------------------------------- + +Any ufunc.reduce function call, as well as other reductions like sum, prod, +any, all, max and min support the ability to choose a subset of the axes to +reduce over. Previously, one could say axis=None to mean all the axes or +axis=# to pick a single axis. Now, one can also say axis=(#,#) to pick a +list of axes for reduction. + +Reduction UFuncs New keepdims= Parameter +---------------------------------------- + +There is a new keepdims= parameter, which if set to True, doesn't throw +away the reduction axes but instead sets them to have size one. When this +option is set, the reduction result will broadcast correctly to the +original operand which was reduced. + +Datetime support +---------------- + +.. note:: The datetime API is *experimental* in 1.7.0, and may undergo changes + in future versions of NumPy. + +There have been a lot of fixes and enhancements to datetime64 compared +to NumPy 1.6: + +* the parser is quite strict about only accepting ISO 8601 dates, with a few + convenience extensions +* converts between units correctly +* datetime arithmetic works correctly +* business day functionality (allows the datetime to be used in contexts where + only certain days of the week are valid) + +The notes in `doc/source/reference/arrays.datetime.rst <https://github.com/numpy/numpy/blob/maintenance/1.7.x/doc/source/reference/arrays.datetime.rst>`_ +(also available in the online docs at `arrays.datetime.html +<https://docs.scipy.org/doc/numpy/reference/arrays.datetime.html>`_) should be +consulted for more details. + +Custom formatter for printing arrays +------------------------------------ + +See the new ``formatter`` parameter of the ``numpy.set_printoptions`` +function. + +New function numpy.random.choice +-------------------------------- + +A generic sampling function has been added which will generate samples from +a given array-like. The samples can be with or without replacement, and +with uniform or given non-uniform probabilities. + +New function isclose +-------------------- + +Returns a boolean array where two arrays are element-wise equal within a +tolerance. Both relative and absolute tolerance can be specified. + +Preliminary multi-dimensional support in the polynomial package +--------------------------------------------------------------- + +Axis keywords have been added to the integration and differentiation +functions and a tensor keyword was added to the evaluation functions. +These additions allow multi-dimensional coefficient arrays to be used in +those functions. New functions for evaluating 2-D and 3-D coefficient +arrays on grids or sets of points were added together with 2-D and 3-D +pseudo-Vandermonde matrices that can be used for fitting. + + +Ability to pad rank-n arrays +---------------------------- + +A pad module containing functions for padding n-dimensional arrays has been +added. The various private padding functions are exposed as options to a +public 'pad' function. Example:: + + pad(a, 5, mode='mean') + +Current modes are ``constant``, ``edge``, ``linear_ramp``, ``maximum``, +``mean``, ``median``, ``minimum``, ``reflect``, ``symmetric``, ``wrap``, and +``<function>``. + + +New argument to searchsorted +---------------------------- + +The function searchsorted now accepts a 'sorter' argument that is a +permutation array that sorts the array to search. + +Build system +------------ + +Added experimental support for the AArch64 architecture. + +C API +----- + +New function ``PyArray_RequireWriteable`` provides a consistent interface +for checking array writeability -- any C code which works with arrays whose +WRITEABLE flag is not known to be True a priori, should make sure to call +this function before writing. + +NumPy C Style Guide added (``doc/C_STYLE_GUIDE.rst.txt``). + +Changes +======= + +General +------- + +The function np.concatenate tries to match the layout of its input arrays. +Previously, the layout did not follow any particular reason, and depended +in an undesirable way on the particular axis chosen for concatenation. A +bug was also fixed which silently allowed out of bounds axis arguments. + +The ufuncs logical_or, logical_and, and logical_not now follow Python's +behavior with object arrays, instead of trying to call methods on the +objects. For example the expression (3 and 'test') produces the string +'test', and now np.logical_and(np.array(3, 'O'), np.array('test', 'O')) +produces 'test' as well. + +The ``.base`` attribute on ndarrays, which is used on views to ensure that the +underlying array owning the memory is not deallocated prematurely, now +collapses out references when you have a view-of-a-view. For example:: + + a = np.arange(10) + b = a[1:] + c = b[1:] + +In numpy 1.6, ``c.base`` is ``b``, and ``c.base.base`` is ``a``. In numpy 1.7, +``c.base`` is ``a``. + +To increase backwards compatibility for software which relies on the old +behaviour of ``.base``, we only 'skip over' objects which have exactly the same +type as the newly created view. This makes a difference if you use ``ndarray`` +subclasses. For example, if we have a mix of ``ndarray`` and ``matrix`` objects +which are all views on the same original ``ndarray``:: + + a = np.arange(10) + b = np.asmatrix(a) + c = b[0, 1:] + d = c[0, 1:] + +then ``d.base`` will be ``b``. This is because ``d`` is a ``matrix`` object, +and so the collapsing process only continues so long as it encounters other +``matrix`` objects. It considers ``c``, ``b``, and ``a`` in that order, and +``b`` is the last entry in that list which is a ``matrix`` object. + +Casting Rules +------------- + +Casting rules have undergone some changes in corner cases, due to the +NA-related work. In particular for combinations of scalar+scalar: + +* the `longlong` type (`q`) now stays `longlong` for operations with any other + number (`? b h i l q p B H I`), previously it was cast as `int_` (`l`). The + `ulonglong` type (`Q`) now stays as `ulonglong` instead of `uint` (`L`). + +* the `timedelta64` type (`m`) can now be mixed with any integer type (`b h i l + q p B H I L Q P`), previously it raised `TypeError`. + +For array + scalar, the above rules just broadcast except the case when +the array and scalars are unsigned/signed integers, then the result gets +converted to the array type (of possibly larger size) as illustrated by the +following examples:: + + >>> (np.zeros((2,), dtype=np.uint8) + np.int16(257)).dtype + dtype('uint16') + >>> (np.zeros((2,), dtype=np.int8) + np.uint16(257)).dtype + dtype('int16') + >>> (np.zeros((2,), dtype=np.int16) + np.uint32(2**17)).dtype + dtype('int32') + +Whether the size gets increased depends on the size of the scalar, for +example:: + + >>> (np.zeros((2,), dtype=np.uint8) + np.int16(255)).dtype + dtype('uint8') + >>> (np.zeros((2,), dtype=np.uint8) + np.int16(256)).dtype + dtype('uint16') + +Also a ``complex128`` scalar + ``float32`` array is cast to ``complex64``. + +In NumPy 1.7 the `datetime64` type (`M`) must be constructed by explicitly +specifying the type as the second argument (e.g. ``np.datetime64(2000, 'Y')``). + + +Deprecations +============ + +General +------- + +Specifying a custom string formatter with a `_format` array attribute is +deprecated. The new `formatter` keyword in ``numpy.set_printoptions`` or +``numpy.array2string`` can be used instead. + +The deprecated imports in the polynomial package have been removed. + +``concatenate`` now raises DepractionWarning for 1D arrays if ``axis != 0``. +Versions of numpy < 1.7.0 ignored axis argument value for 1D arrays. We +allow this for now, but in due course we will raise an error. + +C-API +----- + +Direct access to the fields of PyArrayObject* has been deprecated. Direct +access has been recommended against for many releases. Expect similar +deprecations for PyArray_Descr* and other core objects in the future as +preparation for NumPy 2.0. + +The macros in old_defines.h are deprecated and will be removed in the next +major release (>= 2.0). The sed script tools/replace_old_macros.sed can be +used to replace these macros with the newer versions. + +You can test your code against the deprecated C API by adding a line +composed of ``#define NPY_NO_DEPRECATED_API`` and the target version number, +such as ``NPY_1_7_API_VERSION``, before including any NumPy headers. + +The ``NPY_CHAR`` member of the ``NPY_TYPES`` enum is deprecated and will be +removed in NumPy 1.8. See the discussion at +`gh-2801 <https://github.com/numpy/numpy/issues/2801>`_ for more details. diff --git a/doc/source/release/1.7.1-notes.rst b/doc/source/release/1.7.1-notes.rst new file mode 100644 index 000000000..04216b0df --- /dev/null +++ b/doc/source/release/1.7.1-notes.rst @@ -0,0 +1,27 @@ +========================= +NumPy 1.7.1 Release Notes +========================= + +This is a bugfix only release in the 1.7.x series. +It supports Python 2.4 - 2.7 and 3.1 - 3.3 and is the last series that +supports Python 2.4 - 2.5. + + +Issues fixed +============ + +* gh-2973: Fix `1` is printed during numpy.test() +* gh-2983: BUG: gh-2969: Backport memory leak fix 80b3a34. +* gh-3007: Backport gh-3006 +* gh-2984: Backport fix complex polynomial fit +* gh-2982: BUG: Make nansum work with booleans. +* gh-2985: Backport large sort fixes +* gh-3039: Backport object take +* gh-3105: Backport nditer fix op axes initialization +* gh-3108: BUG: npy-pkg-config ini files were missing after Bento build. +* gh-3124: BUG: PyArray_LexSort allocates too much temporary memory. +* gh-3131: BUG: Exported f2py_size symbol prevents linking multiple f2py modules. +* gh-3117: Backport gh-2992 +* gh-3135: DOC: Add mention of PyArray_SetBaseObject stealing a reference +* gh-3134: DOC: Fix typo in fft docs (the indexing variable is 'm', not 'n'). +* gh-3136: Backport #3128 diff --git a/doc/source/release/1.7.2-notes.rst b/doc/source/release/1.7.2-notes.rst new file mode 100644 index 000000000..b0951bd72 --- /dev/null +++ b/doc/source/release/1.7.2-notes.rst @@ -0,0 +1,55 @@ +========================= +NumPy 1.7.2 Release Notes +========================= + +This is a bugfix only release in the 1.7.x series. +It supports Python 2.4 - 2.7 and 3.1 - 3.3 and is the last series that +supports Python 2.4 - 2.5. + + +Issues fixed +============ + +* gh-3153: Do not reuse nditer buffers when not filled enough +* gh-3192: f2py crashes with UnboundLocalError exception +* gh-442: Concatenate with axis=None now requires equal number of array elements +* gh-2485: Fix for astype('S') string truncate issue +* gh-3312: bug in count_nonzero +* gh-2684: numpy.ma.average casts complex to float under certain conditions +* gh-2403: masked array with named components does not behave as expected +* gh-2495: np.ma.compress treated inputs in wrong order +* gh-576: add __len__ method to ma.mvoid +* gh-3364: reduce performance regression of mmap slicing +* gh-3421: fix non-swapping strided copies in GetStridedCopySwap +* gh-3373: fix small leak in datetime metadata initialization +* gh-2791: add platform specific python include directories to search paths +* gh-3168: fix undefined function and add integer divisions +* gh-3301: memmap does not work with TemporaryFile in python3 +* gh-3057: distutils.misc_util.get_shared_lib_extension returns wrong debug extension +* gh-3472: add module extensions to load_library search list +* gh-3324: Make comparison function (gt, ge, ...) respect __array_priority__ +* gh-3497: np.insert behaves incorrectly with argument 'axis=-1' +* gh-3541: make preprocessor tests consistent in halffloat.c +* gh-3458: array_ass_boolean_subscript() writes 'non-existent' data to array +* gh-2892: Regression in ufunc.reduceat with zero-sized index array +* gh-3608: Regression when filling struct from tuple +* gh-3701: add support for Python 3.4 ast.NameConstant +* gh-3712: do not assume that GIL is enabled in xerbla +* gh-3712: fix LAPACK error handling in lapack_litemodule +* gh-3728: f2py fix decref on wrong object +* gh-3743: Hash changed signature in Python 3.3 +* gh-3793: scalar int hashing broken on 64 bit python3 +* gh-3160: SandboxViolation easyinstalling 1.7.0 on Mac OS X 10.8.3 +* gh-3871: npy_math.h has invalid isinf for Solaris with SUNWspro12.2 +* gh-2561: Disable check for oldstyle classes in python3 +* gh-3900: Ensure NotImplemented is passed on in MaskedArray ufunc's +* gh-2052: del scalar subscript causes segfault +* gh-3832: fix a few uninitialized uses and memleaks +* gh-3971: f2py changed string.lowercase to string.ascii_lowercase for python3 +* gh-3480: numpy.random.binomial raised ValueError for n == 0 +* gh-3992: hypot(inf, 0) shouldn't raise a warning, hypot(inf, inf) wrong result +* gh-4018: Segmentation fault dealing with very large arrays +* gh-4094: fix NaT handling in _strided_to_strided_string_to_datetime +* gh-4051: fix uninitialized use in _strided_to_strided_string_to_datetime +* gh-4123: lexsort segfault +* gh-4141: Fix a few issues that show up with python 3.4b1 diff --git a/doc/source/release/1.8.0-notes.rst b/doc/source/release/1.8.0-notes.rst new file mode 100644 index 000000000..80c39f8bc --- /dev/null +++ b/doc/source/release/1.8.0-notes.rst @@ -0,0 +1,526 @@ +========================= +NumPy 1.8.0 Release Notes +========================= + +This release supports Python 2.6 -2.7 and 3.2 - 3.3. + + +Highlights +========== + + +* New, no 2to3, Python 2 and Python 3 are supported by a common code base. +* New, gufuncs for linear algebra, enabling operations on stacked arrays. +* New, inplace fancy indexing for ufuncs with the ``.at`` method. +* New, ``partition`` function, partial sorting via selection for fast median. +* New, ``nanmean``, ``nanvar``, and ``nanstd`` functions skipping NaNs. +* New, ``full`` and ``full_like`` functions to create value initialized arrays. +* New, ``PyUFunc_RegisterLoopForDescr``, better ufunc support for user dtypes. +* Numerous performance improvements in many areas. + + +Dropped Support +=============== + + +Support for Python versions 2.4 and 2.5 has been dropped, + +Support for SCons has been removed. + + +Future Changes +============== + + +The Datetime64 type remains experimental in this release. In 1.9 there will +probably be some changes to make it more useable. + +The diagonal method currently returns a new array and raises a +FutureWarning. In 1.9 it will return a readonly view. + +Multiple field selection from an array of structured type currently +returns a new array and raises a FutureWarning. In 1.9 it will return a +readonly view. + +The numpy/oldnumeric and numpy/numarray compatibility modules will be +removed in 1.9. + + +Compatibility notes +=================== + + +The doc/sphinxext content has been moved into its own github repository, +and is included in numpy as a submodule. See the instructions in +doc/HOWTO_BUILD_DOCS.rst.txt for how to access the content. + +.. _numpydoc: https://github.com/numpy/numpydoc + +The hash function of numpy.void scalars has been changed. Previously the +pointer to the data was hashed as an integer. Now, the hash function uses +the tuple-hash algorithm to combine the hash functions of the elements of +the scalar, but only if the scalar is read-only. + +Numpy has switched its build system to using 'separate compilation' by +default. In previous releases this was supported, but not default. This +should produce the same results as the old system, but if you're trying to +do something complicated like link numpy statically or using an unusual +compiler, then it's possible you will encounter problems. If so, please +file a bug and as a temporary workaround you can re-enable the old build +system by exporting the shell variable NPY_SEPARATE_COMPILATION=0. + +For the AdvancedNew iterator the ``oa_ndim`` flag should now be -1 to indicate +that no ``op_axes`` and ``itershape`` are passed in. The ``oa_ndim == 0`` +case, now indicates a 0-D iteration and ``op_axes`` being NULL and the old +usage is deprecated. This does not effect the ``NpyIter_New`` or +``NpyIter_MultiNew`` functions. + +The functions nanargmin and nanargmax now return np.iinfo['intp'].min for +the index in all-NaN slices. Previously the functions would raise a ValueError +for array returns and NaN for scalar returns. + +NPY_RELAXED_STRIDES_CHECKING +---------------------------- +There is a new compile time environment variable +``NPY_RELAXED_STRIDES_CHECKING``. If this variable is set to 1, then +numpy will consider more arrays to be C- or F-contiguous -- for +example, it becomes possible to have a column vector which is +considered both C- and F-contiguous simultaneously. The new definition +is more accurate, allows for faster code that makes fewer unnecessary +copies, and simplifies numpy's code internally. However, it may also +break third-party libraries that make too-strong assumptions about the +stride values of C- and F-contiguous arrays. (It is also currently +known that this breaks Cython code using memoryviews, which will be +fixed in Cython.) THIS WILL BECOME THE DEFAULT IN A FUTURE RELEASE, SO +PLEASE TEST YOUR CODE NOW AGAINST NUMPY BUILT WITH:: + + NPY_RELAXED_STRIDES_CHECKING=1 python setup.py install + +You can check whether NPY_RELAXED_STRIDES_CHECKING is in effect by +running:: + + np.ones((10, 1), order="C").flags.f_contiguous + +This will be ``True`` if relaxed strides checking is enabled, and +``False`` otherwise. The typical problem we've seen so far is C code +that works with C-contiguous arrays, and assumes that the itemsize can +be accessed by looking at the last element in the ``PyArray_STRIDES(arr)`` +array. When relaxed strides are in effect, this is not true (and in +fact, it never was true in some corner cases). Instead, use +``PyArray_ITEMSIZE(arr)``. + +For more information check the "Internal memory layout of an ndarray" +section in the documentation. + +Binary operations with non-arrays as second argument +---------------------------------------------------- +Binary operations of the form ``<array-or-subclass> * <non-array-subclass>`` +where ``<non-array-subclass>`` declares an ``__array_priority__`` higher than +that of ``<array-or-subclass>`` will now unconditionally return +*NotImplemented*, giving ``<non-array-subclass>`` a chance to handle the +operation. Previously, `NotImplemented` would only be returned if +``<non-array-subclass>`` actually implemented the reversed operation, and after +a (potentially expensive) array conversion of ``<non-array-subclass>`` had been +attempted. (`bug <https://github.com/numpy/numpy/issues/3375>`_, `pull request +<https://github.com/numpy/numpy/pull/3501>`_) + +Function `median` used with `overwrite_input` only partially sorts array +------------------------------------------------------------------------ +If `median` is used with `overwrite_input` option the input array will now only +be partially sorted instead of fully sorted. + +Fix to financial.npv +-------------------- +The npv function had a bug. Contrary to what the documentation stated, it +summed from indexes ``1`` to ``M`` instead of from ``0`` to ``M - 1``. The +fix changes the returned value. The mirr function called the npv function, +but worked around the problem, so that was also fixed and the return value +of the mirr function remains unchanged. + +Runtime warnings when comparing NaN numbers +------------------------------------------- +Comparing ``NaN`` floating point numbers now raises the ``invalid`` runtime +warning. If a ``NaN`` is expected the warning can be ignored using np.errstate. +E.g.:: + + with np.errstate(invalid='ignore'): + operation() + + +New Features +============ + + +Support for linear algebra on stacked arrays +-------------------------------------------- +The gufunc machinery is now used for np.linalg, allowing operations on +stacked arrays and vectors. For example:: + + >>> a + array([[[ 1., 1.], + [ 0., 1.]], + + [[ 1., 1.], + [ 0., 1.]]]) + + >>> np.linalg.inv(a) + array([[[ 1., -1.], + [ 0., 1.]], + + [[ 1., -1.], + [ 0., 1.]]]) + +In place fancy indexing for ufuncs +---------------------------------- +The function ``at`` has been added to ufunc objects to allow in place +ufuncs with no buffering when fancy indexing is used. For example, the +following will increment the first and second items in the array, and will +increment the third item twice: ``numpy.add.at(arr, [0, 1, 2, 2], 1)`` + +This is what many have mistakenly thought ``arr[[0, 1, 2, 2]] += 1`` would do, +but that does not work as the incremented value of ``arr[2]`` is simply copied +into the third slot in ``arr`` twice, not incremented twice. + +New functions `partition` and `argpartition` +-------------------------------------------- +New functions to partially sort arrays via a selection algorithm. + +A ``partition`` by index ``k`` moves the ``k`` smallest element to the front of +an array. All elements before ``k`` are then smaller or equal than the value +in position ``k`` and all elements following ``k`` are then greater or equal +than the value in position ``k``. The ordering of the values within these +bounds is undefined. +A sequence of indices can be provided to sort all of them into their sorted +position at once iterative partitioning. +This can be used to efficiently obtain order statistics like median or +percentiles of samples. +``partition`` has a linear time complexity of ``O(n)`` while a full sort has +``O(n log(n))``. + +New functions `nanmean`, `nanvar` and `nanstd` +---------------------------------------------- +New nan aware statistical functions are added. In these functions the +results are what would be obtained if nan values were omitted from all +computations. + +New functions `full` and `full_like` +------------------------------------ +New convenience functions to create arrays filled with a specific value; +complementary to the existing `zeros` and `zeros_like` functions. + +IO compatibility with large files +--------------------------------- +Large NPZ files >2GB can be loaded on 64-bit systems. + +Building against OpenBLAS +------------------------- +It is now possible to build numpy against OpenBLAS by editing site.cfg. + +New constant +------------ +Euler's constant is now exposed in numpy as euler_gamma. + +New modes for qr +---------------- +New modes 'complete', 'reduced', and 'raw' have been added to the qr +factorization and the old 'full' and 'economic' modes are deprecated. +The 'reduced' mode replaces the old 'full' mode and is the default as was +the 'full' mode, so backward compatibility can be maintained by not +specifying the mode. + +The 'complete' mode returns a full dimensional factorization, which can be +useful for obtaining a basis for the orthogonal complement of the range +space. The 'raw' mode returns arrays that contain the Householder +reflectors and scaling factors that can be used in the future to apply q +without needing to convert to a matrix. The 'economic' mode is simply +deprecated, there isn't much use for it and it isn't any more efficient +than the 'raw' mode. + +New `invert` argument to `in1d` +------------------------------- +The function `in1d` now accepts a `invert` argument which, when `True`, +causes the returned array to be inverted. + +Advanced indexing using `np.newaxis` +------------------------------------ +It is now possible to use `np.newaxis`/`None` together with index +arrays instead of only in simple indices. This means that +``array[np.newaxis, [0, 1]]`` will now work as expected and select the first +two rows while prepending a new axis to the array. + + +C-API +----- +New ufuncs can now be registered with builtin input types and a custom +output type. Before this change, NumPy wouldn't be able to find the right +ufunc loop function when the ufunc was called from Python, because the ufunc +loop signature matching logic wasn't looking at the output operand type. +Now the correct ufunc loop is found, as long as the user provides an output +argument with the correct output type. + +runtests.py +----------- +A simple test runner script ``runtests.py`` was added. It also builds Numpy via +``setup.py build`` and can be used to run tests easily during development. + + +Improvements +============ + +IO performance improvements +--------------------------- +Performance in reading large files was improved by chunking (see also IO compatibility). + +Performance improvements to `pad` +--------------------------------- +The `pad` function has a new implementation, greatly improving performance for +all inputs except `mode=<function>` (retained for backwards compatibility). +Scaling with dimensionality is dramatically improved for rank >= 4. + +Performance improvements to `isnan`, `isinf`, `isfinite` and `byteswap` +----------------------------------------------------------------------- +`isnan`, `isinf`, `isfinite` and `byteswap` have been improved to take +advantage of compiler builtins to avoid expensive calls to libc. +This improves performance of these operations by about a factor of two on gnu +libc systems. + +Performance improvements via SSE2 vectorization +----------------------------------------------- +Several functions have been optimized to make use of SSE2 CPU SIMD instructions. + +* Float32 and float64: + * base math (`add`, `subtract`, `divide`, `multiply`) + * `sqrt` + * `minimum/maximum` + * `absolute` +* Bool: + * `logical_or` + * `logical_and` + * `logical_not` + +This improves performance of these operations up to 4x/2x for float32/float64 +and up to 10x for bool depending on the location of the data in the CPU caches. +The performance gain is greatest for in-place operations. + +In order to use the improved functions the SSE2 instruction set must be enabled +at compile time. It is enabled by default on x86_64 systems. On x86_32 with a +capable CPU it must be enabled by passing the appropriate flag to the CFLAGS +build variable (-msse2 with gcc). + +Performance improvements to `median` +------------------------------------ +`median` is now implemented in terms of `partition` instead of `sort` which +reduces its time complexity from O(n log(n)) to O(n). +If used with the `overwrite_input` option the array will now only be partially +sorted instead of fully sorted. + + +Overrideable operand flags in ufunc C-API +----------------------------------------- +When creating a ufunc, the default ufunc operand flags can be overridden +via the new op_flags attribute of the ufunc object. For example, to set +the operand flag for the first input to read/write: + +PyObject \*ufunc = PyUFunc_FromFuncAndData(...); +ufunc->op_flags[0] = NPY_ITER_READWRITE; + +This allows a ufunc to perform an operation in place. Also, global nditer flags +can be overridden via the new iter_flags attribute of the ufunc object. +For example, to set the reduce flag for a ufunc: + +ufunc->iter_flags = NPY_ITER_REDUCE_OK; + + +Changes +======= + + +General +------- +The function np.take now allows 0-d arrays as indices. + +The separate compilation mode is now enabled by default. + +Several changes to np.insert and np.delete: + +* Previously, negative indices and indices that pointed past the end of + the array were simply ignored. Now, this will raise a Future or Deprecation + Warning. In the future they will be treated like normal indexing treats + them -- negative indices will wrap around, and out-of-bound indices will + generate an error. +* Previously, boolean indices were treated as if they were integers (always + referring to either the 0th or 1st item in the array). In the future, they + will be treated as masks. In this release, they raise a FutureWarning + warning of this coming change. +* In Numpy 1.7. np.insert already allowed the syntax + `np.insert(arr, 3, [1,2,3])` to insert multiple items at a single position. + In Numpy 1.8. this is also possible for `np.insert(arr, [3], [1, 2, 3])`. + +Padded regions from np.pad are now correctly rounded, not truncated. + +C-API Array Additions +--------------------- +Four new functions have been added to the array C-API. + +* PyArray_Partition +* PyArray_ArgPartition +* PyArray_SelectkindConverter +* PyDataMem_NEW_ZEROED + +C-API Ufunc Additions +--------------------- +One new function has been added to the ufunc C-API that allows to register +an inner loop for user types using the descr. + +* PyUFunc_RegisterLoopForDescr + +C-API Developer Improvements +---------------------------- +The ``PyArray_Type`` instance creation function ``tp_new`` now +uses ``tp_basicsize`` to determine how much memory to allocate. +In previous releases only ``sizeof(PyArrayObject)`` bytes of +memory were allocated, often requiring C-API subtypes to +reimplement ``tp_new``. + +Deprecations +============ + +The 'full' and 'economic' modes of qr factorization are deprecated. + +General +------- +The use of non-integer for indices and most integer arguments has been +deprecated. Previously float indices and function arguments such as axes or +shapes were truncated to integers without warning. For example +`arr.reshape(3., -1)` or `arr[0.]` will trigger a deprecation warning in +NumPy 1.8., and in some future version of NumPy they will raise an error. + + +Authors +======= + +This release contains work by the following people who contributed at least +one patch to this release. The names are in alphabetical order by first name: + +* 87 +* Adam Ginsburg + +* Adam Griffiths + +* Alexander Belopolsky + +* Alex Barth + +* Alex Ford + +* Andreas Hilboll + +* Andreas Kloeckner + +* Andreas Schwab + +* Andrew Horton + +* argriffing + +* Arink Verma + +* Bago Amirbekian + +* Bartosz Telenczuk + +* bebert218 + +* Benjamin Root + +* Bill Spotz + +* Bradley M. Froehle +* Carwyn Pelley + +* Charles Harris +* Chris +* Christian Brueffer + +* Christoph Dann + +* Christoph Gohlke +* Dan Hipschman + +* Daniel + +* Dan Miller + +* daveydave400 + +* David Cournapeau +* David Warde-Farley +* Denis Laxalde +* dmuellner + +* Edward Catmur + +* Egor Zindy + +* endolith +* Eric Firing +* Eric Fode +* Eric Moore + +* Eric Price + +* Fazlul Shahriar + +* Félix Hartmann + +* Fernando Perez +* Frank B + +* Frank Breitling + +* Frederic +* Gabriel +* GaelVaroquaux +* Guillaume Gay + +* Han Genuit +* HaroldMills + +* hklemm + +* jamestwebber + +* Jason Madden + +* Jay Bourque +* jeromekelleher + +* Jesús Gómez + +* jmozmoz + +* jnothman + +* Johannes Schönberger + +* John Benediktsson + +* John Salvatier + +* John Stechschulte + +* Jonathan Waltman + +* Joon Ro + +* Jos de Kloe + +* Joseph Martinot-Lagarde + +* Josh Warner (Mac) + +* Jostein Bø Fløystad + +* Juan Luis Cano Rodríguez + +* Julian Taylor + +* Julien Phalip + +* K.-Michael Aye + +* Kumar Appaiah + +* Lars Buitinck +* Leon Weber + +* Luis Pedro Coelho +* Marcin Juszkiewicz +* Mark Wiebe +* Marten van Kerkwijk + +* Martin Baeuml + +* Martin Spacek +* Martin Teichmann + +* Matt Davis + +* Matthew Brett +* Maximilian Albert + +* m-d-w + +* Michael Droettboom +* mwtoews + +* Nathaniel J. Smith +* Nicolas Scheffer + +* Nils Werner + +* ochoadavid + +* Ondřej Čertík +* ovillellas + +* Paul Ivanov +* Pauli Virtanen +* peterjc +* Ralf Gommers +* Raul Cota + +* Richard Hattersley + +* Robert Costa + +* Robert Kern +* Rob Ruana + +* Ronan Lamy +* Sandro Tosi +* Sascha Peilicke + +* Sebastian Berg +* Skipper Seabold +* Stefan van der Walt +* Steve + +* Takafumi Arakaki + +* Thomas Robitaille + +* Tomas Tomecek + +* Travis E. Oliphant +* Valentin Haenel +* Vladimir Rutsky + +* Warren Weckesser +* Yaroslav Halchenko +* Yury V. Zaytsev + + +A total of 119 people contributed to this release. +People with a "+" by their names contributed a patch for the first time. diff --git a/doc/source/release/1.8.1-notes.rst b/doc/source/release/1.8.1-notes.rst new file mode 100644 index 000000000..ea34e75ac --- /dev/null +++ b/doc/source/release/1.8.1-notes.rst @@ -0,0 +1,96 @@ +========================= +NumPy 1.8.1 Release Notes +========================= + +This is a bugfix only release in the 1.8.x series. + + +Issues fixed +============ + +* gh-4276: Fix mean, var, std methods for object arrays +* gh-4262: remove insecure mktemp usage +* gh-2385: absolute(complex(inf)) raises invalid warning in python3 +* gh-4024: Sequence assignment doesn't raise exception on shape mismatch +* gh-4027: Fix chunked reading of strings longer than BUFFERSIZE +* gh-4109: Fix object scalar return type of 0-d array indices +* gh-4018: fix missing check for memory allocation failure in ufuncs +* gh-4156: high order linalg.norm discards imaginary elements of complex arrays +* gh-4144: linalg: norm fails on longdouble, signed int +* gh-4094: fix NaT handling in _strided_to_strided_string_to_datetime +* gh-4051: fix uninitialized use in _strided_to_strided_string_to_datetime +* gh-4093: Loading compressed .npz file fails under Python 2.6.6 +* gh-4138: segfault with non-native endian memoryview in python 3.4 +* gh-4123: Fix missing NULL check in lexsort +* gh-4170: fix native-only long long check in memoryviews +* gh-4187: Fix large file support on 32 bit +* gh-4152: fromfile: ensure file handle positions are in sync in python3 +* gh-4176: clang compatibility: Typos in conversion_utils +* gh-4223: Fetching a non-integer item caused array return +* gh-4197: fix minor memory leak in memoryview failure case +* gh-4206: fix build with single-threaded python +* gh-4220: add versionadded:: 1.8.0 to ufunc.at docstring +* gh-4267: improve handling of memory allocation failure +* gh-4267: fix use of capi without gil in ufunc.at +* gh-4261: Detect vendor versions of GNU Compilers +* gh-4253: IRR was returning nan instead of valid negative answer +* gh-4254: fix unnecessary byte order flag change for byte arrays +* gh-3263: numpy.random.shuffle clobbers mask of a MaskedArray +* gh-4270: np.random.shuffle not work with flexible dtypes +* gh-3173: Segmentation fault when 'size' argument to random.multinomial +* gh-2799: allow using unique with lists of complex +* gh-3504: fix linspace truncation for integer array scalar +* gh-4191: get_info('openblas') does not read libraries key +* gh-3348: Access violation in _descriptor_from_pep3118_format +* gh-3175: segmentation fault with numpy.array() from bytearray +* gh-4266: histogramdd - wrong result for entries very close to last boundary +* gh-4408: Fix stride_stricks.as_strided function for object arrays +* gh-4225: fix log1p and exmp1 return for np.inf on windows compiler builds +* gh-4359: Fix infinite recursion in str.format of flex arrays +* gh-4145: Incorrect shape of broadcast result with the exponent operator +* gh-4483: Fix commutativity of {dot,multiply,inner}(scalar, matrix_of_objs) +* gh-4466: Delay npyiter size check when size may change +* gh-4485: Buffered stride was erroneously marked fixed +* gh-4354: byte_bounds fails with datetime dtypes +* gh-4486: segfault/error converting from/to high-precision datetime64 objects +* gh-4428: einsum(None, None, None, None) causes segfault +* gh-4134: uninitialized use for for size 1 object reductions + +Changes +======= + +NDIter +------ +When ``NpyIter_RemoveAxis`` is now called, the iterator range will be reset. + +When a multi index is being tracked and an iterator is not buffered, it is +possible to use ``NpyIter_RemoveAxis``. In this case an iterator can shrink +in size. Because the total size of an iterator is limited, the iterator +may be too large before these calls. In this case its size will be set to ``-1`` +and an error issued not at construction time but when removing the multi +index, setting the iterator range, or getting the next function. + +This has no effect on currently working code, but highlights the necessity +of checking for an error return if these conditions can occur. In most +cases the arrays being iterated are as large as the iterator so that such +a problem cannot occur. + +Optional reduced verbosity for np.distutils +------------------------------------------- +Set ``numpy.distutils.system_info.system_info.verbosity = 0`` and then +calls to ``numpy.distutils.system_info.get_info('blas_opt')`` will not +print anything on the output. This is mostly for other packages using +numpy.distutils. + +Deprecations +============ + +C-API +----- + +The utility function npy_PyFile_Dup and npy_PyFile_DupClose are broken by the +internal buffering python 3 applies to its file objects. +To fix this two new functions npy_PyFile_Dup2 and npy_PyFile_DupClose2 are +declared in npy_3kcompat.h and the old functions are deprecated. +Due to the fragile nature of these functions it is recommended to instead use +the python API when possible. diff --git a/doc/source/release/1.8.2-notes.rst b/doc/source/release/1.8.2-notes.rst new file mode 100644 index 000000000..71e549526 --- /dev/null +++ b/doc/source/release/1.8.2-notes.rst @@ -0,0 +1,20 @@ +========================= +NumPy 1.8.2 Release Notes +========================= + +This is a bugfix only release in the 1.8.x series. + +Issues fixed +============ + +* gh-4836: partition produces wrong results for multiple selections in equal ranges +* gh-4656: Make fftpack._raw_fft threadsafe +* gh-4628: incorrect argument order to _copyto in in np.nanmax, np.nanmin +* gh-4642: Hold GIL for converting dtypes types with fields +* gh-4733: fix np.linalg.svd(b, compute_uv=False) +* gh-4853: avoid unaligned simd load on reductions on i386 +* gh-4722: Fix seg fault converting empty string to object +* gh-4613: Fix lack of NULL check in array_richcompare +* gh-4774: avoid unaligned access for strided byteswap +* gh-650: Prevent division by zero when creating arrays from some buffers +* gh-4602: ifort has issues with optimization flag O2, use O1 diff --git a/doc/source/release/1.9.0-notes.rst b/doc/source/release/1.9.0-notes.rst new file mode 100644 index 000000000..7ea29e354 --- /dev/null +++ b/doc/source/release/1.9.0-notes.rst @@ -0,0 +1,448 @@ +========================= +NumPy 1.9.0 Release Notes +========================= + +This release supports Python 2.6 - 2.7 and 3.2 - 3.4. + + +Highlights +========== +* Numerous performance improvements in various areas, most notably indexing and + operations on small arrays are significantly faster. + Indexing operations now also release the GIL. +* Addition of `nanmedian` and `nanpercentile` rounds out the nanfunction set. + + +Dropped Support +=============== + +* The oldnumeric and numarray modules have been removed. +* The doc/pyrex and doc/cython directories have been removed. +* The doc/numpybook directory has been removed. +* The numpy/testing/numpytest.py file has been removed together with + the importall function it contained. + + +Future Changes +============== + +* The numpy/polynomial/polytemplate.py file will be removed in NumPy 1.10.0. +* Default casting for inplace operations will change to 'same_kind' in + Numpy 1.10.0. This will certainly break some code that is currently + ignoring the warning. +* Relaxed stride checking will be the default in 1.10.0 +* String version checks will break because, e.g., '1.9' > '1.10' is True. A + NumpyVersion class has been added that can be used for such comparisons. +* The diagonal and diag functions will return writeable views in 1.10.0 +* The `S` and/or `a` dtypes may be changed to represent Python strings + instead of bytes, in Python 3 these two types are very different. + + +Compatibility notes +=================== + +The diagonal and diag functions return readonly views. +------------------------------------------------------ +In NumPy 1.8, the diagonal and diag functions returned readonly copies, in +NumPy 1.9 they return readonly views, and in 1.10 they will return writeable +views. + +Special scalar float values don't cause upcast to double anymore +---------------------------------------------------------------- +In previous numpy versions operations involving floating point scalars +containing special values ``NaN``, ``Inf`` and ``-Inf`` caused the result +type to be at least ``float64``. As the special values can be represented +in the smallest available floating point type, the upcast is not performed +anymore. + +For example the dtype of: + + ``np.array([1.], dtype=np.float32) * float('nan')`` + +now remains ``float32`` instead of being cast to ``float64``. +Operations involving non-special values have not been changed. + +Percentile output changes +------------------------- +If given more than one percentile to compute numpy.percentile returns an +array instead of a list. A single percentile still returns a scalar. The +array is equivalent to converting the list returned in older versions +to an array via ``np.array``. + +If the ``overwrite_input`` option is used the input is only partially +instead of fully sorted. + +ndarray.tofile exception type +----------------------------- +All ``tofile`` exceptions are now ``IOError``, some were previously +``ValueError``. + +Invalid fill value exceptions +----------------------------- +Two changes to numpy.ma.core._check_fill_value: + +* When the fill value is a string and the array type is not one of + 'OSUV', TypeError is raised instead of the default fill value being used. + +* When the fill value overflows the array type, TypeError is raised instead + of OverflowError. + +Polynomial Classes no longer derived from PolyBase +-------------------------------------------------- +This may cause problems with folks who depended on the polynomial classes +being derived from PolyBase. They are now all derived from the abstract +base class ABCPolyBase. Strictly speaking, there should be a deprecation +involved, but no external code making use of the old baseclass could be +found. + +Using numpy.random.binomial may change the RNG state vs. numpy < 1.9 +-------------------------------------------------------------------- +A bug in one of the algorithms to generate a binomial random variate has +been fixed. This change will likely alter the number of random draws +performed, and hence the sequence location will be different after a +call to distribution.c::rk_binomial_btpe. Any tests which rely on the RNG +being in a known state should be checked and/or updated as a result. + +Random seed enforced to be a 32 bit unsigned integer +---------------------------------------------------- +``np.random.seed`` and ``np.random.RandomState`` now throw a ``ValueError`` +if the seed cannot safely be converted to 32 bit unsigned integers. +Applications that now fail can be fixed by masking the higher 32 bit values to +zero: ``seed = seed & 0xFFFFFFFF``. This is what is done silently in older +versions so the random stream remains the same. + +Argmin and argmax out argument +------------------------------ +The ``out`` argument to ``np.argmin`` and ``np.argmax`` and their +equivalent C-API functions is now checked to match the desired output shape +exactly. If the check fails a ``ValueError`` instead of ``TypeError`` is +raised. + +Einsum +------ +Remove unnecessary broadcasting notation restrictions. +``np.einsum('ijk,j->ijk', A, B)`` can also be written as +``np.einsum('ij...,j->ij...', A, B)`` (ellipsis is no longer required on 'j') + +Indexing +-------- + +The NumPy indexing has seen a complete rewrite in this version. This makes +most advanced integer indexing operations much faster and should have no +other implications. However some subtle changes and deprecations were +introduced in advanced indexing operations: + +* Boolean indexing into scalar arrays will always return a new 1-d array. + This means that ``array(1)[array(True)]`` gives ``array([1])`` and + not the original array. + +* Advanced indexing into one dimensional arrays used to have + (undocumented) special handling regarding repeating the value array in + assignments when the shape of the value array was too small or did not + match. Code using this will raise an error. For compatibility you can + use ``arr.flat[index] = values``, which uses the old code branch. (for + example ``a = np.ones(10); a[np.arange(10)] = [1, 2, 3]``) + +* The iteration order over advanced indexes used to be always C-order. + In NumPy 1.9. the iteration order adapts to the inputs and is not + guaranteed (with the exception of a *single* advanced index which is + never reversed for compatibility reasons). This means that the result + is undefined if multiple values are assigned to the same element. An + example for this is ``arr[[0, 0], [1, 1]] = [1, 2]``, which may set + ``arr[0, 1]`` to either 1 or 2. + +* Equivalent to the iteration order, the memory layout of the advanced + indexing result is adapted for faster indexing and cannot be predicted. + +* All indexing operations return a view or a copy. No indexing operation + will return the original array object. (For example ``arr[...]``) + +* In the future Boolean array-likes (such as lists of python bools) will + always be treated as Boolean indexes and Boolean scalars (including + python ``True``) will be a legal *boolean* index. At this time, this is + already the case for scalar arrays to allow the general + ``positive = a[a > 0]`` to work when ``a`` is zero dimensional. + +* In NumPy 1.8 it was possible to use ``array(True)`` and + ``array(False)`` equivalent to 1 and 0 if the result of the operation + was a scalar. This will raise an error in NumPy 1.9 and, as noted + above, treated as a boolean index in the future. + +* All non-integer array-likes are deprecated, object arrays of custom + integer like objects may have to be cast explicitly. + +* The error reporting for advanced indexing is more informative, however + the error type has changed in some cases. (Broadcasting errors of + indexing arrays are reported as ``IndexError``) + +* Indexing with more then one ellipsis (``...``) is deprecated. + +Non-integer reduction axis indexes are deprecated +------------------------------------------------- +Non-integer axis indexes to reduction ufuncs like `add.reduce` or `sum` are +deprecated. + +``promote_types`` and string dtype +---------------------------------- +``promote_types`` function now returns a valid string length when given an +integer or float dtype as one argument and a string dtype as another +argument. Previously it always returned the input string dtype, even if it +wasn't long enough to store the max integer/float value converted to a +string. + +``can_cast`` and string dtype +----------------------------- +``can_cast`` function now returns False in "safe" casting mode for +integer/float dtype and string dtype if the string dtype length is not long +enough to store the max integer/float value converted to a string. +Previously ``can_cast`` in "safe" mode returned True for integer/float +dtype and a string dtype of any length. + +astype and string dtype +----------------------- +The ``astype`` method now returns an error if the string dtype to cast to +is not long enough in "safe" casting mode to hold the max value of +integer/float array that is being casted. Previously the casting was +allowed even if the result was truncated. + +`npyio.recfromcsv` keyword arguments change +------------------------------------------- +`npyio.recfromcsv` no longer accepts the undocumented `update` keyword, +which used to override the `dtype` keyword. + +The ``doc/swig`` directory moved +-------------------------------- +The ``doc/swig`` directory has been moved to ``tools/swig``. + +The ``npy_3kcompat.h`` header changed +------------------------------------- +The unused ``simple_capsule_dtor`` function has been removed from +``npy_3kcompat.h``. Note that this header is not meant to be used outside +of numpy; other projects should be using their own copy of this file when +needed. + +Negative indices in C-Api ``sq_item`` and ``sq_ass_item`` sequence methods +-------------------------------------------------------------------------- +When directly accessing the ``sq_item`` or ``sq_ass_item`` PyObject slots +for item getting, negative indices will not be supported anymore. +``PySequence_GetItem`` and ``PySequence_SetItem`` however fix negative +indices so that they can be used there. + +NDIter +------ +When ``NpyIter_RemoveAxis`` is now called, the iterator range will be reset. + +When a multi index is being tracked and an iterator is not buffered, it is +possible to use ``NpyIter_RemoveAxis``. In this case an iterator can shrink +in size. Because the total size of an iterator is limited, the iterator +may be too large before these calls. In this case its size will be set to ``-1`` +and an error issued not at construction time but when removing the multi +index, setting the iterator range, or getting the next function. + +This has no effect on currently working code, but highlights the necessity +of checking for an error return if these conditions can occur. In most +cases the arrays being iterated are as large as the iterator so that such +a problem cannot occur. + +This change was already applied to the 1.8.1 release. + +``zeros_like`` for string dtypes now returns empty strings +---------------------------------------------------------- +To match the `zeros` function `zeros_like` now returns an array initialized +with empty strings instead of an array filled with `'0'`. + + +New Features +============ + +Percentile supports more interpolation options +---------------------------------------------- +``np.percentile`` now has the interpolation keyword argument to specify in +which way points should be interpolated if the percentiles fall between two +values. See the documentation for the available options. + +Generalized axis support for median and percentile +-------------------------------------------------- +``np.median`` and ``np.percentile`` now support generalized axis arguments like +ufunc reductions do since 1.7. One can now say axis=(index, index) to pick a +list of axes for the reduction. The ``keepdims`` keyword argument was also +added to allow convenient broadcasting to arrays of the original shape. + +Dtype parameter added to ``np.linspace`` and ``np.logspace`` +------------------------------------------------------------ +The returned data type from the ``linspace`` and ``logspace`` functions can +now be specified using the dtype parameter. + +More general ``np.triu`` and ``np.tril`` broadcasting +----------------------------------------------------- +For arrays with ``ndim`` exceeding 2, these functions will now apply to the +final two axes instead of raising an exception. + +``tobytes`` alias for ``tostring`` method +----------------------------------------- +``ndarray.tobytes`` and ``MaskedArray.tobytes`` have been added as aliases +for ``tostring`` which exports arrays as ``bytes``. This is more consistent +in Python 3 where ``str`` and ``bytes`` are not the same. + +Build system +------------ +Added experimental support for the ppc64le and OpenRISC architecture. + +Compatibility to python ``numbers`` module +------------------------------------------ +All numerical numpy types are now registered with the type hierarchy in +the python ``numbers`` module. + +``increasing`` parameter added to ``np.vander`` +----------------------------------------------- +The ordering of the columns of the Vandermonde matrix can be specified with +this new boolean argument. + +``unique_counts`` parameter added to ``np.unique`` +-------------------------------------------------- +The number of times each unique item comes up in the input can now be +obtained as an optional return value. + +Support for median and percentile in nanfunctions +------------------------------------------------- +The ``np.nanmedian`` and ``np.nanpercentile`` functions behave like +the median and percentile functions except that NaNs are ignored. + +NumpyVersion class added +------------------------ +The class may be imported from numpy.lib and can be used for version +comparison when the numpy version goes to 1.10.devel. For example:: + + >>> from numpy.lib import NumpyVersion + >>> if NumpyVersion(np.__version__) < '1.10.0'): + ... print('Wow, that is an old NumPy version!') + +Allow saving arrays with large number of named columns +------------------------------------------------------ +The numpy storage format 1.0 only allowed the array header to have a total size +of 65535 bytes. This can be exceeded by structured arrays with a large number +of columns. A new format 2.0 has been added which extends the header size to 4 +GiB. `np.save` will automatically save in 2.0 format if the data requires it, +else it will always use the more compatible 1.0 format. + +Full broadcasting support for ``np.cross`` +------------------------------------------ +``np.cross`` now properly broadcasts its two input arrays, even if they +have different number of dimensions. In earlier versions this would result +in either an error being raised, or wrong results computed. + + +Improvements +============ + +Better numerical stability for sum in some cases +------------------------------------------------ +Pairwise summation is now used in the sum method, but only along the fast +axis and for groups of the values <= 8192 in length. This should also +improve the accuracy of var and std in some common cases. + +Percentile implemented in terms of ``np.partition`` +--------------------------------------------------- +``np.percentile`` has been implemented in terms of ``np.partition`` which +only partially sorts the data via a selection algorithm. This improves the +time complexity from ``O(nlog(n))`` to ``O(n)``. + +Performance improvement for ``np.array`` +---------------------------------------- +The performance of converting lists containing arrays to arrays using +``np.array`` has been improved. It is now equivalent in speed to +``np.vstack(list)``. + +Performance improvement for ``np.searchsorted`` +----------------------------------------------- +For the built-in numeric types, ``np.searchsorted`` no longer relies on the +data type's ``compare`` function to perform the search, but is now +implemented by type specific functions. Depending on the size of the +inputs, this can result in performance improvements over 2x. + +Optional reduced verbosity for np.distutils +------------------------------------------- +Set ``numpy.distutils.system_info.system_info.verbosity = 0`` and then +calls to ``numpy.distutils.system_info.get_info('blas_opt')`` will not +print anything on the output. This is mostly for other packages using +numpy.distutils. + +Covariance check in ``np.random.multivariate_normal`` +----------------------------------------------------- +A ``RuntimeWarning`` warning is raised when the covariance matrix is not +positive-semidefinite. + +Polynomial Classes no longer template based +------------------------------------------- +The polynomial classes have been refactored to use an abstract base class +rather than a template in order to implement a common interface. This makes +importing the polynomial package faster as the classes do not need to be +compiled on import. + +More GIL releases +----------------- +Several more functions now release the Global Interpreter Lock allowing more +efficient parallelization using the ``threading`` module. Most notably the GIL is +now released for fancy indexing, ``np.where`` and the ``random`` module now +uses a per-state lock instead of the GIL. + +MaskedArray support for more complicated base classes +----------------------------------------------------- +Built-in assumptions that the baseclass behaved like a plain array are being +removed. In particalur, ``repr`` and ``str`` should now work more reliably. + + +C-API +----- + + +Deprecations +============ + +Non-integer scalars for sequence repetition +------------------------------------------- +Using non-integer numpy scalars to repeat python sequences is deprecated. +For example ``np.float_(2) * [1]`` will be an error in the future. + +``select`` input deprecations +----------------------------- +The integer and empty input to ``select`` is deprecated. In the future only +boolean arrays will be valid conditions and an empty ``condlist`` will be +considered an input error instead of returning the default. + +``rank`` function +----------------- +The ``rank`` function has been deprecated to avoid confusion with +``numpy.linalg.matrix_rank``. + +Object array equality comparisons +--------------------------------- +In the future object array comparisons both `==` and `np.equal` will not +make use of identity checks anymore. For example: + +>>> a = np.array([np.array([1, 2, 3]), 1]) +>>> b = np.array([np.array([1, 2, 3]), 1]) +>>> a == b + +will consistently return False (and in the future an error) even if the array +in `a` and `b` was the same object. + +The equality operator `==` will in the future raise errors like `np.equal` +if broadcasting or element comparisons, etc. fails. + +Comparison with `arr == None` will in the future do an elementwise comparison +instead of just returning False. Code should be using `arr is None`. + +All of these changes will give Deprecation- or FutureWarnings at this time. + +C-API +----- + +The utility function npy_PyFile_Dup and npy_PyFile_DupClose are broken by the +internal buffering python 3 applies to its file objects. +To fix this two new functions npy_PyFile_Dup2 and npy_PyFile_DupClose2 are +declared in npy_3kcompat.h and the old functions are deprecated. +Due to the fragile nature of these functions it is recommended to instead use +the python API when possible. + +This change was already applied to the 1.8.1 release. diff --git a/doc/source/release/1.9.1-notes.rst b/doc/source/release/1.9.1-notes.rst new file mode 100644 index 000000000..4558237f4 --- /dev/null +++ b/doc/source/release/1.9.1-notes.rst @@ -0,0 +1,34 @@ +========================= +NumPy 1.9.1 Release Notes +========================= + +This is a bugfix only release in the 1.9.x series. + +Issues fixed +============ + +* gh-5184: restore linear edge behaviour of gradient to as it was in < 1.9. + The second order behaviour is available via the `edge_order` keyword +* gh-4007: workaround Accelerate sgemv crash on OSX 10.9 +* gh-5100: restore object dtype inference from iterable objects without `len()` +* gh-5163: avoid gcc-4.1.2 (red hat 5) miscompilation causing a crash +* gh-5138: fix nanmedian on arrays containing inf +* gh-5240: fix not returning out array from ufuncs with subok=False set +* gh-5203: copy inherited masks in MaskedArray.__array_finalize__ +* gh-2317: genfromtxt did not handle filling_values=0 correctly +* gh-5067: restore api of npy_PyFile_DupClose in python2 +* gh-5063: cannot convert invalid sequence index to tuple +* gh-5082: Segmentation fault with argmin() on unicode arrays +* gh-5095: don't propagate subtypes from np.where +* gh-5104: np.inner segfaults with SciPy's sparse matrices +* gh-5251: Issue with fromarrays not using correct format for unicode arrays +* gh-5136: Import dummy_threading if importing threading fails +* gh-5148: Make numpy import when run with Python flag '-OO' +* gh-5147: Einsum double contraction in particular order causes ValueError +* gh-479: Make f2py work with intent(in out) +* gh-5170: Make python2 .npy files readable in python3 +* gh-5027: Use 'll' as the default length specifier for long long +* gh-4896: fix build error with MSVC 2013 caused by C99 complex support +* gh-4465: Make PyArray_PutTo respect writeable flag +* gh-5225: fix crash when using arange on datetime without dtype set +* gh-5231: fix build in c99 mode diff --git a/doc/source/release/1.9.2-notes.rst b/doc/source/release/1.9.2-notes.rst new file mode 100644 index 000000000..268f3aa64 --- /dev/null +++ b/doc/source/release/1.9.2-notes.rst @@ -0,0 +1,26 @@ +========================= +NumPy 1.9.2 Release Notes +========================= + +This is a bugfix only release in the 1.9.x series. + +Issues fixed +============ + +* `#5316 <https://github.com/numpy/numpy/issues/5316>`__: fix too large dtype alignment of strings and complex types +* `#5424 <https://github.com/numpy/numpy/issues/5424>`__: fix ma.median when used on ndarrays +* `#5481 <https://github.com/numpy/numpy/issues/5481>`__: Fix astype for structured array fields of different byte order +* `#5354 <https://github.com/numpy/numpy/issues/5354>`__: fix segfault when clipping complex arrays +* `#5524 <https://github.com/numpy/numpy/issues/5524>`__: allow np.argpartition on non ndarrays +* `#5612 <https://github.com/numpy/numpy/issues/5612>`__: Fixes ndarray.fill to accept full range of uint64 +* `#5155 <https://github.com/numpy/numpy/issues/5155>`__: Fix loadtxt with comments=None and a string None data +* `#4476 <https://github.com/numpy/numpy/issues/4476>`__: Masked array view fails if structured dtype has datetime component +* `#5388 <https://github.com/numpy/numpy/issues/5388>`__: Make RandomState.set_state and RandomState.get_state threadsafe +* `#5390 <https://github.com/numpy/numpy/issues/5390>`__: make seed, randint and shuffle threadsafe +* `#5374 <https://github.com/numpy/numpy/issues/5374>`__: Fixed incorrect assert_array_almost_equal_nulp documentation +* `#5393 <https://github.com/numpy/numpy/issues/5393>`__: Add support for ATLAS > 3.9.33. +* `#5313 <https://github.com/numpy/numpy/issues/5313>`__: PyArray_AsCArray caused segfault for 3d arrays +* `#5492 <https://github.com/numpy/numpy/issues/5492>`__: handle out of memory in rfftf +* `#4181 <https://github.com/numpy/numpy/issues/4181>`__: fix a few bugs in the random.pareto docstring +* `#5359 <https://github.com/numpy/numpy/issues/5359>`__: minor changes to linspace docstring +* `#4723 <https://github.com/numpy/numpy/issues/4723>`__: fix a compile issues on AIX diff --git a/doc/source/release/template.rst b/doc/source/release/template.rst new file mode 100644 index 000000000..cde7646df --- /dev/null +++ b/doc/source/release/template.rst @@ -0,0 +1,45 @@ +:orphan: + +========================== +NumPy 1.xx.x Release Notes +========================== + + +Highlights +========== + + +New functions +============= + + +Deprecations +============ + + +Future Changes +============== + + +Expired deprecations +==================== + + +Compatibility notes +=================== + + +C API changes +============= + + +New Features +============ + + +Improvements +============ + + +Changes +======= diff --git a/doc/source/user/basics.broadcasting.rst b/doc/source/user/basics.broadcasting.rst index 65584b1fd..4e9016ee0 100644 --- a/doc/source/user/basics.broadcasting.rst +++ b/doc/source/user/basics.broadcasting.rst @@ -2,6 +2,10 @@ Broadcasting ************ -.. seealso:: :class:`numpy.broadcast` +.. seealso:: + :class:`numpy.broadcast` + + :ref:`array-broadcasting-in-numpy` + An introduction to the concepts discussed here .. automodule:: numpy.doc.broadcasting diff --git a/doc/source/user/basics.dispatch.rst b/doc/source/user/basics.dispatch.rst new file mode 100644 index 000000000..f7b8da262 --- /dev/null +++ b/doc/source/user/basics.dispatch.rst @@ -0,0 +1,8 @@ +.. _basics.dispatch: + +******************************* +Writing custom array containers +******************************* + +.. automodule:: numpy.doc.dispatch + diff --git a/doc/source/user/basics.indexing.rst b/doc/source/user/basics.indexing.rst index 8844adcae..0dca4b884 100644 --- a/doc/source/user/basics.indexing.rst +++ b/doc/source/user/basics.indexing.rst @@ -4,6 +4,10 @@ Indexing ******** -.. seealso:: :ref:`Indexing routines <routines.indexing>` +.. seealso:: + + :ref:`Indexing <arrays.indexing>` + + :ref:`Indexing routines <routines.indexing>` .. automodule:: numpy.doc.indexing diff --git a/doc/source/user/basics.io.genfromtxt.rst b/doc/source/user/basics.io.genfromtxt.rst index 21832e5aa..19e37eabc 100644 --- a/doc/source/user/basics.io.genfromtxt.rst +++ b/doc/source/user/basics.io.genfromtxt.rst @@ -27,13 +27,13 @@ Defining the input ================== The only mandatory argument of :func:`~numpy.genfromtxt` is the source of -the data. It can be a string, a list of strings, or a generator. If a -single string is provided, it is assumed to be the name of a local or -remote file, or an open file-like object with a :meth:`read` method, for -example, a file or :class:`io.StringIO` object. If a list of strings -or a generator returning strings is provided, each string is treated as one -line in a file. When the URL of a remote file is passed, the file is -automatically downloaded to the current directory and opened. +the data. It can be a string, a list of strings, a generator or an open +file-like object with a :meth:`read` method, for example, a file or +:class:`io.StringIO` object. If a single string is provided, it is assumed +to be the name of a local or remote file. If a list of strings or a generator +returning strings is provided, each string is treated as one line in a file. +When the URL of a remote file is passed, the file is automatically downloaded +to the current directory and opened. Recognized file types are text files and archives. Currently, the function recognizes :class:`gzip` and :class:`bz2` (`bzip2`) archives. The type of @@ -521,12 +521,6 @@ provides several convenience functions derived from :func:`~numpy.genfromtxt`. These functions work the same way as the original, but they have different default values. -:func:`~numpy.ndfromtxt` - Always set ``usemask=False``. - The output is always a standard :class:`numpy.ndarray`. -:func:`~numpy.mafromtxt` - Always set ``usemask=True``. - The output is always a :class:`~numpy.ma.MaskedArray` :func:`~numpy.recfromtxt` Returns a standard :class:`numpy.recarray` (if ``usemask=False``) or a :class:`~numpy.ma.MaskedRecords` array (if ``usemaske=True``). The diff --git a/doc/source/user/basics.rst b/doc/source/user/basics.rst index 7875aff6e..e0fc0ece3 100644 --- a/doc/source/user/basics.rst +++ b/doc/source/user/basics.rst @@ -12,4 +12,5 @@ NumPy basics basics.broadcasting basics.byteswapping basics.rec + basics.dispatch basics.subclassing diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst index d224951dd..b4b4371e5 100644 --- a/doc/source/user/building.rst +++ b/doc/source/user/building.rst @@ -56,7 +56,7 @@ Basic Installation To install NumPy run:: - python setup.py install + pip install . To perform an in-place build that can be run from the source folder run:: @@ -69,6 +69,15 @@ Using ``virtualenv`` should work as expected. *Note: for build instructions to do development work on NumPy itself, see* :ref:`development-environment`. +Testing +------- + +Make sure to test your builds. To ensure everything stays in shape, see if all tests pass:: + + $ python runtests.py -v -m full + +For detailed info on testing, see :ref:`testing-builds`. + .. _parallel-builds: Parallel builds @@ -118,12 +127,71 @@ means that g77 has been used. If libgfortran.so is a dependency, gfortran has been used. If both are dependencies, this means both have been used, which is almost always a very bad idea. +Accelerated BLAS/LAPACK libraries +--------------------------------- + +NumPy searches for optimized linear algebra libraries such as BLAS and LAPACK. +There are specific orders for searching these libraries, as described below. + +BLAS +~~~~ + +The default order for the libraries are: + +1. MKL +2. BLIS +3. OpenBLAS +4. ATLAS +5. Accelerate (MacOS) +6. BLAS (NetLIB) + + +If you wish to build against OpenBLAS but you also have BLIS available one +may predefine the order of searching via the environment variable +``NPY_BLAS_ORDER`` which is a comma-separated list of the above names which +is used to determine what to search for, for instance:: + + NPY_BLAS_ORDER=ATLAS,blis,openblas,MKL python setup.py build + +will prefer to use ATLAS, then BLIS, then OpenBLAS and as a last resort MKL. +If neither of these exists the build will fail (names are compared +lower case). + +LAPACK +~~~~~~ + +The default order for the libraries are: + +1. MKL +2. OpenBLAS +3. libFLAME +4. ATLAS +5. Accelerate (MacOS) +6. LAPACK (NetLIB) + + +If you wish to build against OpenBLAS but you also have MKL available one +may predefine the order of searching via the environment variable +``NPY_LAPACK_ORDER`` which is a comma-separated list of the above names, +for instance:: + + NPY_LAPACK_ORDER=ATLAS,openblas,MKL python setup.py build + +will prefer to use ATLAS, then OpenBLAS and as a last resort MKL. +If neither of these exists the build will fail (names are compared +lower case). + + Disabling ATLAS and other accelerated libraries ------------------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Usage of ATLAS and other accelerated libraries in NumPy can be disabled via:: + NPY_BLAS_ORDER= NPY_LAPACK_ORDER= python setup.py build + +or:: + BLAS=None LAPACK=None ATLAS=None python setup.py build diff --git a/doc/source/user/c-info.beyond-basics.rst b/doc/source/user/c-info.beyond-basics.rst index d4d941a5e..dd25861b4 100644 --- a/doc/source/user/c-info.beyond-basics.rst +++ b/doc/source/user/c-info.beyond-basics.rst @@ -300,9 +300,10 @@ An example castfunc is: static void double_to_float(double *from, float* to, npy_intp n, - void* ig1, void* ig2); - while (n--) { - (*to++) = (double) *(from++); + void* ignore1, void* ignore2) { + while (n--) { + (*to++) = (double) *(from++); + } } This could then be registered to convert doubles to floats using the diff --git a/doc/source/user/c-info.how-to-extend.rst b/doc/source/user/c-info.how-to-extend.rst index 9738168d2..00ef8ab74 100644 --- a/doc/source/user/c-info.how-to-extend.rst +++ b/doc/source/user/c-info.how-to-extend.rst @@ -342,7 +342,7 @@ The method is to 4. If you are writing the algorithm, then I recommend that you use the stride information contained in the array to access the elements of - the array (the :c:func:`PyArray_GETPTR` macros make this painless). Then, + the array (the :c:func:`PyArray_GetPtr` macros make this painless). Then, you can relax your requirements so as not to force a single-segment array and the data-copying that might result. @@ -362,8 +362,7 @@ specific builtin data-type ( *e.g.* float), while specifying a particular set of requirements ( *e.g.* contiguous, aligned, and writeable). The syntax is -.. c:function:: PyObject *PyArray_FROM_OTF( \ - PyObject* obj, int typenum, int requirements) +:c:func:`PyArray_FROM_OTF` Return an ndarray from any Python object, *obj*, that can be converted to an array. The number of dimensions in the returned @@ -446,33 +445,25 @@ writeable). The syntax is flags most commonly needed are :c:data:`NPY_ARRAY_IN_ARRAY`, :c:data:`NPY_OUT_ARRAY`, and :c:data:`NPY_ARRAY_INOUT_ARRAY`: - .. c:var:: NPY_ARRAY_IN_ARRAY + :c:data:`NPY_ARRAY_IN_ARRAY` - Equivalent to :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| - :c:data:`NPY_ARRAY_ALIGNED`. This combination of flags is useful - for arrays that must be in C-contiguous order and aligned. - These kinds of arrays are usually input arrays for some - algorithm. + This flag is useful for arrays that must be in C-contiguous + order and aligned. These kinds of arrays are usually input + arrays for some algorithm. - .. c:var:: NPY_ARRAY_OUT_ARRAY + :c:data:`NPY_ARRAY_OUT_ARRAY` - Equivalent to :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| - :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_WRITEABLE`. This - combination of flags is useful to specify an array that is + This flag is useful to specify an array that is in C-contiguous order, is aligned, and can be written to as well. Such an array is usually returned as output (although normally such output arrays are created from scratch). - .. c:var:: NPY_ARRAY_INOUT_ARRAY + :c:data:`NPY_ARRAY_INOUT_ARRAY` - Equivalent to :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| - :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_WRITEABLE` \| - :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` \| - :c:data:`NPY_ARRAY_UPDATEIFCOPY`. This combination of flags is - useful to specify an array that will be used for both + This flag is useful to specify an array that will be used for both input and output. :c:func:`PyArray_ResolveWritebackIfCopy` - must be called before :func:`Py_DECREF` at + must be called before :c:func:`Py_DECREF` at the end of the interface routine to write back the temporary data into the original array passed in. Use of the :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` or @@ -487,16 +478,16 @@ writeable). The syntax is Other useful flags that can be OR'd as additional requirements are: - .. c:var:: NPY_ARRAY_FORCECAST + :c:data:`NPY_ARRAY_FORCECAST` Cast to the desired type, even if it can't be done without losing information. - .. c:var:: NPY_ARRAY_ENSURECOPY + :c:data:`NPY_ARRAY_ENSURECOPY` Make sure the resulting array is a copy of the original. - .. c:var:: NPY_ARRAY_ENSUREARRAY + :c:data:`NPY_ARRAY_ENSUREARRAY` Make sure the resulting object is an actual ndarray and not a sub- class. @@ -513,7 +504,7 @@ writeable). The syntax is Creating a brand-new ndarray ---------------------------- -Quite often new arrays must be created from within extension-module +Quite often, new arrays must be created from within extension-module code. Perhaps an output array is needed and you don't want the caller to have to supply it. Perhaps only a temporary array is needed to hold an intermediate calculation. Whatever the need there are simple ways @@ -521,43 +512,9 @@ to get an ndarray object of whatever data-type is needed. The most general function for doing this is :c:func:`PyArray_NewFromDescr`. All array creation functions go through this heavily re-used code. Because of its flexibility, it can be somewhat confusing to use. As a result, -simpler forms exist that are easier to use. - -.. c:function:: PyObject *PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) - - This function allocates new memory and places it in an ndarray - with *nd* dimensions whose shape is determined by the array of - at least *nd* items pointed to by *dims*. The memory for the - array is uninitialized (unless typenum is :c:data:`NPY_OBJECT` in - which case each element in the array is set to NULL). The - *typenum* argument allows specification of any of the builtin - data-types such as :c:data:`NPY_FLOAT` or :c:data:`NPY_LONG`. The - memory for the array can be set to zero if desired using - :c:func:`PyArray_FILLWBYTE` (return_object, 0). - -.. c:function:: PyObject *PyArray_SimpleNewFromData( \ - int nd, npy_intp* dims, int typenum, void* data) - - Sometimes, you want to wrap memory allocated elsewhere into an - ndarray object for downstream use. This routine makes it - straightforward to do that. The first three arguments are the same - as in :c:func:`PyArray_SimpleNew`, the final argument is a pointer to a - block of contiguous memory that the ndarray should use as it's - data-buffer which will be interpreted in C-style contiguous - fashion. A new reference to an ndarray is returned, but the - ndarray will not own its data. When this ndarray is deallocated, - the pointer will not be freed. - - You should ensure that the provided memory is not freed while the - returned array is in existence. The easiest way to handle this is - if data comes from another reference-counted Python object. The - reference count on this object should be increased after the - pointer is passed in, and the base member of the returned ndarray - should point to the Python object that owns the data. Then, when - the ndarray is deallocated, the base-member will be DECREF'd - appropriately. If you want the memory to be freed as soon as the - ndarray is deallocated then simply set the OWNDATA flag on the - returned ndarray. +simpler forms exist that are easier to use. These forms are part of the +:c:func:`PyArray_SimpleNew` family of functions, which simplify the interface +by providing default values for common use cases. Getting at ndarray memory and accessing elements of the ndarray @@ -573,7 +530,7 @@ specific element of the array is determined only by the array of npy_intp variables, :c:func:`PyArray_STRIDES` (obj). In particular, this c-array of integers shows how many **bytes** must be added to the current element pointer to get to the next element in each dimension. -For arrays less than 4-dimensions there are :c:func:`PyArray_GETPTR{k}` +For arrays less than 4-dimensions there are ``PyArray_GETPTR{k}`` (obj, ...) macros where {k} is the integer 1, 2, 3, or 4 that make using the array strides easier. The arguments .... represent {k} non- negative integer indices into the array. For example, suppose ``E`` is @@ -586,7 +543,7 @@ contiguous arrays have particular striding patterns. Two array flags whether or not the striding pattern of a particular array matches the C-style contiguous or Fortran-style contiguous or neither. Whether or not the striding pattern matches a standard C or Fortran one can be -tested Using :c:func:`PyArray_ISCONTIGUOUS` (obj) and +tested Using :c:func:`PyArray_IS_C_CONTIGUOUS` (obj) and :c:func:`PyArray_ISFORTRAN` (obj) respectively. Most third-party libraries expect contiguous arrays. But, often it is not difficult to support general-purpose striding. I encourage you to use the striding diff --git a/doc/source/user/c-info.python-as-glue.rst b/doc/source/user/c-info.python-as-glue.rst index 01d2a64d1..7b9b096af 100644 --- a/doc/source/user/c-info.python-as-glue.rst +++ b/doc/source/user/c-info.python-as-glue.rst @@ -387,7 +387,7 @@ distribution of the ``add.f`` module (as part of the package Installation of the new package is easy using:: - python setup.py install + pip install . assuming you have the proper permissions to write to the main site- packages directory for the version of Python you are using. For the @@ -744,14 +744,14 @@ around this restriction that allow ctypes to integrate with other objects. 1. Don't set the argtypes attribute of the function object and define an - :obj:`_as_parameter_` method for the object you want to pass in. The - :obj:`_as_parameter_` method must return a Python int which will be passed + ``_as_parameter_`` method for the object you want to pass in. The + ``_as_parameter_`` method must return a Python int which will be passed directly to the function. 2. Set the argtypes attribute to a list whose entries contain objects with a classmethod named from_param that knows how to convert your object to an object that ctypes can understand (an int/long, string, - unicode, or object with the :obj:`_as_parameter_` attribute). + unicode, or object with the ``_as_parameter_`` attribute). NumPy uses both methods with a preference for the second method because it can be safer. The ctypes attribute of the ndarray returns @@ -764,7 +764,7 @@ correct type, shape, and has the correct flags set or risk nasty crashes if the data-pointer to inappropriate arrays are passed in. To implement the second method, NumPy provides the class-factory -function :func:`ndpointer` in the :mod:`ctypeslib` module. This +function :func:`ndpointer` in the :mod:`numpy.ctypeslib` module. This class-factory function produces an appropriate class that can be placed in an argtypes attribute entry of a ctypes function. The class will contain a from_param method which ctypes will use to convert any diff --git a/doc/source/user/numpy-for-matlab-users.rst b/doc/source/user/numpy-for-matlab-users.rst index 399237c21..e53d1ca45 100644 --- a/doc/source/user/numpy-for-matlab-users.rst +++ b/doc/source/user/numpy-for-matlab-users.rst @@ -436,7 +436,7 @@ Linear Algebra Equivalents ``a`` * - ``rand(3,4)`` - - ``random.rand(3,4)`` + - ``random.rand(3,4)`` or ``random.random_sample((3, 4))`` - random 3x4 matrix * - ``linspace(1,3,4)`` @@ -547,7 +547,7 @@ Linear Algebra Equivalents - eigenvalues and eigenvectors of ``a`` * - ``[V,D]=eig(a,b)`` - - ``V,D = np.linalg.eig(a,b)`` + - ``D,V = scipy.linalg.eig(a,b)`` - eigenvalues and eigenvectors of ``a``, ``b`` * - ``[V,D]=eigs(a,k)`` @@ -693,19 +693,19 @@ this is just an example, not a statement of "best practices"): :: - # Make all numpy available via shorter 'num' prefix - import numpy as num + # Make all numpy available via shorter 'np' prefix + import numpy as np # Make all matlib functions accessible at the top level via M.func() import numpy.matlib as M # Make some matlib functions accessible directly at the top level via, e.g. rand(3,3) from numpy.matlib import rand,zeros,ones,empty,eye # Define a Hermitian function def hermitian(A, **kwargs): - return num.transpose(A,**kwargs).conj() + return np.transpose(A,**kwargs).conj() # Make some shortcuts for transpose,hermitian: - # num.transpose(A) --> T(A) + # np.transpose(A) --> T(A) # hermitian(A) --> H(A) - T = num.transpose + T = np.transpose H = hermitian Links diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index 5ef8b145f..a23a7b2c7 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -25,7 +25,7 @@ The Basics NumPy's main object is the homogeneous multidimensional array. It is a table of elements (usually numbers), all of the same type, indexed by a -tuple of positive integers. In NumPy dimensions are called *axes*. +tuple of non-negative integers. In NumPy dimensions are called *axes*. For example, the coordinates of a point in 3D space ``[1, 2, 1]`` has one axis. That axis has 3 elements in it, so we say it has a length @@ -206,8 +206,8 @@ of elements that we want, instead of the step:: `empty_like`, `arange`, `linspace`, - `numpy.random.rand`, - `numpy.random.randn`, + `numpy.random.mtrand.RandomState.rand`, + `numpy.random.mtrand.RandomState.randn`, `fromfunction`, `fromfile` @@ -270,7 +270,7 @@ can change the printing options using ``set_printoptions``. :: - >>> np.set_printoptions(threshold=np.nan) + >>> np.set_printoptions(threshold=sys.maxsize) # sys module should be imported Basic Operations @@ -732,9 +732,9 @@ stacks 1D arrays as columns into a 2D array. It is equivalent to array([[ 4., 3.], [ 2., 8.]]) -On the other hand, the function `row_stack` is equivalent to `vstack` +On the other hand, the function `ma.row_stack` is equivalent to `vstack` for any input arrays. -In general, for arrays of with more than two dimensions, +In general, for arrays with more than two dimensions, `hstack` stacks along their second axes, `vstack` stacks along their first axes, and `concatenate` @@ -884,6 +884,17 @@ The ``copy`` method makes a complete copy of the array and its data. [ 8, 10, 10, 11]]) +Sometimes ``copy`` should be called after slicing if the original array is not required anymore. +For example, suppose ``a`` is a huge intermediate result and the final result ``b`` only contains +a small fraction of ``a``, a deep copy should be made when constructing ``b`` with slicing:: + + >>> a = np.arange(int(1e8)) + >>> b = a[:100].copy() + >>> del a # the memory of ``a`` can be released. + +If ``b = a[:100]`` is used instead, ``a`` is referenced by ``b`` and will persist in memory +even if ``del a`` is executed. + Functions and Methods Overview ------------------------------ @@ -1465,5 +1476,5 @@ Further reading - The `Python tutorial <https://docs.python.org/tutorial/>`__ - :ref:`reference` - `SciPy Tutorial <https://docs.scipy.org/doc/scipy/reference/tutorial/index.html>`__ -- `SciPy Lecture Notes <https://www.scipy-lectures.org>`__ +- `SciPy Lecture Notes <https://scipy-lectures.org>`__ - A `matlab, R, IDL, NumPy/SciPy dictionary <http://mathesaurus.sf.net/>`__ diff --git a/doc/source/user/theory.broadcast_1.gif b/doc/source/user/theory.broadcast_1.gif Binary files differnew file mode 100644 index 000000000..541ec734b --- /dev/null +++ b/doc/source/user/theory.broadcast_1.gif diff --git a/doc/source/user/theory.broadcast_2.gif b/doc/source/user/theory.broadcast_2.gif Binary files differnew file mode 100644 index 000000000..163a8473f --- /dev/null +++ b/doc/source/user/theory.broadcast_2.gif diff --git a/doc/source/user/theory.broadcast_3.gif b/doc/source/user/theory.broadcast_3.gif Binary files differnew file mode 100644 index 000000000..83f61f2df --- /dev/null +++ b/doc/source/user/theory.broadcast_3.gif diff --git a/doc/source/user/theory.broadcast_4.gif b/doc/source/user/theory.broadcast_4.gif Binary files differnew file mode 100644 index 000000000..9b21ff582 --- /dev/null +++ b/doc/source/user/theory.broadcast_4.gif diff --git a/doc/source/user/theory.broadcast_5.png b/doc/source/user/theory.broadcast_5.png Binary files differnew file mode 100644 index 000000000..3aa2f0536 --- /dev/null +++ b/doc/source/user/theory.broadcast_5.png diff --git a/doc/source/user/theory.broadcasting.rst b/doc/source/user/theory.broadcasting.rst new file mode 100644 index 000000000..b37edeacc --- /dev/null +++ b/doc/source/user/theory.broadcasting.rst @@ -0,0 +1,229 @@ +:orphan: + +.. _array-broadcasting-in-numpy: + +=========================== +Array Broadcasting in Numpy +=========================== + +.. + Originally part of the scipy.org wiki, available `here + <https://scipy.github.io/old-wiki/pages/EricsBroadcastingDoc>`_ or from the + `github repo + <https://github.com/scipy/old-wiki/blob/gh-pages/pages/EricsBroadcastingDoc.html>`_ + +Let's explore a more advanced concept in numpy called broadcasting. The +term broadcasting describes how numpy treats arrays with different shapes +during arithmetic operations. Subject to certain constraints, the smaller array +is "broadcast" across the larger array so that they have compatible shapes. +Broadcasting provides a means of vectorizing array operations so that looping +occurs in C instead of Python. It does this without making needless copies of +data and usually leads to efficient algorithm implementations. There are also +cases where broadcasting is a bad idea because it leads to inefficient use of +memory that slows computation. This article provides a gentle introduction to +broadcasting with numerous examples ranging from simple to involved. It also +provides hints on when and when not to use broadcasting. + +numpy operations are usually done element-by-element which requires two arrays +to have exactly the same shape: + +.. code-block:: python + :caption: Example 1 + :name: example-1 + + >>> from numpy import array + >>> a = array([1.0, 2.0, 3.0]) + >>> b = array([2.0, 2.0, 2.0]) + >>> a * b + array([ 2., 4., 6.]) + +numpy's broadcasting rule relaxes this constraint when the arrays' shapes meet +certain constraints. The simplest broadcasting example occurs when an array and +a scalar value are combined in an operation: + +.. code-block:: python + :caption: Example 2 + :name: example-2 + + >>> from numpy import array + >>> a = array([1.0,2.0,3.0]) + >>> b = 2.0 + >>> a * b + array([ 2., 4., 6.]) + +The result is equivalent to the previous example where ``b`` was an array. We +can think of the scalar ``b`` being stretched during the arithmetic operation +into an array with the same shape as ``a``. The new elements in ``b``, as shown +in :ref:`figure-1`, are simply copies of the original scalar. The stretching +analogy is only conceptual. numpy is smart enough to use the original scalar +value without actually making copies so that broadcasting operations are as +memory and computationally efficient as possible. Because :ref:`example-2` +moves less memory, (``b`` is a scalar, not an array) around during the +multiplication, it is about 10% faster than :ref:`example-1` using the standard +numpy on Windows 2000 with one million element arrays. + +.. figure:: theory.broadcast_1.gif + :alt: Vector-Scalar multiplication + :name: figure-1 + + *Figure 1* + + *In the simplest example of broadcasting, the scalar ``b`` is + stretched to become an array of with the same shape as ``a`` so the shapes + are compatible for element-by-element multiplication.* + + +The rule governing whether two arrays have compatible shapes for broadcasting +can be expressed in a single sentence. + +.. admonition:: The Broadcasting Rule + + **In order to broadcast, the size of the trailing axes for both arrays in + an operation must either be the same size or one of them must be one.** + +If this condition is not met, a ``ValueError('frames are not aligned')`` +exception is thrown indicating that the arrays have incompatible shapes. The +size of the result array created by broadcast operations is the maximum size +along each dimension from the input arrays. Note that the rule does not say +anything about the two arrays needing to have the same number of dimensions. +So, for example, if you have a 256 x 256 x 3 array of RGB values, and you want +to scale each color in the image by a different value, you can multiply the +image by a one-dimensional array with 3 values. Lining up the sizes of the +trailing axes of these arrays according to the broadcast rule shows that they +are compatible + ++-------+------------+-------+-------+---+ +|Image | (3d array) | 256 x | 256 x | 3 | ++-------+------------+-------+-------+---+ +|Scale | (1d array) | | | 3 | ++-------+------------+-------+-------+---+ +|Result | (3d array) | 256 x | 256 x | 3 | ++-------+------------+-------+-------+---+ + +In the following example, both the ``A`` and ``B`` arrays have axes with length +one that are expanded to a larger size in a broadcast operation. + ++-------+------------+-----+-----+-----+---+ +|A | (4d array) | 8 x | 1 x | 6 x | 1 | ++-------+------------+-----+-----+-----+---+ +|B | (3d array) | | 7 x | 1 x | 5 | ++-------+------------+-----+-----+-----+---+ +|Result | (4d array) | 8 x | 7 x | 6 x | 5 | ++-------+------------+-----+-----+-----+---+ + +Below, are several code examples and graphical representations that help make +the broadcast rule visually obvious. :ref:`example-3` adds a one-dimensional array +to a two-dimensional array: + +.. code-block:: python + :caption: Example 3 + :name: example-3 + + >>> from numpy import array + >>> a = array([[ 0.0, 0.0, 0.0], + ... [10.0, 10.0, 10.0], + ... [20.0, 20.0, 20.0], + ... [30.0, 30.0, 30.0]]) + >>> b = array([1.0, 2.0, 3.0]) + >>> a + b + array([[ 1., 2., 3.], + [ 11., 12., 13.], + [ 21., 22., 23.], + [ 31., 32., 33.]]) + +As shown in :ref:`figure-2`, ``b`` is added to each row of ``a``. When ``b`` is +longer than the rows of ``a``, as in :ref:`figure-3`, an exception is raised +because of the incompatible shapes. + +.. figure:: theory.broadcast_2.gif + :alt: Matrix-Vector + :name: figure-2 + + *Figure 2* + + *A two dimensional array multiplied by a one dimensional array results in + broadcasting if number of 1-d array elements matches the number of 2-d + array columns.* + +.. figure:: theory.broadcast_3.gif + :alt: Matrix-Vector-with-error + :name: figure-3 + + *Figure 3* + + *When the trailing dimensions of the arrays are unequal, broadcasting fails + because it is impossible to align the values in the rows of the 1st array + with the elements of the 2nd arrays for element-by-element addition.* + +Broadcasting provides a convenient way of taking the outer product (or any +other outer operation) of two arrays. The following example shows an outer +addition operation of two 1-d arrays that produces the same result as +:ref:`example-3` + +.. code-block:: python + :caption: Example 4 + :name: example-4 + + >>> from numpy import array, newaxis + >>> a = array([0.0, 10.0, 20.0, 30.0]) + >>> b = array([1.0, 2.0, 3.0]) + >>> a[:,newaxis] + b + array([[ 1., 2., 3.], + [ 11., 12., 13.], + [ 21., 22., 23.], + [ 31., 32., 33.]]) + +Here the newaxis index operator inserts a new axis into ``a``, making it a +two-dimensional 4x1 array. :ref:`figure-4` illustrates the stretching of both +arrays to produce the desired 4x3 output array. + +.. figure:: theory.broadcast_4.gif + :alt: vector-vector with newaxis + :name: figure-4 + + *Figure 4* + + In some cases, broadcasting stretches both arrays to form an output array + larger than either of the initial arrays.* + +A Practical Example: Vector Quantization. +========================================= + +Broadcasting comes up quite often in real world problems. A typical example +occurs in the vector quantization (VQ) algorithm used in information theory, +classification, and other related areas. The basic operation in VQ [#f0] finds +the closest point in a set of points, called codes in VQ jargon, to a given +point, called the observation. In the very simple, two-dimensional case shown +in :ref:`figure-5`, the values in observation describe the weight and height of an +athlete to be classified. The codes represent different classes of +athletes. [#f1]_ Finding the closest point requires calculating the distance +between observation and each of the codes. The shortest distance provides the +best match. In this example, ``codes[0]`` is the closest class indicating that +the athlete is likely a basketball player. + +.. figure:: theory.broadcast_5.png + :alt: vector quantitization example + :name: figure-5 + + *Figure 5* + + *The basic operation of vector quantization calculates the distance between + an object to be classified, the dark square, and multiple known codes, the + gray circles. In this simple case, the codes represent individual classes. + More complex cases use multiple codes per class.* + + +.. rubric:: Footnotes + +.. [#f0] Vector Quantization J. Makhoul, S. Roucos, and H. Gish, "Vector Quantization in Speech Coding," Proc. IEEE, vol. 73, pp. 1551-1587, Nov. 1985. +.. [#f1] + In this example, weight has more impact on the distance calculation + than height because of the larger values. In practice, it is important to + normalize the height and weight, often by their standard deviation across the + data set, so that both have equal influence on the distance calculation. + +.. note:: + + The code to produce the figures is part of the `AstroML book + <http://www.astroml.org/book_figures/appendix/fig_broadcast_visual.html>`_ + diff --git a/doc/source/user/whatisnumpy.rst b/doc/source/user/whatisnumpy.rst index cd74a8de3..abaa2bfed 100644 --- a/doc/source/user/whatisnumpy.rst +++ b/doc/source/user/whatisnumpy.rst @@ -91,6 +91,11 @@ idiom is even simpler! This last example illustrates two of NumPy's features which are the basis of much of its power: vectorization and broadcasting. +.. _whatis-vectorization: + +Why is NumPy Fast? +------------------ + Vectorization describes the absence of any explicit looping, indexing, etc., in the code - these things are taking place, of course, just "behind the scenes" in optimized, pre-compiled C code. Vectorized @@ -120,9 +125,13 @@ the shape of the larger in such a way that the resulting broadcast is unambiguous. For detailed "rules" of broadcasting see `numpy.doc.broadcasting`. +Who Else Uses NumPy? +-------------------- + NumPy fully supports an object-oriented approach, starting, once again, with `ndarray`. For example, `ndarray` is a class, possessing -numerous methods and attributes. Many of its methods mirror -functions in the outer-most NumPy namespace, giving the programmer -complete freedom to code in whichever paradigm she prefers and/or -which seems most appropriate to the task at hand. +numerous methods and attributes. Many of its methods are mirrored by +functions in the outer-most NumPy namespace, allowing the programmer +to code in whichever paradigm they prefer. This flexibility has allowed the +NumPy array dialect and NumPy `ndarray` class to become the *de-facto* language +of multi-dimensional data interchange used in Python. |
