diff options
Diffstat (limited to 'doc/source')
82 files changed, 2779 insertions, 881 deletions
diff --git a/doc/source/_static/numpy_logo.png b/doc/source/_static/numpy_logo.png Binary files differnew file mode 100644 index 000000000..af8cbe323 --- /dev/null +++ b/doc/source/_static/numpy_logo.png diff --git a/doc/source/_templates/indexcontent.html b/doc/source/_templates/indexcontent.html index 008eaaa7c..294d39233 100644 --- a/doc/source/_templates/indexcontent.html +++ b/doc/source/_templates/indexcontent.html @@ -7,6 +7,8 @@ <span class="linkdescr">start here</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("reference/index") }}">NumPy Reference</a><br/> <span class="linkdescr">reference documentation</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("benchmarking") }}">Benchmarking</a><br/> + <span class="linkdescr">benchmarking NumPy</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("f2py/index") }}">F2Py Guide</a><br/> <span class="linkdescr">f2py documentation</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("dev/index") }}">NumPy Developer Guide</a><br/> diff --git a/doc/source/_templates/indexsidebar.html b/doc/source/_templates/indexsidebar.html index 51e7c4308..4707fc0e8 100644 --- a/doc/source/_templates/indexsidebar.html +++ b/doc/source/_templates/indexsidebar.html @@ -1,4 +1,5 @@ <h3>Resources</h3> <ul> + <li><a href="https://numpy.org/">NumPy.org website</a></li> <li><a href="https://scipy.org/">Scipy.org website</a></li> </ul> diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html index 77da54a00..beaa297db 100644 --- a/doc/source/_templates/layout.html +++ b/doc/source/_templates/layout.html @@ -1,5 +1,15 @@ {% extends "!layout.html" %} +{%- block header %} +<div class="container"> + <div class="top-scipy-org-logo-header" style="background-color: #a2bae8;"> + <a href="{{ pathto('index') }}"> + <img border=0 alt="NumPy" src="{{ pathto('_static/numpy_logo.png', 1) }}"></a> + </div> + </div> +</div> + +{% endblock %} {% block rootrellink %} {% if pagename != 'index' %} <li class="active"><a href="{{ pathto('index') }}">{{ shorttitle|e }}</a></li> diff --git a/doc/source/about.rst b/doc/source/about.rst index 5ac4facbb..3e83833d1 100644 --- a/doc/source/about.rst +++ b/doc/source/about.rst @@ -8,7 +8,7 @@ needed for scientific computing with Python. This package contains: - sophisticated :ref:`(broadcasting) functions <ufuncs>` - basic :ref:`linear algebra functions <routines.linalg>` - basic :ref:`Fourier transforms <routines.fft>` -- sophisticated :ref:`random number capabilities <routines.random>` +- sophisticated :ref:`random number capabilities <numpyrandom>` - tools for integrating Fortran code - tools for integrating C/C++ code diff --git a/doc/source/benchmarking.rst b/doc/source/benchmarking.rst new file mode 100644 index 000000000..9f0eeb03a --- /dev/null +++ b/doc/source/benchmarking.rst @@ -0,0 +1 @@ +.. include:: ../../benchmarks/README.rst diff --git a/doc/source/conf.py b/doc/source/conf.py index 455e9748b..4f312eff5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -19,11 +19,19 @@ needs_sphinx = '1.0' sys.path.insert(0, os.path.abspath('../sphinxext')) -extensions = ['sphinx.ext.autodoc', 'numpydoc', - 'sphinx.ext.intersphinx', 'sphinx.ext.coverage', - 'sphinx.ext.doctest', 'sphinx.ext.autosummary', - 'sphinx.ext.graphviz', 'sphinx.ext.ifconfig', - 'matplotlib.sphinxext.plot_directive'] +extensions = [ + 'sphinx.ext.autodoc', + 'numpydoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.autosummary', + 'sphinx.ext.graphviz', + 'sphinx.ext.ifconfig', + 'matplotlib.sphinxext.plot_directive', + 'IPython.sphinxext.ipython_console_highlighting', + 'IPython.sphinxext.ipython_directive', +] if sphinx.__version__ >= "1.4": extensions.append('sphinx.ext.imgmath') @@ -39,7 +47,7 @@ source_suffix = '.rst' # General substitutions. project = 'NumPy' -copyright = '2008-2018, The SciPy community' +copyright = '2008-2019, The SciPy community' # The default replacements for |version| and |release|, also used in various # other places throughout the built documents. @@ -113,7 +121,9 @@ else: "edit_link": False, "sidebar": "left", "scipy_org_logo": False, - "rootlinks": [] + "rootlinks": [("https://numpy.org/", "NumPy.org"), + ("https://numpy.org/doc", "Docs"), + ] } html_sidebars = {'index': ['indexsidebar.html', 'searchbox.html']} @@ -234,7 +244,7 @@ numpydoc_use_plots = True # ----------------------------------------------------------------------------- import glob -autosummary_generate = glob.glob("reference/*.rst") +autosummary_generate = True # ----------------------------------------------------------------------------- # Coverage checker @@ -355,3 +365,21 @@ def linkcode_resolve(domain, info): else: return "https://github.com/numpy/numpy/blob/v%s/numpy/%s%s" % ( numpy.__version__, fn, linespec) + +from pygments.lexers import CLexer +from pygments import token +from sphinx.highlighting import lexers +import copy + +class NumPyLexer(CLexer): + name = 'NUMPYLEXER' + + tokens = copy.deepcopy(lexers['c'].tokens) + # Extend the regex for valid identifiers with @ + for k, val in tokens.items(): + for i, v in enumerate(val): + if isinstance(v, tuple): + if isinstance(v[0], str): + val[i] = (v[0].replace('a-zA-Z', 'a-zA-Z@'),) + v[1:] + +lexers['NumPyC'] = NumPyLexer(stripnl=False) diff --git a/doc/source/dev/development_environment.rst b/doc/source/dev/development_environment.rst index aa4326f63..ce571926e 100644 --- a/doc/source/dev/development_environment.rst +++ b/doc/source/dev/development_environment.rst @@ -3,18 +3,27 @@ Setting up and using your development environment ================================================= +.. _recommended-development-setup: + Recommended development setup ----------------------------- Since NumPy contains parts written in C and Cython that need to be compiled before use, make sure you have the necessary compilers and Python -development headers installed - see :ref:`building-from-source`. +development headers installed - see :ref:`building-from-source`. Building +NumPy as of version ``1.17`` requires a C99 compliant compiler. For +some older compilers this may require ``export CFLAGS='-std=c99'``. Having compiled code also means that importing NumPy from the development sources needs some additional steps, which are explained below. For the rest of this chapter we assume that you have set up your git repo as described in :ref:`using-git`. +.. _testing-builds: + +Testing builds +-------------- + To build the development version of NumPy and run tests, spawn interactive shells with the Python import paths properly set up etc., do one of:: @@ -43,6 +52,10 @@ When using pytest as a target (the default), you can $ python runtests.py -v -t numpy/core/tests/test_multiarray.py -- -k "MatMul and not vector" +.. note:: + + Remember that all tests of NumPy should pass before commiting your changes. + Using ``runtests.py`` is the recommended approach to running tests. There are also a number of alternatives to it, for example in-place build or installing to a virtualenv. See the FAQ below for details. @@ -86,13 +99,10 @@ Other build options It's possible to do a parallel build with ``numpy.distutils`` with the ``-j`` option; see :ref:`parallel-builds` for more details. -In order to install the development version of NumPy in ``site-packages``, use -``python setup.py install --user``. - A similar approach to in-place builds and use of ``PYTHONPATH`` but outside the source tree is to use:: - $ python setup.py install --prefix /some/owned/folder + $ pip install . --prefix /some/owned/folder $ export PYTHONPATH=/some/owned/folder/lib/python3.4/site-packages @@ -125,6 +135,9 @@ the interpreter, tests can be run like this:: >>> np.test('full') # Also run tests marked as slow >>> np.test('full', verbose=2) # Additionally print test name/file + An example of a successful test : + ``4686 passed, 362 skipped, 9 xfailed, 5 warnings in 213.99 seconds`` + Or a similar way from the command line:: $ python -c "import numpy as np; np.test()" @@ -142,9 +155,9 @@ That also takes extra arguments, like ``--pdb`` which drops you into the Python debugger when a test fails or an exception is raised. Running tests with `tox`_ is also supported. For example, to build NumPy and -run the test suite with Python 3.4, use:: +run the test suite with Python 3.7, use:: - $ tox -e py34 + $ tox -e py37 For more extensive information, see :ref:`testing-guidelines` diff --git a/doc/source/dev/gitwash/development_workflow.rst b/doc/source/dev/development_workflow.rst index 9561e25f7..900431374 100644 --- a/doc/source/dev/gitwash/development_workflow.rst +++ b/doc/source/dev/development_workflow.rst @@ -28,7 +28,7 @@ In short: - *Core developers* If you want to push changes without further review, see the notes :ref:`below <pushing-to-main>`. - + This way of working helps to keep work well organized and the history as clear as possible. @@ -69,7 +69,7 @@ Overview git status # Optional git diff # Optional git add modified_file - git commit + git commit # push the branch to your own Github repo git push origin my-new-feature @@ -112,38 +112,38 @@ In more detail properly formatted and sufficiently detailed commit message. After saving your message and closing the editor, your commit will be saved. For trivial commits, a short commit message can be passed in through the command line - using the ``-m`` flag. For example, ``git commit -am "ENH: Some message"``. - + using the ``-m`` flag. For example, ``git commit -am "ENH: Some message"``. + In some cases, you will see this form of the commit command: ``git commit -a``. The extra ``-a`` flag automatically commits all modified files and removes all deleted files. This can save you some typing of numerous ``git add`` commands; however, it can add unwanted changes to a commit if you're not careful. For more information, see `why the -a flag?`_ - and the - helpful use-case description in the `tangled working copy problem`_. + helpful use-case description in the `tangled working copy problem`_. #. Push the changes to your forked repo on github_:: git push origin my-new-feature For more information, see `git push`_. - + .. note:: - + Assuming you have followed the instructions in these pages, git will create a default link to your github_ repo called ``origin``. In git >= 1.7 you can ensure that the link to origin is permanently set by using the ``--set-upstream`` option:: - + git push --set-upstream origin my-new-feature - + From now on git_ will know that ``my-new-feature`` is related to the ``my-new-feature`` branch in your own github_ repo. Subsequent push calls are then simplified to the following:: git push - + You have to use ``--set-upstream`` for each new branch that you create. - + It may be the case that while you were working on your edits, new commits have been added to ``upstream`` that affect your work. In this case, follow the @@ -194,12 +194,18 @@ Asking for your changes to be merged with the main repo ======================================================= When you feel your work is finished, you can create a pull request (PR). Github -has a nice help page that outlines the process for `filing pull requests`_. +has a nice help page that outlines the process for `filing pull requests`_. If your changes involve modifications to the API or addition/modification of a -function, you should initiate a code review. This involves sending an email to -the `NumPy mailing list`_ with a link to your PR along with a description of -and a motivation for your changes. +function, you should + +- send an email to the `NumPy mailing list`_ with a link to your PR along with + a description of and a motivation for your changes. This may generate + changes and feedback. It might be prudent to start with this step if your + change may be controversial. +- add a release note to the ``doc/release/upcoming_changes/`` directory, + following the instructions and format in the + ``doc/release/upcoming_changes/README.rst`` file. .. _rebasing-on-master: @@ -500,11 +506,11 @@ them to ``upstream`` as follows: git push upstream my-feature-branch:master -.. note:: +.. note:: It's usually a good idea to use the ``-n`` flag to ``git push`` to check first that you're about to push the changes you want to the place you want. -.. include:: git_links.inc +.. include:: gitwash/git_links.inc diff --git a/doc/source/dev/gitwash/development_setup.rst b/doc/source/dev/gitwash/development_setup.rst index 1ebd4b486..9027dda64 100644 --- a/doc/source/dev/gitwash/development_setup.rst +++ b/doc/source/dev/gitwash/development_setup.rst @@ -25,6 +25,8 @@ to the instructions at http://help.github.com/forking/ - please see that page for more detail. We're repeating some of it here just to give the specifics for the NumPy_ project, and to suggest some default names. +.. _set-up-and-configure-a-github-account: + Set up and configure a github_ account ====================================== diff --git a/doc/source/dev/gitwash/following_latest.rst b/doc/source/dev/gitwash/following_latest.rst index ad497bf9a..0e98b4ec4 100644 --- a/doc/source/dev/gitwash/following_latest.rst +++ b/doc/source/dev/gitwash/following_latest.rst @@ -1,9 +1,5 @@ .. _following-latest: -============================= - Following the latest source -============================= - These are the instructions if you just want to follow the latest *NumPy* source, but you don't need to do any development for now. If you do want to contribute a patch (excellent!) or do more extensive diff --git a/doc/source/dev/gitwash/git_development.rst b/doc/source/dev/gitwash/git_development.rst deleted file mode 100644 index 5d7d47f89..000000000 --- a/doc/source/dev/gitwash/git_development.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _git-development: - -===================== - Git for development -===================== - -Contents: - -.. toctree:: - :maxdepth: 2 - - development_setup - configure_git - dot2_dot3 diff --git a/doc/source/dev/gitwash/git_intro.rst b/doc/source/dev/gitwash/git_intro.rst index 3ce322f8f..9d596d4d4 100644 --- a/doc/source/dev/gitwash/git_intro.rst +++ b/doc/source/dev/gitwash/git_intro.rst @@ -1,42 +1,8 @@ -============ -Introduction -============ - -These pages describe a git_ and github_ workflow for the NumPy_ -project. - -There are several different workflows here, for different ways of -working with *NumPy*. - -This is not a comprehensive git_ reference, it's just a workflow for our -own project. It's tailored to the github_ hosting service. You may well -find better or quicker ways of getting stuff done with git_, but these -should get you started. - -For general resources for learning git_ see :ref:`git-resources`. - -.. _install-git: - Install git =========== -Overview --------- - -================ ============= -Debian / Ubuntu ``sudo apt-get install git-core`` -Fedora ``sudo yum install git-core`` -Windows Download and install msysGit_ -OS X Use the git-osx-installer_ -================ ============= - -In detail ---------- - -See the git_ page for the most recent information. - -Have a look at the github_ install help pages available from `github help`_ - -There are good instructions here: http://book.git-scm.com/2_installing_git.html +Developing with git can be done entirely without github. Git is a distributed +version control system. In order to use git on your machine you must `install +it`_. .. include:: git_links.inc diff --git a/doc/source/dev/gitwash/git_links.inc b/doc/source/dev/gitwash/git_links.inc index cebbb3a67..f69a3cf62 100644 --- a/doc/source/dev/gitwash/git_links.inc +++ b/doc/source/dev/gitwash/git_links.inc @@ -10,10 +10,9 @@ .. git stuff .. _git: https://git-scm.com/ -.. _github: https://github.com +.. _github: https://github.com/numpy/numpy .. _github help: https://help.github.com -.. _msysgit: https://code.google.com/p/msysgit/downloads/list -.. _git-osx-installer: https://code.google.com/p/git-osx-installer/downloads/list +.. _`install it`: https://git-scm.com/downloads .. _subversion: http://subversion.tigris.org/ .. _git cheat sheet: http://cheat.errtheblog.com/s/git .. _pro git book: https://git-scm.com/book/ diff --git a/doc/source/dev/gitwash/index.rst b/doc/source/dev/gitwash/index.rst index b867bbd97..afbb5e019 100644 --- a/doc/source/dev/gitwash/index.rst +++ b/doc/source/dev/gitwash/index.rst @@ -1,7 +1,22 @@ .. _using-git: +.. _git-development: + +===================== + Git for development +===================== + +These pages describe a general git_ and github_ workflow. + +This is not a comprehensive git_ reference. It's tailored to the github_ +hosting service. You may well find better or quicker ways of getting stuff done +with git_, but these should get you started. + +For general resources for learning git_ see :ref:`git-resources`. + +Have a look at the github_ install help pages available from `github help`_ + +.. _install-git: -Working with *NumPy* source code -================================ Contents: @@ -10,6 +25,9 @@ Contents: git_intro following_latest - git_development - development_workflow + development_setup + configure_git + dot2_dot3 git_resources + +.. include:: git_links.inc diff --git a/doc/source/dev/governance/people.rst b/doc/source/dev/governance/people.rst index 7b8d3cab0..10af7f221 100644 --- a/doc/source/dev/governance/people.rst +++ b/doc/source/dev/governance/people.rst @@ -48,7 +48,7 @@ NumFOCUS Subcommittee * Jaime Fernández del Río -* Nathaniel Smith +* Sebastian Berg * External member: Thomas Caswell @@ -56,10 +56,7 @@ NumFOCUS Subcommittee Institutional Partners ---------------------- -* UC Berkeley (Stefan van der Walt) +* UC Berkeley (Stefan van der Walt, Matti Picus, Tyler Reddy, Sebastian Berg) +* Quansight (Ralf Gommers, Hameer Abbasi) -Document history ----------------- - -https://github.com/numpy/numpy/commits/master/doc/source/dev/governance/governance.rst diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index 825b93b53..3b409f5ca 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -2,14 +2,260 @@ Contributing to NumPy ##################### +Not a coder? Not a problem! NumPy is multi-faceted, and we can use a lot of help. +These are all activities we'd like to get help with (they're all important, so +we list them in alphabetical order): + +- Code maintenance and development +- Community coordination +- DevOps +- Developing educational content & narrative documentation +- Writing technical documentation +- Fundraising +- Project management +- Marketing +- Translating content +- Website design and development + +The rest of this document discusses working on the NumPy code base and documentation. +We're in the process of updating our descriptions of other activities and roles. +If you are interested in these other activities, please contact us! +You can do this via +the `numpy-discussion mailing list <https://scipy.org/scipylib/mailing-lists.html>`__, +or on GitHub (open an issue or comment on a relevant issue). These are our preferred +communication channels (open source is open by nature!), however if you prefer +to discuss in private first, please reach out to our community coordinators +at `numpy-team@googlegroups.com` or `numpy-team.slack.com` (send an email to +`numpy-team@googlegroups.com` for an invite the first time). + + +Development process - summary +============================= + +Here's the short summary, complete TOC links are below: + +1. If you are a first-time contributor: + + * Go to `https://github.com/numpy/numpy + <https://github.com/numpy/numpy>`_ and click the + "fork" button to create your own copy of the project. + + * Clone the project to your local computer:: + + git clone https://github.com/your-username/numpy.git + + * Change the directory:: + + cd numpy + + * Add the upstream repository:: + + git remote add upstream https://github.com/numpy/numpy.git + + * Now, `git remote -v` will show two remote repositories named: + + - ``upstream``, which refers to the ``numpy`` repository + - ``origin``, which refers to your personal fork + +2. Develop your contribution: + + * Pull the latest changes from upstream:: + + git checkout master + git pull upstream master + + * Create a branch for the feature you want to work on. Since the + branch name will appear in the merge message, use a sensible name + such as 'linspace-speedups':: + + git checkout -b linspace-speedups + + * Commit locally as you progress (``git add`` and ``git commit``) + Use a :ref:`properly formatted<writing-the-commit-message>` commit message, + write tests that fail before your change and pass afterward, run all the + :ref:`tests locally<development-environment>`. Be sure to document any + changed behavior in docstrings, keeping to the NumPy docstring + :ref:`standard<howto-document>`. + +3. To submit your contribution: + + * Push your changes back to your fork on GitHub:: + + git push origin linspace-speedups + + * Enter your GitHub username and password (repeat contributors or advanced + users can remove this step by connecting to GitHub with + :ref:`SSH<set-up-and-configure-a-github-account>` . + + * Go to GitHub. The new branch will show up with a green Pull Request + button. Make sure the title and message are clear, concise, and self- + explanatory. Then click the button to submit it. + + * If your commit introduces a new feature or changes functionality, post on + the `mailing list`_ to explain your changes. For bug fixes, documentation + updates, etc., this is generally not necessary, though if you do not get + any reaction, do feel free to ask for review. + +4. Review process: + + * Reviewers (the other developers and interested community members) will + write inline and/or general comments on your Pull Request (PR) to help + you improve its implementation, documentation and style. Every single + developer working on the project has their code reviewed, and we've come + to see it as friendly conversation from which we all learn and the + overall code quality benefits. Therefore, please don't let the review + discourage you from contributing: its only aim is to improve the quality + of project, not to criticize (we are, after all, very grateful for the + time you're donating!). + + * To update your PR, make your changes on your local repository, commit, + **run tests, and only if they succeed** push to your fork. As soon as + those changes are pushed up (to the same branch as before) the PR will + update automatically. If you have no idea how to fix the test failures, + you may push your changes anyway and ask for help in a PR comment. + + * Various continuous integration (CI) services are triggered after each PR + update to build the code, run unit tests, measure code coverage and check + coding style of your branch. The CI tests must pass before your PR can be + merged. If CI fails, you can find out why by clicking on the "failed" + icon (red cross) and inspecting the build and test log. To avoid overuse + and waste of this resource, + :ref:`test your work<recommended-development-setup>` locally before + committing. + + * A PR must be **approved** by at least one core team member before merging. + Approval means the core team member has carefully reviewed the changes, + and the PR is ready for merging. + +5. Document changes + + Beyond changes to a functions docstring and possible description in the + general documentation, if your change introduces any user-facing + modifications, update the current release notes under + ``doc/release/X.XX-notes.rst`` + + If your change introduces a deprecation, make sure to discuss this first on + GitHub or the mailing list first. If agreement on the deprecation is + reached, follow `NEP 23 deprecation policy <http://www.numpy.org/neps/ + nep-0023-backwards-compatibility.html>`_ to add the deprecation. + +6. Cross referencing issues + + If the PR relates to any issues, you can add the text ``xref gh-xxxx`` where + ``xxxx`` is the number of the issue to github comments. Likewise, if the PR + solves an issue, replace the ``xref`` with ``closes``, ``fixes`` or any of + the other flavors `github accepts <https://help.github.com/en/articles/ + closing-issues-using-keywords>`_. + + In the source code, be sure to preface any issue or PR reference with + ``gh-xxxx``. + +For a more detailed discussion, read on and follow the links at the bottom of +this page. + +Divergence between ``upstream/master`` and your feature branch +-------------------------------------------------------------- + +If GitHub indicates that the branch of your Pull Request can no longer +be merged automatically, you have to incorporate changes that have been made +since you started into your branch. Our recommended way to do this is to +:ref:`rebase on master<rebasing-on-master>`. + +Guidelines +---------- + +* All code should have tests (see `test coverage`_ below for more details). +* All code should be `documented <https://numpydoc.readthedocs.io/ + en/latest/format.html#docstring-standard>`_. +* No changes are ever committed without review and approval by a core + team member.Please ask politely on the PR or on the `mailing list`_ if you + get no response to your pull request within a week. + +Stylistic Guidelines +-------------------- + +* Set up your editor to follow `PEP 8 <https://www.python.org/dev/peps/ + pep-0008/>`_ (remove trailing white space, no tabs, etc.). Check code with + pyflakes / flake8. + +* Use numpy data types instead of strings (``np.uint8`` instead of + ``"uint8"``). + +* Use the following import conventions:: + + import numpy as np + +* For C code, see the :ref:`numpy-c-style-guide<style_guide>` + + +Test coverage +------------- + +Pull requests (PRs) that modify code should either have new tests, or modify existing +tests to fail before the PR and pass afterwards. You should :ref:`run the tests +<development-environment>` before pushing a PR. + +Tests for a module should ideally cover all code in that module, +i.e., statement coverage should be at 100%. + +To measure the test coverage, install +`pytest-cov <https://pytest-cov.readthedocs.io/en/latest/>`__ +and then run:: + + $ python runtests.py --coverage + +This will create a report in ``build/coverage``, which can be viewed with:: + + $ firefox build/coverage/index.html + +Building docs +------------- + +To build docs, run ``make`` from the ``doc`` directory. ``make help`` lists +all targets. For example, to build the HTML documentation, you can run: + +.. code:: sh + + make html + +Then, all the HTML files will be generated in ``doc/build/html/``. +Since the documentation is based on docstrings, the appropriate version of +numpy must be installed in the host python used to run sphinx. + +Requirements +~~~~~~~~~~~~ + +`Sphinx <http://www.sphinx-doc.org/en/stable/>`__ is needed to build +the documentation. Matplotlib and SciPy are also required. + +Fixing Warnings +~~~~~~~~~~~~~~~ + +- "citation not found: R###" There is probably an underscore after a + reference in the first line of a docstring (e.g. [1]\_). Use this + method to find the source file: $ cd doc/build; grep -rin R#### + +- "Duplicate citation R###, other instance in..."" There is probably a + [2] without a [1] in one of the docstrings + +Development process - details +============================= + +The rest of the story + .. toctree:: - :maxdepth: 3 + :maxdepth: 2 conduct/code_of_conduct - gitwash/index + Git Basics <gitwash/index> development_environment + development_workflow + ../benchmarking style_guide releasing governance/index -For core developers: see :ref:`development-workflow`. +NumPy-specific workflow is in :ref:`numpy-development-workflow +<development-workflow>`. + +.. _`mailing list`: https://mail.python.org/mailman/listinfo/numpy-devel diff --git a/doc/source/dev/gitwash/pull_button.png b/doc/source/dev/pull_button.png Binary files differindex e5031681b..e5031681b 100644 --- a/doc/source/dev/gitwash/pull_button.png +++ b/doc/source/dev/pull_button.png diff --git a/doc/source/docs/howto_build_docs.rst b/doc/source/docs/howto_build_docs.rst index cdf490c37..4bb7628c1 100644 --- a/doc/source/docs/howto_build_docs.rst +++ b/doc/source/docs/howto_build_docs.rst @@ -5,7 +5,7 @@ Building the NumPy API and reference docs ========================================= We currently use Sphinx_ for generating the API and reference -documentation for NumPy. You will need Sphinx 1.0.1 or newer. +documentation for NumPy. You will need Sphinx 1.8.3 or newer. If you only want to get the documentation, note that pre-built versions can be found at @@ -30,11 +30,9 @@ In addition, building the documentation requires the Sphinx extension `plot_directive`, which is shipped with Matplotlib_. This Sphinx extension can be installed by installing Matplotlib. You will also need python3.6. -Since large parts of the main documentation are stored in -docstrings, you will need to first build NumPy, and install it so -that the correct version is imported by - - >>> import numpy +Since large parts of the main documentation are obtained from numpy via +``import numpy`` and examining the docstrings, you will need to first build +NumPy, and install it so that the correct version is imported. Note that you can eg. install NumPy to a temporary location and set the PYTHONPATH environment variable appropriately. @@ -46,8 +44,11 @@ generate the docs, so write:: make html in the ``doc/`` directory. If all goes well, this will generate a -``build/html`` subdirectory containing the built documentation. Note -that building the documentation on Windows is currently not actively +``build/html`` subdirectory containing the built documentation. If you get +a message about ``installed numpy != current repo git version``, you must +either override the check by setting ``GITVER`` or re-install NumPy. + +Note that building the documentation on Windows is currently not actively supported, though it should be possible. (See Sphinx_ documentation for more information.) diff --git a/doc/source/f2py/run_main_session.dat b/doc/source/f2py/run_main_session.dat index b9a7e1b0d..be6cacd22 100644 --- a/doc/source/f2py/run_main_session.dat +++ b/doc/source/f2py/run_main_session.dat @@ -8,7 +8,7 @@ Post-processing... Building modules... Building module "scalar"... Wrote C/API module "scalar" to file "./scalarmodule.c" ->>> printr(r) +>>> print(r) {'scalar': {'h': ['/home/users/pearu/src_cvs/f2py/src/fortranobject.h'], 'csrc': ['./scalarmodule.c', '/home/users/pearu/src_cvs/f2py/src/fortranobject.c']}} diff --git a/doc/source/f2py/usage.rst b/doc/source/f2py/usage.rst index 0f5068e0e..5043ec430 100644 --- a/doc/source/f2py/usage.rst +++ b/doc/source/f2py/usage.rst @@ -214,32 +214,7 @@ Python module ``numpy.f2py`` The current Python interface to the ``f2py`` module is not mature and may change in the future. -The following functions are provided by the ``numpy.f2py`` module: -``run_main(<list>)`` - Equivalent to running:: +.. automodule:: numpy.f2py + :members: - f2py <args> - - where ``<args>=string.join(<list>,' ')``, but in Python. Unless - ``-h`` is used, this function returns a dictionary containing - information on generated modules and their dependencies on source - files. For example, the command ``f2py -m scalar scalar.f`` can be - executed from Python as follows - - .. include:: run_main_session.dat - :literal: - - You cannot build extension modules with this function, that is, - using ``-c`` is not allowed. Use ``compile`` command instead, see - below. - -``compile(source, modulename='untitled', extra_args='', verbose=1, source_fn=None)`` - Build extension module from Fortran 77 source string ``source``. - Return 0 if successful. - Note that this function actually calls ``f2py -c ..`` from shell to - ensure safety of the current Python process. - For example, - - .. include:: compile_session.dat - :literal: diff --git a/doc/source/reference/alignment.rst b/doc/source/reference/alignment.rst index c749972b4..ebc8f353c 100644 --- a/doc/source/reference/alignment.rst +++ b/doc/source/reference/alignment.rst @@ -34,6 +34,14 @@ datatype is implemented as ``struct { float real, imag; }``. This has "true" alignment of 4 and "uint" alignment of 8 (equal to the true alignment of ``uint64``). +Some cases where uint and true alignment are different (default gcc linux): + arch type true-aln uint-aln + ---- ---- -------- -------- + x86_64 complex64 4 8 + x86_64 float128 16 8 + x86 float96 4 - + + Variables in Numpy which control and describe alignment ------------------------------------------------------- @@ -82,17 +90,15 @@ Here is how the variables above are used: appropriate N. Otherwise numpy copies by doing ``memcpy(dst, src, N)``. 5. Nditer code: Since this often calls the strided copy code, it must check for "uint alignment". - 6. Cast code: if the array is "uint aligned" this will essentially do - ``*dst = CASTFUNC(*src)``. If not, it does + 6. Cast code: This checks for "true" alignment, as it does + ``*dst = CASTFUNC(*src)`` if aligned. Otherwise, it does ``memmove(srcval, src); dstval = CASTFUNC(srcval); memmove(dst, dstval)`` where dstval/srcval are aligned. -Note that in principle, only "true alignment" is required for casting code. -However, because the casting code and copy code are deeply intertwined they -both use "uint" alignment. This should be safe assuming uint alignment is -always larger than true alignment, though it can cause unnecessary buffering if -an array is "true aligned" but not "uint aligned". If there is ever a big -rewrite of this code it would be good to allow them to use different -alignments. +Note that the strided-copy and strided-cast code are deeply intertwined and so +any arrays being processed by them must be both uint and true aligned, even +though the copy-code only needs uint alignment and the cast code only true +alignment. If there is ever a big rewrite of this code it would be good to +allow them to use different alignments. diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index f17cb932a..39410b2a4 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -6,8 +6,15 @@ Standard array subclasses .. currentmodule:: numpy -The :class:`ndarray` in NumPy is a "new-style" Python -built-in-type. Therefore, it can be inherited from (in Python or in C) +.. note:: + + Subclassing a ``numpy.ndarray`` is possible but if your goal is to create + an array with *modified* behavior, as do dask arrays for distributed + computation and cupy arrays for GPU-based computation, subclassing is + discouraged. Instead, using numpy's + :ref:`dispatch mechanism <basics.dispatch>` is recommended. + +The :class:`ndarray` can be inherited from (in Python or in C) if desired. Therefore, it can form a foundation for many useful classes. Often whether to sub-class the array object or to simply use the core array component as an internal part of a new class is a @@ -43,10 +50,6 @@ NumPy provides several hooks that classes can customize: .. versionadded:: 1.13 - .. note:: The API is `provisional - <https://docs.python.org/3/glossary.html#term-provisional-api>`_, - i.e., we do not yet guarantee backward compatibility. - Any class, ndarray subclass or not, can define this method or set it to :obj:`None` in order to override the behavior of NumPy's ufuncs. This works quite similarly to Python's ``__mul__`` and other binary operation routines. @@ -79,7 +82,7 @@ NumPy provides several hooks that classes can customize: :func:`~numpy.matmul`, which currently is not a Ufunc, but could be relatively easily be rewritten as a (set of) generalized Ufuncs. The same may happen with functions such as :func:`~numpy.median`, - :func:`~numpy.min`, and :func:`~numpy.argsort`. + :func:`~numpy.amin`, and :func:`~numpy.argsort`. Like with some other special methods in python, such as ``__hash__`` and ``__iter__``, it is possible to indicate that your class does *not* @@ -151,6 +154,121 @@ NumPy provides several hooks that classes can customize: :func:`__array_prepare__`, :data:`__array_priority__` mechanism described below for ufuncs (which may eventually be deprecated). +.. py:method:: class.__array_function__(func, types, args, kwargs) + + .. versionadded:: 1.16 + + .. note:: + + - In NumPy 1.17, the protocol is enabled by default, but can be disabled + with ``NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=0``. + - In NumPy 1.16, you need to set the environment variable + ``NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=1`` before importing NumPy to use + NumPy function overrides. + - Eventually, expect to ``__array_function__`` to always be enabled. + + - ``func`` is an arbitrary callable exposed by NumPy's public API, + which was called in the form ``func(*args, **kwargs)``. + - ``types`` is a `collection <collections.abc.Collection>`_ + of unique argument types from the original NumPy function call that + implement ``__array_function__``. + - The tuple ``args`` and dict ``kwargs`` are directly passed on from the + original call. + + As a convenience for ``__array_function__`` implementors, ``types`` + provides all argument types with an ``'__array_function__'`` attribute. + This allows implementors to quickly identify cases where they should defer + to ``__array_function__`` implementations on other arguments. + Implementations should not rely on the iteration order of ``types``. + + Most implementations of ``__array_function__`` will start with two + checks: + + 1. Is the given function something that we know how to overload? + 2. Are all arguments of a type that we know how to handle? + + If these conditions hold, ``__array_function__`` should return the result + from calling its implementation for ``func(*args, **kwargs)``. Otherwise, + it should return the sentinel value ``NotImplemented``, indicating that the + function is not implemented by these types. + + There are no general requirements on the return value from + ``__array_function__``, although most sensible implementations should + probably return array(s) with the same type as one of the function's + arguments. + + It may also be convenient to define a custom decorators (``implements`` + below) for registering ``__array_function__`` implementations. + + .. code:: python + + HANDLED_FUNCTIONS = {} + + class MyArray: + def __array_function__(self, func, types, args, kwargs): + if func not in HANDLED_FUNCTIONS: + return NotImplemented + # Note: this allows subclasses that don't override + # __array_function__ to handle MyArray objects + if not all(issubclass(t, MyArray) for t in types): + return NotImplemented + return HANDLED_FUNCTIONS[func](*args, **kwargs) + + def implements(numpy_function): + """Register an __array_function__ implementation for MyArray objects.""" + def decorator(func): + HANDLED_FUNCTIONS[numpy_function] = func + return func + return decorator + + @implements(np.concatenate) + def concatenate(arrays, axis=0, out=None): + ... # implementation of concatenate for MyArray objects + + @implements(np.broadcast_to) + def broadcast_to(array, shape): + ... # implementation of broadcast_to for MyArray objects + + Note that it is not required for ``__array_function__`` implementations to + include *all* of the corresponding NumPy function's optional arguments + (e.g., ``broadcast_to`` above omits the irrelevant ``subok`` argument). + Optional arguments are only passed in to ``__array_function__`` if they + were explicitly used in the NumPy function call. + + Just like the case for builtin special methods like ``__add__``, properly + written ``__array_function__`` methods should always return + ``NotImplemented`` when an unknown type is encountered. Otherwise, it will + be impossible to correctly override NumPy functions from another object + if the operation also includes one of your objects. + + For the most part, the rules for dispatch with ``__array_function__`` + match those for ``__array_ufunc__``. In particular: + + - NumPy will gather implementations of ``__array_function__`` from all + specified inputs and call them in order: subclasses before + superclasses, and otherwise left to right. Note that in some edge cases + involving subclasses, this differs slightly from the + `current behavior <https://bugs.python.org/issue30140>`_ of Python. + - Implementations of ``__array_function__`` indicate that they can + handle the operation by returning any value other than + ``NotImplemented``. + - If all ``__array_function__`` methods return ``NotImplemented``, + NumPy will raise ``TypeError``. + + If no ``__array_function__`` methods exists, NumPy will default to calling + its own implementation, intended for use on NumPy arrays. This case arises, + for example, when all array-like arguments are Python numbers or lists. + (NumPy arrays do have a ``__array_function__`` method, given below, but it + always returns ``NotImplemented`` if any argument other than a NumPy array + subclass implements ``__array_function__``.) + + One deviation from the current behavior of ``__array_ufunc__`` is that + NumPy will only call ``__array_function__`` on the *first* argument of each + unique type. This matches Python's `rule for calling reflected methods + <https://docs.python.org/3/reference/datamodel.html#object.__ror__>`_, and + this ensures that checking overloads has acceptable performance even when + there are a large number of overloaded arguments. + .. py:method:: class.__array_finalize__(obj) This method is called whenever the system internally allocates a @@ -452,7 +570,7 @@ object, then the Python code:: some code involving val ... -calls ``val = myiter.next()`` repeatedly until :exc:`StopIteration` is +calls ``val = next(myiter)`` repeatedly until :exc:`StopIteration` is raised by the iterator. There are several ways to iterate over an array that may be useful: default iteration, flat iteration, and :math:`N`-dimensional enumeration. diff --git a/doc/source/reference/arrays.dtypes.rst b/doc/source/reference/arrays.dtypes.rst index f2072263f..ab743a8ee 100644 --- a/doc/source/reference/arrays.dtypes.rst +++ b/doc/source/reference/arrays.dtypes.rst @@ -14,7 +14,7 @@ following aspects of the data: 1. Type of the data (integer, float, Python object, etc.) 2. Size of the data (how many bytes is in *e.g.* the integer) 3. Byte order of the data (:term:`little-endian` or :term:`big-endian`) -4. If the data type is :term:`structured`, an aggregate of other +4. If the data type is :term:`structured data type`, an aggregate of other data types, (*e.g.*, describing an array item consisting of an integer and a float), @@ -42,7 +42,7 @@ needed in NumPy. pair: dtype; field Structured data types are formed by creating a data type whose -:term:`fields` contain other data types. Each field has a name by +:term:`field` contain other data types. Each field has a name by which it can be :ref:`accessed <arrays.indexing.fields>`. The parent data type should be of sufficient size to contain all its fields; the parent is nearly always based on the :class:`void` type which allows @@ -145,7 +145,7 @@ Array-scalar types This is true for their sub-classes as well. Note that not all data-type information can be supplied with a - type-object: for example, :term:`flexible` data-types have + type-object: for example, `flexible` data-types have a default *itemsize* of 0, and require an explicitly given size to be useful. @@ -511,7 +511,7 @@ Endianness of this data: dtype.byteorder -Information about sub-data-types in a :term:`structured` data type: +Information about sub-data-types in a :term:`structured data type`: .. autosummary:: :toctree: generated/ @@ -538,6 +538,7 @@ Attributes providing additional information: dtype.isnative dtype.descr dtype.alignment + dtype.base Methods diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index 62d36e28c..8ec8d8330 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -3,6 +3,10 @@ Indexing ======== +.. seealso:: + + :ref:`Indexing basics <basics.indexing>` + .. sectionauthor:: adapted from "Guide to NumPy" by Travis E. Oliphant .. currentmodule:: numpy @@ -57,6 +61,17 @@ interpreted as counting from the end of the array (*i.e.*, if All arrays generated by basic slicing are always :term:`views <view>` of the original array. +.. note:: + + NumPy slicing creates a :term:`view` instead of a copy as in the case of + builtin Python sequences such as string, tuple and list. + Care must be taken when extracting + a small portion from a large array which becomes useless after the + extraction, because the small portion extracted contains a reference + to the large original array whose memory will not be released until + all arrays derived from it are garbage-collected. In such cases an + explicit ``copy()`` is recommended. + The standard rules of sequence slicing apply to basic slicing on a per-dimension basis (including using a step index). Some useful concepts to remember include: @@ -111,9 +126,10 @@ concepts to remember include: [5], [6]]]) -- :const:`Ellipsis` expand to the number of ``:`` objects needed to - make a selection tuple of the same length as ``x.ndim``. There may - only be a single ellipsis present. +- :const:`Ellipsis` expands to the number of ``:`` objects needed for the + selection tuple to index all dimensions. In most cases, this means that + length of the expanded selection tuple is ``x.ndim``. There may only be a + single ellipsis present. .. admonition:: Example @@ -513,14 +529,10 @@ only the part of the data in the specified field. Also :ref:`record array <arrays.classes.rec>` scalars can be "indexed" this way. Indexing into a structured array can also be done with a list of field names, -*e.g.* ``x[['field-name1','field-name2']]``. Currently this returns a new -array containing a copy of the values in the fields specified in the list. -As of NumPy 1.7, returning a copy is being deprecated in favor of returning -a view. A copy will continue to be returned for now, but a FutureWarning -will be issued when writing to the copy. If you depend on the current -behavior, then we suggest copying the returned array explicitly, i.e. use -x[['field-name1','field-name2']].copy(). This will work with both past and -future versions of NumPy. +*e.g.* ``x[['field-name1','field-name2']]``. As of NumPy 1.16 this returns a +view containing only those fields. In older versions of numpy it returned a +copy. See the user guide section on :ref:`structured_arrays` for more +information on multifield indexing. If the accessed field is a sub-array, the dimensions of the sub-array are appended to the shape of the result. diff --git a/doc/source/reference/arrays.ndarray.rst b/doc/source/reference/arrays.ndarray.rst index 306d22f43..8f431bc9c 100644 --- a/doc/source/reference/arrays.ndarray.rst +++ b/doc/source/reference/arrays.ndarray.rst @@ -9,7 +9,7 @@ The N-dimensional array (:class:`ndarray`) An :class:`ndarray` is a (usually fixed-size) multidimensional container of items of the same type and size. The number of dimensions and items in an array is defined by its :attr:`shape <ndarray.shape>`, -which is a :class:`tuple` of *N* positive integers that specify the +which is a :class:`tuple` of *N* non-negative integers that specify the sizes of each dimension. The type of items in the array is specified by a separate :ref:`data-type object (dtype) <arrays.dtypes>`, one of which is associated with each ndarray. @@ -82,10 +82,12 @@ Indexing arrays Arrays can be indexed using an extended Python slicing syntax, ``array[selection]``. Similar syntax is also used for accessing -fields in a :ref:`structured array <arrays.dtypes.field>`. +fields in a :term:`structured data type`. .. seealso:: :ref:`Array Indexing <arrays.indexing>`. +.. _memory-layout: + Internal memory layout of an ndarray ==================================== @@ -127,7 +129,7 @@ strided scheme, and correspond to memory that can be *addressed* by the strides: where :math:`d_j` `= self.shape[j]`. Both the C and Fortran orders are :term:`contiguous`, *i.e.,* -:term:`single-segment`, memory layouts, in which every part of the +single-segment, memory layouts, in which every part of the memory block can be accessed by some combination of the indices. While a C-style and Fortran-style contiguous array, which has the corresponding @@ -143,14 +145,15 @@ different. This can happen in two cases: considered C-style and Fortran-style contiguous. Point 1. means that ``self`` and ``self.squeeze()`` always have the same -contiguity and :term:`aligned` flags value. This also means that even a high -dimensional array could be C-style and Fortran-style contiguous at the same -time. +contiguity and ``aligned`` flags value. This also means +that even a high dimensional array could be C-style and Fortran-style +contiguous at the same time. .. index:: aligned An array is considered aligned if the memory offsets for all elements and the -base offset itself is a multiple of `self.itemsize`. +base offset itself is a multiple of `self.itemsize`. Understanding +`memory-alignment` leads to better performance on most hardware. .. note:: @@ -409,6 +412,7 @@ be performed. .. autosummary:: :toctree: generated/ + ndarray.max ndarray.argmax ndarray.min ndarray.argmin @@ -440,7 +444,7 @@ Each of the arithmetic operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``, ``divmod()``, ``**`` or ``pow()``, ``<<``, ``>>``, ``&``, ``^``, ``|``, ``~``) and the comparisons (``==``, ``<``, ``>``, ``<=``, ``>=``, ``!=``) is equivalent to the corresponding -:term:`universal function` (or :term:`ufunc` for short) in NumPy. For +universal function (or :term:`ufunc` for short) in NumPy. For more information, see the section on :ref:`Universal Functions <ufuncs>`. @@ -461,12 +465,12 @@ Truth value of an array (:func:`bool()`): .. autosummary:: :toctree: generated/ - ndarray.__nonzero__ + ndarray.__bool__ .. note:: Truth-value testing of an array invokes - :meth:`ndarray.__nonzero__`, which raises an error if the number of + :meth:`ndarray.__bool__`, which raises an error if the number of elements in the array is larger than 1, because the truth value of such arrays is ambiguous. Use :meth:`.any() <ndarray.any>` and :meth:`.all() <ndarray.all>` instead to be clear about what is meant @@ -492,7 +496,6 @@ Arithmetic: ndarray.__add__ ndarray.__sub__ ndarray.__mul__ - ndarray.__div__ ndarray.__truediv__ ndarray.__floordiv__ ndarray.__mod__ @@ -527,7 +530,6 @@ Arithmetic, in-place: ndarray.__iadd__ ndarray.__isub__ ndarray.__imul__ - ndarray.__idiv__ ndarray.__itruediv__ ndarray.__ifloordiv__ ndarray.__imod__ @@ -597,19 +599,17 @@ Container customization: (see :ref:`Indexing <arrays.indexing>`) ndarray.__setitem__ ndarray.__contains__ -Conversion; the operations :func:`complex()`, :func:`int()`, -:func:`long()`, :func:`float()`, :func:`oct()`, and -:func:`hex()`. They work only on arrays that have one element in them +Conversion; the operations :func:`int()`, :func:`float()` and +:func:`complex()`. +. They work only on arrays that have one element in them and return the appropriate scalar. .. autosummary:: :toctree: generated/ ndarray.__int__ - ndarray.__long__ ndarray.__float__ - ndarray.__oct__ - ndarray.__hex__ + ndarray.__complex__ String representations: diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index 9c4f05f75..d27d61e2c 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -177,7 +177,7 @@ Any Python object: .. note:: - The data actually stored in :term:`object arrays <object array>` + The data actually stored in object arrays (*i.e.*, arrays having dtype :class:`object_`) are references to Python objects, not the objects themselves. Hence, object arrays behave more like usual Python :class:`lists <list>`, in the sense @@ -188,8 +188,10 @@ Any Python object: on item access, but instead returns the actual object that the array item refers to. -The following data types are :term:`flexible`. They have no predefined -size: the data they describe can be of different length in different +.. index:: flexible + +The following data types are **flexible**: they have no predefined +size and the data they describe can be of different length in different arrays. (In the character codes ``#`` is an integer denoting how many elements the data type consists of.) diff --git a/doc/source/reference/c-api.array.rst b/doc/source/reference/c-api/array.rst index 76aa680ae..a2b56cee7 100644 --- a/doc/source/reference/c-api.array.rst +++ b/doc/source/reference/c-api/array.rst @@ -20,27 +20,44 @@ Array API Array structure and data access ------------------------------- -These macros all access the :c:type:`PyArrayObject` structure members. The input -argument, arr, can be any :c:type:`PyObject *<PyObject>` that is directly interpretable -as a :c:type:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` and its -sub-types). +These macros access the :c:type:`PyArrayObject` structure members and are +defined in ``ndarraytypes.h``. The input argument, *arr*, can be any +:c:type:`PyObject *<PyObject>` that is directly interpretable as a +:c:type:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` +and itssub-types). .. c:function:: int PyArray_NDIM(PyArrayObject *arr) The number of dimensions in the array. -.. c:function:: npy_intp *PyArray_DIMS(PyArrayObject *arr) +.. c:function:: int PyArray_FLAGS(PyArrayObject* arr) - Returns a pointer to the dimensions/shape of the array. The - number of elements matches the number of dimensions - of the array. + Returns an integer representing the :ref:`array-flags<array-flags>`. -.. c:function:: npy_intp *PyArray_SHAPE(PyArrayObject *arr) +.. c:function:: int PyArray_TYPE(PyArrayObject* arr) + + Return the (builtin) typenumber for the elements of this array. + +.. c:function:: int PyArray_SETITEM( \ + PyArrayObject* arr, void* itemptr, PyObject* obj) + + Convert obj and place it in the ndarray, *arr*, at the place + pointed to by itemptr. Return -1 if an error occurs or 0 on + success. + +.. c:function:: void PyArray_ENABLEFLAGS(PyArrayObject* arr, int flags) .. versionadded:: 1.7 - A synonym for PyArray_DIMS, named to be consistent with the - 'shape' usage within Python. + Enables the specified array flags. This function does no validation, + and assumes that you know what you're doing. + +.. c:function:: void PyArray_CLEARFLAGS(PyArrayObject* arr, int flags) + + .. versionadded:: 1.7 + + Clears the specified array flags. This function does no validation, + and assumes that you know what you're doing. .. c:function:: void *PyArray_DATA(PyArrayObject *arr) @@ -53,6 +70,19 @@ sub-types). array then be sure you understand how to access the data in the array to avoid memory and/or alignment problems. +.. c:function:: npy_intp *PyArray_DIMS(PyArrayObject *arr) + + Returns a pointer to the dimensions/shape of the array. The + number of elements matches the number of dimensions + of the array. Can return ``NULL`` for 0-dimensional arrays. + +.. c:function:: npy_intp *PyArray_SHAPE(PyArrayObject *arr) + + .. versionadded:: 1.7 + + A synonym for :c:func:`PyArray_DIMS`, named to be consistent with the + `shape <numpy.ndarray.shape>` usage within Python. + .. c:function:: npy_intp *PyArray_STRIDES(PyArrayObject* arr) Returns a pointer to the strides of the array. The @@ -67,6 +97,27 @@ sub-types). Return the stride in the *n* :math:`^{\textrm{th}}` dimension. +.. c:function:: npy_intp PyArray_ITEMSIZE(PyArrayObject* arr) + + Return the itemsize for the elements of this array. + + Note that, in the old API that was deprecated in version 1.7, this function + had the return type ``int``. + +.. c:function:: npy_intp PyArray_SIZE(PyArrayObject* arr) + + Returns the total size (in number of elements) of the array. + +.. c:function:: npy_intp PyArray_Size(PyArrayObject* obj) + + Returns 0 if *obj* is not a sub-class of ndarray. Otherwise, + returns the total number of elements in the array. Safer version + of :c:func:`PyArray_SIZE` (*obj*). + +.. c:function:: npy_intp PyArray_NBYTES(PyArrayObject* arr) + + Returns the total number of bytes consumed by the array. + .. c:function:: PyObject *PyArray_BASE(PyArrayObject* arr) This returns the base object of the array. In most cases, this @@ -93,60 +144,12 @@ sub-types). A synonym for PyArray_DESCR, named to be consistent with the 'dtype' usage within Python. -.. c:function:: void PyArray_ENABLEFLAGS(PyArrayObject* arr, int flags) - - .. versionadded:: 1.7 - - Enables the specified array flags. This function does no validation, - and assumes that you know what you're doing. - -.. c:function:: void PyArray_CLEARFLAGS(PyArrayObject* arr, int flags) - - .. versionadded:: 1.7 - - Clears the specified array flags. This function does no validation, - and assumes that you know what you're doing. - -.. c:function:: int PyArray_FLAGS(PyArrayObject* arr) - -.. c:function:: npy_intp PyArray_ITEMSIZE(PyArrayObject* arr) - - Return the itemsize for the elements of this array. - - Note that, in the old API that was deprecated in version 1.7, this function - had the return type ``int``. - -.. c:function:: int PyArray_TYPE(PyArrayObject* arr) - - Return the (builtin) typenumber for the elements of this array. - .. c:function:: PyObject *PyArray_GETITEM(PyArrayObject* arr, void* itemptr) - Get a Python object of a builtin type from the ndarray, *arr*, + Get a Python object of a builtin type from the ndarray, *arr*, at the location pointed to by itemptr. Return ``NULL`` on failure. - - `numpy.ndarray.item` is identical to PyArray_GETITEM. - -.. c:function:: int PyArray_SETITEM( \ - PyArrayObject* arr, void* itemptr, PyObject* obj) - - Convert obj and place it in the ndarray, *arr*, at the place - pointed to by itemptr. Return -1 if an error occurs or 0 on - success. - -.. c:function:: npy_intp PyArray_SIZE(PyArrayObject* arr) - - Returns the total size (in number of elements) of the array. - -.. c:function:: npy_intp PyArray_Size(PyArrayObject* obj) - - Returns 0 if *obj* is not a sub-class of ndarray. Otherwise, - returns the total number of elements in the array. Safer version - of :c:func:`PyArray_SIZE` (*obj*). - -.. c:function:: npy_intp PyArray_NBYTES(PyArrayObject* arr) - Returns the total number of bytes consumed by the array. + `numpy.ndarray.item` is identical to PyArray_GETITEM. Data access @@ -199,8 +202,8 @@ From scratch ^^^^^^^^^^^^ .. c:function:: PyObject* PyArray_NewFromDescr( \ - PyTypeObject* subtype, PyArray_Descr* descr, int nd, npy_intp* dims, \ - npy_intp* strides, void* data, int flags, PyObject* obj) + PyTypeObject* subtype, PyArray_Descr* descr, int nd, npy_intp const* dims, \ + npy_intp const* strides, void* data, int flags, PyObject* obj) This function steals a reference to *descr*. The easiest way to get one is using :c:func:`PyArray_DescrFromType`. @@ -219,7 +222,7 @@ From scratch If *data* is ``NULL``, then new unitinialized memory will be allocated and *flags* can be non-zero to indicate a Fortran-style contiguous array. Use - :c:ref:`PyArray_FILLWBYTE` to initialze the memory. + :c:func:`PyArray_FILLWBYTE` to initialize the memory. If *data* is not ``NULL``, then it is assumed to point to the memory to be used for the array and the *flags* argument is used as the @@ -266,8 +269,9 @@ From scratch base-class array. .. c:function:: PyObject* PyArray_New( \ - PyTypeObject* subtype, int nd, npy_intp* dims, int type_num, \ - npy_intp* strides, void* data, int itemsize, int flags, PyObject* obj) + PyTypeObject* subtype, int nd, npy_intp const* dims, int type_num, \ + npy_intp const* strides, void* data, int itemsize, int flags, \ + PyObject* obj) This is similar to :c:func:`PyArray_NewFromDescr` (...) except you specify the data-type descriptor with *type_num* and *itemsize*, @@ -288,29 +292,40 @@ From scratch are passed in they must be consistent with the dimensions, the itemsize, and the data of the array. -.. c:function:: PyObject* PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) +.. c:function:: PyObject* PyArray_SimpleNew(int nd, npy_intp const* dims, int typenum) Create a new uninitialized array of type, *typenum*, whose size in - each of *nd* dimensions is given by the integer array, *dims*. - This function cannot be used to create a flexible-type array (no - itemsize given). + each of *nd* dimensions is given by the integer array, *dims*.The memory + for the array is uninitialized (unless typenum is :c:data:`NPY_OBJECT` + in which case each element in the array is set to NULL). The + *typenum* argument allows specification of any of the builtin + data-types such as :c:data:`NPY_FLOAT` or :c:data:`NPY_LONG`. The + memory for the array can be set to zero if desired using + :c:func:`PyArray_FILLWBYTE` (return_object, 0).This function cannot be + used to create a flexible-type array (no itemsize given). .. c:function:: PyObject* PyArray_SimpleNewFromData( \ - int nd, npy_intp* dims, int typenum, void* data) + int nd, npy_intp const* dims, int typenum, void* data) Create an array wrapper around *data* pointed to by the given pointer. The array flags will have a default that the data area is well-behaved and C-style contiguous. The shape of the array is given by the *dims* c-array of length *nd*. The data-type of the - array is indicated by *typenum*. + array is indicated by *typenum*. If data comes from another + reference-counted Python object, the reference count on this object + should be increased after the pointer is passed in, and the base member + of the returned ndarray should point to the Python object that owns + the data. This will ensure that the provided memory is not + freed while the returned array is in existence. To free memory as soon + as the ndarray is deallocated, set the OWNDATA flag on the returned ndarray. .. c:function:: PyObject* PyArray_SimpleNewFromDescr( \ - int nd, npy_intp* dims, PyArray_Descr* descr) + int nd, npy_int const* dims, PyArray_Descr* descr) - This function steals a reference to *descr* if it is not NULL. + This function steals a reference to *descr*. - Create a new array with the provided data-type descriptor, *descr* - , of the shape determined by *nd* and *dims*. + Create a new array with the provided data-type descriptor, *descr*, + of the shape determined by *nd* and *dims*. .. c:function:: PyArray_FILLWBYTE(PyObject* obj, int val) @@ -319,7 +334,7 @@ From scratch This macro calls memset, so obj must be contiguous. .. c:function:: PyObject* PyArray_Zeros( \ - int nd, npy_intp* dims, PyArray_Descr* dtype, int fortran) + int nd, npy_intp const* dims, PyArray_Descr* dtype, int fortran) Construct a new *nd* -dimensional array with shape given by *dims* and data type given by *dtype*. If *fortran* is non-zero, then a @@ -328,13 +343,13 @@ From scratch corresponds to :c:type:`NPY_OBJECT` ). .. c:function:: PyObject* PyArray_ZEROS( \ - int nd, npy_intp* dims, int type_num, int fortran) + int nd, npy_intp const* dims, int type_num, int fortran) Macro form of :c:func:`PyArray_Zeros` which takes a type-number instead of a data-type object. .. c:function:: PyObject* PyArray_Empty( \ - int nd, npy_intp* dims, PyArray_Descr* dtype, int fortran) + int nd, npy_intp const* dims, PyArray_Descr* dtype, int fortran) Construct a new *nd* -dimensional array with shape given by *dims* and data type given by *dtype*. If *fortran* is non-zero, then a @@ -344,7 +359,7 @@ From scratch filled with :c:data:`Py_None`. .. c:function:: PyObject* PyArray_EMPTY( \ - int nd, npy_intp* dims, int typenum, int fortran) + int nd, npy_intp const* dims, int typenum, int fortran) Macro form of :c:func:`PyArray_Empty` which takes a type-number, *typenum*, instead of a data-type object. @@ -510,6 +525,11 @@ From other objects :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| :c:data:`NPY_ARRAY_WRITEABLE` \| :c:data:`NPY_ARRAY_ALIGNED` + .. c:var:: NPY_ARRAY_OUT_ARRAY + + :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| :c:data:`NPY_ARRAY_ALIGNED` \| + :c:data:`NPY_ARRAY_WRITEABLE` + .. c:var:: NPY_ARRAY_OUT_FARRAY :c:data:`NPY_ARRAY_F_CONTIGUOUS` \| :c:data:`NPY_ARRAY_WRITEABLE` \| @@ -573,8 +593,9 @@ From other objects return NULL; } if (arr == NULL) { + /* ... validate/change dtype, validate flags, ndim, etc ... - // Could make custom strides here too + Could make custom strides here too */ arr = PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, dims, NULL, fortran ? NPY_ARRAY_F_CONTIGUOUS : 0, @@ -588,10 +609,14 @@ From other objects } } else { + /* ... in this case the other parameters weren't filled, just validate and possibly copy arr itself ... + */ } + /* ... use arr ... + */ .. c:function:: PyObject* PyArray_CheckFromAny( \ PyObject* op, PyArray_Descr* dtype, int min_depth, int max_depth, \ @@ -784,7 +809,7 @@ From other objects PyObject* obj, int typenum, int requirements) Combination of :c:func:`PyArray_FROM_OF` and :c:func:`PyArray_FROM_OT` - allowing both a *typenum* and a *flags* argument to be provided.. + allowing both a *typenum* and a *flags* argument to be provided. .. c:function:: PyObject* PyArray_FROMANY( \ PyObject* obj, int typenum, int min, int max, int requirements) @@ -816,17 +841,17 @@ Dealing with types General check of Python Type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. c:function:: PyArray_Check(op) +.. c:function:: PyArray_Check(PyObject *op) Evaluates true if *op* is a Python object whose type is a sub-type of :c:data:`PyArray_Type`. -.. c:function:: PyArray_CheckExact(op) +.. c:function:: PyArray_CheckExact(PyObject *op) Evaluates true if *op* is a Python object with type :c:data:`PyArray_Type`. -.. c:function:: PyArray_HasArrayInterface(op, out) +.. c:function:: PyArray_HasArrayInterface(PyObject *op, PyObject *out) If ``op`` implements any part of the array interface, then ``out`` will contain a new reference to the newly created ndarray using @@ -1375,6 +1400,7 @@ Special functions for NPY_OBJECT Returns 0 for success, -1 for failure. +.. _array-flags: Array flags ----------- @@ -1552,7 +1578,7 @@ Flag checking ^^^^^^^^^^^^^ For all of these macros *arr* must be an instance of a (subclass of) -:c:data:`PyArray_Type`, but no checking is done. +:c:data:`PyArray_Type`. .. c:function:: PyArray_CHKFLAGS(arr, flags) @@ -1650,11 +1676,13 @@ Conversion .. c:function:: PyObject* PyArray_GetField( \ PyArrayObject* self, PyArray_Descr* dtype, int offset) - Equivalent to :meth:`ndarray.getfield<numpy.ndarray.getfield>` (*self*, *dtype*, *offset*). Return - a new array of the given *dtype* using the data in the current - array at a specified *offset* in bytes. The *offset* plus the - itemsize of the new array type must be less than *self* - ->descr->elsize or an error is raised. The same shape and strides + Equivalent to :meth:`ndarray.getfield<numpy.ndarray.getfield>` + (*self*, *dtype*, *offset*). This function `steals a reference + <https://docs.python.org/3/c-api/intro.html?reference-count-details>`_ + to `PyArray_Descr` and returns a new array of the given `dtype` using + the data in the current array at a specified `offset` in bytes. The + `offset` plus the itemsize of the new array type must be less than ``self + ->descr->elsize`` or an error is raised. The same shape and strides as the original array are used. Therefore, this function has the effect of returning a field from a structured array. But, it can also be used to select specific bytes or groups of bytes from any array @@ -1904,22 +1932,23 @@ Item selection and manipulation all values are clipped to the region [0, len(*op*) ). -.. c:function:: PyObject* PyArray_Sort(PyArrayObject* self, int axis) +.. c:function:: PyObject* PyArray_Sort(PyArrayObject* self, int axis, NPY_SORTKIND kind) - Equivalent to :meth:`ndarray.sort<numpy.ndarray.sort>` (*self*, *axis*). Return an array with - the items of *self* sorted along *axis*. + Equivalent to :meth:`ndarray.sort<numpy.ndarray.sort>` (*self*, *axis*, *kind*). + Return an array with the items of *self* sorted along *axis*. The array + is sorted using the algorithm denoted by *kind* , which is an integer/enum pointing + to the type of sorting algorithms used. .. c:function:: PyObject* PyArray_ArgSort(PyArrayObject* self, int axis) - Equivalent to :meth:`ndarray.argsort<numpy.ndarray.argsort>` (*self*, *axis*). Return an array of - indices such that selection of these indices along the given - ``axis`` would return a sorted version of *self*. If *self* - ->descr is a data-type with fields defined, then - self->descr->names is used to determine the sort order. A - comparison where the first field is equal will use the second - field and so on. To alter the sort order of a structured array, create - a new data-type with a different order of names and construct a - view of the array with that new data-type. + Equivalent to :meth:`ndarray.argsort<numpy.ndarray.argsort>` (*self*, *axis*). + Return an array of indices such that selection of these indices + along the given ``axis`` would return a sorted version of *self*. If *self* ->descr + is a data-type with fields defined, then self->descr->names is used + to determine the sort order. A comparison where the first field is equal + will use the second field and so on. To alter the sort order of a + structured array, create a new data-type with a different order of names + and construct a view of the array with that new data-type. .. c:function:: PyObject* PyArray_LexSort(PyObject* sort_keys, int axis) @@ -2023,6 +2052,17 @@ Calculation effect that is obtained by passing in *axis* = :const:`None` in Python (treating the array as a 1-d array). + +.. note:: + + The out argument specifies where to place the result. If out is + NULL, then the output array is created, otherwise the output is + placed in out which must be the correct size and type. A new + reference to the output array is always returned even when out + is not NULL. The caller of the routine has the responsibility + to ``Py_DECREF`` out if not NULL or a memory-leak will occur. + + .. c:function:: PyObject* PyArray_ArgMax( \ PyArrayObject* self, int axis, PyArrayObject* out) @@ -2035,18 +2075,6 @@ Calculation Equivalent to :meth:`ndarray.argmin<numpy.ndarray.argmin>` (*self*, *axis*). Return the index of the smallest element of *self* along *axis*. - - - -.. note:: - - The out argument specifies where to place the result. If out is - NULL, then the output array is created, otherwise the output is - placed in out which must be the correct size and type. A new - reference to the output array is always returned even when out - is not NULL. The caller of the routine has the responsibility - to ``DECREF`` out if not NULL or a memory-leak will occur. - .. c:function:: PyObject* PyArray_Max( \ PyArrayObject* self, int axis, PyArrayObject* out) @@ -2333,8 +2361,8 @@ Other functions ^^^^^^^^^^^^^^^ .. c:function:: Bool PyArray_CheckStrides( \ - int elsize, int nd, npy_intp numbytes, npy_intp* dims, \ - npy_intp* newstrides) + int elsize, int nd, npy_intp numbytes, npy_intp const* dims, \ + npy_intp const* newstrides) Determine if *newstrides* is a strides array consistent with the memory of an *nd* -dimensional array with shape ``dims`` and @@ -2346,14 +2374,14 @@ Other functions *elsize* refer to a single-segment array. Return :c:data:`NPY_TRUE` if *newstrides* is acceptable, otherwise return :c:data:`NPY_FALSE`. -.. c:function:: npy_intp PyArray_MultiplyList(npy_intp* seq, int n) +.. c:function:: npy_intp PyArray_MultiplyList(npy_intp const* seq, int n) -.. c:function:: int PyArray_MultiplyIntList(int* seq, int n) +.. c:function:: int PyArray_MultiplyIntList(int const* seq, int n) Both of these routines multiply an *n* -length array, *seq*, of integers and return the result. No overflow checking is performed. -.. c:function:: int PyArray_CompareLists(npy_intp* l1, npy_intp* l2, int n) +.. c:function:: int PyArray_CompareLists(npy_intp const* l1, npy_intp const* l2, int n) Given two *n* -length arrays of integers, *l1*, and *l2*, return 1 if the lists are identical; otherwise, return 0. @@ -2626,22 +2654,24 @@ cost of a slight overhead. The mode should be one of: - * NPY_NEIGHBORHOOD_ITER_ZERO_PADDING: zero padding. Outside bounds values - will be 0. - * NPY_NEIGHBORHOOD_ITER_ONE_PADDING: one padding, Outside bounds values - will be 1. - * NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING: constant padding. Outside bounds - values will be the same as the first item in fill_value. - * NPY_NEIGHBORHOOD_ITER_MIRROR_PADDING: mirror padding. Outside bounds - values will be as if the array items were mirrored. For example, for the - array [1, 2, 3, 4], x[-2] will be 2, x[-2] will be 1, x[4] will be 4, - x[5] will be 1, etc... - * NPY_NEIGHBORHOOD_ITER_CIRCULAR_PADDING: circular padding. Outside bounds - values will be as if the array was repeated. For example, for the - array [1, 2, 3, 4], x[-2] will be 3, x[-2] will be 4, x[4] will be 1, - x[5] will be 2, etc... - - If the mode is constant filling (NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING), + .. c:macro:: NPY_NEIGHBORHOOD_ITER_ZERO_PADDING + Zero padding. Outside bounds values will be 0. + .. c:macro:: NPY_NEIGHBORHOOD_ITER_ONE_PADDING + One padding, Outside bounds values will be 1. + .. c:macro:: NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING + Constant padding. Outside bounds values will be the + same as the first item in fill_value. + .. c:macro:: NPY_NEIGHBORHOOD_ITER_MIRROR_PADDING + Mirror padding. Outside bounds values will be as if the + array items were mirrored. For example, for the array [1, 2, 3, 4], + x[-2] will be 2, x[-2] will be 1, x[4] will be 4, x[5] will be 1, + etc... + .. c:macro:: NPY_NEIGHBORHOOD_ITER_CIRCULAR_PADDING + Circular padding. Outside bounds values will be as if the array + was repeated. For example, for the array [1, 2, 3, 4], x[-2] will + be 3, x[-2] will be 4, x[4] will be 1, x[5] will be 2, etc... + + If the mode is constant filling (`NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING`), fill_value should point to an array object which holds the filling value (the first item will be the filling value if the array contains more than one item). For other cases, fill_value may be NULL. @@ -2659,22 +2689,22 @@ cost of a slight overhead. .. code-block:: c - PyArrayIterObject \*iter; - PyArrayNeighborhoodIterObject \*neigh_iter; + PyArrayIterObject *iter; + PyArrayNeighborhoodIterObject *neigh_iter; iter = PyArray_IterNew(x); - //For a 3x3 kernel + /*For a 3x3 kernel */ bounds = {-1, 1, -1, 1}; neigh_iter = (PyArrayNeighborhoodIterObject*)PyArrayNeighborhoodIter_New( iter, bounds, NPY_NEIGHBORHOOD_ITER_ZERO_PADDING, NULL); for(i = 0; i < iter->size; ++i) { for (j = 0; j < neigh_iter->size; ++j) { - // Walk around the item currently pointed by iter->dataptr + /* Walk around the item currently pointed by iter->dataptr */ PyArrayNeighborhoodIter_Next(neigh_iter); } - // Move to the next point of iter + /* Move to the next point of iter */ PyArrayIter_Next(iter); PyArrayNeighborhoodIter_Reset(neigh_iter); } @@ -2988,8 +3018,11 @@ to. .. c:function:: int PyArray_SortkindConverter(PyObject* obj, NPY_SORTKIND* sort) Convert Python strings into one of :c:data:`NPY_QUICKSORT` (starts - with 'q' or 'Q') , :c:data:`NPY_HEAPSORT` (starts with 'h' or 'H'), - or :c:data:`NPY_MERGESORT` (starts with 'm' or 'M'). + with 'q' or 'Q'), :c:data:`NPY_HEAPSORT` (starts with 'h' or 'H'), + :c:data:`NPY_MERGESORT` (starts with 'm' or 'M') or :c:data:`NPY_STABLESORT` + (starts with 't' or 'T'). :c:data:`NPY_MERGESORT` and :c:data:`NPY_STABLESORT` + are aliased to each other for backwards compatibility and may refer to one + of several stable sorting algorithms depending on the data type. .. c:function:: int PyArray_SearchsideConverter( \ PyObject* obj, NPY_SEARCHSIDE* side) @@ -3252,19 +3285,19 @@ Memory management Macros to allocate, free, and reallocate memory. These macros are used internally to create arrays. -.. c:function:: npy_intp* PyDimMem_NEW(nd) +.. c:function:: npy_intp* PyDimMem_NEW(int nd) -.. c:function:: PyDimMem_FREE(npy_intp* ptr) +.. c:function:: PyDimMem_FREE(char* ptr) -.. c:function:: npy_intp* PyDimMem_RENEW(npy_intp* ptr, npy_intp newnd) +.. c:function:: npy_intp* PyDimMem_RENEW(void* ptr, size_t newnd) Macros to allocate, free, and reallocate dimension and strides memory. -.. c:function:: PyArray_malloc(nbytes) +.. c:function:: void* PyArray_malloc(size_t nbytes) -.. c:function:: PyArray_free(ptr) +.. c:function:: PyArray_free(void* ptr) -.. c:function:: PyArray_realloc(ptr, nbytes) +.. c:function:: void* PyArray_realloc(npy_intp* ptr, size_t nbytes) These macros use different memory allocators, depending on the constant :c:data:`NPY_USE_PYMEM`. The system malloc is used when @@ -3344,7 +3377,7 @@ Group 1 Useful to release the GIL only if *dtype* does not contain arbitrary Python objects which may need the Python interpreter - during execution of the loop. Equivalent to + during execution of the loop. .. c:function:: NPY_END_THREADS_DESCR(PyArray_Descr *dtype) @@ -3438,6 +3471,10 @@ Other constants The maximum number of dimensions allowed in arrays. +.. c:var:: NPY_MAXARGS + + The maximum number of array arguments that can be used in functions. + .. c:var:: NPY_VERSION The current version of the ndarray object (check to see if this @@ -3466,31 +3503,31 @@ Other constants Miscellaneous Macros ^^^^^^^^^^^^^^^^^^^^ -.. c:function:: PyArray_SAMESHAPE(a1, a2) +.. c:function:: PyArray_SAMESHAPE(PyArrayObject *a1, PyArrayObject *a2) Evaluates as True if arrays *a1* and *a2* have the same shape. -.. c:function:: PyArray_MAX(a,b) +.. c:macro:: PyArray_MAX(a,b) Returns the maximum of *a* and *b*. If (*a*) or (*b*) are expressions they are evaluated twice. -.. c:function:: PyArray_MIN(a,b) +.. c:macro:: PyArray_MIN(a,b) Returns the minimum of *a* and *b*. If (*a*) or (*b*) are expressions they are evaluated twice. -.. c:function:: PyArray_CLT(a,b) +.. c:macro:: PyArray_CLT(a,b) -.. c:function:: PyArray_CGT(a,b) +.. c:macro:: PyArray_CGT(a,b) -.. c:function:: PyArray_CLE(a,b) +.. c:macro:: PyArray_CLE(a,b) -.. c:function:: PyArray_CGE(a,b) +.. c:macro:: PyArray_CGE(a,b) -.. c:function:: PyArray_CEQ(a,b) +.. c:macro:: PyArray_CEQ(a,b) -.. c:function:: PyArray_CNE(a,b) +.. c:macro:: PyArray_CNE(a,b) Implements the complex comparisons between two complex numbers (structures with a real and imag member) using NumPy's definition @@ -3530,14 +3567,26 @@ Enumerated Types .. c:type:: NPY_SORTKIND - A special variable-type which can take on the values :c:data:`NPY_{KIND}` - where ``{KIND}`` is + A special variable-type which can take on different values to indicate + the sorting algorithm being used. + + .. c:var:: NPY_QUICKSORT - **QUICKSORT**, **HEAPSORT**, **MERGESORT** + .. c:var:: NPY_HEAPSORT + + .. c:var:: NPY_MERGESORT + + .. c:var:: NPY_STABLESORT + + Used as an alias of :c:data:`NPY_MERGESORT` and vica versa. .. c:var:: NPY_NSORTS - Defined to be the number of sorts. + Defined to be the number of sorts. It is fixed at three by the need for + backwards compatibility, and consequently :c:data:`NPY_MERGESORT` and + :c:data:`NPY_STABLESORT` are aliased to each other and may refer to one + of several stable sorting algorithms depending on the data type. + .. c:type:: NPY_SCALARKIND diff --git a/doc/source/reference/c-api.config.rst b/doc/source/reference/c-api/config.rst index 60bf61a32..05e6fe44d 100644 --- a/doc/source/reference/c-api.config.rst +++ b/doc/source/reference/c-api/config.rst @@ -101,3 +101,22 @@ Platform information Returns the endianness of the current platform. One of :c:data:`NPY_CPU_BIG`, :c:data:`NPY_CPU_LITTLE`, or :c:data:`NPY_CPU_UNKNOWN_ENDIAN`. + + +Compiler directives +------------------- + +.. c:var:: NPY_LIKELY +.. c:var:: NPY_UNLIKELY +.. c:var:: NPY_UNUSED + + +Interrupt Handling +------------------ + +.. c:var:: NPY_INTERRUPT_H +.. c:var:: NPY_SIGSETJMP +.. c:var:: NPY_SIGLONGJMP +.. c:var:: NPY_SIGJMP_BUF +.. c:var:: NPY_SIGINT_ON +.. c:var:: NPY_SIGINT_OFF diff --git a/doc/source/reference/c-api.coremath.rst b/doc/source/reference/c-api/coremath.rst index 691f73287..7e00322f9 100644 --- a/doc/source/reference/c-api.coremath.rst +++ b/doc/source/reference/c-api/coremath.rst @@ -80,8 +80,9 @@ Floating point classification Useful math constants ~~~~~~~~~~~~~~~~~~~~~ -The following math constants are available in npy_math.h. Single and extended -precision are also available by adding the F and L suffixes respectively. +The following math constants are available in ``npy_math.h``. Single +and extended precision are also available by adding the ``f`` and +``l`` suffixes respectively. .. c:var:: NPY_E @@ -184,7 +185,7 @@ Those can be useful for precise floating point comparison. * NPY_FPE_INVALID Note that :c:func:`npy_get_floatstatus_barrier` is preferable as it prevents - agressive compiler optimizations reordering the call relative to + aggressive compiler optimizations reordering the call relative to the code setting the status, which could lead to incorrect results. .. versionadded:: 1.9.0 @@ -192,7 +193,7 @@ Those can be useful for precise floating point comparison. .. c:function:: int npy_get_floatstatus_barrier(char*) Get floating point status. A pointer to a local variable is passed in to - prevent aggresive compiler optimizations from reodering this function call + prevent aggressive compiler optimizations from reodering this function call relative to the code setting the status, which could lead to incorrect results. @@ -210,7 +211,7 @@ Those can be useful for precise floating point comparison. Clears the floating point status. Returns the previous status mask. Note that :c:func:`npy_clear_floatstatus_barrier` is preferable as it - prevents agressive compiler optimizations reordering the call relative to + prevents aggressive compiler optimizations reordering the call relative to the code setting the status, which could lead to incorrect results. .. versionadded:: 1.9.0 @@ -218,7 +219,7 @@ Those can be useful for precise floating point comparison. .. c:function:: int npy_clear_floatstatus_barrier(char*) Clears the floating point status. A pointer to a local variable is passed in to - prevent aggresive compiler optimizations from reodering this function call. + prevent aggressive compiler optimizations from reodering this function call. Returns the previous status mask. .. versionadded:: 1.15.0 @@ -258,7 +259,7 @@ and co. Half-precision functions ~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 2.0.0 +.. versionadded:: 1.6.0 The header file <numpy/halffloat.h> provides functions to work with IEEE 754-2008 16-bit floating point values. While this format is diff --git a/doc/source/reference/c-api.deprecations.rst b/doc/source/reference/c-api/deprecations.rst index a382017a2..a382017a2 100644 --- a/doc/source/reference/c-api.deprecations.rst +++ b/doc/source/reference/c-api/deprecations.rst diff --git a/doc/source/reference/c-api.dtype.rst b/doc/source/reference/c-api/dtype.rst index 9ac46b284..72e908861 100644 --- a/doc/source/reference/c-api.dtype.rst +++ b/doc/source/reference/c-api/dtype.rst @@ -308,13 +308,45 @@ to the front of the integer name. (unsigned) char -.. c:type:: npy_(u)short +.. c:type:: npy_short - (unsigned) short + short -.. c:type:: npy_(u)int +.. c:type:: npy_ushort - (unsigned) int + unsigned short + +.. c:type:: npy_uint + + unsigned int + +.. c:type:: npy_int + + int + +.. c:type:: npy_int16 + + 16-bit integer + +.. c:type:: npy_uint16 + + 16-bit unsigned integer + +.. c:type:: npy_int32 + + 32-bit integer + +.. c:type:: npy_uint32 + + 32-bit unsigned integer + +.. c:type:: npy_int64 + + 64-bit integer + +.. c:type:: npy_uint64 + + 64-bit unsigned integer .. c:type:: npy_(u)long @@ -324,22 +356,31 @@ to the front of the integer name. (unsigned long long int) -.. c:type:: npy_(u)intp +.. c:type:: npy_intp - (unsigned) Py_intptr_t (an integer that is the size of a pointer on + Py_intptr_t (an integer that is the size of a pointer on + the platform). + +.. c:type:: npy_uintp + + unsigned Py_intptr_t (an integer that is the size of a pointer on the platform). (Complex) Floating point ^^^^^^^^^^^^^^^^^^^^^^^^ +.. c:type:: npy_half + + 16-bit float + .. c:type:: npy_(c)float - float + 32-bit float .. c:type:: npy_(c)double - double + 64-bit double .. c:type:: npy_(c)longdouble diff --git a/doc/source/reference/c-api.generalized-ufuncs.rst b/doc/source/reference/c-api/generalized-ufuncs.rst index b59f077ad..b59f077ad 100644 --- a/doc/source/reference/c-api.generalized-ufuncs.rst +++ b/doc/source/reference/c-api/generalized-ufuncs.rst diff --git a/doc/source/reference/c-api.rst b/doc/source/reference/c-api/index.rst index b8cbe97b2..56fe8e473 100644 --- a/doc/source/reference/c-api.rst +++ b/doc/source/reference/c-api/index.rst @@ -40,12 +40,12 @@ code. .. toctree:: :maxdepth: 2 - c-api.types-and-structures - c-api.config - c-api.dtype - c-api.array - c-api.iterator - c-api.ufunc - c-api.generalized-ufuncs - c-api.coremath - c-api.deprecations + types-and-structures + config + dtype + array + iterator + ufunc + generalized-ufuncs + coremath + deprecations diff --git a/doc/source/reference/c-api.iterator.rst b/doc/source/reference/c-api/iterator.rst index 940452d3c..b77d029cc 100644 --- a/doc/source/reference/c-api.iterator.rst +++ b/doc/source/reference/c-api/iterator.rst @@ -593,25 +593,23 @@ Construction and Destruction code doing iteration can write to this operand to control which elements will be untouched and which ones will be modified. This is useful when the mask should be a combination - of input masks, for example. Mask values can be created - with the :c:func:`NpyMask_Create` function. + of input masks. .. c:var:: NPY_ITER_WRITEMASKED .. versionadded:: 1.7 - Indicates that only elements which the operand with - the ARRAYMASK flag indicates are intended to be modified - by the iteration. In general, the iterator does not enforce - this, it is up to the code doing the iteration to follow - that promise. Code can use the :c:func:`NpyMask_IsExposed` - inline function to test whether the mask at a particular - element allows writing. + This array is the mask for all `writemasked <numpy.nditer>` + operands. Code uses the ``writemasked`` flag which indicates + that only elements where the chosen ARRAYMASK operand is True + will be written to. In general, the iterator does not enforce + this, it is up to the code doing the iteration to follow that + promise. - When this flag is used, and this operand is buffered, this - changes how data is copied from the buffer into the array. + When ``writemasked`` flag is used, and this operand is buffered, + this changes how data is copied from the buffer into the array. A masked copying routine is used, which only copies the - elements in the buffer for which :c:func:`NpyMask_IsExposed` + elements in the buffer for which ``writemasked`` returns true from the corresponding element in the ARRAYMASK operand. @@ -630,7 +628,7 @@ Construction and Destruction .. c:function:: NpyIter* NpyIter_AdvancedNew( \ npy_intp nop, PyArrayObject** op, npy_uint32 flags, NPY_ORDER order, \ NPY_CASTING casting, npy_uint32* op_flags, PyArray_Descr** op_dtypes, \ - int oa_ndim, int** op_axes, npy_intp* itershape, npy_intp buffersize) + int oa_ndim, int** op_axes, npy_intp const* itershape, npy_intp buffersize) Extends :c:func:`NpyIter_MultiNew` with several advanced options providing more control over broadcasting and buffering. @@ -867,7 +865,7 @@ Construction and Destruction } while (iternext2(iter2)); } while (iternext1(iter1)); -.. c:function:: int NpyIter_GotoMultiIndex(NpyIter* iter, npy_intp* multi_index) +.. c:function:: int NpyIter_GotoMultiIndex(NpyIter* iter, npy_intp const* multi_index) Adjusts the iterator to point to the ``ndim`` indices pointed to by ``multi_index``. Returns an error if a multi-index @@ -974,19 +972,6 @@ Construction and Destruction Returns the number of operands in the iterator. - When :c:data:`NPY_ITER_USE_MASKNA` is used on an operand, a new - operand is added to the end of the operand list in the iterator - to track that operand's NA mask. Thus, this equals the number - of construction operands plus the number of operands for - which the flag :c:data:`NPY_ITER_USE_MASKNA` was specified. - -.. c:function:: int NpyIter_GetFirstMaskNAOp(NpyIter* iter) - - .. versionadded:: 1.7 - - Returns the index of the first NA mask operand in the array. This - value is equal to the number of operands passed into the constructor. - .. c:function:: npy_intp* NpyIter_GetAxisStrideArray(NpyIter* iter, int axis) Gets the array of strides for the specified axis. Requires that @@ -1023,16 +1008,6 @@ Construction and Destruction that are being iterated. The result points into ``iter``, so the caller does not gain any references to the PyObjects. -.. c:function:: npy_int8* NpyIter_GetMaskNAIndexArray(NpyIter* iter) - - .. versionadded:: 1.7 - - This gives back a pointer to the ``nop`` indices which map - construction operands with :c:data:`NPY_ITER_USE_MASKNA` flagged - to their corresponding NA mask operands and vice versa. For - operands which were not flagged with :c:data:`NPY_ITER_USE_MASKNA`, - this array contains negative values. - .. c:function:: PyObject* NpyIter_GetIterView(NpyIter* iter, npy_intp i) This gives back a reference to a new ndarray view, which is a view diff --git a/doc/source/reference/c-api.types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index f04d65ee1..336dff211 100644 --- a/doc/source/reference/c-api.types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -1,3 +1,4 @@ + ***************************** Python Types and C-Structures ***************************** @@ -57,8 +58,8 @@ types are place holders that allow the array scalars to fit into a hierarchy of actual Python types. -PyArray_Type ------------- +PyArray_Type and PyArrayObject +------------------------------ .. c:var:: PyArray_Type @@ -74,8 +75,9 @@ PyArray_Type subclasses) will have this structure. For future compatibility, these structure members should normally be accessed using the provided macros. If you need a shorter name, then you can make use - of :c:type:`NPY_AO` which is defined to be equivalent to - :c:type:`PyArrayObject`. + of :c:type:`NPY_AO` (deprecated) which is defined to be equivalent to + :c:type:`PyArrayObject`. Direct access to the struct fields are + deprecated. Use the `PyArray_*(arr)` form instead. .. code-block:: c @@ -91,7 +93,7 @@ PyArray_Type PyObject *weakreflist; } PyArrayObject; -.. c:macro: PyArrayObject.PyObject_HEAD +.. c:macro:: PyArrayObject.PyObject_HEAD This is needed by all Python objects. It consists of (at least) a reference count member ( ``ob_refcnt`` ) and a pointer to the @@ -103,7 +105,8 @@ PyArray_Type .. c:member:: char *PyArrayObject.data - A pointer to the first element of the array. This pointer can + Accessible via :c:data:`PyArray_DATA`, this data member is a + pointer to the first element of the array. This pointer can (and normally should) be recast to the data type of the array. .. c:member:: int PyArrayObject.nd @@ -111,33 +114,38 @@ PyArray_Type An integer providing the number of dimensions for this array. When nd is 0, the array is sometimes called a rank-0 array. Such arrays have undefined dimensions and strides and - cannot be accessed. :c:data:`NPY_MAXDIMS` is the largest number of - dimensions for any array. + cannot be accessed. Macro :c:data:`PyArray_NDIM` defined in + ``ndarraytypes.h`` points to this data member. :c:data:`NPY_MAXDIMS` + is the largest number of dimensions for any array. .. c:member:: npy_intp PyArrayObject.dimensions An array of integers providing the shape in each dimension as long as nd :math:`\geq` 1. The integer is always large enough to hold a pointer on the platform, so the dimension size is - only limited by memory. + only limited by memory. :c:data:`PyArray_DIMS` is the macro + associated with this data member. .. c:member:: npy_intp *PyArrayObject.strides An array of integers providing for each dimension the number of bytes that must be skipped to get to the next element in that - dimension. + dimension. Associated with macro :c:data:`PyArray_STRIDES`. .. c:member:: PyObject *PyArrayObject.base - This member is used to hold a pointer to another Python object that - is related to this array. There are two use cases: 1) If this array - does not own its own memory, then base points to the Python object - that owns it (perhaps another array object), 2) If this array has - the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or - :c:data:NPY_ARRAY_WRITEBACKIFCOPY`: flag set, then this array is - a working copy of a "misbehaved" array. When - ``PyArray_ResolveWritebackIfCopy`` is called, the array pointed to by base - will be updated with the contents of this array. + Pointed to by :c:data:`PyArray_BASE`, this member is used to hold a + pointer to another Python object that is related to this array. + There are two use cases: + + - If this array does not own its own memory, then base points to the + Python object that owns it (perhaps another array object) + - If this array has the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag set, then this array is a working + copy of a "misbehaved" array. + + When ``PyArray_ResolveWritebackIfCopy`` is called, the array pointed to + by base will be updated with the contents of this array. .. c:member:: PyArray_Descr *PyArrayObject.descr @@ -147,11 +155,13 @@ PyArray_Type descriptor structure for each data type supported. This descriptor structure contains useful information about the type as well as a pointer to a table of function pointers to - implement specific functionality. + implement specific functionality. As the name suggests, it is + associated with the macro :c:data:`PyArray_DESCR`. .. c:member:: int PyArrayObject.flags - Flags indicating how the memory pointed to by data is to be + Pointed to by the macro :c:data:`PyArray_FLAGS`, this data member represents + the flags indicating how the memory pointed to by data is to be interpreted. Possible flags are :c:data:`NPY_ARRAY_C_CONTIGUOUS`, :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_OWNDATA`, :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, @@ -163,8 +173,8 @@ PyArray_Type weakref module). -PyArrayDescr_Type ------------------ +PyArrayDescr_Type and PyArray_Descr +----------------------------------- .. c:var:: PyArrayDescr_Type @@ -203,14 +213,17 @@ PyArrayDescr_Type char kind; char type; char byteorder; - char unused; - int flags; + char flags; int type_num; int elsize; int alignment; PyArray_ArrayDescr *subarray; PyObject *fields; + PyObject *names; PyArray_ArrFuncs *f; + PyObject *metadata; + NpyAuxData *c_metadata; + npy_hash_t hash; } PyArray_Descr; .. c:member:: PyTypeObject *PyArray_Descr.typeobj @@ -242,7 +255,7 @@ PyArrayDescr_Type endian), '=' (native), '\|' (irrelevant, ignore). All builtin data- types have byteorder '='. -.. c:member:: int PyArray_Descr.flags +.. c:member:: char PyArray_Descr.flags A data-type bit-flag that determines if the data-type exhibits object- array like behavior. Each bit in this member is a flag which are named @@ -250,11 +263,13 @@ PyArrayDescr_Type .. c:var:: NPY_ITEM_REFCOUNT - .. c:var:: NPY_ITEM_HASOBJECT - Indicates that items of this data-type must be reference counted (using :c:func:`Py_INCREF` and :c:func:`Py_DECREF` ). + .. c:var:: NPY_ITEM_HASOBJECT + + Same as :c:data:`NPY_ITEM_REFCOUNT`. + .. c:var:: NPY_LIST_PICKLE Indicates arrays of this data-type must be converted to a list @@ -377,6 +392,11 @@ PyArrayDescr_Type normally a Python string. These tuples are placed in this dictionary keyed by name (and also title if given). +.. c:member:: PyObject *PyArray_Descr.names + + An ordered tuple of field names. It is NULL if no field is + defined. + .. c:member:: PyArray_ArrFuncs *PyArray_Descr.f A pointer to a structure containing functions that the type needs @@ -384,6 +404,20 @@ PyArrayDescr_Type thing as the universal functions (ufuncs) described later. Their signatures can vary arbitrarily. +.. c:member:: PyObject *PyArray_Descr.metadata + + Metadata about this dtype. + +.. c:member:: NpyAuxData *PyArray_Descr.c_metadata + + Metadata specific to the C implementation + of the particular dtype. Added for NumPy 1.7.0. + +.. c:member:: Npy_hash_t *PyArray_Descr.hash + + Currently unused. Reserved for future use in caching + hash values. + .. c:type:: PyArray_ArrFuncs Functions implementing internal features. Not all of these @@ -508,20 +542,19 @@ PyArrayDescr_Type and ``is2`` *bytes*, respectively. This function requires behaved (though not necessarily contiguous) memory. - .. c:member:: int scanfunc(FILE* fd, void* ip , void* sep , void* arr) + .. c:member:: int scanfunc(FILE* fd, void* ip, void* arr) A pointer to a function that scans (scanf style) one element of the corresponding type from the file descriptor ``fd`` into the array memory pointed to by ``ip``. The array is assumed - to be behaved. If ``sep`` is not NULL, then a separator string - is also scanned from the file before returning. The last - argument ``arr`` is the array to be scanned into. A 0 is - returned if the scan is successful. A negative number - indicates something went wrong: -1 means the end of file was - reached before the separator string could be scanned, -4 means - that the end of file was reached before the element could be - scanned, and -3 means that the element could not be - interpreted from the format string. Requires a behaved array. + to be behaved. + The last argument ``arr`` is the array to be scanned into. + Returns number of receiving arguments successfully assigned (which + may be zero in case a matching failure occurred before the first + receiving argument was assigned), or EOF if input failure occurs + before the first receiving argument was assigned. + This function should be called without holding the Python GIL, and + has to grab it for error reporting. .. c:member:: int fromstr(char* str, void* ip, char** endptr, void* arr) @@ -532,6 +565,8 @@ PyArrayDescr_Type string. The last argument ``arr`` is the array into which ip points (needed for variable-size data- types). Returns 0 on success or -1 on failure. Requires a behaved array. + This function should be called without holding the Python GIL, and + has to grab it for error reporting. .. c:member:: Bool nonzero(void* data, void* arr) @@ -653,25 +688,28 @@ PyArrayDescr_Type The :c:data:`PyArray_Type` typeobject implements many of the features of -Python objects including the tp_as_number, tp_as_sequence, -tp_as_mapping, and tp_as_buffer interfaces. The rich comparison -(tp_richcompare) is also used along with new-style attribute lookup -for methods (tp_methods) and properties (tp_getset). The -:c:data:`PyArray_Type` can also be sub-typed. +:c:type:`Python objects <PyTypeObject>` including the :c:member:`tp_as_number +<PyTypeObject.tp_as_number>`, :c:member:`tp_as_sequence +<PyTypeObject.tp_as_sequence>`, :c:member:`tp_as_mapping +<PyTypeObject.tp_as_mapping>`, and :c:member:`tp_as_buffer +<PyTypeObject.tp_as_buffer>` interfaces. The :c:type:`rich comparison +<richcmpfunc>`) is also used along with new-style attribute lookup for +member (:c:member:`tp_members <PyTypeObject.tp_members>`) and properties +(:c:member:`tp_getset <PyTypeObject.tp_getset>`). +The :c:data:`PyArray_Type` can also be sub-typed. .. tip:: - The tp_as_number methods use a generic approach to call whatever - function has been registered for handling the operation. The - function PyNumeric_SetOps(..) can be used to register functions to - handle particular mathematical operations (for all arrays). When - the umath module is imported, it sets the numeric operations for - all arrays to the corresponding ufuncs. The tp_str and tp_repr - methods can also be altered using PyString_SetStringFunction(...). + The ``tp_as_number`` methods use a generic approach to call whatever + function has been registered for handling the operation. When the + ``_multiarray_umath module`` is imported, it sets the numeric operations + for all arrays to the corresponding ufuncs. This choice can be changed with + :c:func:`PyUFunc_ReplaceLoopBySignature` The ``tp_str`` and ``tp_repr`` + methods can also be altered using :c:func:`PyArray_SetStringFunction`. -PyUFunc_Type ------------- +PyUFunc_Type and PyUFuncObject +------------------------------ .. c:var:: PyUFunc_Type @@ -763,8 +801,8 @@ PyUFunc_Type the identity for this operation. It is only used for a reduce-like call on an empty array. - .. c:member:: void PyUFuncObject.functions(char** args, npy_intp* dims, - npy_intp* steps, void* extradata) + .. c:member:: void PyUFuncObject.functions( \ + char** args, npy_intp* dims, npy_intp* steps, void* extradata) An array of function pointers --- one for each data type supported by the ufunc. This is the vector loop that is called @@ -909,8 +947,8 @@ PyUFunc_Type - :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` if the dim size will be determined from the operands and not from a :ref:`frozen <frozen>` signature -PyArrayIter_Type ----------------- +PyArrayIter_Type and PyArrayIterObject +-------------------------------------- .. c:var:: PyArrayIter_Type @@ -1019,8 +1057,8 @@ with it through the use of the macros :c:func:`PyArray_ITER_NEXT` (it), :c:type:`PyArrayIterObject *`. -PyArrayMultiIter_Type ---------------------- +PyArrayMultiIter_Type and PyArrayMultiIterObject +------------------------------------------------ .. c:var:: PyArrayMultiIter_Type @@ -1081,8 +1119,8 @@ PyArrayMultiIter_Type arrays to be broadcast together. On return, the iterators are adjusted for broadcasting. -PyArrayNeighborhoodIter_Type ----------------------------- +PyArrayNeighborhoodIter_Type and PyArrayNeighborhoodIterObject +-------------------------------------------------------------- .. c:var:: PyArrayNeighborhoodIter_Type @@ -1095,8 +1133,33 @@ PyArrayNeighborhoodIter_Type :c:data:`PyArrayNeighborhoodIter_Type` is the :c:type:`PyArrayNeighborhoodIterObject`. -PyArrayFlags_Type ------------------ + .. code-block:: c + + typedef struct { + PyObject_HEAD + int nd_m1; + npy_intp index, size; + npy_intp coordinates[NPY_MAXDIMS] + npy_intp dims_m1[NPY_MAXDIMS]; + npy_intp strides[NPY_MAXDIMS]; + npy_intp backstrides[NPY_MAXDIMS]; + npy_intp factors[NPY_MAXDIMS]; + PyArrayObject *ao; + char *dataptr; + npy_bool contiguous; + npy_intp bounds[NPY_MAXDIMS][2]; + npy_intp limits[NPY_MAXDIMS][2]; + npy_intp limits_sizes[NPY_MAXDIMS]; + npy_iter_get_dataptr_t translate; + npy_intp nd; + npy_intp dimensions[NPY_MAXDIMS]; + PyArrayIterObject* _internal_iter; + char* constant; + int mode; + } PyArrayNeighborhoodIterObject; + +PyArrayFlags_Type and PyArrayFlagsObject +---------------------------------------- .. c:var:: PyArrayFlags_Type @@ -1106,6 +1169,16 @@ PyArrayFlags_Type attributes or by accessing them as if the object were a dictionary with the flag names as entries. +.. c:type:: PyArrayFlagsObject + + .. code-block:: c + + typedef struct PyArrayFlagsObject { + PyObject_HEAD + PyObject *arr; + int flags; + } PyArrayFlagsObject; + ScalarArrayTypes ---------------- diff --git a/doc/source/reference/c-api.ufunc.rst b/doc/source/reference/c-api/ufunc.rst index 0499ccf5b..92a679510 100644 --- a/doc/source/reference/c-api.ufunc.rst +++ b/doc/source/reference/c-api/ufunc.rst @@ -21,7 +21,17 @@ Constants .. c:var:: PyUFunc_{VALUE} - ``{VALUE}`` can be **One** (1), **Zero** (0), or **None** (-1) + .. c:var:: PyUFunc_One + + .. c:var:: PyUFunc_Zero + + .. c:var:: PyUFunc_MinusOne + + .. c:var:: PyUFunc_ReorderableNone + + .. c:var:: PyUFunc_None + + .. c:var:: PyUFunc_IdentityValue Macros @@ -39,28 +49,6 @@ Macros Used in universal function code to re-acquire the Python GIL if it was released (because loop->obj was not true). -.. c:function:: UFUNC_CHECK_ERROR(loop) - - A macro used internally to check for errors and goto fail if - found. This macro requires a fail label in the current code - block. The *loop* variable must have at least members (obj, - errormask, and errorobj). If *loop* ->obj is nonzero, then - :c:func:`PyErr_Occurred` () is called (meaning the GIL must be held). If - *loop* ->obj is zero, then if *loop* ->errormask is nonzero, - :c:func:`PyUFunc_checkfperr` is called with arguments *loop* ->errormask - and *loop* ->errobj. If the result of this check of the IEEE - floating point registers is true then the code redirects to the - fail label which must be defined. - -.. c:function:: UFUNC_CHECK_STATUS(ret) - - Deprecated: use npy_clear_floatstatus from npy_math.h instead. - - A macro that expands to platform-dependent code. The *ret* - variable can be any integer. The :c:data:`UFUNC_FPE_{ERR}` bits are - set in *ret* according to the status of the corresponding error - flags of the floating point processor. - Functions --------- diff --git a/doc/source/reference/distutils.rst b/doc/source/reference/distutils.rst index 88e533832..46e5ec25e 100644 --- a/doc/source/reference/distutils.rst +++ b/doc/source/reference/distutils.rst @@ -214,102 +214,4 @@ template and placed in the build directory to be used instead. Two forms of template conversion are supported. The first form occurs for files named <file>.ext.src where ext is a recognized Fortran extension (f, f90, f95, f77, for, ftn, pyf). The second form is used -for all other cases. - -.. index:: - single: code generation - -Fortran files -------------- - -This template converter will replicate all **function** and -**subroutine** blocks in the file with names that contain '<...>' -according to the rules in '<...>'. The number of comma-separated words -in '<...>' determines the number of times the block is repeated. What -these words are indicates what that repeat rule, '<...>', should be -replaced with in each block. All of the repeat rules in a block must -contain the same number of comma-separated words indicating the number -of times that block should be repeated. If the word in the repeat rule -needs a comma, leftarrow, or rightarrow, then prepend it with a -backslash ' \'. If a word in the repeat rule matches ' \\<index>' then -it will be replaced with the <index>-th word in the same repeat -specification. There are two forms for the repeat rule: named and -short. - - -Named repeat rule -^^^^^^^^^^^^^^^^^ - -A named repeat rule is useful when the same set of repeats must be -used several times in a block. It is specified using <rule1=item1, -item2, item3,..., itemN>, where N is the number of times the block -should be repeated. On each repeat of the block, the entire -expression, '<...>' will be replaced first with item1, and then with -item2, and so forth until N repeats are accomplished. Once a named -repeat specification has been introduced, the same repeat rule may be -used **in the current block** by referring only to the name -(i.e. <rule1>. - - -Short repeat rule -^^^^^^^^^^^^^^^^^ - -A short repeat rule looks like <item1, item2, item3, ..., itemN>. The -rule specifies that the entire expression, '<...>' should be replaced -first with item1, and then with item2, and so forth until N repeats -are accomplished. - - -Pre-defined names -^^^^^^^^^^^^^^^^^ - -The following predefined named repeat rules are available: - -- <prefix=s,d,c,z> - -- <_c=s,d,c,z> - -- <_t=real, double precision, complex, double complex> - -- <ftype=real, double precision, complex, double complex> - -- <ctype=float, double, complex_float, complex_double> - -- <ftypereal=float, double precision, \\0, \\1> - -- <ctypereal=float, double, \\0, \\1> - - -Other files ------------ - -Non-Fortran files use a separate syntax for defining template blocks -that should be repeated using a variable expansion similar to the -named repeat rules of the Fortran-specific repeats. The template rules -for these files are: - -1. "/\**begin repeat "on a line by itself marks the beginning of - a segment that should be repeated. - -2. Named variable expansions are defined using #name=item1, item2, item3, - ..., itemN# and placed on successive lines. These variables are - replaced in each repeat block with corresponding word. All named - variables in the same repeat block must define the same number of - words. - -3. In specifying the repeat rule for a named variable, item*N is short- - hand for item, item, ..., item repeated N times. In addition, - parenthesis in combination with \*N can be used for grouping several - items that should be repeated. Thus, #name=(item1, item2)*4# is - equivalent to #name=item1, item2, item1, item2, item1, item2, item1, - item2# - -4. "\*/ "on a line by itself marks the end of the variable expansion - naming. The next line is the first line that will be repeated using - the named rules. - -5. Inside the block to be repeated, the variables that should be expanded - are specified as @name@. - -6. "/\**end repeat**/ "on a line by itself marks the previous line - as the last line of the block to be repeated. +for all other cases. See :ref:`templating`. diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index 6accb8535..6742d605a 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -24,7 +24,7 @@ For learning how to use NumPy, see also :ref:`user`. routines distutils distutils_guide - c-api + c-api/index internals swig diff --git a/doc/source/reference/maskedarray.baseclass.rst b/doc/source/reference/maskedarray.baseclass.rst index 427ad1536..204ebfe08 100644 --- a/doc/source/reference/maskedarray.baseclass.rst +++ b/doc/source/reference/maskedarray.baseclass.rst @@ -49,11 +49,11 @@ The :class:`MaskedArray` class .. class:: MaskedArray - A subclass of :class:`~numpy.ndarray` designed to manipulate numerical arrays with missing data. +A subclass of :class:`~numpy.ndarray` designed to manipulate numerical arrays with missing data. - An instance of :class:`MaskedArray` can be thought as the combination of several elements: +An instance of :class:`MaskedArray` can be thought as the combination of several elements: * The :attr:`~MaskedArray.data`, as a regular :class:`numpy.ndarray` of any shape or datatype (the data). * A boolean :attr:`~numpy.ma.MaskedArray.mask` with the same shape as the data, where a ``True`` value indicates that the corresponding element of the data is invalid. @@ -62,89 +62,26 @@ The :class:`MaskedArray` class +.. _ma-attributes: + Attributes and properties of masked arrays ------------------------------------------ .. seealso:: :ref:`Array Attributes <arrays.ndarray.attributes>` +.. autoattribute:: MaskedArray.data -.. attribute:: MaskedArray.data - - Returns the underlying data, as a view of the masked array. - If the underlying data is a subclass of :class:`numpy.ndarray`, it is - returned as such. - - >>> x = ma.array(np.matrix([[1, 2], [3, 4]]), mask=[[0, 1], [1, 0]]) - >>> x.data - matrix([[1, 2], - [3, 4]]) - - The type of the data can be accessed through the :attr:`baseclass` - attribute. - -.. attribute:: MaskedArray.mask - - Returns the underlying mask, as an array with the same shape and structure - as the data, but where all fields are atomically booleans. - A value of ``True`` indicates an invalid entry. - - -.. attribute:: MaskedArray.recordmask - - Returns the mask of the array if it has no named fields. For structured - arrays, returns a ndarray of booleans where entries are ``True`` if **all** - the fields are masked, ``False`` otherwise:: - - >>> x = ma.array([(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)], - ... mask=[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)], - ... dtype=[('a', int), ('b', int)]) - >>> x.recordmask - array([False, False, True, False, False]) - - -.. attribute:: MaskedArray.fill_value - - Returns the value used to fill the invalid entries of a masked array. - The value is either a scalar (if the masked array has no named fields), - or a 0-D ndarray with the same :attr:`dtype` as the masked array if it has - named fields. - - The default filling value depends on the datatype of the array: - - ======== ======== - datatype default - ======== ======== - bool True - int 999999 - float 1.e20 - complex 1.e20+0j - object '?' - string 'N/A' - ======== ======== - - - -.. attribute:: MaskedArray.baseclass - - Returns the class of the underlying data. - - >>> x = ma.array(np.matrix([[1, 2], [3, 4]]), mask=[[0, 0], [1, 0]]) - >>> x.baseclass - <class 'numpy.matrixlib.defmatrix.matrix'> - - -.. attribute:: MaskedArray.sharedmask +.. autoattribute:: MaskedArray.mask - Returns whether the mask of the array is shared between several masked arrays. - If this is the case, any modification to the mask of one array will be - propagated to the others. +.. autoattribute:: MaskedArray.recordmask +.. autoattribute:: MaskedArray.fill_value -.. attribute:: MaskedArray.hardmask +.. autoattribute:: MaskedArray.baseclass - Returns whether the mask is hard (``True``) or soft (``False``). - When the mask is hard, masked entries cannot be unmasked. +.. autoattribute:: MaskedArray.sharedmask +.. autoattribute:: MaskedArray.hardmask As :class:`MaskedArray` is a subclass of :class:`~numpy.ndarray`, a masked array also inherits all the attributes and properties of a :class:`~numpy.ndarray` instance. @@ -184,10 +121,8 @@ Conversion :toctree: generated/ MaskedArray.__float__ - MaskedArray.__hex__ MaskedArray.__int__ MaskedArray.__long__ - MaskedArray.__oct__ MaskedArray.view MaskedArray.astype @@ -311,7 +246,7 @@ Truth value of an array (:func:`bool()`): .. autosummary:: :toctree: generated/ - MaskedArray.__nonzero__ + MaskedArray.__bool__ Arithmetic: @@ -328,7 +263,6 @@ Arithmetic: MaskedArray.__mul__ MaskedArray.__rmul__ MaskedArray.__div__ - MaskedArray.__rdiv__ MaskedArray.__truediv__ MaskedArray.__rtruediv__ MaskedArray.__floordiv__ diff --git a/doc/source/reference/maskedarray.generic.rst b/doc/source/reference/maskedarray.generic.rst index 07ad6c292..7375d60fb 100644 --- a/doc/source/reference/maskedarray.generic.rst +++ b/doc/source/reference/maskedarray.generic.rst @@ -2,7 +2,7 @@ .. _maskedarray.generic: - +.. module:: numpy.ma The :mod:`numpy.ma` module ========================== diff --git a/doc/source/reference/random/bit_generators/bitgenerators.rst b/doc/source/reference/random/bit_generators/bitgenerators.rst new file mode 100644 index 000000000..1474f7dac --- /dev/null +++ b/doc/source/reference/random/bit_generators/bitgenerators.rst @@ -0,0 +1,11 @@ +:orphan: + +BitGenerator +------------ + +.. currentmodule:: numpy.random.bit_generator + +.. autosummary:: + :toctree: generated/ + + BitGenerator diff --git a/doc/source/reference/random/bit_generators/index.rst b/doc/source/reference/random/bit_generators/index.rst new file mode 100644 index 000000000..35d9e5d09 --- /dev/null +++ b/doc/source/reference/random/bit_generators/index.rst @@ -0,0 +1,112 @@ +.. _bit_generator: + +.. currentmodule:: numpy.random + +Bit Generators +-------------- + +The random values produced by :class:`~Generator` +orignate in a BitGenerator. The BitGenerators do not directly provide +random numbers and only contains methods used for seeding, getting or +setting the state, jumping or advancing the state, and for accessing +low-level wrappers for consumption by code that can efficiently +access the functions provided, e.g., `numba <https://numba.pydata.org>`_. + +Supported BitGenerators +======================= + +The included BitGenerators are: + +* PCG-64 - The default. A fast generator that supports many parallel streams + and can be advanced by an arbitrary amount. See the documentation for + :meth:`~.PCG64.advance`. PCG-64 has a period of :math:`2^{128}`. See the `PCG + author's page`_ for more details about this class of PRNG. +* MT19937 - The standard Python BitGenerator. Adds a `~mt19937.MT19937.jumped` + function that returns a new generator with state as-if :math:`2^{128}` draws have + been made. +* Philox - A counter-based generator capable of being advanced an + arbitrary number of steps or generating independent streams. See the + `Random123`_ page for more details about this class of bit generators. +* SFC64 - A fast generator based on random invertible mappings. Usually the + fastest generator of the four. See the `SFC author's page`_ for (a little) + more detail. + +.. _`PCG author's page`: http://www.pcg-random.org/ +.. _`Random123`: https://www.deshawresearch.com/resources_random123.html +.. _`SFC author's page`: http://pracrand.sourceforge.net/RNG_engines.txt + +.. toctree:: + :maxdepth: 1 + + BitGenerator <bitgenerators> + MT19937 <mt19937> + PCG64 <pcg64> + Philox <philox> + SFC64 <sfc64> + +Seeding and Entropy +------------------- + +A BitGenerator provides a stream of random values. In order to generate +reproducible streams, BitGenerators support setting their initial state via a +seed. All of the provided BitGenerators will take an arbitrary-sized +non-negative integer, or a list of such integers, as a seed. BitGenerators +need to take those inputs and process them into a high-quality internal state +for the BitGenerator. All of the BitGenerators in numpy delegate that task to +`~SeedSequence`, which uses hashing techniques to ensure that even low-quality +seeds generate high-quality initial states. + +.. code-block:: python + + from numpy.random import PCG64 + + bg = PCG64(12345678903141592653589793) + +.. end_block + +`~SeedSequence` is designed to be convenient for implementing best practices. +We recommend that a stochastic program defaults to using entropy from the OS so +that each run is different. The program should print out or log that entropy. +In order to reproduce a past value, the program should allow the user to +provide that value through some mechanism, a command-line argument is common, +so that the user can then re-enter that entropy to reproduce the result. +`~SeedSequence` can take care of everything except for communicating with the +user, which is up to you. + +.. code-block:: python + + from numpy.random import PCG64, SeedSequence + + # Get the user's seed somehow, maybe through `argparse`. + # If the user did not provide a seed, it should return `None`. + seed = get_user_seed() + ss = SeedSequence(seed) + print('seed = {}'.format(ss.entropy)) + bg = PCG64(ss) + +.. end_block + +We default to using a 128-bit integer using entropy gathered from the OS. This +is a good amount of entropy to initialize all of the generators that we have in +numpy. We do not recommend using small seeds below 32 bits for general use. +Using just a small set of seeds to instantiate larger state spaces means that +there are some initial states that are impossible to reach. This creates some +biases if everyone uses such values. + +There will not be anything *wrong* with the results, per se; even a seed of +0 is perfectly fine thanks to the processing that `~SeedSequence` does. If you +just need *some* fixed value for unit tests or debugging, feel free to use +whatever seed you like. But if you want to make inferences from the results or +publish them, drawing from a larger set of seeds is good practice. + +If you need to generate a good seed "offline", then ``SeedSequence().entropy`` +or using ``secrets.randbits(128)`` from the standard library are both +convenient ways. + +.. autosummary:: + :toctree: generated/ + + SeedSequence + bit_generator.ISeedSequence + bit_generator.ISpawnableSeedSequence + bit_generator.SeedlessSeedSequence diff --git a/doc/source/reference/random/bit_generators/mt19937.rst b/doc/source/reference/random/bit_generators/mt19937.rst new file mode 100644 index 000000000..25ba1d7b5 --- /dev/null +++ b/doc/source/reference/random/bit_generators/mt19937.rst @@ -0,0 +1,34 @@ +Mersenne Twister (MT19937) +-------------------------- + +.. module:: numpy.random.mt19937 + +.. currentmodule:: numpy.random.mt19937 + +.. autoclass:: MT19937 + :exclude-members: + +State +===== + +.. autosummary:: + :toctree: generated/ + + ~MT19937.state + +Parallel generation +=================== +.. autosummary:: + :toctree: generated/ + + ~MT19937.jumped + +Extending +========= +.. autosummary:: + :toctree: generated/ + + ~MT19937.cffi + ~MT19937.ctypes + + diff --git a/doc/source/reference/random/bit_generators/pcg64.rst b/doc/source/reference/random/bit_generators/pcg64.rst new file mode 100644 index 000000000..7aef1e0dd --- /dev/null +++ b/doc/source/reference/random/bit_generators/pcg64.rst @@ -0,0 +1,33 @@ +Parallel Congruent Generator (64-bit, PCG64) +-------------------------------------------- + +.. module:: numpy.random.pcg64 + +.. currentmodule:: numpy.random.pcg64 + +.. autoclass:: PCG64 + :exclude-members: + +State +===== + +.. autosummary:: + :toctree: generated/ + + ~PCG64.state + +Parallel generation +=================== +.. autosummary:: + :toctree: generated/ + + ~PCG64.advance + ~PCG64.jumped + +Extending +========= +.. autosummary:: + :toctree: generated/ + + ~PCG64.cffi + ~PCG64.ctypes diff --git a/doc/source/reference/random/bit_generators/philox.rst b/doc/source/reference/random/bit_generators/philox.rst new file mode 100644 index 000000000..5e581e094 --- /dev/null +++ b/doc/source/reference/random/bit_generators/philox.rst @@ -0,0 +1,35 @@ +Philox Counter-based RNG +------------------------ + +.. module:: numpy.random.philox + +.. currentmodule:: numpy.random.philox + +.. autoclass:: Philox + :exclude-members: + +State +===== + +.. autosummary:: + :toctree: generated/ + + ~Philox.state + +Parallel generation +=================== +.. autosummary:: + :toctree: generated/ + + ~Philox.advance + ~Philox.jumped + +Extending +========= +.. autosummary:: + :toctree: generated/ + + ~Philox.cffi + ~Philox.ctypes + + diff --git a/doc/source/reference/random/bit_generators/sfc64.rst b/doc/source/reference/random/bit_generators/sfc64.rst new file mode 100644 index 000000000..dc03820ae --- /dev/null +++ b/doc/source/reference/random/bit_generators/sfc64.rst @@ -0,0 +1,28 @@ +SFC64 Small Fast Chaotic PRNG +----------------------------- + +.. module:: numpy.random.sfc64 + +.. currentmodule:: numpy.random.sfc64 + +.. autoclass:: SFC64 + :exclude-members: + +State +===== + +.. autosummary:: + :toctree: generated/ + + ~SFC64.state + +Extending +========= +.. autosummary:: + :toctree: generated/ + + ~SFC64.cffi + ~SFC64.ctypes + + + diff --git a/doc/source/reference/random/entropy.rst b/doc/source/reference/random/entropy.rst new file mode 100644 index 000000000..0664da6f9 --- /dev/null +++ b/doc/source/reference/random/entropy.rst @@ -0,0 +1,6 @@ +System Entropy +============== + +.. module:: numpy.random.entropy + +.. autofunction:: random_entropy diff --git a/doc/source/reference/random/extending.rst b/doc/source/reference/random/extending.rst new file mode 100644 index 000000000..22f9cb7e4 --- /dev/null +++ b/doc/source/reference/random/extending.rst @@ -0,0 +1,165 @@ +.. currentmodule:: numpy.random + +Extending +--------- +The BitGenerators have been designed to be extendable using standard tools for +high-performance Python -- numba and Cython. The `~Generator` object can also +be used with user-provided BitGenerators as long as these export a small set of +required functions. + +Numba +===== +Numba can be used with either CTypes or CFFI. The current iteration of the +BitGenerators all export a small set of functions through both interfaces. + +This example shows how numba can be used to produce Box-Muller normals using +a pure Python implementation which is then compiled. The random numbers are +provided by ``ctypes.next_double``. + +.. code-block:: python + + from numpy.random import PCG64 + import numpy as np + import numba as nb + + x = PCG64() + f = x.ctypes.next_double + s = x.ctypes.state + state_addr = x.ctypes.state_address + + def normals(n, state): + out = np.empty(n) + for i in range((n+1)//2): + x1 = 2.0*f(state) - 1.0 + x2 = 2.0*f(state) - 1.0 + r2 = x1*x1 + x2*x2 + while r2 >= 1.0 or r2 == 0.0: + x1 = 2.0*f(state) - 1.0 + x2 = 2.0*f(state) - 1.0 + r2 = x1*x1 + x2*x2 + g = np.sqrt(-2.0*np.log(r2)/r2) + out[2*i] = g*x1 + if 2*i+1 < n: + out[2*i+1] = g*x2 + return out + + # Compile using Numba + print(normals(10, s).var()) + # Warm up + normalsj = nb.jit(normals, nopython=True) + # Must use state address not state with numba + normalsj(1, state_addr) + %timeit normalsj(1000000, state_addr) + print('1,000,000 Box-Muller (numba/PCG64) randoms') + %timeit np.random.standard_normal(1000000) + print('1,000,000 Box-Muller (NumPy) randoms') + + +Both CTypes and CFFI allow the more complicated distributions to be used +directly in Numba after compiling the file distributions.c into a DLL or so. +An example showing the use of a more complicated distribution is in the +examples folder. + +.. _randomgen_cython: + +Cython +====== + +Cython can be used to unpack the ``PyCapsule`` provided by a BitGenerator. +This example uses `~pcg64.PCG64` and +``random_gauss_zig``, the Ziggurat-based generator for normals, to fill an +array. The usual caveats for writing high-performance code using Cython -- +removing bounds checks and wrap around, providing array alignment information +-- still apply. + +.. code-block:: cython + + import numpy as np + cimport numpy as np + cimport cython + from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer + from numpy.random.common cimport * + from numpy.random.distributions cimport random_gauss_zig + from numpy.random import PCG64 + + + @cython.boundscheck(False) + @cython.wraparound(False) + def normals_zig(Py_ssize_t n): + cdef Py_ssize_t i + cdef bitgen_t *rng + cdef const char *capsule_name = "BitGenerator" + cdef double[::1] random_values + + x = PCG64() + capsule = x.capsule + if not PyCapsule_IsValid(capsule, capsule_name): + raise ValueError("Invalid pointer to anon_func_state") + rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name) + random_values = np.empty(n) + # Best practice is to release GIL and acquire the lock + with x.lock, nogil: + for i in range(n): + random_values[i] = random_gauss_zig(rng) + randoms = np.asarray(random_values) + return randoms + +The BitGenerator can also be directly accessed using the members of the basic +RNG structure. + +.. code-block:: cython + + @cython.boundscheck(False) + @cython.wraparound(False) + def uniforms(Py_ssize_t n): + cdef Py_ssize_t i + cdef bitgen_t *rng + cdef const char *capsule_name = "BitGenerator" + cdef double[::1] random_values + + x = PCG64() + capsule = x.capsule + # Optional check that the capsule if from a BitGenerator + if not PyCapsule_IsValid(capsule, capsule_name): + raise ValueError("Invalid pointer to anon_func_state") + # Cast the pointer + rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name) + random_values = np.empty(n) + with x.lock, nogil: + for i in range(n): + # Call the function + random_values[i] = rng.next_double(rng.state) + randoms = np.asarray(random_values) + return randoms + +These functions along with a minimal setup file are included in the +examples folder. + +New Basic RNGs +============== +`~Generator` can be used with other user-provided BitGenerators. The simplest +way to write a new BitGenerator is to examine the pyx file of one of the +existing BitGenerators. The key structure that must be provided is the +``capsule`` which contains a ``PyCapsule`` to a struct pointer of type +``bitgen_t``, + +.. code-block:: c + + typedef struct bitgen { + void *state; + uint64_t (*next_uint64)(void *st); + uint32_t (*next_uint32)(void *st); + double (*next_double)(void *st); + uint64_t (*next_raw)(void *st); + } bitgen_t; + +which provides 5 pointers. The first is an opaque pointer to the data structure +used by the BitGenerators. The next three are function pointers which return +the next 64- and 32-bit unsigned integers, the next random double and the next +raw value. This final function is used for testing and so can be set to +the next 64-bit unsigned integer function if not needed. Functions inside +``Generator`` use this structure as in + +.. code-block:: c + + bitgen_state->next_uint64(bitgen_state->state) diff --git a/doc/source/reference/random/generator.rst b/doc/source/reference/random/generator.rst new file mode 100644 index 000000000..068143270 --- /dev/null +++ b/doc/source/reference/random/generator.rst @@ -0,0 +1,84 @@ +.. currentmodule:: numpy.random + +Random Generator +---------------- +The `~Generator` provides access to +a wide range of distributions, and served as a replacement for +:class:`~numpy.random.RandomState`. The main difference between +the two is that ``Generator`` relies on an additional BitGenerator to +manage state and generate the random bits, which are then transformed into +random values from useful distributions. The default BitGenerator used by +``Generator`` is `~PCG64`. The BitGenerator +can be changed by passing an instantized BitGenerator to ``Generator``. + + +.. autofunction:: default_rng + +.. autoclass:: Generator + :exclude-members: + +Accessing the BitGenerator +========================== +.. autosummary:: + :toctree: generated/ + + ~numpy.random.Generator.bit_generator + +Simple random data +================== +.. autosummary:: + :toctree: generated/ + + ~numpy.random.Generator.integers + ~numpy.random.Generator.random + ~numpy.random.Generator.choice + ~numpy.random.Generator.bytes + +Permutations +============ +.. autosummary:: + :toctree: generated/ + + ~numpy.random.Generator.shuffle + ~numpy.random.Generator.permutation + +Distributions +============= +.. autosummary:: + :toctree: generated/ + + ~numpy.random.Generator.beta + ~numpy.random.Generator.binomial + ~numpy.random.Generator.chisquare + ~numpy.random.Generator.dirichlet + ~numpy.random.Generator.exponential + ~numpy.random.Generator.f + ~numpy.random.Generator.gamma + ~numpy.random.Generator.geometric + ~numpy.random.Generator.gumbel + ~numpy.random.Generator.hypergeometric + ~numpy.random.Generator.laplace + ~numpy.random.Generator.logistic + ~numpy.random.Generator.lognormal + ~numpy.random.Generator.logseries + ~numpy.random.Generator.multinomial + ~numpy.random.Generator.multivariate_normal + ~numpy.random.Generator.negative_binomial + ~numpy.random.Generator.noncentral_chisquare + ~numpy.random.Generator.noncentral_f + ~numpy.random.Generator.normal + ~numpy.random.Generator.pareto + ~numpy.random.Generator.poisson + ~numpy.random.Generator.power + ~numpy.random.Generator.rayleigh + ~numpy.random.Generator.standard_cauchy + ~numpy.random.Generator.standard_exponential + ~numpy.random.Generator.standard_gamma + ~numpy.random.Generator.standard_normal + ~numpy.random.Generator.standard_t + ~numpy.random.Generator.triangular + ~numpy.random.Generator.uniform + ~numpy.random.Generator.vonmises + ~numpy.random.Generator.wald + ~numpy.random.Generator.weibull + ~numpy.random.Generator.zipf diff --git a/doc/source/reference/random/index.rst b/doc/source/reference/random/index.rst new file mode 100644 index 000000000..01f9981a2 --- /dev/null +++ b/doc/source/reference/random/index.rst @@ -0,0 +1,212 @@ +.. _numpyrandom: + +.. py:module:: numpy.random + +.. currentmodule:: numpy.random + +Random sampling (:mod:`numpy.random`) +===================================== + +Numpy's random number routines produce pseudo random numbers using +combinations of a `BitGenerator` to create sequences and a `Generator` +to use those sequences to sample from different statistical distributions: + +* BitGenerators: Objects that generate random numbers. These are typically + unsigned integer words filled with sequences of either 32 or 64 random bits. +* Generators: Objects that transform sequences of random bits from a + BitGenerator into sequences of numbers that follow a specific probability + distribution (such as uniform, Normal or Binomial) within a specified + interval. + +Since Numpy version 1.17.0 the Generator can be initialized with a +number of different BitGenerators. It exposes many different probability +distributions. See `NEP 19 <https://www.numpy.org/neps/ +nep-0019-rng-policy.html>`_ for context on the updated random Numpy number +routines. The legacy `.RandomState` random number routines are still +available, but limited to a single BitGenerator. + +For convenience and backward compatibility, a single `~.RandomState` +instance's methods are imported into the numpy.random namespace, see +:ref:`legacy` for the complete list. + +Quick Start +----------- + +By default, `~Generator` uses bits provided by `~pcg64.PCG64` which +has better statistical properties than the legacy mt19937 random +number generator in `~.RandomState`. + +.. code-block:: python + + # Uses the old numpy.random.RandomState + from numpy import random + random.standard_normal() + +`~Generator` can be used as a replacement for `~.RandomState`. Both class +instances now hold a internal `BitGenerator` instance to provide the bit +stream, it is accessible as ``gen.bit_generator``. Some long-overdue API +cleanup means that legacy and compatibility methods have been removed from +`~.Generator` + +=================== ============== ============ +`~.RandomState` `~.Generator` Notes +------------------- -------------- ------------ +``random_sample``, ``random`` Compatible with `random.random` +``rand`` +------------------- -------------- ------------ +``randint``, ``integers`` Add an ``endpoint`` kwarg +``random_integers`` +------------------- -------------- ------------ +``tomaxint`` removed Use ``integers(0, np.iinfo(np.int).max,`` + ``endpoint=False)`` +------------------- -------------- ------------ +``seed`` removed Use `~.SeedSequence.spawn` +=================== ============== ============ + +See `new-or-different` for more information + +.. code-block:: python + + # As replacement for RandomState(); default_rng() instantiates Generator with + # the default PCG64 BitGenerator. + from numpy.random import default_rng + rg = default_rng() + rg.standard_normal() + rg.bit_generator + +Something like the following code can be used to support both ``RandomState`` +and ``Generator``, with the understanding that the interfaces are slightly +different + +.. code-block:: python + + try: + rg_integers = rg.integers + except AttributeError: + rg_integers = rg.randint + a = rg_integers(1000) + +Seeds can be passed to any of the BitGenerators. The provided value is mixed +via `~.SeedSequence` to spread a possible sequence of seeds across a wider +range of initialization states for the BitGenerator. Here `~.PCG64` is used and +is wrapped with a `~.Generator`. + +.. code-block:: python + + from numpy.random import Generator, PCG64 + rg = Generator(PCG64(12345)) + rg.standard_normal() + +Introduction +------------ +The new infrastructure takes a different approach to producing random numbers +from the `~.RandomState` object. Random number generation is separated into +two components, a bit generator and a random generator. + +The `BitGenerator` has a limited set of responsibilities. It manages state +and provides functions to produce random doubles and random unsigned 32- and +64-bit values. + +The `random generator <Generator>` takes the +bit generator-provided stream and transforms them into more useful +distributions, e.g., simulated normal random values. This structure allows +alternative bit generators to be used with little code duplication. + +The `Generator` is the user-facing object that is nearly identical to +`.RandomState`. The canonical method to initialize a generator passes a +`~.PCG64` bit generator as the sole argument. + +.. code-block:: python + + from numpy.random import default_rng + rg = default_rng(12345) + rg.random() + +One can also instantiate `Generator` directly with a `BitGenerator` instance. +To use the older `~mt19937.MT19937` algorithm, one can instantiate it directly +and pass it to `Generator`. + +.. code-block:: python + + from numpy.random import Generator, MT19937 + rg = Generator(MT19937(12345)) + rg.random() + +What's New or Different +~~~~~~~~~~~~~~~~~~~~~~~ +.. warning:: + + The Box-Muller method used to produce NumPy's normals is no longer available + in `Generator`. It is not possible to reproduce the exact random + values using Generator for the normal distribution or any other + distribution that relies on the normal such as the `.RandomState.gamma` or + `.RandomState.standard_t`. If you require bitwise backward compatible + streams, use `.RandomState`. + +* The Generator's normal, exponential and gamma functions use 256-step Ziggurat + methods which are 2-10 times faster than NumPy's Box-Muller or inverse CDF + implementations. +* Optional ``dtype`` argument that accepts ``np.float32`` or ``np.float64`` + to produce either single or double prevision uniform random variables for + select distributions +* Optional ``out`` argument that allows existing arrays to be filled for + select distributions +* `~entropy.random_entropy` provides access to the system + source of randomness that is used in cryptographic applications (e.g., + ``/dev/urandom`` on Unix). +* All BitGenerators can produce doubles, uint64s and uint32s via CTypes + (`~.PCG64.ctypes`) and CFFI (`~.PCG64.cffi`). This allows the bit generators + to be used in numba. +* The bit generators can be used in downstream projects via + :ref:`Cython <randomgen_cython>`. +* `~.Generator.integers` is now the canonical way to generate integer + random numbers from a discrete uniform distribution. The ``rand`` and + ``randn`` methods are only available through the legacy `~.RandomState`. + The ``endpoint`` keyword can be used to specify open or closed intervals. + This replaces both ``randint`` and the deprecated ``random_integers``. +* `~.Generator.random` is now the canonical way to generate floating-point + random numbers, which replaces `.RandomState.random_sample`, + `.RandomState.sample`, and `.RandomState.ranf`. This is consistent with + Python's `random.random`. +* All BitGenerators in numpy use `~SeedSequence` to convert seeds into + initialized states. + +See :ref:`new-or-different` for a complete list of improvements and +differences from the traditional ``Randomstate``. + +Parallel Generation +~~~~~~~~~~~~~~~~~~~ + +The included generators can be used in parallel, distributed applications in +one of three ways: + +* :ref:`seedsequence-spawn` +* :ref:`independent-streams` +* :ref:`parallel-jumped` + +Concepts +-------- +.. toctree:: + :maxdepth: 1 + + generator + legacy mtrand <legacy> + BitGenerators, SeedSequences <bit_generators/index> + +Features +-------- +.. toctree:: + :maxdepth: 2 + + Parallel Applications <parallel> + Multithreaded Generation <multithreading> + new-or-different + Comparing Performance <performance> + extending + Reading System Entropy <entropy> + +Original Source +~~~~~~~~~~~~~~~ + +This package was developed independently of NumPy and was integrated in version +1.17.0. The original repo is at https://github.com/bashtage/randomgen. diff --git a/doc/source/reference/random/legacy.rst b/doc/source/reference/random/legacy.rst new file mode 100644 index 000000000..04d4d3569 --- /dev/null +++ b/doc/source/reference/random/legacy.rst @@ -0,0 +1,125 @@ +.. currentmodule:: numpy.random + +.. _legacy: + +Legacy Random Generation +------------------------ +The `~mtrand.RandomState` provides access to +legacy generators. This generator is considered frozen and will have +no further improvements. It is guaranteed to produce the same values +as the final point release of NumPy v1.16. These all depend on Box-Muller +normals or inverse CDF exponentials or gammas. This class should only be used +if it is essential to have randoms that are identical to what +would have been produced by previous versions of NumPy. + +`~mtrand.RandomState` adds additional information +to the state which is required when using Box-Muller normals since these +are produced in pairs. It is important to use +`~mtrand.RandomState.get_state`, and not the underlying bit generators +`state`, when accessing the state so that these extra values are saved. + +Although we provide the `~mt19937.MT19937` BitGenerator for use independent of +`~mtrand.RandomState`, note that its default seeding uses `~SeedSequence` +rather than the legacy seeding algorithm. `~mtrand.RandomState` will use the +legacy seeding algorithm. The methods to use the legacy seeding algorithm are +currently private as the main reason to use them is just to implement +`~mtrand.RandomState`. However, one can reset the state of `~mt19937.MT19937` +using the state of the `~mtrand.RandomState`: + +.. code-block:: python + + from numpy.random import MT19937 + from numpy.random import RandomState + + rs = RandomState(12345) + mt19937 = MT19937() + mt19937.state = rs.get_state() + rs2 = RandomState(mt19937) + + # Same output + rs.standard_normal() + rs2.standard_normal() + + rs.random() + rs2.random() + + rs.standard_exponential() + rs2.standard_exponential() + + +.. currentmodule:: numpy.random.mtrand + +.. autoclass:: RandomState + :exclude-members: + +Seeding and State +================= + +.. autosummary:: + :toctree: generated/ + + ~RandomState.get_state + ~RandomState.set_state + ~RandomState.seed + +Simple random data +================== +.. autosummary:: + :toctree: generated/ + + ~RandomState.rand + ~RandomState.randn + ~RandomState.randint + ~RandomState.random_integers + ~RandomState.random_sample + ~RandomState.choice + ~RandomState.bytes + +Permutations +============ +.. autosummary:: + :toctree: generated/ + + ~RandomState.shuffle + ~RandomState.permutation + +Distributions +============= +.. autosummary:: + :toctree: generated/ + + ~RandomState.beta + ~RandomState.binomial + ~RandomState.chisquare + ~RandomState.dirichlet + ~RandomState.exponential + ~RandomState.f + ~RandomState.gamma + ~RandomState.geometric + ~RandomState.gumbel + ~RandomState.hypergeometric + ~RandomState.laplace + ~RandomState.logistic + ~RandomState.lognormal + ~RandomState.logseries + ~RandomState.multinomial + ~RandomState.multivariate_normal + ~RandomState.negative_binomial + ~RandomState.noncentral_chisquare + ~RandomState.noncentral_f + ~RandomState.normal + ~RandomState.pareto + ~RandomState.poisson + ~RandomState.power + ~RandomState.rayleigh + ~RandomState.standard_cauchy + ~RandomState.standard_exponential + ~RandomState.standard_gamma + ~RandomState.standard_normal + ~RandomState.standard_t + ~RandomState.triangular + ~RandomState.uniform + ~RandomState.vonmises + ~RandomState.wald + ~RandomState.weibull + ~RandomState.zipf diff --git a/doc/source/reference/random/multithreading.rst b/doc/source/reference/random/multithreading.rst new file mode 100644 index 000000000..6883d3672 --- /dev/null +++ b/doc/source/reference/random/multithreading.rst @@ -0,0 +1,108 @@ +Multithreaded Generation +======================== + +The four core distributions (:meth:`~.Generator.random`, +:meth:`~.Generator.standard_normal`, :meth:`~.Generator.standard_exponential`, +and :meth:`~.Generator.standard_gamma`) all allow existing arrays to be filled +using the ``out`` keyword argument. Existing arrays need to be contiguous and +well-behaved (writable and aligned). Under normal circumstances, arrays +created using the common constructors such as :meth:`numpy.empty` will satisfy +these requirements. + +This example makes use of Python 3 :mod:`concurrent.futures` to fill an array +using multiple threads. Threads are long-lived so that repeated calls do not +require any additional overheads from thread creation. The underlying +BitGenerator is `PCG64` which is fast, has a long period and supports +using `PCG64.jumped` to return a new generator while advancing the +state. The random numbers generated are reproducible in the sense that the same +seed will produce the same outputs. + +.. code-block:: ipython + + from numpy.random import Generator, PCG64 + import multiprocessing + import concurrent.futures + import numpy as np + + class MultithreadedRNG(object): + def __init__(self, n, seed=None, threads=None): + rg = PCG64(seed) + if threads is None: + threads = multiprocessing.cpu_count() + self.threads = threads + + self._random_generators = [rg] + last_rg = rg + for _ in range(0, threads-1): + new_rg = last_rg.jumped() + self._random_generators.append(new_rg) + last_rg = new_rg + + self.n = n + self.executor = concurrent.futures.ThreadPoolExecutor(threads) + self.values = np.empty(n) + self.step = np.ceil(n / threads).astype(np.int) + + def fill(self): + def _fill(random_state, out, first, last): + random_state.standard_normal(out=out[first:last]) + + futures = {} + for i in range(self.threads): + args = (_fill, + self._random_generators[i], + self.values, + i * self.step, + (i + 1) * self.step) + futures[self.executor.submit(*args)] = i + concurrent.futures.wait(futures) + + def __del__(self): + self.executor.shutdown(False) + + +The multithreaded random number generator can be used to fill an array. +The ``values`` attributes shows the zero-value before the fill and the +random value after. + +.. code-block:: ipython + + In [2]: mrng = MultithreadedRNG(10000000, seed=0) + ...: print(mrng.values[-1]) + 0.0 + + In [3]: mrng.fill() + ...: print(mrng.values[-1]) + 3.296046120254392 + +The time required to produce using multiple threads can be compared to +the time required to generate using a single thread. + +.. code-block:: ipython + + In [4]: print(mrng.threads) + ...: %timeit mrng.fill() + + 4 + 32.8 ms ± 2.71 ms per loop (mean ± std. dev. of 7 runs, 10 loops each) + +The single threaded call directly uses the BitGenerator. + +.. code-block:: ipython + + In [5]: values = np.empty(10000000) + ...: rg = Generator(PCG64()) + ...: %timeit rg.standard_normal(out=values) + + 99.6 ms ± 222 µs per loop (mean ± std. dev. of 7 runs, 10 loops each) + +The gains are substantial and the scaling is reasonable even for large that +are only moderately large. The gains are even larger when compared to a call +that does not use an existing array due to array creation overhead. + +.. code-block:: ipython + + In [6]: rg = Generator(PCG64()) + ...: %timeit rg.standard_normal(10000000) + + 125 ms ± 309 µs per loop (mean ± std. dev. of 7 runs, 10 loops each) diff --git a/doc/source/reference/random/new-or-different.rst b/doc/source/reference/random/new-or-different.rst new file mode 100644 index 000000000..5442f46c9 --- /dev/null +++ b/doc/source/reference/random/new-or-different.rst @@ -0,0 +1,118 @@ +.. _new-or-different: + +.. currentmodule:: numpy.random + +What's New or Different +----------------------- + +.. warning:: + + The Box-Muller method used to produce NumPy's normals is no longer available + in `Generator`. It is not possible to reproduce the exact random + values using ``Generator`` for the normal distribution or any other + distribution that relies on the normal such as the `gamma` or + `standard_t`. If you require bitwise backward compatible + streams, use `RandomState`. + +Quick comparison of legacy `mtrand <legacy>`_ to the new `Generator` + +================== ==================== ============= +Feature Older Equivalent Notes +------------------ -------------------- ------------- +`~.Generator` `~.RandomState` ``Generator`` requires a stream + source, called a `BitGenerator + <bit_generators>` A number of these + are provided. ``RandomState`` uses + the Mersenne Twister `~.MT19937` by + default, but can also be instantiated + with any BitGenerator. +------------------ -------------------- ------------- +``random`` ``random_sample``, Access the values in a BitGenerator, + ``rand`` convert them to ``float64`` in the + interval ``[0.0.,`` `` 1.0)``. + In addition to the ``size`` kwarg, now + supports ``dtype='d'`` or ``dtype='f'``, + and an ``out`` kwarg to fill a user- + supplied array. + + Many other distributions are also + supported. +------------------ -------------------- ------------- +``integers`` ``randint``, Use the ``endpoint`` kwarg to adjust + ``random_integers`` the inclusion or exclution of the + ``high`` interval endpoint +================== ==================== ============= + +And in more detail: + +* `~.entropy.random_entropy` provides access to the system + source of randomness that is used in cryptographic applications (e.g., + ``/dev/urandom`` on Unix). +* Simulate from the complex normal distribution + (`~.Generator.complex_normal`) +* The normal, exponential and gamma generators use 256-step Ziggurat + methods which are 2-10 times faster than NumPy's default implementation in + `~.Generator.standard_normal`, `~.Generator.standard_exponential` or + `~.Generator.standard_gamma`. +* `~.Generator.integers` is now the canonical way to generate integer + random numbers from a discrete uniform distribution. The ``rand`` and + ``randn`` methods are only available through the legacy `~.RandomState`. + This replaces both ``randint`` and the deprecated ``random_integers``. +* The Box-Muller method used to produce NumPy's normals is no longer available. +* All bit generators can produce doubles, uint64s and + uint32s via CTypes (`~PCG64.ctypes`) and CFFI (`~PCG64.cffi`). + This allows these bit generators to be used in numba. +* The bit generators can be used in downstream projects via + Cython. + + +.. ipython:: python + + from numpy.random import Generator, PCG64 + import numpy.random + rg = Generator(PCG64()) + %timeit rg.standard_normal(100000) + %timeit numpy.random.standard_normal(100000) + +.. ipython:: python + + %timeit rg.standard_exponential(100000) + %timeit numpy.random.standard_exponential(100000) + +.. ipython:: python + + %timeit rg.standard_gamma(3.0, 100000) + %timeit numpy.random.standard_gamma(3.0, 100000) + +* Optional ``dtype`` argument that accepts ``np.float32`` or ``np.float64`` + to produce either single or double prevision uniform random variables for + select distributions + + * Uniforms (`~.Generator.random` and `~.Generator.integers`) + * Normals (`~.Generator.standard_normal`) + * Standard Gammas (`~.Generator.standard_gamma`) + * Standard Exponentials (`~.Generator.standard_exponential`) + +.. ipython:: python + + rg = Generator(PCG64(0)) + rg.random(3, dtype='d') + rg.random(3, dtype='f') + +* Optional ``out`` argument that allows existing arrays to be filled for + select distributions + + * Uniforms (`~.Generator.random`) + * Normals (`~.Generator.standard_normal`) + * Standard Gammas (`~.Generator.standard_gamma`) + * Standard Exponentials (`~.Generator.standard_exponential`) + + This allows multithreading to fill large arrays in chunks using suitable + BitGenerators in parallel. + +.. ipython:: python + + existing = np.zeros(4) + rg.random(out=existing[:2]) + print(existing) + diff --git a/doc/source/reference/random/parallel.rst b/doc/source/reference/random/parallel.rst new file mode 100644 index 000000000..2f79f22d8 --- /dev/null +++ b/doc/source/reference/random/parallel.rst @@ -0,0 +1,193 @@ +Parallel Random Number Generation +================================= + +There are three strategies implemented that can be used to produce +repeatable pseudo-random numbers across multiple processes (local +or distributed). + +.. currentmodule:: numpy.random + +.. _seedsequence-spawn: + +`~SeedSequence` spawning +------------------------ + +`~SeedSequence` `implements an algorithm`_ to process a user-provided seed, +typically as an integer of some size, and to convert it into an initial state for +a `~BitGenerator`. It uses hashing techniques to ensure that low-quality seeds +are turned into high quality initial states (at least, with very high +probability). + +For example, `~mt19937.MT19937` has a state consisting of 624 +`uint32` integers. A naive way to take a 32-bit integer seed would be to just set +the last element of the state to the 32-bit seed and leave the rest 0s. This is +a valid state for `~mt19937.MT19937`, but not a good one. The Mersenne Twister +algorithm `suffers if there are too many 0s`_. Similarly, two adjacent 32-bit +integer seeds (i.e. ``12345`` and ``12346``) would produce very similar +streams. + +`~SeedSequence` avoids these problems by using successions of integer hashes +with good `avalanche properties`_ to ensure that flipping any bit in the input +input has about a 50% chance of flipping any bit in the output. Two input seeds +that are very close to each other will produce initial states that are very far +from each other (with very high probability). It is also constructed in such +a way that you can provide arbitrary-sized integers or lists of integers. +`~SeedSequence` will take all of the bits that you provide and mix them +together to produce however many bits the consuming `~BitGenerator` needs to +initialize itself. + +These properties together mean that we can safely mix together the usual +user-provided seed with simple incrementing counters to get `~BitGenerator` +states that are (to very high probability) independent of each other. We can +wrap this together into an API that is easy to use and difficult to misuse. + +.. code-block:: python + + from numpy.random import SeedSequence, default_rng + + ss = SeedSequence(12345) + + # Spawn off 10 child SeedSequences to pass to child processes. + child_seeds = ss.spawn(10) + streams = [default_rng(s) for s in child_seeds] + +.. end_block + +Child `~SeedSequence` objects can also spawn to make grandchildren, and so on. +Each `~SeedSequence` has its position in the tree of spawned `~SeedSequence` +objects mixed in with the user-provided seed to generate independent (with very +high probability) streams. + +.. code-block:: python + + grandchildren = child_seeds[0].spawn(4) + grand_streams = [default_rng(s) for s in grandchildren] + +.. end_block + +This feature lets you make local decisions about when and how to split up +streams without coordination between processes. You do not have to preallocate +space to avoid overlapping or request streams from a common global service. This +general "tree-hashing" scheme is `not unique to numpy`_ but not yet widespread. +Python has increasingly-flexible mechanisms for parallelization available, and +this scheme fits in very well with that kind of use. + +Using this scheme, an upper bound on the probability of a collision can be +estimated if one knows the number of streams that you derive. `~SeedSequence` +hashes its inputs, both the seed and the spawn-tree-path, down to a 128-bit +pool by default. The probability that there is a collision in +that pool, pessimistically-estimated ([1]_), will be about :math:`n^2*2^{-128}` where +`n` is the number of streams spawned. If a program uses an aggressive million +streams, about :math:`2^{20}`, then the probability that at least one pair of +them are identical is about :math:`2^{-88}`, which is in solidly-ignorable +territory ([2]_). + +.. [1] The algorithm is carefully designed to eliminate a number of possible + ways to collide. For example, if one only does one level of spawning, it + is guaranteed that all states will be unique. But it's easier to + estimate the naive upper bound on a napkin and take comfort knowing + that the probability is actually lower. + +.. [2] In this calculation, we can ignore the amount of numbers drawn from each + stream. Each of the PRNGs we provide has some extra protection built in + that avoids overlaps if the `~SeedSequence` pools differ in the + slightest bit. `~pcg64.PCG64` has :math:`2^{127}` separate cycles + determined by the seed in addition to the position in the + :math:`2^{128}` long period for each cycle, so one has to both get on or + near the same cycle *and* seed a nearby position in the cycle. + `~philox.Philox` has completely independent cycles determined by the seed. + `~sfc64.SFC64` incorporates a 64-bit counter so every unique seed is at + least :math:`2^{64}` iterations away from any other seed. And + finally, `~mt19937.MT19937` has just an unimaginably huge period. Getting + a collision internal to `~SeedSequence` is the way a failure would be + observed. + +.. _`implements an algorithm`: http://www.pcg-random.org/posts/developing-a-seed_seq-alternative.html +.. _`suffers if there are too many 0s`: http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html +.. _`avalanche properties`: https://en.wikipedia.org/wiki/Avalanche_effect +.. _`not unique to numpy`: https://www.iro.umontreal.ca/~lecuyer/myftp/papers/parallel-rng-imacs.pdf + + +.. _independent-streams: + +Independent Streams +------------------- + +:class:`~philox.Philox` is a counter-based RNG based which generates values by +encrypting an incrementing counter using weak cryptographic primitives. The +seed determines the key that is used for the encryption. Unique keys create +unique, independent streams. :class:`~philox.Philox` lets you bypass the +seeding algorithm to directly set the 128-bit key. Similar, but different, keys +will still create independent streams. + +.. code-block:: python + + import secrets + from numpy.random import Philox + + # 128-bit number as a seed + root_seed = secrets.getrandbits(128) + streams = [Philox(key=root_seed + stream_id) for stream_id in range(10)] + +.. end_block + +This scheme does require that you avoid reusing stream IDs. This may require +coordination between the parallel processes. + + +.. _parallel-jumped: + +Jumping the BitGenerator state +------------------------------ + +``jumped`` advances the state of the BitGenerator *as-if* a large number of +random numbers have been drawn, and returns a new instance with this state. +The specific number of draws varies by BitGenerator, and ranges from +:math:`2^{64}` to :math:`2^{128}`. Additionally, the *as-if* draws also depend +on the size of the default random number produced by the specific BitGenerator. +The BitGenerators that support ``jumped``, along with the period of the +BitGenerator, the size of the jump and the bits in the default unsigned random +are listed below. + ++-----------------+-------------------------+-------------------------+-------------------------+ +| BitGenerator | Period | Jump Size | Bits | ++=================+=========================+=========================+=========================+ +| MT19937 | :math:`2^{19937}` | :math:`2^{128}` | 32 | ++-----------------+-------------------------+-------------------------+-------------------------+ +| PCG64 | :math:`2^{128}` | :math:`~2^{127}` ([3]_) | 64 | ++-----------------+-------------------------+-------------------------+-------------------------+ +| Philox | :math:`2^{256}` | :math:`2^{128}` | 64 | ++-----------------+-------------------------+-------------------------+-------------------------+ + +.. [3] The jump size is :math:`(\phi-1)*2^{128}` where :math:`\phi` is the + golden ratio. As the jumps wrap around the period, the actual distances + between neighboring streams will slowly grow smaller than the jump size, + but using the golden ratio this way is a classic method of constructing + a low-discrepancy sequence that spreads out the states around the period + optimally. You will not be able to jump enough to make those distances + small enough to overlap in your lifetime. + +``jumped`` can be used to produce long blocks which should be long enough to not +overlap. + +.. code-block:: python + + import secrets + from numpy.random import PCG64 + + seed = secrets.getrandbits(128) + blocked_rng = [] + rng = PCG64(seed) + for i in range(10): + blocked_rng.append(rng.jumped(i)) + +.. end_block + +When using ``jumped``, one does have to take care not to jump to a stream that +was already used. In the above example, one could not later use +``blocked_rng[0].jumped()`` as it would overlap with ``blocked_rng[1]``. Like +with the independent streams, if the main process here wants to split off 10 +more streams by jumping, then it needs to start with ``range(10, 20)``, +otherwise it would recreate the same streams. On the other hand, if you +carefully construct the streams, then you are guaranteed to have streams that +do not overlap. diff --git a/doc/source/reference/random/performance.py b/doc/source/reference/random/performance.py new file mode 100644 index 000000000..28a42eb0d --- /dev/null +++ b/doc/source/reference/random/performance.py @@ -0,0 +1,87 @@ +from collections import OrderedDict +from timeit import repeat + +import pandas as pd + +import numpy as np +from numpy.random import MT19937, PCG64, Philox, SFC64 + +PRNGS = [MT19937, PCG64, Philox, SFC64] + +funcs = OrderedDict() +integers = 'integers(0, 2**{bits},size=1000000, dtype="uint{bits}")' +funcs['32-bit Unsigned Ints'] = integers.format(bits=32) +funcs['64-bit Unsigned Ints'] = integers.format(bits=64) +funcs['Uniforms'] = 'random(size=1000000)' +funcs['Normals'] = 'standard_normal(size=1000000)' +funcs['Exponentials'] = 'standard_exponential(size=1000000)' +funcs['Gammas'] = 'standard_gamma(3.0,size=1000000)' +funcs['Binomials'] = 'binomial(9, .1, size=1000000)' +funcs['Laplaces'] = 'laplace(size=1000000)' +funcs['Poissons'] = 'poisson(3.0, size=1000000)' + +setup = """ +from numpy.random import {prng}, Generator +rg = Generator({prng}()) +""" + +test = "rg.{func}" +table = OrderedDict() +for prng in PRNGS: + print(prng) + col = OrderedDict() + for key in funcs: + t = repeat(test.format(func=funcs[key]), + setup.format(prng=prng().__class__.__name__), + number=1, repeat=3) + col[key] = 1000 * min(t) + col = pd.Series(col) + table[prng().__class__.__name__] = col + +npfuncs = OrderedDict() +npfuncs.update(funcs) +npfuncs['32-bit Unsigned Ints'] = 'randint(2**32,dtype="uint32",size=1000000)' +npfuncs['64-bit Unsigned Ints'] = 'randint(2**64,dtype="uint64",size=1000000)' +setup = """ +from numpy.random import RandomState +rg = RandomState() +""" +col = {} +for key in npfuncs: + t = repeat(test.format(func=npfuncs[key]), + setup.format(prng=prng().__class__.__name__), + number=1, repeat=3) + col[key] = 1000 * min(t) +table['RandomState'] = pd.Series(col) + +columns = ['MT19937','PCG64','Philox','SFC64', 'RandomState'] +table = pd.DataFrame(table) +order = np.log(table).mean().sort_values().index +table = table.T +table = table.reindex(columns) +table = table.T +table = table.reindex([k for k in funcs], axis=0) +print(table.to_csv(float_format='%0.1f')) + + +rel = table.loc[:, ['RandomState']].values @ np.ones( + (1, table.shape[1])) / table +rel.pop('RandomState') +rel = rel.T +rel['Overall'] = np.exp(np.log(rel).mean(1)) +rel *= 100 +rel = np.round(rel) +rel = rel.T +print(rel.to_csv(float_format='%0d')) + +# Cross-platform table +rows = ['32-bit Unsigned Ints','64-bit Unsigned Ints','Uniforms','Normals','Exponentials'] +xplat = rel.reindex(rows, axis=0) +xplat = 100 * (xplat / xplat.MT19937.values[:,None]) +overall = np.exp(np.log(xplat).mean(0)) +xplat = xplat.T.copy() +xplat['Overall']=overall +print(xplat.T.round(1)) + + + diff --git a/doc/source/reference/random/performance.rst b/doc/source/reference/random/performance.rst new file mode 100644 index 000000000..2d5fca496 --- /dev/null +++ b/doc/source/reference/random/performance.rst @@ -0,0 +1,153 @@ +Performance +----------- + +.. currentmodule:: numpy.random + +Recommendation +************** +The recommended generator for general use is :class:`~pcg64.PCG64`. It is +statistically high quality, full-featured, and fast on most platforms, but +somewhat slow when compiled for 32-bit processes. + +:class:`~philox.Philox` is fairly slow, but its statistical properties have +very high quality, and it is easy to get assuredly-independent stream by using +unique keys. If that is the style you wish to use for parallel streams, or you +are porting from another system that uses that style, then +:class:`~philox.Philox` is your choice. + +:class:`~sfc64.SFC64` is statistically high quality and very fast. However, it +lacks jumpability. If you are not using that capability and want lots of speed, +even on 32-bit processes, this is your choice. + +:class:`~mt19937.MT19937` `fails some statistical tests`_ and is not especially +fast compared to modern PRNGs. For these reasons, we mostly do not recommend +using it on its own, only through the legacy `~.RandomState` for +reproducing old results. That said, it has a very long history as a default in +many systems. + +.. _`fails some statistical tests`: https://www.iro.umontreal.ca/~lecuyer/myftp/papers/testu01.pdf + +Timings +******* + +The timings below are the time in ns to produce 1 random value from a +specific distribution. The original :class:`~mt19937.MT19937` generator is +much slower since it requires 2 32-bit values to equal the output of the +faster generators. + +Integer performance has a similar ordering. + +The pattern is similar for other, more complex generators. The normal +performance of the legacy :class:`~.RandomState` generator is much +lower than the other since it uses the Box-Muller transformation rather +than the Ziggurat generator. The performance gap for Exponentials is also +large due to the cost of computing the log function to invert the CDF. +The column labeled MT19973 is used the same 32-bit generator as +:class:`~.RandomState` but produces random values using +:class:`~Generator`. + +.. csv-table:: + :header: ,MT19937,PCG64,Philox,SFC64,RandomState + :widths: 14,14,14,14,14,14 + + 32-bit Unsigned Ints,3.2,2.7,4.9,2.7,3.2 + 64-bit Unsigned Ints,5.6,3.7,6.3,2.9,5.7 + Uniforms,7.3,4.1,8.1,3.1,7.3 + Normals,13.1,10.2,13.5,7.8,34.6 + Exponentials,7.9,5.4,8.5,4.1,40.3 + Gammas,34.8,28.0,34.7,25.1,58.1 + Binomials,25.0,21.4,26.1,19.5,25.2 + Laplaces,45.1,40.7,45.5,38.1,45.6 + Poissons,67.6,52.4,69.2,46.4,78.1 + +The next table presents the performance in percentage relative to values +generated by the legacy generator, `RandomState(MT19937())`. The overall +performance was computed using a geometric mean. + +.. csv-table:: + :header: ,MT19937,PCG64,Philox,SFC64 + :widths: 14,14,14,14,14 + + 32-bit Unsigned Ints,101,121,67,121 + 64-bit Unsigned Ints,102,156,91,199 + Uniforms,100,179,90,235 + Normals,263,338,257,443 + Exponentials,507,752,474,985 + Gammas,167,207,167,231 + Binomials,101,118,96,129 + Laplaces,101,112,100,120 + Poissons,116,149,113,168 + Overall,144,192,132,225 + +.. note:: + + All timings were taken using Linux on a i5-3570 processor. + +Performance on different Operating Systems +****************************************** +Performance differs across platforms due to compiler and hardware availability +(e.g., register width) differences. The default bit generator has been chosen +to perform well on 64-bit platforms. Performance on 32-bit operating systems +is very different. + +The values reported are normalized relative to the speed of MT19937 in +each table. A value of 100 indicates that the performance matches the MT19937. +Higher values indicate improved performance. These values cannot be compared +across tables. + +64-bit Linux +~~~~~~~~~~~~ + +=================== ========= ======= ======== ======= +Distribution MT19937 PCG64 Philox SFC64 +=================== ========= ======= ======== ======= +32-bit Unsigned Int 100 119.8 67.7 120.2 +64-bit Unsigned Int 100 152.9 90.8 213.3 +Uniforms 100 179.0 87.0 232.0 +Normals 100 128.5 99.2 167.8 +Exponentials 100 148.3 93.0 189.3 +**Overall** 100 144.3 86.8 180.0 +=================== ========= ======= ======== ======= + + +64-bit Windows +~~~~~~~~~~~~~~ +The relative performance on 64-bit Linux and 64-bit Windows is broadly similar. + + +=================== ========= ======= ======== ======= +Distribution MT19937 PCG64 Philox SFC64 +=================== ========= ======= ======== ======= +32-bit Unsigned Int 100 129.1 35.0 135.0 +64-bit Unsigned Int 100 146.9 35.7 176.5 +Uniforms 100 165.0 37.0 192.0 +Normals 100 128.5 48.5 158.0 +Exponentials 100 151.6 39.0 172.8 +**Overall** 100 143.6 38.7 165.7 +=================== ========= ======= ======== ======= + + +32-bit Windows +~~~~~~~~~~~~~~ + +The performance of 64-bit generators on 32-bit Windows is much lower than on 64-bit +operating systems due to register width. MT19937, the generator that has been +in NumPy since 2005, operates on 32-bit integers. + +=================== ========= ======= ======== ======= +Distribution MT19937 PCG64 Philox SFC64 +=================== ========= ======= ======== ======= +32-bit Unsigned Int 100 30.5 21.1 77.9 +64-bit Unsigned Int 100 26.3 19.2 97.0 +Uniforms 100 28.0 23.0 106.0 +Normals 100 40.1 31.3 112.6 +Exponentials 100 33.7 26.3 109.8 +**Overall** 100 31.4 23.8 99.8 +=================== ========= ======= ======== ======= + + +.. note:: + + Linux timings used Ubuntu 18.04 and GCC 7.4. Windows timings were made on + Windows 10 using Microsoft C/C++ Optimizing Compiler Version 19 (Visual + Studio 2015). All timings were produced on a i5-3570 processor. diff --git a/doc/source/reference/routines.char.rst b/doc/source/reference/routines.char.rst index 7413e3615..ed8393855 100644 --- a/doc/source/reference/routines.char.rst +++ b/doc/source/reference/routines.char.rst @@ -1,11 +1,13 @@ String operations ***************** -.. currentmodule:: numpy.core.defchararray +.. currentmodule:: numpy.char -This module provides a set of vectorized string operations for arrays -of type `numpy.string_` or `numpy.unicode_`. All of them are based on -the string methods in the Python standard library. +.. module:: numpy.char + +The `numpy.char` module provides a set of vectorized string +operations for arrays of type `numpy.string_` or `numpy.unicode_`. +All of them are based on the string methods in the Python standard library. String operations ----------------- @@ -20,6 +22,7 @@ String operations center decode encode + expandtabs join ljust lower @@ -55,6 +58,7 @@ comparison. less_equal greater less + compare_chararrays String information ------------------ @@ -63,9 +67,11 @@ String information :toctree: generated/ count + endswith find index isalpha + isalnum isdecimal isdigit islower @@ -76,6 +82,7 @@ String information rfind rindex startswith + str_len Convenience class ----------------- @@ -83,4 +90,6 @@ Convenience class .. autosummary:: :toctree: generated/ + array + asarray chararray diff --git a/doc/source/reference/routines.ctypeslib.rst b/doc/source/reference/routines.ctypeslib.rst index b04713b61..562638e9c 100644 --- a/doc/source/reference/routines.ctypeslib.rst +++ b/doc/source/reference/routines.ctypeslib.rst @@ -1,3 +1,5 @@ +.. module:: numpy.ctypeslib + *********************************************************** C-Types Foreign Function Interface (:mod:`numpy.ctypeslib`) *********************************************************** @@ -6,6 +8,7 @@ C-Types Foreign Function Interface (:mod:`numpy.ctypeslib`) .. autofunction:: as_array .. autofunction:: as_ctypes +.. autofunction:: as_ctypes_type .. autofunction:: ctypes_load_library .. autofunction:: load_library .. autofunction:: ndpointer diff --git a/doc/source/reference/routines.dtype.rst b/doc/source/reference/routines.dtype.rst index ec8d2981d..e9189ca07 100644 --- a/doc/source/reference/routines.dtype.rst +++ b/doc/source/reference/routines.dtype.rst @@ -17,11 +17,9 @@ Data type routines Creating data types ------------------- - .. autosummary:: :toctree: generated/ - dtype format_parser @@ -53,3 +51,4 @@ Miscellaneous typename sctype2char mintypecode + maximum_sctype diff --git a/doc/source/reference/routines.linalg.rst b/doc/source/reference/routines.linalg.rst index 0520df413..d42e77ad8 100644 --- a/doc/source/reference/routines.linalg.rst +++ b/doc/source/reference/routines.linalg.rst @@ -1,8 +1,23 @@ .. _routines.linalg: +.. module:: numpy.linalg + Linear algebra (:mod:`numpy.linalg`) ************************************ +The NumPy linear algebra functions rely on BLAS and LAPACK to provide efficient +low level implementations of standard linear algebra algorithms. Those +libraries may be provided by NumPy itself using C versions of a subset of their +reference implementations but, when possible, highly optimized libraries that +take advantage of specialized processor functionality are preferred. Examples +of such libraries are OpenBLAS_, MKL (TM), and ATLAS. Because those libraries +are multithreaded and processor dependent, environmental variables and external +packages such as threadpoolctl_ may be needed to control the number of threads +or specify the processor architecture. + +.. _OpenBLAS: https://www.openblas.net/ +.. _threadpoolctl: https://github.com/joblib/threadpoolctl + .. currentmodule:: numpy Matrix and vector products diff --git a/doc/source/reference/routines.ma.rst b/doc/source/reference/routines.ma.rst index 15f2ba0a4..491bb6bff 100644 --- a/doc/source/reference/routines.ma.rst +++ b/doc/source/reference/routines.ma.rst @@ -68,9 +68,6 @@ Inspecting the array ma.is_masked ma.is_mask - ma.MaskedArray.data - ma.MaskedArray.mask - ma.MaskedArray.recordmask ma.MaskedArray.all ma.MaskedArray.any @@ -80,6 +77,12 @@ Inspecting the array ma.size +.. autosummary:: + + ma.MaskedArray.data + ma.MaskedArray.mask + ma.MaskedArray.recordmask + _____ Manipulating a MaskedArray @@ -285,8 +288,10 @@ Filling a masked array ma.MaskedArray.get_fill_value ma.MaskedArray.set_fill_value - ma.MaskedArray.fill_value +.. autosummary:: + + ma.MaskedArray.fill_value _____ diff --git a/doc/source/reference/routines.math.rst b/doc/source/reference/routines.math.rst index 821363987..3c2f96830 100644 --- a/doc/source/reference/routines.math.rst +++ b/doc/source/reference/routines.math.rst @@ -141,6 +141,7 @@ Handling complex numbers real imag conj + conjugate Miscellaneous diff --git a/doc/source/reference/routines.matlib.rst b/doc/source/reference/routines.matlib.rst index a35eaec78..c7f675425 100644 --- a/doc/source/reference/routines.matlib.rst +++ b/doc/source/reference/routines.matlib.rst @@ -1,3 +1,5 @@ +.. module:: numpy.matlib + Matrix library (:mod:`numpy.matlib`) ************************************ diff --git a/doc/source/reference/routines.other.rst b/doc/source/reference/routines.other.rst index 45b9ac3d9..28c9a1ad1 100644 --- a/doc/source/reference/routines.other.rst +++ b/doc/source/reference/routines.other.rst @@ -5,14 +5,6 @@ Miscellaneous routines .. currentmodule:: numpy -Buffer objects --------------- -.. autosummary:: - :toctree: generated/ - - getbuffer - newbuffer - Performance tuning ------------------ .. autosummary:: @@ -29,6 +21,7 @@ Memory ranges shares_memory may_share_memory + byte_bounds Array mixins ------------ @@ -43,3 +36,21 @@ NumPy version comparison :toctree: generated/ lib.NumpyVersion + +Utility +------- + +.. autosummary:: + :toctree: generated/ + + get_include + deprecate + deprecate_with_doc + +Matlab-like Functions +--------------------- +.. autosummary:: + :toctree: generated/ + + who + disp
\ No newline at end of file diff --git a/doc/source/reference/routines.polynomials.package.rst b/doc/source/reference/routines.polynomials.package.rst index 61cb57fbb..7e40d9f00 100644 --- a/doc/source/reference/routines.polynomials.package.rst +++ b/doc/source/reference/routines.polynomials.package.rst @@ -1,3 +1,5 @@ +.. module:: numpy.polynomial + Polynomial Package ================== diff --git a/doc/source/reference/routines.polynomials.polynomial.rst b/doc/source/reference/routines.polynomials.polynomial.rst index 8194ca867..365c8da98 100644 --- a/doc/source/reference/routines.polynomials.polynomial.rst +++ b/doc/source/reference/routines.polynomials.polynomial.rst @@ -1,3 +1,5 @@ +.. module:: numpy.polynomial.polynomial + Polynomial Module (:mod:`numpy.polynomial.polynomial`) ====================================================== diff --git a/doc/source/reference/routines.random.rst b/doc/source/reference/routines.random.rst deleted file mode 100644 index c8b097d7d..000000000 --- a/doc/source/reference/routines.random.rst +++ /dev/null @@ -1,81 +0,0 @@ -.. _routines.random: - -Random sampling (:mod:`numpy.random`) -************************************* - -.. currentmodule:: numpy.random - -Simple random data -================== -.. autosummary:: - :toctree: generated/ - - rand - randn - randint - random_integers - random_sample - random - ranf - sample - choice - bytes - -Permutations -============ -.. autosummary:: - :toctree: generated/ - - shuffle - permutation - -Distributions -============= -.. autosummary:: - :toctree: generated/ - - beta - binomial - chisquare - dirichlet - exponential - f - gamma - geometric - gumbel - hypergeometric - laplace - logistic - lognormal - logseries - multinomial - multivariate_normal - negative_binomial - noncentral_chisquare - noncentral_f - normal - pareto - poisson - power - rayleigh - standard_cauchy - standard_exponential - standard_gamma - standard_normal - standard_t - triangular - uniform - vonmises - wald - weibull - zipf - -Random generator -================ -.. autosummary:: - :toctree: generated/ - - RandomState - seed - get_state - set_state diff --git a/doc/source/reference/routines.rst b/doc/source/reference/routines.rst index a9e80480b..7a9b97d77 100644 --- a/doc/source/reference/routines.rst +++ b/doc/source/reference/routines.rst @@ -41,7 +41,7 @@ indentation. routines.other routines.padding routines.polynomials - routines.random + random/index routines.set routines.sort routines.statistics diff --git a/doc/source/reference/routines.testing.rst b/doc/source/reference/routines.testing.rst index 5a52a40d6..c676dec07 100644 --- a/doc/source/reference/routines.testing.rst +++ b/doc/source/reference/routines.testing.rst @@ -1,4 +1,4 @@ -.. _numpy-testing: +.. module:: numpy.testing Test Support (:mod:`numpy.testing`) =================================== diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst index 3cc956887..d00e88b34 100644 --- a/doc/source/reference/ufuncs.rst +++ b/doc/source/reference/ufuncs.rst @@ -16,7 +16,7 @@ A universal function (or :term:`ufunc` for short) is a function that operates on :class:`ndarrays <ndarray>` in an element-by-element fashion, supporting :ref:`array broadcasting <ufuncs.broadcasting>`, :ref:`type casting <ufuncs.casting>`, and several other standard features. That -is, a ufunc is a ":term:`vectorized`" wrapper for a function that +is, a ufunc is a ":term:`vectorized <vectorization>`" wrapper for a function that takes a fixed number of specific inputs and produces a fixed number of specific outputs. @@ -59,7 +59,7 @@ understood by four rules: entry in that dimension will be used for all calculations along that dimension. In other words, the stepping machinery of the :term:`ufunc` will simply not step along that dimension (the - :term:`stride` will be 0 for that dimension). + :ref:`stride <memory-layout>` will be 0 for that dimension). Broadcasting is used throughout NumPy to decide how to handle disparately shaped arrays; for example, all arithmetic operations (``+``, @@ -70,7 +70,7 @@ arrays before operation. .. index:: broadcastable -A set of arrays is called ":term:`broadcastable`" to the same shape if +A set of arrays is called "broadcastable" to the same shape if the above rules produce a valid result, *i.e.*, one of the following is true: @@ -118,7 +118,7 @@ all output arrays will be passed to the :obj:`~class.__array_prepare__` and the highest :obj:`~class.__array_priority__` of any other input to the universal function. The default :obj:`~class.__array_priority__` of the ndarray is 0.0, and the default :obj:`~class.__array_priority__` of a subtype -is 1.0. Matrices have :obj:`~class.__array_priority__` equal to 10.0. +is 0.0. Matrices have :obj:`~class.__array_priority__` equal to 10.0. All ufuncs can also take output arguments. If necessary, output will be cast to the data-type(s) of the provided output array(s). If a class @@ -586,6 +586,7 @@ Math operations sign heaviside conj + conjugate exp exp2 log diff --git a/doc/source/release.rst b/doc/source/release.rst index 1cf215549..f8d83726f 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -2,6 +2,11 @@ Release Notes ************* +.. include:: ../release/1.17.0-notes.rst +.. include:: ../release/1.16.4-notes.rst +.. include:: ../release/1.16.3-notes.rst +.. include:: ../release/1.16.2-notes.rst +.. include:: ../release/1.16.1-notes.rst .. include:: ../release/1.16.0-notes.rst .. include:: ../release/1.15.4-notes.rst .. include:: ../release/1.15.3-notes.rst diff --git a/doc/source/user/basics.dispatch.rst b/doc/source/user/basics.dispatch.rst new file mode 100644 index 000000000..f7b8da262 --- /dev/null +++ b/doc/source/user/basics.dispatch.rst @@ -0,0 +1,8 @@ +.. _basics.dispatch: + +******************************* +Writing custom array containers +******************************* + +.. automodule:: numpy.doc.dispatch + diff --git a/doc/source/user/basics.indexing.rst b/doc/source/user/basics.indexing.rst index 8844adcae..0dca4b884 100644 --- a/doc/source/user/basics.indexing.rst +++ b/doc/source/user/basics.indexing.rst @@ -4,6 +4,10 @@ Indexing ******** -.. seealso:: :ref:`Indexing routines <routines.indexing>` +.. seealso:: + + :ref:`Indexing <arrays.indexing>` + + :ref:`Indexing routines <routines.indexing>` .. automodule:: numpy.doc.indexing diff --git a/doc/source/user/basics.io.genfromtxt.rst b/doc/source/user/basics.io.genfromtxt.rst index 21832e5aa..6ef80bf8e 100644 --- a/doc/source/user/basics.io.genfromtxt.rst +++ b/doc/source/user/basics.io.genfromtxt.rst @@ -521,12 +521,6 @@ provides several convenience functions derived from :func:`~numpy.genfromtxt`. These functions work the same way as the original, but they have different default values. -:func:`~numpy.ndfromtxt` - Always set ``usemask=False``. - The output is always a standard :class:`numpy.ndarray`. -:func:`~numpy.mafromtxt` - Always set ``usemask=True``. - The output is always a :class:`~numpy.ma.MaskedArray` :func:`~numpy.recfromtxt` Returns a standard :class:`numpy.recarray` (if ``usemask=False``) or a :class:`~numpy.ma.MaskedRecords` array (if ``usemaske=True``). The diff --git a/doc/source/user/basics.rst b/doc/source/user/basics.rst index 7875aff6e..e0fc0ece3 100644 --- a/doc/source/user/basics.rst +++ b/doc/source/user/basics.rst @@ -12,4 +12,5 @@ NumPy basics basics.broadcasting basics.byteswapping basics.rec + basics.dispatch basics.subclassing diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst index d224951dd..b4b4371e5 100644 --- a/doc/source/user/building.rst +++ b/doc/source/user/building.rst @@ -56,7 +56,7 @@ Basic Installation To install NumPy run:: - python setup.py install + pip install . To perform an in-place build that can be run from the source folder run:: @@ -69,6 +69,15 @@ Using ``virtualenv`` should work as expected. *Note: for build instructions to do development work on NumPy itself, see* :ref:`development-environment`. +Testing +------- + +Make sure to test your builds. To ensure everything stays in shape, see if all tests pass:: + + $ python runtests.py -v -m full + +For detailed info on testing, see :ref:`testing-builds`. + .. _parallel-builds: Parallel builds @@ -118,12 +127,71 @@ means that g77 has been used. If libgfortran.so is a dependency, gfortran has been used. If both are dependencies, this means both have been used, which is almost always a very bad idea. +Accelerated BLAS/LAPACK libraries +--------------------------------- + +NumPy searches for optimized linear algebra libraries such as BLAS and LAPACK. +There are specific orders for searching these libraries, as described below. + +BLAS +~~~~ + +The default order for the libraries are: + +1. MKL +2. BLIS +3. OpenBLAS +4. ATLAS +5. Accelerate (MacOS) +6. BLAS (NetLIB) + + +If you wish to build against OpenBLAS but you also have BLIS available one +may predefine the order of searching via the environment variable +``NPY_BLAS_ORDER`` which is a comma-separated list of the above names which +is used to determine what to search for, for instance:: + + NPY_BLAS_ORDER=ATLAS,blis,openblas,MKL python setup.py build + +will prefer to use ATLAS, then BLIS, then OpenBLAS and as a last resort MKL. +If neither of these exists the build will fail (names are compared +lower case). + +LAPACK +~~~~~~ + +The default order for the libraries are: + +1. MKL +2. OpenBLAS +3. libFLAME +4. ATLAS +5. Accelerate (MacOS) +6. LAPACK (NetLIB) + + +If you wish to build against OpenBLAS but you also have MKL available one +may predefine the order of searching via the environment variable +``NPY_LAPACK_ORDER`` which is a comma-separated list of the above names, +for instance:: + + NPY_LAPACK_ORDER=ATLAS,openblas,MKL python setup.py build + +will prefer to use ATLAS, then OpenBLAS and as a last resort MKL. +If neither of these exists the build will fail (names are compared +lower case). + + Disabling ATLAS and other accelerated libraries ------------------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Usage of ATLAS and other accelerated libraries in NumPy can be disabled via:: + NPY_BLAS_ORDER= NPY_LAPACK_ORDER= python setup.py build + +or:: + BLAS=None LAPACK=None ATLAS=None python setup.py build diff --git a/doc/source/user/c-info.how-to-extend.rst b/doc/source/user/c-info.how-to-extend.rst index 9738168d2..00ef8ab74 100644 --- a/doc/source/user/c-info.how-to-extend.rst +++ b/doc/source/user/c-info.how-to-extend.rst @@ -342,7 +342,7 @@ The method is to 4. If you are writing the algorithm, then I recommend that you use the stride information contained in the array to access the elements of - the array (the :c:func:`PyArray_GETPTR` macros make this painless). Then, + the array (the :c:func:`PyArray_GetPtr` macros make this painless). Then, you can relax your requirements so as not to force a single-segment array and the data-copying that might result. @@ -362,8 +362,7 @@ specific builtin data-type ( *e.g.* float), while specifying a particular set of requirements ( *e.g.* contiguous, aligned, and writeable). The syntax is -.. c:function:: PyObject *PyArray_FROM_OTF( \ - PyObject* obj, int typenum, int requirements) +:c:func:`PyArray_FROM_OTF` Return an ndarray from any Python object, *obj*, that can be converted to an array. The number of dimensions in the returned @@ -446,33 +445,25 @@ writeable). The syntax is flags most commonly needed are :c:data:`NPY_ARRAY_IN_ARRAY`, :c:data:`NPY_OUT_ARRAY`, and :c:data:`NPY_ARRAY_INOUT_ARRAY`: - .. c:var:: NPY_ARRAY_IN_ARRAY + :c:data:`NPY_ARRAY_IN_ARRAY` - Equivalent to :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| - :c:data:`NPY_ARRAY_ALIGNED`. This combination of flags is useful - for arrays that must be in C-contiguous order and aligned. - These kinds of arrays are usually input arrays for some - algorithm. + This flag is useful for arrays that must be in C-contiguous + order and aligned. These kinds of arrays are usually input + arrays for some algorithm. - .. c:var:: NPY_ARRAY_OUT_ARRAY + :c:data:`NPY_ARRAY_OUT_ARRAY` - Equivalent to :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| - :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_WRITEABLE`. This - combination of flags is useful to specify an array that is + This flag is useful to specify an array that is in C-contiguous order, is aligned, and can be written to as well. Such an array is usually returned as output (although normally such output arrays are created from scratch). - .. c:var:: NPY_ARRAY_INOUT_ARRAY + :c:data:`NPY_ARRAY_INOUT_ARRAY` - Equivalent to :c:data:`NPY_ARRAY_C_CONTIGUOUS` \| - :c:data:`NPY_ARRAY_ALIGNED` \| :c:data:`NPY_ARRAY_WRITEABLE` \| - :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` \| - :c:data:`NPY_ARRAY_UPDATEIFCOPY`. This combination of flags is - useful to specify an array that will be used for both + This flag is useful to specify an array that will be used for both input and output. :c:func:`PyArray_ResolveWritebackIfCopy` - must be called before :func:`Py_DECREF` at + must be called before :c:func:`Py_DECREF` at the end of the interface routine to write back the temporary data into the original array passed in. Use of the :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` or @@ -487,16 +478,16 @@ writeable). The syntax is Other useful flags that can be OR'd as additional requirements are: - .. c:var:: NPY_ARRAY_FORCECAST + :c:data:`NPY_ARRAY_FORCECAST` Cast to the desired type, even if it can't be done without losing information. - .. c:var:: NPY_ARRAY_ENSURECOPY + :c:data:`NPY_ARRAY_ENSURECOPY` Make sure the resulting array is a copy of the original. - .. c:var:: NPY_ARRAY_ENSUREARRAY + :c:data:`NPY_ARRAY_ENSUREARRAY` Make sure the resulting object is an actual ndarray and not a sub- class. @@ -513,7 +504,7 @@ writeable). The syntax is Creating a brand-new ndarray ---------------------------- -Quite often new arrays must be created from within extension-module +Quite often, new arrays must be created from within extension-module code. Perhaps an output array is needed and you don't want the caller to have to supply it. Perhaps only a temporary array is needed to hold an intermediate calculation. Whatever the need there are simple ways @@ -521,43 +512,9 @@ to get an ndarray object of whatever data-type is needed. The most general function for doing this is :c:func:`PyArray_NewFromDescr`. All array creation functions go through this heavily re-used code. Because of its flexibility, it can be somewhat confusing to use. As a result, -simpler forms exist that are easier to use. - -.. c:function:: PyObject *PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) - - This function allocates new memory and places it in an ndarray - with *nd* dimensions whose shape is determined by the array of - at least *nd* items pointed to by *dims*. The memory for the - array is uninitialized (unless typenum is :c:data:`NPY_OBJECT` in - which case each element in the array is set to NULL). The - *typenum* argument allows specification of any of the builtin - data-types such as :c:data:`NPY_FLOAT` or :c:data:`NPY_LONG`. The - memory for the array can be set to zero if desired using - :c:func:`PyArray_FILLWBYTE` (return_object, 0). - -.. c:function:: PyObject *PyArray_SimpleNewFromData( \ - int nd, npy_intp* dims, int typenum, void* data) - - Sometimes, you want to wrap memory allocated elsewhere into an - ndarray object for downstream use. This routine makes it - straightforward to do that. The first three arguments are the same - as in :c:func:`PyArray_SimpleNew`, the final argument is a pointer to a - block of contiguous memory that the ndarray should use as it's - data-buffer which will be interpreted in C-style contiguous - fashion. A new reference to an ndarray is returned, but the - ndarray will not own its data. When this ndarray is deallocated, - the pointer will not be freed. - - You should ensure that the provided memory is not freed while the - returned array is in existence. The easiest way to handle this is - if data comes from another reference-counted Python object. The - reference count on this object should be increased after the - pointer is passed in, and the base member of the returned ndarray - should point to the Python object that owns the data. Then, when - the ndarray is deallocated, the base-member will be DECREF'd - appropriately. If you want the memory to be freed as soon as the - ndarray is deallocated then simply set the OWNDATA flag on the - returned ndarray. +simpler forms exist that are easier to use. These forms are part of the +:c:func:`PyArray_SimpleNew` family of functions, which simplify the interface +by providing default values for common use cases. Getting at ndarray memory and accessing elements of the ndarray @@ -573,7 +530,7 @@ specific element of the array is determined only by the array of npy_intp variables, :c:func:`PyArray_STRIDES` (obj). In particular, this c-array of integers shows how many **bytes** must be added to the current element pointer to get to the next element in each dimension. -For arrays less than 4-dimensions there are :c:func:`PyArray_GETPTR{k}` +For arrays less than 4-dimensions there are ``PyArray_GETPTR{k}`` (obj, ...) macros where {k} is the integer 1, 2, 3, or 4 that make using the array strides easier. The arguments .... represent {k} non- negative integer indices into the array. For example, suppose ``E`` is @@ -586,7 +543,7 @@ contiguous arrays have particular striding patterns. Two array flags whether or not the striding pattern of a particular array matches the C-style contiguous or Fortran-style contiguous or neither. Whether or not the striding pattern matches a standard C or Fortran one can be -tested Using :c:func:`PyArray_ISCONTIGUOUS` (obj) and +tested Using :c:func:`PyArray_IS_C_CONTIGUOUS` (obj) and :c:func:`PyArray_ISFORTRAN` (obj) respectively. Most third-party libraries expect contiguous arrays. But, often it is not difficult to support general-purpose striding. I encourage you to use the striding diff --git a/doc/source/user/c-info.python-as-glue.rst b/doc/source/user/c-info.python-as-glue.rst index 01d2a64d1..7b9b096af 100644 --- a/doc/source/user/c-info.python-as-glue.rst +++ b/doc/source/user/c-info.python-as-glue.rst @@ -387,7 +387,7 @@ distribution of the ``add.f`` module (as part of the package Installation of the new package is easy using:: - python setup.py install + pip install . assuming you have the proper permissions to write to the main site- packages directory for the version of Python you are using. For the @@ -744,14 +744,14 @@ around this restriction that allow ctypes to integrate with other objects. 1. Don't set the argtypes attribute of the function object and define an - :obj:`_as_parameter_` method for the object you want to pass in. The - :obj:`_as_parameter_` method must return a Python int which will be passed + ``_as_parameter_`` method for the object you want to pass in. The + ``_as_parameter_`` method must return a Python int which will be passed directly to the function. 2. Set the argtypes attribute to a list whose entries contain objects with a classmethod named from_param that knows how to convert your object to an object that ctypes can understand (an int/long, string, - unicode, or object with the :obj:`_as_parameter_` attribute). + unicode, or object with the ``_as_parameter_`` attribute). NumPy uses both methods with a preference for the second method because it can be safer. The ctypes attribute of the ndarray returns @@ -764,7 +764,7 @@ correct type, shape, and has the correct flags set or risk nasty crashes if the data-pointer to inappropriate arrays are passed in. To implement the second method, NumPy provides the class-factory -function :func:`ndpointer` in the :mod:`ctypeslib` module. This +function :func:`ndpointer` in the :mod:`numpy.ctypeslib` module. This class-factory function produces an appropriate class that can be placed in an argtypes attribute entry of a ctypes function. The class will contain a from_param method which ctypes will use to convert any diff --git a/doc/source/user/numpy-for-matlab-users.rst b/doc/source/user/numpy-for-matlab-users.rst index 399237c21..e53d1ca45 100644 --- a/doc/source/user/numpy-for-matlab-users.rst +++ b/doc/source/user/numpy-for-matlab-users.rst @@ -436,7 +436,7 @@ Linear Algebra Equivalents ``a`` * - ``rand(3,4)`` - - ``random.rand(3,4)`` + - ``random.rand(3,4)`` or ``random.random_sample((3, 4))`` - random 3x4 matrix * - ``linspace(1,3,4)`` @@ -547,7 +547,7 @@ Linear Algebra Equivalents - eigenvalues and eigenvectors of ``a`` * - ``[V,D]=eig(a,b)`` - - ``V,D = np.linalg.eig(a,b)`` + - ``D,V = scipy.linalg.eig(a,b)`` - eigenvalues and eigenvectors of ``a``, ``b`` * - ``[V,D]=eigs(a,k)`` @@ -693,19 +693,19 @@ this is just an example, not a statement of "best practices"): :: - # Make all numpy available via shorter 'num' prefix - import numpy as num + # Make all numpy available via shorter 'np' prefix + import numpy as np # Make all matlib functions accessible at the top level via M.func() import numpy.matlib as M # Make some matlib functions accessible directly at the top level via, e.g. rand(3,3) from numpy.matlib import rand,zeros,ones,empty,eye # Define a Hermitian function def hermitian(A, **kwargs): - return num.transpose(A,**kwargs).conj() + return np.transpose(A,**kwargs).conj() # Make some shortcuts for transpose,hermitian: - # num.transpose(A) --> T(A) + # np.transpose(A) --> T(A) # hermitian(A) --> H(A) - T = num.transpose + T = np.transpose H = hermitian Links diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index 5ef8b145f..a23a7b2c7 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -25,7 +25,7 @@ The Basics NumPy's main object is the homogeneous multidimensional array. It is a table of elements (usually numbers), all of the same type, indexed by a -tuple of positive integers. In NumPy dimensions are called *axes*. +tuple of non-negative integers. In NumPy dimensions are called *axes*. For example, the coordinates of a point in 3D space ``[1, 2, 1]`` has one axis. That axis has 3 elements in it, so we say it has a length @@ -206,8 +206,8 @@ of elements that we want, instead of the step:: `empty_like`, `arange`, `linspace`, - `numpy.random.rand`, - `numpy.random.randn`, + `numpy.random.mtrand.RandomState.rand`, + `numpy.random.mtrand.RandomState.randn`, `fromfunction`, `fromfile` @@ -270,7 +270,7 @@ can change the printing options using ``set_printoptions``. :: - >>> np.set_printoptions(threshold=np.nan) + >>> np.set_printoptions(threshold=sys.maxsize) # sys module should be imported Basic Operations @@ -732,9 +732,9 @@ stacks 1D arrays as columns into a 2D array. It is equivalent to array([[ 4., 3.], [ 2., 8.]]) -On the other hand, the function `row_stack` is equivalent to `vstack` +On the other hand, the function `ma.row_stack` is equivalent to `vstack` for any input arrays. -In general, for arrays of with more than two dimensions, +In general, for arrays with more than two dimensions, `hstack` stacks along their second axes, `vstack` stacks along their first axes, and `concatenate` @@ -884,6 +884,17 @@ The ``copy`` method makes a complete copy of the array and its data. [ 8, 10, 10, 11]]) +Sometimes ``copy`` should be called after slicing if the original array is not required anymore. +For example, suppose ``a`` is a huge intermediate result and the final result ``b`` only contains +a small fraction of ``a``, a deep copy should be made when constructing ``b`` with slicing:: + + >>> a = np.arange(int(1e8)) + >>> b = a[:100].copy() + >>> del a # the memory of ``a`` can be released. + +If ``b = a[:100]`` is used instead, ``a`` is referenced by ``b`` and will persist in memory +even if ``del a`` is executed. + Functions and Methods Overview ------------------------------ @@ -1465,5 +1476,5 @@ Further reading - The `Python tutorial <https://docs.python.org/tutorial/>`__ - :ref:`reference` - `SciPy Tutorial <https://docs.scipy.org/doc/scipy/reference/tutorial/index.html>`__ -- `SciPy Lecture Notes <https://www.scipy-lectures.org>`__ +- `SciPy Lecture Notes <https://scipy-lectures.org>`__ - A `matlab, R, IDL, NumPy/SciPy dictionary <http://mathesaurus.sf.net/>`__ diff --git a/doc/source/user/whatisnumpy.rst b/doc/source/user/whatisnumpy.rst index cd74a8de3..abaa2bfed 100644 --- a/doc/source/user/whatisnumpy.rst +++ b/doc/source/user/whatisnumpy.rst @@ -91,6 +91,11 @@ idiom is even simpler! This last example illustrates two of NumPy's features which are the basis of much of its power: vectorization and broadcasting. +.. _whatis-vectorization: + +Why is NumPy Fast? +------------------ + Vectorization describes the absence of any explicit looping, indexing, etc., in the code - these things are taking place, of course, just "behind the scenes" in optimized, pre-compiled C code. Vectorized @@ -120,9 +125,13 @@ the shape of the larger in such a way that the resulting broadcast is unambiguous. For detailed "rules" of broadcasting see `numpy.doc.broadcasting`. +Who Else Uses NumPy? +-------------------- + NumPy fully supports an object-oriented approach, starting, once again, with `ndarray`. For example, `ndarray` is a class, possessing -numerous methods and attributes. Many of its methods mirror -functions in the outer-most NumPy namespace, giving the programmer -complete freedom to code in whichever paradigm she prefers and/or -which seems most appropriate to the task at hand. +numerous methods and attributes. Many of its methods are mirrored by +functions in the outer-most NumPy namespace, allowing the programmer +to code in whichever paradigm they prefer. This flexibility has allowed the +NumPy array dialect and NumPy `ndarray` class to become the *de-facto* language +of multi-dimensional data interchange used in Python. |
