diff options
| author | Jason R. Coombs <jaraco@jaraco.com> | 2021-10-22 11:48:14 -0400 |
|---|---|---|
| committer | Jason R. Coombs <jaraco@jaraco.com> | 2021-10-22 11:48:14 -0400 |
| commit | 13c55c0cee1351c88551bf081ae56ed3e60b4493 (patch) | |
| tree | 05efd2817e1779b23f2f79d23348bfaa699e39a5 | |
| parent | f7e70d0fbf7488198026631b435b3e7faaf3dab2 (diff) | |
| parent | fee9d17d2a9113dbdc4bcfeca8b040af75bcf2f9 (diff) | |
| download | python-setuptools-git-13c55c0cee1351c88551bf081ae56ed3e60b4493.tar.gz | |
Merge branch 'main' into debt/remove-legacy-version
167 files changed, 13181 insertions, 3362 deletions
diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d156dedb..0ed35632 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 51.1.0 +current_version = 58.2.0 commit = True tag = True diff --git a/.codecov.yml b/.codecov.yml index 69cb7601..7510dfc6 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1 +1,5 @@ comment: false +coverage: + status: + project: + threshold: 0.5% diff --git a/.coveragerc b/.coveragerc index 45823064..6a34e662 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,7 @@ [run] -omit = .tox/* +omit = + # leading `*/` for pytest-dev/pytest-cov#456 + */.tox/* [report] show_missing = True diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..6385b573 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +indent_style = tab +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.py] +indent_style = space + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 @@ -1,14 +1,15 @@ [flake8] max-line-length = 88 -exclude = + +# jaraco/skeleton#34 +max-complexity = 10 + +extend-exclude = + build setuptools/_vendor + setuptools/_distutils pkg_resources/_vendor -ignore = - # W503 violates spec https://github.com/PyCQA/pycodestyle/issues/513 - W503 - # W504 has issues https://github.com/OCA/maintainer-quality-tools/issues/545 - W504 + +extend-ignore = # Black creates whitespace before colon E203 - setuptools/site-patch.py F821 - setuptools/py*compat.py F811 diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..73911ec8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,129 @@ +--- +name: 🐛 Bug report +description: >- + Create a report to help us improve when + something is not working correctly +title: '[BUG] ' +labels: +- bug +- Needs Triage + +body: +- type: markdown + attributes: + value: > + **Thank you for wanting to report a bug in setuptools!** + + + ⚠ + Verify first that your issue is not + [already reported on GitHub][issue search] and keep in mind and + keep in mind that we may have to keep the current behavior because + [every change breaks someone's workflow][XKCD 1172]. + We try to be mindful about this. + + Also test if the latest release and main branch are affected too. + + + If you are seeking community support, please consider + [starting a discussion][Discussions]. + + + Thank you for your collaboration! + + + [Discussions]: https://github.com/pypa/setuptools/discussions + + [issue search]: https://github.com/pypa/setuptools/search?q=is%3Aissue&type=issues + + [XKCD 1172]: https://xkcd.com/1172/ + +- type: markdown + attributes: + value: >- + **Environment** +- type: input + attributes: + label: setuptools version + placeholder: For example, setuptools===60.4.2 + validations: + required: true +- type: input + attributes: + label: Python version + placeholder: For example, Python 3.10 + validations: + required: true +- type: input + attributes: + label: OS + placeholder: For example, Gentoo Linux, RHEL 8, Arch Linux, macOS etc. + validations: + required: true +- type: textarea + attributes: + label: Additional environment information + description: >- + Feel free to add more information about your environment here. + placeholder: >- + This is only happening when I run setuptools on my fridge's patched firmware 🤯 + +- type: textarea + attributes: + label: Description + description: >- + A clear and concise description of what the bug is. + placeholder: >- + I tried doing X and I expected it to result in Y because the docs + mentioned Z but what happened next what totally unexpected! + And here's why... + validations: + required: true + +- type: textarea + attributes: + label: Expected behavior + description: >- + A clear and concise description of what you expected to happen. + placeholder: >- + I tried doing X and I expected it to result in Y. I'm confused... + validations: + required: true + +- type: textarea + attributes: + label: How to Reproduce + description: >- + Describe the steps to reproduce this bug. + placeholder: | + 1. Integrate setuptools via '...' + 2. Then run '...' + 3. An error occurs. + validations: + required: true + +- type: textarea + attributes: + label: Output + description: >- + Paste the output of the steps above, including the commands + themselves and setuptools' output/traceback etc. + value: | + ```console + + ``` + validations: + required: true + + +- type: checkboxes + attributes: + label: Code of Conduct + description: | + Read the [PSF Code of Conduct][CoC] first. + + [CoC]: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md + options: + - label: I agree to follow the PSF Code of Conduct + required: true +... diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..dde102ca --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,15 @@ +# Ref: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository#configuring-the-template-chooser +blank_issues_enabled: false # default: true +contact_links: +- name: 🤔 Have questions or need support? + url: https://github.com/pypa/setuptools/discussions + about: This is a place for the community to exchange ideas and recipes +- name: 💬 Discourse + url: https://discuss.python.org/c/packaging + about: | + Please ask typical Q&A here: general ideas for Python packaging, + questions about structuring projects and so on +- name: >- + 💬 IRC: #pypa @ Freenode + url: https://webchat.freenode.net/#pypa + about: Chat with devs diff --git a/.github/ISSUE_TEMPLATE/documentation-report.yml b/.github/ISSUE_TEMPLATE/documentation-report.yml new file mode 100644 index 00000000..238ce896 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation-report.yml @@ -0,0 +1,91 @@ +--- +name: 📝 Documentation Report +title: '[Docs] ' +description: Ask us about docs +labels: +- documentation +- Needs Triage + +body: +- type: markdown + attributes: + value: > + **Thank you for wanting to report a problem with setuptools + documentation!** + + + Please fill out your suggestions below. If the problem seems + straightforward, feel free to go ahead and + submit a pull request instead! + + + ⚠ + Verify first that your issue is not [already reported on + GitHub][issue search]. + + + If you are seeking community support, please consider + [starting a discussion][Discussions]. + + + Thank you for your collaboration! + + + [issue search]: https://github.com/pypa/setuptools/search?q=is%3Aissue&type=issues + + [Discussions]: https://github.com/pypa/setuptools/discussions + +- type: textarea + attributes: + label: Summary + description: > + Explain the problem briefly below, add suggestions to wording + or structure. + + + **HINT:** Did you know the documentation has a `View on GitHub` + link on every page? Feel free to use it to start a pull request + right from the GitHub UI! + placeholder: >- + I was reading the setuptools documentation of version X and I'm + having problems understanding Y. It would be very helpful if that + got rephrased as Z. + validations: + required: true + +- type: textarea + attributes: + label: OS / Environment + description: >- + Provide all relevant information below, e.g. OS version, + browser, etc. + placeholder: Fedora 33, Firefox etc. + + +- type: textarea + attributes: + label: Additional Information + description: > + Describe how this improves the documentation, e.g. before/after + situation or screenshots. + + + **HINT:** You can paste https://gist.github.com links for larger files. + placeholder: >- + When the improvement is applied, it makes it more straightforward + to understand X. + validations: + required: true + + +- type: checkboxes + attributes: + label: Code of Conduct + description: | + Read the [PSF Code of Conduct][CoC] first. + + [CoC]: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md + options: + - label: I agree to follow the PSF Code of Conduct + required: true +... diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 00000000..88ae6741 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,104 @@ +--- +name: ✨ Feature request +description: Suggest an idea for setuptools +title: '[FR] ' +labels: +- enhancement +- Needs Triage + +body: +- type: markdown + attributes: + value: > + **Thank you for wanting to suggest a feature for setuptools!** + + + 💡 + Before you go ahead with your request, please first consider if it + would be useful for majority of the setuptools users. As a general + rule of thumb, any feature that is only of interest to a small sub + group should be implemented in a third-party plugin or maybe even + just your project alone. Be mindful of the fact that the core + setuptools features have a broad impact. + + + <details> + <summary> + ❗ Every change breaks someone's workflow. + </summary> + + + [ + ](https://xkcd.com/1172/) + </details> + + + ⚠ + Verify first that your idea is not [already requested on GitHub][issue search]. + + + + [issue search]: https://github.com/pypa/setuptools/search?q=is%3Aissue&type=issues + +- type: textarea + attributes: + label: What's the problem this feature will solve? + description: >- + What are you trying to do, that you are unable to achieve + with setuptools as it currently stands? + placeholder: >- + I'm trying to do X and I'm missing feature Y for this to be + easily achievable. + validations: + required: true + +- type: textarea + attributes: + label: Describe the solution you'd like + description: > + Clear and concise description of what you want to happen. + + + Provide examples of real world use cases that this would enable + and how it solves the problem described above. + placeholder: >- + When I do X, I want to achieve Y in a situation when Z. + validations: + required: true + +- type: textarea + attributes: + label: Alternative Solutions + description: >- + Have you tried to workaround the problem using other tools? Or a + different approach to solving this issue? Please elaborate here. + placeholder: >- + I tried doing X, Y and Z. But they are subobpimal because of P. + +- type: textarea + attributes: + label: Additional context + description: > + Add any other context, links, etc. about the feature here. + Describe how the feature would be used, why it is needed and what + it would solve. + + + **HINT:** You can paste https://gist.github.com links for + larger files. + placeholder: >- + I asked on https://stackoverflow.com/.... and the community + advised me to do X, Y and Z. + + +- type: checkboxes + attributes: + label: Code of Conduct + description: | + Read the [PSF Code of Conduct][CoC] first. + + [CoC]: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md + options: + - label: I agree to follow the PSF Code of Conduct + required: true +... diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..89ff3396 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + allow: + - dependency-type: "all" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 31b94144..b348056b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: Automated Tests +name: tests on: [push, pull_request] @@ -6,7 +6,11 @@ jobs: test: strategy: matrix: - python: [3.6, 3.8, 3.9, pypy3] + python: + - pypy3 + - 3.6 + - 3.9 + - 3.10.0-alpha - 3.10.99 platform: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: @@ -19,7 +23,14 @@ jobs: run: | python -m pip install tox - name: Run tests - run: tox + run: tox -- --cov-report xml + - name: Publish coverage + if: false # disabled for #2727 + uses: codecov/codecov-action@v1 + with: + flags: >- # Mark which lines are covered by which envs + ${{ runner.os }}, + ${{ matrix.python }} release: needs: test @@ -40,4 +51,3 @@ jobs: env: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TIDELIFT_TOKEN: ${{ secrets.TIDELIFT_TOKEN }} @@ -6,6 +6,7 @@ docs/build include lib distribute.egg-info +foo.egg-info setuptools.egg-info .coverage .eggs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 6639c78c..00000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -repos: -- repo: https://github.com/psf/black - rev: stable - hooks: - - id: black - -- repo: https://github.com/asottile/blacken-docs - rev: v1.8.0 - hooks: - - id: blacken-docs diff --git a/.readthedocs.yml b/.readthedocs.yml index 850d79c4..cc698548 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,4 +1,6 @@ version: 2 python: install: - - requirements: docs/requirements.txt + - path: . + extra_requirements: + - docs diff --git a/CHANGES.rst b/CHANGES.rst index fb5725cc..0a002fe5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,410 @@ +v58.2.0 +------- + + +Changes +^^^^^^^ +* #2757: Add windows arm64 launchers for scripts generated by easy_install. +* #2800: Added ``--owner`` and ``--group`` options to the ``sdist`` command, + for specifying file ownership within the produced tarball (similarly + to the corresponding distutils ``sdist`` options). + +Documentation changes +^^^^^^^^^^^^^^^^^^^^^ +* #2792: Document how the legacy and non-legacy versions are compared, and reference to the `PEP 440 <https://www.python.org/dev/peps/pep-0440/>`_ scheme. + + +v58.1.0 +------- + + +Changes +^^^^^^^ +* #2796: Merge with pypa/distutils@02e9f65ab0 + + +v58.0.4 +------- + + +Misc +^^^^ +* #2773: Retain case in setup.cfg during sdist. + + +v58.0.3 +------- + + +Misc +^^^^ +* #2777: Build does not fail fast when ``use_2to3`` is supplied but set to a false value. + + +v58.0.2 +------- + + +Misc +^^^^ +* #2769: Build now fails fast when ``use_2to3`` is supplied. + + +v58.0.1 +------- + + +Misc +^^^^ +* #2765: In Distribution.finalize_options, suppress known removed entry points to avoid issues with older Setuptools. + + +v58.0.0 +------- + + +Breaking Changes +^^^^^^^^^^^^^^^^ +* #2086: Removed support for 2to3 during builds. Projects should port to a unified codebase or pin to an older version of Setuptools using PEP 518 build-requires. + +Documentation changes +^^^^^^^^^^^^^^^^^^^^^ +* #2746: add python_requires example + + +v57.5.0 +------- + + +Changes +^^^^^^^ +* #2712: Added implicit globbing support for `[options.data_files]` values. + +Documentation changes +^^^^^^^^^^^^^^^^^^^^^ +* #2737: fix various syntax and style errors in code snippets in docs + + +v57.4.0 +------- + + +Changes +^^^^^^^ +* #2722: Added support for ``SETUPTOOLS_EXT_SUFFIX`` environment variable to override the suffix normally detected from the ``sysconfig`` module. + + +v57.3.0 +------- + + +Changes +^^^^^^^ +* #2465: Documentation is now published using the Furo theme. + + +v57.2.0 +------- + + +Changes +^^^^^^^ +* #2724: Added detection of Windows ARM64 build environments using the ``VSCMD_ARG_TGT_ARCH`` environment variable. + + +v57.1.0 +------- + + +Changes +^^^^^^^ +* #2692: Globs are now sorted in 'license_files' restoring reproducibility by eliminating variance from disk order. +* #2714: Update to distutils at pypa/distutils@e2627b7. +* #2715: Removed reliance on deprecated ssl.match_hostname by removing the ssl support. Now any index operations rely on the native SSL implementation. + +Documentation changes +^^^^^^^^^^^^^^^^^^^^^ +* #2604: Revamped the backward/cross tool compatibility section to remove + some confusion. + Add some examples and the version since when ``entry_points`` are + supported in declarative configuration. + Tried to make the reading flow a bit leaner, gather some informations + that were a bit dispersed. + + +v57.0.0 +------- + + +Breaking Changes +^^^^^^^^^^^^^^^^ +* #2645: License files excluded via the ``MANIFEST.in`` but matched by either + the ``license_file`` (deprecated) or ``license_files`` options, + will be nevertheless included in the source distribution. - by :user:`cdce8p` + +Changes +^^^^^^^ +* #2628: Write long description in message payload of PKG-INFO file. - by :user:`cdce8p` +* #2645: Added ``License-File`` (multiple) to the output package metadata. + The field will contain the path of a license file, matched by the + ``license_file`` (deprecated) and ``license_files`` options, + relative to ``.dist-info``. - by :user:`cdce8p` +* #2678: Moved Setuptools' own entry points into declarative config. +* #2680: Vendored `more_itertools <https://pypi.org/project/more-itertools>`_ for Setuptools. +* #2681: Setuptools own setup.py no longer declares setup_requires, but instead expects wheel to be installed as declared by pyproject.toml. + +Misc +^^^^ +* #2650: Updated the docs build tooling to support the latest version of + Towncrier and show the previews of not-yet-released setuptools versions + in the changelog -- :user:`webknjaz` + + +v56.2.0 +------- + + +Changes +^^^^^^^ +* #2640: Fixed handling of multiline license strings. - by :user:`cdce8p` +* #2641: Setuptools will now always try to use the latest supported + metadata version for ``PKG-INFO``. - by :user:`cdce8p` + + +v56.1.0 +------- + + +Changes +^^^^^^^ +* #2653: Incorporated assorted changes from pypa/distutils. +* #2657: Adopted docs from distutils. +* #2663: Added Visual Studio Express 2017 support -- by :user:`dofuuz` + +Misc +^^^^ +* #2644: Fixed ``DeprecationWarning`` due to ``threading.Thread.setDaemon`` in tests -- by :user:`tirkarthi` +* #2654: Made the changelog generator compatible + with Towncrier >= 19.9 -- :user:`webknjaz` +* #2664: Relax the deprecation message in the distutils hack. + + +v56.0.0 +------- + + +Deprecations +^^^^^^^^^^^^ +* #2620: The ``license_file`` option is now marked as deprecated. + Use ``license_files`` instead. -- by :user:`cdce8p` + +Breaking Changes +^^^^^^^^^^^^^^^^ +* #2620: If neither ``license_file`` nor ``license_files`` is specified, the ``sdist`` + option will now auto-include files that match the following patterns: + ``LICEN[CS]E*``, ``COPYING*``, ``NOTICE*``, ``AUTHORS*``. + This matches the behavior of ``bdist_wheel``. -- by :user:`cdce8p` + +Changes +^^^^^^^ +* #2620: The ``license_file`` and ``license_files`` options now support glob patterns. -- by :user:`cdce8p` +* #2632: Implemented ``VendorImporter.find_spec()`` method to get rid + of ``ImportWarning`` that Python 3.10 emits when only the old-style + importer hooks are present -- by :user:`webknjaz` + +Documentation changes +^^^^^^^^^^^^^^^^^^^^^ +* #2620: Added documentation for the ``license_files`` option. -- by :user:`cdce8p` + + +v55.0.0 +------- + + +Breaking Changes +^^^^^^^^^^^^^^^^ +* #2566: Remove the deprecated ``bdist_wininst`` command. Binary packages should be built as wheels instead. -- by :user:`hroncok` + + +v54.2.0 +------- + + +Changes +^^^^^^^ +* #2608: Added informative error message to PEP 517 build failures owing to + an empty ``setup.py`` -- by :user:`layday` + + +v54.1.3 +------- + +No significant changes. + + +v54.1.2 +------- + + +Misc +^^^^ +* #2595: Reduced scope of dash deprecation warning to Setuptools/distutils only -- by :user:`melissa-kun-li` + + +v54.1.1 +------- + + +Documentation changes +^^^^^^^^^^^^^^^^^^^^^ +* #2584: Added ``sphinx-inline-tabs`` extension to allow for comparison of ``setup.py`` and its equivalent ``setup.cfg`` -- by :user:`amy-lei` + +Misc +^^^^ +* #2592: Made option keys in the ``[metadata]`` section of ``setup.cfg`` case-sensitive. Users having + uppercase option spellings will get a warning suggesting to make them to lowercase + -- by :user:`melissa-kun-li` + + +v54.1.0 +------- + + +Changes +^^^^^^^ +* #1608: Removed the conversion of dashes to underscores in the :code:`extras_require` and :code:`data_files` of :code:`setup.cfg` to support the usage of dashes. Method will warn users when they use a dash-separated key which in the future will only allow an underscore. Note: the method performs the dash to underscore conversion to preserve compatibility, but future versions will no longer support it -- by :user:`melissa-kun-li` + + +v54.0.0 +------- + + +Breaking Changes +^^^^^^^^^^^^^^^^ +* #2582: Simplified build-from-source story by providing bootstrapping metadata in a separate egg-info directory. Build requirements no longer include setuptools itself. Sdist once again includes the pyproject.toml. Project can no longer be installed from source on pip 19.x, but install from source is still supported on pip < 19 and pip >= 20 and install from wheel is still supported with pip >= 9. + +Changes +^^^^^^^ +* #1932: Handled :code:`AttributeError` by raising :code:`DistutilsSetupError` in :code:`dist.check_specifier()` when specifier is not a string -- by :user:`melissa-kun-li` +* #2570: Correctly parse cmdclass in setup.cfg. + +Documentation changes +^^^^^^^^^^^^^^^^^^^^^ +* #2553: Added userguide example for markers in extras_require -- by :user:`pwoolvett` + + +v53.1.0 +------- + + +Changes +^^^^^^^ +* #1937: Preserved case-sensitivity of keys in setup.cfg so that entry point names are case-sensitive. Changed sensitivity of configparser. NOTE: Any projects relying on case-insensitivity will need to adapt to accept the original case as published. -- by :user:`melissa-kun-li` +* #2573: Fixed error in uploading a Sphinx doc with the :code:`upload_docs` command. An html builder will be used. + Note: :code:`upload_docs` is deprecated for PyPi, but is supported for other sites -- by :user:`melissa-kun-li` + + +v53.0.0 +------- + + +Breaking Changes +^^^^^^^^^^^^^^^^ +* #1527: Removed bootstrap script. Now Setuptools requires pip or another pep517-compliant builder such as 'build' to build. Now Setuptools can be installed from Github main branch. + + +v52.0.0 +------- + + +Breaking Changes +^^^^^^^^^^^^^^^^ +* #2537: Remove fallback support for fetch_build_eggs using easy_install. Now pip is required for setup_requires to succeed. +* #2544: Removed 'easy_install' top-level model (runpy entry point) and 'easy_install' console script. +* #2545: Removed support for eggsecutables. + +Changes +^^^^^^^ +* #2459: Tests now run in parallel via pytest-xdist, completing in about half the time. Special thanks to :user:`webknjaz` for hard work implementing test isolation. To run without parallelization, disable the plugin with ``tox -- -p no:xdist``. + + +v51.3.3 +------- + + +Misc +^^^^ +* #2539: Fix AttributeError in Description validation. + + +v51.3.2 +------- + + +Misc +^^^^ +* #1390: Validation of Description field now is more lenient, emitting a warning and mangling the value to be valid (replacing newlines with spaces). + + +v51.3.1 +------- + + +Misc +^^^^ +* #2536: Reverted tag deduplication handling. + + +v51.3.0 +------- + + +Changes +^^^^^^^ +* #1390: Newlines in metadata description/Summary now trigger a ValueError. +* #2481: Define ``create_module()`` and ``exec_module()`` methods in ``VendorImporter`` + to get rid of ``ImportWarning`` -- by :user:`hroncok` +* #2489: ``pkg_resources`` behavior for zipimport now matches the regular behavior, and finds + ``.egg-info`` (previoulsy would only find ``.dist-info``) -- by :user:`thatch` +* #2529: Fixed an issue where version tags may be added multiple times + + +v51.2.0 +------- + + +Changes +^^^^^^^ +* #2493: Use importlib.import_module() rather than the deprecated loader.load_module() + in pkg_resources namespace delaration -- by :user:`encukou` + +Documentation changes +^^^^^^^^^^^^^^^^^^^^^ +* #2525: Fix typo in the document page about entry point. -- by :user:`jtr109` + +Misc +^^^^ +* #2534: Avoid hitting network during test_easy_install. + + +v51.1.2 +------- + + +Misc +^^^^ +* #2505: Disable inclusion of package data as it causes 'tests' to be included as data. + + +v51.1.1 +------- + + +Misc +^^^^ +* #2534: Avoid hitting network during test_virtualenv.test_test_command. + + v51.1.0 ------- @@ -371,6 +778,7 @@ v47.2.0 Changes ^^^^^^^ * #2194: Editable-installed entry points now load significantly faster on Python versions 3.8+. +* #1471: Incidentally fixed by #2194 on Python 3.8 or when importlib_metadata is present. v47.1.1 @@ -4049,7 +4457,7 @@ how it parses version numbers. (platform.mac_ver() fails) * Distribute #103: test_get_script_header_jython_workaround not run anymore under py3 with C or POSIX local. Contributed by Arfrever. -* Distribute #104: remvoved the assertion when the installation fails, +* Distribute #104: removed the assertion when the installation fails, with a nicer message for the end user. * Distribute #100: making sure there's no SandboxViolation when the setup script patches setuptools. @@ -4186,7 +4594,7 @@ setuptools bootstrapping ^^^^^^^^^^^^^ -* The boostrap process leave setuptools alone if detected in the system +* The bootstrap process leave setuptools alone if detected in the system and --root or --prefix is provided, but is not in the same location. This closes Distribute #10. diff --git a/MANIFEST.in b/MANIFEST.in index 92bd4f69..3e8f09de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,12 @@ -recursive-include setuptools *.py *.exe *.xml +recursive-include setuptools *.py *.exe *.xml *.tmpl recursive-include tests *.py recursive-include setuptools/tests *.html recursive-include docs *.py *.txt *.rst *.conf *.css *.css_t Makefile indexsidebar.html recursive-include setuptools/_vendor *.py *.txt recursive-include pkg_resources *.py *.txt recursive-include pkg_resources/tests/data * +recursive-include tools * +recursive-include changelog.d * include *.py include *.rst include MANIFEST.in @@ -13,4 +15,3 @@ include launcher.c include msvc-build-launcher.cmd include pytest.ini include tox.ini -exclude pyproject.toml # Temporary workaround for #1644. @@ -6,9 +6,9 @@ .. _PyPI link: https://pypi.org/project/setuptools -.. image:: https://github.com/jaraco/setuptools/workflows/Automated%20Tests/badge.svg - :target: https://github.com/jaraco/setuptools/actions?query=workflow%3A%22Automated+Tests%22 - :alt: Automated Tests +.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg + :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22 + :alt: tests .. image:: https://img.shields.io/badge/code%20style-black-000000.svg :target: https://github.com/psf/black @@ -17,6 +17,9 @@ .. image:: https://img.shields.io/readthedocs/setuptools/latest.svg :target: https://setuptools.readthedocs.io +.. image:: https://img.shields.io/badge/skeleton-2021-informational + :target: https://blog.jaraco.com/skeleton + .. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white :target: https://codecov.io/gh/pypa/setuptools diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index c31edfed..5f40996a 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -8,6 +8,11 @@ import warnings is_pypy = '__pypy__' in sys.builtin_module_names +warnings.filterwarnings('ignore', + r'.+ distutils\b.+ deprecated', + DeprecationWarning) + + def warn_distutils_present(): if 'distutils' not in sys.modules: return diff --git a/bootstrap.egg-info/PKG-INFO b/bootstrap.egg-info/PKG-INFO new file mode 100644 index 00000000..6e11ceeb --- /dev/null +++ b/bootstrap.egg-info/PKG-INFO @@ -0,0 +1,2 @@ +Name: setuptools-bootstrap +Version: 1.0 diff --git a/bootstrap.egg-info/entry_points.txt b/bootstrap.egg-info/entry_points.txt new file mode 100644 index 00000000..834d674e --- /dev/null +++ b/bootstrap.egg-info/entry_points.txt @@ -0,0 +1,14 @@ +[distutils.commands] +egg_info = setuptools.command.egg_info:egg_info + +[distutils.setup_keywords] +include_package_data = setuptools.dist:assert_bool +install_requires = setuptools.dist:check_requirements +extras_require = setuptools.dist:check_extras +entry_points = setuptools.dist:check_entry_points + +[egg_info.writers] +PKG-INFO = setuptools.command.egg_info:write_pkg_info +dependency_links.txt = setuptools.command.egg_info:overwrite_arg +entry_points.txt = setuptools.command.egg_info:write_entries +requires.txt = setuptools.command.egg_info:write_requirements diff --git a/bootstrap.py b/bootstrap.py index 118671f6..229b9965 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -1,57 +1,7 @@ -""" -If setuptools is not already installed in the environment, it's not possible -to invoke setuptools' own commands. This routine will bootstrap this local -environment by creating a minimal egg-info directory and then invoking the -egg-info command to flesh out the egg-info directory. -""" +import warnings -import os -import sys -import textwrap -import subprocess -import io +msg = "bootstrap.py is no longer needed. Use a PEP-517-compatible builder instead." -minimal_egg_info = textwrap.dedent(""" - [distutils.commands] - egg_info = setuptools.command.egg_info:egg_info - [distutils.setup_keywords] - include_package_data = setuptools.dist:assert_bool - install_requires = setuptools.dist:check_requirements - extras_require = setuptools.dist:check_extras - entry_points = setuptools.dist:check_entry_points - - [egg_info.writers] - PKG-INFO = setuptools.command.egg_info:write_pkg_info - dependency_links.txt = setuptools.command.egg_info:overwrite_arg - entry_points.txt = setuptools.command.egg_info:write_entries - requires.txt = setuptools.command.egg_info:write_requirements - """) - - -def ensure_egg_info(): - if os.path.exists('setuptools.egg-info'): - return - print("adding minimal entry_points") - add_minimal_info() - run_egg_info() - - -def add_minimal_info(): - """ - Build a minimal egg-info, enough to invoke egg_info - """ - - os.mkdir('setuptools.egg-info') - with io.open('setuptools.egg-info/entry_points.txt', 'w') as ep: - ep.write(minimal_egg_info) - - -def run_egg_info(): - cmd = [sys.executable, 'setup.py', 'egg_info'] - print("Regenerating egg_info") - subprocess.check_call(cmd) - - -__name__ == '__main__' and ensure_egg_info() +__name__ == '__main__' and warnings.warn(msg) diff --git a/changelog.d/1988-change.rst b/changelog.d/1988-change.rst new file mode 100644 index 00000000..4f104639 --- /dev/null +++ b/changelog.d/1988-change.rst @@ -0,0 +1,2 @@ +Deprecated the ``bdist_rpm`` command. Binary packages should be built as wheels instead. +-- by :user:`hugovk` diff --git a/changelog.d/2762.misc.rst b/changelog.d/2762.misc.rst new file mode 100644 index 00000000..cc9bf86c --- /dev/null +++ b/changelog.d/2762.misc.rst @@ -0,0 +1,2 @@ +Changed codecov.yml to configure the threshold to be lower +-- by :user:`tanvimoharir` diff --git a/changelog.d/2785-change.rst b/changelog.d/2785-change.rst new file mode 100644 index 00000000..7e9631b7 --- /dev/null +++ b/changelog.d/2785-change.rst @@ -0,0 +1,2 @@ +Replace confirparser's readfp with read_file, deprecated since Python 3.2. +-- by :user:`hugovk` diff --git a/changelog.d/README.rst b/changelog.d/README.rst index 30831edc..49b4d563 100644 --- a/changelog.d/README.rst +++ b/changelog.d/README.rst @@ -76,7 +76,7 @@ File :file:`changelog.d/1354.misc.rst`: .. code-block:: rst - Added ``towncrier`` for changelog managment -- by :user:`pganssle` + Added ``towncrier`` for changelog management -- by :user:`pganssle` File :file:`changelog.d/2355.change.rst`: diff --git a/docs/_templates/indexsidebar.html b/docs/_templates/indexsidebar.html deleted file mode 100644 index d803b8a3..00000000 --- a/docs/_templates/indexsidebar.html +++ /dev/null @@ -1,15 +0,0 @@ -<h3 class="donation">For Enterprise</h3> - -<p> -Professionally-supported {{ project }} is available with the -<a href="https://tidelift.com/subscription/pkg/pypi-{{ project }}?utm_source=pypi-{{ project }}&utm_medium=referral">Tidelift Subscription</a>. -</p> - -<h3>Download</h3> - -<p>Current version: <b>{{ version }}</b></p> -<p>Get Setuptools from the <a href="https://pypi.org/project/setuptools/"> Python Package Index</a> - -<h3>Questions? Suggestions? Contributions?</h3> - -<p>Visit the <a href="{{ package_url }}">Project page</a> </p> diff --git a/docs/_templates/tidelift-sidebar.html b/docs/_templates/tidelift-sidebar.html deleted file mode 100644 index ce48f46b..00000000 --- a/docs/_templates/tidelift-sidebar.html +++ /dev/null @@ -1,6 +0,0 @@ -<h3 class="donation">For Enterprise</h3> - -<p> -Professionally-supported {{ project }} is available with the -<a href="https://tidelift.com/subscription/pkg/pypi-{{ project }}?utm_source=pypi-{{ project }}&utm_medium=referral">Tidelift Subscription</a>. -</p> diff --git a/docs/_theme/nature/static/nature.css_t b/docs/_theme/nature/static/nature.css_t deleted file mode 100644 index 1a654264..00000000 --- a/docs/_theme/nature/static/nature.css_t +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Sphinx stylesheet -- default theme - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - -@import url("basic.css"); - -/* -- page layout ----------------------------------------------------------- */ - -body { - font-family: Arial, sans-serif; - font-size: 100%; - background-color: #111111; - color: #555555; - margin: 0; - padding: 0; -} - -div.documentwrapper { - float: left; - width: 100%; -} - -div.bodywrapper { - margin: 0 0 0 300px; -} - -hr{ - border: 1px solid #B1B4B6; -} - -div.document { - background-color: #fafafa; -} - -div.body { - background-color: #ffffff; - color: #3E4349; - padding: 1em 30px 30px 30px; - font-size: 0.9em; -} - -div.footer { - color: #555; - width: 100%; - padding: 13px 0; - text-align: center; - font-size: 75%; -} - -div.footer a { - color: #444444; -} - -div.related { - background-color: #6BA81E; - line-height: 36px; - color: #ffffff; - text-shadow: 0px 1px 0 #444444; - font-size: 1.1em; -} - -div.related a { - color: #E2F3CC; -} - -div.related .right { - font-size: 0.9em; -} - -div.sphinxsidebar { - font-size: 0.9em; - line-height: 1.5em; - width: 300px; -} - -div.sphinxsidebarwrapper{ - padding: 20px 0; -} - -div.sphinxsidebar h3, -div.sphinxsidebar h4 { - font-family: Arial, sans-serif; - color: #222222; - font-size: 1.2em; - font-weight: bold; - margin: 0; - padding: 5px 10px; - text-shadow: 1px 1px 0 white -} - -div.sphinxsidebar h3 a { - color: #444444; -} - -div.sphinxsidebar p { - color: #888888; - padding: 5px 20px; - margin: 0.5em 0px; -} - -div.sphinxsidebar p.topless { -} - -div.sphinxsidebar ul { - margin: 10px 10px 10px 20px; - padding: 0; - color: #000000; -} - -div.sphinxsidebar a { - color: #444444; -} - -div.sphinxsidebar a:hover { - color: #E32E00; -} - -div.sphinxsidebar input { - border: 1px solid #cccccc; - font-family: sans-serif; - font-size: 1.1em; - padding: 0.15em 0.3em; -} - -div.sphinxsidebar input[type=text]{ - margin-left: 20px; -} - -/* -- body styles ----------------------------------------------------------- */ - -a { - color: #005B81; - text-decoration: none; -} - -a:hover { - color: #E32E00; -} - -div.body h1, -div.body h2, -div.body h3, -div.body h4, -div.body h5, -div.body h6 { - font-family: Arial, sans-serif; - font-weight: normal; - color: #212224; - margin: 30px 0px 10px 0px; - padding: 5px 0 5px 0px; - text-shadow: 0px 1px 0 white; - border-bottom: 1px solid #C8D5E3; -} - -div.body h1 { margin-top: 0; font-size: 200%; } -div.body h2 { font-size: 150%; } -div.body h3 { font-size: 120%; } -div.body h4 { font-size: 110%; } -div.body h5 { font-size: 100%; } -div.body h6 { font-size: 100%; } - -a.headerlink { - color: #c60f0f; - font-size: 0.8em; - padding: 0 4px 0 4px; - text-decoration: none; -} - -a.headerlink:hover { - background-color: #c60f0f; - color: white; -} - -div.body p, div.body dd, div.body li { - line-height: 1.8em; -} - -div.admonition p.admonition-title + p { - display: inline; -} - -div.highlight{ - background-color: white; -} - -div.note { - background-color: #eeeeee; - border: 1px solid #cccccc; -} - -div.seealso { - background-color: #ffffcc; - border: 1px solid #ffff66; -} - -div.topic { - background-color: #fafafa; - border-width: 0; -} - -div.warning { - background-color: #ffe4e4; - border: 1px solid #ff6666; -} - -p.admonition-title { - display: inline; -} - -p.admonition-title:after { - content: ":"; -} - -pre { - padding: 10px; - background-color: #fafafa; - color: #222222; - line-height: 1.5em; - font-size: 1.1em; - margin: 1.5em 0 1.5em 0; - -webkit-box-shadow: 0px 0px 4px #d8d8d8; - -moz-box-shadow: 0px 0px 4px #d8d8d8; - box-shadow: 0px 0px 4px #d8d8d8; -} - -tt { - color: #222222; - padding: 1px 2px; - font-size: 1.2em; - font-family: monospace; -} - -#table-of-contents ul { - padding-left: 2em; -} - diff --git a/docs/_theme/nature/static/pygments.css b/docs/_theme/nature/static/pygments.css deleted file mode 100644 index 652b7612..00000000 --- a/docs/_theme/nature/static/pygments.css +++ /dev/null @@ -1,54 +0,0 @@ -.c { color: #999988; font-style: italic } /* Comment */ -.k { font-weight: bold } /* Keyword */ -.o { font-weight: bold } /* Operator */ -.cm { color: #999988; font-style: italic } /* Comment.Multiline */ -.cp { color: #999999; font-weight: bold } /* Comment.preproc */ -.c1 { color: #999988; font-style: italic } /* Comment.Single */ -.gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ -.ge { font-style: italic } /* Generic.Emph */ -.gr { color: #aa0000 } /* Generic.Error */ -.gh { color: #999999 } /* Generic.Heading */ -.gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ -.go { color: #111 } /* Generic.Output */ -.gp { color: #555555 } /* Generic.Prompt */ -.gs { font-weight: bold } /* Generic.Strong */ -.gu { color: #aaaaaa } /* Generic.Subheading */ -.gt { color: #aa0000 } /* Generic.Traceback */ -.kc { font-weight: bold } /* Keyword.Constant */ -.kd { font-weight: bold } /* Keyword.Declaration */ -.kp { font-weight: bold } /* Keyword.Pseudo */ -.kr { font-weight: bold } /* Keyword.Reserved */ -.kt { color: #445588; font-weight: bold } /* Keyword.Type */ -.m { color: #009999 } /* Literal.Number */ -.s { color: #bb8844 } /* Literal.String */ -.na { color: #008080 } /* Name.Attribute */ -.nb { color: #999999 } /* Name.Builtin */ -.nc { color: #445588; font-weight: bold } /* Name.Class */ -.no { color: #ff99ff } /* Name.Constant */ -.ni { color: #800080 } /* Name.Entity */ -.ne { color: #990000; font-weight: bold } /* Name.Exception */ -.nf { color: #990000; font-weight: bold } /* Name.Function */ -.nn { color: #555555 } /* Name.Namespace */ -.nt { color: #000080 } /* Name.Tag */ -.nv { color: purple } /* Name.Variable */ -.ow { font-weight: bold } /* Operator.Word */ -.mf { color: #009999 } /* Literal.Number.Float */ -.mh { color: #009999 } /* Literal.Number.Hex */ -.mi { color: #009999 } /* Literal.Number.Integer */ -.mo { color: #009999 } /* Literal.Number.Oct */ -.sb { color: #bb8844 } /* Literal.String.Backtick */ -.sc { color: #bb8844 } /* Literal.String.Char */ -.sd { color: #bb8844 } /* Literal.String.Doc */ -.s2 { color: #bb8844 } /* Literal.String.Double */ -.se { color: #bb8844 } /* Literal.String.Escape */ -.sh { color: #bb8844 } /* Literal.String.Heredoc */ -.si { color: #bb8844 } /* Literal.String.Interpol */ -.sx { color: #bb8844 } /* Literal.String.Other */ -.sr { color: #808000 } /* Literal.String.Regex */ -.s1 { color: #bb8844 } /* Literal.String.Single */ -.ss { color: #bb8844 } /* Literal.String.Symbol */ -.bp { color: #999999 } /* Name.Builtin.Pseudo */ -.vc { color: #ff99ff } /* Name.Variable.Class */ -.vg { color: #ff99ff } /* Name.Variable.Global */ -.vi { color: #ff99ff } /* Name.Variable.Instance */ -.il { color: #009999 } /* Literal.Number.Integer.Long */
\ No newline at end of file diff --git a/docs/_theme/nature/theme.conf b/docs/_theme/nature/theme.conf deleted file mode 100644 index 1cc40044..00000000 --- a/docs/_theme/nature/theme.conf +++ /dev/null @@ -1,4 +0,0 @@ -[theme] -inherit = basic -stylesheet = nature.css -pygments_style = tango diff --git a/docs/build_meta.rst b/docs/build_meta.rst index c36e2bab..27df70a2 100644 --- a/docs/build_meta.rst +++ b/docs/build_meta.rst @@ -5,9 +5,9 @@ Build System Support What is it? ------------- -Python packaging has come `a long way <https://www.bernat.tech/pep-517-518/>`_. +Python packaging has come `a long way <https://bernat.tech/posts/pep-517-518/>`_. -The traditional ``setuptools`` way of packgaging Python modules +The traditional ``setuptools`` way of packaging Python modules uses a ``setup()`` function within the ``setup.py`` script. Commands such as ``python setup.py bdist`` or ``python setup.py bdist_wheel`` generate a distribution bundle and ``python setup.py install`` installs the distribution. @@ -54,7 +54,11 @@ setuptools, the content would be:: [build-system] requires = ["setuptools", "wheel"] - build-backend = "setuptools.build_meta" + build-backend = "setuptools.build_meta" + +The ``setuptools`` package implements the ``build_sdist`` +command and the ``wheel`` package implements the ``build_wheel`` +command; both are required to be compliant with PEP 517. Use ``setuptools``' :ref:`declarative config <declarative config>` to specify the package information:: @@ -67,14 +71,11 @@ specify the package information:: [options] packages = find: -Now generate the distribution. Although the PyPA is still working to -`provide a recommended tool <https://github.com/pypa/packaging-problems/issues/219>`_ -to build packages, the `pep517 package <https://pypi.org/project/pep517>`_ -provides this functionality. To build the package:: +Now generate the distribution. To build the package, use +`PyPA build <https://pypa-build.readthedocs.io/en/latest/>`_:: - $ pip install -q pep517 - $ mkdir dist - $ python -m pep517.build . + $ pip install -q build + $ python -m build And now it's done! The ``.whl`` file and ``.tar.gz`` can then be distributed and installed:: diff --git a/docs/conf.py b/docs/conf.py index 8cb959df..8f1e1f4c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,11 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import subprocess -import sys -import os - - extensions = ['sphinx.ext.autodoc', 'jaraco.packaging.sphinx', 'rst.linker'] master_doc = "index" @@ -81,14 +73,12 @@ link_files = { ), } +# Be strict about any broken references: +nitpicky = True -# hack to run the bootstrap script so that jaraco.packaging.sphinx -# can invoke setup.py -'READTHEDOCS' in os.environ and subprocess.check_call( - [sys.executable, '-m', 'bootstrap'], - cwd=os.path.join(os.path.dirname(__file__), os.path.pardir), -) - +intersphinx_mapping = { + 'pypa-build': ('https://pypa-build.readthedocs.io/en/latest/', None) +} # Add support for linking usernames github_url = 'https://github.com' @@ -96,16 +86,67 @@ github_sponsors_url = f'{github_url}/sponsors' extlinks = { 'user': (f'{github_sponsors_url}/%s', '@'), # noqa: WPS323 } -extensions += ['sphinx.ext.extlinks'] - -# Be strict about any broken references: -nitpicky = True +extensions += ['sphinx.ext.extlinks', 'sphinx.ext.intersphinx'] # Ref: https://github.com/python-attrs/attrs/pull/571/files\ # #diff-85987f48f1258d9ee486e3191495582dR82 default_role = 'any' -# Custom sidebar templates, maps document names to template names. -html_theme = 'alabaster' -templates_path = ['_templates'] -html_sidebars = {'index': ['tidelift-sidebar.html']} +# HTML theme +html_theme = 'furo' + +# Add support for inline tabs +extensions += ['sphinx_inline_tabs'] + +# Support for distutils + +# Ref: https://stackoverflow.com/a/30624034/595220 +nitpick_ignore = [ + ('c:func', 'SHGetSpecialFolderPath'), # ref to MS docs + ('envvar', 'DISTUTILS_DEBUG'), # undocumented + ('envvar', 'HOME'), # undocumented + ('envvar', 'PLAT'), # undocumented + ('py:attr', 'CCompiler.language_map'), # undocumented + ('py:attr', 'CCompiler.language_order'), # undocumented + ('py:class', 'distutils.dist.Distribution'), # undocumented + ('py:class', 'distutils.extension.Extension'), # undocumented + ('py:class', 'BorlandCCompiler'), # undocumented + ('py:class', 'CCompiler'), # undocumented + ('py:class', 'CygwinCCompiler'), # undocumented + ('py:class', 'distutils.dist.DistributionMetadata'), # undocumented + ('py:class', 'FileList'), # undocumented + ('py:class', 'IShellLink'), # ref to MS docs + ('py:class', 'MSVCCompiler'), # undocumented + ('py:class', 'OptionDummy'), # undocumented + ('py:class', 'UnixCCompiler'), # undocumented + ('py:exc', 'CompileError'), # undocumented + ('py:exc', 'DistutilsExecError'), # undocumented + ('py:exc', 'DistutilsFileError'), # undocumented + ('py:exc', 'LibError'), # undocumented + ('py:exc', 'LinkError'), # undocumented + ('py:exc', 'PreprocessError'), # undocumented + ('py:func', 'distutils.CCompiler.new_compiler'), # undocumented + # undocumented: + ('py:func', 'distutils.dist.DistributionMetadata.read_pkg_file'), + ('py:func', 'distutils.file_util._copy_file_contents'), # undocumented + ('py:func', 'distutils.log.debug'), # undocumented + ('py:func', 'distutils.spawn.find_executable'), # undocumented + ('py:func', 'distutils.spawn.spawn'), # undocumented + # TODO: check https://docutils.rtfd.io in the future + ('py:mod', 'docutils'), # there's no Sphinx site documenting this +] + +# Allow linking objects on other Sphinx sites seamlessly: +intersphinx_mapping.update( + python=('https://docs.python.org/3', None), + python2=('https://docs.python.org/2', None), +) + +# Add support for the unreleased "next-version" change notes +extensions += ['sphinxcontrib.towncrier'] +# Extension needs a path from here to the towncrier config. +towncrier_draft_working_directory = '..' +# Avoid an empty section for unpublished changes. +towncrier_draft_include_empty = False + +extensions += ['jaraco.tidelift'] diff --git a/docs/deprecated/distutils/_setuptools_disclaimer.rst b/docs/deprecated/distutils/_setuptools_disclaimer.rst new file mode 100644 index 00000000..cc758583 --- /dev/null +++ b/docs/deprecated/distutils/_setuptools_disclaimer.rst @@ -0,0 +1,5 @@ +.. note:: + + This document is being retained solely until the ``setuptools`` documentation + at https://setuptools.readthedocs.io/en/latest/setuptools.html + independently covers all of the relevant information currently included here. diff --git a/docs/deprecated/distutils/apiref.rst b/docs/deprecated/distutils/apiref.rst new file mode 100644 index 00000000..f721fc16 --- /dev/null +++ b/docs/deprecated/distutils/apiref.rst @@ -0,0 +1,2053 @@ +.. _api-reference: + +************* +API Reference +************* + +.. seealso:: + + `New and changed setup.py arguments in setuptools`_ + The ``setuptools`` project adds new capabilities to the ``setup`` function + and other APIs, makes the API consistent across different Python versions, + and is hence recommended over using ``distutils`` directly. + +.. _New and changed setup.py arguments in setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords + +.. include:: ./_setuptools_disclaimer.rst + +:mod:`distutils.core` --- Core Distutils functionality +====================================================== + +.. module:: distutils.core + :synopsis: The core Distutils functionality + + +The :mod:`distutils.core` module is the only module that needs to be installed +to use the Distutils. It provides the :func:`setup` (which is called from the +setup script). Indirectly provides the :class:`distutils.dist.Distribution` and +:class:`distutils.cmd.Command` class. + + +.. function:: setup(arguments) + + The basic do-everything function that does most everything you could ever ask + for from a Distutils method. + + The setup function takes a large number of arguments. These are laid out in the + following table. + + .. tabularcolumns:: |l|L|L| + + +--------------------+--------------------------------+-------------------------------------------------------------+ + | argument name | value | type | + +====================+================================+=============================================================+ + | *name* | The name of the package | a string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *version* | The version number of the | a string | + | | package; see | | + | | :mod:`distutils.version` | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *description* | A single line describing the | a string | + | | package | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *long_description* | Longer description of the | a string | + | | package | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *author* | The name of the package author | a string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *author_email* | The email address of the | a string | + | | package author | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *maintainer* | The name of the current | a string | + | | maintainer, if different from | | + | | the author. Note that if | | + | | the maintainer is provided, | | + | | distutils will use it as the | | + | | author in :file:`PKG-INFO` | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *maintainer_email* | The email address of the | a string | + | | current maintainer, if | | + | | different from the author | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *url* | A URL for the package | a string | + | | (homepage) | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *download_url* | A URL to download the package | a string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *packages* | A list of Python packages that | a list of strings | + | | distutils will manipulate | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *py_modules* | A list of Python modules that | a list of strings | + | | distutils will manipulate | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *scripts* | A list of standalone script | a list of strings | + | | files to be built and | | + | | installed | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *ext_modules* | A list of Python extensions to | a list of instances of | + | | be built | :class:`distutils.core.Extension` | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *classifiers* | A list of categories for the | a list of strings; valid classifiers are listed on `PyPI | + | | package | <https://pypi.org/classifiers>`_. | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *distclass* | the :class:`Distribution` | a subclass of | + | | class to use | :class:`distutils.core.Distribution` | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *script_name* | The name of the setup.py | a string | + | | script - defaults to | | + | | ``sys.argv[0]`` | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *script_args* | Arguments to supply to the | a list of strings | + | | setup script | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *options* | default options for the setup | a dictionary | + | | script | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *license* | The license for the package | a string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *keywords* | Descriptive meta-data, see | a list of strings or a comma-separated string | + | | :pep:`314` | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *platforms* | | a list of strings or a comma-separated string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *cmdclass* | A mapping of command names to | a dictionary | + | | :class:`Command` subclasses | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *data_files* | A list of data files to | a list | + | | install | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *package_dir* | A mapping of package to | a dictionary | + | | directory names | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + + + +.. function:: run_setup(script_name[, script_args=None, stop_after='run']) + + Run a setup script in a somewhat controlled environment, and return the + :class:`distutils.dist.Distribution` instance that drives things. This is + useful if you need to find out the distribution meta-data (passed as keyword + args from *script* to :func:`setup`), or the contents of the config files or + command-line. + + *script_name* is a file that will be read and run with :func:`exec`. ``sys.argv[0]`` + will be replaced with *script* for the duration of the call. *script_args* is a + list of strings; if supplied, ``sys.argv[1:]`` will be replaced by *script_args* + for the duration of the call. + + *stop_after* tells :func:`setup` when to stop processing; possible values: + + .. tabularcolumns:: |l|L| + + +---------------+---------------------------------------------+ + | value | description | + +===============+=============================================+ + | *init* | Stop after the :class:`Distribution` | + | | instance has been created and populated | + | | with the keyword arguments to :func:`setup` | + +---------------+---------------------------------------------+ + | *config* | Stop after config files have been parsed | + | | (and their data stored in the | + | | :class:`Distribution` instance) | + +---------------+---------------------------------------------+ + | *commandline* | Stop after the command-line | + | | (``sys.argv[1:]`` or *script_args*) have | + | | been parsed (and the data stored in the | + | | :class:`Distribution` instance.) | + +---------------+---------------------------------------------+ + | *run* | Stop after all commands have been run (the | + | | same as if :func:`setup` had been called | + | | in the usual way). This is the default | + | | value. | + +---------------+---------------------------------------------+ + +In addition, the :mod:`distutils.core` module exposed a number of classes that +live elsewhere. + +* :class:`~distutils.extension.Extension` from :mod:`distutils.extension` + +* :class:`~distutils.cmd.Command` from :mod:`distutils.cmd` + +* :class:`~distutils.dist.Distribution` from :mod:`distutils.dist` + +A short description of each of these follows, but see the relevant module for +the full reference. + + +.. class:: Extension + + The Extension class describes a single C or C++ extension module in a setup + script. It accepts the following keyword arguments in its constructor: + + .. tabularcolumns:: |l|L|l| + + +------------------------+--------------------------------+---------------------------+ + | argument name | value | type | + +========================+================================+===========================+ + | *name* | the full name of the | a string | + | | extension, including any | | + | | packages --- ie. *not* a | | + | | filename or pathname, but | | + | | Python dotted name | | + +------------------------+--------------------------------+---------------------------+ + | *sources* | list of source filenames, | a list of strings | + | | relative to the distribution | | + | | root (where the setup script | | + | | lives), in Unix form | | + | | (slash-separated) for | | + | | portability. | | + | | Source files may be C, C++, | | + | | SWIG (.i), platform-specific | | + | | resource files, or whatever | | + | | else is recognized by the | | + | | :command:`build_ext` command | | + | | as source for a Python | | + | | extension. | | + +------------------------+--------------------------------+---------------------------+ + | *include_dirs* | list of directories to search | a list of strings | + | | for C/C++ header files (in | | + | | Unix form for portability) | | + +------------------------+--------------------------------+---------------------------+ + | *define_macros* | list of macros to define; each | a list of tuples | + | | macro is defined using a | | + | | 2-tuple ``(name, value)``, | | + | | where *value* is | | + | | either the string to define it | | + | | to or ``None`` to define it | | + | | without a particular value | | + | | (equivalent of ``#define FOO`` | | + | | in source or :option:`!-DFOO` | | + | | on Unix C compiler command | | + | | line) | | + +------------------------+--------------------------------+---------------------------+ + | *undef_macros* | list of macros to undefine | a list of strings | + | | explicitly | | + +------------------------+--------------------------------+---------------------------+ + | *library_dirs* | list of directories to search | a list of strings | + | | for C/C++ libraries at link | | + | | time | | + +------------------------+--------------------------------+---------------------------+ + | *libraries* | list of library names (not | a list of strings | + | | filenames or paths) to link | | + | | against | | + +------------------------+--------------------------------+---------------------------+ + | *runtime_library_dirs* | list of directories to search | a list of strings | + | | for C/C++ libraries at run | | + | | time (for shared extensions, | | + | | this is when the extension is | | + | | loaded) | | + +------------------------+--------------------------------+---------------------------+ + | *extra_objects* | list of extra files to link | a list of strings | + | | with (eg. object files not | | + | | implied by 'sources', static | | + | | library that must be | | + | | explicitly specified, binary | | + | | resource files, etc.) | | + +------------------------+--------------------------------+---------------------------+ + | *extra_compile_args* | any extra platform- and | a list of strings | + | | compiler-specific information | | + | | to use when compiling the | | + | | source files in 'sources'. For | | + | | platforms and compilers where | | + | | a command line makes sense, | | + | | this is typically a list of | | + | | command-line arguments, but | | + | | for other platforms it could | | + | | be anything. | | + +------------------------+--------------------------------+---------------------------+ + | *extra_link_args* | any extra platform- and | a list of strings | + | | compiler-specific information | | + | | to use when linking object | | + | | files together to create the | | + | | extension (or to create a new | | + | | static Python interpreter). | | + | | Similar interpretation as for | | + | | 'extra_compile_args'. | | + +------------------------+--------------------------------+---------------------------+ + | *export_symbols* | list of symbols to be exported | a list of strings | + | | from a shared extension. Not | | + | | used on all platforms, and not | | + | | generally necessary for Python | | + | | extensions, which typically | | + | | export exactly one symbol: | | + | | ``init`` + extension_name. | | + +------------------------+--------------------------------+---------------------------+ + | *depends* | list of files that the | a list of strings | + | | extension depends on | | + +------------------------+--------------------------------+---------------------------+ + | *language* | extension language (i.e. | a string | + | | ``'c'``, ``'c++'``, | | + | | ``'objc'``). Will be detected | | + | | from the source extensions if | | + | | not provided. | | + +------------------------+--------------------------------+---------------------------+ + | *optional* | specifies that a build failure | a boolean | + | | in the extension should not | | + | | abort the build process, but | | + | | simply skip the extension. | | + +------------------------+--------------------------------+---------------------------+ + + .. versionchanged:: 3.8 + + On Unix, C extensions are no longer linked to libpython except on + Android and Cygwin. + + +.. class:: Distribution + + A :class:`Distribution` describes how to build, install and package up a Python + software package. + + See the :func:`setup` function for a list of keyword arguments accepted by the + Distribution constructor. :func:`setup` creates a Distribution instance. + + .. versionchanged:: 3.7 + :class:`~distutils.core.Distribution` now warns if ``classifiers``, + ``keywords`` and ``platforms`` fields are not specified as a list or + a string. + +.. class:: Command + + A :class:`Command` class (or rather, an instance of one of its subclasses) + implement a single distutils command. + + +:mod:`distutils.ccompiler` --- CCompiler base class +=================================================== + +.. module:: distutils.ccompiler + :synopsis: Abstract CCompiler class + + +This module provides the abstract base class for the :class:`CCompiler` +classes. A :class:`CCompiler` instance can be used for all the compile and +link steps needed to build a single project. Methods are provided to set +options for the compiler --- macro definitions, include directories, link path, +libraries and the like. + +This module provides the following functions. + + +.. function:: gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) + + Generate linker options for searching library directories and linking with + specific libraries. *libraries* and *library_dirs* are, respectively, lists of + library names (not filenames!) and search directories. Returns a list of + command-line options suitable for use with some compiler (depending on the two + format strings passed in). + + +.. function:: gen_preprocess_options(macros, include_dirs) + + Generate C pre-processor options (:option:`!-D`, :option:`!-U`, :option:`!-I`) as + used by at least two types of compilers: the typical Unix compiler and Visual + C++. *macros* is the usual thing, a list of 1- or 2-tuples, where ``(name,)`` + means undefine (:option:`!-U`) macro *name*, and ``(name, value)`` means define + (:option:`!-D`) macro *name* to *value*. *include_dirs* is just a list of + directory names to be added to the header file search path (:option:`!-I`). + Returns a list of command-line options suitable for either Unix compilers or + Visual C++. + + +.. function:: get_default_compiler(osname, platform) + + Determine the default compiler to use for the given platform. + + *osname* should be one of the standard Python OS names (i.e. the ones returned + by ``os.name``) and *platform* the common value returned by ``sys.platform`` for + the platform in question. + + The default values are ``os.name`` and ``sys.platform`` in case the parameters + are not given. + + +.. function:: new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0) + + Factory function to generate an instance of some CCompiler subclass for the + supplied platform/compiler combination. *plat* defaults to ``os.name`` (eg. + ``'posix'``, ``'nt'``), and *compiler* defaults to the default compiler for + that platform. Currently only ``'posix'`` and ``'nt'`` are supported, and the + default compilers are "traditional Unix interface" (:class:`UnixCCompiler` + class) and Visual C++ (:class:`MSVCCompiler` class). Note that it's perfectly + possible to ask for a Unix compiler object under Windows, and a Microsoft + compiler object under Unix---if you supply a value for *compiler*, *plat* is + ignored. + + .. % Is the posix/nt only thing still true? Mac OS X seems to work, and + .. % returns a UnixCCompiler instance. How to document this... hmm. + + +.. function:: show_compilers() + + Print list of available compilers (used by the :option:`!--help-compiler` options + to :command:`build`, :command:`build_ext`, :command:`build_clib`). + + +.. class:: CCompiler([verbose=0, dry_run=0, force=0]) + + The abstract base class :class:`CCompiler` defines the interface that must be + implemented by real compiler classes. The class also has some utility methods + used by several compiler classes. + + The basic idea behind a compiler abstraction class is that each instance can be + used for all the compile/link steps in building a single project. Thus, + attributes common to all of those compile and link steps --- include + directories, macros to define, libraries to link against, etc. --- are + attributes of the compiler instance. To allow for variability in how individual + files are treated, most of those attributes may be varied on a per-compilation + or per-link basis. + + The constructor for each subclass creates an instance of the Compiler object. + Flags are *verbose* (show verbose output), *dry_run* (don't actually execute the + steps) and *force* (rebuild everything, regardless of dependencies). All of + these flags default to ``0`` (off). Note that you probably don't want to + instantiate :class:`CCompiler` or one of its subclasses directly - use the + :func:`distutils.CCompiler.new_compiler` factory function instead. + + The following methods allow you to manually alter compiler options for the + instance of the Compiler class. + + + .. method:: CCompiler.add_include_dir(dir) + + Add *dir* to the list of directories that will be searched for header files. + The compiler is instructed to search directories in the order in which they are + supplied by successive calls to :meth:`add_include_dir`. + + + .. method:: CCompiler.set_include_dirs(dirs) + + Set the list of directories that will be searched to *dirs* (a list of strings). + Overrides any preceding calls to :meth:`add_include_dir`; subsequent calls to + :meth:`add_include_dir` add to the list passed to :meth:`set_include_dirs`. + This does not affect any list of standard include directories that the compiler + may search by default. + + + .. method:: CCompiler.add_library(libname) + + Add *libname* to the list of libraries that will be included in all links driven + by this compiler object. Note that *libname* should \*not\* be the name of a + file containing a library, but the name of the library itself: the actual + filename will be inferred by the linker, the compiler, or the compiler class + (depending on the platform). + + The linker will be instructed to link against libraries in the order they were + supplied to :meth:`add_library` and/or :meth:`set_libraries`. It is perfectly + valid to duplicate library names; the linker will be instructed to link against + libraries as many times as they are mentioned. + + + .. method:: CCompiler.set_libraries(libnames) + + Set the list of libraries to be included in all links driven by this compiler + object to *libnames* (a list of strings). This does not affect any standard + system libraries that the linker may include by default. + + + .. method:: CCompiler.add_library_dir(dir) + + Add *dir* to the list of directories that will be searched for libraries + specified to :meth:`add_library` and :meth:`set_libraries`. The linker will be + instructed to search for libraries in the order they are supplied to + :meth:`add_library_dir` and/or :meth:`set_library_dirs`. + + + .. method:: CCompiler.set_library_dirs(dirs) + + Set the list of library search directories to *dirs* (a list of strings). This + does not affect any standard library search path that the linker may search by + default. + + + .. method:: CCompiler.add_runtime_library_dir(dir) + + Add *dir* to the list of directories that will be searched for shared libraries + at runtime. + + + .. method:: CCompiler.set_runtime_library_dirs(dirs) + + Set the list of directories to search for shared libraries at runtime to *dirs* + (a list of strings). This does not affect any standard search path that the + runtime linker may search by default. + + + .. method:: CCompiler.define_macro(name[, value=None]) + + Define a preprocessor macro for all compilations driven by this compiler object. + The optional parameter *value* should be a string; if it is not supplied, then + the macro will be defined without an explicit value and the exact outcome + depends on the compiler used. + + .. XXX true? does ANSI say anything about this? + + + .. method:: CCompiler.undefine_macro(name) + + Undefine a preprocessor macro for all compilations driven by this compiler + object. If the same macro is defined by :meth:`define_macro` and + undefined by :meth:`undefine_macro` the last call takes precedence + (including multiple redefinitions or undefinitions). If the macro is + redefined/undefined on a per-compilation basis (ie. in the call to + :meth:`compile`), then that takes precedence. + + + .. method:: CCompiler.add_link_object(object) + + Add *object* to the list of object files (or analogues, such as explicitly named + library files or the output of "resource compilers") to be included in every + link driven by this compiler object. + + + .. method:: CCompiler.set_link_objects(objects) + + Set the list of object files (or analogues) to be included in every link to + *objects*. This does not affect any standard object files that the linker may + include by default (such as system libraries). + + The following methods implement methods for autodetection of compiler options, + providing some functionality similar to GNU :program:`autoconf`. + + + .. method:: CCompiler.detect_language(sources) + + Detect the language of a given file, or list of files. Uses the instance + attributes :attr:`~CCompiler.language_map` (a dictionary), and :attr:`~CCompiler.language_order` (a + list) to do the job. + + + .. method:: CCompiler.find_library_file(dirs, lib[, debug=0]) + + Search the specified list of directories for a static or shared library file + *lib* and return the full path to that file. If *debug* is true, look for a + debugging version (if that makes sense on the current platform). Return + ``None`` if *lib* wasn't found in any of the specified directories. + + + .. method:: CCompiler.has_function(funcname [, includes=None, include_dirs=None, libraries=None, library_dirs=None]) + + Return a boolean indicating whether *funcname* is supported on the current + platform. The optional arguments can be used to augment the compilation + environment by providing additional include files and paths and libraries and + paths. + + + .. method:: CCompiler.library_dir_option(dir) + + Return the compiler option to add *dir* to the list of directories searched for + libraries. + + + .. method:: CCompiler.library_option(lib) + + Return the compiler option to add *lib* to the list of libraries linked into the + shared library or executable. + + + .. method:: CCompiler.runtime_library_dir_option(dir) + + Return the compiler option to add *dir* to the list of directories searched for + runtime libraries. + + + .. method:: CCompiler.set_executables(**args) + + Define the executables (and options for them) that will be run to perform the + various stages of compilation. The exact set of executables that may be + specified here depends on the compiler class (via the 'executables' class + attribute), but most will have: + + +--------------+------------------------------------------+ + | attribute | description | + +==============+==========================================+ + | *compiler* | the C/C++ compiler | + +--------------+------------------------------------------+ + | *linker_so* | linker used to create shared objects and | + | | libraries | + +--------------+------------------------------------------+ + | *linker_exe* | linker used to create binary executables | + +--------------+------------------------------------------+ + | *archiver* | static library creator | + +--------------+------------------------------------------+ + + On platforms with a command-line (Unix, DOS/Windows), each of these is a string + that will be split into executable name and (optional) list of arguments. + (Splitting the string is done similarly to how Unix shells operate: words are + delimited by spaces, but quotes and backslashes can override this. See + :func:`distutils.util.split_quoted`.) + + The following methods invoke stages in the build process. + + + .. method:: CCompiler.compile(sources[, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None]) + + Compile one or more source files. Generates object files (e.g. transforms a + :file:`.c` file to a :file:`.o` file.) + + *sources* must be a list of filenames, most likely C/C++ files, but in reality + anything that can be handled by a particular compiler and compiler class (eg. + :class:`MSVCCompiler` can handle resource files in *sources*). Return a list of + object filenames, one per source filename in *sources*. Depending on the + implementation, not all source files will necessarily be compiled, but all + corresponding object filenames will be returned. + + If *output_dir* is given, object files will be put under it, while retaining + their original path component. That is, :file:`foo/bar.c` normally compiles to + :file:`foo/bar.o` (for a Unix implementation); if *output_dir* is *build*, then + it would compile to :file:`build/foo/bar.o`. + + *macros*, if given, must be a list of macro definitions. A macro definition is + either a ``(name, value)`` 2-tuple or a ``(name,)`` 1-tuple. The former defines + a macro; if the value is ``None``, the macro is defined without an explicit + value. The 1-tuple case undefines a macro. Later + definitions/redefinitions/undefinitions take precedence. + + *include_dirs*, if given, must be a list of strings, the directories to add to + the default include file search path for this compilation only. + + *debug* is a boolean; if true, the compiler will be instructed to output debug + symbols in (or alongside) the object file(s). + + *extra_preargs* and *extra_postargs* are implementation-dependent. On platforms + that have the notion of a command-line (e.g. Unix, DOS/Windows), they are most + likely lists of strings: extra command-line arguments to prepend/append to the + compiler command line. On other platforms, consult the implementation class + documentation. In any event, they are intended as an escape hatch for those + occasions when the abstract compiler framework doesn't cut the mustard. + + *depends*, if given, is a list of filenames that all targets depend on. If a + source file is older than any file in depends, then the source file will be + recompiled. This supports dependency tracking, but only at a coarse + granularity. + + Raises :exc:`CompileError` on failure. + + + .. method:: CCompiler.create_static_lib(objects, output_libname[, output_dir=None, debug=0, target_lang=None]) + + Link a bunch of stuff together to create a static library file. The "bunch of + stuff" consists of the list of object files supplied as *objects*, the extra + object files supplied to :meth:`add_link_object` and/or + :meth:`set_link_objects`, the libraries supplied to :meth:`add_library` and/or + :meth:`set_libraries`, and the libraries supplied as *libraries* (if any). + + *output_libname* should be a library name, not a filename; the filename will be + inferred from the library name. *output_dir* is the directory where the library + file will be put. + + .. XXX defaults to what? + + *debug* is a boolean; if true, debugging information will be included in the + library (note that on most platforms, it is the compile step where this matters: + the *debug* flag is included here just for consistency). + + *target_lang* is the target language for which the given objects are being + compiled. This allows specific linkage time treatment of certain languages. + + Raises :exc:`LibError` on failure. + + + .. method:: CCompiler.link(target_desc, objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) + + Link a bunch of stuff together to create an executable or shared library file. + + The "bunch of stuff" consists of the list of object files supplied as *objects*. + *output_filename* should be a filename. If *output_dir* is supplied, + *output_filename* is relative to it (i.e. *output_filename* can provide + directory components if needed). + + *libraries* is a list of libraries to link against. These are library names, + not filenames, since they're translated into filenames in a platform-specific + way (eg. *foo* becomes :file:`libfoo.a` on Unix and :file:`foo.lib` on + DOS/Windows). However, they can include a directory component, which means the + linker will look in that specific directory rather than searching all the normal + locations. + + *library_dirs*, if supplied, should be a list of directories to search for + libraries that were specified as bare library names (ie. no directory + component). These are on top of the system default and those supplied to + :meth:`add_library_dir` and/or :meth:`set_library_dirs`. *runtime_library_dirs* + is a list of directories that will be embedded into the shared library and used + to search for other shared libraries that \*it\* depends on at run-time. (This + may only be relevant on Unix.) + + *export_symbols* is a list of symbols that the shared library will export. + (This appears to be relevant only on Windows.) + + *debug* is as for :meth:`compile` and :meth:`create_static_lib`, with the + slight distinction that it actually matters on most platforms (as opposed to + :meth:`create_static_lib`, which includes a *debug* flag mostly for form's + sake). + + *extra_preargs* and *extra_postargs* are as for :meth:`compile` (except of + course that they supply command-line arguments for the particular linker being + used). + + *target_lang* is the target language for which the given objects are being + compiled. This allows specific linkage time treatment of certain languages. + + Raises :exc:`LinkError` on failure. + + + .. method:: CCompiler.link_executable(objects, output_progname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, target_lang=None]) + + Link an executable. *output_progname* is the name of the file executable, while + *objects* are a list of object filenames to link in. Other arguments are as for + the :meth:`link` method. + + + .. method:: CCompiler.link_shared_lib(objects, output_libname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) + + Link a shared library. *output_libname* is the name of the output library, + while *objects* is a list of object filenames to link in. Other arguments are + as for the :meth:`link` method. + + + .. method:: CCompiler.link_shared_object(objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) + + Link a shared object. *output_filename* is the name of the shared object that + will be created, while *objects* is a list of object filenames to link in. + Other arguments are as for the :meth:`link` method. + + + .. method:: CCompiler.preprocess(source[, output_file=None, macros=None, include_dirs=None, extra_preargs=None, extra_postargs=None]) + + Preprocess a single C/C++ source file, named in *source*. Output will be written + to file named *output_file*, or *stdout* if *output_file* not supplied. + *macros* is a list of macro definitions as for :meth:`compile`, which will + augment the macros set with :meth:`define_macro` and :meth:`undefine_macro`. + *include_dirs* is a list of directory names that will be added to the default + list, in the same way as :meth:`add_include_dir`. + + Raises :exc:`PreprocessError` on failure. + + The following utility methods are defined by the :class:`CCompiler` class, for + use by the various concrete subclasses. + + + .. method:: CCompiler.executable_filename(basename[, strip_dir=0, output_dir='']) + + Returns the filename of the executable for the given *basename*. Typically for + non-Windows platforms this is the same as the basename, while Windows will get + a :file:`.exe` added. + + + .. method:: CCompiler.library_filename(libname[, lib_type='static', strip_dir=0, output_dir='']) + + Returns the filename for the given library name on the current platform. On Unix + a library with *lib_type* of ``'static'`` will typically be of the form + :file:`liblibname.a`, while a *lib_type* of ``'dynamic'`` will be of the form + :file:`liblibname.so`. + + + .. method:: CCompiler.object_filenames(source_filenames[, strip_dir=0, output_dir='']) + + Returns the name of the object files for the given source files. + *source_filenames* should be a list of filenames. + + + .. method:: CCompiler.shared_object_filename(basename[, strip_dir=0, output_dir='']) + + Returns the name of a shared object file for the given file name *basename*. + + + .. method:: CCompiler.execute(func, args[, msg=None, level=1]) + + Invokes :func:`distutils.util.execute`. This method invokes a Python function + *func* with the given arguments *args*, after logging and taking into account + the *dry_run* flag. + + + .. method:: CCompiler.spawn(cmd) + + Invokes :func:`distutils.spawn.spawn`. This invokes an external process to run + the given command. + + + .. method:: CCompiler.mkpath(name[, mode=511]) + + Invokes :func:`distutils.dir_util.mkpath`. This creates a directory and any + missing ancestor directories. + + + .. method:: CCompiler.move_file(src, dst) + + Invokes :meth:`distutils.file_util.move_file`. Renames *src* to *dst*. + + + .. method:: CCompiler.announce(msg[, level=1]) + + Write a message using :func:`distutils.log.debug`. + + + .. method:: CCompiler.warn(msg) + + Write a warning message *msg* to standard error. + + + .. method:: CCompiler.debug_print(msg) + + If the *debug* flag is set on this :class:`CCompiler` instance, print *msg* to + standard output, otherwise do nothing. + +.. % \subsection{Compiler-specific modules} +.. % +.. % The following modules implement concrete subclasses of the abstract +.. % \class{CCompiler} class. They should not be instantiated directly, but should +.. % be created using \function{distutils.ccompiler.new_compiler()} factory +.. % function. + + +:mod:`distutils.unixccompiler` --- Unix C Compiler +================================================== + +.. module:: distutils.unixccompiler + :synopsis: UNIX C Compiler + + +This module provides the :class:`UnixCCompiler` class, a subclass of +:class:`CCompiler` that handles the typical Unix-style command-line C compiler: + +* macros defined with :option:`!-Dname[=value]` + +* macros undefined with :option:`!-Uname` + +* include search directories specified with :option:`!-Idir` + +* libraries specified with :option:`!-llib` + +* library search directories specified with :option:`!-Ldir` + +* compile handled by :program:`cc` (or similar) executable with :option:`!-c` + option: compiles :file:`.c` to :file:`.o` + +* link static library handled by :program:`ar` command (possibly with + :program:`ranlib`) + +* link shared library handled by :program:`cc` :option:`!-shared` + + +:mod:`distutils.msvccompiler` --- Microsoft Compiler +==================================================== + +.. module:: distutils.msvccompiler + :synopsis: Microsoft Compiler + +.. XXX: This is *waaaaay* out of date! + +This module provides :class:`MSVCCompiler`, an implementation of the abstract +:class:`CCompiler` class for Microsoft Visual Studio. Typically, extension +modules need to be compiled with the same compiler that was used to compile +Python. For Python 2.3 and earlier, the compiler was Visual Studio 6. For Python +2.4 and 2.5, the compiler is Visual Studio .NET 2003. + +:class:`MSVCCompiler` will normally choose the right compiler, linker etc. on +its own. To override this choice, the environment variables *DISTUTILS_USE_SDK* +and *MSSdk* must be both set. *MSSdk* indicates that the current environment has +been setup by the SDK's ``SetEnv.Cmd`` script, or that the environment variables +had been registered when the SDK was installed; *DISTUTILS_USE_SDK* indicates +that the distutils user has made an explicit choice to override the compiler +selection by :class:`MSVCCompiler`. + + +:mod:`distutils.bcppcompiler` --- Borland Compiler +================================================== + +.. module:: distutils.bcppcompiler + + +This module provides :class:`BorlandCCompiler`, a subclass of the abstract +:class:`CCompiler` class for the Borland C++ compiler. + + +:mod:`distutils.cygwinccompiler` --- Cygwin Compiler +==================================================== + +.. module:: distutils.cygwinccompiler + + +This module provides the :class:`CygwinCCompiler` class, a subclass of +:class:`UnixCCompiler` that handles the Cygwin port of the GNU C compiler to +Windows. It also contains the Mingw32CCompiler class which handles the mingw32 +port of GCC (same as cygwin in no-cygwin mode). + + +:mod:`distutils.archive_util` --- Archiving utilities +====================================================== + +.. module:: distutils.archive_util + :synopsis: Utility functions for creating archive files (tarballs, zip files, ...) + + +This module provides a few functions for creating archive files, such as +tarballs or zipfiles. + + +.. function:: make_archive(base_name, format[, root_dir=None, base_dir=None, verbose=0, dry_run=0]) + + Create an archive file (eg. ``zip`` or ``tar``). *base_name* is the name of + the file to create, minus any format-specific extension; *format* is the + archive format: one of ``zip``, ``tar``, ``gztar``, ``bztar``, ``xztar``, or + ``ztar``. *root_dir* is a directory that will be the root directory of the + archive; ie. we typically ``chdir`` into *root_dir* before creating the + archive. *base_dir* is the directory where we start archiving from; ie. + *base_dir* will be the common prefix of all files and directories in the + archive. *root_dir* and *base_dir* both default to the current directory. + Returns the name of the archive file. + + .. versionchanged:: 3.5 + Added support for the ``xztar`` format. + + +.. function:: make_tarball(base_name, base_dir[, compress='gzip', verbose=0, dry_run=0]) + + 'Create an (optional compressed) archive as a tar file from all files in and + under *base_dir*. *compress* must be ``'gzip'`` (the default), + ``'bzip2'``, ``'xz'``, ``'compress'``, or ``None``. For the ``'compress'`` + method the compression utility named by :program:`compress` must be on the + default program search path, so this is probably Unix-specific. The output + tar file will be named :file:`base_dir.tar`, possibly plus the appropriate + compression extension (``.gz``, ``.bz2``, ``.xz`` or ``.Z``). Return the + output filename. + + .. versionchanged:: 3.5 + Added support for the ``xz`` compression. + + +.. function:: make_zipfile(base_name, base_dir[, verbose=0, dry_run=0]) + + Create a zip file from all files in and under *base_dir*. The output zip file + will be named *base_name* + :file:`.zip`. Uses either the :mod:`zipfile` Python + module (if available) or the InfoZIP :file:`zip` utility (if installed and + found on the default search path). If neither tool is available, raises + :exc:`DistutilsExecError`. Returns the name of the output zip file. + + +:mod:`distutils.dep_util` --- Dependency checking +================================================= + +.. module:: distutils.dep_util + :synopsis: Utility functions for simple dependency checking + + +This module provides functions for performing simple, timestamp-based +dependency of files and groups of files; also, functions based entirely on such +timestamp dependency analysis. + + +.. function:: newer(source, target) + + Return true if *source* exists and is more recently modified than *target*, or + if *source* exists and *target* doesn't. Return false if both exist and *target* + is the same age or newer than *source*. Raise :exc:`DistutilsFileError` if + *source* does not exist. + + +.. function:: newer_pairwise(sources, targets) + + Walk two filename lists in parallel, testing if each source is newer than its + corresponding target. Return a pair of lists (*sources*, *targets*) where + source is newer than target, according to the semantics of :func:`newer`. + + .. % % equivalent to a listcomp... + + +.. function:: newer_group(sources, target[, missing='error']) + + Return true if *target* is out-of-date with respect to any file listed in + *sources*. In other words, if *target* exists and is newer than every file in + *sources*, return false; otherwise return true. *missing* controls what we do + when a source file is missing; the default (``'error'``) is to blow up with an + :exc:`OSError` from inside :func:`os.stat`; if it is ``'ignore'``, we silently + drop any missing source files; if it is ``'newer'``, any missing source files + make us assume that *target* is out-of-date (this is handy in "dry-run" mode: + it'll make you pretend to carry out commands that wouldn't work because inputs + are missing, but that doesn't matter because you're not actually going to run + the commands). + + +:mod:`distutils.dir_util` --- Directory tree operations +======================================================= + +.. module:: distutils.dir_util + :synopsis: Utility functions for operating on directories and directory trees + + +This module provides functions for operating on directories and trees of +directories. + + +.. function:: mkpath(name[, mode=0o777, verbose=0, dry_run=0]) + + Create a directory and any missing ancestor directories. If the directory + already exists (or if *name* is the empty string, which means the current + directory, which of course exists), then do nothing. Raise + :exc:`DistutilsFileError` if unable to create some directory along the way (eg. + some sub-path exists, but is a file rather than a directory). If *verbose* is + true, print a one-line summary of each mkdir to stdout. Return the list of + directories actually created. + + +.. function:: create_tree(base_dir, files[, mode=0o777, verbose=0, dry_run=0]) + + Create all the empty directories under *base_dir* needed to put *files* there. + *base_dir* is just the name of a directory which doesn't necessarily exist + yet; *files* is a list of filenames to be interpreted relative to *base_dir*. + *base_dir* + the directory portion of every file in *files* will be created if + it doesn't already exist. *mode*, *verbose* and *dry_run* flags are as for + :func:`mkpath`. + + +.. function:: copy_tree(src, dst[, preserve_mode=1, preserve_times=1, preserve_symlinks=0, update=0, verbose=0, dry_run=0]) + + Copy an entire directory tree *src* to a new location *dst*. Both *src* and + *dst* must be directory names. If *src* is not a directory, raise + :exc:`DistutilsFileError`. If *dst* does not exist, it is created with + :func:`mkpath`. The end result of the copy is that every file in *src* is + copied to *dst*, and directories under *src* are recursively copied to *dst*. + Return the list of files that were copied or might have been copied, using their + output name. The return value is unaffected by *update* or *dry_run*: it is + simply the list of all files under *src*, with the names changed to be under + *dst*. + + *preserve_mode* and *preserve_times* are the same as for + :func:`distutils.file_util.copy_file`; note that they only apply to + regular files, not to + directories. If *preserve_symlinks* is true, symlinks will be copied as + symlinks (on platforms that support them!); otherwise (the default), the + destination of the symlink will be copied. *update* and *verbose* are the same + as for :func:`~distutils.file_util.copy_file`. + + Files in *src* that begin with :file:`.nfs` are skipped (more information on + these files is available in answer D2 of the `NFS FAQ page + <http://nfs.sourceforge.net/#section_d>`_). + + .. versionchanged:: 3.3.1 + NFS files are ignored. + +.. function:: remove_tree(directory[, verbose=0, dry_run=0]) + + Recursively remove *directory* and all files and directories underneath it. Any + errors are ignored (apart from being reported to ``sys.stdout`` if *verbose* is + true). + + +:mod:`distutils.file_util` --- Single file operations +===================================================== + +.. module:: distutils.file_util + :synopsis: Utility functions for operating on single files + + +This module contains some utility functions for operating on individual files. + + +.. function:: copy_file(src, dst[, preserve_mode=1, preserve_times=1, update=0, link=None, verbose=0, dry_run=0]) + + Copy file *src* to *dst*. If *dst* is a directory, then *src* is copied there + with the same name; otherwise, it must be a filename. (If the file exists, it + will be ruthlessly clobbered.) If *preserve_mode* is true (the default), the + file's mode (type and permission bits, or whatever is analogous on the + current platform) is copied. If *preserve_times* is true (the default), the + last-modified and last-access times are copied as well. If *update* is true, + *src* will only be copied if *dst* does not exist, or if *dst* does exist but + is older than *src*. + + *link* allows you to make hard links (using :func:`os.link`) or symbolic links + (using :func:`os.symlink`) instead of copying: set it to ``'hard'`` or + ``'sym'``; if it is ``None`` (the default), files are copied. Don't set *link* + on systems that don't support it: :func:`copy_file` doesn't check if hard or + symbolic linking is available. It uses :func:`~distutils.file_util._copy_file_contents` to copy file + contents. + + Return a tuple ``(dest_name, copied)``: *dest_name* is the actual name of the + output file, and *copied* is true if the file was copied (or would have been + copied, if *dry_run* true). + + .. % XXX if the destination file already exists, we clobber it if + .. % copying, but blow up if linking. Hmmm. And I don't know what + .. % macostools.copyfile() does. Should definitely be consistent, and + .. % should probably blow up if destination exists and we would be + .. % changing it (ie. it's not already a hard/soft link to src OR + .. % (not update) and (src newer than dst)). + + +.. function:: move_file(src, dst[, verbose, dry_run]) + + Move file *src* to *dst*. If *dst* is a directory, the file will be moved into + it with the same name; otherwise, *src* is just renamed to *dst*. Returns the + new full name of the file. + + .. warning:: + + Handles cross-device moves on Unix using :func:`copy_file`. What about + other systems? + + +.. function:: write_file(filename, contents) + + Create a file called *filename* and write *contents* (a sequence of strings + without line terminators) to it. + + +:mod:`distutils.util` --- Miscellaneous other utility functions +=============================================================== + +.. module:: distutils.util + :synopsis: Miscellaneous other utility functions + + +This module contains other assorted bits and pieces that don't fit into any +other utility module. + + +.. function:: get_platform() + + Return a string that identifies the current platform. This is used mainly to + distinguish platform-specific build directories and platform-specific built + distributions. Typically includes the OS name and version and the + architecture (as supplied by 'os.uname()'), although the exact information + included depends on the OS; e.g., on Linux, the kernel version isn't + particularly important. + + Examples of returned values: + + * ``linux-i586`` + * ``linux-alpha`` + * ``solaris-2.6-sun4u`` + + For non-POSIX platforms, currently just returns ``sys.platform``. + + For Mac OS X systems the OS version reflects the minimal version on which + binaries will run (that is, the value of ``MACOSX_DEPLOYMENT_TARGET`` + during the build of Python), not the OS version of the current system. + + For universal binary builds on Mac OS X the architecture value reflects + the universal binary status instead of the architecture of the current + processor. For 32-bit universal binaries the architecture is ``fat``, + for 64-bit universal binaries the architecture is ``fat64``, and + for 4-way universal binaries the architecture is ``universal``. Starting + from Python 2.7 and Python 3.2 the architecture ``fat3`` is used for + a 3-way universal build (ppc, i386, x86_64) and ``intel`` is used for + a universal build with the i386 and x86_64 architectures + + Examples of returned values on Mac OS X: + + * ``macosx-10.3-ppc`` + + * ``macosx-10.3-fat`` + + * ``macosx-10.5-universal`` + + * ``macosx-10.6-intel`` + + For AIX, Python 3.9 and later return a string starting with "aix", followed + by additional fields (separated by ``'-'``) that represent the combined + values of AIX Version, Release and Technology Level (first field), Build Date + (second field), and bit-size (third field). Python 3.8 and earlier returned + only a single additional field with the AIX Version and Release. + + Examples of returned values on AIX: + + * ``aix-5307-0747-32`` # 32-bit build on AIX ``oslevel -s``: 5300-07-00-0000 + + * ``aix-7105-1731-64`` # 64-bit build on AIX ``oslevel -s``: 7100-05-01-1731 + + * ``aix-7.2`` # Legacy form reported in Python 3.8 and earlier + + .. versionchanged:: 3.9 + The AIX platform string format now also includes the technology level, + build date, and ABI bit-size. + + +.. function:: convert_path(pathname) + + Return 'pathname' as a name that will work on the native filesystem, i.e. split + it on '/' and put it back together again using the current directory separator. + Needed because filenames in the setup script are always supplied in Unix style, + and have to be converted to the local convention before we can actually use them + in the filesystem. Raises :exc:`ValueError` on non-Unix-ish systems if + *pathname* either starts or ends with a slash. + + +.. function:: change_root(new_root, pathname) + + Return *pathname* with *new_root* prepended. If *pathname* is relative, this is + equivalent to ``os.path.join(new_root,pathname)`` Otherwise, it requires making + *pathname* relative and then joining the two, which is tricky on DOS/Windows. + + +.. function:: check_environ() + + Ensure that 'os.environ' has all the environment variables we guarantee that + users can use in config files, command-line options, etc. Currently this + includes: + + * :envvar:`HOME` - user's home directory (Unix only) + * :envvar:`PLAT` - description of the current platform, including hardware and + OS (see :func:`get_platform`) + + +.. function:: subst_vars(s, local_vars) + + Perform shell/Perl-style variable substitution on *s*. Every occurrence of + ``$`` followed by a name is considered a variable, and variable is substituted + by the value found in the *local_vars* dictionary, or in ``os.environ`` if it's + not in *local_vars*. *os.environ* is first checked/augmented to guarantee that + it contains certain values: see :func:`check_environ`. Raise :exc:`ValueError` + for any variables not found in either *local_vars* or ``os.environ``. + + Note that this is not a fully-fledged string interpolation function. A valid + ``$variable`` can consist only of upper and lower case letters, numbers and an + underscore. No { } or ( ) style quoting is available. + + +.. function:: split_quoted(s) + + Split a string up according to Unix shell-like rules for quotes and backslashes. + In short: words are delimited by spaces, as long as those spaces are not escaped + by a backslash, or inside a quoted string. Single and double quotes are + equivalent, and the quote characters can be backslash-escaped. The backslash is + stripped from any two-character escape sequence, leaving only the escaped + character. The quote characters are stripped from any quoted string. Returns a + list of words. + + .. % Should probably be moved into the standard library. + + +.. function:: execute(func, args[, msg=None, verbose=0, dry_run=0]) + + Perform some action that affects the outside world (for instance, writing to the + filesystem). Such actions are special because they are disabled by the + *dry_run* flag. This method takes care of all that bureaucracy for you; all + you have to do is supply the function to call and an argument tuple for it (to + embody the "external action" being performed), and an optional message to print. + + +.. function:: strtobool(val) + + Convert a string representation of truth to true (1) or false (0). + + True values are ``y``, ``yes``, ``t``, ``true``, ``on`` and ``1``; false values + are ``n``, ``no``, ``f``, ``false``, ``off`` and ``0``. Raises + :exc:`ValueError` if *val* is anything else. + + +.. function:: byte_compile(py_files[, optimize=0, force=0, prefix=None, base_dir=None, verbose=1, dry_run=0, direct=None]) + + Byte-compile a collection of Python source files to :file:`.pyc` files in a + :file:`__pycache__` subdirectory (see :pep:`3147` and :pep:`488`). + *py_files* is a list of files to compile; any files that don't end in + :file:`.py` are silently skipped. *optimize* must be one of the following: + + * ``0`` - don't optimize + * ``1`` - normal optimization (like ``python -O``) + * ``2`` - extra optimization (like ``python -OO``) + + If *force* is true, all files are recompiled regardless of timestamps. + + The source filename encoded in each :term:`bytecode` file defaults to the filenames + listed in *py_files*; you can modify these with *prefix* and *basedir*. + *prefix* is a string that will be stripped off of each source filename, and + *base_dir* is a directory name that will be prepended (after *prefix* is + stripped). You can supply either or both (or neither) of *prefix* and + *base_dir*, as you wish. + + If *dry_run* is true, doesn't actually do anything that would affect the + filesystem. + + Byte-compilation is either done directly in this interpreter process with the + standard :mod:`py_compile` module, or indirectly by writing a temporary script + and executing it. Normally, you should let :func:`byte_compile` figure out to + use direct compilation or not (see the source for details). The *direct* flag + is used by the script generated in indirect mode; unless you know what you're + doing, leave it set to ``None``. + + .. versionchanged:: 3.2.3 + Create ``.pyc`` files with an :func:`import magic tag + <imp.get_tag>` in their name, in a :file:`__pycache__` subdirectory + instead of files without tag in the current directory. + + .. versionchanged:: 3.5 + Create ``.pyc`` files according to :pep:`488`. + + +.. function:: rfc822_escape(header) + + Return a version of *header* escaped for inclusion in an :rfc:`822` header, by + ensuring there are 8 spaces space after each newline. Note that it does no other + modification of the string. + + .. % this _can_ be replaced + +.. % \subsection{Distutils objects} + + +:mod:`distutils.dist` --- The Distribution class +================================================ + +.. module:: distutils.dist + :synopsis: Provides the Distribution class, which represents the module distribution being + built/installed/distributed + + +This module provides the :class:`~distutils.core.Distribution` class, which +represents the module distribution being built/installed/distributed. + + +:mod:`distutils.extension` --- The Extension class +================================================== + +.. module:: distutils.extension + :synopsis: Provides the Extension class, used to describe C/C++ extension modules in setup + scripts + + +This module provides the :class:`~distutils.extension.Extension` class, +used to describe C/C++ extension modules in setup scripts. + +.. % \subsection{Ungrouped modules} +.. % The following haven't been moved into a more appropriate section yet. + + +:mod:`distutils.debug` --- Distutils debug mode +=============================================== + +.. module:: distutils.debug + :synopsis: Provides the debug flag for distutils + + +This module provides the DEBUG flag. + + +:mod:`distutils.errors` --- Distutils exceptions +================================================ + +.. module:: distutils.errors + :synopsis: Provides standard distutils exceptions + + +Provides exceptions used by the Distutils modules. Note that Distutils modules +may raise standard exceptions; in particular, SystemExit is usually raised for +errors that are obviously the end-user's fault (eg. bad command-line arguments). + +This module is safe to use in ``from ... import *`` mode; it only exports +symbols whose names start with ``Distutils`` and end with ``Error``. + + +:mod:`distutils.fancy_getopt` --- Wrapper around the standard getopt module +=========================================================================== + +.. module:: distutils.fancy_getopt + :synopsis: Additional getopt functionality + + +This module provides a wrapper around the standard :mod:`getopt` module that +provides the following additional features: + +* short and long options are tied together + +* options have help strings, so :func:`fancy_getopt` could potentially create a + complete usage summary + +* options set attributes of a passed-in object + +* boolean options can have "negative aliases" --- eg. if :option:`!--quiet` is + the "negative alias" of :option:`!--verbose`, then :option:`!--quiet` on the + command line sets *verbose* to false. + +.. function:: fancy_getopt(options, negative_opt, object, args) + + Wrapper function. *options* is a list of ``(long_option, short_option, + help_string)`` 3-tuples as described in the constructor for + :class:`FancyGetopt`. *negative_opt* should be a dictionary mapping option names + to option names, both the key and value should be in the *options* list. + *object* is an object which will be used to store values (see the :meth:`~FancyGetopt.getopt` + method of the :class:`FancyGetopt` class). *args* is the argument list. Will use + ``sys.argv[1:]`` if you pass ``None`` as *args*. + + +.. function:: wrap_text(text, width) + + Wraps *text* to less than *width* wide. + + +.. class:: FancyGetopt([option_table=None]) + + The option_table is a list of 3-tuples: ``(long_option, short_option, + help_string)`` + + If an option takes an argument, its *long_option* should have ``'='`` appended; + *short_option* should just be a single character, no ``':'`` in any case. + *short_option* should be ``None`` if a *long_option* doesn't have a + corresponding *short_option*. All option tuples must have long options. + +The :class:`FancyGetopt` class provides the following methods: + + +.. method:: FancyGetopt.getopt([args=None, object=None]) + + Parse command-line options in args. Store as attributes on *object*. + + If *args* is ``None`` or not supplied, uses ``sys.argv[1:]``. If *object* is + ``None`` or not supplied, creates a new :class:`OptionDummy` instance, stores + option values there, and returns a tuple ``(args, object)``. If *object* is + supplied, it is modified in place and :func:`getopt` just returns *args*; in + both cases, the returned *args* is a modified copy of the passed-in *args* list, + which is left untouched. + + .. % and args returned are? + + +.. method:: FancyGetopt.get_option_order() + + Returns the list of ``(option, value)`` tuples processed by the previous run of + :meth:`getopt` Raises :exc:`RuntimeError` if :meth:`getopt` hasn't been called + yet. + + +.. method:: FancyGetopt.generate_help([header=None]) + + Generate help text (a list of strings, one per suggested line of output) from + the option table for this :class:`FancyGetopt` object. + + If supplied, prints the supplied *header* at the top of the help. + + +:mod:`distutils.filelist` --- The FileList class +================================================ + +.. module:: distutils.filelist + :synopsis: The FileList class, used for poking about the file system and + building lists of files. + + +This module provides the :class:`FileList` class, used for poking about the +filesystem and building lists of files. + + +:mod:`distutils.log` --- Simple :pep:`282`-style logging +======================================================== + +.. module:: distutils.log + :synopsis: A simple logging mechanism, :pep:`282`-style + + +:mod:`distutils.spawn` --- Spawn a sub-process +============================================== + +.. module:: distutils.spawn + :synopsis: Provides the spawn() function + + +This module provides the :func:`~distutils.spawn.spawn` function, a +front-end to various platform-specific functions for launching another +program in a sub-process. +Also provides :func:`~distutils.spawn.find_executable` to search the path for a given executable +name. + + +:mod:`distutils.sysconfig` --- System configuration information +=============================================================== + +.. module:: distutils.sysconfig + :synopsis: Low-level access to configuration information of the Python interpreter. +.. moduleauthor:: Fred L. Drake, Jr. <fdrake@acm.org> +.. moduleauthor:: Greg Ward <gward@python.net> +.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org> + + +The :mod:`distutils.sysconfig` module provides access to Python's low-level +configuration information. The specific configuration variables available +depend heavily on the platform and configuration. The specific variables depend +on the build process for the specific version of Python being run; the variables +are those found in the :file:`Makefile` and configuration header that are +installed with Python on Unix systems. The configuration header is called +:file:`pyconfig.h` for Python versions starting with 2.2, and :file:`config.h` +for earlier versions of Python. + +Some additional functions are provided which perform some useful manipulations +for other parts of the :mod:`distutils` package. + + +.. data:: PREFIX + + The result of ``os.path.normpath(sys.prefix)``. + + +.. data:: EXEC_PREFIX + + The result of ``os.path.normpath(sys.exec_prefix)``. + + +.. function:: get_config_var(name) + + Return the value of a single variable. This is equivalent to + ``get_config_vars().get(name)``. + + +.. function:: get_config_vars(...) + + Return a set of variable definitions. If there are no arguments, this returns a + dictionary mapping names of configuration variables to values. If arguments are + provided, they should be strings, and the return value will be a sequence giving + the associated values. If a given name does not have a corresponding value, + ``None`` will be included for that variable. + + +.. function:: get_config_h_filename() + + Return the full path name of the configuration header. For Unix, this will be + the header generated by the :program:`configure` script; for other platforms the + header will have been supplied directly by the Python source distribution. The + file is a platform-specific text file. + + +.. function:: get_makefile_filename() + + Return the full path name of the :file:`Makefile` used to build Python. For + Unix, this will be a file generated by the :program:`configure` script; the + meaning for other platforms will vary. The file is a platform-specific text + file, if it exists. This function is only useful on POSIX platforms. + + +.. function:: get_python_inc([plat_specific[, prefix]]) + + Return the directory for either the general or platform-dependent C include + files. If *plat_specific* is true, the platform-dependent include directory is + returned; if false or omitted, the platform-independent directory is returned. + If *prefix* is given, it is used as either the prefix instead of + :const:`PREFIX`, or as the exec-prefix instead of :const:`EXEC_PREFIX` if + *plat_specific* is true. + + +.. function:: get_python_lib([plat_specific[, standard_lib[, prefix]]]) + + Return the directory for either the general or platform-dependent library + installation. If *plat_specific* is true, the platform-dependent include + directory is returned; if false or omitted, the platform-independent directory + is returned. If *prefix* is given, it is used as either the prefix instead of + :const:`PREFIX`, or as the exec-prefix instead of :const:`EXEC_PREFIX` if + *plat_specific* is true. If *standard_lib* is true, the directory for the + standard library is returned rather than the directory for the installation of + third-party extensions. + +The following function is only intended for use within the :mod:`distutils` +package. + + +.. function:: customize_compiler(compiler) + + Do any platform-specific customization of a + :class:`distutils.ccompiler.CCompiler` instance. + + This function is only needed on Unix at this time, but should be called + consistently to support forward-compatibility. It inserts the information that + varies across Unix flavors and is stored in Python's :file:`Makefile`. This + information includes the selected compiler, compiler and linker options, and the + extension used by the linker for shared objects. + +This function is even more special-purpose, and should only be used from +Python's own build procedures. + + +.. function:: set_python_build() + + Inform the :mod:`distutils.sysconfig` module that it is being used as part of + the build process for Python. This changes a lot of relative locations for + files, allowing them to be located in the build area rather than in an installed + Python. + + +:mod:`distutils.text_file` --- The TextFile class +================================================= + +.. module:: distutils.text_file + :synopsis: Provides the TextFile class, a simple interface to text files + + +This module provides the :class:`TextFile` class, which gives an interface to +text files that (optionally) takes care of stripping comments, ignoring blank +lines, and joining lines with backslashes. + + +.. class:: TextFile([filename=None, file=None, **options]) + + This class provides a file-like object that takes care of all the things you + commonly want to do when processing a text file that has some line-by-line + syntax: strip comments (as long as ``#`` is your comment character), skip blank + lines, join adjacent lines by escaping the newline (ie. backslash at end of + line), strip leading and/or trailing whitespace. All of these are optional and + independently controllable. + + The class provides a :meth:`warn` method so you can generate warning messages + that report physical line number, even if the logical line in question spans + multiple physical lines. Also provides :meth:`unreadline` for implementing + line-at-a-time lookahead. + + :class:`TextFile` instances are create with either *filename*, *file*, or both. + :exc:`RuntimeError` is raised if both are ``None``. *filename* should be a + string, and *file* a file object (or something that provides :meth:`readline` + and :meth:`close` methods). It is recommended that you supply at least + *filename*, so that :class:`TextFile` can include it in warning messages. If + *file* is not supplied, :class:`TextFile` creates its own using the + :func:`open` built-in function. + + The options are all boolean, and affect the values returned by :meth:`readline` + + .. tabularcolumns:: |l|L|l| + + +------------------+--------------------------------+---------+ + | option name | description | default | + +==================+================================+=========+ + | *strip_comments* | strip from ``'#'`` to | true | + | | end-of-line, as well as any | | + | | whitespace leading up to the | | + | | ``'#'``\ ---unless it is | | + | | escaped by a backslash | | + +------------------+--------------------------------+---------+ + | *lstrip_ws* | strip leading whitespace from | false | + | | each line before returning it | | + +------------------+--------------------------------+---------+ + | *rstrip_ws* | strip trailing whitespace | true | + | | (including line terminator!) | | + | | from each line before | | + | | returning it. | | + +------------------+--------------------------------+---------+ + | *skip_blanks* | skip lines that are empty | true | + | | \*after\* stripping comments | | + | | and whitespace. (If both | | + | | lstrip_ws and rstrip_ws are | | + | | false, then some lines may | | + | | consist of solely whitespace: | | + | | these will \*not\* be skipped, | | + | | even if *skip_blanks* is | | + | | true.) | | + +------------------+--------------------------------+---------+ + | *join_lines* | if a backslash is the last | false | + | | non-newline character on a | | + | | line after stripping comments | | + | | and whitespace, join the | | + | | following line to it to form | | + | | one logical line; if N | | + | | consecutive lines end with a | | + | | backslash, then N+1 physical | | + | | lines will be joined to form | | + | | one logical line. | | + +------------------+--------------------------------+---------+ + | *collapse_join* | strip leading whitespace from | false | + | | lines that are joined to their | | + | | predecessor; only matters if | | + | | ``(join_lines and not | | + | | lstrip_ws)`` | | + +------------------+--------------------------------+---------+ + + Note that since *rstrip_ws* can strip the trailing newline, the semantics of + :meth:`readline` must differ from those of the built-in file object's + :meth:`readline` method! In particular, :meth:`readline` returns ``None`` for + end-of-file: an empty string might just be a blank line (or an all-whitespace + line), if *rstrip_ws* is true but *skip_blanks* is not. + + + .. method:: TextFile.open(filename) + + Open a new file *filename*. This overrides any *file* or *filename* + constructor arguments. + + + .. method:: TextFile.close() + + Close the current file and forget everything we know about it (including the + filename and the current line number). + + + .. method:: TextFile.warn(msg[,line=None]) + + Print (to stderr) a warning message tied to the current logical line in the + current file. If the current logical line in the file spans multiple physical + lines, the warning refers to the whole range, such as ``"lines 3-5"``. If + *line* is supplied, it overrides the current line number; it may be a list or + tuple to indicate a range of physical lines, or an integer for a single + physical line. + + + .. method:: TextFile.readline() + + Read and return a single logical line from the current file (or from an internal + buffer if lines have previously been "unread" with :meth:`unreadline`). If the + *join_lines* option is true, this may involve reading multiple physical lines + concatenated into a single string. Updates the current line number, so calling + :meth:`warn` after :meth:`readline` emits a warning about the physical line(s) + just read. Returns ``None`` on end-of-file, since the empty string can occur + if *rstrip_ws* is true but *strip_blanks* is not. + + + .. method:: TextFile.readlines() + + Read and return the list of all logical lines remaining in the current file. + This updates the current line number to the last line of the file. + + + .. method:: TextFile.unreadline(line) + + Push *line* (a string) onto an internal buffer that will be checked by future + :meth:`readline` calls. Handy for implementing a parser with line-at-a-time + lookahead. Note that lines that are "unread" with :meth:`unreadline` are not + subsequently re-cleansed (whitespace stripped, or whatever) when read with + :meth:`readline`. If multiple calls are made to :meth:`unreadline` before a call + to :meth:`readline`, the lines will be returned most in most recent first order. + + +:mod:`distutils.version` --- Version number classes +=================================================== + +.. module:: distutils.version + :synopsis: Implements classes that represent module version numbers. + + +.. % todo +.. % \section{Distutils Commands} +.. % +.. % This part of Distutils implements the various Distutils commands, such +.. % as \code{build}, \code{install} \&c. Each command is implemented as a +.. % separate module, with the command name as the name of the module. + + +:mod:`distutils.cmd` --- Abstract base class for Distutils commands +=================================================================== + +.. module:: distutils.cmd + :synopsis: Provides the abstract base class :class:`~distutils.cmd.Command`. This class + is subclassed by the modules in the distutils.command subpackage. + + +This module supplies the abstract base class :class:`Command`. + + +.. class:: Command(dist) + + Abstract base class for defining command classes, the "worker bees" of the + Distutils. A useful analogy for command classes is to think of them as + subroutines with local variables called *options*. The options are declared + in :meth:`initialize_options` and defined (given their final values) in + :meth:`finalize_options`, both of which must be defined by every command + class. The distinction between the two is necessary because option values + might come from the outside world (command line, config file, ...), and any + options dependent on other options must be computed after these outside + influences have been processed --- hence :meth:`finalize_options`. The body + of the subroutine, where it does all its work based on the values of its + options, is the :meth:`run` method, which must also be implemented by every + command class. + + The class constructor takes a single argument *dist*, a + :class:`~distutils.core.Distribution` instance. + + +Creating a new Distutils command +================================ + +This section outlines the steps to create a new Distutils command. + +A new command lives in a module in the :mod:`distutils.command` package. There +is a sample template in that directory called :file:`command_template`. Copy +this file to a new module with the same name as the new command you're +implementing. This module should implement a class with the same name as the +module (and the command). So, for instance, to create the command +``peel_banana`` (so that users can run ``setup.py peel_banana``), you'd copy +:file:`command_template` to :file:`distutils/command/peel_banana.py`, then edit +it so that it's implementing the class ``peel_banana``, a subclass of +:class:`distutils.cmd.Command`. + +Subclasses of :class:`Command` must define the following methods. + +.. method:: Command.initialize_options() + + Set default values for all the options that this command supports. Note that + these defaults may be overridden by other commands, by the setup script, by + config files, or by the command-line. Thus, this is not the place to code + dependencies between options; generally, :meth:`initialize_options` + implementations are just a bunch of ``self.foo = None`` assignments. + + +.. method:: Command.finalize_options() + + Set final values for all the options that this command supports. This is + always called as late as possible, ie. after any option assignments from the + command-line or from other commands have been done. Thus, this is the place + to code option dependencies: if *foo* depends on *bar*, then it is safe to + set *foo* from *bar* as long as *foo* still has the same value it was + assigned in :meth:`initialize_options`. + + +.. method:: Command.run() + + A command's raison d'etre: carry out the action it exists to perform, controlled + by the options initialized in :meth:`initialize_options`, customized by other + commands, the setup script, the command-line, and config files, and finalized in + :meth:`finalize_options`. All terminal output and filesystem interaction should + be done by :meth:`run`. + + +.. attribute:: Command.sub_commands + + *sub_commands* formalizes the notion of a "family" of commands, + e.g. ``install`` as the parent with sub-commands ``install_lib``, + ``install_headers``, etc. The parent of a family of commands defines + *sub_commands* as a class attribute; it's a list of 2-tuples ``(command_name, + predicate)``, with *command_name* a string and *predicate* a function, a + string or ``None``. *predicate* is a method of the parent command that + determines whether the corresponding command is applicable in the current + situation. (E.g. ``install_headers`` is only applicable if we have any C + header files to install.) If *predicate* is ``None``, that command is always + applicable. + + *sub_commands* is usually defined at the *end* of a class, because + predicates can be methods of the class, so they must already have been + defined. The canonical example is the :command:`install` command. + + +:mod:`distutils.command` --- Individual Distutils commands +========================================================== + +.. module:: distutils.command + :synopsis: Contains one module for each standard Distutils command. + + +.. % \subsubsection{Individual Distutils commands} +.. % todo + + +:mod:`distutils.command.bdist` --- Build a binary installer +=========================================================== + +.. module:: distutils.command.bdist + :synopsis: Build a binary installer for a package + + +.. % todo + + +:mod:`distutils.command.bdist_packager` --- Abstract base class for packagers +============================================================================= + +.. module:: distutils.command.bdist_packager + :synopsis: Abstract base class for packagers + + +.. % todo + + +:mod:`distutils.command.bdist_dumb` --- Build a "dumb" installer +================================================================ + +.. module:: distutils.command.bdist_dumb + :synopsis: Build a "dumb" installer - a simple archive of files + + +.. % todo + + +:mod:`distutils.command.bdist_msi` --- Build a Microsoft Installer binary package +================================================================================= + +.. module:: distutils.command.bdist_msi + :synopsis: Build a binary distribution as a Windows MSI file + +.. class:: bdist_msi + +.. deprecated:: 3.9 + Use bdist_wheel (wheel packages) instead. + + Builds a `Windows Installer`_ (.msi) binary package. + + .. _Windows Installer: https://msdn.microsoft.com/en-us/library/cc185688(VS.85).aspx + + In most cases, the ``bdist_msi`` installer is a better choice than the + ``bdist_wininst`` installer, because it provides better support for + Win64 platforms, allows administrators to perform non-interactive + installations, and allows installation through group policies. + + +:mod:`distutils.command.bdist_rpm` --- Build a binary distribution as a Redhat RPM and SRPM +=========================================================================================== + +.. module:: distutils.command.bdist_rpm + :synopsis: Build a binary distribution as a Redhat RPM and SRPM + + +.. % todo + + +:mod:`distutils.command.bdist_wininst` --- Build a Windows installer +==================================================================== + +.. module:: distutils.command.bdist_wininst + :synopsis: Build a Windows installer + +.. deprecated:: 3.8 + Use bdist_wheel (wheel packages) instead. + + +.. % todo + + +:mod:`distutils.command.sdist` --- Build a source distribution +============================================================== + +.. module:: distutils.command.sdist + :synopsis: Build a source distribution + + +.. % todo + + +:mod:`distutils.command.build` --- Build all files of a package +=============================================================== + +.. module:: distutils.command.build + :synopsis: Build all files of a package + + +.. % todo + + +:mod:`distutils.command.build_clib` --- Build any C libraries in a package +========================================================================== + +.. module:: distutils.command.build_clib + :synopsis: Build any C libraries in a package + + +.. % todo + + +:mod:`distutils.command.build_ext` --- Build any extensions in a package +======================================================================== + +.. module:: distutils.command.build_ext + :synopsis: Build any extensions in a package + + +.. % todo + + +:mod:`distutils.command.build_py` --- Build the .py/.pyc files of a package +=========================================================================== + +.. module:: distutils.command.build_py + :synopsis: Build the .py/.pyc files of a package + + +.. class:: build_py + + +:mod:`distutils.command.build_scripts` --- Build the scripts of a package +========================================================================= + +.. module:: distutils.command.build_scripts + :synopsis: Build the scripts of a package + + +.. % todo + + +:mod:`distutils.command.clean` --- Clean a package build area +============================================================= + +.. module:: distutils.command.clean + :synopsis: Clean a package build area + +This command removes the temporary files created by :command:`build` +and its subcommands, like intermediary compiled object files. With +the ``--all`` option, the complete build directory will be removed. + +Extension modules built :ref:`in place <distutils-build-ext-inplace>` +will not be cleaned, as they are not in the build directory. + + +:mod:`distutils.command.config` --- Perform package configuration +================================================================= + +.. module:: distutils.command.config + :synopsis: Perform package configuration + + +.. % todo + + +:mod:`distutils.command.install` --- Install a package +====================================================== + +.. module:: distutils.command.install + :synopsis: Install a package + + +.. % todo + + +:mod:`distutils.command.install_data` --- Install data files from a package +=========================================================================== + +.. module:: distutils.command.install_data + :synopsis: Install data files from a package + + +.. % todo + + +:mod:`distutils.command.install_headers` --- Install C/C++ header files from a package +====================================================================================== + +.. module:: distutils.command.install_headers + :synopsis: Install C/C++ header files from a package + + +.. % todo + + +:mod:`distutils.command.install_lib` --- Install library files from a package +============================================================================= + +.. module:: distutils.command.install_lib + :synopsis: Install library files from a package + + +.. % todo + + +:mod:`distutils.command.install_scripts` --- Install script files from a package +================================================================================ + +.. module:: distutils.command.install_scripts + :synopsis: Install script files from a package + + +.. % todo + + +:mod:`distutils.command.register` --- Register a module with the Python Package Index +===================================================================================== + +.. module:: distutils.command.register + :synopsis: Register a module with the Python Package Index + + +The ``register`` command registers the package with the Python Package Index. +This is described in more detail in :pep:`301`. + +.. % todo + + +:mod:`distutils.command.check` --- Check the meta-data of a package +=================================================================== + +.. module:: distutils.command.check + :synopsis: Check the meta-data of a package + + +The ``check`` command performs some tests on the meta-data of a package. +For example, it verifies that all required meta-data are provided as +the arguments passed to the :func:`~distutils.core.setup` function. + +.. % todo diff --git a/docs/deprecated/distutils/builtdist.rst b/docs/deprecated/distutils/builtdist.rst new file mode 100644 index 00000000..e032c03e --- /dev/null +++ b/docs/deprecated/distutils/builtdist.rst @@ -0,0 +1,479 @@ +.. _built-dist: + +**************************** +Creating Built Distributions +**************************** + +.. include:: ./_setuptools_disclaimer.rst + +A "built distribution" is what you're probably used to thinking of either as a +"binary package" or an "installer" (depending on your background). It's not +necessarily binary, though, because it might contain only Python source code +and/or byte-code; and we don't call it a package, because that word is already +spoken for in Python. (And "installer" is a term specific to the world of +mainstream desktop systems.) + +A built distribution is how you make life as easy as possible for installers of +your module distribution: for users of RPM-based Linux systems, it's a binary +RPM; for Windows users, it's an executable installer; for Debian-based Linux +users, it's a Debian package; and so forth. Obviously, no one person will be +able to create built distributions for every platform under the sun, so the +Distutils are designed to enable module developers to concentrate on their +specialty---writing code and creating source distributions---while an +intermediary species called *packagers* springs up to turn source distributions +into built distributions for as many platforms as there are packagers. + +Of course, the module developer could be their own packager; or the packager could +be a volunteer "out there" somewhere who has access to a platform which the +original developer does not; or it could be software periodically grabbing new +source distributions and turning them into built distributions for as many +platforms as the software has access to. Regardless of who they are, a packager +uses the setup script and the :command:`bdist` command family to generate built +distributions. + +As a simple example, if I run the following command in the Distutils source +tree:: + + python setup.py bdist + +then the Distutils builds my module distribution (the Distutils itself in this +case), does a "fake" installation (also in the :file:`build` directory), and +creates the default type of built distribution for my platform. The default +format for built distributions is a "dumb" tar file on Unix, and a simple +executable installer on Windows. (That tar file is considered "dumb" because it +has to be unpacked in a specific location to work.) + +Thus, the above command on a Unix system creates +:file:`Distutils-1.0.{plat}.tar.gz`; unpacking this tarball from the right place +installs the Distutils just as though you had downloaded the source distribution +and run ``python setup.py install``. (The "right place" is either the root of +the filesystem or Python's :file:`{prefix}` directory, depending on the options +given to the :command:`bdist_dumb` command; the default is to make dumb +distributions relative to :file:`{prefix}`.) + +Obviously, for pure Python distributions, this isn't any simpler than just +running ``python setup.py install``\ ---but for non-pure distributions, which +include extensions that would need to be compiled, it can mean the difference +between someone being able to use your extensions or not. And creating "smart" +built distributions, such as an RPM package or an executable installer for +Windows, is far more convenient for users even if your distribution doesn't +include any extensions. + +The :command:`bdist` command has a :option:`!--formats` option, similar to the +:command:`sdist` command, which you can use to select the types of built +distribution to generate: for example, :: + + python setup.py bdist --format=zip + +would, when run on a Unix system, create +:file:`Distutils-1.0.{plat}.zip`\ ---again, this archive would be unpacked +from the root directory to install the Distutils. + +The available formats for built distributions are: + ++-------------+------------------------------+---------+ +| Format | Description | Notes | ++=============+==============================+=========+ +| ``gztar`` | gzipped tar file | \(1) | +| | (:file:`.tar.gz`) | | ++-------------+------------------------------+---------+ +| ``bztar`` | bzipped tar file | | +| | (:file:`.tar.bz2`) | | ++-------------+------------------------------+---------+ +| ``xztar`` | xzipped tar file | | +| | (:file:`.tar.xz`) | | ++-------------+------------------------------+---------+ +| ``ztar`` | compressed tar file | \(3) | +| | (:file:`.tar.Z`) | | ++-------------+------------------------------+---------+ +| ``tar`` | tar file (:file:`.tar`) | | ++-------------+------------------------------+---------+ +| ``zip`` | zip file (:file:`.zip`) | (2),(4) | ++-------------+------------------------------+---------+ +| ``rpm`` | RPM | \(5) | ++-------------+------------------------------+---------+ +| ``pkgtool`` | Solaris :program:`pkgtool` | | ++-------------+------------------------------+---------+ +| ``sdux`` | HP-UX :program:`swinstall` | | ++-------------+------------------------------+---------+ +| ``wininst`` | self-extracting ZIP file for | \(4) | +| | Windows | | ++-------------+------------------------------+---------+ +| ``msi`` | Microsoft Installer. | | ++-------------+------------------------------+---------+ + +.. versionchanged:: 3.5 + Added support for the ``xztar`` format. + + +Notes: + +(1) + default on Unix + +(2) + default on Windows + +(3) + requires external :program:`compress` utility. + +(4) + requires either external :program:`zip` utility or :mod:`zipfile` module (part + of the standard Python library since Python 1.6) + +(5) + requires external :program:`rpm` utility, version 3.0.4 or better (use ``rpm + --version`` to find out which version you have) + +You don't have to use the :command:`bdist` command with the :option:`!--formats` +option; you can also use the command that directly implements the format you're +interested in. Some of these :command:`bdist` "sub-commands" actually generate +several similar formats; for instance, the :command:`bdist_dumb` command +generates all the "dumb" archive formats (``tar``, ``gztar``, ``bztar``, +``xztar``, ``ztar``, and ``zip``), and :command:`bdist_rpm` generates both +binary and source RPMs. The :command:`bdist` sub-commands, and the formats +generated by each, are: + ++--------------------------+-------------------------------------+ +| Command | Formats | ++==========================+=====================================+ +| :command:`bdist_dumb` | tar, gztar, bztar, xztar, ztar, zip | ++--------------------------+-------------------------------------+ +| :command:`bdist_rpm` | rpm, srpm | ++--------------------------+-------------------------------------+ +| :command:`bdist_wininst` | wininst | ++--------------------------+-------------------------------------+ +| :command:`bdist_msi` | msi | ++--------------------------+-------------------------------------+ + +.. note:: + bdist_wininst is deprecated since Python 3.8. + +.. note:: + bdist_msi is deprecated since Python 3.9. + +The following sections give details on the individual :command:`bdist_\*` +commands. + + +.. .. _creating-dumb: + +.. Creating dumb built distributions +.. ================================= + +.. XXX Need to document absolute vs. prefix-relative packages here, but first + I have to implement it! + + +.. _creating-rpms: + +Creating RPM packages +===================== + +The RPM format is used by many popular Linux distributions, including Red Hat, +SuSE, and Mandrake. If one of these (or any of the other RPM-based Linux +distributions) is your usual environment, creating RPM packages for other users +of that same distribution is trivial. Depending on the complexity of your module +distribution and differences between Linux distributions, you may also be able +to create RPMs that work on different RPM-based distributions. + +The usual way to create an RPM of your module distribution is to run the +:command:`bdist_rpm` command:: + + python setup.py bdist_rpm + +or the :command:`bdist` command with the :option:`!--format` option:: + + python setup.py bdist --formats=rpm + +The former allows you to specify RPM-specific options; the latter allows you to +easily specify multiple formats in one run. If you need to do both, you can +explicitly specify multiple :command:`bdist_\*` commands and their options:: + + python setup.py bdist_rpm --packager="John Doe <jdoe@example.org>" \ + bdist_wininst --target-version="2.0" + +Creating RPM packages is driven by a :file:`.spec` file, much as using the +Distutils is driven by the setup script. To make your life easier, the +:command:`bdist_rpm` command normally creates a :file:`.spec` file based on the +information you supply in the setup script, on the command line, and in any +Distutils configuration files. Various options and sections in the +:file:`.spec` file are derived from options in the setup script as follows: + ++------------------------------------------+----------------------------------------------+ +| RPM :file:`.spec` file option or section | Distutils setup script option | ++==========================================+==============================================+ +| Name | ``name`` | ++------------------------------------------+----------------------------------------------+ +| Summary (in preamble) | ``description`` | ++------------------------------------------+----------------------------------------------+ +| Version | ``version`` | ++------------------------------------------+----------------------------------------------+ +| Vendor | ``author`` and ``author_email``, | +| | or --- & ``maintainer`` and | +| | ``maintainer_email`` | ++------------------------------------------+----------------------------------------------+ +| Copyright | ``license`` | ++------------------------------------------+----------------------------------------------+ +| Url | ``url`` | ++------------------------------------------+----------------------------------------------+ +| %description (section) | ``long_description`` | ++------------------------------------------+----------------------------------------------+ + +Additionally, there are many options in :file:`.spec` files that don't have +corresponding options in the setup script. Most of these are handled through +options to the :command:`bdist_rpm` command as follows: + ++-------------------------------+-----------------------------+-------------------------+ +| RPM :file:`.spec` file option | :command:`bdist_rpm` option | default value | +| or section | | | ++===============================+=============================+=========================+ +| Release | ``release`` | "1" | ++-------------------------------+-----------------------------+-------------------------+ +| Group | ``group`` | "Development/Libraries" | ++-------------------------------+-----------------------------+-------------------------+ +| Vendor | ``vendor`` | (see above) | ++-------------------------------+-----------------------------+-------------------------+ +| Packager | ``packager`` | (none) | ++-------------------------------+-----------------------------+-------------------------+ +| Provides | ``provides`` | (none) | ++-------------------------------+-----------------------------+-------------------------+ +| Requires | ``requires`` | (none) | ++-------------------------------+-----------------------------+-------------------------+ +| Conflicts | ``conflicts`` | (none) | ++-------------------------------+-----------------------------+-------------------------+ +| Obsoletes | ``obsoletes`` | (none) | ++-------------------------------+-----------------------------+-------------------------+ +| Distribution | ``distribution_name`` | (none) | ++-------------------------------+-----------------------------+-------------------------+ +| BuildRequires | ``build_requires`` | (none) | ++-------------------------------+-----------------------------+-------------------------+ +| Icon | ``icon`` | (none) | ++-------------------------------+-----------------------------+-------------------------+ + +Obviously, supplying even a few of these options on the command-line would be +tedious and error-prone, so it's usually best to put them in the setup +configuration file, :file:`setup.cfg`\ ---see section :ref:`setup-config`. If +you distribute or package many Python module distributions, you might want to +put options that apply to all of them in your personal Distutils configuration +file (:file:`~/.pydistutils.cfg`). If you want to temporarily disable +this file, you can pass the :option:`!--no-user-cfg` option to :file:`setup.py`. + +There are three steps to building a binary RPM package, all of which are +handled automatically by the Distutils: + +#. create a :file:`.spec` file, which describes the package (analogous to the + Distutils setup script; in fact, much of the information in the setup script + winds up in the :file:`.spec` file) + +#. create the source RPM + +#. create the "binary" RPM (which may or may not contain binary code, depending + on whether your module distribution contains Python extensions) + +Normally, RPM bundles the last two steps together; when you use the Distutils, +all three steps are typically bundled together. + +If you wish, you can separate these three steps. You can use the +:option:`!--spec-only` option to make :command:`bdist_rpm` just create the +:file:`.spec` file and exit; in this case, the :file:`.spec` file will be +written to the "distribution directory"---normally :file:`dist/`, but +customizable with the :option:`!--dist-dir` option. (Normally, the :file:`.spec` +file winds up deep in the "build tree," in a temporary directory created by +:command:`bdist_rpm`.) + +.. % \XXX{this isn't implemented yet---is it needed?!} +.. % You can also specify a custom \file{.spec} file with the +.. % \longprogramopt{spec-file} option; used in conjunction with +.. % \longprogramopt{spec-only}, this gives you an opportunity to customize +.. % the \file{.spec} file manually: +.. % +.. % \ begin{verbatim} +.. % > python setup.py bdist_rpm --spec-only +.. % # ...edit dist/FooBar-1.0.spec +.. % > python setup.py bdist_rpm --spec-file=dist/FooBar-1.0.spec +.. % \ end{verbatim} +.. % +.. % (Although a better way to do this is probably to override the standard +.. % \command{bdist\_rpm} command with one that writes whatever else you want +.. % to the \file{.spec} file.) + + +.. _creating-wininst: + +Creating Windows Installers +=========================== + +.. warning:: + bdist_wininst is deprecated since Python 3.8. + +.. warning:: + bdist_msi is deprecated since Python 3.9. + +Executable installers are the natural format for binary distributions on +Windows. They display a nice graphical user interface, display some information +about the module distribution to be installed taken from the metadata in the +setup script, let the user select a few options, and start or cancel the +installation. + +Since the metadata is taken from the setup script, creating Windows installers +is usually as easy as running:: + + python setup.py bdist_wininst + +or the :command:`bdist` command with the :option:`!--formats` option:: + + python setup.py bdist --formats=wininst + +If you have a pure module distribution (only containing pure Python modules and +packages), the resulting installer will be version independent and have a name +like :file:`foo-1.0.win32.exe`. Note that creating ``wininst`` binary +distributions in only supported on Windows systems. + +If you have a non-pure distribution, the extensions can only be created on a +Windows platform, and will be Python version dependent. The installer filename +will reflect this and now has the form :file:`foo-1.0.win32-py2.0.exe`. You +have to create a separate installer for every Python version you want to +support. + +The installer will try to compile pure modules into :term:`bytecode` after installation +on the target system in normal and optimizing mode. If you don't want this to +happen for some reason, you can run the :command:`bdist_wininst` command with +the :option:`!--no-target-compile` and/or the :option:`!--no-target-optimize` +option. + +By default the installer will display the cool "Python Powered" logo when it is +run, but you can also supply your own 152x261 bitmap which must be a Windows +:file:`.bmp` file with the :option:`!--bitmap` option. + +The installer will also display a large title on the desktop background window +when it is run, which is constructed from the name of your distribution and the +version number. This can be changed to another text by using the +:option:`!--title` option. + +The installer file will be written to the "distribution directory" --- normally +:file:`dist/`, but customizable with the :option:`!--dist-dir` option. + +.. _cross-compile-windows: + +Cross-compiling on Windows +========================== + +Starting with Python 2.6, distutils is capable of cross-compiling between +Windows platforms. In practice, this means that with the correct tools +installed, you can use a 32bit version of Windows to create 64bit extensions +and vice-versa. + +To build for an alternate platform, specify the :option:`!--plat-name` option +to the build command. Valid values are currently 'win32', and 'win-amd64'. +For example, on a 32bit version of Windows, you could execute:: + + python setup.py build --plat-name=win-amd64 + +to build a 64bit version of your extension. The Windows Installers also +support this option, so the command:: + + python setup.py build --plat-name=win-amd64 bdist_wininst + +would create a 64bit installation executable on your 32bit version of Windows. + +To cross-compile, you must download the Python source code and cross-compile +Python itself for the platform you are targeting - it is not possible from a +binary installation of Python (as the .lib etc file for other platforms are +not included.) In practice, this means the user of a 32 bit operating +system will need to use Visual Studio 2008 to open the +:file:`PCbuild/PCbuild.sln` solution in the Python source tree and build the +"x64" configuration of the 'pythoncore' project before cross-compiling +extensions is possible. + +Note that by default, Visual Studio 2008 does not install 64bit compilers or +tools. You may need to reexecute the Visual Studio setup process and select +these tools (using Control Panel->[Add/Remove] Programs is a convenient way to +check or modify your existing install.) + +.. _postinstallation-script: + +The Postinstallation script +--------------------------- + +Starting with Python 2.3, a postinstallation script can be specified with the +:option:`!--install-script` option. The basename of the script must be +specified, and the script filename must also be listed in the scripts argument +to the setup function. + +This script will be run at installation time on the target system after all the +files have been copied, with ``argv[1]`` set to :option:`!-install`, and again at +uninstallation time before the files are removed with ``argv[1]`` set to +:option:`!-remove`. + +The installation script runs embedded in the windows installer, every output +(``sys.stdout``, ``sys.stderr``) is redirected into a buffer and will be +displayed in the GUI after the script has finished. + +Some functions especially useful in this context are available as additional +built-in functions in the installation script. + + +.. function:: directory_created(path) + file_created(path) + + These functions should be called when a directory or file is created by the + postinstall script at installation time. It will register *path* with the + uninstaller, so that it will be removed when the distribution is uninstalled. + To be safe, directories are only removed if they are empty. + + +.. function:: get_special_folder_path(csidl_string) + + This function can be used to retrieve special folder locations on Windows like + the Start Menu or the Desktop. It returns the full path to the folder. + *csidl_string* must be one of the following strings:: + + "CSIDL_APPDATA" + + "CSIDL_COMMON_STARTMENU" + "CSIDL_STARTMENU" + + "CSIDL_COMMON_DESKTOPDIRECTORY" + "CSIDL_DESKTOPDIRECTORY" + + "CSIDL_COMMON_STARTUP" + "CSIDL_STARTUP" + + "CSIDL_COMMON_PROGRAMS" + "CSIDL_PROGRAMS" + + "CSIDL_FONTS" + + If the folder cannot be retrieved, :exc:`OSError` is raised. + + Which folders are available depends on the exact Windows version, and probably + also the configuration. For details refer to Microsoft's documentation of the + :c:func:`SHGetSpecialFolderPath` function. + + +.. function:: create_shortcut(target, description, filename[, arguments[, workdir[, iconpath[, iconindex]]]]) + + This function creates a shortcut. *target* is the path to the program to be + started by the shortcut. *description* is the description of the shortcut. + *filename* is the title of the shortcut that the user will see. *arguments* + specifies the command line arguments, if any. *workdir* is the working directory + for the program. *iconpath* is the file containing the icon for the shortcut, + and *iconindex* is the index of the icon in the file *iconpath*. Again, for + details consult the Microsoft documentation for the :class:`IShellLink` + interface. + + +Vista User Access Control (UAC) +=============================== + +Starting with Python 2.6, bdist_wininst supports a :option:`!--user-access-control` +option. The default is 'none' (meaning no UAC handling is done), and other +valid values are 'auto' (meaning prompt for UAC elevation if Python was +installed for all users) and 'force' (meaning always prompt for elevation). + +.. note:: + bdist_wininst is deprecated since Python 3.8. + +.. note:: + bdist_msi is deprecated since Python 3.9. diff --git a/docs/deprecated/distutils/commandref.rst b/docs/deprecated/distutils/commandref.rst new file mode 100644 index 00000000..0f6fe2ab --- /dev/null +++ b/docs/deprecated/distutils/commandref.rst @@ -0,0 +1,106 @@ +.. _reference: + +***************** +Command Reference +***************** + +.. include:: ./_setuptools_disclaimer.rst + +.. % \section{Building modules: the \protect\command{build} command family} +.. % \label{build-cmds} +.. % \subsubsection{\protect\command{build}} +.. % \label{build-cmd} +.. % \subsubsection{\protect\command{build\_py}} +.. % \label{build-py-cmd} +.. % \subsubsection{\protect\command{build\_ext}} +.. % \label{build-ext-cmd} +.. % \subsubsection{\protect\command{build\_clib}} +.. % \label{build-clib-cmd} + + +.. _install-cmd: + +Installing modules: the :command:`install` command family +========================================================= + +The install command ensures that the build commands have been run and then runs +the subcommands :command:`install_lib`, :command:`install_data` and +:command:`install_scripts`. + +.. % \subsubsection{\protect\command{install\_lib}} +.. % \label{install-lib-cmd} + + +.. _install-data-cmd: + +:command:`install_data` +----------------------- + +This command installs all data files provided with the distribution. + + +.. _install-scripts-cmd: + +:command:`install_scripts` +-------------------------- + +This command installs all (Python) scripts in the distribution. + +.. % \subsection{Cleaning up: the \protect\command{clean} command} +.. % \label{clean-cmd} + + +.. _sdist-cmd: + +Creating a source distribution: the :command:`sdist` command +============================================================ + +.. XXX fragment moved down from above: needs context! + +The manifest template commands are: + ++-------------------------------------------+-----------------------------------------------+ +| Command | Description | ++===========================================+===============================================+ +| :command:`include pat1 pat2 ...` | include all files matching any of the listed | +| | patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`exclude pat1 pat2 ...` | exclude all files matching any of the listed | +| | patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`recursive-include dir pat1 pat2 | include all files under *dir* matching any of | +| ...` | the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`recursive-exclude dir pat1 pat2 | exclude all files under *dir* matching any of | +| ...` | the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`global-include pat1 pat2 ...` | include all files anywhere in the source tree | +| | matching --- & any of the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`global-exclude pat1 pat2 ...` | exclude all files anywhere in the source tree | +| | matching --- & any of the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`prune dir` | exclude all files under *dir* | ++-------------------------------------------+-----------------------------------------------+ +| :command:`graft dir` | include all files under *dir* | ++-------------------------------------------+-----------------------------------------------+ + +The patterns here are Unix-style "glob" patterns: ``*`` matches any sequence of +regular filename characters, ``?`` matches any single regular filename +character, and ``[range]`` matches any of the characters in *range* (e.g., +``a-z``, ``a-zA-Z``, ``a-f0-9_.``). The definition of "regular filename +character" is platform-specific: on Unix it is anything except slash; on Windows +anything except backslash or colon. + +.. XXX Windows support not there yet + +.. % \section{Creating a built distribution: the +.. % \protect\command{bdist} command family} +.. % \label{bdist-cmds} + +.. % \subsection{\protect\command{bdist}} +.. % \subsection{\protect\command{bdist\_dumb}} +.. % \subsection{\protect\command{bdist\_rpm}} +.. % \subsection{\protect\command{bdist\_wininst}} + + diff --git a/docs/deprecated/distutils/configfile.rst b/docs/deprecated/distutils/configfile.rst new file mode 100644 index 00000000..328936fb --- /dev/null +++ b/docs/deprecated/distutils/configfile.rst @@ -0,0 +1,144 @@ +.. _setup-config: + +************************************ +Writing the Setup Configuration File +************************************ + +.. include:: ./_setuptools_disclaimer.rst + +Often, it's not possible to write down everything needed to build a distribution +*a priori*: you may need to get some information from the user, or from the +user's system, in order to proceed. As long as that information is fairly +simple---a list of directories to search for C header files or libraries, for +example---then providing a configuration file, :file:`setup.cfg`, for users to +edit is a cheap and easy way to solicit it. Configuration files also let you +provide default values for any command option, which the installer can then +override either on the command-line or by editing the config file. + +The setup configuration file is a useful middle-ground between the setup +script---which, ideally, would be opaque to installers [#]_---and the command-line to +the setup script, which is outside of your control and entirely up to the +installer. In fact, :file:`setup.cfg` (and any other Distutils configuration +files present on the target system) are processed after the contents of the +setup script, but before the command-line. This has several useful +consequences: + +.. % (If you have more advanced needs, such as determining which extensions +.. % to build based on what capabilities are present on the target system, +.. % then you need the Distutils ``auto-configuration'' facility. This +.. % started to appear in Distutils 0.9 but, as of this writing, isn't mature +.. % or stable enough yet for real-world use.) + +* installers can override some of what you put in :file:`setup.py` by editing + :file:`setup.cfg` + +* you can provide non-standard defaults for options that are not easily set in + :file:`setup.py` + +* installers can override anything in :file:`setup.cfg` using the command-line + options to :file:`setup.py` + +The basic syntax of the configuration file is simple: + +.. code-block:: ini + + [command] + option=value + ... + +where *command* is one of the Distutils commands (e.g. :command:`build_py`, +:command:`install`), and *option* is one of the options that command supports. +Any number of options can be supplied for each command, and any number of +command sections can be included in the file. Blank lines are ignored, as are +comments, which run from a ``'#'`` character until the end of the line. Long +option values can be split across multiple lines simply by indenting the +continuation lines. + +You can find out the list of options supported by a particular command with the +universal :option:`!--help` option, e.g. + +.. code-block:: shell-session + + $ python setup.py --help build_ext + [...] + Options for 'build_ext' command: + --build-lib (-b) directory for compiled extension modules + --build-temp (-t) directory for temporary files (build by-products) + --inplace (-i) ignore build-lib and put compiled extensions into the + source directory alongside your pure Python modules + --include-dirs (-I) list of directories to search for header files + --define (-D) C preprocessor macros to define + --undef (-U) C preprocessor macros to undefine + --swig-opts list of SWIG command line options + [...] + +Note that an option spelled :option:`!--foo-bar` on the command-line is spelled +``foo_bar`` in configuration files. + +.. _distutils-build-ext-inplace: + +For example, say you want your extensions to be built "in-place"---that is, you +have an extension ``pkg.ext``, and you want the compiled extension file +(:file:`ext.so` on Unix, say) to be put in the same source directory as your +pure Python modules ``pkg.mod1`` and ``pkg.mod2``. You can always use the +:option:`!--inplace` option on the command-line to ensure this: + +.. code-block:: sh + + python setup.py build_ext --inplace + +But this requires that you always specify the :command:`build_ext` command +explicitly, and remember to provide :option:`!--inplace`. An easier way is to +"set and forget" this option, by encoding it in :file:`setup.cfg`, the +configuration file for this distribution: + +.. code-block:: ini + + [build_ext] + inplace=1 + +This will affect all builds of this module distribution, whether or not you +explicitly specify :command:`build_ext`. If you include :file:`setup.cfg` in +your source distribution, it will also affect end-user builds---which is +probably a bad idea for this option, since always building extensions in-place +would break installation of the module distribution. In certain peculiar cases, +though, modules are built right in their installation directory, so this is +conceivably a useful ability. (Distributing extensions that expect to be built +in their installation directory is almost always a bad idea, though.) + +Another example: certain commands take a lot of options that don't change from +run to run; for example, :command:`bdist_rpm` needs to know everything required +to generate a "spec" file for creating an RPM distribution. Some of this +information comes from the setup script, and some is automatically generated by +the Distutils (such as the list of files installed). But some of it has to be +supplied as options to :command:`bdist_rpm`, which would be very tedious to do +on the command-line for every run. Hence, here is a snippet from the Distutils' +own :file:`setup.cfg`: + +.. code-block:: ini + + [bdist_rpm] + release = 1 + packager = Greg Ward <gward@python.net> + doc_files = CHANGES.txt + README.txt + USAGE.txt + doc/ + examples/ + +Note that the ``doc_files`` option is simply a whitespace-separated string +split across multiple lines for readability. + + +.. seealso:: + + :ref:`inst-config-syntax` in "Installing Python Modules" + More information on the configuration files is available in the manual for + system administrators. + + +.. rubric:: Footnotes + +.. [#] This ideal probably won't be achieved until auto-configuration is fully + supported by the Distutils. + diff --git a/docs/deprecated/distutils/examples.rst b/docs/deprecated/distutils/examples.rst new file mode 100644 index 00000000..d0984655 --- /dev/null +++ b/docs/deprecated/distutils/examples.rst @@ -0,0 +1,340 @@ +.. _distutils_examples: + +****************** +Distutils Examples +****************** + +.. include:: ./_setuptools_disclaimer.rst + +This chapter provides a number of basic examples to help get started with +distutils. Additional information about using distutils can be found in the +Distutils Cookbook. + + +.. seealso:: + + `Distutils Cookbook <https://wiki.python.org/moin/Distutils/Cookbook>`_ + Collection of recipes showing how to achieve more control over distutils. + + +.. _pure-mod: + +Pure Python distribution (by module) +==================================== + +If you're just distributing a couple of modules, especially if they don't live +in a particular package, you can specify them individually using the +``py_modules`` option in the setup script. + +In the simplest case, you'll have two files to worry about: a setup script and +the single module you're distributing, :file:`foo.py` in this example:: + + <root>/ + setup.py + foo.py + +(In all diagrams in this section, *<root>* will refer to the distribution root +directory.) A minimal setup script to describe this situation would be:: + + from distutils.core import setup + setup(name='foo', + version='1.0', + py_modules=['foo'], + ) + +Note that the name of the distribution is specified independently with the +``name`` option, and there's no rule that says it has to be the same as +the name of the sole module in the distribution (although that's probably a good +convention to follow). However, the distribution name is used to generate +filenames, so you should stick to letters, digits, underscores, and hyphens. + +Since ``py_modules`` is a list, you can of course specify multiple +modules, eg. if you're distributing modules ``foo`` and ``bar``, your +setup might look like this:: + + <root>/ + setup.py + foo.py + bar.py + +and the setup script might be :: + + from distutils.core import setup + setup(name='foobar', + version='1.0', + py_modules=['foo', 'bar'], + ) + +You can put module source files into another directory, but if you have enough +modules to do that, it's probably easier to specify modules by package rather +than listing them individually. + + +.. _pure-pkg: + +Pure Python distribution (by package) +===================================== + +If you have more than a couple of modules to distribute, especially if they are +in multiple packages, it's probably easier to specify whole packages rather than +individual modules. This works even if your modules are not in a package; you +can just tell the Distutils to process modules from the root package, and that +works the same as any other package (except that you don't have to have an +:file:`__init__.py` file). + +The setup script from the last example could also be written as :: + + from distutils.core import setup + setup(name='foobar', + version='1.0', + packages=[''], + ) + +(The empty string stands for the root package.) + +If those two files are moved into a subdirectory, but remain in the root +package, e.g.:: + + <root>/ + setup.py + src/ foo.py + bar.py + +then you would still specify the root package, but you have to tell the +Distutils where source files in the root package live:: + + from distutils.core import setup + setup(name='foobar', + version='1.0', + package_dir={'': 'src'}, + packages=[''], + ) + +More typically, though, you will want to distribute multiple modules in the same +package (or in sub-packages). For example, if the ``foo`` and ``bar`` +modules belong in package ``foobar``, one way to layout your source tree is +:: + + <root>/ + setup.py + foobar/ + __init__.py + foo.py + bar.py + +This is in fact the default layout expected by the Distutils, and the one that +requires the least work to describe in your setup script:: + + from distutils.core import setup + setup(name='foobar', + version='1.0', + packages=['foobar'], + ) + +If you want to put modules in directories not named for their package, then you +need to use the ``package_dir`` option again. For example, if the +:file:`src` directory holds modules in the ``foobar`` package:: + + <root>/ + setup.py + src/ + __init__.py + foo.py + bar.py + +an appropriate setup script would be :: + + from distutils.core import setup + setup(name='foobar', + version='1.0', + package_dir={'foobar': 'src'}, + packages=['foobar'], + ) + +Or, you might put modules from your main package right in the distribution +root:: + + <root>/ + setup.py + __init__.py + foo.py + bar.py + +in which case your setup script would be :: + + from distutils.core import setup + setup(name='foobar', + version='1.0', + package_dir={'foobar': ''}, + packages=['foobar'], + ) + +(The empty string also stands for the current directory.) + +If you have sub-packages, they must be explicitly listed in ``packages``, +but any entries in ``package_dir`` automatically extend to sub-packages. +(In other words, the Distutils does *not* scan your source tree, trying to +figure out which directories correspond to Python packages by looking for +:file:`__init__.py` files.) Thus, if the default layout grows a sub-package:: + + <root>/ + setup.py + foobar/ + __init__.py + foo.py + bar.py + subfoo/ + __init__.py + blah.py + +then the corresponding setup script would be :: + + from distutils.core import setup + setup(name='foobar', + version='1.0', + packages=['foobar', 'foobar.subfoo'], + ) + + +.. _single-ext: + +Single extension module +======================= + +Extension modules are specified using the ``ext_modules`` option. +``package_dir`` has no effect on where extension source files are found; +it only affects the source for pure Python modules. The simplest case, a +single extension module in a single C source file, is:: + + <root>/ + setup.py + foo.c + +If the ``foo`` extension belongs in the root package, the setup script for +this could be :: + + from distutils.core import setup + from distutils.extension import Extension + setup(name='foobar', + version='1.0', + ext_modules=[Extension('foo', ['foo.c'])], + ) + +If the extension actually belongs in a package, say ``foopkg``, then + +With exactly the same source tree layout, this extension can be put in the +``foopkg`` package simply by changing the name of the extension:: + + from distutils.core import setup + from distutils.extension import Extension + setup(name='foobar', + version='1.0', + ext_modules=[Extension('foopkg.foo', ['foo.c'])], + ) + +Checking a package +================== + +The ``check`` command allows you to verify if your package meta-data +meet the minimum requirements to build a distribution. + +To run it, just call it using your :file:`setup.py` script. If something is +missing, ``check`` will display a warning. + +Let's take an example with a simple script:: + + from distutils.core import setup + + setup(name='foobar') + +Running the ``check`` command will display some warnings: + +.. code-block:: shell-session + + $ python setup.py check + running check + warning: check: missing required meta-data: version, url + warning: check: missing meta-data: either (author and author_email) or + (maintainer and maintainer_email) should be supplied + + +If you use the reStructuredText syntax in the ``long_description`` field and +`docutils`_ is installed you can check if the syntax is fine with the +``check`` command, using the ``restructuredtext`` option. + +For example, if the :file:`setup.py` script is changed like this:: + + from distutils.core import setup + + desc = """\ + My description + ============== + + This is the description of the ``foobar`` package. + """ + + setup(name='foobar', version='1', author='tarek', + author_email='tarek@ziade.org', + url='http://example.com', long_description=desc) + +Where the long description is broken, ``check`` will be able to detect it +by using the :mod:`docutils` parser: + +.. code-block:: shell-session + + $ python setup.py check --restructuredtext + running check + warning: check: Title underline too short. (line 2) + warning: check: Could not finish the parsing. + +Reading the metadata +===================== + +The :func:`distutils.core.setup` function provides a command-line interface +that allows you to query the metadata fields of a project through the +``setup.py`` script of a given project: + +.. code-block:: shell-session + + $ python setup.py --name + distribute + +This call reads the ``name`` metadata by running the +:func:`distutils.core.setup` function. Although, when a source or binary +distribution is created with Distutils, the metadata fields are written +in a static file called :file:`PKG-INFO`. When a Distutils-based project is +installed in Python, the :file:`PKG-INFO` file is copied alongside the modules +and packages of the distribution under :file:`NAME-VERSION-pyX.X.egg-info`, +where ``NAME`` is the name of the project, ``VERSION`` its version as defined +in the Metadata, and ``pyX.X`` the major and minor version of Python like +``2.7`` or ``3.2``. + +You can read back this static file, by using the +:class:`distutils.dist.DistributionMetadata` class and its +:func:`~distutils.dist.DistributionMetadata.read_pkg_file` method:: + + >>> from distutils.dist import DistributionMetadata + >>> metadata = DistributionMetadata() + >>> metadata.read_pkg_file(open('distribute-0.6.8-py2.7.egg-info')) + >>> metadata.name + 'distribute' + >>> metadata.version + '0.6.8' + >>> metadata.description + 'Easily download, build, install, upgrade, and uninstall Python packages' + +Notice that the class can also be instantiated with a metadata file path to +loads its values:: + + >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' + >>> DistributionMetadata(pkg_info_path).name + 'distribute' + + +.. % \section{Multiple extension modules} +.. % \label{multiple-ext} + +.. % \section{Putting it all together} + + +.. _docutils: http://docutils.sourceforge.net diff --git a/docs/deprecated/distutils/extending.rst b/docs/deprecated/distutils/extending.rst new file mode 100644 index 00000000..c99d3c79 --- /dev/null +++ b/docs/deprecated/distutils/extending.rst @@ -0,0 +1,98 @@ +.. _extending-distutils: + +******************* +Extending Distutils +******************* + +.. include:: ./_setuptools_disclaimer.rst + +Distutils can be extended in various ways. Most extensions take the form of new +commands or replacements for existing commands. New commands may be written to +support new types of platform-specific packaging, for example, while +replacements for existing commands may be made to modify details of how the +command operates on a package. + +Most extensions of the distutils are made within :file:`setup.py` scripts that +want to modify existing commands; many simply add a few file extensions that +should be copied into packages in addition to :file:`.py` files as a +convenience. + +Most distutils command implementations are subclasses of the +:class:`distutils.cmd.Command` class. New commands may directly inherit from +:class:`~distutils.cmd.Command`, while replacements often derive from :class:`~distutils.cmd.Command` +indirectly, directly subclassing the command they are replacing. Commands are +required to derive from :class:`~distutils.cmd.Command`. + +.. % \section{Extending existing commands} +.. % \label{extend-existing} + +.. % \section{Writing new commands} +.. % \label{new-commands} +.. % \XXX{Would an uninstall command be a good example here?} + + +Integrating new commands +======================== + +There are different ways to integrate new command implementations into +distutils. The most difficult is to lobby for the inclusion of the new features +in distutils itself, and wait for (and require) a version of Python that +provides that support. This is really hard for many reasons. + +The most common, and possibly the most reasonable for most needs, is to include +the new implementations with your :file:`setup.py` script, and cause the +:func:`distutils.core.setup` function use them:: + + from distutils.command.build_py import build_py as _build_py + from distutils.core import setup + + class build_py(_build_py): + """Specialized Python source builder.""" + + # implement whatever needs to be different... + + setup(cmdclass={'build_py': build_py}, + ...) + +This approach is most valuable if the new implementations must be used to use a +particular package, as everyone interested in the package will need to have the +new command implementation. + +Beginning with Python 2.4, a third option is available, intended to allow new +commands to be added which can support existing :file:`setup.py` scripts without +requiring modifications to the Python installation. This is expected to allow +third-party extensions to provide support for additional packaging systems, but +the commands can be used for anything distutils commands can be used for. A new +configuration option, ``command_packages`` (command-line option +:option:`!--command-packages`), can be used to specify additional packages to be +searched for modules implementing commands. Like all distutils options, this +can be specified on the command line or in a configuration file. This option +can only be set in the ``[global]`` section of a configuration file, or before +any commands on the command line. If set in a configuration file, it can be +overridden from the command line; setting it to an empty string on the command +line causes the default to be used. This should never be set in a configuration +file provided with a package. + +This new option can be used to add any number of packages to the list of +packages searched for command implementations; multiple package names should be +separated by commas. When not specified, the search is only performed in the +:mod:`distutils.command` package. When :file:`setup.py` is run with the option +``--command-packages distcmds,buildcmds``, however, the packages +:mod:`distutils.command`, ``distcmds``, and ``buildcmds`` will be searched +in that order. New commands are expected to be implemented in modules of the +same name as the command by classes sharing the same name. Given the example +command line option above, the command :command:`bdist_openpkg` could be +implemented by the class ``distcmds.bdist_openpkg.bdist_openpkg`` or +``buildcmds.bdist_openpkg.bdist_openpkg``. + + +Adding new distribution types +============================= + +Commands that create distributions (files in the :file:`dist/` directory) need +to add ``(command, filename)`` pairs to ``self.distribution.dist_files`` so that +:command:`upload` can upload it to PyPI. The *filename* in the pair contains no +path information, only the name of the file itself. In dry-run mode, pairs +should still be added to represent what would have been created. + + diff --git a/docs/deprecated/distutils/index.rst b/docs/deprecated/distutils/index.rst new file mode 100644 index 00000000..1f72a255 --- /dev/null +++ b/docs/deprecated/distutils/index.rst @@ -0,0 +1,42 @@ +.. _distutils-index: + +############################################## + Distributing Python Modules (Legacy version) +############################################## + +:Authors: Greg Ward, Anthony Baxter +:Email: distutils-sig@python.org + +.. seealso:: + + :ref:`distributing-index` + The up to date module distribution documentations + +.. include:: ./_setuptools_disclaimer.rst + +.. note:: + + This guide only covers the basic tools for building and distributing + extensions that are provided as part of this version of Python. Third party + tools offer easier to use and more secure alternatives. Refer to the `quick + recommendations section <https://packaging.python.org/guides/tool-recommendations/>`__ + in the Python Packaging User Guide for more information. + +This document describes the Python Distribution Utilities ("Distutils") from +the module developer's point of view, describing the underlying capabilities +that ``setuptools`` builds on to allow Python developers to make Python modules +and extensions readily available to a wider audience. + +.. toctree:: + :maxdepth: 2 + :numbered: + + introduction.rst + setupscript.rst + configfile.rst + sourcedist.rst + builtdist.rst + examples.rst + extending.rst + commandref.rst + apiref.rst diff --git a/docs/deprecated/distutils/introduction.rst b/docs/deprecated/distutils/introduction.rst new file mode 100644 index 00000000..7491b965 --- /dev/null +++ b/docs/deprecated/distutils/introduction.rst @@ -0,0 +1,214 @@ +.. _distutils-intro: + +**************************** +An Introduction to Distutils +**************************** + +.. include:: ./_setuptools_disclaimer.rst + +This document covers using the Distutils to distribute your Python modules, +concentrating on the role of developer/distributor: if you're looking for +information on installing Python modules, you should refer to the +:ref:`install-index` chapter. + + +.. _distutils-concepts: + +Concepts & Terminology +====================== + +Using the Distutils is quite simple, both for module developers and for +users/administrators installing third-party modules. As a developer, your +responsibilities (apart from writing solid, well-documented and well-tested +code, of course!) are: + +* write a setup script (:file:`setup.py` by convention) + +* (optional) write a setup configuration file + +* create a source distribution + +* (optional) create one or more built (binary) distributions + +Each of these tasks is covered in this document. + +Not all module developers have access to a multitude of platforms, so it's not +always feasible to expect them to create a multitude of built distributions. It +is hoped that a class of intermediaries, called *packagers*, will arise to +address this need. Packagers will take source distributions released by module +developers, build them on one or more platforms, and release the resulting built +distributions. Thus, users on the most popular platforms will be able to +install most popular Python module distributions in the most natural way for +their platform, without having to run a single setup script or compile a line of +code. + + +.. _distutils-simple-example: + +A Simple Example +================ + +The setup script is usually quite simple, although since it's written in Python, +there are no arbitrary limits to what you can do with it, though you should be +careful about putting arbitrarily expensive operations in your setup script. +Unlike, say, Autoconf-style configure scripts, the setup script may be run +multiple times in the course of building and installing your module +distribution. + +If all you want to do is distribute a module called ``foo``, contained in a +file :file:`foo.py`, then your setup script can be as simple as this:: + + from distutils.core import setup + setup(name='foo', + version='1.0', + py_modules=['foo'], + ) + +Some observations: + +* most information that you supply to the Distutils is supplied as keyword + arguments to the :func:`~distutils.core.setup` function + +* those keyword arguments fall into two categories: package metadata (name, + version number) and information about what's in the package (a list of pure + Python modules, in this case) + +* modules are specified by module name, not filename (the same will hold true + for packages and extensions) + +* it's recommended that you supply a little more metadata, in particular your + name, email address and a URL for the project (see section :ref:`setup-script` + for an example) + +To create a source distribution for this module, you would create a setup +script, :file:`setup.py`, containing the above code, and run this command from a +terminal:: + + python setup.py sdist + +For Windows, open a command prompt window (:menuselection:`Start --> +Accessories`) and change the command to:: + + setup.py sdist + +:command:`sdist` will create an archive file (e.g., tarball on Unix, ZIP file on Windows) +containing your setup script :file:`setup.py`, and your module :file:`foo.py`. +The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and +will unpack into a directory :file:`foo-1.0`. + +If an end-user wishes to install your ``foo`` module, all they have to do is +download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and---from the +:file:`foo-1.0` directory---run :: + + python setup.py install + +which will ultimately copy :file:`foo.py` to the appropriate directory for +third-party modules in their Python installation. + +This simple example demonstrates some fundamental concepts of the Distutils. +First, both developers and installers have the same basic user interface, i.e. +the setup script. The difference is which Distutils *commands* they use: the +:command:`sdist` command is almost exclusively for module developers, while +:command:`install` is more often for installers (although most developers will +want to install their own code occasionally). + +If you want to make things really easy for your users, you can create one or +more built distributions for them. For instance, if you are running on a +Windows machine, and want to make things easy for other Windows users, you can +create an executable installer (the most appropriate type of built distribution +for this platform) with the :command:`bdist_wininst` command. For example:: + + python setup.py bdist_wininst + +will create an executable installer, :file:`foo-1.0.win32.exe`, in the current +directory. + +Other useful built distribution formats are RPM, implemented by the +:command:`bdist_rpm` command, Solaris :program:`pkgtool` +(:command:`bdist_pkgtool`), and HP-UX :program:`swinstall` +(:command:`bdist_sdux`). For example, the following command will create an RPM +file called :file:`foo-1.0.noarch.rpm`:: + + python setup.py bdist_rpm + +(The :command:`bdist_rpm` command uses the :command:`rpm` executable, therefore +this has to be run on an RPM-based system such as Red Hat Linux, SuSE Linux, or +Mandrake Linux.) + +You can find out what distribution formats are available at any time by running +:: + + python setup.py bdist --help-formats + + +.. _python-terms: + +General Python terminology +========================== + +If you're reading this document, you probably have a good idea of what modules, +extensions, and so forth are. Nevertheless, just to be sure that everyone is +operating from a common starting point, we offer the following glossary of +common Python terms: + +module + the basic unit of code reusability in Python: a block of code imported by some + other code. Three types of modules concern us here: pure Python modules, + extension modules, and packages. + +pure Python module + a module written in Python and contained in a single :file:`.py` file (and + possibly associated :file:`.pyc` files). Sometimes referred to as a + "pure module." + +extension module + a module written in the low-level language of the Python implementation: C/C++ + for Python, Java for Jython. Typically contained in a single dynamically + loadable pre-compiled file, e.g. a shared object (:file:`.so`) file for Python + extensions on Unix, a DLL (given the :file:`.pyd` extension) for Python + extensions on Windows, or a Java class file for Jython extensions. (Note that + currently, the Distutils only handles C/C++ extensions for Python.) + +package + a module that contains other modules; typically contained in a directory in the + filesystem and distinguished from other directories by the presence of a file + :file:`__init__.py`. + +root package + the root of the hierarchy of packages. (This isn't really a package, since it + doesn't have an :file:`__init__.py` file. But we have to call it something.) + The vast majority of the standard library is in the root package, as are many + small, standalone third-party modules that don't belong to a larger module + collection. Unlike regular packages, modules in the root package can be found in + many directories: in fact, every directory listed in ``sys.path`` contributes + modules to the root package. + + +.. _distutils-term: + +Distutils-specific terminology +============================== + +The following terms apply more specifically to the domain of distributing Python +modules using the Distutils: + +module distribution + a collection of Python modules distributed together as a single downloadable + resource and meant to be installed *en masse*. Examples of some well-known + module distributions are NumPy, SciPy, Pillow, + or mxBase. (This would be called a *package*, except that term is + already taken in the Python context: a single module distribution may contain + zero, one, or many Python packages.) + +pure module distribution + a module distribution that contains only pure Python modules and packages. + Sometimes referred to as a "pure distribution." + +non-pure module distribution + a module distribution that contains at least one extension module. Sometimes + referred to as a "non-pure distribution." + +distribution root + the top-level directory of your source tree (or source distribution); the + directory where :file:`setup.py` exists. Generally :file:`setup.py` will be + run from this directory. diff --git a/docs/deprecated/distutils/packageindex.rst b/docs/deprecated/distutils/packageindex.rst new file mode 100644 index 00000000..ccb9a598 --- /dev/null +++ b/docs/deprecated/distutils/packageindex.rst @@ -0,0 +1,16 @@ +:orphan: + +.. _package-index: + +******************************* +The Python Package Index (PyPI) +******************************* + +The `Python Package Index (PyPI)`_ stores metadata describing distributions +packaged with distutils and other publishing tools, as well the distribution +archives themselves. + +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. + +.. _Python Package Index (PyPI): https://pypi.org diff --git a/docs/deprecated/distutils/setupscript.rst b/docs/deprecated/distutils/setupscript.rst new file mode 100644 index 00000000..f49c4f89 --- /dev/null +++ b/docs/deprecated/distutils/setupscript.rst @@ -0,0 +1,715 @@ +.. _setup-script: + +************************ +Writing the Setup Script +************************ + +.. include:: ./_setuptools_disclaimer.rst + +The setup script is the centre of all activity in building, distributing, and +installing modules using the Distutils. The main purpose of the setup script is +to describe your module distribution to the Distutils, so that the various +commands that operate on your modules do the right thing. As we saw in section +:ref:`distutils-simple-example` above, the setup script consists mainly of a call to :func:`~distutils.core.setup`, and most information +supplied to the Distutils by the module developer is supplied as keyword +arguments to :func:`~distutils.core.setup`. + +Here's a slightly more involved example, which we'll follow for the next couple +of sections: the Distutils' own setup script. (Keep in mind that although the +Distutils are included with Python 1.6 and later, they also have an independent +existence so that Python 1.5.2 users can use them to install other module +distributions. The Distutils' own setup script, shown here, is used to install +the package into Python 1.5.2.) :: + + #!/usr/bin/env python + + from distutils.core import setup + + setup(name='Distutils', + version='1.0', + description='Python Distribution Utilities', + author='Greg Ward', + author_email='gward@python.net', + url='https://www.python.org/sigs/distutils-sig/', + packages=['distutils', 'distutils.command'], + ) + +There are only two differences between this and the trivial one-file +distribution presented in section :ref:`distutils-simple-example`: more metadata, and the +specification of pure Python modules by package, rather than by module. This is +important since the Distutils consist of a couple of dozen modules split into +(so far) two packages; an explicit list of every module would be tedious to +generate and difficult to maintain. For more information on the additional +meta-data, see section :ref:`meta-data`. + +Note that any pathnames (files or directories) supplied in the setup script +should be written using the Unix convention, i.e. slash-separated. The +Distutils will take care of converting this platform-neutral representation into +whatever is appropriate on your current platform before actually using the +pathname. This makes your setup script portable across operating systems, which +of course is one of the major goals of the Distutils. In this spirit, all +pathnames in this document are slash-separated. + +This, of course, only applies to pathnames given to Distutils functions. If +you, for example, use standard Python functions such as :func:`glob.glob` or +:func:`os.listdir` to specify files, you should be careful to write portable +code instead of hardcoding path separators:: + + glob.glob(os.path.join('mydir', 'subdir', '*.html')) + os.listdir(os.path.join('mydir', 'subdir')) + + +.. _listing-packages: + +Listing whole packages +====================== + +The ``packages`` option tells the Distutils to process (build, distribute, +install, etc.) all pure Python modules found in each package mentioned in the +``packages`` list. In order to do this, of course, there has to be a +correspondence between package names and directories in the filesystem. The +default correspondence is the most obvious one, i.e. package :mod:`distutils` is +found in the directory :file:`distutils` relative to the distribution root. +Thus, when you say ``packages = ['foo']`` in your setup script, you are +promising that the Distutils will find a file :file:`foo/__init__.py` (which +might be spelled differently on your system, but you get the idea) relative to +the directory where your setup script lives. If you break this promise, the +Distutils will issue a warning but still process the broken package anyway. + +If you use a different convention to lay out your source directory, that's no +problem: you just have to supply the ``package_dir`` option to tell the +Distutils about your convention. For example, say you keep all Python source +under :file:`lib`, so that modules in the "root package" (i.e., not in any +package at all) are in :file:`lib`, modules in the ``foo`` package are in +:file:`lib/foo`, and so forth. Then you would put :: + + package_dir = {'': 'lib'} + +in your setup script. The keys to this dictionary are package names, and an +empty package name stands for the root package. The values are directory names +relative to your distribution root. In this case, when you say ``packages = +['foo']``, you are promising that the file :file:`lib/foo/__init__.py` exists. + +Another possible convention is to put the ``foo`` package right in +:file:`lib`, the ``foo.bar`` package in :file:`lib/bar`, etc. This would be +written in the setup script as :: + + package_dir = {'foo': 'lib'} + +A ``package: dir`` entry in the ``package_dir`` dictionary implicitly +applies to all packages below *package*, so the ``foo.bar`` case is +automatically handled here. In this example, having ``packages = ['foo', +'foo.bar']`` tells the Distutils to look for :file:`lib/__init__.py` and +:file:`lib/bar/__init__.py`. (Keep in mind that although ``package_dir`` +applies recursively, you must explicitly list all packages in +``packages``: the Distutils will *not* recursively scan your source tree +looking for any directory with an :file:`__init__.py` file.) + + +.. _listing-modules: + +Listing individual modules +========================== + +For a small module distribution, you might prefer to list all modules rather +than listing packages---especially the case of a single module that goes in the +"root package" (i.e., no package at all). This simplest case was shown in +section :ref:`distutils-simple-example`; here is a slightly more involved example:: + + py_modules = ['mod1', 'pkg.mod2'] + +This describes two modules, one of them in the "root" package, the other in the +``pkg`` package. Again, the default package/directory layout implies that +these two modules can be found in :file:`mod1.py` and :file:`pkg/mod2.py`, and +that :file:`pkg/__init__.py` exists as well. And again, you can override the +package/directory correspondence using the ``package_dir`` option. + + +.. _describing-extensions: + +Describing extension modules +============================ + +Just as writing Python extension modules is a bit more complicated than writing +pure Python modules, describing them to the Distutils is a bit more complicated. +Unlike pure modules, it's not enough just to list modules or packages and expect +the Distutils to go out and find the right files; you have to specify the +extension name, source file(s), and any compile/link requirements (include +directories, libraries to link with, etc.). + +.. XXX read over this section + +All of this is done through another keyword argument to +:func:`~distutils.core.setup`, the +``ext_modules`` option. ``ext_modules`` is just a list of +:class:`~distutils.core.Extension` instances, each of which describes a +single extension module. +Suppose your distribution includes a single extension, called ``foo`` and +implemented by :file:`foo.c`. If no additional instructions to the +compiler/linker are needed, describing this extension is quite simple:: + + Extension('foo', ['foo.c']) + +The :class:`~distutils.extension.Extension` class can be imported from :mod:`distutils.core` along +with :func:`~distutils.core.setup`. Thus, the setup script for a module distribution that +contains only this one extension and nothing else might be:: + + from distutils.core import setup, Extension + setup(name='foo', + version='1.0', + ext_modules=[Extension('foo', ['foo.c'])], + ) + +The :class:`~distutils.extension.Extension` class (actually, the underlying extension-building +machinery implemented by the :command:`build_ext` command) supports a great deal +of flexibility in describing Python extensions, which is explained in the +following sections. + + +Extension names and packages +---------------------------- + +The first argument to the :class:`~distutils.core.Extension` constructor is +always the name of the extension, including any package names. For example, :: + + Extension('foo', ['src/foo1.c', 'src/foo2.c']) + +describes an extension that lives in the root package, while :: + + Extension('pkg.foo', ['src/foo1.c', 'src/foo2.c']) + +describes the same extension in the ``pkg`` package. The source files and +resulting object code are identical in both cases; the only difference is where +in the filesystem (and therefore where in Python's namespace hierarchy) the +resulting extension lives. + +If you have a number of extensions all in the same package (or all under the +same base package), use the ``ext_package`` keyword argument to +:func:`~distutils.core.setup`. For example, :: + + setup(..., + ext_package='pkg', + ext_modules=[Extension('foo', ['foo.c']), + Extension('subpkg.bar', ['bar.c'])], + ) + +will compile :file:`foo.c` to the extension ``pkg.foo``, and +:file:`bar.c` to ``pkg.subpkg.bar``. + + +Extension source files +---------------------- + +The second argument to the :class:`~distutils.core.Extension` constructor is +a list of source +files. Since the Distutils currently only support C, C++, and Objective-C +extensions, these are normally C/C++/Objective-C source files. (Be sure to use +appropriate extensions to distinguish C++ source files: :file:`.cc` and +:file:`.cpp` seem to be recognized by both Unix and Windows compilers.) + +However, you can also include SWIG interface (:file:`.i`) files in the list; the +:command:`build_ext` command knows how to deal with SWIG extensions: it will run +SWIG on the interface file and compile the resulting C/C++ file into your +extension. + +.. XXX SWIG support is rough around the edges and largely untested! + +This warning notwithstanding, options to SWIG can be currently passed like +this:: + + setup(..., + ext_modules=[Extension('_foo', ['foo.i'], + swig_opts=['-modern', '-I../include'])], + py_modules=['foo'], + ) + +Or on the commandline like this:: + + > python setup.py build_ext --swig-opts="-modern -I../include" + +On some platforms, you can include non-source files that are processed by the +compiler and included in your extension. Currently, this just means Windows +message text (:file:`.mc`) files and resource definition (:file:`.rc`) files for +Visual C++. These will be compiled to binary resource (:file:`.res`) files and +linked into the executable. + + +Preprocessor options +-------------------- + +Three optional arguments to :class:`~distutils.core.Extension` will help if +you need to specify include directories to search or preprocessor macros to +define/undefine: ``include_dirs``, ``define_macros``, and ``undef_macros``. + +For example, if your extension requires header files in the :file:`include` +directory under your distribution root, use the ``include_dirs`` option:: + + Extension('foo', ['foo.c'], include_dirs=['include']) + +You can specify absolute directories there; if you know that your extension will +only be built on Unix systems with X11R6 installed to :file:`/usr`, you can get +away with :: + + Extension('foo', ['foo.c'], include_dirs=['/usr/include/X11']) + +You should avoid this sort of non-portable usage if you plan to distribute your +code: it's probably better to write C code like :: + + #include <X11/Xlib.h> + +If you need to include header files from some other Python extension, you can +take advantage of the fact that header files are installed in a consistent way +by the Distutils :command:`install_headers` command. For example, the Numerical +Python header files are installed (on a standard Unix installation) to +:file:`/usr/local/include/python1.5/Numerical`. (The exact location will differ +according to your platform and Python installation.) Since the Python include +directory---\ :file:`/usr/local/include/python1.5` in this case---is always +included in the search path when building Python extensions, the best approach +is to write C code like :: + + #include <Numerical/arrayobject.h> + +If you must put the :file:`Numerical` include directory right into your header +search path, though, you can find that directory using the Distutils +:mod:`distutils.sysconfig` module:: + + from distutils.sysconfig import get_python_inc + incdir = os.path.join(get_python_inc(plat_specific=1), 'Numerical') + setup(..., + Extension(..., include_dirs=[incdir]), + ) + +Even though this is quite portable---it will work on any Python installation, +regardless of platform---it's probably easier to just write your C code in the +sensible way. + +You can define and undefine pre-processor macros with the ``define_macros`` and +``undef_macros`` options. ``define_macros`` takes a list of ``(name, value)`` +tuples, where ``name`` is the name of the macro to define (a string) and +``value`` is its value: either a string or ``None``. (Defining a macro ``FOO`` +to ``None`` is the equivalent of a bare ``#define FOO`` in your C source: with +most compilers, this sets ``FOO`` to the string ``1``.) ``undef_macros`` is +just a list of macros to undefine. + +For example:: + + Extension(..., + define_macros=[('NDEBUG', '1'), + ('HAVE_STRFTIME', None)], + undef_macros=['HAVE_FOO', 'HAVE_BAR']) + +is the equivalent of having this at the top of every C source file:: + + #define NDEBUG 1 + #define HAVE_STRFTIME + #undef HAVE_FOO + #undef HAVE_BAR + + +Library options +--------------- + +You can also specify the libraries to link against when building your extension, +and the directories to search for those libraries. The ``libraries`` option is +a list of libraries to link against, ``library_dirs`` is a list of directories +to search for libraries at link-time, and ``runtime_library_dirs`` is a list of +directories to search for shared (dynamically loaded) libraries at run-time. + +For example, if you need to link against libraries known to be in the standard +library search path on target systems :: + + Extension(..., + libraries=['gdbm', 'readline']) + +If you need to link with libraries in a non-standard location, you'll have to +include the location in ``library_dirs``:: + + Extension(..., + library_dirs=['/usr/X11R6/lib'], + libraries=['X11', 'Xt']) + +(Again, this sort of non-portable construct should be avoided if you intend to +distribute your code.) + +.. XXX Should mention clib libraries here or somewhere else! + + +Other options +------------- + +There are still some other options which can be used to handle special cases. + +The ``optional`` option is a boolean; if it is true, +a build failure in the extension will not abort the build process, but +instead simply not install the failing extension. + +The ``extra_objects`` option is a list of object files to be passed to the +linker. These files must not have extensions, as the default extension for the +compiler is used. + +``extra_compile_args`` and ``extra_link_args`` can be used to +specify additional command line options for the respective compiler and linker +command lines. + +``export_symbols`` is only useful on Windows. It can contain a list of +symbols (functions or variables) to be exported. This option is not needed when +building compiled extensions: Distutils will automatically add ``initmodule`` +to the list of exported symbols. + +The ``depends`` option is a list of files that the extension depends on +(for example header files). The build command will call the compiler on the +sources to rebuild extension if any on this files has been modified since the +previous build. + +Relationships between Distributions and Packages +================================================ + +A distribution may relate to packages in three specific ways: + +#. It can require packages or modules. + +#. It can provide packages or modules. + +#. It can obsolete packages or modules. + +These relationships can be specified using keyword arguments to the +:func:`distutils.core.setup` function. + +Dependencies on other Python modules and packages can be specified by supplying +the *requires* keyword argument to :func:`~distutils.core.setup`. The +value must be a list of +strings. Each string specifies a package that is required, and optionally what +versions are sufficient. + +To specify that any version of a module or package is required, the string +should consist entirely of the module or package name. Examples include +``'mymodule'`` and ``'xml.parsers.expat'``. + +If specific versions are required, a sequence of qualifiers can be supplied in +parentheses. Each qualifier may consist of a comparison operator and a version +number. The accepted comparison operators are:: + + < > == + <= >= != + +These can be combined by using multiple qualifiers separated by commas (and +optional whitespace). In this case, all of the qualifiers must be matched; a +logical AND is used to combine the evaluations. + +Let's look at a bunch of examples: + ++-------------------------+----------------------------------------------+ +| Requires Expression | Explanation | ++=========================+==============================================+ +| ``==1.0`` | Only version ``1.0`` is compatible | ++-------------------------+----------------------------------------------+ +| ``>1.0, !=1.5.1, <2.0`` | Any version after ``1.0`` and before ``2.0`` | +| | is compatible, except ``1.5.1`` | ++-------------------------+----------------------------------------------+ + +Now that we can specify dependencies, we also need to be able to specify what we +provide that other distributions can require. This is done using the *provides* +keyword argument to :func:`~distutils.core.setup`. The value for this keyword is a list of +strings, each of which names a Python module or package, and optionally +identifies the version. If the version is not specified, it is assumed to match +that of the distribution. + +Some examples: + ++---------------------+----------------------------------------------+ +| Provides Expression | Explanation | ++=====================+==============================================+ +| ``mypkg`` | Provide ``mypkg``, using the distribution | +| | version | ++---------------------+----------------------------------------------+ +| ``mypkg (1.1)`` | Provide ``mypkg`` version 1.1, regardless of | +| | the distribution version | ++---------------------+----------------------------------------------+ + +A package can declare that it obsoletes other packages using the *obsoletes* +keyword argument. The value for this is similar to that of the *requires* +keyword: a list of strings giving module or package specifiers. Each specifier +consists of a module or package name optionally followed by one or more version +qualifiers. Version qualifiers are given in parentheses after the module or +package name. + +The versions identified by the qualifiers are those that are obsoleted by the +distribution being described. If no qualifiers are given, all versions of the +named module or package are understood to be obsoleted. + +.. _distutils-installing-scripts: + +Installing Scripts +================== + +So far we have been dealing with pure and non-pure Python modules, which are +usually not run by themselves but imported by scripts. + +Scripts are files containing Python source code, intended to be started from the +command line. Scripts don't require Distutils to do anything very complicated. +The only clever feature is that if the first line of the script starts with +``#!`` and contains the word "python", the Distutils will adjust the first line +to refer to the current interpreter location. By default, it is replaced with +the current interpreter location. The :option:`!--executable` (or :option:`!-e`) +option will allow the interpreter path to be explicitly overridden. + +The ``scripts`` option simply is a list of files to be handled in this +way. From the PyXML setup script:: + + setup(..., + scripts=['scripts/xmlproc_parse', 'scripts/xmlproc_val'] + ) + +.. versionchanged:: 3.1 + All the scripts will also be added to the ``MANIFEST`` file if no template is + provided. See :ref:`manifest`. + + +.. _distutils-installing-package-data: + +Installing Package Data +======================= + +Often, additional files need to be installed into a package. These files are +often data that's closely related to the package's implementation, or text files +containing documentation that might be of interest to programmers using the +package. These files are called :dfn:`package data`. + +Package data can be added to packages using the ``package_data`` keyword +argument to the :func:`~distutils.core.setup` function. The value must be a mapping from +package name to a list of relative path names that should be copied into the +package. The paths are interpreted as relative to the directory containing the +package (information from the ``package_dir`` mapping is used if appropriate); +that is, the files are expected to be part of the package in the source +directories. They may contain glob patterns as well. + +The path names may contain directory portions; any necessary directories will be +created in the installation. + +For example, if a package should contain a subdirectory with several data files, +the files can be arranged like this in the source tree:: + + setup.py + src/ + mypkg/ + __init__.py + module.py + data/ + tables.dat + spoons.dat + forks.dat + +The corresponding call to :func:`~distutils.core.setup` might be:: + + setup(..., + packages=['mypkg'], + package_dir={'mypkg': 'src/mypkg'}, + package_data={'mypkg': ['data/*.dat']}, + ) + + +.. versionchanged:: 3.1 + All the files that match ``package_data`` will be added to the ``MANIFEST`` + file if no template is provided. See :ref:`manifest`. + + +.. _distutils-additional-files: + +Installing Additional Files +=========================== + +The ``data_files`` option can be used to specify additional files needed +by the module distribution: configuration files, message catalogs, data files, +anything which doesn't fit in the previous categories. + +``data_files`` specifies a sequence of (*directory*, *files*) pairs in the +following way:: + + setup(..., + data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']), + ('config', ['cfg/data.cfg'])], + ) + +Each (*directory*, *files*) pair in the sequence specifies the installation +directory and the files to install there. + +Each file name in *files* is interpreted relative to the :file:`setup.py` +script at the top of the package source distribution. Note that you can +specify the directory where the data files will be installed, but you cannot +rename the data files themselves. + +The *directory* should be a relative path. It is interpreted relative to the +installation prefix (Python's ``sys.prefix`` for system installations; +``site.USER_BASE`` for user installations). Distutils allows *directory* to be +an absolute installation path, but this is discouraged since it is +incompatible with the wheel packaging format. No directory information from +*files* is used to determine the final location of the installed file; only +the name of the file is used. + +You can specify the ``data_files`` options as a simple sequence of files +without specifying a target directory, but this is not recommended, and the +:command:`install` command will print a warning in this case. To install data +files directly in the target directory, an empty string should be given as the +directory. + +.. versionchanged:: 3.1 + All the files that match ``data_files`` will be added to the ``MANIFEST`` + file if no template is provided. See :ref:`manifest`. + + +.. _meta-data: + +Additional meta-data +==================== + +The setup script may include additional meta-data beyond the name and version. +This information includes: + ++----------------------+---------------------------+-----------------+--------+ +| Meta-Data | Description | Value | Notes | ++======================+===========================+=================+========+ +| ``name`` | name of the package | short string | \(1) | ++----------------------+---------------------------+-----------------+--------+ +| ``version`` | version of this release | short string | (1)(2) | ++----------------------+---------------------------+-----------------+--------+ +| ``author`` | package author's name | short string | \(3) | ++----------------------+---------------------------+-----------------+--------+ +| ``author_email`` | email address of the | email address | \(3) | +| | package author | | | ++----------------------+---------------------------+-----------------+--------+ +| ``maintainer`` | package maintainer's name | short string | \(3) | ++----------------------+---------------------------+-----------------+--------+ +| ``maintainer_email`` | email address of the | email address | \(3) | +| | package maintainer | | | ++----------------------+---------------------------+-----------------+--------+ +| ``url`` | home page for the package | URL | \(1) | ++----------------------+---------------------------+-----------------+--------+ +| ``description`` | short, summary | short string | | +| | description of the | | | +| | package | | | ++----------------------+---------------------------+-----------------+--------+ +| ``long_description`` | longer description of the | long string | \(4) | +| | package | | | ++----------------------+---------------------------+-----------------+--------+ +| ``download_url`` | location where the | URL | | +| | package may be downloaded | | | ++----------------------+---------------------------+-----------------+--------+ +| ``classifiers`` | a list of classifiers | list of strings | (6)(7) | ++----------------------+---------------------------+-----------------+--------+ +| ``platforms`` | a list of platforms | list of strings | (6)(8) | ++----------------------+---------------------------+-----------------+--------+ +| ``keywords`` | a list of keywords | list of strings | (6)(8) | ++----------------------+---------------------------+-----------------+--------+ +| ``license`` | license for the package | short string | \(5) | ++----------------------+---------------------------+-----------------+--------+ + +Notes: + +(1) + These fields are required. + +(2) + It is recommended that versions take the form *major.minor[.patch[.sub]]*. + +(3) + Either the author or the maintainer must be identified. If maintainer is + provided, distutils lists it as the author in :file:`PKG-INFO`. + +(4) + The ``long_description`` field is used by PyPI when you publish a package, + to build its project page. + +(5) + The ``license`` field is a text indicating the license covering the + package where the license is not a selection from the "License" Trove + classifiers. See the ``Classifier`` field. Notice that + there's a ``licence`` distribution option which is deprecated but still + acts as an alias for ``license``. + +(6) + This field must be a list. + +(7) + The valid classifiers are listed on + `PyPI <https://pypi.org/classifiers>`_. + +(8) + To preserve backward compatibility, this field also accepts a string. If + you pass a comma-separated string ``'foo, bar'``, it will be converted to + ``['foo', 'bar']``, Otherwise, it will be converted to a list of one + string. + +'short string' + A single line of text, not more than 200 characters. + +'long string' + Multiple lines of plain text in reStructuredText format (see + http://docutils.sourceforge.net/). + +'list of strings' + See below. + +Encoding the version information is an art in itself. Python packages generally +adhere to the version format *major.minor[.patch][sub]*. The major number is 0 +for initial, experimental releases of software. It is incremented for releases +that represent major milestones in a package. The minor number is incremented +when important new features are added to the package. The patch number +increments when bug-fix releases are made. Additional trailing version +information is sometimes used to indicate sub-releases. These are +"a1,a2,...,aN" (for alpha releases, where functionality and API may change), +"b1,b2,...,bN" (for beta releases, which only fix bugs) and "pr1,pr2,...,prN" +(for final pre-release release testing). Some examples: + +0.1.0 + the first, experimental release of a package + +1.0.1a2 + the second alpha release of the first patch version of 1.0 + +``classifiers`` must be specified in a list:: + + setup(..., + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Environment :: Web Environment', + 'Intended Audience :: End Users/Desktop', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Python Software Foundation License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Programming Language :: Python', + 'Topic :: Communications :: Email', + 'Topic :: Office/Business', + 'Topic :: Software Development :: Bug Tracking', + ], + ) + +.. versionchanged:: 3.7 + :class:`~distutils.core.setup` now warns when ``classifiers``, ``keywords`` + or ``platforms`` fields are not specified as a list or a string. + +.. _debug-setup-script: + +Debugging the setup script +========================== + +Sometimes things go wrong, and the setup script doesn't do what the developer +wants. + +Distutils catches any exceptions when running the setup script, and print a +simple error message before the script is terminated. The motivation for this +behaviour is to not confuse administrators who don't know much about Python and +are trying to install a package. If they get a big long traceback from deep +inside the guts of Distutils, they may think the package or the Python +installation is broken because they don't read all the way down to the bottom +and see that it's a permission problem. + +On the other hand, this doesn't help the developer to find the cause of the +failure. For this purpose, the :envvar:`DISTUTILS_DEBUG` environment variable can be set +to anything except an empty string, and distutils will now print detailed +information about what it is doing, dump the full traceback when an exception +occurs, and print the whole command line when an external program (like a C +compiler) fails. diff --git a/docs/deprecated/distutils/sourcedist.rst b/docs/deprecated/distutils/sourcedist.rst new file mode 100644 index 00000000..0600663d --- /dev/null +++ b/docs/deprecated/distutils/sourcedist.rst @@ -0,0 +1,242 @@ +.. _source-dist: + +****************************** +Creating a Source Distribution +****************************** + +.. include:: ./_setuptools_disclaimer.rst + +As shown in section :ref:`distutils-simple-example`, you use the :command:`sdist` command +to create a source distribution. In the simplest case, :: + + python setup.py sdist + +(assuming you haven't specified any :command:`sdist` options in the setup script +or config file), :command:`sdist` creates the archive of the default format for +the current platform. The default format is a gzip'ed tar file +(:file:`.tar.gz`) on Unix, and ZIP file on Windows. + +You can specify as many formats as you like using the :option:`!--formats` +option, for example:: + + python setup.py sdist --formats=gztar,zip + +to create a gzipped tarball and a zip file. The available formats are: + ++-----------+-------------------------+---------+ +| Format | Description | Notes | ++===========+=========================+=========+ +| ``zip`` | zip file (:file:`.zip`) | (1),(3) | ++-----------+-------------------------+---------+ +| ``gztar`` | gzip'ed tar file | \(2) | +| | (:file:`.tar.gz`) | | ++-----------+-------------------------+---------+ +| ``bztar`` | bzip2'ed tar file | | +| | (:file:`.tar.bz2`) | | ++-----------+-------------------------+---------+ +| ``xztar`` | xz'ed tar file | | +| | (:file:`.tar.xz`) | | ++-----------+-------------------------+---------+ +| ``ztar`` | compressed tar file | \(4) | +| | (:file:`.tar.Z`) | | ++-----------+-------------------------+---------+ +| ``tar`` | tar file (:file:`.tar`) | | ++-----------+-------------------------+---------+ + +.. versionchanged:: 3.5 + Added support for the ``xztar`` format. + +Notes: + +(1) + default on Windows + +(2) + default on Unix + +(3) + requires either external :program:`zip` utility or :mod:`zipfile` module (part + of the standard Python library since Python 1.6) + +(4) + requires the :program:`compress` program. Notice that this format is now + pending for deprecation and will be removed in the future versions of Python. + +When using any ``tar`` format (``gztar``, ``bztar``, ``xztar``, ``ztar`` or +``tar``), under Unix you can specify the ``owner`` and ``group`` names +that will be set for each member of the archive. + +For example, if you want all files of the archive to be owned by root:: + + python setup.py sdist --owner=root --group=root + + +.. _manifest: + +Specifying the files to distribute +================================== + +If you don't supply an explicit list of files (or instructions on how to +generate one), the :command:`sdist` command puts a minimal default set into the +source distribution: + +* all Python source files implied by the ``py_modules`` and + ``packages`` options + +* all C source files mentioned in the ``ext_modules`` or + ``libraries`` options + + .. XXX getting C library sources currently broken---no + :meth:`get_source_files` method in :file:`build_clib.py`! + +* scripts identified by the ``scripts`` option + See :ref:`distutils-installing-scripts`. + +* anything that looks like a test script: :file:`test/test\*.py` (currently, the + Distutils don't do anything with test scripts except include them in source + distributions, but in the future there will be a standard for testing Python + module distributions) + +* Any of the standard README files (:file:`README`, :file:`README.txt`, + or :file:`README.rst`), :file:`setup.py` (or whatever you called your setup + script), and :file:`setup.cfg`. + +* all files that matches the ``package_data`` metadata. + See :ref:`distutils-installing-package-data`. + +* all files that matches the ``data_files`` metadata. + See :ref:`distutils-additional-files`. + +Sometimes this is enough, but usually you will want to specify additional files +to distribute. The typical way to do this is to write a *manifest template*, +called :file:`MANIFEST.in` by default. The manifest template is just a list of +instructions for how to generate your manifest file, :file:`MANIFEST`, which is +the exact list of files to include in your source distribution. The +:command:`sdist` command processes this template and generates a manifest based +on its instructions and what it finds in the filesystem. + +If you prefer to roll your own manifest file, the format is simple: one filename +per line, regular files (or symlinks to them) only. If you do supply your own +:file:`MANIFEST`, you must specify everything: the default set of files +described above does not apply in this case. + +.. versionchanged:: 3.1 + An existing generated :file:`MANIFEST` will be regenerated without + :command:`sdist` comparing its modification time to the one of + :file:`MANIFEST.in` or :file:`setup.py`. + +.. versionchanged:: 3.1.3 + :file:`MANIFEST` files start with a comment indicating they are generated. + Files without this comment are not overwritten or removed. + +.. versionchanged:: 3.2.2 + :command:`sdist` will read a :file:`MANIFEST` file if no :file:`MANIFEST.in` + exists, like it used to do. + +.. versionchanged:: 3.7 + :file:`README.rst` is now included in the list of distutils standard READMEs. + + +The manifest template has one command per line, where each command specifies a +set of files to include or exclude from the source distribution. For an +example, again we turn to the Distutils' own manifest template: + +.. code-block:: none + + include *.txt + recursive-include examples *.txt *.py + prune examples/sample?/build + +The meanings should be fairly clear: include all files in the distribution root +matching :file:`\*.txt`, all files anywhere under the :file:`examples` directory +matching :file:`\*.txt` or :file:`\*.py`, and exclude all directories matching +:file:`examples/sample?/build`. All of this is done *after* the standard +include set, so you can exclude files from the standard set with explicit +instructions in the manifest template. (Or, you can use the +:option:`!--no-defaults` option to disable the standard set entirely.) There are +several other commands available in the manifest template mini-language; see +section :ref:`sdist-cmd`. + +The order of commands in the manifest template matters: initially, we have the +list of default files as described above, and each command in the template adds +to or removes from that list of files. Once we have fully processed the +manifest template, we remove files that should not be included in the source +distribution: + +* all files in the Distutils "build" tree (default :file:`build/`) + +* all files in directories named :file:`RCS`, :file:`CVS`, :file:`.svn`, + :file:`.hg`, :file:`.git`, :file:`.bzr` or :file:`_darcs` + +Now we have our complete list of files, which is written to the manifest for +future reference, and then used to build the source distribution archive(s). + +You can disable the default set of included files with the +:option:`!--no-defaults` option, and you can disable the standard exclude set +with :option:`!--no-prune`. + +Following the Distutils' own manifest template, let's trace how the +:command:`sdist` command builds the list of files to include in the Distutils +source distribution: + +#. include all Python source files in the :file:`distutils` and + :file:`distutils/command` subdirectories (because packages corresponding to + those two directories were mentioned in the ``packages`` option in the + setup script---see section :ref:`setup-script`) + +#. include :file:`README.txt`, :file:`setup.py`, and :file:`setup.cfg` (standard + files) + +#. include :file:`test/test\*.py` (standard files) + +#. include :file:`\*.txt` in the distribution root (this will find + :file:`README.txt` a second time, but such redundancies are weeded out later) + +#. include anything matching :file:`\*.txt` or :file:`\*.py` in the sub-tree + under :file:`examples`, + +#. exclude all files in the sub-trees starting at directories matching + :file:`examples/sample?/build`\ ---this may exclude files included by the + previous two steps, so it's important that the ``prune`` command in the manifest + template comes after the ``recursive-include`` command + +#. exclude the entire :file:`build` tree, and any :file:`RCS`, :file:`CVS`, + :file:`.svn`, :file:`.hg`, :file:`.git`, :file:`.bzr` and :file:`_darcs` + directories + +Just like in the setup script, file and directory names in the manifest template +should always be slash-separated; the Distutils will take care of converting +them to the standard representation on your platform. That way, the manifest +template is portable across operating systems. + + +.. _manifest-options: + +Manifest-related options +======================== + +The normal course of operations for the :command:`sdist` command is as follows: + +* if the manifest file (:file:`MANIFEST` by default) exists and the first line + does not have a comment indicating it is generated from :file:`MANIFEST.in`, + then it is used as is, unaltered + +* if the manifest file doesn't exist or has been previously automatically + generated, read :file:`MANIFEST.in` and create the manifest + +* if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest + with just the default file set + +* use the list of files now in :file:`MANIFEST` (either just generated or read + in) to create the source distribution archive(s) + +There are a couple of options that modify this behaviour. First, use the +:option:`!--no-defaults` and :option:`!--no-prune` to disable the standard +"include" and "exclude" sets. + +Second, you might just want to (re)generate the manifest, but not create a source +distribution:: + + python setup.py sdist --manifest-only + +:option:`!-o` is a shortcut for :option:`!--manifest-only`. diff --git a/docs/deprecated/distutils/uploading.rst b/docs/deprecated/distutils/uploading.rst new file mode 100644 index 00000000..4c391cab --- /dev/null +++ b/docs/deprecated/distutils/uploading.rst @@ -0,0 +1,8 @@ +:orphan: + +*************************************** +Uploading Packages to the Package Index +*************************************** + +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. diff --git a/docs/deprecated/easy_install.rst b/docs/deprecated/easy_install.rst index fac7b8fc..76c3f608 100644 --- a/docs/deprecated/easy_install.rst +++ b/docs/deprecated/easy_install.rst @@ -25,9 +25,6 @@ own packages work better with EasyInstall, or provide EasyInstall-like features without requiring your users to use EasyInstall directly, you'll probably want to check out the full documentation as well.) -.. contents:: **Table of Contents** - - Using "Easy Install" ==================== diff --git a/docs/deprecated/index.rst b/docs/deprecated/index.rst index ca80767a..ce2ac006 100644 --- a/docs/deprecated/index.rst +++ b/docs/deprecated/index.rst @@ -16,5 +16,6 @@ objectives. python3 python_eggs easy_install + distutils/index distutils-legacy functionalities diff --git a/docs/deprecated/python3.rst b/docs/deprecated/python3.rst deleted file mode 100644 index 6b55fe78..00000000 --- a/docs/deprecated/python3.rst +++ /dev/null @@ -1,94 +0,0 @@ -===================================================== -Supporting both Python 2 and Python 3 with Setuptools -===================================================== - -Starting with Distribute version 0.6.2 and Setuptools 0.7, the Setuptools -project supported Python 3. Installing and -using setuptools for Python 3 code works exactly the same as for Python 2 -code. - -Setuptools provides a facility to invoke 2to3 on the code as a part of the -build process, by setting the keyword parameter ``use_2to3`` to True, but -the Setuptools project strongly recommends instead developing a unified codebase -using `six <https://pypi.org/project/six/>`_, -`future <https://pypi.org/project/future/>`_, or another compatibility -library. - - -Using 2to3 -========== - -Setuptools attempts to make the porting process easier by automatically -running -2to3 as a part of running tests. To do so, you need to configure the -setup.py so that you can run the unit tests with ``python setup.py test``. - -See :ref:`test` for more information on this. - -Once you have the tests running under Python 2, you can add the use_2to3 -keyword parameters to setup(), and start running the tests under Python 3. -The test command will now first run the build command during which the code -will be converted with 2to3, and the tests will then be run from the build -directory, as opposed from the source directory as is normally done. - -Setuptools will convert all Python files, and also all doctests in Python -files. However, if you have doctests located in separate text files, these -will not automatically be converted. By adding them to the -``convert_2to3_doctests`` keyword parameter Setuptools will convert them as -well. - -By default, the conversion uses all fixers in the ``lib2to3.fixers`` package. -To use additional fixers, the parameter ``use_2to3_fixers`` can be set -to a list of names of packages containing fixers. To exclude fixers, the -parameter ``use_2to3_exclude_fixers`` can be set to fixer names to be -skipped. - -An example setup.py might look something like this:: - - from setuptools import setup - - setup( - name='your.module', - version='1.0', - description='This is your awesome module', - author='You', - author_email='your@email', - package_dir={'': 'src'}, - packages=['your', 'you.module'], - test_suite='your.module.tests', - use_2to3=True, - convert_2to3_doctests=['src/your/module/README.txt'], - use_2to3_fixers=['your.fixers'], - use_2to3_exclude_fixers=['lib2to3.fixes.fix_import'], - ) - -Differential conversion ------------------------ - -Note that a file will only be copied and converted during the build process -if the source file has been changed. If you add a file to the doctests -that should be converted, it will not be converted the next time you run -the tests, since it hasn't been modified. You need to remove it from the -build directory. Also if you run the build, install or test commands before -adding the use_2to3 parameter, you will have to remove the build directory -before you run the test command, as the files otherwise will seem updated, -and no conversion will happen. - -In general, if code doesn't seem to be converted, deleting the build directory -and trying again is a good safeguard against the build directory getting -"out of sync" with the source directory. - -Distributing Python 3 modules -============================= - -You can distribute your modules with Python 3 support in different ways. A -normal source distribution will work, but can be slow in installing, as the -2to3 process will be run during the install. But you can also distribute -the module in binary format, such as a binary egg. That egg will contain the -already converted code, and hence no 2to3 conversion is needed during install. - -Advanced features -================= - -If you don't want to run the 2to3 conversion on the doctests in Python files, -you can turn that off by setting ``setuptools.use_2to3_on_doctests = False``. diff --git a/docs/deprecated/python_eggs.rst b/docs/deprecated/python_eggs.rst index 6c0456de..59d1adc5 100644 --- a/docs/deprecated/python_eggs.rst +++ b/docs/deprecated/python_eggs.rst @@ -6,9 +6,6 @@ STOP! This is not the first document you should read! -.. contents:: **Table of Contents** - - ---------------------- Eggs and their Formats ---------------------- @@ -676,4 +673,3 @@ to facilitate introspection of installed scripts, and their relationship to installed eggs. For example, an uninstallation tool could use this data to identify what scripts can safely be removed, and/or identify what scripts would stop working if a particular egg is uninstalled. - diff --git a/docs/development/developer-guide.rst b/docs/development/developer-guide.rst index 13b07e7e..d6ed2c1f 100644 --- a/docs/development/developer-guide.rst +++ b/docs/development/developer-guide.rst @@ -5,9 +5,6 @@ Developer's Guide for Setuptools If you want to know more about contributing on Setuptools, this is the place. -.. contents:: **Table of Contents** - - ------------------- Recommended Reading ------------------- @@ -61,7 +58,7 @@ jump to the in-depth discussion about any subject referenced. Making a pull request --------------------- -When making a pull request, please +When making a pull request, please :ref:`include a short summary of the changes <Adding change notes with your PRs>` and a reference to any issue tickets that the PR is intended to solve. diff --git a/docs/history.rst b/docs/history.rst index faf7adfe..ce7e77ab 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -5,6 +5,8 @@ History ******* +.. towncrier-draft-entries:: DRAFT, unreleased as on |today| + .. include:: ../CHANGES (links).rst Credits diff --git a/docs/index.rst b/docs/index.rst index b9a79647..2833ab96 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,13 +1,12 @@ -Documentation -============= +setuptools +========== Setuptools is a fully-featured, actively-maintained, and stable library designed to facilitate packaging Python projects. -Documentation content: - .. toctree:: :maxdepth: 1 + :hidden: User guide <userguide/index> build_meta @@ -18,3 +17,5 @@ Documentation content: Development guide <development/index> Backward compatibility & deprecated practice <deprecated/index> Changelog <history> + +.. tidelift-referral-banner:: diff --git a/docs/pkg_resources.rst b/docs/pkg_resources.rst index 19d43244..fb5fc077 100644 --- a/docs/pkg_resources.rst +++ b/docs/pkg_resources.rst @@ -10,8 +10,12 @@ eggs, support for merging packages that have separately-distributed modules or subpackages, and APIs for managing Python's current "working set" of active packages. - -.. contents:: **Table of Contents** +Use of ``pkg_resources`` is discouraged in favor of +`importlib.resources <https://docs.python.org/3/library/importlib.html#module-importlib.resources>`_, +`importlib.metadata <https://docs.python.org/3/library/importlib.metadata.html>`_, +and their backports (`resources <https://pypi.org/project/importlib_resources>`_, +`metadata <https://pypi.org/project/importlib_metadata>`_). +Please consider using those libraries instead of pkg_resources. -------- @@ -1222,7 +1226,7 @@ Resource Extraction If you are implementing an ``IResourceProvider`` and/or ``IMetadataProvider`` for a new distribution archive format, you may need to use the following -``IResourceManager`` methods to co-ordinate extraction of resources to the +``IResourceManager`` methods to coordinate extraction of resources to the filesystem. If you're not implementing an archive format, however, you have no need to use these methods. Unlike the other methods listed above, they are *not* available as top-level functions tied to the global ``ResourceManager``; diff --git a/docs/python 2 sunset.rst b/docs/python 2 sunset.rst index f7b7ee25..225d6551 100644 --- a/docs/python 2 sunset.rst +++ b/docs/python 2 sunset.rst @@ -50,7 +50,7 @@ versions of Setuptools installed, here are some things to try. Then ``pip install setuptools``. 2. If possible, attempt to replicate the problem in a second environment (virtual machine, friend's computer, etc). If the issue is isolated to just - one unique enviornment, first determine what is different about those + one unique environment, first determine what is different about those environments (or reinstall/reset the failing one to defaults). 3. End users who are not themselves the maintainers for the package they are trying to install should contact the support channels for the relevant diff --git a/docs/references/keywords.rst b/docs/references/keywords.rst index 03ce9fa2..c26b9d49 100644 --- a/docs/references/keywords.rst +++ b/docs/references/keywords.rst @@ -76,6 +76,17 @@ Keywords ``license`` A string specifying the license of the package. +``license_file`` + + .. warning:: + ``license_file`` is deprecated. Use ``license_files`` instead. + +``license_files`` + + A list of glob patterns for license related files that should be included. + If neither ``license_file`` nor ``license_files`` is specified, this option + defaults to ``LICEN[CS]E*``, ``COPYING*``, ``NOTICE*``, and ``AUTHORS*``. + ``keywords`` A list of strings or a comma-separated string providing descriptive meta-data. See: `PEP 0314`_. @@ -319,21 +330,6 @@ Keywords mess with it. For more details on how this argument works, see the section below on :ref:`Automatic Resource Extraction`. -``use_2to3`` - Convert the source code from Python 2 to Python 3 with 2to3 during the - build process. See :doc:`../deprecated/python3` for more details. - -``convert_2to3_doctests`` - List of doctest source files that need to be converted with 2to3. - See :doc:`../deprecated/python3` for more details. - -``use_2to3_fixers`` - A list of modules to search for additional fixers to be used during - the 2to3 conversion. See :doc:`../deprecated/python3` for more details. - -``use_2to3_exclude_fixers`` - List of fixer names to be skipped. - ``project_urls`` An arbitrary map of URL names to hyperlinks, allowing more extensible documentation of where various resources can be found than the simple diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 104d68fa..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -# keep these in sync with setup.cfg -sphinx -jaraco.packaging>=6.1 -rst.linker>=1.9 -pygments-github-lexers==0.0.5 - -setuptools>=34 diff --git a/docs/setuptools.rst b/docs/setuptools.rst index 1000a0ce..c5a89adc 100644 --- a/docs/setuptools.rst +++ b/docs/setuptools.rst @@ -47,14 +47,11 @@ Feature Highlights: * Full support for PEP 420 via ``find_namespace_packages()``, which is also backwards compatible to the existing ``find_packages()`` for Python >= 3.3. -.. contents:: **Table of Contents** - - ----------------- Developer's Guide ----------------- - +The developer's guide has been updated. See the :doc:`most recent version <userguide/index>`. @@ -135,7 +132,7 @@ only a ``setuptools.setup()`` call. .. note:: :pep:`517` doesn't support editable installs so this is currently - incompatible with ``pip install -e .``, as :pep:`517` does not support editable installs. + incompatible with ``pip install -e .``. This means that you can have a Python project with all build configuration specified in ``setup.cfg``, without a ``setup.py`` file, if you **can rely @@ -157,7 +154,7 @@ To use this feature: ] build-backend = "setuptools.build_meta" -* Use a :pep:`517` compatible build frontend, such as ``pip >= 19`` or ``pep517``. +* Use a :pep:`517` compatible build frontend, such as ``pip >= 19`` or ``build``. .. warning:: diff --git a/docs/userguide/declarative_config.rst b/docs/userguide/declarative_config.rst index bc66869b..c50fcf6d 100644 --- a/docs/userguide/declarative_config.rst +++ b/docs/userguide/declarative_config.rst @@ -18,15 +18,7 @@ to the ``setup()`` function (declarative config). This approach not only allows automation scenarios but also reduces boilerplate code in some cases. -.. note:: - - This implementation has limited compatibility with the distutils2-like - ``setup.cfg`` sections used by the ``pbr`` and ``d2to1`` packages. - - Namely: only metadata-related keys from ``metadata`` section are supported - (except for ``description-file``); keys from ``files``, ``entry_points`` - and ``backwards_compat`` are not supported. - +.. _example-setup-config: .. code-block:: ini @@ -58,6 +50,10 @@ boilerplate code in some cases. * = *.txt, *.rst hello = *.msg + [options.entry_points] + console_scripts = + executable-name = package.module:function + [options.extras_require] pdf = ReportLab>=1.2; RXP rest = docutils>=0.3; pack ==1.1, ==1.3 @@ -72,6 +68,7 @@ boilerplate code in some cases. site.d/00_default.conf host.d/00_default.conf data = data/img/logo.png, data/svg/icon.svg + fonts = data/fonts/*.ttf, data/fonts/*.otf Metadata and options are set in the config sections of the same name. @@ -156,10 +153,9 @@ Special directives: * ``file:`` - Value is read from a list of files and then concatenated - -.. note:: - The ``file:`` directive is sandboxed and won't reach anything outside - the directory containing ``setup.py``. + .. note:: + The ``file:`` directive is sandboxed and won't reach anything outside + the directory containing ``setup.py``. Metadata @@ -169,11 +165,11 @@ Metadata The aliases given below are supported for compatibility reasons, but their use is not advised. -============================== ================= ================= =============== ===== +============================== ================= ================= =============== ========== Key Aliases Type Minimum Version Notes -============================== ================= ================= =============== ===== +============================== ================= ================= =============== ========== name str -version attr:, file:, str 39.2.0 (1) +version attr:, file:, str 39.2.0 [#meta-1]_ url home-page str download_url download-url str project_urls dict 38.3.0 @@ -183,8 +179,7 @@ maintainer str maintainer_email maintainer-email str classifiers classifier file:, list-comma license str -license_file str -license_files list-comma +license_files license_file list-comma 42.0.0 description summary file:, str long_description long-description file:, str long_description_content_type str 38.6.0 @@ -193,56 +188,102 @@ platforms platform list-comma provides list-comma requires list-comma obsoletes list-comma -============================== ================= ================= =============== ===== +============================== ================= ================= =============== ========== -.. note:: - A version loaded using the ``file:`` directive must comply with PEP 440. - It is easy to accidentally put something other than a valid version - string in such a file, so validation is stricter in this case. +**Notes**: + +.. [#meta-1] The ``version`` file attribute has only been supported since 39.2.0. + + A version loaded using the ``file:`` directive must comply with PEP 440. + It is easy to accidentally put something other than a valid version + string in such a file, so validation is stricter in this case. -Notes: -1. The ``version`` file attribute has only been supported since 39.2.0. Options ------- -======================= =================================== =============== ===== +======================= =================================== =============== ========= Key Type Minimum Version Notes -======================= =================================== =============== ===== +======================= =================================== =============== ========= zip_safe bool setup_requires list-semi install_requires list-semi -extras_require section +extras_require section [#opt-2]_ python_requires str -entry_points file:, section -use_2to3 bool -use_2to3_fixers list-comma -use_2to3_exclude_fixers list-comma -convert_2to3_doctests list-comma +entry_points file:, section 51.0.0 scripts list-comma eager_resources list-comma dependency_links list-comma tests_require list-semi include_package_data bool -packages find:, find_namespace:, list-comma +packages find:, find_namespace:, list-comma [#opt-3]_ package_dir dict -package_data section (1) +package_data section [#opt-1]_ exclude_package_data section namespace_packages list-comma py_modules list-comma data_files dict 40.6.0 -======================= =================================== =============== ===== - -.. note:: - - **packages** - The ``find:`` and ``find_namespace:`` directive can be further configured - in a dedicated subsection ``options.packages.find``. This subsection - accepts the same keys as the ``setuptools.find_packages`` and the - ``setuptools.find_namespace_packages`` function: - ``where``, ``include``, and ``exclude``. - - **find_namespace directive** - The ``find_namespace:`` directive is supported since Python >=3.3. - -Notes: -1. In the ``package_data`` section, a key named with a single asterisk (``*``) -refers to all packages, in lieu of the empty string used in ``setup.py``. +======================= =================================== =============== ========= + +**Notes**: + +.. [#opt-1] In the ``package_data`` section, a key named with a single asterisk + (``*``) refers to all packages, in lieu of the empty string used in ``setup.py``. + +.. [#opt-2] In the ``extras_require`` section, values are parsed as ``list-semi``. + This implies that in order to include markers, they **must** be *dangling*: + + .. code-block:: ini + + [options.extras_require] + rest = docutils>=0.3; pack ==1.1, ==1.3 + pdf = + ReportLab>=1.2 + RXP + importlib-metadata; python_version < "3.8" + +.. [#opt-3] The ``find:`` and ``find_namespace:`` directive can be further configured + in a dedicated subsection ``options.packages.find``. This subsection accepts the + same keys as the ``setuptools.find_packages`` and the + ``setuptools.find_namespace_packages`` function: + ``where``, ``include``, and ``exclude``. + + The ``find_namespace:`` directive is supported since Python >=3.3. + + +Compatibility with other tools +============================== + +Historically, several tools explored declarative package configuration +in parallel. And several of them chose to place the packaging +configuration within the project's :file:`setup.cfg` file. +One of the first was ``distutils2``, which development has stopped in +2013. Other include ``pbr`` which is still under active development or +``d2to1``, which was a plug-in that backports declarative configuration +to ``distutils``, but has had no release since Oct. 2015. +As a way to harmonize packaging tools, ``setuptools``, having held the +position of *de facto* standard, has gradually integrated those +features as part of its core features. + +Still this has lead to some confusion and feature incompatibilities: + +- some tools support features others don't; +- some have similar features but the declarative syntax differs; + +The table below tries to summarize the differences. But, please, refer +to each tool documentation for up-to-date information. + +=========================== ========== ========== ===== === +feature setuptools distutils2 d2to1 pbr +=========================== ========== ========== ===== === +[metadata] description-file S Y Y Y +[files] S Y Y Y +entry_points Y Y Y S +[backwards_compat] N Y Y Y +=========================== ========== ========== ===== === + +Y: supported, N: unsupported, S: syntax differs (see +:ref:`above example<example-setup-config>`). + +Also note that some features were only recently added to ``setuptools``. +Please refer to the previous sections to find out when. diff --git a/docs/userguide/dependency_management.rst b/docs/userguide/dependency_management.rst index 354a9f8c..23578a57 100644 --- a/docs/userguide/dependency_management.rst +++ b/docs/userguide/dependency_management.rst @@ -3,7 +3,7 @@ Dependencies Management in Setuptools ===================================== There are three types of dependency styles offered by setuptools: -1) build system requirement, required dependency and 3) optional +1) build system requirement, 2) required dependency and 3) optional dependency. .. Note:: @@ -11,16 +11,14 @@ dependency. version by following `PEP 440 <https://www.python.org/dev/peps/pep-0440/>`_ -.. contents:: - Build system requirement ======================== Package requirement ------------------- After organizing all the scripts and files and getting ready for packaging, -there needs to be a way to tell Python what programs it need to actually -do the packgaging (in our case, ``setuptools`` of course). Usually, +there needs to be a way to tell Python what programs it needs to actually +do the packaging (in our case, ``setuptools`` of course). Usually, you also need the ``wheel`` package as well since it is recommended that you upload a ``.whl`` file to PyPI alongside your ``.tar.gz`` file. Unlike the other two types of dependency keyword, this one is specified in your @@ -47,32 +45,36 @@ Declaring required dependency This is where a package declares its core dependencies, without which it won't be able to run. ``setuptools`` support automatically download and install these dependencies when the package is installed. Although there is more -finess to it, let's start with a simple example. +finesse to it, let's start with a simple example. -.. code-block:: ini +.. tab:: setup.cfg - [options] - #... - install_requires = - docutils - BazSpam ==1.1 + .. code-block:: ini -.. code-block:: python + [options] + #... + install_requires = + docutils + BazSpam ==1.1 - setup( - #..., - install_requires = [ - 'docutils', - 'BazSpam ==1.1' - ] - ) +.. tab:: setup.py + + .. code-block:: python + + setup( + ..., + install_requires=[ + 'docutils', + 'BazSpam ==1.1', + ], + ) When your project is installed (e.g. using pip), all of the dependencies not already installed will be located (via PyPI), downloaded, built (if necessary), and installed and 2) Any scripts in your project will be installed with wrappers that verify the availability of the specified dependencies at runtime. - + Platform specific dependencies ------------------------------ @@ -82,41 +84,50 @@ specific dependencies. For example, the ``enum`` package was added in Python 3.4, therefore, package that depends on it can elect to install it only when the Python version is older than 3.4. To accomplish this -.. code-block:: ini - - [options] - #... - install_requires = - enum34;python_version<'3.4' +.. tab:: setup.cfg -.. code-block:: python + .. code-block:: ini - setup( + [options] #... - install_requires=[ - "enum34;python_version<'3.4'",] - ) + install_requires = + enum34;python_version<'3.4' + +.. tab:: setup.py + + .. code-block:: python + + setup( + ..., + install_requires=[ + "enum34;python_version<'3.4'", + ], + ) Similarly, if you also wish to declare ``pywin32`` with a minimal version of 1.0 and only install it if the user is using a Windows operating system: -.. code-block:: ini - - [options] - #... - install_requires = - enum34;python_version<'3.4' - pywin32 >= 1.0;platform_system=='Windows' +.. tab:: setup.cfg -.. code-block:: python + .. code-block:: ini - setup( + [options] #... - install_requires=[ - "enum34;python_version<'3.4'", - "pywin32 >= 1.0;platform_system=='Windows'" - ] - ) + install_requires = + enum34;python_version<'3.4' + pywin32 >= 1.0;platform_system=='Windows' + +.. tab:: setup.py + + .. code-block:: python + + setup( + ..., + install_requires=[ + "enum34;python_version<'3.4'", + "pywin32 >= 1.0;platform_system=='Windows'", + ], + ) The environmental markers that may be used for testing platform types are detailed in `PEP 508 <https://www.python.org/dev/peps/pep-0508/>`_. @@ -181,20 +192,24 @@ The ``dependency_links`` option takes the form of a list of URL strings. For example, this will cause a search of the specified page for eggs or source distributions, if the package's dependencies aren't already installed: -.. code-block:: ini - - [options] - #... - dependency_links = http://peak.telecommunity.com/snapshots/ +.. tab:: setup.cfg -.. code-block:: python + .. code-block:: ini - setup( + [options] #... - dependency_links=[ - "http://peak.telecommunity.com/snapshots/" - ], - ) + dependency_links = http://peak.telecommunity.com/snapshots/ + +.. tab:: setup.py + + .. code-block:: python + + setup( + ..., + dependency_links=[ + "http://peak.telecommunity.com/snapshots/", + ], + ) Optional dependencies @@ -202,7 +217,7 @@ Optional dependencies Setuptools allows you to declare dependencies that only get installed under specific circumstances. These dependencies are specified with ``extras_require`` keyword and are only installed if another package depends on it (either -directly or indirectly) This makes it convenient to declare dependencies for +directly or indirectly) This makes it convenient to declare dependencies for ancillary functions such as "tests" and "docs". .. note:: @@ -211,82 +226,98 @@ ancillary functions such as "tests" and "docs". For example, Package-A offers optional PDF support and requires two other dependencies for it to work: -.. code-block:: ini +.. tab:: setup.cfg - [metadata] - name = Package-A + .. code-block:: ini - [options.extras_require] - PDF = ReportLab>=1.2; RXP + [metadata] + name = Package-A + [options.extras_require] + PDF = ReportLab>=1.2; RXP -.. code-block:: python - setup( - name="Project-A", - #... - extras_require={ - "PDF": ["ReportLab>=1.2", "RXP"], - } - ) +.. tab:: setup.py + + .. code-block:: python -The name ``PDF`` is an arbitary identifier of such a list of dependencies, to + setup( + name="Project-A", + ..., + extras_require={ + "PDF": ["ReportLab>=1.2", "RXP"], + }, + ) + +The name ``PDF`` is an arbitrary identifier of such a list of dependencies, to which other components can refer and have them installed. There are two common use cases. First is the console_scripts entry point: -.. code-block:: ini +.. tab:: setup.cfg - [metadata] - name = Project A - #... + .. code-block:: ini - [options] - #... - entry_points= - [console_scripts] - rst2pdf = project_a.tools.pdfgen [PDF] - rst2html = project_a.tools.htmlgen - -.. code-block:: python - - setup( - name = "Project-A" - #..., - entry_points={ - "console_scripts": [ - "rst2pdf = project_a.tools.pdfgen [PDF]", - "rst2html = project_a.tools.htmlgen", - ], - } - ) + [metadata] + name = Project A + #... -When the script ``rst2pdf`` is run, it will trigger the installation of -the two dependencies ``PDF`` maps to. + [options] + #... + entry_points= + [console_scripts] + rst2pdf = project_a.tools.pdfgen [PDF] + rst2html = project_a.tools.htmlgen + +.. tab:: setup.py + + .. code-block:: python + + setup( + name="Project-A", + ..., + entry_points={ + "console_scripts": [ + "rst2pdf = project_a.tools.pdfgen [PDF]", + "rst2html = project_a.tools.htmlgen", + ], + }, + ) + +This syntax indicates that the entry point (in this case a console script) +is only valid when the PDF extra is installed. It is up to the installer +to determine how to handle the situation where PDF was not indicated +(e.g. omit the console script, provide a warning when attempting to load +the entry point, assume the extras are present and let the implementation +fail later). The second use case is that other package can use this "extra" for their own dependencies. For example, if "Project-B" needs "project A" with PDF support installed, it might declare the dependency like this: -.. code-block:: ini +.. tab:: setup.cfg - [metadata] - name = Project-B - #... + .. code-block:: ini - [options] - #... - install_requires = - Project-A[PDF] + [metadata] + name = Project-B + #... + + [options] + #... + install_requires = + Project-A[PDF] + +.. tab:: setup.py -.. code-block:: python + .. code-block:: python - setup( - name="Project-B", - install_requires=["Project-A[PDF]"], - ... - ) + setup( + name="Project-B", + install_requires=["Project-A[PDF]"], + ..., + ) This will cause ReportLab to be installed along with project A, if project B is installed -- even if project A was already installed. In this way, a project @@ -310,4 +341,25 @@ In some cases, you might need to specify the minimum required python version. This is handled with the ``python_requires`` keyword supplied to ``setup.cfg`` or ``setup.py``. -Example WIP + +.. tab:: setup.cfg + + .. code-block:: ini + + [metadata] + name = Project-B + #... + + [options] + #... + python_requires = >=3.6 + +.. tab:: setup.py + + .. code-block:: python + + setup( + name="Project-B", + python_requires=[">=3.6"], + ..., + ) diff --git a/docs/userguide/development_mode.rst b/docs/userguide/development_mode.rst index bce724a7..90bc5676 100644 --- a/docs/userguide/development_mode.rst +++ b/docs/userguide/development_mode.rst @@ -3,9 +3,9 @@ Under normal circumstances, the ``distutils`` assume that you are going to build a distribution of your project, not use it in its "raw" or "unbuilt" -form. If you were to use the ``distutils`` that way, you would have to rebuild -and reinstall your project every time you made a change to it during -development. +form. However, if you were to use the ``distutils`` to build a distribution, +you would have to rebuild and reinstall your project every time you made a +change to it during development. Another problem that sometimes comes up with the ``distutils`` is that you may need to do development on two related projects at the same time. You may need @@ -28,14 +28,6 @@ Python's ``site-packages`` directory, it will also update the ``easy-install.pth`` file to include your project's source code, thereby making it available on ``sys.path`` for all programs using that Python installation. -If you have enabled the ``use_2to3`` flag, then of course the ``.egg-link`` -will not link directly to your source code when run under Python 3, since -that source code would be made for Python 2 and not work under Python 3. -Instead the ``setup.py develop`` will build Python 3 code under the ``build`` -directory, and link there. This means that after doing code changes you will -have to run ``setup.py build`` before these changes are picked up by your -Python 3 installation. - In addition, the ``develop`` command creates wrapper scripts in the target script directory that will run your in-development scripts after ensuring that all your ``install_requires`` packages are available on ``sys.path``. diff --git a/docs/userguide/distribution.rst b/docs/userguide/distribution.rst index 377f7bb4..2872dacd 100644 --- a/docs/userguide/distribution.rst +++ b/docs/userguide/distribution.rst @@ -25,7 +25,7 @@ the command line ahead of the ``sdist`` or ``bdist`` commands that you want to generate a daily build or snapshot for. See the section below on the :ref:`egg_info <egg_info>` command for more details. -(Also, before you release your project, be sure to see the section above on +(Also, before you release your project, be sure to see the section on :ref:`Specifying Your Project's Version` for more information about how pre- and post-release tags affect how version numbers are interpreted. This is important in order to make sure that dependency processing tools will know @@ -161,7 +161,10 @@ without Cython. Specifying Your Project's Version --------------------------------- -Setuptools can work well with most versioning schemes; there are, however, a +Setuptools can work well with most versioning schemes. Over the years, +setuptools has tried to closely follow the +`PEP 440 <https://www.python.org/dev/peps/pep-0440/>`_ scheme, but it +also supports legacy versions. There are, however, a few special things to watch out for, in order to ensure that setuptools and other tools can always tell what version of your package is newer than another version. Knowing these things will also help you correctly specify what @@ -181,9 +184,13 @@ tag. Pre-release tags make a version be considered *older* than the version they are appended to. So, revision ``2.4`` is *newer* than revision ``2.4c1``, which in turn is newer than ``2.4b1`` or ``2.4a1``. Postrelease tags make a version be considered *newer* than the version they are appended to. So, -revisions like ``2.4-1`` and ``2.4pl3`` are newer than ``2.4``, but are *older* +revisions like ``2.4-1`` are newer than ``2.4``, but *older* than ``2.4.1`` (which has a higher release number). +In the case of legacy versions (for example, ``2.4pl1``), they are considered +older than non-legacy versions. Taking that in count, a revision ``2.4pl1`` +is *older* than ``2.4`` + A pre-release tag is a series of letters that are alphabetically before "final". Some examples of prerelease tags would include ``alpha``, ``beta``, ``a``, ``c``, ``dev``, and so on. You do not have to place a dot or dash diff --git a/docs/userguide/entry_point.rst b/docs/userguide/entry_point.rst index edab4465..21edc697 100644 --- a/docs/userguide/entry_point.rst +++ b/docs/userguide/entry_point.rst @@ -28,7 +28,7 @@ with ``__init__.py`` as: .. code-block:: python - def helloworld(): + def hello_world(): print("Hello world") and ``__main__.py`` providing a hook: @@ -36,6 +36,7 @@ and ``__main__.py`` providing a hook: .. code-block:: python from . import hello_world + if __name__ == '__main__': hello_world() @@ -64,7 +65,7 @@ After installing the package, a user may invoke that function by simply calling The syntax for entry points is specified as follows: -.. code-block:: +.. code-block:: ini <name> = [<package>.[<subpackage>.]]<module>[:<object>.<object>] @@ -102,7 +103,7 @@ module (part of stdlib since Python 3.8) or its backport, For example, to find the console script entry points from the example above: -.. code-block:: python +.. code-block:: pycon >>> from importlib import metadata >>> eps = metadata.entry_points()['console_scripts'] @@ -121,13 +122,14 @@ method to import and load that entry point (module or object). Then, a different project wishing to load 'my.plugins' plugins could run the following routine to load (and invoke) such plugins: -.. code-block:: python +.. code-block:: pycon >>> from importlib import metadata >>> eps = metadata.entry_points()['my.plugins'] >>> for ep in eps: ... plugin = ep.load() ... plugin() + ... The project soliciting the entry points needs not to have any dependency or prior knowledge about the libraries implementing the entry points, and @@ -139,7 +141,7 @@ Dependency Management ===================== Some entry points may require additional dependencies to properly function. -For such an entry point, declare in square brakets any number of dependency +For such an entry point, declare in square brackets any number of dependency ``extras`` following the entry point definition. Such entry points will only be viable if their extras were declared and installed. See the :doc:`guide on dependencies management <dependency_management>` for diff --git a/docs/userguide/extension.rst b/docs/userguide/extension.rst index 4de24ec9..93b59501 100644 --- a/docs/userguide/extension.rst +++ b/docs/userguide/extension.rst @@ -202,7 +202,7 @@ called "foobar", you would write a function something like this: .. code-block:: python def find_files_for_foobar(dirname): - # loop to yield paths that start with `dirname` + ... # loop to yield paths that start with `dirname` And you would register it in a setup script using something like this:: diff --git a/docs/userguide/keywords.rst b/docs/userguide/keywords.rst index 268e4f42..5388ffea 100644 --- a/docs/userguide/keywords.rst +++ b/docs/userguide/keywords.rst @@ -157,18 +157,6 @@ unless you need the associated ``setuptools`` feature. mess with it. For more details on how this argument works, see the section below on :ref:`Automatic Resource Extraction`. -``use_2to3`` - Convert the source code from Python 2 to Python 3 with 2to3 during the - build process. See :doc:`../deprecated/python3` for more details. - -``convert_2to3_doctests`` - List of doctest source files that need to be converted with 2to3. - See :doc:`../deprecated/python3` for more details. - -``use_2to3_fixers`` - A list of modules to search for additional fixers to be used during - the 2to3 conversion. See :doc:`../deprecated/python3` for more details. - ``project_urls`` An arbitrary map of URL names to hyperlinks, allowing more extensible documentation of where various resources can be found than the simple diff --git a/docs/userguide/package_discovery.rst b/docs/userguide/package_discovery.rst index de4ef668..61da2d66 100644 --- a/docs/userguide/package_discovery.rst +++ b/docs/userguide/package_discovery.rst @@ -19,36 +19,45 @@ Package Discovery and Namespace Package support for namespace package. Normally, you would specify the package to be included manually in the following manner: -.. code-block:: ini - - [options] - #... - packages = - mypkg1 - mypkg2 +.. tab:: setup.cfg -.. code-block:: python + .. code-block:: ini - setup( + [options] #... - packages = ['mypkg1', 'mypkg2'] - ) + packages = + mypkg1 + mypkg2 -This can get tiresome reallly quickly. To speed things up, we introduce two +.. tab:: setup.py + + .. code-block:: python + + setup( + # ... + packages=['mypkg1', 'mypkg2'] + ) + +This can get tiresome really quickly. To speed things up, we introduce two functions provided by setuptools: -.. code-block:: ini +.. tab:: setup.cfg - [options] - packages = find: - #or - packages = find_namespace: + .. code-block:: ini -.. code-block:: python + [options] + packages = find: + #or + packages = find_namespace: + +.. tab:: setup.py - from setuptools import find_packages - #or - from setuptools import find_namespace_packages + .. code-block:: python + + from setuptools import find_packages + + # or + from setuptools import find_namespace_packages Using ``find:`` or ``find_packages`` @@ -71,30 +80,34 @@ it, consider the following directory To have your setup.cfg or setup.py to automatically include packages found in ``src`` that starts with the name ``pkg`` and not ``additional``: -.. code-block:: ini +.. tab:: setup.cfg - [options] - packages = find: - package_dir = - =src + .. code-block:: ini - [options.packages.find] - where = src - include = pkg* - exclude = additional + [options] + packages = find: + package_dir = + =src -.. code-block:: python + [options.packages.find] + where = src + include = pkg* + exclude = additional - setup( - #... - packages = find_packages( - where = 'src', - include = ['pkg*',], - exclude = ['additional',] - ), - package_dir = {"":"src"} - #... - ) +.. tab:: setup.py + + .. code-block:: python + + setup( + # ... + packages=find_packages( + where='src', + include=['pkg*'], + exclude=['additional'], + ), + package_dir={"": "src"} + # ... + ) .. _Namespace Packages: @@ -117,7 +130,7 @@ If both ``Desktop`` and ``Library`` are on your ``PYTHONPATH``, then a namespace package called ``timmins`` will be created automatically for you when you invoke the import mechanism, allowing you to accomplish the following -.. code-block:: python +.. code-block:: pycon >>> import timmins.foo >>> import timmins.bar @@ -144,7 +157,7 @@ to use ``find_namespace:``: =src packages = find_namespace: - [options.packages.find_namespace] + [options.packages.find] where = src When you install the zipped distribution, ``timmins.foo`` would become @@ -195,17 +208,21 @@ following: And the ``namespace_packages`` keyword in your ``setup.cfg`` or ``setup.py``: -.. code-block:: ini +.. tab:: setup.cfg - [options] - namespace_packages = timmins + .. code-block:: ini -.. code-block:: python + [options] + namespace_packages = timmins + +.. tab:: setup.py + + .. code-block:: python - setup( - # ... - namespace_packages = ['timmins'] - ) + setup( + # ... + namespace_packages=['timmins'] + ) And your directory should look like this diff --git a/docs/userguide/quickstart.rst b/docs/userguide/quickstart.rst index 1d557d47..bcb282ed 100644 --- a/docs/userguide/quickstart.rst +++ b/docs/userguide/quickstart.rst @@ -2,8 +2,6 @@ ``setuptools`` Quickstart ========================== -.. contents:: - Installation ============ @@ -22,7 +20,7 @@ be generated with whatever tools that provides a ``build sdist``-alike functionality. While this may appear cumbersome, given the added pieces, it in fact tremendously enhances the portability of your package. The change is driven under :pep:`PEP 517 <517#build-requirements>`. To learn more about Python packaging in general, -navigate to the `bottom <Resources on python packaging>`_ of this page. +navigate to the :ref:`bottom <packaging-resources>` of this page. Basic Use @@ -37,33 +35,52 @@ package your project: requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" -Then, you will need a ``setup.cfg`` to specify your package information, -such as metadata, contents, dependencies, etc. Here we demonstrate the minimum +Then, you will need a ``setup.cfg`` or ``setup.py`` to specify your package +information, such as metadata, contents, dependencies, etc. Here we demonstrate +the minimum -.. code-block:: ini +.. tab:: setup.cfg - [metadata] - name = mypackage - version = 0.0.1 + .. code-block:: ini - [options] - packages = mypackage - install_requires = - requests - importlib; python_version == "2.6" + [metadata] + name = mypackage + version = 0.0.1 + + [options] + packages = mypackage + install_requires = + requests + importlib; python_version == "2.6" + +.. tab:: setup.py + + .. code-block:: python + + from setuptools import setup + + setup( + name='mypackage', + version='0.0.1', + packages=['mypackage'], + install_requires=[ + 'requests', + 'importlib; python_version == "2.6"', + ], + ) This is what your project would look like:: ~/mypackage/ pyproject.toml - setup.cfg + setup.cfg # or setup.py mypackage/__init__.py -Then, you need an installer, such as `pep517 <https://pypi.org/project/pep517/>`_ -which you can obtain via ``pip install pep517``. After downloading it, invoke -the installer:: +Then, you need an builder, such as :std:doc:`PyPA build <pypa-build:index>` +which you can obtain via ``pip install build``. After downloading it, invoke +the builder:: - python -m pep517.build . + python -m build You now have your distribution ready (e.g. a ``tar.gz`` file and a ``.whl`` file in the ``dist`` directory), which you can upload to PyPI! @@ -154,7 +171,7 @@ Including Data Files ==================== The distutils have traditionally allowed installation of "data files", which are placed in a platform-specific location. Setuptools offers three ways to -specify data files to be included in your packages. For the simpliest use, you +specify data files to be included in your packages. For the simplest use, you can simply use the ``include_package_data`` keyword: .. code-block:: ini @@ -170,7 +187,7 @@ For more details, see :doc:`datafiles` Development mode ================ ``setuptools`` allows you to install a package without copying any files -to your interpretor directory (e.g. the ``site-packages`` directory). This +to your interpreter directory (e.g. the ``site-packages`` directory). This allows you to modify your source code and have the changes take effect without you having to rebuild and reinstall. This is currently incompatible with PEP 517 and therefore it requires a ``setup.py`` script with the following @@ -183,8 +200,8 @@ Then:: pip install --editable . -This creates a link file in your interpretor site package directory which -associate with your source code. For more information, see: (WIP) +This creates a link file in your interpreter site package directory which +associate with your source code. For more information, see :doc:`development_mode`. Uploading your package to PyPI @@ -196,13 +213,14 @@ basic use here. Transitioning from ``setup.py`` to ``setup.cfg`` -================================================== -To avoid executing arbitary scripts and boilerplate code, we are transitioning +================================================ +To avoid executing arbitrary scripts and boilerplate code, we are transitioning into a full-fledged ``setup.cfg`` to declare your package information instead of running ``setup()``. This inevitably brings challenges due to a different syntax. Here we provide a quick guide to understanding how ``setup.cfg`` is parsed by ``setuptool`` to ease the pain of transition. +.. _packaging-resources: Resources on Python packaging ============================= diff --git a/easy_install.py b/easy_install.py deleted file mode 100644 index d87e9840..00000000 --- a/easy_install.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Run the EasyInstall command""" - -if __name__ == '__main__': - from setuptools.command.easy_install import main - main() diff --git a/msvc-build-launcher-arm64.cmd b/msvc-build-launcher-arm64.cmd new file mode 100644 index 00000000..8e63506b --- /dev/null +++ b/msvc-build-launcher-arm64.cmd @@ -0,0 +1,19 @@ +@echo off + +REM Build with jaraco/windows Docker image + +set PATH_OLD=%PATH% +set PATH=C:\BuildTools\VC\Auxiliary\Build;%PATH_OLD% + +REM now for arm 64-bit +REM Cross compile for arm64 +call VCVARSx86_arm64 +if "%ERRORLEVEL%"=="0" ( + cl /D "GUI=0" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:arm64 /SUBSYSTEM:CONSOLE /out:setuptools/cli-arm64.exe + cl /D "GUI=1" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:arm64 /SUBSYSTEM:WINDOWS /out:setuptools/gui-arm64.exe +) else ( + echo Visual Studio 2019 with arm64 toolchain not installed +) + +set PATH=%PATH_OLD% + diff --git a/netlify.toml b/netlify.toml deleted file mode 100644 index 5828132e..00000000 --- a/netlify.toml +++ /dev/null @@ -1,7 +0,0 @@ -# Configuration for pull request documentation previews via Netlify - -# Netlify relies on there being a ./runtime.txt to indicate Python 3. - -[build] - publish = "build/html" - command = "pip install tox && tox -e docs" diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index afe6078c..b6bc9b97 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -37,6 +37,7 @@ import textwrap import inspect import ntpath import posixpath +import importlib from pkgutil import get_importer try: @@ -691,7 +692,8 @@ class WorkingSet: keys2.append(dist.key) self._added_new(dist) - def resolve(self, requirements, env=None, installer=None, + # FIXME: 'WorkingSet.resolve' is too complex (11) + def resolve(self, requirements, env=None, installer=None, # noqa: C901 replace_conflicting=False, extras=None): """List all distributions needed to (recursively) meet `requirements` @@ -1740,7 +1742,8 @@ class ZipProvider(EggProvider): timestamp = time.mktime(date_time) return timestamp, size - def _extract_resource(self, manager, zip_path): + # FIXME: 'ZipProvider._extract_resource' is too complex (12) + def _extract_resource(self, manager, zip_path): # noqa: C901 if zip_path in self._index(): for name in self._index()[zip_path]: @@ -1978,7 +1981,7 @@ def find_eggs_in_zip(importer, path_item, only=False): dists = find_eggs_in_zip(zipimport.zipimporter(subpath), subpath) for dist in dists: yield dist - elif subitem.lower().endswith('.dist-info'): + elif subitem.lower().endswith(('.dist-info', '.egg-info')): subpath = os.path.join(path_item, subitem) submeta = EggMetadata(zipimport.zipimporter(subpath)) submeta.egg_info = subpath @@ -2168,7 +2171,7 @@ def _handle_ns(packageName, path_item): if subpath is not None: path = module.__path__ path.append(subpath) - loader.load_module(packageName) + importlib.import_module(packageName) _rebuild_mod_path(path, packageName, module) return subpath @@ -2791,7 +2794,8 @@ class Distribution: """Return the EntryPoint object for `group`+`name`, or ``None``""" return self.get_entry_map(group).get(name) - def insert_on(self, path, loc=None, replace=False): + # FIXME: 'Distribution.insert_on' is too complex (13) + def insert_on(self, path, loc=None, replace=False): # noqa: C901 """Ensure self.location is on path If replace=False (default): diff --git a/pkg_resources/_vendor/pyparsing.py b/pkg_resources/_vendor/pyparsing.py index cf75e1e5..4cae7883 100644 --- a/pkg_resources/_vendor/pyparsing.py +++ b/pkg_resources/_vendor/pyparsing.py @@ -1625,7 +1625,7 @@ class ParserElement(object): (see L{I{parseWithTabs}<parseWithTabs>})
- define your parse action using the full C{(s,loc,toks)} signature, and
reference the input string using the parse action's C{s} argument
- - explictly expand the tabs in your input string before calling
+ - explicitly expand the tabs in your input string before calling
C{parseString}
Example::
diff --git a/pkg_resources/extern/__init__.py b/pkg_resources/extern/__init__.py index 4dc3beb2..fed59295 100644 --- a/pkg_resources/extern/__init__.py +++ b/pkg_resources/extern/__init__.py @@ -1,3 +1,4 @@ +import importlib.util import sys @@ -20,17 +21,10 @@ class VendorImporter: yield self.vendor_pkg + '.' yield '' - def find_module(self, fullname, path=None): - """ - Return self when fullname starts with root_name and the - target module is one vendored through this importer. - """ + def _module_matches_namespace(self, fullname): + """Figure out if the target module is vendored.""" root, base, target = fullname.partition(self.root_name + '.') - if root: - return - if not any(map(target.startswith, self.vendored_names)): - return - return self + return not root and any(map(target.startswith, self.vendored_names)) def load_module(self, fullname): """ @@ -54,6 +48,19 @@ class VendorImporter: "distribution.".format(**locals()) ) + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + + def find_spec(self, fullname, path=None, target=None): + """Return a module spec for vendored names.""" + return ( + importlib.util.spec_from_loader(fullname, self) + if self._module_matches_namespace(fullname) else None + ) + def install(self): """ Install this importer into sys.meta_path if not already present. diff --git a/pkg_resources/tests/data/my-test-package-zip/my-test-package.zip b/pkg_resources/tests/data/my-test-package-zip/my-test-package.zip Binary files differnew file mode 100644 index 00000000..81f9a017 --- /dev/null +++ b/pkg_resources/tests/data/my-test-package-zip/my-test-package.zip diff --git a/pkg_resources/tests/test_find_distributions.py b/pkg_resources/tests/test_find_distributions.py index f9594422..b01b4827 100644 --- a/pkg_resources/tests/test_find_distributions.py +++ b/pkg_resources/tests/test_find_distributions.py @@ -32,3 +32,12 @@ class TestFindDistributions: assert [dist.project_name for dist in dists] == ['my-test-package'] dists = pkg_resources.find_distributions(str(target_dir), only=True) assert not list(dists) + + def test_zipped_sdist_one_level_removed(self, target_dir): + (TESTS_DATA_DIR / 'my-test-package-zip').copy(target_dir) + dists = pkg_resources.find_distributions( + str(target_dir / "my-test-package.zip")) + assert [dist.project_name for dist in dists] == ['my-test-package'] + dists = pkg_resources.find_distributions( + str(target_dir / "my-test-package.zip"), only=True) + assert not list(dists) diff --git a/pyproject.toml b/pyproject.toml index 658514d3..70e3473d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,5 @@ [build-system] requires = [ - # avoid self install on Python 2; ref #1996 - "setuptools >= 40.8; python_version > '3'", "wheel", ] build-backend = "setuptools.build_meta" @@ -12,20 +10,21 @@ skip-string-normalization = true [tool.setuptools_scm] -# jaraco/skeleton#22 -[tool.jaraco.pytest.plugins.black] +[pytest.enabler.black] #addopts = "--black" -# jaraco/skeleton#22 -[tool.jaraco.pytest.plugins.mypy] +[pytest.enabler.mypy] #addopts = "--mypy" -[tool.jaraco.pytest.plugins.flake8] +[pytest.enabler.flake8] addopts = "--flake8" -[tool.jaraco.pytest.plugins.cov] +[pytest.enabler.cov] addopts = "--cov" +[pytest.enabler.xdist] +addopts = "-n auto" + [tool.towncrier] package = "setuptools" package_dir = "setuptools" @@ -8,10 +8,16 @@ doctest_optionflags=ALLOW_UNICODE ELLIPSIS # workaround for warning pytest-dev/pytest#6178 junit_family=xunit2 filterwarnings= - # https://github.com/pytest-dev/pytest/issues/6928 - ignore:direct construction of .*Item has been deprecated:DeprecationWarning # Fail on warnings error + + ## upstream + # Suppress deprecation warning in flake8 + ignore:SelectableGroups dict interface is deprecated::flake8 + # Suppress deprecation warning in pypa/packaging#433 + ignore:The distutils package is deprecated::packaging.tags + ## end upstream + # https://github.com/pypa/setuptools/issues/1823 ignore:bdist_wininst command is deprecated # Suppress this error; unimportant for CI tests @@ -20,6 +26,3 @@ filterwarnings= ignore:Parent module 'setuptools' not found while handling absolute import:RuntimeWarning # Suppress use of bytes for filenames on Windows until fixed #2016 ignore:The Windows bytes API has been deprecated:DeprecationWarning - # https://github.com/pypa/setuptools/issues/2081 - ignore:lib2to3 package is deprecated:PendingDeprecationWarning - ignore:lib2to3 package is deprecated:DeprecationWarning diff --git a/runtime.txt b/runtime.txt deleted file mode 100644 index 475ba515..00000000 --- a/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -3.7 @@ -1,7 +1,6 @@ [metadata] -license_file = LICENSE name = setuptools -version = 51.1.0 +version = 58.2.0 author = Python Packaging Authority author_email = distutils-sig@python.org description = Easily download, build, install, upgrade, and uninstall Python packages @@ -13,63 +12,128 @@ classifiers = License :: OSI Approved :: MIT License Programming Language :: Python :: 3 Programming Language :: Python :: 3 :: Only - Topic :: Software Development :: Libraries :: Python Modules - Topic :: System :: Archiving :: Packaging - Topic :: System :: Systems Administration - Topic :: Utilities + Topic :: Software Development :: Libraries :: Python Modules + Topic :: System :: Archiving :: Packaging + Topic :: System :: Systems Administration + Topic :: Utilities keywords = CPAN PyPI distutils eggs package management project_urls = - Documentation = https://setuptools.readthedocs.io/ + Documentation = https://setuptools.readthedocs.io/ [options] -packages = find: -py_modules = easy_install -include_package_data = true +packages = find_namespace: +# disabled as it causes tests to be included #2505 +# include_package_data = true python_requires = >=3.6 install_requires = [options.packages.find] -exclude = *.tests +exclude = + build* + dist* + docs* + tests* + *.tests + tools* [options.extras_require] testing = # upstream - pytest >= 3.5, !=3.7.3 - pytest-checkdocs >= 1.2.3 + pytest >= 4.6 + pytest-checkdocs >= 2.4 pytest-flake8 - pytest-black >= 0.3.7; python_implementation != "PyPy" + pytest-black >= 0.3.7; \ + # workaround for jaraco/skeleton#22 + python_implementation != "PyPy" pytest-cov - pytest-mypy; python_implementation != "PyPy" - # jaraco/skeleton#22 - jaraco.test >= 3.2.0 + pytest-mypy; \ + # workaround for jaraco/skeleton#22 + python_implementation != "PyPy" + pytest-enabler >= 1.0.1 # local - mock - flake8-2020 - virtualenv>=13.0.0 - pytest-virtualenv>=1.2.7 - wheel - paver - pip>=19.1 # For proper file:// URLs support. - jaraco.envs + mock + flake8-2020 + virtualenv>=13.0.0 + pytest-virtualenv>=1.2.7 + wheel + paver + pip>=19.1 # For proper file:// URLs support. + jaraco.envs + pytest-xdist + sphinx + jaraco.path>=3.2.0 docs = - # Keep these in sync with docs/requirements.txt # upstream sphinx - jaraco.packaging >= 6.1 + jaraco.packaging >= 8.2 rst.linker >= 1.9 + jaraco.tidelift >= 1.4 # local - pygments-github-lexers==0.0.5 + pygments-github-lexers==0.0.5 + sphinx-inline-tabs + sphinxcontrib-towncrier + furo ssl = - wincertstore==0.2; sys_platform=='win32' certs = - certifi==2016.9.26 [options.entry_points] +distutils.commands = + alias = setuptools.command.alias:alias + bdist_egg = setuptools.command.bdist_egg:bdist_egg + bdist_rpm = setuptools.command.bdist_rpm:bdist_rpm + build_clib = setuptools.command.build_clib:build_clib + build_ext = setuptools.command.build_ext:build_ext + build_py = setuptools.command.build_py:build_py + develop = setuptools.command.develop:develop + dist_info = setuptools.command.dist_info:dist_info + easy_install = setuptools.command.easy_install:easy_install + egg_info = setuptools.command.egg_info:egg_info + install = setuptools.command.install:install + install_egg_info = setuptools.command.install_egg_info:install_egg_info + install_lib = setuptools.command.install_lib:install_lib + install_scripts = setuptools.command.install_scripts:install_scripts + rotate = setuptools.command.rotate:rotate + saveopts = setuptools.command.saveopts:saveopts + sdist = setuptools.command.sdist:sdist + setopt = setuptools.command.setopt:setopt + test = setuptools.command.test:test + upload_docs = setuptools.command.upload_docs:upload_docs +setuptools.finalize_distribution_options = + parent_finalize = setuptools.dist:_Distribution.finalize_options + keywords = setuptools.dist:Distribution._finalize_setup_keywords +distutils.setup_keywords = + eager_resources = setuptools.dist:assert_string_list + namespace_packages = setuptools.dist:check_nsp + extras_require = setuptools.dist:check_extras + install_requires = setuptools.dist:check_requirements + tests_require = setuptools.dist:check_requirements + setup_requires = setuptools.dist:check_requirements + python_requires = setuptools.dist:check_specifier + entry_points = setuptools.dist:check_entry_points + test_suite = setuptools.dist:check_test_suite + zip_safe = setuptools.dist:assert_bool + package_data = setuptools.dist:check_package_data + exclude_package_data = setuptools.dist:check_package_data + include_package_data = setuptools.dist:assert_bool + packages = setuptools.dist:check_packages + dependency_links = setuptools.dist:assert_string_list + test_loader = setuptools.dist:check_importable + test_runner = setuptools.dist:check_importable + use_2to3 = setuptools.dist:invalid_unless_false +egg_info.writers = + PKG-INFO = setuptools.command.egg_info:write_pkg_info + requires.txt = setuptools.command.egg_info:write_requirements + entry_points.txt = setuptools.command.egg_info:write_entries + eager_resources.txt = setuptools.command.egg_info:overwrite_arg + namespace_packages.txt = setuptools.command.egg_info:overwrite_arg + top_level.txt = setuptools.command.egg_info:write_toplevel_names + depends.txt = setuptools.command.egg_info:warn_depends_obsolete + dependency_links.txt = setuptools.command.egg_info:overwrite_arg [egg_info] tag_build = .post @@ -10,42 +10,6 @@ from setuptools.command.install import install here = os.path.dirname(__file__) -def require_metadata(): - "Prevent improper installs without necessary metadata. See #659" - egg_info_dir = os.path.join(here, 'setuptools.egg-info') - if not os.path.exists(egg_info_dir): - msg = ( - "Cannot build setuptools without metadata. " - "Run `bootstrap.py`." - ) - raise RuntimeError(msg) - - -def read_commands(): - command_ns = {} - cmd_module_path = 'setuptools/command/__init__.py' - init_path = os.path.join(here, cmd_module_path) - with open(init_path) as init_file: - exec(init_file.read(), command_ns) - return command_ns['__all__'] - - -def _gen_console_scripts(): - yield "easy_install = setuptools.command.easy_install:main" - - # Gentoo distributions manage the python-version-specific scripts - # themselves, so those platforms define an environment variable to - # suppress the creation of the version-specific scripts. - var_names = ( - 'SETUPTOOLS_DISABLE_VERSIONED_EASY_INSTALL_SCRIPT', - 'DISTRIBUTE_DISABLE_VERSIONED_EASY_INSTALL_SCRIPT', - ) - if any(os.environ.get(var) not in (None, "", "0") for var in var_names): - return - tmpl = "easy_install-{shortver} = setuptools.command.easy_install:main" - yield tmpl.format(shortver='{}.{}'.format(*sys.version_info)) - - package_data = dict( setuptools=['script (dev).tmpl', 'script.tmpl', 'site-patch.py'], ) @@ -55,19 +19,12 @@ force_windows_specific_files = ( not in ("", "0", "false", "no") ) -include_windows_files = ( - sys.platform == 'win32' or - os.name == 'java' and os._name == 'nt' or - force_windows_specific_files -) +include_windows_files = sys.platform == 'win32' or force_windows_specific_files if include_windows_files: package_data.setdefault('setuptools', []).extend(['*.exe']) package_data.setdefault('setuptools.command', []).extend(['*.xml']) -needs_wheel = set(['release', 'bdist_wheel']).intersection(sys.argv) -wheel = ['wheel'] if needs_wheel else [] - def pypi_link(pkg_filename): """ @@ -120,74 +77,11 @@ class install_with_pth(install): setup_params = dict( - src_root=None, cmdclass={'install': install_with_pth}, package_data=package_data, - entry_points={ - "distutils.commands": [ - "%(cmd)s = setuptools.command.%(cmd)s:%(cmd)s" % locals() - for cmd in read_commands() - ], - "setuptools.finalize_distribution_options": [ - "parent_finalize = setuptools.dist:_Distribution.finalize_options", - "keywords = setuptools.dist:Distribution._finalize_setup_keywords", - "2to3_doctests = " - "setuptools.dist:Distribution._finalize_2to3_doctests", - ], - "distutils.setup_keywords": [ - "eager_resources = setuptools.dist:assert_string_list", - "namespace_packages = setuptools.dist:check_nsp", - "extras_require = setuptools.dist:check_extras", - "install_requires = setuptools.dist:check_requirements", - "tests_require = setuptools.dist:check_requirements", - "setup_requires = setuptools.dist:check_requirements", - "python_requires = setuptools.dist:check_specifier", - "entry_points = setuptools.dist:check_entry_points", - "test_suite = setuptools.dist:check_test_suite", - "zip_safe = setuptools.dist:assert_bool", - "package_data = setuptools.dist:check_package_data", - "exclude_package_data = setuptools.dist:check_package_data", - "include_package_data = setuptools.dist:assert_bool", - "packages = setuptools.dist:check_packages", - "dependency_links = setuptools.dist:assert_string_list", - "test_loader = setuptools.dist:check_importable", - "test_runner = setuptools.dist:check_importable", - "use_2to3 = setuptools.dist:assert_bool", - "convert_2to3_doctests = setuptools.dist:assert_string_list", - "use_2to3_fixers = setuptools.dist:assert_string_list", - "use_2to3_exclude_fixers = setuptools.dist:assert_string_list", - ], - "egg_info.writers": [ - "PKG-INFO = setuptools.command.egg_info:write_pkg_info", - "requires.txt = setuptools.command.egg_info:write_requirements", - "entry_points.txt = setuptools.command.egg_info:write_entries", - "eager_resources.txt = setuptools.command.egg_info:overwrite_arg", - ( - "namespace_packages.txt = " - "setuptools.command.egg_info:overwrite_arg" - ), - "top_level.txt = setuptools.command.egg_info:write_toplevel_names", - "depends.txt = setuptools.command.egg_info:warn_depends_obsolete", - "dependency_links.txt = setuptools.command.egg_info:overwrite_arg", - ], - "console_scripts": list(_gen_console_scripts()), - "setuptools.installation": - ['eggsecutable = setuptools.command.easy_install:bootstrap'], - }, - dependency_links=[ - pypi_link( - 'certifi-2016.9.26.tar.gz#md5=baa81e951a29958563689d868ef1064d', - ), - pypi_link( - 'wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2', - ), - ], - setup_requires=[ - ] + wheel, ) if __name__ == '__main__': # allow setup.py to run from another directory here and os.chdir(here) - require_metadata() dist = setuptools.setup(**setup_params) diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 4d9b8357..9d6f0bc0 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -21,21 +21,20 @@ from . import monkey __all__ = [ - 'setup', 'Distribution', 'Command', 'Extension', 'Require', + 'setup', + 'Distribution', + 'Command', + 'Extension', + 'Require', 'SetuptoolsDeprecationWarning', - 'find_packages', 'find_namespace_packages', + 'find_packages', + 'find_namespace_packages', ] __version__ = setuptools.version.__version__ bootstrap_install_from = None -# If we run 2to3 on .py files, should we also convert docstrings? -# Default: yes; assume that we can detect doctests reliably -run_2to3_on_doctests = True -# Standard package names for fixer packages -lib2to3_fixer_packages = ['lib2to3.fixes'] - class PackageFinder: """ @@ -60,10 +59,13 @@ class PackageFinder: shell style wildcard patterns just like 'exclude'. """ - return list(cls._find_packages_iter( - convert_path(where), - cls._build_filter('ez_setup', '*__pycache__', *exclude), - cls._build_filter(*include))) + return list( + cls._find_packages_iter( + convert_path(where), + cls._build_filter('ez_setup', '*__pycache__', *exclude), + cls._build_filter(*include), + ) + ) @classmethod def _find_packages_iter(cls, where, exclude, include): @@ -82,7 +84,7 @@ class PackageFinder: package = rel_path.replace(os.path.sep, '.') # Skip directory trees that are not valid packages - if ('.' in dir or not cls._looks_like_package(full_path)): + if '.' in dir or not cls._looks_like_package(full_path): continue # Should this package be included? @@ -125,12 +127,10 @@ def _install_setup_requires(attrs): A minimal version of a distribution for supporting the fetch_build_eggs interface. """ + def __init__(self, attrs): _incl = 'dependency_links', 'setup_requires' - filtered = { - k: attrs[k] - for k in set(_incl) & set(attrs) - } + filtered = {k: attrs[k] for k in set(_incl) & set(attrs)} distutils.core.Distribution.__init__(self, filtered) def finalize_options(self): @@ -178,8 +178,9 @@ class Command(_Command): setattr(self, option, default) return default elif not isinstance(val, str): - raise DistutilsOptionError("'%s' must be a %s (got `%s`)" - % (option, what, val)) + raise DistutilsOptionError( + "'%s' must be a %s (got `%s`)" % (option, what, val) + ) return val def ensure_string_list(self, option): @@ -200,8 +201,8 @@ class Command(_Command): ok = False if not ok: raise DistutilsOptionError( - "'%s' must be a list of strings (got %r)" - % (option, val)) + "'%s' must be a list of strings (got %r)" % (option, val) + ) def reinitialize_command(self, command, reinit_subcommands=0, **kw): cmd = _Command.reinitialize_command(self, command, reinit_subcommands) diff --git a/setuptools/_distutils/_msvccompiler.py b/setuptools/_distutils/_msvccompiler.py index e9af4cf5..b7a06082 100644 --- a/setuptools/_distutils/_msvccompiler.py +++ b/setuptools/_distutils/_msvccompiler.py @@ -248,7 +248,7 @@ class MSVCCompiler(CCompiler) : # Future releases of Python 3.x will include all past # versions of vcruntime*.dll for compatibility. self.compile_options = [ - '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG', '/MD' + '/nologo', '/O2', '/W3', '/GL', '/DNDEBUG', '/MD' ] self.compile_options_debug = [ diff --git a/setuptools/_distutils/ccompiler.py b/setuptools/_distutils/ccompiler.py index 57bb94e8..48d160d2 100644 --- a/setuptools/_distutils/ccompiler.py +++ b/setuptools/_distutils/ccompiler.py @@ -392,7 +392,7 @@ class CCompiler: return output_dir, macros, include_dirs def _prep_compile(self, sources, output_dir, depends=None): - """Decide which souce files must be recompiled. + """Decide which source files must be recompiled. Determine the list of object files corresponding to 'sources', and figure out which ones really need to be recompiled. @@ -792,6 +792,8 @@ int main (int argc, char **argv) { objects = self.compile([fname], include_dirs=include_dirs) except CompileError: return False + finally: + os.remove(fname) try: self.link_executable(objects, "a.out", @@ -799,6 +801,11 @@ int main (int argc, char **argv) { library_dirs=library_dirs) except (LinkError, TypeError): return False + else: + os.remove("a.out") + finally: + for fn in objects: + os.remove(fn) return True def find_library_file (self, dirs, lib, debug=0): diff --git a/setuptools/_distutils/command/build.py b/setuptools/_distutils/command/build.py index a86df0bc..4355a632 100644 --- a/setuptools/_distutils/command/build.py +++ b/setuptools/_distutils/command/build.py @@ -102,7 +102,7 @@ class build(Command): # particular module distribution -- if user didn't supply it, pick # one of 'build_purelib' or 'build_platlib'. if self.build_lib is None: - if self.distribution.ext_modules: + if self.distribution.has_ext_modules(): self.build_lib = self.build_platlib else: self.build_lib = self.build_purelib diff --git a/setuptools/_distutils/command/build_ext.py b/setuptools/_distutils/command/build_ext.py index bbb34833..22628baf 100644 --- a/setuptools/_distutils/command/build_ext.py +++ b/setuptools/_distutils/command/build_ext.py @@ -220,7 +220,7 @@ class build_ext(Command): # For extensions under Cygwin, Python's library directory must be # appended to library_dirs if sys.platform[:6] == 'cygwin': - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(os.path.join(sys.prefix, "lib", "python" + get_python_version(), @@ -690,13 +690,15 @@ class build_ext(Command): provided, "PyInit_" + module_name. Only relevant on Windows, where the .pyd file (DLL) must export the module "PyInit_" function. """ - suffix = '_' + ext.name.split('.')[-1] + name = ext.name.split('.')[-1] try: # Unicode module name support as defined in PEP-489 # https://www.python.org/dev/peps/pep-0489/#export-hook-name - suffix.encode('ascii') + name.encode('ascii') except UnicodeEncodeError: - suffix = 'U' + suffix.encode('punycode').replace(b'-', b'_').decode('ascii') + suffix = 'U_' + name.encode('punycode').replace(b'-', b'_').decode('ascii') + else: + suffix = "_" + name initfunc_name = "PyInit" + suffix if initfunc_name not in ext.export_symbols: diff --git a/setuptools/_distutils/command/build_py.py b/setuptools/_distutils/command/build_py.py index edc2171c..7ef9bcef 100644 --- a/setuptools/_distutils/command/build_py.py +++ b/setuptools/_distutils/command/build_py.py @@ -9,7 +9,7 @@ import glob from distutils.core import Command from distutils.errors import * -from distutils.util import convert_path, Mixin2to3 +from distutils.util import convert_path from distutils import log class build_py (Command): @@ -390,27 +390,3 @@ class build_py (Command): if self.optimize > 0: byte_compile(files, optimize=self.optimize, force=self.force, prefix=prefix, dry_run=self.dry_run) - -class build_py_2to3(build_py, Mixin2to3): - def run(self): - self.updated_files = [] - - # Base class code - if self.py_modules: - self.build_modules() - if self.packages: - self.build_packages() - self.build_package_data() - - # 2to3 - self.run_2to3(self.updated_files) - - # Remaining base class code - self.byte_compile(self.get_outputs(include_bytecode=0)) - - def build_module(self, module, module_file, package): - res = build_py.build_module(self, module, module_file, package) - if res[1]: - # file was copied - self.updated_files.append(res[0]) - return res diff --git a/setuptools/_distutils/command/build_scripts.py b/setuptools/_distutils/command/build_scripts.py index ccc70e64..e3312cf0 100644 --- a/setuptools/_distutils/command/build_scripts.py +++ b/setuptools/_distutils/command/build_scripts.py @@ -7,7 +7,7 @@ from stat import ST_MODE from distutils import sysconfig from distutils.core import Command from distutils.dep_util import newer -from distutils.util import convert_path, Mixin2to3 +from distutils.util import convert_path from distutils import log import tokenize @@ -150,11 +150,3 @@ class build_scripts(Command): os.chmod(file, newmode) # XXX should we modify self.outfiles? return outfiles, updated_files - -class build_scripts_2to3(build_scripts, Mixin2to3): - - def copy_scripts(self): - outfiles, updated_files = build_scripts.copy_scripts(self) - if not self.dry_run: - self.run_2to3(updated_files) - return outfiles, updated_files diff --git a/setuptools/_distutils/command/install.py b/setuptools/_distutils/command/install.py index 13feeb89..866e2d59 100644 --- a/setuptools/_distutils/command/install.py +++ b/setuptools/_distutils/command/install.py @@ -348,7 +348,7 @@ class install(Command): # module distribution is pure or not. Of course, if the user # already specified install_lib, use their selection. if self.install_lib is None: - if self.distribution.ext_modules: # has extensions: non-pure + if self.distribution.has_ext_modules(): # has extensions: non-pure self.install_lib = self.install_platlib else: self.install_lib = self.install_purelib @@ -470,6 +470,7 @@ class install(Command): """Sets the install directories by applying the install schemes.""" # it's the caller's problem if they supply a bad name! if (hasattr(sys, 'pypy_version_info') and + sys.version_info < (3, 8) and not name.endswith(('_user', '_home'))): if os.name == 'nt': name = 'pypy_nt' diff --git a/setuptools/_distutils/cygwinccompiler.py b/setuptools/_distutils/cygwinccompiler.py index 66c12dd3..f1c38e39 100644 --- a/setuptools/_distutils/cygwinccompiler.py +++ b/setuptools/_distutils/cygwinccompiler.py @@ -44,6 +44,8 @@ cygwin in no-cygwin mode). # (ld supports -shared) # * mingw gcc 3.2/ld 2.13 works # (ld supports -shared) +# * llvm-mingw with Clang 11 works +# (lld supports -shared) import os import sys @@ -109,41 +111,46 @@ class CygwinCCompiler(UnixCCompiler): "Compiling may fail because of undefined preprocessor macros." % details) - self.gcc_version, self.ld_version, self.dllwrap_version = \ - get_versions() - self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % - (self.gcc_version, - self.ld_version, - self.dllwrap_version) ) - - # ld_version >= "2.10.90" and < "2.13" should also be able to use - # gcc -mdll instead of dllwrap - # Older dllwraps had own version numbers, newer ones use the - # same as the rest of binutils ( also ld ) - # dllwrap 2.10.90 is buggy - if self.ld_version >= "2.10.90": - self.linker_dll = "gcc" - else: - self.linker_dll = "dllwrap" + self.cc = os.environ.get('CC', 'gcc') + self.cxx = os.environ.get('CXX', 'g++') + + if ('gcc' in self.cc): # Start gcc workaround + self.gcc_version, self.ld_version, self.dllwrap_version = \ + get_versions() + self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % + (self.gcc_version, + self.ld_version, + self.dllwrap_version) ) + + # ld_version >= "2.10.90" and < "2.13" should also be able to use + # gcc -mdll instead of dllwrap + # Older dllwraps had own version numbers, newer ones use the + # same as the rest of binutils ( also ld ) + # dllwrap 2.10.90 is buggy + if self.ld_version >= "2.10.90": + self.linker_dll = self.cc + else: + self.linker_dll = "dllwrap" - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + else: # Assume linker is up to date + self.linker_dll = self.cc shared_option = "-shared" - else: - shared_option = "-mdll -static" - # Hard-code GCC because that's what this is all about. - # XXX optimization, warnings etc. should be customizable. - self.set_executables(compiler='gcc -mcygwin -O -Wall', - compiler_so='gcc -mcygwin -mdll -O -Wall', - compiler_cxx='g++ -mcygwin -O -Wall', - linker_exe='gcc -mcygwin', + self.set_executables(compiler='%s -mcygwin -O -Wall' % self.cc, + compiler_so='%s -mcygwin -mdll -O -Wall' % self.cc, + compiler_cxx='%s -mcygwin -O -Wall' % self.cxx, + linker_exe='%s -mcygwin' % self.cc, linker_so=('%s -mcygwin %s' % (self.linker_dll, shared_option))) # cygwin and mingw32 need different sets of libraries - if self.gcc_version == "2.91.57": + if ('gcc' in self.cc and self.gcc_version == "2.91.57"): # cygwin shouldn't need msvcrt, but without the dlls will crash # (gcc version 2.91.57) -- perhaps something about initialization self.dll_libraries=["msvcrt"] @@ -281,26 +288,26 @@ class Mingw32CCompiler(CygwinCCompiler): # ld_version >= "2.13" support -shared so use it instead of # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: + if ('gcc' in self.cc and self.ld_version < "2.13"): shared_option = "-mdll -static" + else: + shared_option = "-shared" # A real mingw32 doesn't need to specify a different entry point, # but cygwin 2.91.57 in no-cygwin-mode needs it. - if self.gcc_version <= "2.91.57": + if ('gcc' in self.cc and self.gcc_version <= "2.91.57"): entry_point = '--entry _DllMain@12' else: entry_point = '' - if is_cygwingcc(): + if is_cygwincc(self.cc): raise CCompilerError( 'Cygwin gcc cannot be used with --compiler=mingw32') - self.set_executables(compiler='gcc -O -Wall', - compiler_so='gcc -mdll -O -Wall', - compiler_cxx='g++ -O -Wall', - linker_exe='gcc', + self.set_executables(compiler='%s -O -Wall' % self.cc, + compiler_so='%s -mdll -O -Wall' % self.cc, + compiler_cxx='%s -O -Wall' % self.cxx, + linker_exe='%s' % self.cc, linker_so='%s %s %s' % (self.linker_dll, shared_option, entry_point)) @@ -351,6 +358,10 @@ def check_config_h(): if "GCC" in sys.version: return CONFIG_H_OK, "sys.version mentions 'GCC'" + # Clang would also work + if "Clang" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'Clang'" + # let's see if __GNUC__ is mentioned in python.h fn = sysconfig.get_config_h_filename() try: @@ -397,7 +408,7 @@ def get_versions(): commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] return tuple([_find_exe_version(cmd) for cmd in commands]) -def is_cygwingcc(): - '''Try to determine if the gcc that would be used is from cygwin.''' - out_string = check_output(['gcc', '-dumpmachine']) +def is_cygwincc(cc): + '''Try to determine if the compiler that would be used is from cygwin.''' + out_string = check_output([cc, '-dumpmachine']) return out_string.strip().endswith(b'cygwin') diff --git a/setuptools/_distutils/filelist.py b/setuptools/_distutils/filelist.py index c92d5fdb..82a77384 100644 --- a/setuptools/_distutils/filelist.py +++ b/setuptools/_distutils/filelist.py @@ -4,13 +4,16 @@ Provides the FileList class, used for poking about the filesystem and building lists of files. """ -import os, re +import os +import re import fnmatch import functools + from distutils.util import convert_path from distutils.errors import DistutilsTemplateError, DistutilsInternalError from distutils import log + class FileList: """A list of files built by on exploring the filesystem and filtered by applying various patterns to what we find there. @@ -46,7 +49,7 @@ class FileList: if DEBUG: print(msg) - # -- List-like methods --------------------------------------------- + # Collection methods def append(self, item): self.files.append(item) @@ -61,8 +64,7 @@ class FileList: for sort_tuple in sortable_files: self.files.append(os.path.join(*sort_tuple)) - - # -- Other miscellaneous utility methods --------------------------- + # Other miscellaneous utility methods def remove_duplicates(self): # Assumes list has been sorted! @@ -70,8 +72,7 @@ class FileList: if self.files[i] == self.files[i - 1]: del self.files[i] - - # -- "File template" methods --------------------------------------- + # "File template" methods def _parse_template_line(self, line): words = line.split() @@ -146,9 +147,11 @@ class FileList: (dir, ' '.join(patterns))) for pattern in patterns: if not self.include_pattern(pattern, prefix=dir): - log.warn(("warning: no files found matching '%s' " - "under directory '%s'"), - pattern, dir) + msg = ( + "warning: no files found matching '%s' " + "under directory '%s'" + ) + log.warn(msg, pattern, dir) elif action == 'recursive-exclude': self.debug_print("recursive-exclude %s %s" % @@ -174,8 +177,7 @@ class FileList: raise DistutilsInternalError( "this cannot happen: invalid action '%s'" % action) - - # -- Filtering/selection methods ----------------------------------- + # Filtering/selection methods def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): """Select strings (presumably filenames) from 'self.files' that @@ -219,9 +221,8 @@ class FileList: files_found = True return files_found - - def exclude_pattern (self, pattern, - anchor=1, prefix=None, is_regex=0): + def exclude_pattern( + self, pattern, anchor=1, prefix=None, is_regex=0): """Remove strings (presumably filenames) from 'files' that match 'pattern'. Other parameters are the same as for 'include_pattern()', above. @@ -240,21 +241,47 @@ class FileList: return files_found -# ---------------------------------------------------------------------- # Utility functions def _find_all_simple(path): """ Find all files under 'path' """ + all_unique = _UniqueDirs.filter(os.walk(path, followlinks=True)) results = ( os.path.join(base, file) - for base, dirs, files in os.walk(path, followlinks=True) + for base, dirs, files in all_unique for file in files ) return filter(os.path.isfile, results) +class _UniqueDirs(set): + """ + Exclude previously-seen dirs from walk results, + avoiding infinite recursion. + Ref https://bugs.python.org/issue44497. + """ + def __call__(self, walk_item): + """ + Given an item from an os.walk result, determine + if the item represents a unique dir for this instance + and if not, prevent further traversal. + """ + base, dirs, files = walk_item + stat = os.stat(base) + candidate = stat.st_dev, stat.st_ino + found = candidate in self + if found: + del dirs[:] + self.add(candidate) + return not found + + @classmethod + def filter(cls, items): + return filter(cls(), items) + + def findall(dir=os.curdir): """ Find all files under 'dir' and return the list of full filenames. @@ -319,7 +346,8 @@ def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0): if os.sep == '\\': sep = r'\\' pattern_re = pattern_re[len(start): len(pattern_re) - len(end)] - pattern_re = r'%s\A%s%s.*%s%s' % (start, prefix_re, sep, pattern_re, end) + pattern_re = r'%s\A%s%s.*%s%s' % ( + start, prefix_re, sep, pattern_re, end) else: # no prefix -- respect anchor flag if anchor: pattern_re = r'%s\A%s' % (start, pattern_re[len(start):]) diff --git a/setuptools/_distutils/msvc9compiler.py b/setuptools/_distutils/msvc9compiler.py index 6934e964..a1b3b02f 100644 --- a/setuptools/_distutils/msvc9compiler.py +++ b/setuptools/_distutils/msvc9compiler.py @@ -399,13 +399,13 @@ class MSVCCompiler(CCompiler) : self.preprocess_options = None if self.__arch == "x86": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', + self.compile_options = [ '/nologo', '/O2', '/MD', '/W3', '/DNDEBUG'] self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/Z7', '/D_DEBUG'] else: # Win64 - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , + self.compile_options = [ '/nologo', '/O2', '/MD', '/W3', '/GS-' , '/DNDEBUG'] self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', '/Z7', '/D_DEBUG'] diff --git a/setuptools/_distutils/msvccompiler.py b/setuptools/_distutils/msvccompiler.py index d5857cb1..2d447b85 100644 --- a/setuptools/_distutils/msvccompiler.py +++ b/setuptools/_distutils/msvccompiler.py @@ -283,13 +283,13 @@ class MSVCCompiler(CCompiler) : self.preprocess_options = None if self.__arch == "Intel": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GX' , + self.compile_options = [ '/nologo', '/O2', '/MD', '/W3', '/GX' , '/DNDEBUG'] self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GX', '/Z7', '/D_DEBUG'] else: # Win64 - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , + self.compile_options = [ '/nologo', '/O2', '/MD', '/W3', '/GS-' , '/DNDEBUG'] self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', '/Z7', '/D_DEBUG'] diff --git a/setuptools/_distutils/spawn.py b/setuptools/_distutils/spawn.py index fc592d4a..6e1c89f1 100644 --- a/setuptools/_distutils/spawn.py +++ b/setuptools/_distutils/spawn.py @@ -15,11 +15,6 @@ from distutils.debug import DEBUG from distutils import log -if sys.platform == 'darwin': - _cfg_target = None - _cfg_target_split = None - - def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None): """Run another program, specified as a command list 'cmd', in a new process. @@ -40,7 +35,7 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None): # in, protect our %-formatting code against horrible death cmd = list(cmd) - log.info(' '.join(cmd)) + log.info(subprocess.list2cmdline(cmd)) if dry_run: return @@ -52,24 +47,10 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None): env = env if env is not None else dict(os.environ) if sys.platform == 'darwin': - global _cfg_target, _cfg_target_split - if _cfg_target is None: - from distutils import sysconfig - _cfg_target = sysconfig.get_config_var( - 'MACOSX_DEPLOYMENT_TARGET') or '' - if _cfg_target: - _cfg_target_split = [int(x) for x in _cfg_target.split('.')] - if _cfg_target: - # ensure that the deployment target of build process is not less - # than that used when the interpreter was built. This ensures - # extension modules are built with correct compatibility values - cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) - if _cfg_target_split > [int(x) for x in cur_target.split('.')]: - my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' - 'now "%s" but "%s" during configure' - % (cur_target, _cfg_target)) - raise DistutilsPlatformError(my_msg) - env.update(MACOSX_DEPLOYMENT_TARGET=cur_target) + from distutils.util import MACOSX_VERSION_VAR, get_macosx_target_ver + macosx_target_ver = get_macosx_target_ver() + if macosx_target_ver: + env[MACOSX_VERSION_VAR] = macosx_target_ver try: proc = subprocess.Popen(cmd, env=env) diff --git a/setuptools/_distutils/sysconfig.py b/setuptools/_distutils/sysconfig.py index 879b6981..8832b3ec 100644 --- a/setuptools/_distutils/sysconfig.py +++ b/setuptools/_distutils/sysconfig.py @@ -99,9 +99,9 @@ def get_python_inc(plat_specific=0, prefix=None): """ if prefix is None: prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - if IS_PYPY: - return os.path.join(prefix, 'include') - elif os.name == "posix": + if os.name == "posix": + if IS_PYPY and sys.version_info < (3, 8): + return os.path.join(prefix, 'include') if python_build: # Assume the executable is in the build directory. The # pyconfig.h file should be in the same directory. Since @@ -113,7 +113,8 @@ def get_python_inc(plat_specific=0, prefix=None): else: incdir = os.path.join(get_config_var('srcdir'), 'Include') return os.path.normpath(incdir) - python_dir = 'python' + get_python_version() + build_flags + implementation = 'pypy' if IS_PYPY else 'python' + python_dir = implementation + get_python_version() + build_flags return os.path.join(prefix, "include", python_dir) elif os.name == "nt": if python_build: @@ -142,7 +143,8 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): If 'prefix' is supplied, use it instead of sys.base_prefix or sys.base_exec_prefix -- i.e., ignore 'plat_specific'. """ - if IS_PYPY: + + if IS_PYPY and sys.version_info < (3, 8): # PyPy-specific schema if prefix is None: prefix = PREFIX @@ -164,8 +166,9 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): else: # Pure Python libdir = "lib" + implementation = 'pypy' if IS_PYPY else 'python' libpython = os.path.join(prefix, libdir, - "python" + get_python_version()) + implementation + get_python_version()) if standard_lib: return libpython else: @@ -211,10 +214,9 @@ def customize_compiler(compiler): if 'CC' in os.environ: newcc = os.environ['CC'] - if (sys.platform == 'darwin' - and 'LDSHARED' not in os.environ + if('LDSHARED' not in os.environ and ldshared.startswith(cc)): - # On OS X, if CC is overridden, use that as the default + # If CC is overridden, use that as the default # command for LDSHARED as well ldshared = newcc + ldshared[len(cc):] cc = newcc @@ -252,6 +254,9 @@ def customize_compiler(compiler): linker_exe=cc, archiver=archiver) + if 'RANLIB' in os.environ and compiler.executables.get('ranlib', None): + compiler.set_executables(ranlib=os.environ['RANLIB']) + compiler.shared_lib_extension = shlib_suffix diff --git a/setuptools/_distutils/tests/test_build_ext.py b/setuptools/_distutils/tests/test_build_ext.py index 5a72458c..85ecf4b7 100644 --- a/setuptools/_distutils/tests/test_build_ext.py +++ b/setuptools/_distutils/tests/test_build_ext.py @@ -316,7 +316,7 @@ class BuildExtTestCase(TempdirManager, self.assertRegex(cmd.get_ext_filename(modules[0].name), r'foo(_d)?\..*') self.assertRegex(cmd.get_ext_filename(modules[1].name), r'föö(_d)?\..*') self.assertEqual(cmd.get_export_symbols(modules[0]), ['PyInit_foo']) - self.assertEqual(cmd.get_export_symbols(modules[1]), ['PyInitU_f_gkaa']) + self.assertEqual(cmd.get_export_symbols(modules[1]), ['PyInitU_f_1gaa']) def test_compiler_option(self): # cmd.compiler is an option and diff --git a/setuptools/_distutils/tests/test_filelist.py b/setuptools/_distutils/tests/test_filelist.py index d8e4b39f..9ec507b5 100644 --- a/setuptools/_distutils/tests/test_filelist.py +++ b/setuptools/_distutils/tests/test_filelist.py @@ -331,6 +331,16 @@ class FindAllTestCase(unittest.TestCase): expected = [file1] self.assertEqual(filelist.findall(temp_dir), expected) + @os_helper.skip_unless_symlink + def test_symlink_loop(self): + with os_helper.temp_dir() as temp_dir: + link = os.path.join(temp_dir, 'link-to-parent') + content = os.path.join(temp_dir, 'somefile') + os_helper.create_empty_file(content) + os.symlink('.', link) + files = filelist.findall(temp_dir) + assert len(files) == 1 + def test_suite(): return unittest.TestSuite([ diff --git a/setuptools/_distutils/tests/test_sysconfig.py b/setuptools/_distutils/tests/test_sysconfig.py index c7571942..80cd1599 100644 --- a/setuptools/_distutils/tests/test_sysconfig.py +++ b/setuptools/_distutils/tests/test_sysconfig.py @@ -9,6 +9,7 @@ import unittest from distutils import sysconfig from distutils.ccompiler import get_default_compiler +from distutils.unixccompiler import UnixCCompiler from distutils.tests import support from test.support import run_unittest, swap_item @@ -84,9 +85,14 @@ class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): # make sure AR gets caught class compiler: compiler_type = 'unix' + executables = UnixCCompiler.executables + + def __init__(self): + self.exes = {} def set_executables(self, **kw): - self.exes = kw + for k, v in kw.items(): + self.exes[k] = v sysconfig_vars = { 'AR': 'sc_ar', @@ -125,6 +131,7 @@ class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): os.environ['ARFLAGS'] = '--env-arflags' os.environ['CFLAGS'] = '--env-cflags' os.environ['CPPFLAGS'] = '--env-cppflags' + os.environ['RANLIB'] = 'env_ranlib' comp = self.customize_compiler() self.assertEqual(comp.exes['archiver'], @@ -145,6 +152,12 @@ class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): ' --env-cppflags')) self.assertEqual(comp.shared_lib_extension, 'sc_shutil_suffix') + if sys.platform == "darwin": + self.assertEqual(comp.exes['ranlib'], + 'env_ranlib') + else: + self.assertTrue('ranlib' not in comp.exes) + del os.environ['AR'] del os.environ['CC'] del os.environ['CPP'] @@ -154,6 +167,7 @@ class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): del os.environ['ARFLAGS'] del os.environ['CFLAGS'] del os.environ['CPPFLAGS'] + del os.environ['RANLIB'] comp = self.customize_compiler() self.assertEqual(comp.exes['archiver'], @@ -171,6 +185,7 @@ class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): self.assertEqual(comp.exes['linker_so'], 'sc_ldshared') self.assertEqual(comp.shared_lib_extension, 'sc_shutil_suffix') + self.assertTrue('ranlib' not in comp.exes) def test_parse_makefile_base(self): self.makefile = TESTFN diff --git a/setuptools/_distutils/tests/test_unixccompiler.py b/setuptools/_distutils/tests/test_unixccompiler.py index 1828ba1a..ee2fe99c 100644 --- a/setuptools/_distutils/tests/test_unixccompiler.py +++ b/setuptools/_distutils/tests/test_unixccompiler.py @@ -1,4 +1,5 @@ """Tests for distutils.unixccompiler.""" +import os import sys import unittest from test.support import run_unittest @@ -6,7 +7,9 @@ from test.support import run_unittest from .py38compat import EnvironmentVarGuard from distutils import sysconfig +from distutils.errors import DistutilsPlatformError from distutils.unixccompiler import UnixCCompiler +from distutils.util import _clear_cached_macosx_ver class UnixCCompilerTestCase(unittest.TestCase): @@ -26,18 +29,90 @@ class UnixCCompilerTestCase(unittest.TestCase): @unittest.skipIf(sys.platform == 'win32', "can't test on Windows") def test_runtime_libdir_option(self): - # Issue#5900 + # Issue #5900; GitHub Issue #37 # # Ensure RUNPATH is added to extension modules with RPATH if # GNU ld is used # darwin sys.platform = 'darwin' - self.assertEqual(self.cc.rpath_foo(), '-L/foo') + darwin_ver_var = 'MACOSX_DEPLOYMENT_TARGET' + darwin_rpath_flag = '-Wl,-rpath,/foo' + darwin_lib_flag = '-L/foo' + + # (macOS version from syscfg, macOS version from env var) -> flag + # Version value of None generates two tests: as None and as empty string + # Expected flag value of None means an mismatch exception is expected + darwin_test_cases = [ + ((None , None ), darwin_lib_flag), + ((None , '11' ), darwin_rpath_flag), + (('10' , None ), darwin_lib_flag), + (('10.3' , None ), darwin_lib_flag), + (('10.3.1', None ), darwin_lib_flag), + (('10.5' , None ), darwin_rpath_flag), + (('10.5.1', None ), darwin_rpath_flag), + (('10.3' , '10.3' ), darwin_lib_flag), + (('10.3' , '10.5' ), darwin_rpath_flag), + (('10.5' , '10.3' ), darwin_lib_flag), + (('10.5' , '11' ), darwin_rpath_flag), + (('10.4' , '10' ), None), + ] + + def make_darwin_gcv(syscfg_macosx_ver): + def gcv(var): + if var == darwin_ver_var: + return syscfg_macosx_ver + return "xxx" + return gcv + + def do_darwin_test(syscfg_macosx_ver, env_macosx_ver, expected_flag): + env = os.environ + msg = "macOS version = (sysconfig=%r, env=%r)" % \ + (syscfg_macosx_ver, env_macosx_ver) + + # Save + old_gcv = sysconfig.get_config_var + old_env_macosx_ver = env.get(darwin_ver_var) + + # Setup environment + _clear_cached_macosx_ver() + sysconfig.get_config_var = make_darwin_gcv(syscfg_macosx_ver) + if env_macosx_ver is not None: + env[darwin_ver_var] = env_macosx_ver + elif darwin_ver_var in env: + env.pop(darwin_ver_var) + + # Run the test + if expected_flag is not None: + self.assertEqual(self.cc.rpath_foo(), expected_flag, msg=msg) + else: + with self.assertRaisesRegex(DistutilsPlatformError, + darwin_ver_var + r' mismatch', msg=msg): + self.cc.rpath_foo() + + # Restore + if old_env_macosx_ver is not None: + env[darwin_ver_var] = old_env_macosx_ver + elif darwin_ver_var in env: + env.pop(darwin_ver_var) + sysconfig.get_config_var = old_gcv + _clear_cached_macosx_ver() + + for macosx_vers, expected_flag in darwin_test_cases: + syscfg_macosx_ver, env_macosx_ver = macosx_vers + do_darwin_test(syscfg_macosx_ver, env_macosx_ver, expected_flag) + # Bonus test cases with None interpreted as empty string + if syscfg_macosx_ver is None: + do_darwin_test("", env_macosx_ver, expected_flag) + if env_macosx_ver is None: + do_darwin_test(syscfg_macosx_ver, "", expected_flag) + if syscfg_macosx_ver is None and env_macosx_ver is None: + do_darwin_test("", "", expected_flag) + + old_gcv = sysconfig.get_config_var # hp-ux sys.platform = 'hp-ux' - old_gcv = sysconfig.get_config_var def gcv(v): return 'xxx' sysconfig.get_config_var = gcv @@ -65,6 +140,14 @@ class UnixCCompilerTestCase(unittest.TestCase): sysconfig.get_config_var = gcv self.assertEqual(self.cc.rpath_foo(), '-Wl,--enable-new-dtags,-R/foo') + def gcv(v): + if v == 'CC': + return 'gcc -pthread -B /bar' + elif v == 'GNULD': + return 'yes' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), '-Wl,--enable-new-dtags,-R/foo') + # GCC non-GNULD sys.platform = 'bar' def gcv(v): @@ -94,7 +177,7 @@ class UnixCCompilerTestCase(unittest.TestCase): elif v == 'GNULD': return 'yes' sysconfig.get_config_var = gcv - self.assertEqual(self.cc.rpath_foo(), '-R/foo') + self.assertEqual(self.cc.rpath_foo(), '-Wl,--enable-new-dtags,-R/foo') # non-GCC non-GNULD sys.platform = 'bar' @@ -104,10 +187,10 @@ class UnixCCompilerTestCase(unittest.TestCase): elif v == 'GNULD': return 'no' sysconfig.get_config_var = gcv - self.assertEqual(self.cc.rpath_foo(), '-R/foo') + self.assertEqual(self.cc.rpath_foo(), '-Wl,-R/foo') - @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for OS X') - def test_osx_cc_overrides_ldshared(self): + @unittest.skipIf(sys.platform == 'win32', "can't test on Windows") + def test_cc_overrides_ldshared(self): # Issue #18080: # ensure that setting CC env variable also changes default linker def gcv(v): @@ -127,8 +210,8 @@ class UnixCCompilerTestCase(unittest.TestCase): sysconfig.customize_compiler(self.cc) self.assertEqual(self.cc.linker_so[0], 'my_cc') - @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for OS X') - def test_osx_explicit_ldshared(self): + @unittest.skipIf(sys.platform == 'win32', "can't test on Windows") + def test_explicit_ldshared(self): # Issue #18080: # ensure that setting CC env variable does not change # explicit LDSHARED setting for linker diff --git a/setuptools/_distutils/unixccompiler.py b/setuptools/_distutils/unixccompiler.py index 4d7a6de7..a07e5988 100644 --- a/setuptools/_distutils/unixccompiler.py +++ b/setuptools/_distutils/unixccompiler.py @@ -13,7 +13,7 @@ the "typical" Unix-style command-line C compiler: * link shared library handled by 'cc -shared' """ -import os, sys, re +import os, sys, re, shlex from distutils import sysconfig from distutils.dep_util import newer @@ -231,33 +231,30 @@ class UnixCCompiler(CCompiler): # this time, there's no way to determine this information from # the configuration data stored in the Python installation, so # we use this hack. - compiler = os.path.basename(sysconfig.get_config_var("CC")) + compiler = os.path.basename(shlex.split(sysconfig.get_config_var("CC"))[0]) if sys.platform[:6] == "darwin": - # MacOSX's linker doesn't understand the -R flag at all - return "-L" + dir + from distutils.util import get_macosx_target_ver, split_version + macosx_target_ver = get_macosx_target_ver() + if macosx_target_ver and split_version(macosx_target_ver) >= [10, 5]: + return "-Wl,-rpath," + dir + else: # no support for -rpath on earlier macOS versions + return "-L" + dir elif sys.platform[:7] == "freebsd": return "-Wl,-rpath=" + dir elif sys.platform[:5] == "hp-ux": if self._is_gcc(compiler): return ["-Wl,+s", "-L" + dir] return ["+s", "-L" + dir] + + # For all compilers, `-Wl` is the presumed way to + # pass a compiler option to the linker and `-R` is + # the way to pass an RPATH. + if sysconfig.get_config_var("GNULD") == "yes": + # GNU ld needs an extra option to get a RUNPATH + # instead of just an RPATH. + return "-Wl,--enable-new-dtags,-R" + dir else: - if self._is_gcc(compiler): - # gcc on non-GNU systems does not need -Wl, but can - # use it anyway. Since distutils has always passed in - # -Wl whenever gcc was used in the past it is probably - # safest to keep doing so. - if sysconfig.get_config_var("GNULD") == "yes": - # GNU ld needs an extra option to get a RUNPATH - # instead of just an RPATH. - return "-Wl,--enable-new-dtags,-R" + dir - else: - return "-Wl,-R" + dir - else: - # No idea how --enable-new-dtags would be passed on to - # ld if this system was using GNU ld. Don't know if a - # system like this even exists. - return "-R" + dir + return "-Wl,-R" + dir def library_option(self, lib): return "-l" + lib diff --git a/setuptools/_distutils/util.py b/setuptools/_distutils/util.py index f5aca794..afc23c4e 100644 --- a/setuptools/_distutils/util.py +++ b/setuptools/_distutils/util.py @@ -103,11 +103,66 @@ def get_platform(): 'x86' : 'win32', 'x64' : 'win-amd64', 'arm' : 'win-arm32', + 'arm64': 'win-arm64', } return TARGET_TO_PLAT.get(os.environ.get('VSCMD_ARG_TGT_ARCH')) or get_host_platform() else: return get_host_platform() + +if sys.platform == 'darwin': + _syscfg_macosx_ver = None # cache the version pulled from sysconfig +MACOSX_VERSION_VAR = 'MACOSX_DEPLOYMENT_TARGET' + +def _clear_cached_macosx_ver(): + """For testing only. Do not call.""" + global _syscfg_macosx_ver + _syscfg_macosx_ver = None + +def get_macosx_target_ver_from_syscfg(): + """Get the version of macOS latched in the Python interpreter configuration. + Returns the version as a string or None if can't obtain one. Cached.""" + global _syscfg_macosx_ver + if _syscfg_macosx_ver is None: + from distutils import sysconfig + ver = sysconfig.get_config_var(MACOSX_VERSION_VAR) or '' + if ver: + _syscfg_macosx_ver = ver + return _syscfg_macosx_ver + +def get_macosx_target_ver(): + """Return the version of macOS for which we are building. + + The target version defaults to the version in sysconfig latched at time + the Python interpreter was built, unless overridden by an environment + variable. If neither source has a value, then None is returned""" + + syscfg_ver = get_macosx_target_ver_from_syscfg() + env_ver = os.environ.get(MACOSX_VERSION_VAR) + + if env_ver: + # Validate overridden version against sysconfig version, if have both. + # Ensure that the deployment target of the build process is not less + # than 10.3 if the interpreter was built for 10.3 or later. This + # ensures extension modules are built with correct compatibility + # values, specifically LDSHARED which can use + # '-undefined dynamic_lookup' which only works on >= 10.3. + if syscfg_ver and split_version(syscfg_ver) >= [10, 3] and \ + split_version(env_ver) < [10, 3]: + my_msg = ('$' + MACOSX_VERSION_VAR + ' mismatch: ' + 'now "%s" but "%s" during configure; ' + 'must use 10.3 or later' + % (env_ver, syscfg_ver)) + raise DistutilsPlatformError(my_msg) + return env_ver + return syscfg_ver + + +def split_version(s): + """Convert a dot-separated string into a list of numbers for comparisons""" + return [int(n) for n in s.split('.')] + + def convert_path (pathname): """Return 'pathname' as a name that will work on the native filesystem, i.e. split it on '/' and put it back together again using the current @@ -478,84 +533,3 @@ def rfc822_escape (header): lines = header.split('\n') sep = '\n' + 8 * ' ' return sep.join(lines) - -# 2to3 support - -def run_2to3(files, fixer_names=None, options=None, explicit=None): - """Invoke 2to3 on a list of Python files. - The files should all come from the build area, as the - modification is done in-place. To reduce the build time, - only files modified since the last invocation of this - function should be passed in the files argument.""" - - if not files: - return - - # Make this class local, to delay import of 2to3 - from lib2to3.refactor import RefactoringTool, get_fixers_from_package - class DistutilsRefactoringTool(RefactoringTool): - def log_error(self, msg, *args, **kw): - log.error(msg, *args) - - def log_message(self, msg, *args): - log.info(msg, *args) - - def log_debug(self, msg, *args): - log.debug(msg, *args) - - if fixer_names is None: - fixer_names = get_fixers_from_package('lib2to3.fixes') - r = DistutilsRefactoringTool(fixer_names, options=options) - r.refactor(files, write=True) - -def copydir_run_2to3(src, dest, template=None, fixer_names=None, - options=None, explicit=None): - """Recursively copy a directory, only copying new and changed files, - running run_2to3 over all newly copied Python modules afterward. - - If you give a template string, it's parsed like a MANIFEST.in. - """ - from distutils.dir_util import mkpath - from distutils.file_util import copy_file - from distutils.filelist import FileList - filelist = FileList() - curdir = os.getcwd() - os.chdir(src) - try: - filelist.findall() - finally: - os.chdir(curdir) - filelist.files[:] = filelist.allfiles - if template: - for line in template.splitlines(): - line = line.strip() - if not line: continue - filelist.process_template_line(line) - copied = [] - for filename in filelist.files: - outname = os.path.join(dest, filename) - mkpath(os.path.dirname(outname)) - res = copy_file(os.path.join(src, filename), outname, update=1) - if res[1]: copied.append(outname) - run_2to3([fn for fn in copied if fn.lower().endswith('.py')], - fixer_names=fixer_names, options=options, explicit=explicit) - return copied - -class Mixin2to3: - '''Mixin class for commands that run 2to3. - To configure 2to3, setup scripts may either change - the class variables, or inherit from individual commands - to override how 2to3 is invoked.''' - - # provide list of fixers to run; - # defaults to all from lib2to3.fixers - fixer_names = None - - # options dictionary - options = None - - # list of fixers to invoke even though they are marked as explicit - explicit = None - - def run_2to3(self, files): - return run_2to3(files, self.fixer_names, self.options, self.explicit) diff --git a/setuptools/_imp.py b/setuptools/_imp.py index 451e45a8..47efd792 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -41,12 +41,12 @@ def find_module(module, paths=None): spec.loader, importlib.machinery.FrozenImporter): kind = PY_FROZEN path = None # imp compabilty - suffix = mode = '' # imp compability + suffix = mode = '' # imp compatibility elif spec.origin == 'built-in' or static and issubclass( spec.loader, importlib.machinery.BuiltinImporter): kind = C_BUILTIN path = None # imp compabilty - suffix = mode = '' # imp compability + suffix = mode = '' # imp compatibility elif spec.has_location: path = spec.origin suffix = os.path.splitext(path)[1] diff --git a/setuptools/_vendor/more_itertools/__init__.py b/setuptools/_vendor/more_itertools/__init__.py new file mode 100644 index 00000000..19a169fc --- /dev/null +++ b/setuptools/_vendor/more_itertools/__init__.py @@ -0,0 +1,4 @@ +from .more import * # noqa +from .recipes import * # noqa + +__version__ = '8.8.0' diff --git a/setuptools/_vendor/more_itertools/__init__.pyi b/setuptools/_vendor/more_itertools/__init__.pyi new file mode 100644 index 00000000..96f6e36c --- /dev/null +++ b/setuptools/_vendor/more_itertools/__init__.pyi @@ -0,0 +1,2 @@ +from .more import * +from .recipes import * diff --git a/setuptools/_vendor/more_itertools/more.py b/setuptools/_vendor/more_itertools/more.py new file mode 100644 index 00000000..0f7d282a --- /dev/null +++ b/setuptools/_vendor/more_itertools/more.py @@ -0,0 +1,3825 @@ +import warnings + +from collections import Counter, defaultdict, deque, abc +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor +from functools import partial, reduce, wraps +from heapq import merge, heapify, heapreplace, heappop +from itertools import ( + chain, + compress, + count, + cycle, + dropwhile, + groupby, + islice, + repeat, + starmap, + takewhile, + tee, + zip_longest, +) +from math import exp, factorial, floor, log +from queue import Empty, Queue +from random import random, randrange, uniform +from operator import itemgetter, mul, sub, gt, lt +from sys import hexversion, maxsize +from time import monotonic + +from .recipes import ( + consume, + flatten, + pairwise, + powerset, + take, + unique_everseen, +) + +__all__ = [ + 'AbortThread', + 'adjacent', + 'always_iterable', + 'always_reversible', + 'bucket', + 'callback_iter', + 'chunked', + 'circular_shifts', + 'collapse', + 'collate', + 'consecutive_groups', + 'consumer', + 'countable', + 'count_cycle', + 'mark_ends', + 'difference', + 'distinct_combinations', + 'distinct_permutations', + 'distribute', + 'divide', + 'exactly_n', + 'filter_except', + 'first', + 'groupby_transform', + 'ilen', + 'interleave_longest', + 'interleave', + 'intersperse', + 'islice_extended', + 'iterate', + 'ichunked', + 'is_sorted', + 'last', + 'locate', + 'lstrip', + 'make_decorator', + 'map_except', + 'map_reduce', + 'nth_or_last', + 'nth_permutation', + 'nth_product', + 'numeric_range', + 'one', + 'only', + 'padded', + 'partitions', + 'set_partitions', + 'peekable', + 'repeat_last', + 'replace', + 'rlocate', + 'rstrip', + 'run_length', + 'sample', + 'seekable', + 'SequenceView', + 'side_effect', + 'sliced', + 'sort_together', + 'split_at', + 'split_after', + 'split_before', + 'split_when', + 'split_into', + 'spy', + 'stagger', + 'strip', + 'substrings', + 'substrings_indexes', + 'time_limited', + 'unique_to_each', + 'unzip', + 'windowed', + 'with_iter', + 'UnequalIterablesError', + 'zip_equal', + 'zip_offset', + 'windowed_complete', + 'all_unique', + 'value_chain', + 'product_index', + 'combination_index', + 'permutation_index', +] + +_marker = object() + + +def chunked(iterable, n, strict=False): + """Break *iterable* into lists of length *n*: + + >>> list(chunked([1, 2, 3, 4, 5, 6], 3)) + [[1, 2, 3], [4, 5, 6]] + + By the default, the last yielded list will have fewer than *n* elements + if the length of *iterable* is not divisible by *n*: + + >>> list(chunked([1, 2, 3, 4, 5, 6, 7, 8], 3)) + [[1, 2, 3], [4, 5, 6], [7, 8]] + + To use a fill-in value instead, see the :func:`grouper` recipe. + + If the length of *iterable* is not divisible by *n* and *strict* is + ``True``, then ``ValueError`` will be raised before the last + list is yielded. + + """ + iterator = iter(partial(take, n, iter(iterable)), []) + if strict: + + def ret(): + for chunk in iterator: + if len(chunk) != n: + raise ValueError('iterable is not divisible by n.') + yield chunk + + return iter(ret()) + else: + return iterator + + +def first(iterable, default=_marker): + """Return the first item of *iterable*, or *default* if *iterable* is + empty. + + >>> first([0, 1, 2, 3]) + 0 + >>> first([], 'some default') + 'some default' + + If *default* is not provided and there are no items in the iterable, + raise ``ValueError``. + + :func:`first` is useful when you have a generator of expensive-to-retrieve + values and want any arbitrary one. It is marginally shorter than + ``next(iter(iterable), default)``. + + """ + try: + return next(iter(iterable)) + except StopIteration as e: + if default is _marker: + raise ValueError( + 'first() was called on an empty iterable, and no ' + 'default value was provided.' + ) from e + return default + + +def last(iterable, default=_marker): + """Return the last item of *iterable*, or *default* if *iterable* is + empty. + + >>> last([0, 1, 2, 3]) + 3 + >>> last([], 'some default') + 'some default' + + If *default* is not provided and there are no items in the iterable, + raise ``ValueError``. + """ + try: + if isinstance(iterable, Sequence): + return iterable[-1] + # Work around https://bugs.python.org/issue38525 + elif hasattr(iterable, '__reversed__') and (hexversion != 0x030800F0): + return next(reversed(iterable)) + else: + return deque(iterable, maxlen=1)[-1] + except (IndexError, TypeError, StopIteration): + if default is _marker: + raise ValueError( + 'last() was called on an empty iterable, and no default was ' + 'provided.' + ) + return default + + +def nth_or_last(iterable, n, default=_marker): + """Return the nth or the last item of *iterable*, + or *default* if *iterable* is empty. + + >>> nth_or_last([0, 1, 2, 3], 2) + 2 + >>> nth_or_last([0, 1], 2) + 1 + >>> nth_or_last([], 0, 'some default') + 'some default' + + If *default* is not provided and there are no items in the iterable, + raise ``ValueError``. + """ + return last(islice(iterable, n + 1), default=default) + + +class peekable: + """Wrap an iterator to allow lookahead and prepending elements. + + Call :meth:`peek` on the result to get the value that will be returned + by :func:`next`. This won't advance the iterator: + + >>> p = peekable(['a', 'b']) + >>> p.peek() + 'a' + >>> next(p) + 'a' + + Pass :meth:`peek` a default value to return that instead of raising + ``StopIteration`` when the iterator is exhausted. + + >>> p = peekable([]) + >>> p.peek('hi') + 'hi' + + peekables also offer a :meth:`prepend` method, which "inserts" items + at the head of the iterable: + + >>> p = peekable([1, 2, 3]) + >>> p.prepend(10, 11, 12) + >>> next(p) + 10 + >>> p.peek() + 11 + >>> list(p) + [11, 12, 1, 2, 3] + + peekables can be indexed. Index 0 is the item that will be returned by + :func:`next`, index 1 is the item after that, and so on: + The values up to the given index will be cached. + + >>> p = peekable(['a', 'b', 'c', 'd']) + >>> p[0] + 'a' + >>> p[1] + 'b' + >>> next(p) + 'a' + + Negative indexes are supported, but be aware that they will cache the + remaining items in the source iterator, which may require significant + storage. + + To check whether a peekable is exhausted, check its truth value: + + >>> p = peekable(['a', 'b']) + >>> if p: # peekable has items + ... list(p) + ['a', 'b'] + >>> if not p: # peekable is exhausted + ... list(p) + [] + + """ + + def __init__(self, iterable): + self._it = iter(iterable) + self._cache = deque() + + def __iter__(self): + return self + + def __bool__(self): + try: + self.peek() + except StopIteration: + return False + return True + + def peek(self, default=_marker): + """Return the item that will be next returned from ``next()``. + + Return ``default`` if there are no items left. If ``default`` is not + provided, raise ``StopIteration``. + + """ + if not self._cache: + try: + self._cache.append(next(self._it)) + except StopIteration: + if default is _marker: + raise + return default + return self._cache[0] + + def prepend(self, *items): + """Stack up items to be the next ones returned from ``next()`` or + ``self.peek()``. The items will be returned in + first in, first out order:: + + >>> p = peekable([1, 2, 3]) + >>> p.prepend(10, 11, 12) + >>> next(p) + 10 + >>> list(p) + [11, 12, 1, 2, 3] + + It is possible, by prepending items, to "resurrect" a peekable that + previously raised ``StopIteration``. + + >>> p = peekable([]) + >>> next(p) + Traceback (most recent call last): + ... + StopIteration + >>> p.prepend(1) + >>> next(p) + 1 + >>> next(p) + Traceback (most recent call last): + ... + StopIteration + + """ + self._cache.extendleft(reversed(items)) + + def __next__(self): + if self._cache: + return self._cache.popleft() + + return next(self._it) + + def _get_slice(self, index): + # Normalize the slice's arguments + step = 1 if (index.step is None) else index.step + if step > 0: + start = 0 if (index.start is None) else index.start + stop = maxsize if (index.stop is None) else index.stop + elif step < 0: + start = -1 if (index.start is None) else index.start + stop = (-maxsize - 1) if (index.stop is None) else index.stop + else: + raise ValueError('slice step cannot be zero') + + # If either the start or stop index is negative, we'll need to cache + # the rest of the iterable in order to slice from the right side. + if (start < 0) or (stop < 0): + self._cache.extend(self._it) + # Otherwise we'll need to find the rightmost index and cache to that + # point. + else: + n = min(max(start, stop) + 1, maxsize) + cache_len = len(self._cache) + if n >= cache_len: + self._cache.extend(islice(self._it, n - cache_len)) + + return list(self._cache)[index] + + def __getitem__(self, index): + if isinstance(index, slice): + return self._get_slice(index) + + cache_len = len(self._cache) + if index < 0: + self._cache.extend(self._it) + elif index >= cache_len: + self._cache.extend(islice(self._it, index + 1 - cache_len)) + + return self._cache[index] + + +def collate(*iterables, **kwargs): + """Return a sorted merge of the items from each of several already-sorted + *iterables*. + + >>> list(collate('ACDZ', 'AZ', 'JKL')) + ['A', 'A', 'C', 'D', 'J', 'K', 'L', 'Z', 'Z'] + + Works lazily, keeping only the next value from each iterable in memory. Use + :func:`collate` to, for example, perform a n-way mergesort of items that + don't fit in memory. + + If a *key* function is specified, the iterables will be sorted according + to its result: + + >>> key = lambda s: int(s) # Sort by numeric value, not by string + >>> list(collate(['1', '10'], ['2', '11'], key=key)) + ['1', '2', '10', '11'] + + + If the *iterables* are sorted in descending order, set *reverse* to + ``True``: + + >>> list(collate([5, 3, 1], [4, 2, 0], reverse=True)) + [5, 4, 3, 2, 1, 0] + + If the elements of the passed-in iterables are out of order, you might get + unexpected results. + + On Python 3.5+, this function is an alias for :func:`heapq.merge`. + + """ + warnings.warn( + "collate is no longer part of more_itertools, use heapq.merge", + DeprecationWarning, + ) + return merge(*iterables, **kwargs) + + +def consumer(func): + """Decorator that automatically advances a PEP-342-style "reverse iterator" + to its first yield point so you don't have to call ``next()`` on it + manually. + + >>> @consumer + ... def tally(): + ... i = 0 + ... while True: + ... print('Thing number %s is %s.' % (i, (yield))) + ... i += 1 + ... + >>> t = tally() + >>> t.send('red') + Thing number 0 is red. + >>> t.send('fish') + Thing number 1 is fish. + + Without the decorator, you would have to call ``next(t)`` before + ``t.send()`` could be used. + + """ + + @wraps(func) + def wrapper(*args, **kwargs): + gen = func(*args, **kwargs) + next(gen) + return gen + + return wrapper + + +def ilen(iterable): + """Return the number of items in *iterable*. + + >>> ilen(x for x in range(1000000) if x % 3 == 0) + 333334 + + This consumes the iterable, so handle with care. + + """ + # This approach was selected because benchmarks showed it's likely the + # fastest of the known implementations at the time of writing. + # See GitHub tracker: #236, #230. + counter = count() + deque(zip(iterable, counter), maxlen=0) + return next(counter) + + +def iterate(func, start): + """Return ``start``, ``func(start)``, ``func(func(start))``, ... + + >>> from itertools import islice + >>> list(islice(iterate(lambda x: 2*x, 1), 10)) + [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + """ + while True: + yield start + start = func(start) + + +def with_iter(context_manager): + """Wrap an iterable in a ``with`` statement, so it closes once exhausted. + + For example, this will close the file when the iterator is exhausted:: + + upper_lines = (line.upper() for line in with_iter(open('foo'))) + + Any context manager which returns an iterable is a candidate for + ``with_iter``. + + """ + with context_manager as iterable: + yield from iterable + + +def one(iterable, too_short=None, too_long=None): + """Return the first item from *iterable*, which is expected to contain only + that item. Raise an exception if *iterable* is empty or has more than one + item. + + :func:`one` is useful for ensuring that an iterable contains only one item. + For example, it can be used to retrieve the result of a database query + that is expected to return a single row. + + If *iterable* is empty, ``ValueError`` will be raised. You may specify a + different exception with the *too_short* keyword: + + >>> it = [] + >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: too many items in iterable (expected 1)' + >>> too_short = IndexError('too few items') + >>> one(it, too_short=too_short) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + IndexError: too few items + + Similarly, if *iterable* contains more than one item, ``ValueError`` will + be raised. You may specify a different exception with the *too_long* + keyword: + + >>> it = ['too', 'many'] + >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Expected exactly one item in iterable, but got 'too', + 'many', and perhaps more. + >>> too_long = RuntimeError + >>> one(it, too_long=too_long) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + RuntimeError + + Note that :func:`one` attempts to advance *iterable* twice to ensure there + is only one item. See :func:`spy` or :func:`peekable` to check iterable + contents less destructively. + + """ + it = iter(iterable) + + try: + first_value = next(it) + except StopIteration as e: + raise ( + too_short or ValueError('too few items in iterable (expected 1)') + ) from e + + try: + second_value = next(it) + except StopIteration: + pass + else: + msg = ( + 'Expected exactly one item in iterable, but got {!r}, {!r}, ' + 'and perhaps more.'.format(first_value, second_value) + ) + raise too_long or ValueError(msg) + + return first_value + + +def distinct_permutations(iterable, r=None): + """Yield successive distinct permutations of the elements in *iterable*. + + >>> sorted(distinct_permutations([1, 0, 1])) + [(0, 1, 1), (1, 0, 1), (1, 1, 0)] + + Equivalent to ``set(permutations(iterable))``, except duplicates are not + generated and thrown away. For larger input sequences this is much more + efficient. + + Duplicate permutations arise when there are duplicated elements in the + input iterable. The number of items returned is + `n! / (x_1! * x_2! * ... * x_n!)`, where `n` is the total number of + items input, and each `x_i` is the count of a distinct item in the input + sequence. + + If *r* is given, only the *r*-length permutations are yielded. + + >>> sorted(distinct_permutations([1, 0, 1], r=2)) + [(0, 1), (1, 0), (1, 1)] + >>> sorted(distinct_permutations(range(3), r=2)) + [(0, 1), (0, 2), (1, 0), (1, 2), (2, 0), (2, 1)] + + """ + # Algorithm: https://w.wiki/Qai + def _full(A): + while True: + # Yield the permutation we have + yield tuple(A) + + # Find the largest index i such that A[i] < A[i + 1] + for i in range(size - 2, -1, -1): + if A[i] < A[i + 1]: + break + # If no such index exists, this permutation is the last one + else: + return + + # Find the largest index j greater than j such that A[i] < A[j] + for j in range(size - 1, i, -1): + if A[i] < A[j]: + break + + # Swap the value of A[i] with that of A[j], then reverse the + # sequence from A[i + 1] to form the new permutation + A[i], A[j] = A[j], A[i] + A[i + 1 :] = A[: i - size : -1] # A[i + 1:][::-1] + + # Algorithm: modified from the above + def _partial(A, r): + # Split A into the first r items and the last r items + head, tail = A[:r], A[r:] + right_head_indexes = range(r - 1, -1, -1) + left_tail_indexes = range(len(tail)) + + while True: + # Yield the permutation we have + yield tuple(head) + + # Starting from the right, find the first index of the head with + # value smaller than the maximum value of the tail - call it i. + pivot = tail[-1] + for i in right_head_indexes: + if head[i] < pivot: + break + pivot = head[i] + else: + return + + # Starting from the left, find the first value of the tail + # with a value greater than head[i] and swap. + for j in left_tail_indexes: + if tail[j] > head[i]: + head[i], tail[j] = tail[j], head[i] + break + # If we didn't find one, start from the right and find the first + # index of the head with a value greater than head[i] and swap. + else: + for j in right_head_indexes: + if head[j] > head[i]: + head[i], head[j] = head[j], head[i] + break + + # Reverse head[i + 1:] and swap it with tail[:r - (i + 1)] + tail += head[: i - r : -1] # head[i + 1:][::-1] + i += 1 + head[i:], tail[:] = tail[: r - i], tail[r - i :] + + items = sorted(iterable) + + size = len(items) + if r is None: + r = size + + if 0 < r <= size: + return _full(items) if (r == size) else _partial(items, r) + + return iter(() if r else ((),)) + + +def intersperse(e, iterable, n=1): + """Intersperse filler element *e* among the items in *iterable*, leaving + *n* items between each filler element. + + >>> list(intersperse('!', [1, 2, 3, 4, 5])) + [1, '!', 2, '!', 3, '!', 4, '!', 5] + + >>> list(intersperse(None, [1, 2, 3, 4, 5], n=2)) + [1, 2, None, 3, 4, None, 5] + + """ + if n == 0: + raise ValueError('n must be > 0') + elif n == 1: + # interleave(repeat(e), iterable) -> e, x_0, e, e, x_1, e, x_2... + # islice(..., 1, None) -> x_0, e, e, x_1, e, x_2... + return islice(interleave(repeat(e), iterable), 1, None) + else: + # interleave(filler, chunks) -> [e], [x_0, x_1], [e], [x_2, x_3]... + # islice(..., 1, None) -> [x_0, x_1], [e], [x_2, x_3]... + # flatten(...) -> x_0, x_1, e, x_2, x_3... + filler = repeat([e]) + chunks = chunked(iterable, n) + return flatten(islice(interleave(filler, chunks), 1, None)) + + +def unique_to_each(*iterables): + """Return the elements from each of the input iterables that aren't in the + other input iterables. + + For example, suppose you have a set of packages, each with a set of + dependencies:: + + {'pkg_1': {'A', 'B'}, 'pkg_2': {'B', 'C'}, 'pkg_3': {'B', 'D'}} + + If you remove one package, which dependencies can also be removed? + + If ``pkg_1`` is removed, then ``A`` is no longer necessary - it is not + associated with ``pkg_2`` or ``pkg_3``. Similarly, ``C`` is only needed for + ``pkg_2``, and ``D`` is only needed for ``pkg_3``:: + + >>> unique_to_each({'A', 'B'}, {'B', 'C'}, {'B', 'D'}) + [['A'], ['C'], ['D']] + + If there are duplicates in one input iterable that aren't in the others + they will be duplicated in the output. Input order is preserved:: + + >>> unique_to_each("mississippi", "missouri") + [['p', 'p'], ['o', 'u', 'r']] + + It is assumed that the elements of each iterable are hashable. + + """ + pool = [list(it) for it in iterables] + counts = Counter(chain.from_iterable(map(set, pool))) + uniques = {element for element in counts if counts[element] == 1} + return [list(filter(uniques.__contains__, it)) for it in pool] + + +def windowed(seq, n, fillvalue=None, step=1): + """Return a sliding window of width *n* over the given iterable. + + >>> all_windows = windowed([1, 2, 3, 4, 5], 3) + >>> list(all_windows) + [(1, 2, 3), (2, 3, 4), (3, 4, 5)] + + When the window is larger than the iterable, *fillvalue* is used in place + of missing values: + + >>> list(windowed([1, 2, 3], 4)) + [(1, 2, 3, None)] + + Each window will advance in increments of *step*: + + >>> list(windowed([1, 2, 3, 4, 5, 6], 3, fillvalue='!', step=2)) + [(1, 2, 3), (3, 4, 5), (5, 6, '!')] + + To slide into the iterable's items, use :func:`chain` to add filler items + to the left: + + >>> iterable = [1, 2, 3, 4] + >>> n = 3 + >>> padding = [None] * (n - 1) + >>> list(windowed(chain(padding, iterable), 3)) + [(None, None, 1), (None, 1, 2), (1, 2, 3), (2, 3, 4)] + """ + if n < 0: + raise ValueError('n must be >= 0') + if n == 0: + yield tuple() + return + if step < 1: + raise ValueError('step must be >= 1') + + window = deque(maxlen=n) + i = n + for _ in map(window.append, seq): + i -= 1 + if not i: + i = step + yield tuple(window) + + size = len(window) + if size < n: + yield tuple(chain(window, repeat(fillvalue, n - size))) + elif 0 < i < min(step, n): + window += (fillvalue,) * i + yield tuple(window) + + +def substrings(iterable): + """Yield all of the substrings of *iterable*. + + >>> [''.join(s) for s in substrings('more')] + ['m', 'o', 'r', 'e', 'mo', 'or', 're', 'mor', 'ore', 'more'] + + Note that non-string iterables can also be subdivided. + + >>> list(substrings([0, 1, 2])) + [(0,), (1,), (2,), (0, 1), (1, 2), (0, 1, 2)] + + """ + # The length-1 substrings + seq = [] + for item in iter(iterable): + seq.append(item) + yield (item,) + seq = tuple(seq) + item_count = len(seq) + + # And the rest + for n in range(2, item_count + 1): + for i in range(item_count - n + 1): + yield seq[i : i + n] + + +def substrings_indexes(seq, reverse=False): + """Yield all substrings and their positions in *seq* + + The items yielded will be a tuple of the form ``(substr, i, j)``, where + ``substr == seq[i:j]``. + + This function only works for iterables that support slicing, such as + ``str`` objects. + + >>> for item in substrings_indexes('more'): + ... print(item) + ('m', 0, 1) + ('o', 1, 2) + ('r', 2, 3) + ('e', 3, 4) + ('mo', 0, 2) + ('or', 1, 3) + ('re', 2, 4) + ('mor', 0, 3) + ('ore', 1, 4) + ('more', 0, 4) + + Set *reverse* to ``True`` to yield the same items in the opposite order. + + + """ + r = range(1, len(seq) + 1) + if reverse: + r = reversed(r) + return ( + (seq[i : i + L], i, i + L) for L in r for i in range(len(seq) - L + 1) + ) + + +class bucket: + """Wrap *iterable* and return an object that buckets it iterable into + child iterables based on a *key* function. + + >>> iterable = ['a1', 'b1', 'c1', 'a2', 'b2', 'c2', 'b3'] + >>> s = bucket(iterable, key=lambda x: x[0]) # Bucket by 1st character + >>> sorted(list(s)) # Get the keys + ['a', 'b', 'c'] + >>> a_iterable = s['a'] + >>> next(a_iterable) + 'a1' + >>> next(a_iterable) + 'a2' + >>> list(s['b']) + ['b1', 'b2', 'b3'] + + The original iterable will be advanced and its items will be cached until + they are used by the child iterables. This may require significant storage. + + By default, attempting to select a bucket to which no items belong will + exhaust the iterable and cache all values. + If you specify a *validator* function, selected buckets will instead be + checked against it. + + >>> from itertools import count + >>> it = count(1, 2) # Infinite sequence of odd numbers + >>> key = lambda x: x % 10 # Bucket by last digit + >>> validator = lambda x: x in {1, 3, 5, 7, 9} # Odd digits only + >>> s = bucket(it, key=key, validator=validator) + >>> 2 in s + False + >>> list(s[2]) + [] + + """ + + def __init__(self, iterable, key, validator=None): + self._it = iter(iterable) + self._key = key + self._cache = defaultdict(deque) + self._validator = validator or (lambda x: True) + + def __contains__(self, value): + if not self._validator(value): + return False + + try: + item = next(self[value]) + except StopIteration: + return False + else: + self._cache[value].appendleft(item) + + return True + + def _get_values(self, value): + """ + Helper to yield items from the parent iterator that match *value*. + Items that don't match are stored in the local cache as they + are encountered. + """ + while True: + # If we've cached some items that match the target value, emit + # the first one and evict it from the cache. + if self._cache[value]: + yield self._cache[value].popleft() + # Otherwise we need to advance the parent iterator to search for + # a matching item, caching the rest. + else: + while True: + try: + item = next(self._it) + except StopIteration: + return + item_value = self._key(item) + if item_value == value: + yield item + break + elif self._validator(item_value): + self._cache[item_value].append(item) + + def __iter__(self): + for item in self._it: + item_value = self._key(item) + if self._validator(item_value): + self._cache[item_value].append(item) + + yield from self._cache.keys() + + def __getitem__(self, value): + if not self._validator(value): + return iter(()) + + return self._get_values(value) + + +def spy(iterable, n=1): + """Return a 2-tuple with a list containing the first *n* elements of + *iterable*, and an iterator with the same items as *iterable*. + This allows you to "look ahead" at the items in the iterable without + advancing it. + + There is one item in the list by default: + + >>> iterable = 'abcdefg' + >>> head, iterable = spy(iterable) + >>> head + ['a'] + >>> list(iterable) + ['a', 'b', 'c', 'd', 'e', 'f', 'g'] + + You may use unpacking to retrieve items instead of lists: + + >>> (head,), iterable = spy('abcdefg') + >>> head + 'a' + >>> (first, second), iterable = spy('abcdefg', 2) + >>> first + 'a' + >>> second + 'b' + + The number of items requested can be larger than the number of items in + the iterable: + + >>> iterable = [1, 2, 3, 4, 5] + >>> head, iterable = spy(iterable, 10) + >>> head + [1, 2, 3, 4, 5] + >>> list(iterable) + [1, 2, 3, 4, 5] + + """ + it = iter(iterable) + head = take(n, it) + + return head.copy(), chain(head, it) + + +def interleave(*iterables): + """Return a new iterable yielding from each iterable in turn, + until the shortest is exhausted. + + >>> list(interleave([1, 2, 3], [4, 5], [6, 7, 8])) + [1, 4, 6, 2, 5, 7] + + For a version that doesn't terminate after the shortest iterable is + exhausted, see :func:`interleave_longest`. + + """ + return chain.from_iterable(zip(*iterables)) + + +def interleave_longest(*iterables): + """Return a new iterable yielding from each iterable in turn, + skipping any that are exhausted. + + >>> list(interleave_longest([1, 2, 3], [4, 5], [6, 7, 8])) + [1, 4, 6, 2, 5, 7, 3, 8] + + This function produces the same output as :func:`roundrobin`, but may + perform better for some inputs (in particular when the number of iterables + is large). + + """ + i = chain.from_iterable(zip_longest(*iterables, fillvalue=_marker)) + return (x for x in i if x is not _marker) + + +def collapse(iterable, base_type=None, levels=None): + """Flatten an iterable with multiple levels of nesting (e.g., a list of + lists of tuples) into non-iterable types. + + >>> iterable = [(1, 2), ([3, 4], [[5], [6]])] + >>> list(collapse(iterable)) + [1, 2, 3, 4, 5, 6] + + Binary and text strings are not considered iterable and + will not be collapsed. + + To avoid collapsing other types, specify *base_type*: + + >>> iterable = ['ab', ('cd', 'ef'), ['gh', 'ij']] + >>> list(collapse(iterable, base_type=tuple)) + ['ab', ('cd', 'ef'), 'gh', 'ij'] + + Specify *levels* to stop flattening after a certain level: + + >>> iterable = [('a', ['b']), ('c', ['d'])] + >>> list(collapse(iterable)) # Fully flattened + ['a', 'b', 'c', 'd'] + >>> list(collapse(iterable, levels=1)) # Only one level flattened + ['a', ['b'], 'c', ['d']] + + """ + + def walk(node, level): + if ( + ((levels is not None) and (level > levels)) + or isinstance(node, (str, bytes)) + or ((base_type is not None) and isinstance(node, base_type)) + ): + yield node + return + + try: + tree = iter(node) + except TypeError: + yield node + return + else: + for child in tree: + yield from walk(child, level + 1) + + yield from walk(iterable, 0) + + +def side_effect(func, iterable, chunk_size=None, before=None, after=None): + """Invoke *func* on each item in *iterable* (or on each *chunk_size* group + of items) before yielding the item. + + `func` must be a function that takes a single argument. Its return value + will be discarded. + + *before* and *after* are optional functions that take no arguments. They + will be executed before iteration starts and after it ends, respectively. + + `side_effect` can be used for logging, updating progress bars, or anything + that is not functionally "pure." + + Emitting a status message: + + >>> from more_itertools import consume + >>> func = lambda item: print('Received {}'.format(item)) + >>> consume(side_effect(func, range(2))) + Received 0 + Received 1 + + Operating on chunks of items: + + >>> pair_sums = [] + >>> func = lambda chunk: pair_sums.append(sum(chunk)) + >>> list(side_effect(func, [0, 1, 2, 3, 4, 5], 2)) + [0, 1, 2, 3, 4, 5] + >>> list(pair_sums) + [1, 5, 9] + + Writing to a file-like object: + + >>> from io import StringIO + >>> from more_itertools import consume + >>> f = StringIO() + >>> func = lambda x: print(x, file=f) + >>> before = lambda: print(u'HEADER', file=f) + >>> after = f.close + >>> it = [u'a', u'b', u'c'] + >>> consume(side_effect(func, it, before=before, after=after)) + >>> f.closed + True + + """ + try: + if before is not None: + before() + + if chunk_size is None: + for item in iterable: + func(item) + yield item + else: + for chunk in chunked(iterable, chunk_size): + func(chunk) + yield from chunk + finally: + if after is not None: + after() + + +def sliced(seq, n, strict=False): + """Yield slices of length *n* from the sequence *seq*. + + >>> list(sliced((1, 2, 3, 4, 5, 6), 3)) + [(1, 2, 3), (4, 5, 6)] + + By the default, the last yielded slice will have fewer than *n* elements + if the length of *seq* is not divisible by *n*: + + >>> list(sliced((1, 2, 3, 4, 5, 6, 7, 8), 3)) + [(1, 2, 3), (4, 5, 6), (7, 8)] + + If the length of *seq* is not divisible by *n* and *strict* is + ``True``, then ``ValueError`` will be raised before the last + slice is yielded. + + This function will only work for iterables that support slicing. + For non-sliceable iterables, see :func:`chunked`. + + """ + iterator = takewhile(len, (seq[i : i + n] for i in count(0, n))) + if strict: + + def ret(): + for _slice in iterator: + if len(_slice) != n: + raise ValueError("seq is not divisible by n.") + yield _slice + + return iter(ret()) + else: + return iterator + + +def split_at(iterable, pred, maxsplit=-1, keep_separator=False): + """Yield lists of items from *iterable*, where each list is delimited by + an item where callable *pred* returns ``True``. + + >>> list(split_at('abcdcba', lambda x: x == 'b')) + [['a'], ['c', 'd', 'c'], ['a']] + + >>> list(split_at(range(10), lambda n: n % 2 == 1)) + [[0], [2], [4], [6], [8], []] + + At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, + then there is no limit on the number of splits: + + >>> list(split_at(range(10), lambda n: n % 2 == 1, maxsplit=2)) + [[0], [2], [4, 5, 6, 7, 8, 9]] + + By default, the delimiting items are not included in the output. + The include them, set *keep_separator* to ``True``. + + >>> list(split_at('abcdcba', lambda x: x == 'b', keep_separator=True)) + [['a'], ['b'], ['c', 'd', 'c'], ['b'], ['a']] + + """ + if maxsplit == 0: + yield list(iterable) + return + + buf = [] + it = iter(iterable) + for item in it: + if pred(item): + yield buf + if keep_separator: + yield [item] + if maxsplit == 1: + yield list(it) + return + buf = [] + maxsplit -= 1 + else: + buf.append(item) + yield buf + + +def split_before(iterable, pred, maxsplit=-1): + """Yield lists of items from *iterable*, where each list ends just before + an item for which callable *pred* returns ``True``: + + >>> list(split_before('OneTwo', lambda s: s.isupper())) + [['O', 'n', 'e'], ['T', 'w', 'o']] + + >>> list(split_before(range(10), lambda n: n % 3 == 0)) + [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] + + At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, + then there is no limit on the number of splits: + + >>> list(split_before(range(10), lambda n: n % 3 == 0, maxsplit=2)) + [[0, 1, 2], [3, 4, 5], [6, 7, 8, 9]] + """ + if maxsplit == 0: + yield list(iterable) + return + + buf = [] + it = iter(iterable) + for item in it: + if pred(item) and buf: + yield buf + if maxsplit == 1: + yield [item] + list(it) + return + buf = [] + maxsplit -= 1 + buf.append(item) + if buf: + yield buf + + +def split_after(iterable, pred, maxsplit=-1): + """Yield lists of items from *iterable*, where each list ends with an + item where callable *pred* returns ``True``: + + >>> list(split_after('one1two2', lambda s: s.isdigit())) + [['o', 'n', 'e', '1'], ['t', 'w', 'o', '2']] + + >>> list(split_after(range(10), lambda n: n % 3 == 0)) + [[0], [1, 2, 3], [4, 5, 6], [7, 8, 9]] + + At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, + then there is no limit on the number of splits: + + >>> list(split_after(range(10), lambda n: n % 3 == 0, maxsplit=2)) + [[0], [1, 2, 3], [4, 5, 6, 7, 8, 9]] + + """ + if maxsplit == 0: + yield list(iterable) + return + + buf = [] + it = iter(iterable) + for item in it: + buf.append(item) + if pred(item) and buf: + yield buf + if maxsplit == 1: + yield list(it) + return + buf = [] + maxsplit -= 1 + if buf: + yield buf + + +def split_when(iterable, pred, maxsplit=-1): + """Split *iterable* into pieces based on the output of *pred*. + *pred* should be a function that takes successive pairs of items and + returns ``True`` if the iterable should be split in between them. + + For example, to find runs of increasing numbers, split the iterable when + element ``i`` is larger than element ``i + 1``: + + >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], lambda x, y: x > y)) + [[1, 2, 3, 3], [2, 5], [2, 4], [2]] + + At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, + then there is no limit on the number of splits: + + >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], + ... lambda x, y: x > y, maxsplit=2)) + [[1, 2, 3, 3], [2, 5], [2, 4, 2]] + + """ + if maxsplit == 0: + yield list(iterable) + return + + it = iter(iterable) + try: + cur_item = next(it) + except StopIteration: + return + + buf = [cur_item] + for next_item in it: + if pred(cur_item, next_item): + yield buf + if maxsplit == 1: + yield [next_item] + list(it) + return + buf = [] + maxsplit -= 1 + + buf.append(next_item) + cur_item = next_item + + yield buf + + +def split_into(iterable, sizes): + """Yield a list of sequential items from *iterable* of length 'n' for each + integer 'n' in *sizes*. + + >>> list(split_into([1,2,3,4,5,6], [1,2,3])) + [[1], [2, 3], [4, 5, 6]] + + If the sum of *sizes* is smaller than the length of *iterable*, then the + remaining items of *iterable* will not be returned. + + >>> list(split_into([1,2,3,4,5,6], [2,3])) + [[1, 2], [3, 4, 5]] + + If the sum of *sizes* is larger than the length of *iterable*, fewer items + will be returned in the iteration that overruns *iterable* and further + lists will be empty: + + >>> list(split_into([1,2,3,4], [1,2,3,4])) + [[1], [2, 3], [4], []] + + When a ``None`` object is encountered in *sizes*, the returned list will + contain items up to the end of *iterable* the same way that itertools.slice + does: + + >>> list(split_into([1,2,3,4,5,6,7,8,9,0], [2,3,None])) + [[1, 2], [3, 4, 5], [6, 7, 8, 9, 0]] + + :func:`split_into` can be useful for grouping a series of items where the + sizes of the groups are not uniform. An example would be where in a row + from a table, multiple columns represent elements of the same feature + (e.g. a point represented by x,y,z) but, the format is not the same for + all columns. + """ + # convert the iterable argument into an iterator so its contents can + # be consumed by islice in case it is a generator + it = iter(iterable) + + for size in sizes: + if size is None: + yield list(it) + return + else: + yield list(islice(it, size)) + + +def padded(iterable, fillvalue=None, n=None, next_multiple=False): + """Yield the elements from *iterable*, followed by *fillvalue*, such that + at least *n* items are emitted. + + >>> list(padded([1, 2, 3], '?', 5)) + [1, 2, 3, '?', '?'] + + If *next_multiple* is ``True``, *fillvalue* will be emitted until the + number of items emitted is a multiple of *n*:: + + >>> list(padded([1, 2, 3, 4], n=3, next_multiple=True)) + [1, 2, 3, 4, None, None] + + If *n* is ``None``, *fillvalue* will be emitted indefinitely. + + """ + it = iter(iterable) + if n is None: + yield from chain(it, repeat(fillvalue)) + elif n < 1: + raise ValueError('n must be at least 1') + else: + item_count = 0 + for item in it: + yield item + item_count += 1 + + remaining = (n - item_count) % n if next_multiple else n - item_count + for _ in range(remaining): + yield fillvalue + + +def repeat_last(iterable, default=None): + """After the *iterable* is exhausted, keep yielding its last element. + + >>> list(islice(repeat_last(range(3)), 5)) + [0, 1, 2, 2, 2] + + If the iterable is empty, yield *default* forever:: + + >>> list(islice(repeat_last(range(0), 42), 5)) + [42, 42, 42, 42, 42] + + """ + item = _marker + for item in iterable: + yield item + final = default if item is _marker else item + yield from repeat(final) + + +def distribute(n, iterable): + """Distribute the items from *iterable* among *n* smaller iterables. + + >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6]) + >>> list(group_1) + [1, 3, 5] + >>> list(group_2) + [2, 4, 6] + + If the length of *iterable* is not evenly divisible by *n*, then the + length of the returned iterables will not be identical: + + >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7]) + >>> [list(c) for c in children] + [[1, 4, 7], [2, 5], [3, 6]] + + If the length of *iterable* is smaller than *n*, then the last returned + iterables will be empty: + + >>> children = distribute(5, [1, 2, 3]) + >>> [list(c) for c in children] + [[1], [2], [3], [], []] + + This function uses :func:`itertools.tee` and may require significant + storage. If you need the order items in the smaller iterables to match the + original iterable, see :func:`divide`. + + """ + if n < 1: + raise ValueError('n must be at least 1') + + children = tee(iterable, n) + return [islice(it, index, None, n) for index, it in enumerate(children)] + + +def stagger(iterable, offsets=(-1, 0, 1), longest=False, fillvalue=None): + """Yield tuples whose elements are offset from *iterable*. + The amount by which the `i`-th item in each tuple is offset is given by + the `i`-th item in *offsets*. + + >>> list(stagger([0, 1, 2, 3])) + [(None, 0, 1), (0, 1, 2), (1, 2, 3)] + >>> list(stagger(range(8), offsets=(0, 2, 4))) + [(0, 2, 4), (1, 3, 5), (2, 4, 6), (3, 5, 7)] + + By default, the sequence will end when the final element of a tuple is the + last item in the iterable. To continue until the first element of a tuple + is the last item in the iterable, set *longest* to ``True``:: + + >>> list(stagger([0, 1, 2, 3], longest=True)) + [(None, 0, 1), (0, 1, 2), (1, 2, 3), (2, 3, None), (3, None, None)] + + By default, ``None`` will be used to replace offsets beyond the end of the + sequence. Specify *fillvalue* to use some other value. + + """ + children = tee(iterable, len(offsets)) + + return zip_offset( + *children, offsets=offsets, longest=longest, fillvalue=fillvalue + ) + + +class UnequalIterablesError(ValueError): + def __init__(self, details=None): + msg = 'Iterables have different lengths' + if details is not None: + msg += (': index 0 has length {}; index {} has length {}').format( + *details + ) + + super().__init__(msg) + + +def _zip_equal_generator(iterables): + for combo in zip_longest(*iterables, fillvalue=_marker): + for val in combo: + if val is _marker: + raise UnequalIterablesError() + yield combo + + +def zip_equal(*iterables): + """``zip`` the input *iterables* together, but raise + ``UnequalIterablesError`` if they aren't all the same length. + + >>> it_1 = range(3) + >>> it_2 = iter('abc') + >>> list(zip_equal(it_1, it_2)) + [(0, 'a'), (1, 'b'), (2, 'c')] + + >>> it_1 = range(3) + >>> it_2 = iter('abcd') + >>> list(zip_equal(it_1, it_2)) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + more_itertools.more.UnequalIterablesError: Iterables have different + lengths + + """ + if hexversion >= 0x30A00A6: + warnings.warn( + ( + 'zip_equal will be removed in a future version of ' + 'more-itertools. Use the builtin zip function with ' + 'strict=True instead.' + ), + DeprecationWarning, + ) + # Check whether the iterables are all the same size. + try: + first_size = len(iterables[0]) + for i, it in enumerate(iterables[1:], 1): + size = len(it) + if size != first_size: + break + else: + # If we didn't break out, we can use the built-in zip. + return zip(*iterables) + + # If we did break out, there was a mismatch. + raise UnequalIterablesError(details=(first_size, i, size)) + # If any one of the iterables didn't have a length, start reading + # them until one runs out. + except TypeError: + return _zip_equal_generator(iterables) + + +def zip_offset(*iterables, offsets, longest=False, fillvalue=None): + """``zip`` the input *iterables* together, but offset the `i`-th iterable + by the `i`-th item in *offsets*. + + >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1))) + [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e')] + + This can be used as a lightweight alternative to SciPy or pandas to analyze + data sets in which some series have a lead or lag relationship. + + By default, the sequence will end when the shortest iterable is exhausted. + To continue until the longest iterable is exhausted, set *longest* to + ``True``. + + >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1), longest=True)) + [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e'), (None, 'f')] + + By default, ``None`` will be used to replace offsets beyond the end of the + sequence. Specify *fillvalue* to use some other value. + + """ + if len(iterables) != len(offsets): + raise ValueError("Number of iterables and offsets didn't match") + + staggered = [] + for it, n in zip(iterables, offsets): + if n < 0: + staggered.append(chain(repeat(fillvalue, -n), it)) + elif n > 0: + staggered.append(islice(it, n, None)) + else: + staggered.append(it) + + if longest: + return zip_longest(*staggered, fillvalue=fillvalue) + + return zip(*staggered) + + +def sort_together(iterables, key_list=(0,), key=None, reverse=False): + """Return the input iterables sorted together, with *key_list* as the + priority for sorting. All iterables are trimmed to the length of the + shortest one. + + This can be used like the sorting function in a spreadsheet. If each + iterable represents a column of data, the key list determines which + columns are used for sorting. + + By default, all iterables are sorted using the ``0``-th iterable:: + + >>> iterables = [(4, 3, 2, 1), ('a', 'b', 'c', 'd')] + >>> sort_together(iterables) + [(1, 2, 3, 4), ('d', 'c', 'b', 'a')] + + Set a different key list to sort according to another iterable. + Specifying multiple keys dictates how ties are broken:: + + >>> iterables = [(3, 1, 2), (0, 1, 0), ('c', 'b', 'a')] + >>> sort_together(iterables, key_list=(1, 2)) + [(2, 3, 1), (0, 0, 1), ('a', 'c', 'b')] + + To sort by a function of the elements of the iterable, pass a *key* + function. Its arguments are the elements of the iterables corresponding to + the key list:: + + >>> names = ('a', 'b', 'c') + >>> lengths = (1, 2, 3) + >>> widths = (5, 2, 1) + >>> def area(length, width): + ... return length * width + >>> sort_together([names, lengths, widths], key_list=(1, 2), key=area) + [('c', 'b', 'a'), (3, 2, 1), (1, 2, 5)] + + Set *reverse* to ``True`` to sort in descending order. + + >>> sort_together([(1, 2, 3), ('c', 'b', 'a')], reverse=True) + [(3, 2, 1), ('a', 'b', 'c')] + + """ + if key is None: + # if there is no key function, the key argument to sorted is an + # itemgetter + key_argument = itemgetter(*key_list) + else: + # if there is a key function, call it with the items at the offsets + # specified by the key function as arguments + key_list = list(key_list) + if len(key_list) == 1: + # if key_list contains a single item, pass the item at that offset + # as the only argument to the key function + key_offset = key_list[0] + key_argument = lambda zipped_items: key(zipped_items[key_offset]) + else: + # if key_list contains multiple items, use itemgetter to return a + # tuple of items, which we pass as *args to the key function + get_key_items = itemgetter(*key_list) + key_argument = lambda zipped_items: key( + *get_key_items(zipped_items) + ) + + return list( + zip(*sorted(zip(*iterables), key=key_argument, reverse=reverse)) + ) + + +def unzip(iterable): + """The inverse of :func:`zip`, this function disaggregates the elements + of the zipped *iterable*. + + The ``i``-th iterable contains the ``i``-th element from each element + of the zipped iterable. The first element is used to to determine the + length of the remaining elements. + + >>> iterable = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] + >>> letters, numbers = unzip(iterable) + >>> list(letters) + ['a', 'b', 'c', 'd'] + >>> list(numbers) + [1, 2, 3, 4] + + This is similar to using ``zip(*iterable)``, but it avoids reading + *iterable* into memory. Note, however, that this function uses + :func:`itertools.tee` and thus may require significant storage. + + """ + head, iterable = spy(iter(iterable)) + if not head: + # empty iterable, e.g. zip([], [], []) + return () + # spy returns a one-length iterable as head + head = head[0] + iterables = tee(iterable, len(head)) + + def itemgetter(i): + def getter(obj): + try: + return obj[i] + except IndexError: + # basically if we have an iterable like + # iter([(1, 2, 3), (4, 5), (6,)]) + # the second unzipped iterable would fail at the third tuple + # since it would try to access tup[1] + # same with the third unzipped iterable and the second tuple + # to support these "improperly zipped" iterables, + # we create a custom itemgetter + # which just stops the unzipped iterables + # at first length mismatch + raise StopIteration + + return getter + + return tuple(map(itemgetter(i), it) for i, it in enumerate(iterables)) + + +def divide(n, iterable): + """Divide the elements from *iterable* into *n* parts, maintaining + order. + + >>> group_1, group_2 = divide(2, [1, 2, 3, 4, 5, 6]) + >>> list(group_1) + [1, 2, 3] + >>> list(group_2) + [4, 5, 6] + + If the length of *iterable* is not evenly divisible by *n*, then the + length of the returned iterables will not be identical: + + >>> children = divide(3, [1, 2, 3, 4, 5, 6, 7]) + >>> [list(c) for c in children] + [[1, 2, 3], [4, 5], [6, 7]] + + If the length of the iterable is smaller than n, then the last returned + iterables will be empty: + + >>> children = divide(5, [1, 2, 3]) + >>> [list(c) for c in children] + [[1], [2], [3], [], []] + + This function will exhaust the iterable before returning and may require + significant storage. If order is not important, see :func:`distribute`, + which does not first pull the iterable into memory. + + """ + if n < 1: + raise ValueError('n must be at least 1') + + try: + iterable[:0] + except TypeError: + seq = tuple(iterable) + else: + seq = iterable + + q, r = divmod(len(seq), n) + + ret = [] + stop = 0 + for i in range(1, n + 1): + start = stop + stop += q + 1 if i <= r else q + ret.append(iter(seq[start:stop])) + + return ret + + +def always_iterable(obj, base_type=(str, bytes)): + """If *obj* is iterable, return an iterator over its items:: + + >>> obj = (1, 2, 3) + >>> list(always_iterable(obj)) + [1, 2, 3] + + If *obj* is not iterable, return a one-item iterable containing *obj*:: + + >>> obj = 1 + >>> list(always_iterable(obj)) + [1] + + If *obj* is ``None``, return an empty iterable: + + >>> obj = None + >>> list(always_iterable(None)) + [] + + By default, binary and text strings are not considered iterable:: + + >>> obj = 'foo' + >>> list(always_iterable(obj)) + ['foo'] + + If *base_type* is set, objects for which ``isinstance(obj, base_type)`` + returns ``True`` won't be considered iterable. + + >>> obj = {'a': 1} + >>> list(always_iterable(obj)) # Iterate over the dict's keys + ['a'] + >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit + [{'a': 1}] + + Set *base_type* to ``None`` to avoid any special handling and treat objects + Python considers iterable as iterable: + + >>> obj = 'foo' + >>> list(always_iterable(obj, base_type=None)) + ['f', 'o', 'o'] + """ + if obj is None: + return iter(()) + + if (base_type is not None) and isinstance(obj, base_type): + return iter((obj,)) + + try: + return iter(obj) + except TypeError: + return iter((obj,)) + + +def adjacent(predicate, iterable, distance=1): + """Return an iterable over `(bool, item)` tuples where the `item` is + drawn from *iterable* and the `bool` indicates whether + that item satisfies the *predicate* or is adjacent to an item that does. + + For example, to find whether items are adjacent to a ``3``:: + + >>> list(adjacent(lambda x: x == 3, range(6))) + [(False, 0), (False, 1), (True, 2), (True, 3), (True, 4), (False, 5)] + + Set *distance* to change what counts as adjacent. For example, to find + whether items are two places away from a ``3``: + + >>> list(adjacent(lambda x: x == 3, range(6), distance=2)) + [(False, 0), (True, 1), (True, 2), (True, 3), (True, 4), (True, 5)] + + This is useful for contextualizing the results of a search function. + For example, a code comparison tool might want to identify lines that + have changed, but also surrounding lines to give the viewer of the diff + context. + + The predicate function will only be called once for each item in the + iterable. + + See also :func:`groupby_transform`, which can be used with this function + to group ranges of items with the same `bool` value. + + """ + # Allow distance=0 mainly for testing that it reproduces results with map() + if distance < 0: + raise ValueError('distance must be at least 0') + + i1, i2 = tee(iterable) + padding = [False] * distance + selected = chain(padding, map(predicate, i1), padding) + adjacent_to_selected = map(any, windowed(selected, 2 * distance + 1)) + return zip(adjacent_to_selected, i2) + + +def groupby_transform(iterable, keyfunc=None, valuefunc=None, reducefunc=None): + """An extension of :func:`itertools.groupby` that can apply transformations + to the grouped data. + + * *keyfunc* is a function computing a key value for each item in *iterable* + * *valuefunc* is a function that transforms the individual items from + *iterable* after grouping + * *reducefunc* is a function that transforms each group of items + + >>> iterable = 'aAAbBBcCC' + >>> keyfunc = lambda k: k.upper() + >>> valuefunc = lambda v: v.lower() + >>> reducefunc = lambda g: ''.join(g) + >>> list(groupby_transform(iterable, keyfunc, valuefunc, reducefunc)) + [('A', 'aaa'), ('B', 'bbb'), ('C', 'ccc')] + + Each optional argument defaults to an identity function if not specified. + + :func:`groupby_transform` is useful when grouping elements of an iterable + using a separate iterable as the key. To do this, :func:`zip` the iterables + and pass a *keyfunc* that extracts the first element and a *valuefunc* + that extracts the second element:: + + >>> from operator import itemgetter + >>> keys = [0, 0, 1, 1, 1, 2, 2, 2, 3] + >>> values = 'abcdefghi' + >>> iterable = zip(keys, values) + >>> grouper = groupby_transform(iterable, itemgetter(0), itemgetter(1)) + >>> [(k, ''.join(g)) for k, g in grouper] + [(0, 'ab'), (1, 'cde'), (2, 'fgh'), (3, 'i')] + + Note that the order of items in the iterable is significant. + Only adjacent items are grouped together, so if you don't want any + duplicate groups, you should sort the iterable by the key function. + + """ + ret = groupby(iterable, keyfunc) + if valuefunc: + ret = ((k, map(valuefunc, g)) for k, g in ret) + if reducefunc: + ret = ((k, reducefunc(g)) for k, g in ret) + + return ret + + +class numeric_range(abc.Sequence, abc.Hashable): + """An extension of the built-in ``range()`` function whose arguments can + be any orderable numeric type. + + With only *stop* specified, *start* defaults to ``0`` and *step* + defaults to ``1``. The output items will match the type of *stop*: + + >>> list(numeric_range(3.5)) + [0.0, 1.0, 2.0, 3.0] + + With only *start* and *stop* specified, *step* defaults to ``1``. The + output items will match the type of *start*: + + >>> from decimal import Decimal + >>> start = Decimal('2.1') + >>> stop = Decimal('5.1') + >>> list(numeric_range(start, stop)) + [Decimal('2.1'), Decimal('3.1'), Decimal('4.1')] + + With *start*, *stop*, and *step* specified the output items will match + the type of ``start + step``: + + >>> from fractions import Fraction + >>> start = Fraction(1, 2) # Start at 1/2 + >>> stop = Fraction(5, 2) # End at 5/2 + >>> step = Fraction(1, 2) # Count by 1/2 + >>> list(numeric_range(start, stop, step)) + [Fraction(1, 2), Fraction(1, 1), Fraction(3, 2), Fraction(2, 1)] + + If *step* is zero, ``ValueError`` is raised. Negative steps are supported: + + >>> list(numeric_range(3, -1, -1.0)) + [3.0, 2.0, 1.0, 0.0] + + Be aware of the limitations of floating point numbers; the representation + of the yielded numbers may be surprising. + + ``datetime.datetime`` objects can be used for *start* and *stop*, if *step* + is a ``datetime.timedelta`` object: + + >>> import datetime + >>> start = datetime.datetime(2019, 1, 1) + >>> stop = datetime.datetime(2019, 1, 3) + >>> step = datetime.timedelta(days=1) + >>> items = iter(numeric_range(start, stop, step)) + >>> next(items) + datetime.datetime(2019, 1, 1, 0, 0) + >>> next(items) + datetime.datetime(2019, 1, 2, 0, 0) + + """ + + _EMPTY_HASH = hash(range(0, 0)) + + def __init__(self, *args): + argc = len(args) + if argc == 1: + (self._stop,) = args + self._start = type(self._stop)(0) + self._step = type(self._stop - self._start)(1) + elif argc == 2: + self._start, self._stop = args + self._step = type(self._stop - self._start)(1) + elif argc == 3: + self._start, self._stop, self._step = args + elif argc == 0: + raise TypeError( + 'numeric_range expected at least ' + '1 argument, got {}'.format(argc) + ) + else: + raise TypeError( + 'numeric_range expected at most ' + '3 arguments, got {}'.format(argc) + ) + + self._zero = type(self._step)(0) + if self._step == self._zero: + raise ValueError('numeric_range() arg 3 must not be zero') + self._growing = self._step > self._zero + self._init_len() + + def __bool__(self): + if self._growing: + return self._start < self._stop + else: + return self._start > self._stop + + def __contains__(self, elem): + if self._growing: + if self._start <= elem < self._stop: + return (elem - self._start) % self._step == self._zero + else: + if self._start >= elem > self._stop: + return (self._start - elem) % (-self._step) == self._zero + + return False + + def __eq__(self, other): + if isinstance(other, numeric_range): + empty_self = not bool(self) + empty_other = not bool(other) + if empty_self or empty_other: + return empty_self and empty_other # True if both empty + else: + return ( + self._start == other._start + and self._step == other._step + and self._get_by_index(-1) == other._get_by_index(-1) + ) + else: + return False + + def __getitem__(self, key): + if isinstance(key, int): + return self._get_by_index(key) + elif isinstance(key, slice): + step = self._step if key.step is None else key.step * self._step + + if key.start is None or key.start <= -self._len: + start = self._start + elif key.start >= self._len: + start = self._stop + else: # -self._len < key.start < self._len + start = self._get_by_index(key.start) + + if key.stop is None or key.stop >= self._len: + stop = self._stop + elif key.stop <= -self._len: + stop = self._start + else: # -self._len < key.stop < self._len + stop = self._get_by_index(key.stop) + + return numeric_range(start, stop, step) + else: + raise TypeError( + 'numeric range indices must be ' + 'integers or slices, not {}'.format(type(key).__name__) + ) + + def __hash__(self): + if self: + return hash((self._start, self._get_by_index(-1), self._step)) + else: + return self._EMPTY_HASH + + def __iter__(self): + values = (self._start + (n * self._step) for n in count()) + if self._growing: + return takewhile(partial(gt, self._stop), values) + else: + return takewhile(partial(lt, self._stop), values) + + def __len__(self): + return self._len + + def _init_len(self): + if self._growing: + start = self._start + stop = self._stop + step = self._step + else: + start = self._stop + stop = self._start + step = -self._step + distance = stop - start + if distance <= self._zero: + self._len = 0 + else: # distance > 0 and step > 0: regular euclidean division + q, r = divmod(distance, step) + self._len = int(q) + int(r != self._zero) + + def __reduce__(self): + return numeric_range, (self._start, self._stop, self._step) + + def __repr__(self): + if self._step == 1: + return "numeric_range({}, {})".format( + repr(self._start), repr(self._stop) + ) + else: + return "numeric_range({}, {}, {})".format( + repr(self._start), repr(self._stop), repr(self._step) + ) + + def __reversed__(self): + return iter( + numeric_range( + self._get_by_index(-1), self._start - self._step, -self._step + ) + ) + + def count(self, value): + return int(value in self) + + def index(self, value): + if self._growing: + if self._start <= value < self._stop: + q, r = divmod(value - self._start, self._step) + if r == self._zero: + return int(q) + else: + if self._start >= value > self._stop: + q, r = divmod(self._start - value, -self._step) + if r == self._zero: + return int(q) + + raise ValueError("{} is not in numeric range".format(value)) + + def _get_by_index(self, i): + if i < 0: + i += self._len + if i < 0 or i >= self._len: + raise IndexError("numeric range object index out of range") + return self._start + i * self._step + + +def count_cycle(iterable, n=None): + """Cycle through the items from *iterable* up to *n* times, yielding + the number of completed cycles along with each item. If *n* is omitted the + process repeats indefinitely. + + >>> list(count_cycle('AB', 3)) + [(0, 'A'), (0, 'B'), (1, 'A'), (1, 'B'), (2, 'A'), (2, 'B')] + + """ + iterable = tuple(iterable) + if not iterable: + return iter(()) + counter = count() if n is None else range(n) + return ((i, item) for i in counter for item in iterable) + + +def mark_ends(iterable): + """Yield 3-tuples of the form ``(is_first, is_last, item)``. + + >>> list(mark_ends('ABC')) + [(True, False, 'A'), (False, False, 'B'), (False, True, 'C')] + + Use this when looping over an iterable to take special action on its first + and/or last items: + + >>> iterable = ['Header', 100, 200, 'Footer'] + >>> total = 0 + >>> for is_first, is_last, item in mark_ends(iterable): + ... if is_first: + ... continue # Skip the header + ... if is_last: + ... continue # Skip the footer + ... total += item + >>> print(total) + 300 + """ + it = iter(iterable) + + try: + b = next(it) + except StopIteration: + return + + try: + for i in count(): + a = b + b = next(it) + yield i == 0, False, a + + except StopIteration: + yield i == 0, True, a + + +def locate(iterable, pred=bool, window_size=None): + """Yield the index of each item in *iterable* for which *pred* returns + ``True``. + + *pred* defaults to :func:`bool`, which will select truthy items: + + >>> list(locate([0, 1, 1, 0, 1, 0, 0])) + [1, 2, 4] + + Set *pred* to a custom function to, e.g., find the indexes for a particular + item. + + >>> list(locate(['a', 'b', 'c', 'b'], lambda x: x == 'b')) + [1, 3] + + If *window_size* is given, then the *pred* function will be called with + that many items. This enables searching for sub-sequences: + + >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] + >>> pred = lambda *args: args == (1, 2, 3) + >>> list(locate(iterable, pred=pred, window_size=3)) + [1, 5, 9] + + Use with :func:`seekable` to find indexes and then retrieve the associated + items: + + >>> from itertools import count + >>> from more_itertools import seekable + >>> source = (3 * n + 1 if (n % 2) else n // 2 for n in count()) + >>> it = seekable(source) + >>> pred = lambda x: x > 100 + >>> indexes = locate(it, pred=pred) + >>> i = next(indexes) + >>> it.seek(i) + >>> next(it) + 106 + + """ + if window_size is None: + return compress(count(), map(pred, iterable)) + + if window_size < 1: + raise ValueError('window size must be at least 1') + + it = windowed(iterable, window_size, fillvalue=_marker) + return compress(count(), starmap(pred, it)) + + +def lstrip(iterable, pred): + """Yield the items from *iterable*, but strip any from the beginning + for which *pred* returns ``True``. + + For example, to remove a set of items from the start of an iterable: + + >>> iterable = (None, False, None, 1, 2, None, 3, False, None) + >>> pred = lambda x: x in {None, False, ''} + >>> list(lstrip(iterable, pred)) + [1, 2, None, 3, False, None] + + This function is analogous to to :func:`str.lstrip`, and is essentially + an wrapper for :func:`itertools.dropwhile`. + + """ + return dropwhile(pred, iterable) + + +def rstrip(iterable, pred): + """Yield the items from *iterable*, but strip any from the end + for which *pred* returns ``True``. + + For example, to remove a set of items from the end of an iterable: + + >>> iterable = (None, False, None, 1, 2, None, 3, False, None) + >>> pred = lambda x: x in {None, False, ''} + >>> list(rstrip(iterable, pred)) + [None, False, None, 1, 2, None, 3] + + This function is analogous to :func:`str.rstrip`. + + """ + cache = [] + cache_append = cache.append + cache_clear = cache.clear + for x in iterable: + if pred(x): + cache_append(x) + else: + yield from cache + cache_clear() + yield x + + +def strip(iterable, pred): + """Yield the items from *iterable*, but strip any from the + beginning and end for which *pred* returns ``True``. + + For example, to remove a set of items from both ends of an iterable: + + >>> iterable = (None, False, None, 1, 2, None, 3, False, None) + >>> pred = lambda x: x in {None, False, ''} + >>> list(strip(iterable, pred)) + [1, 2, None, 3] + + This function is analogous to :func:`str.strip`. + + """ + return rstrip(lstrip(iterable, pred), pred) + + +class islice_extended: + """An extension of :func:`itertools.islice` that supports negative values + for *stop*, *start*, and *step*. + + >>> iterable = iter('abcdefgh') + >>> list(islice_extended(iterable, -4, -1)) + ['e', 'f', 'g'] + + Slices with negative values require some caching of *iterable*, but this + function takes care to minimize the amount of memory required. + + For example, you can use a negative step with an infinite iterator: + + >>> from itertools import count + >>> list(islice_extended(count(), 110, 99, -2)) + [110, 108, 106, 104, 102, 100] + + You can also use slice notation directly: + + >>> iterable = map(str, count()) + >>> it = islice_extended(iterable)[10:20:2] + >>> list(it) + ['10', '12', '14', '16', '18'] + + """ + + def __init__(self, iterable, *args): + it = iter(iterable) + if args: + self._iterable = _islice_helper(it, slice(*args)) + else: + self._iterable = it + + def __iter__(self): + return self + + def __next__(self): + return next(self._iterable) + + def __getitem__(self, key): + if isinstance(key, slice): + return islice_extended(_islice_helper(self._iterable, key)) + + raise TypeError('islice_extended.__getitem__ argument must be a slice') + + +def _islice_helper(it, s): + start = s.start + stop = s.stop + if s.step == 0: + raise ValueError('step argument must be a non-zero integer or None.') + step = s.step or 1 + + if step > 0: + start = 0 if (start is None) else start + + if start < 0: + # Consume all but the last -start items + cache = deque(enumerate(it, 1), maxlen=-start) + len_iter = cache[-1][0] if cache else 0 + + # Adjust start to be positive + i = max(len_iter + start, 0) + + # Adjust stop to be positive + if stop is None: + j = len_iter + elif stop >= 0: + j = min(stop, len_iter) + else: + j = max(len_iter + stop, 0) + + # Slice the cache + n = j - i + if n <= 0: + return + + for index, item in islice(cache, 0, n, step): + yield item + elif (stop is not None) and (stop < 0): + # Advance to the start position + next(islice(it, start, start), None) + + # When stop is negative, we have to carry -stop items while + # iterating + cache = deque(islice(it, -stop), maxlen=-stop) + + for index, item in enumerate(it): + cached_item = cache.popleft() + if index % step == 0: + yield cached_item + cache.append(item) + else: + # When both start and stop are positive we have the normal case + yield from islice(it, start, stop, step) + else: + start = -1 if (start is None) else start + + if (stop is not None) and (stop < 0): + # Consume all but the last items + n = -stop - 1 + cache = deque(enumerate(it, 1), maxlen=n) + len_iter = cache[-1][0] if cache else 0 + + # If start and stop are both negative they are comparable and + # we can just slice. Otherwise we can adjust start to be negative + # and then slice. + if start < 0: + i, j = start, stop + else: + i, j = min(start - len_iter, -1), None + + for index, item in list(cache)[i:j:step]: + yield item + else: + # Advance to the stop position + if stop is not None: + m = stop + 1 + next(islice(it, m, m), None) + + # stop is positive, so if start is negative they are not comparable + # and we need the rest of the items. + if start < 0: + i = start + n = None + # stop is None and start is positive, so we just need items up to + # the start index. + elif stop is None: + i = None + n = start + 1 + # Both stop and start are positive, so they are comparable. + else: + i = None + n = start - stop + if n <= 0: + return + + cache = list(islice(it, n)) + + yield from cache[i::step] + + +def always_reversible(iterable): + """An extension of :func:`reversed` that supports all iterables, not + just those which implement the ``Reversible`` or ``Sequence`` protocols. + + >>> print(*always_reversible(x for x in range(3))) + 2 1 0 + + If the iterable is already reversible, this function returns the + result of :func:`reversed()`. If the iterable is not reversible, + this function will cache the remaining items in the iterable and + yield them in reverse order, which may require significant storage. + """ + try: + return reversed(iterable) + except TypeError: + return reversed(list(iterable)) + + +def consecutive_groups(iterable, ordering=lambda x: x): + """Yield groups of consecutive items using :func:`itertools.groupby`. + The *ordering* function determines whether two items are adjacent by + returning their position. + + By default, the ordering function is the identity function. This is + suitable for finding runs of numbers: + + >>> iterable = [1, 10, 11, 12, 20, 30, 31, 32, 33, 40] + >>> for group in consecutive_groups(iterable): + ... print(list(group)) + [1] + [10, 11, 12] + [20] + [30, 31, 32, 33] + [40] + + For finding runs of adjacent letters, try using the :meth:`index` method + of a string of letters: + + >>> from string import ascii_lowercase + >>> iterable = 'abcdfgilmnop' + >>> ordering = ascii_lowercase.index + >>> for group in consecutive_groups(iterable, ordering): + ... print(list(group)) + ['a', 'b', 'c', 'd'] + ['f', 'g'] + ['i'] + ['l', 'm', 'n', 'o', 'p'] + + Each group of consecutive items is an iterator that shares it source with + *iterable*. When an an output group is advanced, the previous group is + no longer available unless its elements are copied (e.g., into a ``list``). + + >>> iterable = [1, 2, 11, 12, 21, 22] + >>> saved_groups = [] + >>> for group in consecutive_groups(iterable): + ... saved_groups.append(list(group)) # Copy group elements + >>> saved_groups + [[1, 2], [11, 12], [21, 22]] + + """ + for k, g in groupby( + enumerate(iterable), key=lambda x: x[0] - ordering(x[1]) + ): + yield map(itemgetter(1), g) + + +def difference(iterable, func=sub, *, initial=None): + """This function is the inverse of :func:`itertools.accumulate`. By default + it will compute the first difference of *iterable* using + :func:`operator.sub`: + + >>> from itertools import accumulate + >>> iterable = accumulate([0, 1, 2, 3, 4]) # produces 0, 1, 3, 6, 10 + >>> list(difference(iterable)) + [0, 1, 2, 3, 4] + + *func* defaults to :func:`operator.sub`, but other functions can be + specified. They will be applied as follows:: + + A, B, C, D, ... --> A, func(B, A), func(C, B), func(D, C), ... + + For example, to do progressive division: + + >>> iterable = [1, 2, 6, 24, 120] + >>> func = lambda x, y: x // y + >>> list(difference(iterable, func)) + [1, 2, 3, 4, 5] + + If the *initial* keyword is set, the first element will be skipped when + computing successive differences. + + >>> it = [10, 11, 13, 16] # from accumulate([1, 2, 3], initial=10) + >>> list(difference(it, initial=10)) + [1, 2, 3] + + """ + a, b = tee(iterable) + try: + first = [next(b)] + except StopIteration: + return iter([]) + + if initial is not None: + first = [] + + return chain(first, starmap(func, zip(b, a))) + + +class SequenceView(Sequence): + """Return a read-only view of the sequence object *target*. + + :class:`SequenceView` objects are analogous to Python's built-in + "dictionary view" types. They provide a dynamic view of a sequence's items, + meaning that when the sequence updates, so does the view. + + >>> seq = ['0', '1', '2'] + >>> view = SequenceView(seq) + >>> view + SequenceView(['0', '1', '2']) + >>> seq.append('3') + >>> view + SequenceView(['0', '1', '2', '3']) + + Sequence views support indexing, slicing, and length queries. They act + like the underlying sequence, except they don't allow assignment: + + >>> view[1] + '1' + >>> view[1:-1] + ['1', '2'] + >>> len(view) + 4 + + Sequence views are useful as an alternative to copying, as they don't + require (much) extra storage. + + """ + + def __init__(self, target): + if not isinstance(target, Sequence): + raise TypeError + self._target = target + + def __getitem__(self, index): + return self._target[index] + + def __len__(self): + return len(self._target) + + def __repr__(self): + return '{}({})'.format(self.__class__.__name__, repr(self._target)) + + +class seekable: + """Wrap an iterator to allow for seeking backward and forward. This + progressively caches the items in the source iterable so they can be + re-visited. + + Call :meth:`seek` with an index to seek to that position in the source + iterable. + + To "reset" an iterator, seek to ``0``: + + >>> from itertools import count + >>> it = seekable((str(n) for n in count())) + >>> next(it), next(it), next(it) + ('0', '1', '2') + >>> it.seek(0) + >>> next(it), next(it), next(it) + ('0', '1', '2') + >>> next(it) + '3' + + You can also seek forward: + + >>> it = seekable((str(n) for n in range(20))) + >>> it.seek(10) + >>> next(it) + '10' + >>> it.seek(20) # Seeking past the end of the source isn't a problem + >>> list(it) + [] + >>> it.seek(0) # Resetting works even after hitting the end + >>> next(it), next(it), next(it) + ('0', '1', '2') + + Call :meth:`peek` to look ahead one item without advancing the iterator: + + >>> it = seekable('1234') + >>> it.peek() + '1' + >>> list(it) + ['1', '2', '3', '4'] + >>> it.peek(default='empty') + 'empty' + + Before the iterator is at its end, calling :func:`bool` on it will return + ``True``. After it will return ``False``: + + >>> it = seekable('5678') + >>> bool(it) + True + >>> list(it) + ['5', '6', '7', '8'] + >>> bool(it) + False + + You may view the contents of the cache with the :meth:`elements` method. + That returns a :class:`SequenceView`, a view that updates automatically: + + >>> it = seekable((str(n) for n in range(10))) + >>> next(it), next(it), next(it) + ('0', '1', '2') + >>> elements = it.elements() + >>> elements + SequenceView(['0', '1', '2']) + >>> next(it) + '3' + >>> elements + SequenceView(['0', '1', '2', '3']) + + By default, the cache grows as the source iterable progresses, so beware of + wrapping very large or infinite iterables. Supply *maxlen* to limit the + size of the cache (this of course limits how far back you can seek). + + >>> from itertools import count + >>> it = seekable((str(n) for n in count()), maxlen=2) + >>> next(it), next(it), next(it), next(it) + ('0', '1', '2', '3') + >>> list(it.elements()) + ['2', '3'] + >>> it.seek(0) + >>> next(it), next(it), next(it), next(it) + ('2', '3', '4', '5') + >>> next(it) + '6' + + """ + + def __init__(self, iterable, maxlen=None): + self._source = iter(iterable) + if maxlen is None: + self._cache = [] + else: + self._cache = deque([], maxlen) + self._index = None + + def __iter__(self): + return self + + def __next__(self): + if self._index is not None: + try: + item = self._cache[self._index] + except IndexError: + self._index = None + else: + self._index += 1 + return item + + item = next(self._source) + self._cache.append(item) + return item + + def __bool__(self): + try: + self.peek() + except StopIteration: + return False + return True + + def peek(self, default=_marker): + try: + peeked = next(self) + except StopIteration: + if default is _marker: + raise + return default + if self._index is None: + self._index = len(self._cache) + self._index -= 1 + return peeked + + def elements(self): + return SequenceView(self._cache) + + def seek(self, index): + self._index = index + remainder = index - len(self._cache) + if remainder > 0: + consume(self, remainder) + + +class run_length: + """ + :func:`run_length.encode` compresses an iterable with run-length encoding. + It yields groups of repeated items with the count of how many times they + were repeated: + + >>> uncompressed = 'abbcccdddd' + >>> list(run_length.encode(uncompressed)) + [('a', 1), ('b', 2), ('c', 3), ('d', 4)] + + :func:`run_length.decode` decompresses an iterable that was previously + compressed with run-length encoding. It yields the items of the + decompressed iterable: + + >>> compressed = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] + >>> list(run_length.decode(compressed)) + ['a', 'b', 'b', 'c', 'c', 'c', 'd', 'd', 'd', 'd'] + + """ + + @staticmethod + def encode(iterable): + return ((k, ilen(g)) for k, g in groupby(iterable)) + + @staticmethod + def decode(iterable): + return chain.from_iterable(repeat(k, n) for k, n in iterable) + + +def exactly_n(iterable, n, predicate=bool): + """Return ``True`` if exactly ``n`` items in the iterable are ``True`` + according to the *predicate* function. + + >>> exactly_n([True, True, False], 2) + True + >>> exactly_n([True, True, False], 1) + False + >>> exactly_n([0, 1, 2, 3, 4, 5], 3, lambda x: x < 3) + True + + The iterable will be advanced until ``n + 1`` truthy items are encountered, + so avoid calling it on infinite iterables. + + """ + return len(take(n + 1, filter(predicate, iterable))) == n + + +def circular_shifts(iterable): + """Return a list of circular shifts of *iterable*. + + >>> circular_shifts(range(4)) + [(0, 1, 2, 3), (1, 2, 3, 0), (2, 3, 0, 1), (3, 0, 1, 2)] + """ + lst = list(iterable) + return take(len(lst), windowed(cycle(lst), len(lst))) + + +def make_decorator(wrapping_func, result_index=0): + """Return a decorator version of *wrapping_func*, which is a function that + modifies an iterable. *result_index* is the position in that function's + signature where the iterable goes. + + This lets you use itertools on the "production end," i.e. at function + definition. This can augment what the function returns without changing the + function's code. + + For example, to produce a decorator version of :func:`chunked`: + + >>> from more_itertools import chunked + >>> chunker = make_decorator(chunked, result_index=0) + >>> @chunker(3) + ... def iter_range(n): + ... return iter(range(n)) + ... + >>> list(iter_range(9)) + [[0, 1, 2], [3, 4, 5], [6, 7, 8]] + + To only allow truthy items to be returned: + + >>> truth_serum = make_decorator(filter, result_index=1) + >>> @truth_serum(bool) + ... def boolean_test(): + ... return [0, 1, '', ' ', False, True] + ... + >>> list(boolean_test()) + [1, ' ', True] + + The :func:`peekable` and :func:`seekable` wrappers make for practical + decorators: + + >>> from more_itertools import peekable + >>> peekable_function = make_decorator(peekable) + >>> @peekable_function() + ... def str_range(*args): + ... return (str(x) for x in range(*args)) + ... + >>> it = str_range(1, 20, 2) + >>> next(it), next(it), next(it) + ('1', '3', '5') + >>> it.peek() + '7' + >>> next(it) + '7' + + """ + # See https://sites.google.com/site/bbayles/index/decorator_factory for + # notes on how this works. + def decorator(*wrapping_args, **wrapping_kwargs): + def outer_wrapper(f): + def inner_wrapper(*args, **kwargs): + result = f(*args, **kwargs) + wrapping_args_ = list(wrapping_args) + wrapping_args_.insert(result_index, result) + return wrapping_func(*wrapping_args_, **wrapping_kwargs) + + return inner_wrapper + + return outer_wrapper + + return decorator + + +def map_reduce(iterable, keyfunc, valuefunc=None, reducefunc=None): + """Return a dictionary that maps the items in *iterable* to categories + defined by *keyfunc*, transforms them with *valuefunc*, and + then summarizes them by category with *reducefunc*. + + *valuefunc* defaults to the identity function if it is unspecified. + If *reducefunc* is unspecified, no summarization takes place: + + >>> keyfunc = lambda x: x.upper() + >>> result = map_reduce('abbccc', keyfunc) + >>> sorted(result.items()) + [('A', ['a']), ('B', ['b', 'b']), ('C', ['c', 'c', 'c'])] + + Specifying *valuefunc* transforms the categorized items: + + >>> keyfunc = lambda x: x.upper() + >>> valuefunc = lambda x: 1 + >>> result = map_reduce('abbccc', keyfunc, valuefunc) + >>> sorted(result.items()) + [('A', [1]), ('B', [1, 1]), ('C', [1, 1, 1])] + + Specifying *reducefunc* summarizes the categorized items: + + >>> keyfunc = lambda x: x.upper() + >>> valuefunc = lambda x: 1 + >>> reducefunc = sum + >>> result = map_reduce('abbccc', keyfunc, valuefunc, reducefunc) + >>> sorted(result.items()) + [('A', 1), ('B', 2), ('C', 3)] + + You may want to filter the input iterable before applying the map/reduce + procedure: + + >>> all_items = range(30) + >>> items = [x for x in all_items if 10 <= x <= 20] # Filter + >>> keyfunc = lambda x: x % 2 # Evens map to 0; odds to 1 + >>> categories = map_reduce(items, keyfunc=keyfunc) + >>> sorted(categories.items()) + [(0, [10, 12, 14, 16, 18, 20]), (1, [11, 13, 15, 17, 19])] + >>> summaries = map_reduce(items, keyfunc=keyfunc, reducefunc=sum) + >>> sorted(summaries.items()) + [(0, 90), (1, 75)] + + Note that all items in the iterable are gathered into a list before the + summarization step, which may require significant storage. + + The returned object is a :obj:`collections.defaultdict` with the + ``default_factory`` set to ``None``, such that it behaves like a normal + dictionary. + + """ + valuefunc = (lambda x: x) if (valuefunc is None) else valuefunc + + ret = defaultdict(list) + for item in iterable: + key = keyfunc(item) + value = valuefunc(item) + ret[key].append(value) + + if reducefunc is not None: + for key, value_list in ret.items(): + ret[key] = reducefunc(value_list) + + ret.default_factory = None + return ret + + +def rlocate(iterable, pred=bool, window_size=None): + """Yield the index of each item in *iterable* for which *pred* returns + ``True``, starting from the right and moving left. + + *pred* defaults to :func:`bool`, which will select truthy items: + + >>> list(rlocate([0, 1, 1, 0, 1, 0, 0])) # Truthy at 1, 2, and 4 + [4, 2, 1] + + Set *pred* to a custom function to, e.g., find the indexes for a particular + item: + + >>> iterable = iter('abcb') + >>> pred = lambda x: x == 'b' + >>> list(rlocate(iterable, pred)) + [3, 1] + + If *window_size* is given, then the *pred* function will be called with + that many items. This enables searching for sub-sequences: + + >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] + >>> pred = lambda *args: args == (1, 2, 3) + >>> list(rlocate(iterable, pred=pred, window_size=3)) + [9, 5, 1] + + Beware, this function won't return anything for infinite iterables. + If *iterable* is reversible, ``rlocate`` will reverse it and search from + the right. Otherwise, it will search from the left and return the results + in reverse order. + + See :func:`locate` to for other example applications. + + """ + if window_size is None: + try: + len_iter = len(iterable) + return (len_iter - i - 1 for i in locate(reversed(iterable), pred)) + except TypeError: + pass + + return reversed(list(locate(iterable, pred, window_size))) + + +def replace(iterable, pred, substitutes, count=None, window_size=1): + """Yield the items from *iterable*, replacing the items for which *pred* + returns ``True`` with the items from the iterable *substitutes*. + + >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1] + >>> pred = lambda x: x == 0 + >>> substitutes = (2, 3) + >>> list(replace(iterable, pred, substitutes)) + [1, 1, 2, 3, 1, 1, 2, 3, 1, 1] + + If *count* is given, the number of replacements will be limited: + + >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1, 0] + >>> pred = lambda x: x == 0 + >>> substitutes = [None] + >>> list(replace(iterable, pred, substitutes, count=2)) + [1, 1, None, 1, 1, None, 1, 1, 0] + + Use *window_size* to control the number of items passed as arguments to + *pred*. This allows for locating and replacing subsequences. + + >>> iterable = [0, 1, 2, 5, 0, 1, 2, 5] + >>> window_size = 3 + >>> pred = lambda *args: args == (0, 1, 2) # 3 items passed to pred + >>> substitutes = [3, 4] # Splice in these items + >>> list(replace(iterable, pred, substitutes, window_size=window_size)) + [3, 4, 5, 3, 4, 5] + + """ + if window_size < 1: + raise ValueError('window_size must be at least 1') + + # Save the substitutes iterable, since it's used more than once + substitutes = tuple(substitutes) + + # Add padding such that the number of windows matches the length of the + # iterable + it = chain(iterable, [_marker] * (window_size - 1)) + windows = windowed(it, window_size) + + n = 0 + for w in windows: + # If the current window matches our predicate (and we haven't hit + # our maximum number of replacements), splice in the substitutes + # and then consume the following windows that overlap with this one. + # For example, if the iterable is (0, 1, 2, 3, 4...) + # and the window size is 2, we have (0, 1), (1, 2), (2, 3)... + # If the predicate matches on (0, 1), we need to zap (0, 1) and (1, 2) + if pred(*w): + if (count is None) or (n < count): + n += 1 + yield from substitutes + consume(windows, window_size - 1) + continue + + # If there was no match (or we've reached the replacement limit), + # yield the first item from the window. + if w and (w[0] is not _marker): + yield w[0] + + +def partitions(iterable): + """Yield all possible order-preserving partitions of *iterable*. + + >>> iterable = 'abc' + >>> for part in partitions(iterable): + ... print([''.join(p) for p in part]) + ['abc'] + ['a', 'bc'] + ['ab', 'c'] + ['a', 'b', 'c'] + + This is unrelated to :func:`partition`. + + """ + sequence = list(iterable) + n = len(sequence) + for i in powerset(range(1, n)): + yield [sequence[i:j] for i, j in zip((0,) + i, i + (n,))] + + +def set_partitions(iterable, k=None): + """ + Yield the set partitions of *iterable* into *k* parts. Set partitions are + not order-preserving. + + >>> iterable = 'abc' + >>> for part in set_partitions(iterable, 2): + ... print([''.join(p) for p in part]) + ['a', 'bc'] + ['ab', 'c'] + ['b', 'ac'] + + + If *k* is not given, every set partition is generated. + + >>> iterable = 'abc' + >>> for part in set_partitions(iterable): + ... print([''.join(p) for p in part]) + ['abc'] + ['a', 'bc'] + ['ab', 'c'] + ['b', 'ac'] + ['a', 'b', 'c'] + + """ + L = list(iterable) + n = len(L) + if k is not None: + if k < 1: + raise ValueError( + "Can't partition in a negative or zero number of groups" + ) + elif k > n: + return + + def set_partitions_helper(L, k): + n = len(L) + if k == 1: + yield [L] + elif n == k: + yield [[s] for s in L] + else: + e, *M = L + for p in set_partitions_helper(M, k - 1): + yield [[e], *p] + for p in set_partitions_helper(M, k): + for i in range(len(p)): + yield p[:i] + [[e] + p[i]] + p[i + 1 :] + + if k is None: + for k in range(1, n + 1): + yield from set_partitions_helper(L, k) + else: + yield from set_partitions_helper(L, k) + + +class time_limited: + """ + Yield items from *iterable* until *limit_seconds* have passed. + If the time limit expires before all items have been yielded, the + ``timed_out`` parameter will be set to ``True``. + + >>> from time import sleep + >>> def generator(): + ... yield 1 + ... yield 2 + ... sleep(0.2) + ... yield 3 + >>> iterable = time_limited(0.1, generator()) + >>> list(iterable) + [1, 2] + >>> iterable.timed_out + True + + Note that the time is checked before each item is yielded, and iteration + stops if the time elapsed is greater than *limit_seconds*. If your time + limit is 1 second, but it takes 2 seconds to generate the first item from + the iterable, the function will run for 2 seconds and not yield anything. + + """ + + def __init__(self, limit_seconds, iterable): + if limit_seconds < 0: + raise ValueError('limit_seconds must be positive') + self.limit_seconds = limit_seconds + self._iterable = iter(iterable) + self._start_time = monotonic() + self.timed_out = False + + def __iter__(self): + return self + + def __next__(self): + item = next(self._iterable) + if monotonic() - self._start_time > self.limit_seconds: + self.timed_out = True + raise StopIteration + + return item + + +def only(iterable, default=None, too_long=None): + """If *iterable* has only one item, return it. + If it has zero items, return *default*. + If it has more than one item, raise the exception given by *too_long*, + which is ``ValueError`` by default. + + >>> only([], default='missing') + 'missing' + >>> only([1]) + 1 + >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Expected exactly one item in iterable, but got 1, 2, + and perhaps more.' + >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError + + Note that :func:`only` attempts to advance *iterable* twice to ensure there + is only one item. See :func:`spy` or :func:`peekable` to check + iterable contents less destructively. + """ + it = iter(iterable) + first_value = next(it, default) + + try: + second_value = next(it) + except StopIteration: + pass + else: + msg = ( + 'Expected exactly one item in iterable, but got {!r}, {!r}, ' + 'and perhaps more.'.format(first_value, second_value) + ) + raise too_long or ValueError(msg) + + return first_value + + +def ichunked(iterable, n): + """Break *iterable* into sub-iterables with *n* elements each. + :func:`ichunked` is like :func:`chunked`, but it yields iterables + instead of lists. + + If the sub-iterables are read in order, the elements of *iterable* + won't be stored in memory. + If they are read out of order, :func:`itertools.tee` is used to cache + elements as necessary. + + >>> from itertools import count + >>> all_chunks = ichunked(count(), 4) + >>> c_1, c_2, c_3 = next(all_chunks), next(all_chunks), next(all_chunks) + >>> list(c_2) # c_1's elements have been cached; c_3's haven't been + [4, 5, 6, 7] + >>> list(c_1) + [0, 1, 2, 3] + >>> list(c_3) + [8, 9, 10, 11] + + """ + source = iter(iterable) + + while True: + # Check to see whether we're at the end of the source iterable + item = next(source, _marker) + if item is _marker: + return + + # Clone the source and yield an n-length slice + source, it = tee(chain([item], source)) + yield islice(it, n) + + # Advance the source iterable + consume(source, n) + + +def distinct_combinations(iterable, r): + """Yield the distinct combinations of *r* items taken from *iterable*. + + >>> list(distinct_combinations([0, 0, 1], 2)) + [(0, 0), (0, 1)] + + Equivalent to ``set(combinations(iterable))``, except duplicates are not + generated and thrown away. For larger input sequences this is much more + efficient. + + """ + if r < 0: + raise ValueError('r must be non-negative') + elif r == 0: + yield () + return + pool = tuple(iterable) + generators = [unique_everseen(enumerate(pool), key=itemgetter(1))] + current_combo = [None] * r + level = 0 + while generators: + try: + cur_idx, p = next(generators[-1]) + except StopIteration: + generators.pop() + level -= 1 + continue + current_combo[level] = p + if level + 1 == r: + yield tuple(current_combo) + else: + generators.append( + unique_everseen( + enumerate(pool[cur_idx + 1 :], cur_idx + 1), + key=itemgetter(1), + ) + ) + level += 1 + + +def filter_except(validator, iterable, *exceptions): + """Yield the items from *iterable* for which the *validator* function does + not raise one of the specified *exceptions*. + + *validator* is called for each item in *iterable*. + It should be a function that accepts one argument and raises an exception + if that item is not valid. + + >>> iterable = ['1', '2', 'three', '4', None] + >>> list(filter_except(int, iterable, ValueError, TypeError)) + ['1', '2', '4'] + + If an exception other than one given by *exceptions* is raised by + *validator*, it is raised like normal. + """ + for item in iterable: + try: + validator(item) + except exceptions: + pass + else: + yield item + + +def map_except(function, iterable, *exceptions): + """Transform each item from *iterable* with *function* and yield the + result, unless *function* raises one of the specified *exceptions*. + + *function* is called to transform each item in *iterable*. + It should be a accept one argument. + + >>> iterable = ['1', '2', 'three', '4', None] + >>> list(map_except(int, iterable, ValueError, TypeError)) + [1, 2, 4] + + If an exception other than one given by *exceptions* is raised by + *function*, it is raised like normal. + """ + for item in iterable: + try: + yield function(item) + except exceptions: + pass + + +def _sample_unweighted(iterable, k): + # Implementation of "Algorithm L" from the 1994 paper by Kim-Hung Li: + # "Reservoir-Sampling Algorithms of Time Complexity O(n(1+log(N/n)))". + + # Fill up the reservoir (collection of samples) with the first `k` samples + reservoir = take(k, iterable) + + # Generate random number that's the largest in a sample of k U(0,1) numbers + # Largest order statistic: https://en.wikipedia.org/wiki/Order_statistic + W = exp(log(random()) / k) + + # The number of elements to skip before changing the reservoir is a random + # number with a geometric distribution. Sample it using random() and logs. + next_index = k + floor(log(random()) / log(1 - W)) + + for index, element in enumerate(iterable, k): + + if index == next_index: + reservoir[randrange(k)] = element + # The new W is the largest in a sample of k U(0, `old_W`) numbers + W *= exp(log(random()) / k) + next_index += floor(log(random()) / log(1 - W)) + 1 + + return reservoir + + +def _sample_weighted(iterable, k, weights): + # Implementation of "A-ExpJ" from the 2006 paper by Efraimidis et al. : + # "Weighted random sampling with a reservoir". + + # Log-transform for numerical stability for weights that are small/large + weight_keys = (log(random()) / weight for weight in weights) + + # Fill up the reservoir (collection of samples) with the first `k` + # weight-keys and elements, then heapify the list. + reservoir = take(k, zip(weight_keys, iterable)) + heapify(reservoir) + + # The number of jumps before changing the reservoir is a random variable + # with an exponential distribution. Sample it using random() and logs. + smallest_weight_key, _ = reservoir[0] + weights_to_skip = log(random()) / smallest_weight_key + + for weight, element in zip(weights, iterable): + if weight >= weights_to_skip: + # The notation here is consistent with the paper, but we store + # the weight-keys in log-space for better numerical stability. + smallest_weight_key, _ = reservoir[0] + t_w = exp(weight * smallest_weight_key) + r_2 = uniform(t_w, 1) # generate U(t_w, 1) + weight_key = log(r_2) / weight + heapreplace(reservoir, (weight_key, element)) + smallest_weight_key, _ = reservoir[0] + weights_to_skip = log(random()) / smallest_weight_key + else: + weights_to_skip -= weight + + # Equivalent to [element for weight_key, element in sorted(reservoir)] + return [heappop(reservoir)[1] for _ in range(k)] + + +def sample(iterable, k, weights=None): + """Return a *k*-length list of elements chosen (without replacement) + from the *iterable*. Like :func:`random.sample`, but works on iterables + of unknown length. + + >>> iterable = range(100) + >>> sample(iterable, 5) # doctest: +SKIP + [81, 60, 96, 16, 4] + + An iterable with *weights* may also be given: + + >>> iterable = range(100) + >>> weights = (i * i + 1 for i in range(100)) + >>> sampled = sample(iterable, 5, weights=weights) # doctest: +SKIP + [79, 67, 74, 66, 78] + + The algorithm can also be used to generate weighted random permutations. + The relative weight of each item determines the probability that it + appears late in the permutation. + + >>> data = "abcdefgh" + >>> weights = range(1, len(data) + 1) + >>> sample(data, k=len(data), weights=weights) # doctest: +SKIP + ['c', 'a', 'b', 'e', 'g', 'd', 'h', 'f'] + """ + if k == 0: + return [] + + iterable = iter(iterable) + if weights is None: + return _sample_unweighted(iterable, k) + else: + weights = iter(weights) + return _sample_weighted(iterable, k, weights) + + +def is_sorted(iterable, key=None, reverse=False): + """Returns ``True`` if the items of iterable are in sorted order, and + ``False`` otherwise. *key* and *reverse* have the same meaning that they do + in the built-in :func:`sorted` function. + + >>> is_sorted(['1', '2', '3', '4', '5'], key=int) + True + >>> is_sorted([5, 4, 3, 1, 2], reverse=True) + False + + The function returns ``False`` after encountering the first out-of-order + item. If there are no out-of-order items, the iterable is exhausted. + """ + + compare = lt if reverse else gt + it = iterable if (key is None) else map(key, iterable) + return not any(starmap(compare, pairwise(it))) + + +class AbortThread(BaseException): + pass + + +class callback_iter: + """Convert a function that uses callbacks to an iterator. + + Let *func* be a function that takes a `callback` keyword argument. + For example: + + >>> def func(callback=None): + ... for i, c in [(1, 'a'), (2, 'b'), (3, 'c')]: + ... if callback: + ... callback(i, c) + ... return 4 + + + Use ``with callback_iter(func)`` to get an iterator over the parameters + that are delivered to the callback. + + >>> with callback_iter(func) as it: + ... for args, kwargs in it: + ... print(args) + (1, 'a') + (2, 'b') + (3, 'c') + + The function will be called in a background thread. The ``done`` property + indicates whether it has completed execution. + + >>> it.done + True + + If it completes successfully, its return value will be available + in the ``result`` property. + + >>> it.result + 4 + + Notes: + + * If the function uses some keyword argument besides ``callback``, supply + *callback_kwd*. + * If it finished executing, but raised an exception, accessing the + ``result`` property will raise the same exception. + * If it hasn't finished executing, accessing the ``result`` + property from within the ``with`` block will raise ``RuntimeError``. + * If it hasn't finished executing, accessing the ``result`` property from + outside the ``with`` block will raise a + ``more_itertools.AbortThread`` exception. + * Provide *wait_seconds* to adjust how frequently the it is polled for + output. + + """ + + def __init__(self, func, callback_kwd='callback', wait_seconds=0.1): + self._func = func + self._callback_kwd = callback_kwd + self._aborted = False + self._future = None + self._wait_seconds = wait_seconds + self._executor = ThreadPoolExecutor(max_workers=1) + self._iterator = self._reader() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._aborted = True + self._executor.shutdown() + + def __iter__(self): + return self + + def __next__(self): + return next(self._iterator) + + @property + def done(self): + if self._future is None: + return False + return self._future.done() + + @property + def result(self): + if not self.done: + raise RuntimeError('Function has not yet completed') + + return self._future.result() + + def _reader(self): + q = Queue() + + def callback(*args, **kwargs): + if self._aborted: + raise AbortThread('canceled by user') + + q.put((args, kwargs)) + + self._future = self._executor.submit( + self._func, **{self._callback_kwd: callback} + ) + + while True: + try: + item = q.get(timeout=self._wait_seconds) + except Empty: + pass + else: + q.task_done() + yield item + + if self._future.done(): + break + + remaining = [] + while True: + try: + item = q.get_nowait() + except Empty: + break + else: + q.task_done() + remaining.append(item) + q.join() + yield from remaining + + +def windowed_complete(iterable, n): + """ + Yield ``(beginning, middle, end)`` tuples, where: + + * Each ``middle`` has *n* items from *iterable* + * Each ``beginning`` has the items before the ones in ``middle`` + * Each ``end`` has the items after the ones in ``middle`` + + >>> iterable = range(7) + >>> n = 3 + >>> for beginning, middle, end in windowed_complete(iterable, n): + ... print(beginning, middle, end) + () (0, 1, 2) (3, 4, 5, 6) + (0,) (1, 2, 3) (4, 5, 6) + (0, 1) (2, 3, 4) (5, 6) + (0, 1, 2) (3, 4, 5) (6,) + (0, 1, 2, 3) (4, 5, 6) () + + Note that *n* must be at least 0 and most equal to the length of + *iterable*. + + This function will exhaust the iterable and may require significant + storage. + """ + if n < 0: + raise ValueError('n must be >= 0') + + seq = tuple(iterable) + size = len(seq) + + if n > size: + raise ValueError('n must be <= len(seq)') + + for i in range(size - n + 1): + beginning = seq[:i] + middle = seq[i : i + n] + end = seq[i + n :] + yield beginning, middle, end + + +def all_unique(iterable, key=None): + """ + Returns ``True`` if all the elements of *iterable* are unique (no two + elements are equal). + + >>> all_unique('ABCB') + False + + If a *key* function is specified, it will be used to make comparisons. + + >>> all_unique('ABCb') + True + >>> all_unique('ABCb', str.lower) + False + + The function returns as soon as the first non-unique element is + encountered. Iterables with a mix of hashable and unhashable items can + be used, but the function will be slower for unhashable items. + """ + seenset = set() + seenset_add = seenset.add + seenlist = [] + seenlist_add = seenlist.append + for element in map(key, iterable) if key else iterable: + try: + if element in seenset: + return False + seenset_add(element) + except TypeError: + if element in seenlist: + return False + seenlist_add(element) + return True + + +def nth_product(index, *args): + """Equivalent to ``list(product(*args))[index]``. + + The products of *args* can be ordered lexicographically. + :func:`nth_product` computes the product at sort position *index* without + computing the previous products. + + >>> nth_product(8, range(2), range(2), range(2), range(2)) + (1, 0, 0, 0) + + ``IndexError`` will be raised if the given *index* is invalid. + """ + pools = list(map(tuple, reversed(args))) + ns = list(map(len, pools)) + + c = reduce(mul, ns) + + if index < 0: + index += c + + if not 0 <= index < c: + raise IndexError + + result = [] + for pool, n in zip(pools, ns): + result.append(pool[index % n]) + index //= n + + return tuple(reversed(result)) + + +def nth_permutation(iterable, r, index): + """Equivalent to ``list(permutations(iterable, r))[index]``` + + The subsequences of *iterable* that are of length *r* where order is + important can be ordered lexicographically. :func:`nth_permutation` + computes the subsequence at sort position *index* directly, without + computing the previous subsequences. + + >>> nth_permutation('ghijk', 2, 5) + ('h', 'i') + + ``ValueError`` will be raised If *r* is negative or greater than the length + of *iterable*. + ``IndexError`` will be raised if the given *index* is invalid. + """ + pool = list(iterable) + n = len(pool) + + if r is None or r == n: + r, c = n, factorial(n) + elif not 0 <= r < n: + raise ValueError + else: + c = factorial(n) // factorial(n - r) + + if index < 0: + index += c + + if not 0 <= index < c: + raise IndexError + + if c == 0: + return tuple() + + result = [0] * r + q = index * factorial(n) // c if r < n else index + for d in range(1, n + 1): + q, i = divmod(q, d) + if 0 <= n - d < r: + result[n - d] = i + if q == 0: + break + + return tuple(map(pool.pop, result)) + + +def value_chain(*args): + """Yield all arguments passed to the function in the same order in which + they were passed. If an argument itself is iterable then iterate over its + values. + + >>> list(value_chain(1, 2, 3, [4, 5, 6])) + [1, 2, 3, 4, 5, 6] + + Binary and text strings are not considered iterable and are emitted + as-is: + + >>> list(value_chain('12', '34', ['56', '78'])) + ['12', '34', '56', '78'] + + + Multiple levels of nesting are not flattened. + + """ + for value in args: + if isinstance(value, (str, bytes)): + yield value + continue + try: + yield from value + except TypeError: + yield value + + +def product_index(element, *args): + """Equivalent to ``list(product(*args)).index(element)`` + + The products of *args* can be ordered lexicographically. + :func:`product_index` computes the first index of *element* without + computing the previous products. + + >>> product_index([8, 2], range(10), range(5)) + 42 + + ``ValueError`` will be raised if the given *element* isn't in the product + of *args*. + """ + index = 0 + + for x, pool in zip_longest(element, args, fillvalue=_marker): + if x is _marker or pool is _marker: + raise ValueError('element is not a product of args') + + pool = tuple(pool) + index = index * len(pool) + pool.index(x) + + return index + + +def combination_index(element, iterable): + """Equivalent to ``list(combinations(iterable, r)).index(element)`` + + The subsequences of *iterable* that are of length *r* can be ordered + lexicographically. :func:`combination_index` computes the index of the + first *element*, without computing the previous combinations. + + >>> combination_index('adf', 'abcdefg') + 10 + + ``ValueError`` will be raised if the given *element* isn't one of the + combinations of *iterable*. + """ + element = enumerate(element) + k, y = next(element, (None, None)) + if k is None: + return 0 + + indexes = [] + pool = enumerate(iterable) + for n, x in pool: + if x == y: + indexes.append(n) + tmp, y = next(element, (None, None)) + if tmp is None: + break + else: + k = tmp + else: + raise ValueError('element is not a combination of iterable') + + n, _ = last(pool, default=(n, None)) + + # Python versiosn below 3.8 don't have math.comb + index = 1 + for i, j in enumerate(reversed(indexes), start=1): + j = n - j + if i <= j: + index += factorial(j) // (factorial(i) * factorial(j - i)) + + return factorial(n + 1) // (factorial(k + 1) * factorial(n - k)) - index + + +def permutation_index(element, iterable): + """Equivalent to ``list(permutations(iterable, r)).index(element)``` + + The subsequences of *iterable* that are of length *r* where order is + important can be ordered lexicographically. :func:`permutation_index` + computes the index of the first *element* directly, without computing + the previous permutations. + + >>> permutation_index([1, 3, 2], range(5)) + 19 + + ``ValueError`` will be raised if the given *element* isn't one of the + permutations of *iterable*. + """ + index = 0 + pool = list(iterable) + for i, x in zip(range(len(pool), -1, -1), element): + r = pool.index(x) + index = index * i + r + del pool[r] + + return index + + +class countable: + """Wrap *iterable* and keep a count of how many items have been consumed. + + The ``items_seen`` attribute starts at ``0`` and increments as the iterable + is consumed: + + >>> iterable = map(str, range(10)) + >>> it = countable(iterable) + >>> it.items_seen + 0 + >>> next(it), next(it) + ('0', '1') + >>> list(it) + ['2', '3', '4', '5', '6', '7', '8', '9'] + >>> it.items_seen + 10 + """ + + def __init__(self, iterable): + self._it = iter(iterable) + self.items_seen = 0 + + def __iter__(self): + return self + + def __next__(self): + item = next(self._it) + self.items_seen += 1 + + return item diff --git a/setuptools/_vendor/more_itertools/more.pyi b/setuptools/_vendor/more_itertools/more.pyi new file mode 100644 index 00000000..2fba9cb3 --- /dev/null +++ b/setuptools/_vendor/more_itertools/more.pyi @@ -0,0 +1,480 @@ +"""Stubs for more_itertools.more""" + +from typing import ( + Any, + Callable, + Container, + Dict, + Generic, + Hashable, + Iterable, + Iterator, + List, + Optional, + Reversible, + Sequence, + Sized, + Tuple, + Union, + TypeVar, + type_check_only, +) +from types import TracebackType +from typing_extensions import ContextManager, Protocol, Type, overload + +# Type and type variable definitions +_T = TypeVar('_T') +_U = TypeVar('_U') +_V = TypeVar('_V') +_W = TypeVar('_W') +_T_co = TypeVar('_T_co', covariant=True) +_GenFn = TypeVar('_GenFn', bound=Callable[..., Iterator[object]]) +_Raisable = Union[BaseException, 'Type[BaseException]'] + +@type_check_only +class _SizedIterable(Protocol[_T_co], Sized, Iterable[_T_co]): ... + +@type_check_only +class _SizedReversible(Protocol[_T_co], Sized, Reversible[_T_co]): ... + +def chunked( + iterable: Iterable[_T], n: int, strict: bool = ... +) -> Iterator[List[_T]]: ... +@overload +def first(iterable: Iterable[_T]) -> _T: ... +@overload +def first(iterable: Iterable[_T], default: _U) -> Union[_T, _U]: ... +@overload +def last(iterable: Iterable[_T]) -> _T: ... +@overload +def last(iterable: Iterable[_T], default: _U) -> Union[_T, _U]: ... +@overload +def nth_or_last(iterable: Iterable[_T], n: int) -> _T: ... +@overload +def nth_or_last( + iterable: Iterable[_T], n: int, default: _U +) -> Union[_T, _U]: ... + +class peekable(Generic[_T], Iterator[_T]): + def __init__(self, iterable: Iterable[_T]) -> None: ... + def __iter__(self) -> peekable[_T]: ... + def __bool__(self) -> bool: ... + @overload + def peek(self) -> _T: ... + @overload + def peek(self, default: _U) -> Union[_T, _U]: ... + def prepend(self, *items: _T) -> None: ... + def __next__(self) -> _T: ... + @overload + def __getitem__(self, index: int) -> _T: ... + @overload + def __getitem__(self, index: slice) -> List[_T]: ... + +def collate(*iterables: Iterable[_T], **kwargs: Any) -> Iterable[_T]: ... +def consumer(func: _GenFn) -> _GenFn: ... +def ilen(iterable: Iterable[object]) -> int: ... +def iterate(func: Callable[[_T], _T], start: _T) -> Iterator[_T]: ... +def with_iter( + context_manager: ContextManager[Iterable[_T]], +) -> Iterator[_T]: ... +def one( + iterable: Iterable[_T], + too_short: Optional[_Raisable] = ..., + too_long: Optional[_Raisable] = ..., +) -> _T: ... +def distinct_permutations( + iterable: Iterable[_T], r: Optional[int] = ... +) -> Iterator[Tuple[_T, ...]]: ... +def intersperse( + e: _U, iterable: Iterable[_T], n: int = ... +) -> Iterator[Union[_T, _U]]: ... +def unique_to_each(*iterables: Iterable[_T]) -> List[List[_T]]: ... +@overload +def windowed( + seq: Iterable[_T], n: int, *, step: int = ... +) -> Iterator[Tuple[Optional[_T], ...]]: ... +@overload +def windowed( + seq: Iterable[_T], n: int, fillvalue: _U, step: int = ... +) -> Iterator[Tuple[Union[_T, _U], ...]]: ... +def substrings(iterable: Iterable[_T]) -> Iterator[Tuple[_T, ...]]: ... +def substrings_indexes( + seq: Sequence[_T], reverse: bool = ... +) -> Iterator[Tuple[Sequence[_T], int, int]]: ... + +class bucket(Generic[_T, _U], Container[_U]): + def __init__( + self, + iterable: Iterable[_T], + key: Callable[[_T], _U], + validator: Optional[Callable[[object], object]] = ..., + ) -> None: ... + def __contains__(self, value: object) -> bool: ... + def __iter__(self) -> Iterator[_U]: ... + def __getitem__(self, value: object) -> Iterator[_T]: ... + +def spy( + iterable: Iterable[_T], n: int = ... +) -> Tuple[List[_T], Iterator[_T]]: ... +def interleave(*iterables: Iterable[_T]) -> Iterator[_T]: ... +def interleave_longest(*iterables: Iterable[_T]) -> Iterator[_T]: ... +def collapse( + iterable: Iterable[Any], + base_type: Optional[type] = ..., + levels: Optional[int] = ..., +) -> Iterator[Any]: ... +@overload +def side_effect( + func: Callable[[_T], object], + iterable: Iterable[_T], + chunk_size: None = ..., + before: Optional[Callable[[], object]] = ..., + after: Optional[Callable[[], object]] = ..., +) -> Iterator[_T]: ... +@overload +def side_effect( + func: Callable[[List[_T]], object], + iterable: Iterable[_T], + chunk_size: int, + before: Optional[Callable[[], object]] = ..., + after: Optional[Callable[[], object]] = ..., +) -> Iterator[_T]: ... +def sliced( + seq: Sequence[_T], n: int, strict: bool = ... +) -> Iterator[Sequence[_T]]: ... +def split_at( + iterable: Iterable[_T], + pred: Callable[[_T], object], + maxsplit: int = ..., + keep_separator: bool = ..., +) -> Iterator[List[_T]]: ... +def split_before( + iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... +) -> Iterator[List[_T]]: ... +def split_after( + iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... +) -> Iterator[List[_T]]: ... +def split_when( + iterable: Iterable[_T], + pred: Callable[[_T, _T], object], + maxsplit: int = ..., +) -> Iterator[List[_T]]: ... +def split_into( + iterable: Iterable[_T], sizes: Iterable[Optional[int]] +) -> Iterator[List[_T]]: ... +@overload +def padded( + iterable: Iterable[_T], + *, + n: Optional[int] = ..., + next_multiple: bool = ... +) -> Iterator[Optional[_T]]: ... +@overload +def padded( + iterable: Iterable[_T], + fillvalue: _U, + n: Optional[int] = ..., + next_multiple: bool = ..., +) -> Iterator[Union[_T, _U]]: ... +@overload +def repeat_last(iterable: Iterable[_T]) -> Iterator[_T]: ... +@overload +def repeat_last( + iterable: Iterable[_T], default: _U +) -> Iterator[Union[_T, _U]]: ... +def distribute(n: int, iterable: Iterable[_T]) -> List[Iterator[_T]]: ... +@overload +def stagger( + iterable: Iterable[_T], + offsets: _SizedIterable[int] = ..., + longest: bool = ..., +) -> Iterator[Tuple[Optional[_T], ...]]: ... +@overload +def stagger( + iterable: Iterable[_T], + offsets: _SizedIterable[int] = ..., + longest: bool = ..., + fillvalue: _U = ..., +) -> Iterator[Tuple[Union[_T, _U], ...]]: ... + +class UnequalIterablesError(ValueError): + def __init__( + self, details: Optional[Tuple[int, int, int]] = ... + ) -> None: ... + +def zip_equal(*iterables: Iterable[_T]) -> Iterator[Tuple[_T, ...]]: ... +@overload +def zip_offset( + *iterables: Iterable[_T], offsets: _SizedIterable[int], longest: bool = ... +) -> Iterator[Tuple[Optional[_T], ...]]: ... +@overload +def zip_offset( + *iterables: Iterable[_T], + offsets: _SizedIterable[int], + longest: bool = ..., + fillvalue: _U +) -> Iterator[Tuple[Union[_T, _U], ...]]: ... +def sort_together( + iterables: Iterable[Iterable[_T]], + key_list: Iterable[int] = ..., + key: Optional[Callable[..., Any]] = ..., + reverse: bool = ..., +) -> List[Tuple[_T, ...]]: ... +def unzip(iterable: Iterable[Sequence[_T]]) -> Tuple[Iterator[_T], ...]: ... +def divide(n: int, iterable: Iterable[_T]) -> List[Iterator[_T]]: ... +def always_iterable( + obj: object, + base_type: Union[ + type, Tuple[Union[type, Tuple[Any, ...]], ...], None + ] = ..., +) -> Iterator[Any]: ... +def adjacent( + predicate: Callable[[_T], bool], + iterable: Iterable[_T], + distance: int = ..., +) -> Iterator[Tuple[bool, _T]]: ... +def groupby_transform( + iterable: Iterable[_T], + keyfunc: Optional[Callable[[_T], _U]] = ..., + valuefunc: Optional[Callable[[_T], _V]] = ..., + reducefunc: Optional[Callable[..., _W]] = ..., +) -> Iterator[Tuple[_T, _W]]: ... + +class numeric_range(Generic[_T, _U], Sequence[_T], Hashable, Reversible[_T]): + @overload + def __init__(self, __stop: _T) -> None: ... + @overload + def __init__(self, __start: _T, __stop: _T) -> None: ... + @overload + def __init__(self, __start: _T, __stop: _T, __step: _U) -> None: ... + def __bool__(self) -> bool: ... + def __contains__(self, elem: object) -> bool: ... + def __eq__(self, other: object) -> bool: ... + @overload + def __getitem__(self, key: int) -> _T: ... + @overload + def __getitem__(self, key: slice) -> numeric_range[_T, _U]: ... + def __hash__(self) -> int: ... + def __iter__(self) -> Iterator[_T]: ... + def __len__(self) -> int: ... + def __reduce__( + self, + ) -> Tuple[Type[numeric_range[_T, _U]], Tuple[_T, _T, _U]]: ... + def __repr__(self) -> str: ... + def __reversed__(self) -> Iterator[_T]: ... + def count(self, value: _T) -> int: ... + def index(self, value: _T) -> int: ... # type: ignore + +def count_cycle( + iterable: Iterable[_T], n: Optional[int] = ... +) -> Iterable[Tuple[int, _T]]: ... +def mark_ends( + iterable: Iterable[_T], +) -> Iterable[Tuple[bool, bool, _T]]: ... +def locate( + iterable: Iterable[object], + pred: Callable[..., Any] = ..., + window_size: Optional[int] = ..., +) -> Iterator[int]: ... +def lstrip( + iterable: Iterable[_T], pred: Callable[[_T], object] +) -> Iterator[_T]: ... +def rstrip( + iterable: Iterable[_T], pred: Callable[[_T], object] +) -> Iterator[_T]: ... +def strip( + iterable: Iterable[_T], pred: Callable[[_T], object] +) -> Iterator[_T]: ... + +class islice_extended(Generic[_T], Iterator[_T]): + def __init__( + self, iterable: Iterable[_T], *args: Optional[int] + ) -> None: ... + def __iter__(self) -> islice_extended[_T]: ... + def __next__(self) -> _T: ... + def __getitem__(self, index: slice) -> islice_extended[_T]: ... + +def always_reversible(iterable: Iterable[_T]) -> Iterator[_T]: ... +def consecutive_groups( + iterable: Iterable[_T], ordering: Callable[[_T], int] = ... +) -> Iterator[Iterator[_T]]: ... +@overload +def difference( + iterable: Iterable[_T], + func: Callable[[_T, _T], _U] = ..., + *, + initial: None = ... +) -> Iterator[Union[_T, _U]]: ... +@overload +def difference( + iterable: Iterable[_T], func: Callable[[_T, _T], _U] = ..., *, initial: _U +) -> Iterator[_U]: ... + +class SequenceView(Generic[_T], Sequence[_T]): + def __init__(self, target: Sequence[_T]) -> None: ... + @overload + def __getitem__(self, index: int) -> _T: ... + @overload + def __getitem__(self, index: slice) -> Sequence[_T]: ... + def __len__(self) -> int: ... + +class seekable(Generic[_T], Iterator[_T]): + def __init__( + self, iterable: Iterable[_T], maxlen: Optional[int] = ... + ) -> None: ... + def __iter__(self) -> seekable[_T]: ... + def __next__(self) -> _T: ... + def __bool__(self) -> bool: ... + @overload + def peek(self) -> _T: ... + @overload + def peek(self, default: _U) -> Union[_T, _U]: ... + def elements(self) -> SequenceView[_T]: ... + def seek(self, index: int) -> None: ... + +class run_length: + @staticmethod + def encode(iterable: Iterable[_T]) -> Iterator[Tuple[_T, int]]: ... + @staticmethod + def decode(iterable: Iterable[Tuple[_T, int]]) -> Iterator[_T]: ... + +def exactly_n( + iterable: Iterable[_T], n: int, predicate: Callable[[_T], object] = ... +) -> bool: ... +def circular_shifts(iterable: Iterable[_T]) -> List[Tuple[_T, ...]]: ... +def make_decorator( + wrapping_func: Callable[..., _U], result_index: int = ... +) -> Callable[..., Callable[[Callable[..., Any]], Callable[..., _U]]]: ... +@overload +def map_reduce( + iterable: Iterable[_T], + keyfunc: Callable[[_T], _U], + valuefunc: None = ..., + reducefunc: None = ..., +) -> Dict[_U, List[_T]]: ... +@overload +def map_reduce( + iterable: Iterable[_T], + keyfunc: Callable[[_T], _U], + valuefunc: Callable[[_T], _V], + reducefunc: None = ..., +) -> Dict[_U, List[_V]]: ... +@overload +def map_reduce( + iterable: Iterable[_T], + keyfunc: Callable[[_T], _U], + valuefunc: None = ..., + reducefunc: Callable[[List[_T]], _W] = ..., +) -> Dict[_U, _W]: ... +@overload +def map_reduce( + iterable: Iterable[_T], + keyfunc: Callable[[_T], _U], + valuefunc: Callable[[_T], _V], + reducefunc: Callable[[List[_V]], _W], +) -> Dict[_U, _W]: ... +def rlocate( + iterable: Iterable[_T], + pred: Callable[..., object] = ..., + window_size: Optional[int] = ..., +) -> Iterator[int]: ... +def replace( + iterable: Iterable[_T], + pred: Callable[..., object], + substitutes: Iterable[_U], + count: Optional[int] = ..., + window_size: int = ..., +) -> Iterator[Union[_T, _U]]: ... +def partitions(iterable: Iterable[_T]) -> Iterator[List[List[_T]]]: ... +def set_partitions( + iterable: Iterable[_T], k: Optional[int] = ... +) -> Iterator[List[List[_T]]]: ... + +class time_limited(Generic[_T], Iterator[_T]): + def __init__( + self, limit_seconds: float, iterable: Iterable[_T] + ) -> None: ... + def __iter__(self) -> islice_extended[_T]: ... + def __next__(self) -> _T: ... + +@overload +def only( + iterable: Iterable[_T], *, too_long: Optional[_Raisable] = ... +) -> Optional[_T]: ... +@overload +def only( + iterable: Iterable[_T], default: _U, too_long: Optional[_Raisable] = ... +) -> Union[_T, _U]: ... +def ichunked(iterable: Iterable[_T], n: int) -> Iterator[Iterator[_T]]: ... +def distinct_combinations( + iterable: Iterable[_T], r: int +) -> Iterator[Tuple[_T, ...]]: ... +def filter_except( + validator: Callable[[Any], object], + iterable: Iterable[_T], + *exceptions: Type[BaseException] +) -> Iterator[_T]: ... +def map_except( + function: Callable[[Any], _U], + iterable: Iterable[_T], + *exceptions: Type[BaseException] +) -> Iterator[_U]: ... +def sample( + iterable: Iterable[_T], + k: int, + weights: Optional[Iterable[float]] = ..., +) -> List[_T]: ... +def is_sorted( + iterable: Iterable[_T], + key: Optional[Callable[[_T], _U]] = ..., + reverse: bool = False, +) -> bool: ... + +class AbortThread(BaseException): + pass + +class callback_iter(Generic[_T], Iterator[_T]): + def __init__( + self, + func: Callable[..., Any], + callback_kwd: str = ..., + wait_seconds: float = ..., + ) -> None: ... + def __enter__(self) -> callback_iter[_T]: ... + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> Optional[bool]: ... + def __iter__(self) -> callback_iter[_T]: ... + def __next__(self) -> _T: ... + def _reader(self) -> Iterator[_T]: ... + @property + def done(self) -> bool: ... + @property + def result(self) -> Any: ... + +def windowed_complete( + iterable: Iterable[_T], n: int +) -> Iterator[Tuple[_T, ...]]: ... +def all_unique( + iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = ... +) -> bool: ... +def nth_product(index: int, *args: Iterable[_T]) -> Tuple[_T, ...]: ... +def nth_permutation( + iterable: Iterable[_T], r: int, index: int +) -> Tuple[_T, ...]: ... +def value_chain(*args: Union[_T, Iterable[_T]]) -> Iterable[_T]: ... +def product_index(element: Iterable[_T], *args: Iterable[_T]) -> int: ... +def combination_index( + element: Iterable[_T], iterable: Iterable[_T] +) -> int: ... +def permutation_index( + element: Iterable[_T], iterable: Iterable[_T] +) -> int: ... + +class countable(Generic[_T], Iterator[_T]): + def __init__(self, iterable: Iterable[_T]) -> None: ... + def __iter__(self) -> countable[_T]: ... + def __next__(self) -> _T: ... diff --git a/setuptools/_vendor/more_itertools/py.typed b/setuptools/_vendor/more_itertools/py.typed new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/setuptools/_vendor/more_itertools/py.typed diff --git a/setuptools/_vendor/more_itertools/recipes.py b/setuptools/_vendor/more_itertools/recipes.py new file mode 100644 index 00000000..521abd7c --- /dev/null +++ b/setuptools/_vendor/more_itertools/recipes.py @@ -0,0 +1,620 @@ +"""Imported from the recipes section of the itertools documentation. + +All functions taken from the recipes section of the itertools library docs +[1]_. +Some backward-compatible usability improvements have been made. + +.. [1] http://docs.python.org/library/itertools.html#recipes + +""" +import warnings +from collections import deque +from itertools import ( + chain, + combinations, + count, + cycle, + groupby, + islice, + repeat, + starmap, + tee, + zip_longest, +) +import operator +from random import randrange, sample, choice + +__all__ = [ + 'all_equal', + 'consume', + 'convolve', + 'dotproduct', + 'first_true', + 'flatten', + 'grouper', + 'iter_except', + 'ncycles', + 'nth', + 'nth_combination', + 'padnone', + 'pad_none', + 'pairwise', + 'partition', + 'powerset', + 'prepend', + 'quantify', + 'random_combination_with_replacement', + 'random_combination', + 'random_permutation', + 'random_product', + 'repeatfunc', + 'roundrobin', + 'tabulate', + 'tail', + 'take', + 'unique_everseen', + 'unique_justseen', +] + + +def take(n, iterable): + """Return first *n* items of the iterable as a list. + + >>> take(3, range(10)) + [0, 1, 2] + + If there are fewer than *n* items in the iterable, all of them are + returned. + + >>> take(10, range(3)) + [0, 1, 2] + + """ + return list(islice(iterable, n)) + + +def tabulate(function, start=0): + """Return an iterator over the results of ``func(start)``, + ``func(start + 1)``, ``func(start + 2)``... + + *func* should be a function that accepts one integer argument. + + If *start* is not specified it defaults to 0. It will be incremented each + time the iterator is advanced. + + >>> square = lambda x: x ** 2 + >>> iterator = tabulate(square, -3) + >>> take(4, iterator) + [9, 4, 1, 0] + + """ + return map(function, count(start)) + + +def tail(n, iterable): + """Return an iterator over the last *n* items of *iterable*. + + >>> t = tail(3, 'ABCDEFG') + >>> list(t) + ['E', 'F', 'G'] + + """ + return iter(deque(iterable, maxlen=n)) + + +def consume(iterator, n=None): + """Advance *iterable* by *n* steps. If *n* is ``None``, consume it + entirely. + + Efficiently exhausts an iterator without returning values. Defaults to + consuming the whole iterator, but an optional second argument may be + provided to limit consumption. + + >>> i = (x for x in range(10)) + >>> next(i) + 0 + >>> consume(i, 3) + >>> next(i) + 4 + >>> consume(i) + >>> next(i) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + StopIteration + + If the iterator has fewer items remaining than the provided limit, the + whole iterator will be consumed. + + >>> i = (x for x in range(3)) + >>> consume(i, 5) + >>> next(i) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + StopIteration + + """ + # Use functions that consume iterators at C speed. + if n is None: + # feed the entire iterator into a zero-length deque + deque(iterator, maxlen=0) + else: + # advance to the empty slice starting at position n + next(islice(iterator, n, n), None) + + +def nth(iterable, n, default=None): + """Returns the nth item or a default value. + + >>> l = range(10) + >>> nth(l, 3) + 3 + >>> nth(l, 20, "zebra") + 'zebra' + + """ + return next(islice(iterable, n, None), default) + + +def all_equal(iterable): + """ + Returns ``True`` if all the elements are equal to each other. + + >>> all_equal('aaaa') + True + >>> all_equal('aaab') + False + + """ + g = groupby(iterable) + return next(g, True) and not next(g, False) + + +def quantify(iterable, pred=bool): + """Return the how many times the predicate is true. + + >>> quantify([True, False, True]) + 2 + + """ + return sum(map(pred, iterable)) + + +def pad_none(iterable): + """Returns the sequence of elements and then returns ``None`` indefinitely. + + >>> take(5, pad_none(range(3))) + [0, 1, 2, None, None] + + Useful for emulating the behavior of the built-in :func:`map` function. + + See also :func:`padded`. + + """ + return chain(iterable, repeat(None)) + + +padnone = pad_none + + +def ncycles(iterable, n): + """Returns the sequence elements *n* times + + >>> list(ncycles(["a", "b"], 3)) + ['a', 'b', 'a', 'b', 'a', 'b'] + + """ + return chain.from_iterable(repeat(tuple(iterable), n)) + + +def dotproduct(vec1, vec2): + """Returns the dot product of the two iterables. + + >>> dotproduct([10, 10], [20, 20]) + 400 + + """ + return sum(map(operator.mul, vec1, vec2)) + + +def flatten(listOfLists): + """Return an iterator flattening one level of nesting in a list of lists. + + >>> list(flatten([[0, 1], [2, 3]])) + [0, 1, 2, 3] + + See also :func:`collapse`, which can flatten multiple levels of nesting. + + """ + return chain.from_iterable(listOfLists) + + +def repeatfunc(func, times=None, *args): + """Call *func* with *args* repeatedly, returning an iterable over the + results. + + If *times* is specified, the iterable will terminate after that many + repetitions: + + >>> from operator import add + >>> times = 4 + >>> args = 3, 5 + >>> list(repeatfunc(add, times, *args)) + [8, 8, 8, 8] + + If *times* is ``None`` the iterable will not terminate: + + >>> from random import randrange + >>> times = None + >>> args = 1, 11 + >>> take(6, repeatfunc(randrange, times, *args)) # doctest:+SKIP + [2, 4, 8, 1, 8, 4] + + """ + if times is None: + return starmap(func, repeat(args)) + return starmap(func, repeat(args, times)) + + +def _pairwise(iterable): + """Returns an iterator of paired items, overlapping, from the original + + >>> take(4, pairwise(count())) + [(0, 1), (1, 2), (2, 3), (3, 4)] + + On Python 3.10 and above, this is an alias for :func:`itertools.pairwise`. + + """ + a, b = tee(iterable) + next(b, None) + yield from zip(a, b) + + +try: + from itertools import pairwise as itertools_pairwise +except ImportError: + pairwise = _pairwise +else: + + def pairwise(iterable): + yield from itertools_pairwise(iterable) + + pairwise.__doc__ = _pairwise.__doc__ + + +def grouper(iterable, n, fillvalue=None): + """Collect data into fixed-length chunks or blocks. + + >>> list(grouper('ABCDEFG', 3, 'x')) + [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'x', 'x')] + + """ + if isinstance(iterable, int): + warnings.warn( + "grouper expects iterable as first parameter", DeprecationWarning + ) + n, iterable = iterable, n + args = [iter(iterable)] * n + return zip_longest(fillvalue=fillvalue, *args) + + +def roundrobin(*iterables): + """Yields an item from each iterable, alternating between them. + + >>> list(roundrobin('ABC', 'D', 'EF')) + ['A', 'D', 'E', 'B', 'F', 'C'] + + This function produces the same output as :func:`interleave_longest`, but + may perform better for some inputs (in particular when the number of + iterables is small). + + """ + # Recipe credited to George Sakkis + pending = len(iterables) + nexts = cycle(iter(it).__next__ for it in iterables) + while pending: + try: + for next in nexts: + yield next() + except StopIteration: + pending -= 1 + nexts = cycle(islice(nexts, pending)) + + +def partition(pred, iterable): + """ + Returns a 2-tuple of iterables derived from the input iterable. + The first yields the items that have ``pred(item) == False``. + The second yields the items that have ``pred(item) == True``. + + >>> is_odd = lambda x: x % 2 != 0 + >>> iterable = range(10) + >>> even_items, odd_items = partition(is_odd, iterable) + >>> list(even_items), list(odd_items) + ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9]) + + If *pred* is None, :func:`bool` is used. + + >>> iterable = [0, 1, False, True, '', ' '] + >>> false_items, true_items = partition(None, iterable) + >>> list(false_items), list(true_items) + ([0, False, ''], [1, True, ' ']) + + """ + if pred is None: + pred = bool + + evaluations = ((pred(x), x) for x in iterable) + t1, t2 = tee(evaluations) + return ( + (x for (cond, x) in t1 if not cond), + (x for (cond, x) in t2 if cond), + ) + + +def powerset(iterable): + """Yields all possible subsets of the iterable. + + >>> list(powerset([1, 2, 3])) + [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] + + :func:`powerset` will operate on iterables that aren't :class:`set` + instances, so repeated elements in the input will produce repeated elements + in the output. Use :func:`unique_everseen` on the input to avoid generating + duplicates: + + >>> seq = [1, 1, 0] + >>> list(powerset(seq)) + [(), (1,), (1,), (0,), (1, 1), (1, 0), (1, 0), (1, 1, 0)] + >>> from more_itertools import unique_everseen + >>> list(powerset(unique_everseen(seq))) + [(), (1,), (0,), (1, 0)] + + """ + s = list(iterable) + return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1)) + + +def unique_everseen(iterable, key=None): + """ + Yield unique elements, preserving order. + + >>> list(unique_everseen('AAAABBBCCDAABBB')) + ['A', 'B', 'C', 'D'] + >>> list(unique_everseen('ABBCcAD', str.lower)) + ['A', 'B', 'C', 'D'] + + Sequences with a mix of hashable and unhashable items can be used. + The function will be slower (i.e., `O(n^2)`) for unhashable items. + + Remember that ``list`` objects are unhashable - you can use the *key* + parameter to transform the list to a tuple (which is hashable) to + avoid a slowdown. + + >>> iterable = ([1, 2], [2, 3], [1, 2]) + >>> list(unique_everseen(iterable)) # Slow + [[1, 2], [2, 3]] + >>> list(unique_everseen(iterable, key=tuple)) # Faster + [[1, 2], [2, 3]] + + Similary, you may want to convert unhashable ``set`` objects with + ``key=frozenset``. For ``dict`` objects, + ``key=lambda x: frozenset(x.items())`` can be used. + + """ + seenset = set() + seenset_add = seenset.add + seenlist = [] + seenlist_add = seenlist.append + use_key = key is not None + + for element in iterable: + k = key(element) if use_key else element + try: + if k not in seenset: + seenset_add(k) + yield element + except TypeError: + if k not in seenlist: + seenlist_add(k) + yield element + + +def unique_justseen(iterable, key=None): + """Yields elements in order, ignoring serial duplicates + + >>> list(unique_justseen('AAAABBBCCDAABBB')) + ['A', 'B', 'C', 'D', 'A', 'B'] + >>> list(unique_justseen('ABBCcAD', str.lower)) + ['A', 'B', 'C', 'A', 'D'] + + """ + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + + +def iter_except(func, exception, first=None): + """Yields results from a function repeatedly until an exception is raised. + + Converts a call-until-exception interface to an iterator interface. + Like ``iter(func, sentinel)``, but uses an exception instead of a sentinel + to end the loop. + + >>> l = [0, 1, 2] + >>> list(iter_except(l.pop, IndexError)) + [2, 1, 0] + + """ + try: + if first is not None: + yield first() + while 1: + yield func() + except exception: + pass + + +def first_true(iterable, default=None, pred=None): + """ + Returns the first true value in the iterable. + + If no true value is found, returns *default* + + If *pred* is not None, returns the first item for which + ``pred(item) == True`` . + + >>> first_true(range(10)) + 1 + >>> first_true(range(10), pred=lambda x: x > 5) + 6 + >>> first_true(range(10), default='missing', pred=lambda x: x > 9) + 'missing' + + """ + return next(filter(pred, iterable), default) + + +def random_product(*args, repeat=1): + """Draw an item at random from each of the input iterables. + + >>> random_product('abc', range(4), 'XYZ') # doctest:+SKIP + ('c', 3, 'Z') + + If *repeat* is provided as a keyword argument, that many items will be + drawn from each iterable. + + >>> random_product('abcd', range(4), repeat=2) # doctest:+SKIP + ('a', 2, 'd', 3) + + This equivalent to taking a random selection from + ``itertools.product(*args, **kwarg)``. + + """ + pools = [tuple(pool) for pool in args] * repeat + return tuple(choice(pool) for pool in pools) + + +def random_permutation(iterable, r=None): + """Return a random *r* length permutation of the elements in *iterable*. + + If *r* is not specified or is ``None``, then *r* defaults to the length of + *iterable*. + + >>> random_permutation(range(5)) # doctest:+SKIP + (3, 4, 0, 1, 2) + + This equivalent to taking a random selection from + ``itertools.permutations(iterable, r)``. + + """ + pool = tuple(iterable) + r = len(pool) if r is None else r + return tuple(sample(pool, r)) + + +def random_combination(iterable, r): + """Return a random *r* length subsequence of the elements in *iterable*. + + >>> random_combination(range(5), 3) # doctest:+SKIP + (2, 3, 4) + + This equivalent to taking a random selection from + ``itertools.combinations(iterable, r)``. + + """ + pool = tuple(iterable) + n = len(pool) + indices = sorted(sample(range(n), r)) + return tuple(pool[i] for i in indices) + + +def random_combination_with_replacement(iterable, r): + """Return a random *r* length subsequence of elements in *iterable*, + allowing individual elements to be repeated. + + >>> random_combination_with_replacement(range(3), 5) # doctest:+SKIP + (0, 0, 1, 2, 2) + + This equivalent to taking a random selection from + ``itertools.combinations_with_replacement(iterable, r)``. + + """ + pool = tuple(iterable) + n = len(pool) + indices = sorted(randrange(n) for i in range(r)) + return tuple(pool[i] for i in indices) + + +def nth_combination(iterable, r, index): + """Equivalent to ``list(combinations(iterable, r))[index]``. + + The subsequences of *iterable* that are of length *r* can be ordered + lexicographically. :func:`nth_combination` computes the subsequence at + sort position *index* directly, without computing the previous + subsequences. + + >>> nth_combination(range(5), 3, 5) + (0, 3, 4) + + ``ValueError`` will be raised If *r* is negative or greater than the length + of *iterable*. + ``IndexError`` will be raised if the given *index* is invalid. + """ + pool = tuple(iterable) + n = len(pool) + if (r < 0) or (r > n): + raise ValueError + + c = 1 + k = min(r, n - r) + for i in range(1, k + 1): + c = c * (n - k + i) // i + + if index < 0: + index += c + + if (index < 0) or (index >= c): + raise IndexError + + result = [] + while r: + c, n, r = c * r // n, n - 1, r - 1 + while index >= c: + index -= c + c, n = c * (n - r) // n, n - 1 + result.append(pool[-1 - n]) + + return tuple(result) + + +def prepend(value, iterator): + """Yield *value*, followed by the elements in *iterator*. + + >>> value = '0' + >>> iterator = ['1', '2', '3'] + >>> list(prepend(value, iterator)) + ['0', '1', '2', '3'] + + To prepend multiple values, see :func:`itertools.chain` + or :func:`value_chain`. + + """ + return chain([value], iterator) + + +def convolve(signal, kernel): + """Convolve the iterable *signal* with the iterable *kernel*. + + >>> signal = (1, 2, 3, 4, 5) + >>> kernel = [3, 2, 1] + >>> list(convolve(signal, kernel)) + [3, 8, 14, 20, 26, 14, 5] + + Note: the input arguments are not interchangeable, as the *kernel* + is immediately consumed and stored. + + """ + kernel = tuple(kernel)[::-1] + n = len(kernel) + window = deque([0], maxlen=n) * n + for x in chain(signal, repeat(0, n - 1)): + window.append(x) + yield sum(map(operator.mul, kernel, window)) diff --git a/setuptools/_vendor/more_itertools/recipes.pyi b/setuptools/_vendor/more_itertools/recipes.pyi new file mode 100644 index 00000000..5e39d963 --- /dev/null +++ b/setuptools/_vendor/more_itertools/recipes.pyi @@ -0,0 +1,103 @@ +"""Stubs for more_itertools.recipes""" +from typing import ( + Any, + Callable, + Iterable, + Iterator, + List, + Optional, + Tuple, + TypeVar, + Union, +) +from typing_extensions import overload, Type + +# Type and type variable definitions +_T = TypeVar('_T') +_U = TypeVar('_U') + +def take(n: int, iterable: Iterable[_T]) -> List[_T]: ... +def tabulate( + function: Callable[[int], _T], start: int = ... +) -> Iterator[_T]: ... +def tail(n: int, iterable: Iterable[_T]) -> Iterator[_T]: ... +def consume(iterator: Iterable[object], n: Optional[int] = ...) -> None: ... +@overload +def nth(iterable: Iterable[_T], n: int) -> Optional[_T]: ... +@overload +def nth(iterable: Iterable[_T], n: int, default: _U) -> Union[_T, _U]: ... +def all_equal(iterable: Iterable[object]) -> bool: ... +def quantify( + iterable: Iterable[_T], pred: Callable[[_T], bool] = ... +) -> int: ... +def pad_none(iterable: Iterable[_T]) -> Iterator[Optional[_T]]: ... +def padnone(iterable: Iterable[_T]) -> Iterator[Optional[_T]]: ... +def ncycles(iterable: Iterable[_T], n: int) -> Iterator[_T]: ... +def dotproduct(vec1: Iterable[object], vec2: Iterable[object]) -> object: ... +def flatten(listOfLists: Iterable[Iterable[_T]]) -> Iterator[_T]: ... +def repeatfunc( + func: Callable[..., _U], times: Optional[int] = ..., *args: Any +) -> Iterator[_U]: ... +def pairwise(iterable: Iterable[_T]) -> Iterator[Tuple[_T, _T]]: ... +@overload +def grouper( + iterable: Iterable[_T], n: int +) -> Iterator[Tuple[Optional[_T], ...]]: ... +@overload +def grouper( + iterable: Iterable[_T], n: int, fillvalue: _U +) -> Iterator[Tuple[Union[_T, _U], ...]]: ... +@overload +def grouper( # Deprecated interface + iterable: int, n: Iterable[_T] +) -> Iterator[Tuple[Optional[_T], ...]]: ... +@overload +def grouper( # Deprecated interface + iterable: int, n: Iterable[_T], fillvalue: _U +) -> Iterator[Tuple[Union[_T, _U], ...]]: ... +def roundrobin(*iterables: Iterable[_T]) -> Iterator[_T]: ... +def partition( + pred: Optional[Callable[[_T], object]], iterable: Iterable[_T] +) -> Tuple[Iterator[_T], Iterator[_T]]: ... +def powerset(iterable: Iterable[_T]) -> Iterator[Tuple[_T, ...]]: ... +def unique_everseen( + iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = ... +) -> Iterator[_T]: ... +def unique_justseen( + iterable: Iterable[_T], key: Optional[Callable[[_T], object]] = ... +) -> Iterator[_T]: ... +@overload +def iter_except( + func: Callable[[], _T], exception: Type[BaseException], first: None = ... +) -> Iterator[_T]: ... +@overload +def iter_except( + func: Callable[[], _T], + exception: Type[BaseException], + first: Callable[[], _U], +) -> Iterator[Union[_T, _U]]: ... +@overload +def first_true( + iterable: Iterable[_T], *, pred: Optional[Callable[[_T], object]] = ... +) -> Optional[_T]: ... +@overload +def first_true( + iterable: Iterable[_T], + default: _U, + pred: Optional[Callable[[_T], object]] = ..., +) -> Union[_T, _U]: ... +def random_product( + *args: Iterable[_T], repeat: int = ... +) -> Tuple[_T, ...]: ... +def random_permutation( + iterable: Iterable[_T], r: Optional[int] = ... +) -> Tuple[_T, ...]: ... +def random_combination(iterable: Iterable[_T], r: int) -> Tuple[_T, ...]: ... +def random_combination_with_replacement( + iterable: Iterable[_T], r: int +) -> Tuple[_T, ...]: ... +def nth_combination( + iterable: Iterable[_T], r: int, index: int +) -> Tuple[_T, ...]: ... +def prepend(value: _T, iterator: Iterable[_U]) -> Iterator[Union[_T, _U]]: ... +def convolve(signal: Iterable[_T], kernel: Iterable[_T]) -> Iterator[_T]: ... diff --git a/setuptools/_vendor/pyparsing.py b/setuptools/_vendor/pyparsing.py index cf75e1e5..4cae7883 100644 --- a/setuptools/_vendor/pyparsing.py +++ b/setuptools/_vendor/pyparsing.py @@ -1625,7 +1625,7 @@ class ParserElement(object): (see L{I{parseWithTabs}<parseWithTabs>})
- define your parse action using the full C{(s,loc,toks)} signature, and
reference the input string using the parse action's C{s} argument
- - explictly expand the tabs in your input string before calling
+ - explicitly expand the tabs in your input string before calling
C{parseString}
Example::
diff --git a/setuptools/_vendor/vendored.txt b/setuptools/_vendor/vendored.txt index b1190436..d9804741 100644 --- a/setuptools/_vendor/vendored.txt +++ b/setuptools/_vendor/vendored.txt @@ -1,3 +1,4 @@ packaging==20.4 pyparsing==2.2.1 ordered-set==3.1.1 +more_itertools==8.8.0 diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 0ce190b8..0f702848 100644 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -125,6 +125,56 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): os.chmod(target, unix_attributes) +def _resolve_tar_file_or_dir(tar_obj, tar_member_obj): + """Resolve any links and extract link targets as normal files.""" + while tar_member_obj is not None and ( + tar_member_obj.islnk() or tar_member_obj.issym()): + linkpath = tar_member_obj.linkname + if tar_member_obj.issym(): + base = posixpath.dirname(tar_member_obj.name) + linkpath = posixpath.join(base, linkpath) + linkpath = posixpath.normpath(linkpath) + tar_member_obj = tar_obj._getmember(linkpath) + + is_file_or_dir = ( + tar_member_obj is not None and + (tar_member_obj.isfile() or tar_member_obj.isdir()) + ) + if is_file_or_dir: + return tar_member_obj + + raise LookupError('Got unknown file type') + + +def _iter_open_tar(tar_obj, extract_dir, progress_filter): + """Emit member-destination pairs from a tar archive.""" + # don't do any chowning! + tar_obj.chown = lambda *args: None + + with contextlib.closing(tar_obj): + for member in tar_obj: + name = member.name + # don't extract absolute paths or ones with .. in them + if name.startswith('/') or '..' in name.split('/'): + continue + + prelim_dst = os.path.join(extract_dir, *name.split('/')) + + try: + member = _resolve_tar_file_or_dir(tar_obj, member) + except LookupError: + continue + + final_dst = progress_filter(name, prelim_dst) + if not final_dst: + continue + + if final_dst.endswith(os.sep): + final_dst = final_dst[:-1] + + yield member, final_dst + + def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir` @@ -138,38 +188,18 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): raise UnrecognizedFormat( "%s is not a compressed or uncompressed tar file" % (filename,) ) from e - with contextlib.closing(tarobj): - # don't do any chowning! - tarobj.chown = lambda *args: None - for member in tarobj: - name = member.name - # don't extract absolute paths or ones with .. in them - if not name.startswith('/') and '..' not in name.split('/'): - prelim_dst = os.path.join(extract_dir, *name.split('/')) - - # resolve any links and to extract the link targets as normal - # files - while member is not None and ( - member.islnk() or member.issym()): - linkpath = member.linkname - if member.issym(): - base = posixpath.dirname(member.name) - linkpath = posixpath.join(base, linkpath) - linkpath = posixpath.normpath(linkpath) - member = tarobj._getmember(linkpath) - - if member is not None and (member.isfile() or member.isdir()): - final_dst = progress_filter(name, prelim_dst) - if final_dst: - if final_dst.endswith(os.sep): - final_dst = final_dst[:-1] - try: - # XXX Ugh - tarobj._extract_member(member, final_dst) - except tarfile.ExtractError: - # chown/chmod/mkfifo/mknode/makedev failed - pass - return True + + for member, final_dst in _iter_open_tar( + tarobj, extract_dir, progress_filter, + ): + try: + # XXX Ugh + tarobj._extract_member(member, final_dst) + except tarfile.ExtractError: + # chown/chmod/mkfifo/mknode/makedev failed + pass + + return True extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index b9e8a2b3..9dfb2f24 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -101,7 +101,12 @@ def _file_with_extension(directory, extension): f for f in os.listdir(directory) if f.endswith(extension) ) - file, = matching + try: + file, = matching + except ValueError: + raise ValueError( + 'No distribution was found. Ensure that `setup.py` ' + 'is not empty and that it calls `setup()`.') return file diff --git a/setuptools/cli-arm64.exe b/setuptools/cli-arm64.exe Binary files differnew file mode 100644 index 00000000..7a87ce48 --- /dev/null +++ b/setuptools/cli-arm64.exe diff --git a/setuptools/command/__init__.py b/setuptools/command/__init__.py index 743f5588..b966dcea 100644 --- a/setuptools/command/__init__.py +++ b/setuptools/command/__init__.py @@ -1,15 +1,6 @@ -__all__ = [ - 'alias', 'bdist_egg', 'bdist_rpm', 'build_ext', 'build_py', 'develop', - 'easy_install', 'egg_info', 'install', 'install_lib', 'rotate', 'saveopts', - 'sdist', 'setopt', 'test', 'install_egg_info', 'install_scripts', - 'bdist_wininst', 'upload_docs', 'build_clib', 'dist_info', -] - from distutils.command.bdist import bdist import sys -from setuptools.command import install_scripts - if 'egg' not in bdist.format_commands: bdist.format_command['egg'] = ('bdist_egg', "Python .egg file") bdist.format_commands.append('egg') diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index a88efb45..e6b1609f 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -2,7 +2,6 @@ Build .egg distributions""" -from distutils.errors import DistutilsSetupError from distutils.dir_util import remove_tree, mkpath from distutils import log from types import CodeType @@ -11,12 +10,10 @@ import os import re import textwrap import marshal -import warnings from pkg_resources import get_build_platform, Distribution, ensure_directory -from pkg_resources import EntryPoint from setuptools.extension import Library -from setuptools import Command, SetuptoolsDeprecationWarning +from setuptools import Command from sysconfig import get_path, get_python_version @@ -153,7 +150,7 @@ class bdist_egg(Command): self.run_command(cmdname) return cmd - def run(self): + def run(self): # noqa: C901 # is too complex (14) # FIXME # Generate metadata first self.run_command("egg_info") # We run install_lib before install_data, because some data hacks @@ -268,49 +265,7 @@ class bdist_egg(Command): return analyze_egg(self.bdist_dir, self.stubs) def gen_header(self): - epm = EntryPoint.parse_map(self.distribution.entry_points or '') - ep = epm.get('setuptools.installation', {}).get('eggsecutable') - if ep is None: - return 'w' # not an eggsecutable, do it the usual way. - - warnings.warn( - "Eggsecutables are deprecated and will be removed in a future " - "version.", - SetuptoolsDeprecationWarning - ) - - if not ep.attrs or ep.extras: - raise DistutilsSetupError( - "eggsecutable entry point (%r) cannot have 'extras' " - "or refer to a module" % (ep,) - ) - - pyver = '{}.{}'.format(*sys.version_info) - pkg = ep.module_name - full = '.'.join(ep.attrs) - base = ep.attrs[0] - basename = os.path.basename(self.egg_output) - - header = ( - "#!/bin/sh\n" - 'if [ `basename $0` = "%(basename)s" ]\n' - 'then exec python%(pyver)s -c "' - "import sys, os; sys.path.insert(0, os.path.abspath('$0')); " - "from %(pkg)s import %(base)s; sys.exit(%(full)s())" - '" "$@"\n' - 'else\n' - ' echo $0 is not the correct name for this egg file.\n' - ' echo Please rename it back to %(basename)s and try again.\n' - ' exec false\n' - 'fi\n' - ) % locals() - - if not self.dry_run: - mkpath(os.path.dirname(self.egg_output), dry_run=self.dry_run) - f = open(self.egg_output, 'w') - f.write(header) - f.close() - return 'a' + return 'w' def copy_metadata_to(self, target_dir): "Copy metadata (egg info) to the target_dir" diff --git a/setuptools/command/bdist_rpm.py b/setuptools/command/bdist_rpm.py index 0eb1b9c2..98bf5dea 100644 --- a/setuptools/command/bdist_rpm.py +++ b/setuptools/command/bdist_rpm.py @@ -1,4 +1,7 @@ import distutils.command.bdist_rpm as orig +import warnings + +from setuptools import SetuptoolsDeprecationWarning class bdist_rpm(orig.bdist_rpm): @@ -11,6 +14,12 @@ class bdist_rpm(orig.bdist_rpm): """ def run(self): + warnings.warn( + "bdist_rpm is deprecated and will be removed in a future " + "version. Use bdist_wheel (wheel packages) instead.", + SetuptoolsDeprecationWarning, + ) + # ensure distro name is up-to-date self.run_command('egg_info') diff --git a/setuptools/command/bdist_wininst.py b/setuptools/command/bdist_wininst.py deleted file mode 100644 index ff4b6345..00000000 --- a/setuptools/command/bdist_wininst.py +++ /dev/null @@ -1,30 +0,0 @@ -import distutils.command.bdist_wininst as orig -import warnings - -from setuptools import SetuptoolsDeprecationWarning - - -class bdist_wininst(orig.bdist_wininst): - def reinitialize_command(self, command, reinit_subcommands=0): - """ - Supplement reinitialize_command to work around - http://bugs.python.org/issue20819 - """ - cmd = self.distribution.reinitialize_command( - command, reinit_subcommands) - if command in ('install', 'install_lib'): - cmd.install_lib = None - return cmd - - def run(self): - warnings.warn( - "bdist_wininst is deprecated and will be removed in a future " - "version. Use bdist_wheel (wheel packages) instead.", - SetuptoolsDeprecationWarning - ) - - self._is_running = True - try: - orig.bdist_wininst.run(self) - finally: - self._is_running = False diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 03a72b4f..c59eff8b 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -104,14 +104,20 @@ class build_ext(_build_ext): self.write_stub(package_dir or os.curdir, ext, True) def get_ext_filename(self, fullname): - filename = _build_ext.get_ext_filename(self, fullname) + so_ext = os.getenv('SETUPTOOLS_EXT_SUFFIX') + if so_ext: + filename = os.path.join(*fullname.split('.')) + so_ext + else: + filename = _build_ext.get_ext_filename(self, fullname) + so_ext = get_config_var('EXT_SUFFIX') + if fullname in self.ext_map: ext = self.ext_map[fullname] use_abi3 = getattr(ext, 'py_limited_api') and get_abi3_suffix() if use_abi3: - so_ext = get_config_var('EXT_SUFFIX') filename = filename[:-len(so_ext)] - filename = filename + get_abi3_suffix() + so_ext = get_abi3_suffix() + filename = filename + so_ext if isinstance(ext, Library): fn, ext = os.path.splitext(filename) return self.shlib_compiler.library_filename(fn, libtype) diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index b30aa129..6a615433 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -8,21 +8,14 @@ import io import distutils.errors import itertools import stat - -try: - from setuptools.lib2to3_ex import Mixin2to3 -except Exception: - - class Mixin2to3: - def run_2to3(self, files, doctests=True): - "do nothing" +from setuptools.extern.more_itertools import unique_everseen def make_writable(target): os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE) -class build_py(orig.build_py, Mixin2to3): +class build_py(orig.build_py): """Enhanced 'build_py' command that includes data files with packages The data files are specified via a 'package_data' argument to 'setup()'. @@ -35,12 +28,10 @@ class build_py(orig.build_py, Mixin2to3): def finalize_options(self): orig.build_py.finalize_options(self) self.package_data = self.distribution.package_data - self.exclude_package_data = (self.distribution.exclude_package_data or - {}) + self.exclude_package_data = self.distribution.exclude_package_data or {} if 'data_files' in self.__dict__: del self.__dict__['data_files'] self.__updated_files = [] - self.__doctests_2to3 = [] def run(self): """Build modules, packages, and copy data files to build directory""" @@ -54,10 +45,6 @@ class build_py(orig.build_py, Mixin2to3): self.build_packages() self.build_package_data() - self.run_2to3(self.__updated_files, False) - self.run_2to3(self.__updated_files, True) - self.run_2to3(self.__doctests_2to3, True) - # Only compile actual .py files, using our base class' idea of what our # output files are. self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0)) @@ -70,8 +57,7 @@ class build_py(orig.build_py, Mixin2to3): return orig.build_py.__getattr__(self, attr) def build_module(self, module, module_file, package): - outfile, copied = orig.build_py.build_module(self, module, module_file, - package) + outfile, copied = orig.build_py.build_module(self, module, module_file, package) if copied: self.__updated_files.append(outfile) return outfile, copied @@ -122,9 +108,6 @@ class build_py(orig.build_py, Mixin2to3): outf, copied = self.copy_file(srcfile, target) make_writable(target) srcfile = os.path.abspath(srcfile) - if (copied and - srcfile in self.distribution.convert_2to3_doctests): - self.__doctests_2to3.append(outf) def analyze_manifest(self): self.manifest_files = mf = {} @@ -201,20 +184,13 @@ class build_py(orig.build_py, Mixin2to3): package, src_dir, ) - match_groups = ( - fnmatch.filter(files, pattern) - for pattern in patterns - ) + match_groups = (fnmatch.filter(files, pattern) for pattern in patterns) # flatten the groups of matches into an iterable of matches matches = itertools.chain.from_iterable(match_groups) bad = set(matches) - keepers = ( - fn - for fn in files - if fn not in bad - ) + keepers = (fn for fn in files if fn not in bad) # ditch dupes - return list(_unique_everseen(keepers)) + return list(unique_everseen(keepers)) @staticmethod def _get_platform_patterns(spec, package, src_dir): @@ -235,36 +211,22 @@ class build_py(orig.build_py, Mixin2to3): ) -# from Python docs -def _unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen = set() - seen_add = seen.add - if key is None: - for element in itertools.filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element - - def assert_relative(path): if not os.path.isabs(path): return path from distutils.errors import DistutilsSetupError - msg = textwrap.dedent(""" + msg = ( + textwrap.dedent( + """ Error: setup script specifies an absolute path: %s setup() arguments must *always* be /-separated paths relative to the setup.py directory, *never* absolute paths. - """).lstrip() % path + """ + ).lstrip() + % path + ) raise DistutilsSetupError(msg) diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py index faf8c988..24fb0a7c 100644 --- a/setuptools/command/develop.py +++ b/setuptools/command/develop.py @@ -63,7 +63,8 @@ class develop(namespaces.DevelopInstaller, easy_install): target = pkg_resources.normalize_path(self.egg_base) egg_path = pkg_resources.normalize_path( - os.path.join(self.install_dir, self.egg_path)) + os.path.join(self.install_dir, self.egg_path) + ) if egg_path != target: raise DistutilsOptionError( "--egg-path must be a relative path from the install" @@ -74,7 +75,7 @@ class develop(namespaces.DevelopInstaller, easy_install): self.dist = pkg_resources.Distribution( target, pkg_resources.PathMetadata(target, os.path.abspath(ei.egg_info)), - project_name=ei.egg_name + project_name=ei.egg_name, ) self.setup_path = self._resolve_setup_path( @@ -99,41 +100,18 @@ class develop(namespaces.DevelopInstaller, easy_install): if resolved != pkg_resources.normalize_path(os.curdir): raise DistutilsOptionError( "Can't get a consistent path to setup script from" - " installation directory", resolved, - pkg_resources.normalize_path(os.curdir)) + " installation directory", + resolved, + pkg_resources.normalize_path(os.curdir), + ) return path_to_setup def install_for_development(self): - if getattr(self.distribution, 'use_2to3', False): - # If we run 2to3 we can not do this inplace: - - # Ensure metadata is up-to-date - self.reinitialize_command('build_py', inplace=0) - self.run_command('build_py') - bpy_cmd = self.get_finalized_command("build_py") - build_path = pkg_resources.normalize_path(bpy_cmd.build_lib) - - # Build extensions - self.reinitialize_command('egg_info', egg_base=build_path) - self.run_command('egg_info') - - self.reinitialize_command('build_ext', inplace=0) - self.run_command('build_ext') - - # Fixup egg-link and easy-install.pth - ei_cmd = self.get_finalized_command("egg_info") - self.egg_path = build_path - self.dist.location = build_path - # XXX - self.dist._provider = pkg_resources.PathMetadata( - build_path, ei_cmd.egg_info) - else: - # Without 2to3 inplace works fine: - self.run_command('egg_info') + self.run_command('egg_info') - # Build extensions in-place - self.reinitialize_command('build_ext', inplace=1) - self.run_command('build_ext') + # Build extensions in-place + self.reinitialize_command('build_ext', inplace=1) + self.run_command('build_ext') if setuptools.bootstrap_install_from: self.easy_install(setuptools.bootstrap_install_from) @@ -156,8 +134,7 @@ class develop(namespaces.DevelopInstaller, easy_install): egg_link_file = open(self.egg_link) contents = [line.rstrip() for line in egg_link_file] egg_link_file.close() - if contents not in ([self.egg_path], - [self.egg_path, self.setup_path]): + if contents not in ([self.egg_path], [self.egg_path, self.setup_path]): log.warn("Link points to %s: uninstall aborted", contents) return if not self.dry_run: diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 9ec83b7d..b88c3e9a 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -6,7 +6,7 @@ A tool for doing automatic download/extract/build of distutils-based Python packages. For detailed documentation, see the accompanying EasyInstall.txt file, or visit the `EasyInstall home page`__. -__ https://setuptools.readthedocs.io/en/latest/easy_install.html +__ https://setuptools.readthedocs.io/en/latest/deprecated/easy_install.html """ @@ -67,7 +67,7 @@ warnings.filterwarnings("default", category=pkg_resources.PEP440Warning) __all__ = [ 'samefile', 'easy_install', 'PthDistributions', 'extract_wininst_cfg', - 'main', 'get_exe_prefixes', + 'get_exe_prefixes', ] @@ -226,7 +226,7 @@ class easy_install(Command): print(tmpl.format(**locals())) raise SystemExit() - def finalize_options(self): + def finalize_options(self): # noqa: C901 # is too complex (25) # FIXME self.version and self._render_version() py_version = sys.version.split()[0] @@ -437,7 +437,7 @@ class easy_install(Command): def warn_deprecated_options(self): pass - def check_site_dir(self): + def check_site_dir(self): # noqa: C901 # is too complex (12) # FIXME """Verify that self.install_dir is .pth-capable dir, if needed""" instdir = normalize_path(self.install_dir) @@ -513,7 +513,7 @@ class easy_install(Command): For information on other options, you may wish to consult the documentation at: - https://setuptools.readthedocs.io/en/latest/easy_install.html + https://setuptools.readthedocs.io/en/latest/deprecated/easy_install.html Please make the appropriate changes for your system and try again. """).lstrip() # noqa @@ -713,7 +713,10 @@ class easy_install(Command): if getattr(self, attrname) is None: setattr(self, attrname, scheme[key]) - def process_distribution(self, requirement, dist, deps=True, *info): + # FIXME: 'easy_install.process_distribution' is too complex (12) + def process_distribution( # noqa: C901 + self, requirement, dist, deps=True, *info, + ): self.update_pth(dist) self.package_index.add(dist) if dist in self.local_index[dist.key]: @@ -837,12 +840,19 @@ class easy_install(Command): def install_eggs(self, spec, dist_filename, tmpdir): # .egg dirs or files are already built, so just return them - if dist_filename.lower().endswith('.egg'): - return [self.install_egg(dist_filename, tmpdir)] - elif dist_filename.lower().endswith('.exe'): - return [self.install_exe(dist_filename, tmpdir)] - elif dist_filename.lower().endswith('.whl'): - return [self.install_wheel(dist_filename, tmpdir)] + installer_map = { + '.egg': self.install_egg, + '.exe': self.install_exe, + '.whl': self.install_wheel, + } + try: + install_dist = installer_map[ + dist_filename.lower()[-4:] + ] + except KeyError: + pass + else: + return [install_dist(dist_filename, tmpdir)] # Anything else, try to extract and build setup_base = tmpdir @@ -887,7 +897,8 @@ class easy_install(Command): metadata = EggMetadata(zipimport.zipimporter(egg_path)) return Distribution.from_filename(egg_path, metadata=metadata) - def install_egg(self, egg_path, tmpdir): + # FIXME: 'easy_install.install_egg' is too complex (11) + def install_egg(self, egg_path, tmpdir): # noqa: C901 destination = os.path.join( self.install_dir, os.path.basename(egg_path), @@ -986,7 +997,8 @@ class easy_install(Command): # install the .egg return self.install_egg(egg_path, tmpdir) - def exe_to_egg(self, dist_filename, egg_tmp): + # FIXME: 'easy_install.exe_to_egg' is too complex (12) + def exe_to_egg(self, dist_filename, egg_tmp): # noqa: C901 """Extract a bdist_wininst to the directories an egg would use""" # Check for .pth file and set up prefix translations prefixes = get_exe_prefixes(dist_filename) @@ -1178,22 +1190,24 @@ class easy_install(Command): for key, val in ei_opts.items(): if key not in fetch_directives: continue - fetch_options[key.replace('_', '-')] = val[1] + fetch_options[key] = val[1] # create a settings dictionary suitable for `edit_config` settings = dict(easy_install=fetch_options) cfg_filename = os.path.join(base, 'setup.cfg') setopt.edit_config(cfg_filename, settings) - def update_pth(self, dist): + def update_pth(self, dist): # noqa: C901 # is too complex (11) # FIXME if self.pth_file is None: return for d in self.pth_file[dist.key]: # drop old entries - if self.multi_version or d.location != dist.location: - log.info("Removing %s from easy-install.pth file", d) - self.pth_file.remove(d) - if d.location in self.shadow_path: - self.shadow_path.remove(d.location) + if not self.multi_version and d.location == dist.location: + continue + + log.info("Removing %s from easy-install.pth file", d) + self.pth_file.remove(d) + if d.location in self.shadow_path: + self.shadow_path.remove(d.location) if not self.multi_version: if dist.location in self.pth_file.paths: @@ -1207,19 +1221,21 @@ class easy_install(Command): if dist.location not in self.shadow_path: self.shadow_path.append(dist.location) - if not self.dry_run: + if self.dry_run: + return - self.pth_file.save() + self.pth_file.save() - if dist.key == 'setuptools': - # Ensure that setuptools itself never becomes unavailable! - # XXX should this check for latest version? - filename = os.path.join(self.install_dir, 'setuptools.pth') - if os.path.islink(filename): - os.unlink(filename) - f = open(filename, 'wt') - f.write(self.pth_file.make_relative(dist.location) + '\n') - f.close() + if dist.key != 'setuptools': + return + + # Ensure that setuptools itself never becomes unavailable! + # XXX should this check for latest version? + filename = os.path.join(self.install_dir, 'setuptools.pth') + if os.path.islink(filename): + os.unlink(filename) + with open(filename, 'wt') as f: + f.write(self.pth_file.make_relative(dist.location) + '\n') def unpack_progress(self, src, dst): # Progress filter for unpacking @@ -1290,7 +1306,7 @@ class easy_install(Command): * You can set up the installation directory to support ".pth" files by using one of the approaches described here: - https://setuptools.readthedocs.io/en/latest/easy_install.html#custom-installation-locations + https://setuptools.readthedocs.io/en/latest/deprecated/easy_install.html#custom-installation-locations Please make the appropriate changes for your system and try again. @@ -1360,58 +1376,63 @@ def get_site_dirs(): if sys.exec_prefix != sys.prefix: prefixes.append(sys.exec_prefix) for prefix in prefixes: - if prefix: - if sys.platform in ('os2emx', 'riscos'): - sitedirs.append(os.path.join(prefix, "Lib", "site-packages")) - elif os.sep == '/': - sitedirs.extend([ - os.path.join( - prefix, - "lib", - "python{}.{}".format(*sys.version_info), - "site-packages", - ), - os.path.join(prefix, "lib", "site-python"), - ]) - else: - sitedirs.extend([ + if not prefix: + continue + + if sys.platform in ('os2emx', 'riscos'): + sitedirs.append(os.path.join(prefix, "Lib", "site-packages")) + elif os.sep == '/': + sitedirs.extend([ + os.path.join( prefix, - os.path.join(prefix, "lib", "site-packages"), - ]) - if sys.platform == 'darwin': - # for framework builds *only* we add the standard Apple - # locations. Currently only per-user, but /Library and - # /Network/Library could be added too - if 'Python.framework' in prefix: - home = os.environ.get('HOME') - if home: - home_sp = os.path.join( - home, - 'Library', - 'Python', - '{}.{}'.format(*sys.version_info), - 'site-packages', - ) - sitedirs.append(home_sp) + "lib", + "python{}.{}".format(*sys.version_info), + "site-packages", + ), + os.path.join(prefix, "lib", "site-python"), + ]) + else: + sitedirs.extend([ + prefix, + os.path.join(prefix, "lib", "site-packages"), + ]) + if sys.platform != 'darwin': + continue + + # for framework builds *only* we add the standard Apple + # locations. Currently only per-user, but /Library and + # /Network/Library could be added too + if 'Python.framework' not in prefix: + continue + + home = os.environ.get('HOME') + if not home: + continue + + home_sp = os.path.join( + home, + 'Library', + 'Python', + '{}.{}'.format(*sys.version_info), + 'site-packages', + ) + sitedirs.append(home_sp) lib_paths = get_path('purelib'), get_path('platlib') - for site_lib in lib_paths: - if site_lib not in sitedirs: - sitedirs.append(site_lib) + + sitedirs.extend(s for s in lib_paths if s not in sitedirs) if site.ENABLE_USER_SITE: sitedirs.append(site.USER_SITE) - try: + with contextlib.suppress(AttributeError): sitedirs.extend(site.getsitepackages()) - except AttributeError: - pass sitedirs = list(map(normalize_path, sitedirs)) return sitedirs -def expand_paths(inputs): +def expand_paths(inputs): # noqa: C901 # is too complex (11) # FIXME """Yield sys.path directories that might contain "old-style" packages""" seen = {} @@ -1443,13 +1464,18 @@ def expand_paths(inputs): # Yield existing non-dupe, non-import directory lines from it for line in lines: - if not line.startswith("import"): - line = normalize_path(line.rstrip()) - if line not in seen: - seen[line] = 1 - if not os.path.isdir(line): - continue - yield line, os.listdir(line) + if line.startswith("import"): + continue + + line = normalize_path(line.rstrip()) + if line in seen: + continue + + seen[line] = 1 + if not os.path.isdir(line): + continue + + yield line, os.listdir(line) def extract_wininst_cfg(dist_filename): @@ -1482,7 +1508,7 @@ def extract_wininst_cfg(dist_filename): # Now the config is in bytes, but for RawConfigParser, it should # be text, so decode it. config = config.decode(sys.getfilesystemencoding()) - cfg.readfp(io.StringIO(config)) + cfg.read_file(io.StringIO(config)) except configparser.Error: return None if not cfg.has_section('metadata') or not cfg.has_section('Setup'): @@ -2167,7 +2193,7 @@ class WindowsScriptWriter(ScriptWriter): @classmethod def _adjust_header(cls, type_, orig_header): """ - Make sure 'pythonw' is used for gui and and 'python' is used for + Make sure 'pythonw' is used for gui and 'python' is used for console (regardless of what sys.executable is). """ pattern = 'pythonw.exe' @@ -2237,7 +2263,10 @@ def get_win_launcher(type): """ launcher_fn = '%s.exe' % type if is_64bit(): - launcher_fn = launcher_fn.replace(".", "-64.") + if get_platform() == "win-arm64": + launcher_fn = launcher_fn.replace(".", "-arm64.") + else: + launcher_fn = launcher_fn.replace(".", "-64.") else: launcher_fn = launcher_fn.replace(".", "-32.") return resource_string('setuptools', launcher_fn) @@ -2258,60 +2287,6 @@ def current_umask(): return tmp -def bootstrap(): - # This function is called when setuptools*.egg is run using /bin/sh - import setuptools - - argv0 = os.path.dirname(setuptools.__path__[0]) - sys.argv[0] = argv0 - sys.argv.append(argv0) - main() - - -def main(argv=None, **kw): - from setuptools import setup - from setuptools.dist import Distribution - - class DistributionWithoutHelpCommands(Distribution): - common_usage = "" - - def _show_help(self, *args, **kw): - with _patch_usage(): - Distribution._show_help(self, *args, **kw) - - if argv is None: - argv = sys.argv[1:] - - with _patch_usage(): - setup( - script_args=['-q', 'easy_install', '-v'] + argv, - script_name=sys.argv[0] or 'easy_install', - distclass=DistributionWithoutHelpCommands, - **kw - ) - - -@contextlib.contextmanager -def _patch_usage(): - import distutils.core - USAGE = textwrap.dedent(""" - usage: %(script)s [options] requirement_or_url ... - or: %(script)s --help - """).lstrip() - - def gen_usage(script_name): - return USAGE % dict( - script=os.path.basename(script_name), - ) - - saved = distutils.core.gen_usage - distutils.core.gen_usage = gen_usage - try: - yield - finally: - distutils.core.gen_usage = saved - - class EasyInstallDeprecationWarning(SetuptoolsDeprecationWarning): """ Warning for EasyInstall deprecations, bypassing suppression. diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 97e10d99..57bc7982 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -8,6 +8,7 @@ from distutils.util import convert_path from distutils import log import distutils.errors import distutils.filelist +import functools import os import re import sys @@ -31,7 +32,7 @@ from setuptools.extern import packaging from setuptools import SetuptoolsDeprecationWarning -def translate_pattern(glob): +def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME """ Translate a file path glob like '*.txt' in to a regular expression. This differs from fnmatch.translate which allows wildcards to match @@ -332,70 +333,74 @@ class FileList(_FileList): # patterns, (dir and patterns), or (dir_pattern). (action, patterns, dir, dir_pattern) = self._parse_template_line(line) + action_map = { + 'include': self.include, + 'exclude': self.exclude, + 'global-include': self.global_include, + 'global-exclude': self.global_exclude, + 'recursive-include': functools.partial( + self.recursive_include, dir, + ), + 'recursive-exclude': functools.partial( + self.recursive_exclude, dir, + ), + 'graft': self.graft, + 'prune': self.prune, + } + log_map = { + 'include': "warning: no files found matching '%s'", + 'exclude': ( + "warning: no previously-included files found " + "matching '%s'" + ), + 'global-include': ( + "warning: no files found matching '%s' " + "anywhere in distribution" + ), + 'global-exclude': ( + "warning: no previously-included files matching " + "'%s' found anywhere in distribution" + ), + 'recursive-include': ( + "warning: no files found matching '%s' " + "under directory '%s'" + ), + 'recursive-exclude': ( + "warning: no previously-included files matching " + "'%s' found under directory '%s'" + ), + 'graft': "warning: no directories found matching '%s'", + 'prune': "no previously-included directories found matching '%s'", + } + + try: + process_action = action_map[action] + except KeyError: + raise DistutilsInternalError( + "this cannot happen: invalid action '{action!s}'". + format(action=action), + ) + # OK, now we know that the action is valid and we have the # right number of words on the line for that action -- so we # can proceed with minimal error-checking. - if action == 'include': - self.debug_print("include " + ' '.join(patterns)) - for pattern in patterns: - if not self.include(pattern): - log.warn("warning: no files found matching '%s'", pattern) - - elif action == 'exclude': - self.debug_print("exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.exclude(pattern): - log.warn(("warning: no previously-included files " - "found matching '%s'"), pattern) - - elif action == 'global-include': - self.debug_print("global-include " + ' '.join(patterns)) - for pattern in patterns: - if not self.global_include(pattern): - log.warn(("warning: no files found matching '%s' " - "anywhere in distribution"), pattern) - - elif action == 'global-exclude': - self.debug_print("global-exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.global_exclude(pattern): - log.warn(("warning: no previously-included files matching " - "'%s' found anywhere in distribution"), - pattern) - - elif action == 'recursive-include': - self.debug_print("recursive-include %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.recursive_include(dir, pattern): - log.warn(("warning: no files found matching '%s' " - "under directory '%s'"), - pattern, dir) - - elif action == 'recursive-exclude': - self.debug_print("recursive-exclude %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.recursive_exclude(dir, pattern): - log.warn(("warning: no previously-included files matching " - "'%s' found under directory '%s'"), - pattern, dir) - - elif action == 'graft': - self.debug_print("graft " + dir_pattern) - if not self.graft(dir_pattern): - log.warn("warning: no directories found matching '%s'", - dir_pattern) - - elif action == 'prune': - self.debug_print("prune " + dir_pattern) - if not self.prune(dir_pattern): - log.warn(("no previously-included directories found " - "matching '%s'"), dir_pattern) - - else: - raise DistutilsInternalError( - "this cannot happen: invalid action '%s'" % action) + + action_is_recursive = action.startswith('recursive-') + if action in {'graft', 'prune'}: + patterns = [dir_pattern] + extra_log_args = (dir, ) if action_is_recursive else () + log_tmpl = log_map[action] + + self.debug_print( + ' '.join( + [action] + + ([dir] if action_is_recursive else []) + + patterns, + ) + ) + for pattern in patterns: + if not process_action(pattern): + log.warn(log_tmpl, pattern, *extra_log_args) def _remove_files(self, predicate): """ @@ -536,6 +541,7 @@ class manifest_maker(sdist): self.add_defaults() if os.path.exists(self.template): self.read_template() + self.add_license_files() self.prune_file_list() self.filelist.sort() self.filelist.remove_duplicates() @@ -570,7 +576,6 @@ class manifest_maker(sdist): def add_defaults(self): sdist.add_defaults(self) - self.check_license() self.filelist.append(self.template) self.filelist.append(self.manifest) rcfiles = list(walk_revctrl()) @@ -587,6 +592,13 @@ class manifest_maker(sdist): ei_cmd = self.get_finalized_command('egg_info') self.filelist.graft(ei_cmd.egg_info) + def add_license_files(self): + license_files = self.distribution.metadata.license_files or [] + for lf in license_files: + log.info("adding license file '%s'", lf) + pass + self.filelist.extend(license_files) + def prune_file_list(self): build = self.get_finalized_command('build') base_dir = self.distribution.get_fullname() diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py index 8c9a15e2..9cd8eb06 100644 --- a/setuptools/command/install_scripts.py +++ b/setuptools/command/install_scripts.py @@ -1,5 +1,6 @@ from distutils import log import distutils.command.install_scripts as orig +from distutils.errors import DistutilsModuleError import os import sys @@ -35,7 +36,7 @@ class install_scripts(orig.install_scripts): try: bw_cmd = self.get_finalized_command("bdist_wininst") is_wininst = getattr(bw_cmd, '_is_running', False) - except ImportError: + except (ImportError, DistutilsModuleError): is_wininst = False writer = ei.ScriptWriter if is_wininst: diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 887b7efa..e8062f2e 100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -5,8 +5,6 @@ import sys import io import contextlib -from setuptools.extern import ordered_set - from .py36compat import sdist_add_defaults import pkg_resources @@ -33,6 +31,10 @@ class sdist(sdist_add_defaults, orig.sdist): ('dist-dir=', 'd', "directory to put the source distribution archive(s) in " "[default: dist]"), + ('owner=', 'u', + "Owner name used when creating a tar file [default: current user]"), + ('group=', 'g', + "Group name used when creating a tar file [default: current group]"), ] negative_opt = {} @@ -189,34 +191,3 @@ class sdist(sdist_add_defaults, orig.sdist): continue self.filelist.append(line) manifest.close() - - def check_license(self): - """Checks if license_file' or 'license_files' is configured and adds any - valid paths to 'self.filelist'. - """ - - files = ordered_set.OrderedSet() - - opts = self.distribution.get_option_dict('metadata') - - # ignore the source of the value - _, license_file = opts.get('license_file', (None, None)) - - if license_file is None: - log.debug("'license_file' option was not specified") - else: - files.add(license_file) - - try: - files.update(self.distribution.metadata.license_files) - except TypeError: - log.warn("warning: 'license_files' option is malformed") - - for f in files: - if not os.path.exists(f): - log.warn( - "warning: Failed to find the configured license file '%s'", - f) - files.remove(f) - - self.filelist.extend(files) diff --git a/setuptools/command/setopt.py b/setuptools/command/setopt.py index e18057c8..6358c045 100644 --- a/setuptools/command/setopt.py +++ b/setuptools/command/setopt.py @@ -39,6 +39,7 @@ def edit_config(filename, settings, dry_run=False): """ log.debug("Reading configuration from %s", filename) opts = configparser.RawConfigParser() + opts.optionxform = lambda x: x opts.read([filename]) for section, options in settings.items(): if options is None: diff --git a/setuptools/command/test.py b/setuptools/command/test.py index cf71ad01..4a389e4d 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -8,15 +8,21 @@ from distutils.errors import DistutilsError, DistutilsOptionError from distutils import log from unittest import TestLoader -from pkg_resources import (resource_listdir, resource_exists, normalize_path, - working_set, _namespace_packages, evaluate_marker, - add_activation_listener, require, EntryPoint) +from pkg_resources import ( + resource_listdir, + resource_exists, + normalize_path, + working_set, + evaluate_marker, + add_activation_listener, + require, + EntryPoint, +) from setuptools import Command -from .build_py import _unique_everseen +from setuptools.extern.more_itertools import unique_everseen class ScanningLoader(TestLoader): - def __init__(self): TestLoader.__init__(self) self._visited = set() @@ -73,8 +79,11 @@ class test(Command): user_options = [ ('test-module=', 'm', "Run 'test_suite' in specified module"), - ('test-suite=', 's', - "Run single test, case or suite (e.g. 'module.test_suite')"), + ( + 'test-suite=', + 's', + "Run single test, case or suite (e.g. 'module.test_suite')", + ), ('test-runner=', 'r', "Test runner to use"), ] @@ -124,30 +133,11 @@ class test(Command): @contextlib.contextmanager def project_on_sys_path(self, include_dists=[]): - with_2to3 = getattr(self.distribution, 'use_2to3', False) - - if with_2to3: - # If we run 2to3 we can not do this inplace: + self.run_command('egg_info') - # Ensure metadata is up-to-date - self.reinitialize_command('build_py', inplace=0) - self.run_command('build_py') - bpy_cmd = self.get_finalized_command("build_py") - build_path = normalize_path(bpy_cmd.build_lib) - - # Build extensions - self.reinitialize_command('egg_info', egg_base=build_path) - self.run_command('egg_info') - - self.reinitialize_command('build_ext', inplace=0) - self.run_command('build_ext') - else: - # Without 2to3 inplace works fine: - self.run_command('egg_info') - - # Build extensions in-place - self.reinitialize_command('build_ext', inplace=1) - self.run_command('build_ext') + # Build extensions in-place + self.reinitialize_command('build_ext', inplace=1) + self.run_command('build_ext') ei_cmd = self.get_finalized_command("egg_info") @@ -182,7 +172,7 @@ class test(Command): orig_pythonpath = os.environ.get('PYTHONPATH', nothing) current_pythonpath = os.environ.get('PYTHONPATH', '') try: - prefix = os.pathsep.join(_unique_everseen(paths)) + prefix = os.pathsep.join(unique_everseen(paths)) to_join = filter(None, [prefix, current_pythonpath]) new_path = os.pathsep.join(to_join) if new_path: @@ -203,7 +193,8 @@ class test(Command): ir_d = dist.fetch_build_eggs(dist.install_requires) tr_d = dist.fetch_build_eggs(dist.tests_require or []) er_d = dist.fetch_build_eggs( - v for k, v in dist.extras_require.items() + v + for k, v in dist.extras_require.items() if k.startswith(':') and evaluate_marker(k[1:]) ) return itertools.chain(ir_d, tr_d, er_d) @@ -232,23 +223,10 @@ class test(Command): self.run_tests() def run_tests(self): - # Purge modules under test from sys.modules. The test loader will - # re-import them from the build location. Required when 2to3 is used - # with namespace packages. - if getattr(self.distribution, 'use_2to3', False): - module = self.test_suite.split('.')[0] - if module in _namespace_packages: - del_modules = [] - if module in sys.modules: - del_modules.append(module) - module += '.' - for name in sys.modules: - if name.startswith(module): - del_modules.append(name) - list(map(sys.modules.__delitem__, del_modules)) - test = unittest.main( - None, None, self._argv, + None, + None, + self._argv, testLoader=self._resolve_as_ep(self.test_loader), testRunner=self._resolve_as_ep(self.test_runner), exit=False, diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 2559458a..845bff44 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -2,7 +2,7 @@ """upload_docs Implements a Distutils 'upload_docs' subcommand (upload documentation to -PyPI's pythonhosted.org). +sites other than PyPi such as devpi). """ from base64 import standard_b64encode @@ -31,7 +31,7 @@ class upload_docs(upload): # supported by Warehouse (and won't be). DEFAULT_REPOSITORY = 'https://pypi.python.org/pypi/' - description = 'Upload documentation to PyPI' + description = 'Upload documentation to sites other than PyPi such as devpi' user_options = [ ('repository=', 'r', @@ -59,7 +59,7 @@ class upload_docs(upload): if self.upload_dir is None: if self.has_sphinx(): build_sphinx = self.get_finalized_command('build_sphinx') - self.target_dir = build_sphinx.builder_target_dir + self.target_dir = dict(build_sphinx.builder_target_dirs)['html'] else: build = self.get_finalized_command('build') self.target_dir = os.path.join(build.build_base, 'docs') @@ -67,7 +67,7 @@ class upload_docs(upload): self.ensure_dirname('upload_dir') self.target_dir = self.upload_dir if 'pypi.python.org' in self.repository: - log.warn("Upload_docs command is deprecated. Use RTD instead.") + log.warn("Upload_docs command is deprecated for PyPi. Use RTD instead.") self.announce('Using upload directory %s' % self.target_dir) def create_zipfile(self, filename): diff --git a/setuptools/config.py b/setuptools/config.py index af3a3bcb..e3e44c25 100644 --- a/setuptools/config.py +++ b/setuptools/config.py @@ -9,6 +9,7 @@ import importlib from collections import defaultdict from functools import partial from functools import wraps +from glob import iglob import contextlib from distutils.errors import DistutilsOptionError, DistutilsFileError @@ -20,6 +21,7 @@ class StaticModule: """ Attempt to load the module by the name """ + def __init__(self, name): spec = importlib.util.find_spec(name) with open(spec.origin) as strm: @@ -55,8 +57,7 @@ def patch_path(path): sys.path.remove(path) -def read_configuration( - filepath, find_others=False, ignore_option_errors=False): +def read_configuration(filepath, find_others=False, ignore_option_errors=False): """Read given configuration file and returns options from it as a dict. :param str|unicode filepath: Path to configuration file @@ -77,8 +78,7 @@ def read_configuration( filepath = os.path.abspath(filepath) if not os.path.isfile(filepath): - raise DistutilsFileError( - 'Configuration file %s does not exist.' % filepath) + raise DistutilsFileError('Configuration file %s does not exist.' % filepath) current_directory = os.getcwd() os.chdir(os.path.dirname(filepath)) @@ -93,8 +93,8 @@ def read_configuration( _Distribution.parse_config_files(dist, filenames=filenames) handlers = parse_configuration( - dist, dist.command_options, - ignore_option_errors=ignore_option_errors) + dist, dist.command_options, ignore_option_errors=ignore_option_errors + ) finally: os.chdir(current_directory) @@ -132,8 +132,7 @@ def configuration_to_dict(handlers): return config_dict -def parse_configuration( - distribution, command_options, ignore_option_errors=False): +def parse_configuration(distribution, command_options, ignore_option_errors=False): """Performs additional parsing of configuration options for a distribution. @@ -147,13 +146,15 @@ def parse_configuration( If False exceptions are propagated as expected. :rtype: list """ - options = ConfigOptionsHandler( - distribution, command_options, ignore_option_errors) + options = ConfigOptionsHandler(distribution, command_options, ignore_option_errors) options.parse() meta = ConfigMetadataHandler( - distribution.metadata, command_options, ignore_option_errors, - distribution.package_dir) + distribution.metadata, + command_options, + ignore_option_errors, + distribution.package_dir, + ) meta.parse() return meta, options @@ -195,7 +196,8 @@ class ConfigHandler: def parsers(self): """Metadata item name to parser function mapping.""" raise NotImplementedError( - '%s must provide .parsers property' % self.__class__.__name__) + '%s must provide .parsers property' % self.__class__.__name__ + ) def __setitem__(self, option_name, value): unknown = tuple() @@ -256,6 +258,34 @@ class ConfigHandler: return [chunk.strip() for chunk in value if chunk.strip()] @classmethod + def _parse_list_glob(cls, value, separator=','): + """Equivalent to _parse_list() but expands any glob patterns using glob(). + + However, unlike with glob() calls, the results remain relative paths. + + :param value: + :param separator: List items separator character. + :rtype: list + """ + glob_characters = ('*', '?', '[', ']', '{', '}') + values = cls._parse_list(value, separator=separator) + expanded_values = [] + for value in values: + + # Has globby characters? + if any(char in value for char in glob_characters): + # then expand the glob pattern while keeping paths *relative*: + expanded_values.extend(sorted( + os.path.relpath(path, os.getcwd()) + for path in iglob(os.path.abspath(value)))) + + else: + # take the value as-is: + expanded_values.append(value) + + return expanded_values + + @classmethod def _parse_dict(cls, value): """Represents value as a dict. @@ -268,7 +298,8 @@ class ConfigHandler: key, sep, val = line.partition(separator) if sep != separator: raise DistutilsOptionError( - 'Unable to parse option value to dict: %s' % value) + 'Unable to parse option value to dict: %s' % value + ) result[key.strip()] = val.strip() return result @@ -294,13 +325,16 @@ class ConfigHandler: :param key: :rtype: callable """ + def parser(value): exclude_directive = 'file:' if value.startswith(exclude_directive): raise ValueError( 'Only strings are accepted for the {0} field, ' - 'files are not accepted'.format(key)) + 'files are not accepted'.format(key) + ) return value + return parser @classmethod @@ -325,20 +359,18 @@ class ConfigHandler: if not value.startswith(include_directive): return value - spec = value[len(include_directive):] + spec = value[len(include_directive) :] filepaths = (os.path.abspath(path.strip()) for path in spec.split(',')) return '\n'.join( cls._read_file(path) for path in filepaths - if (cls._assert_local(path) or True) - and os.path.isfile(path) + if (cls._assert_local(path) or True) and os.path.isfile(path) ) @staticmethod def _assert_local(filepath): if not filepath.startswith(os.getcwd()): - raise DistutilsOptionError( - '`file:` directive can not access %s' % filepath) + raise DistutilsOptionError('`file:` directive can not access %s' % filepath) @staticmethod def _read_file(filepath): @@ -400,6 +432,7 @@ class ConfigHandler: :param parse_methods: :rtype: callable """ + def parse(value): parsed = value @@ -453,22 +486,25 @@ class ConfigHandler: self, # Dots in section names are translated into dunderscores. ('parse_section%s' % method_postfix).replace('.', '__'), - None) + None, + ) if section_parser_method is None: raise DistutilsOptionError( - 'Unsupported distribution option section: [%s.%s]' % ( - self.section_prefix, section_name)) + 'Unsupported distribution option section: [%s.%s]' + % (self.section_prefix, section_name) + ) section_parser_method(section_options) def _deprecated_config_handler(self, func, msg, warning_class): - """ this function will wrap around parameters that are deprecated + """this function will wrap around parameters that are deprecated :param msg: deprecation message :param warning_class: class of warning exception to be raised :param func: function to be wrapped around """ + @wraps(func) def config_handler(*args, **kwargs): warnings.warn(msg, warning_class) @@ -494,10 +530,12 @@ class ConfigMetadataHandler(ConfigHandler): """ - def __init__(self, target_obj, options, ignore_option_errors=False, - package_dir=None): - super(ConfigMetadataHandler, self).__init__(target_obj, options, - ignore_option_errors) + def __init__( + self, target_obj, options, ignore_option_errors=False, package_dir=None + ): + super(ConfigMetadataHandler, self).__init__( + target_obj, options, ignore_option_errors + ) self.package_dir = package_dir @property @@ -516,10 +554,17 @@ class ConfigMetadataHandler(ConfigHandler): parse_list, "The requires parameter is deprecated, please use " "install_requires for runtime dependencies.", - DeprecationWarning), + DeprecationWarning, + ), 'obsoletes': parse_list, 'classifiers': self._get_parser_compound(parse_file, parse_list), 'license': exclude_files_parser('license'), + 'license_file': self._deprecated_config_handler( + exclude_files_parser('license_file'), + "The license_file parameter is deprecated, " + "use license_files instead.", + DeprecationWarning, + ), 'license_files': parse_list, 'description': parse_file, 'long_description': parse_file, @@ -574,15 +619,12 @@ class ConfigOptionsHandler(ConfigHandler): parse_list_semicolon = partial(self._parse_list, separator=';') parse_bool = self._parse_bool parse_dict = self._parse_dict + parse_cmdclass = self._parse_cmdclass return { 'zip_safe': parse_bool, - 'use_2to3': parse_bool, 'include_package_data': parse_bool, 'package_dir': parse_dict, - 'use_2to3_fixers': parse_list, - 'use_2to3_exclude_fixers': parse_list, - 'convert_2to3_doctests': parse_list, 'scripts': parse_list, 'eager_resources': parse_list, 'dependency_links': parse_list, @@ -594,8 +636,21 @@ class ConfigOptionsHandler(ConfigHandler): 'entry_points': self._parse_file, 'py_modules': parse_list, 'python_requires': SpecifierSet, + 'cmdclass': parse_cmdclass, } + def _parse_cmdclass(self, value): + def resolve_class(qualified_class_name): + idx = qualified_class_name.rfind('.') + class_name = qualified_class_name[idx + 1 :] + pkg_name = qualified_class_name[:idx] + + module = __import__(pkg_name) + + return getattr(module, class_name) + + return {k: resolve_class(v) for k, v in self._parse_dict(value).items()} + def _parse_packages(self, value): """Parses `packages` option value. @@ -612,7 +667,8 @@ class ConfigOptionsHandler(ConfigHandler): # Read function arguments from a dedicated section. find_kwargs = self.parse_section_packages__find( - self.sections.get('packages.find', {})) + self.sections.get('packages.find', {}) + ) if findns: from setuptools import find_namespace_packages as find_packages @@ -628,13 +684,13 @@ class ConfigOptionsHandler(ConfigHandler): :param dict section_options: """ - section_data = self._parse_section_to_dict( - section_options, self._parse_list) + section_data = self._parse_section_to_dict(section_options, self._parse_list) valid_keys = ['where', 'include', 'exclude'] find_kwargs = dict( - [(k, v) for k, v in section_data.items() if k in valid_keys and v]) + [(k, v) for k, v in section_data.items() if k in valid_keys and v] + ) where = find_kwargs.get('where') if where is not None: @@ -672,8 +728,7 @@ class ConfigOptionsHandler(ConfigHandler): :param dict section_options: """ - self['exclude_package_data'] = self._parse_package_data( - section_options) + self['exclude_package_data'] = self._parse_package_data(section_options) def parse_section_extras_require(self, section_options): """Parses `extras_require` configuration file section. @@ -682,12 +737,13 @@ class ConfigOptionsHandler(ConfigHandler): """ parse_list = partial(self._parse_list, separator=';') self['extras_require'] = self._parse_section_to_dict( - section_options, parse_list) + section_options, parse_list + ) def parse_section_data_files(self, section_options): """Parses `data_files` configuration file section. :param dict section_options: """ - parsed = self._parse_section_to_dict(section_options, self._parse_list) + parsed = self._parse_section_to_dict(section_options, self._parse_list_glob) self['data_files'] = [(k, v) for k, v in parsed.items()] diff --git a/setuptools/dist.py b/setuptools/dist.py index 2c088ef8..8e2111a5 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -11,10 +11,14 @@ import distutils.log import distutils.core import distutils.cmd import distutils.dist +import distutils.command from distutils.util import strtobool from distutils.debug import DEBUG from distutils.fancy_getopt import translate_longopt +from glob import iglob import itertools +import textwrap +from typing import List, Optional, TYPE_CHECKING from collections import defaultdict from email import message_from_file @@ -25,15 +29,20 @@ from distutils.version import StrictVersion from setuptools.extern import packaging from setuptools.extern import ordered_set +from setuptools.extern.more_itertools import unique_everseen from . import SetuptoolsDeprecationWarning import setuptools +import setuptools.command from setuptools import windows_support from setuptools.monkey import get_unpatched from setuptools.config import parse_configuration import pkg_resources +if TYPE_CHECKING: + from email.message import Message + __import__('setuptools.extern.packaging.specifiers') __import__('setuptools.extern.packaging.version') @@ -45,83 +54,108 @@ def _get_unpatched(cls): def get_metadata_version(self): mv = getattr(self, 'metadata_version', None) - if mv is None: - if self.long_description_content_type or self.provides_extras: - mv = StrictVersion('2.1') - elif (self.maintainer is not None or - self.maintainer_email is not None or - getattr(self, 'python_requires', None) is not None or - self.project_urls): - mv = StrictVersion('1.2') - elif (self.provides or self.requires or self.obsoletes or - self.classifiers or self.download_url): - mv = StrictVersion('1.1') - else: - mv = StrictVersion('1.0') - + mv = StrictVersion('2.1') self.metadata_version = mv - return mv -def read_pkg_file(self, file): - """Reads the metadata values from a file object.""" - msg = message_from_file(file) +def rfc822_unescape(content: str) -> str: + """Reverse RFC-822 escaping by removing leading whitespaces from content.""" + lines = content.splitlines() + if len(lines) == 1: + return lines[0].lstrip() + return '\n'.join((lines[0].lstrip(), textwrap.dedent('\n'.join(lines[1:])))) + - def _read_field(name): - value = msg[name] - if value == 'UNKNOWN': - return None +def _read_field_from_msg(msg: "Message", field: str) -> Optional[str]: + """Read Message header field.""" + value = msg[field] + if value == 'UNKNOWN': + return None + return value + + +def _read_field_unescaped_from_msg(msg: "Message", field: str) -> Optional[str]: + """Read Message header field and apply rfc822_unescape.""" + value = _read_field_from_msg(msg, field) + if value is None: return value + return rfc822_unescape(value) + - def _read_list(name): - values = msg.get_all(name, None) - if values == []: - return None - return values +def _read_list_from_msg(msg: "Message", field: str) -> Optional[List[str]]: + """Read Message header field and return all results as list.""" + values = msg.get_all(field, None) + if values == []: + return None + return values + + +def _read_payload_from_msg(msg: "Message") -> Optional[str]: + value = msg.get_payload().strip() + if value == 'UNKNOWN': + return None + return value + + +def read_pkg_file(self, file): + """Reads the metadata values from a file object.""" + msg = message_from_file(file) self.metadata_version = StrictVersion(msg['metadata-version']) - self.name = _read_field('name') - self.version = _read_field('version') - self.description = _read_field('summary') + self.name = _read_field_from_msg(msg, 'name') + self.version = _read_field_from_msg(msg, 'version') + self.description = _read_field_from_msg(msg, 'summary') # we are filling author only. - self.author = _read_field('author') + self.author = _read_field_from_msg(msg, 'author') self.maintainer = None - self.author_email = _read_field('author-email') + self.author_email = _read_field_from_msg(msg, 'author-email') self.maintainer_email = None - self.url = _read_field('home-page') - self.license = _read_field('license') + self.url = _read_field_from_msg(msg, 'home-page') + self.license = _read_field_unescaped_from_msg(msg, 'license') if 'download-url' in msg: - self.download_url = _read_field('download-url') + self.download_url = _read_field_from_msg(msg, 'download-url') else: self.download_url = None - self.long_description = _read_field('description') - self.description = _read_field('summary') + self.long_description = _read_field_unescaped_from_msg(msg, 'description') + if self.long_description is None and self.metadata_version >= StrictVersion('2.1'): + self.long_description = _read_payload_from_msg(msg) + self.description = _read_field_from_msg(msg, 'summary') if 'keywords' in msg: - self.keywords = _read_field('keywords').split(',') + self.keywords = _read_field_from_msg(msg, 'keywords').split(',') - self.platforms = _read_list('platform') - self.classifiers = _read_list('classifier') + self.platforms = _read_list_from_msg(msg, 'platform') + self.classifiers = _read_list_from_msg(msg, 'classifier') # PEP 314 - these fields only exist in 1.1 if self.metadata_version == StrictVersion('1.1'): - self.requires = _read_list('requires') - self.provides = _read_list('provides') - self.obsoletes = _read_list('obsoletes') + self.requires = _read_list_from_msg(msg, 'requires') + self.provides = _read_list_from_msg(msg, 'provides') + self.obsoletes = _read_list_from_msg(msg, 'obsoletes') else: self.requires = None self.provides = None self.obsoletes = None + self.license_files = _read_list_from_msg(msg, 'license-file') + + +def single_line(val): + # quick and dirty validation for description pypa/setuptools#1390 + if '\n' in val: + # TODO after 2021-07-31: Replace with `raise ValueError("newlines not allowed")` + warnings.warn("newlines not allowed and will break in the future") + val = val.replace('\n', ' ') + return val + # Based on Python 3.5 version -def write_pkg_file(self, file): - """Write the PKG-INFO format data to a file object. - """ +def write_pkg_file(self, file): # noqa: C901 # is too complex (14) # FIXME + """Write the PKG-INFO format data to a file object.""" version = self.get_metadata_version() def write_field(key, value): @@ -130,44 +164,34 @@ def write_pkg_file(self, file): write_field('Metadata-Version', str(version)) write_field('Name', self.get_name()) write_field('Version', self.get_version()) - write_field('Summary', self.get_description()) + write_field('Summary', single_line(self.get_description())) write_field('Home-page', self.get_url()) - if version < StrictVersion('1.2'): - write_field('Author', self.get_contact()) - write_field('Author-email', self.get_contact_email()) - else: - optional_fields = ( - ('Author', 'author'), - ('Author-email', 'author_email'), - ('Maintainer', 'maintainer'), - ('Maintainer-email', 'maintainer_email'), - ) - - for field, attr in optional_fields: - attr_val = getattr(self, attr) + optional_fields = ( + ('Author', 'author'), + ('Author-email', 'author_email'), + ('Maintainer', 'maintainer'), + ('Maintainer-email', 'maintainer_email'), + ) - if attr_val is not None: - write_field(field, attr_val) + for field, attr in optional_fields: + attr_val = getattr(self, attr, None) + if attr_val is not None: + write_field(field, attr_val) - write_field('License', self.get_license()) + license = rfc822_escape(self.get_license()) + write_field('License', license) if self.download_url: write_field('Download-URL', self.download_url) for project_url in self.project_urls.items(): write_field('Project-URL', '%s, %s' % project_url) - long_desc = rfc822_escape(self.get_long_description()) - write_field('Description', long_desc) - keywords = ','.join(self.get_keywords()) if keywords: write_field('Keywords', keywords) - if version >= StrictVersion('1.2'): - for platform in self.get_platforms(): - write_field('Platform', platform) - else: - self._write_list(file, 'Platform', self.get_platforms()) + for platform in self.get_platforms(): + write_field('Platform', platform) self._write_list(file, 'Classifier', self.get_classifiers()) @@ -182,14 +206,15 @@ def write_pkg_file(self, file): # PEP 566 if self.long_description_content_type: - write_field( - 'Description-Content-Type', - self.long_description_content_type - ) + write_field('Description-Content-Type', self.long_description_content_type) if self.provides_extras: for extra in self.provides_extras: write_field('Provides-Extra', extra) + self._write_list(file, 'License-File', self.license_files or []) + + file.write("\n%s\n\n" % self.get_long_description()) + sequence = tuple, list @@ -200,8 +225,7 @@ def check_importable(dist, attr, value): assert not ep.extras except (TypeError, ValueError, AttributeError, AssertionError) as e: raise DistutilsSetupError( - "%r must be importable 'module:attrs' string (got %r)" - % (attr, value) + "%r must be importable 'module:attrs' string (got %r)" % (attr, value) ) from e @@ -226,14 +250,16 @@ def check_nsp(dist, attr, value): for nsp in ns_packages: if not dist.has_contents_for(nsp): raise DistutilsSetupError( - "Distribution contains no modules or packages for " + - "namespace package %r" % nsp + "Distribution contains no modules or packages for " + + "namespace package %r" % nsp ) parent, sep, child = nsp.rpartition('.') if parent and parent not in ns_packages: distutils.log.warn( "WARNING: %r is declared as a package namespace, but %r" - " is not: please correct this in setup.py", nsp, parent + " is not: please correct this in setup.py", + nsp, + parent, ) @@ -263,6 +289,13 @@ def assert_bool(dist, attr, value): raise DistutilsSetupError(tmpl.format(attr=attr, value=value)) +def invalid_unless_false(dist, attr, value): + if not value: + warnings.warn(f"{attr} is ignored.", DistDeprecationWarning) + return + raise DistutilsSetupError(f"{attr} is invalid.") + + def check_requirements(dist, attr, value): """Verify that install_requires is a valid requirements list""" try: @@ -274,23 +307,18 @@ def check_requirements(dist, attr, value): "{attr!r} must be a string or list of strings " "containing valid project/version requirement specifiers; {error}" ) - raise DistutilsSetupError( - tmpl.format(attr=attr, error=error) - ) from error + raise DistutilsSetupError(tmpl.format(attr=attr, error=error)) from error def check_specifier(dist, attr, value): """Verify that value is a valid version specifier""" try: packaging.specifiers.SpecifierSet(value) - except packaging.specifiers.InvalidSpecifier as error: + except (packaging.specifiers.InvalidSpecifier, AttributeError) as error: tmpl = ( - "{attr!r} must be a string " - "containing valid version specifiers; {error}" + "{attr!r} must be a string " "containing valid version specifiers; {error}" ) - raise DistutilsSetupError( - tmpl.format(attr=attr, error=error) - ) from error + raise DistutilsSetupError(tmpl.format(attr=attr, error=error)) from error def check_entry_points(dist, attr, value): @@ -311,12 +339,12 @@ def check_package_data(dist, attr, value): if not isinstance(value, dict): raise DistutilsSetupError( "{!r} must be a dictionary mapping package names to lists of " - "string wildcard patterns".format(attr)) + "string wildcard patterns".format(attr) + ) for k, v in value.items(): if not isinstance(k, str): raise DistutilsSetupError( - "keys of {!r} dict must be strings (got {!r})" - .format(attr, k) + "keys of {!r} dict must be strings (got {!r})".format(attr, k) ) assert_string_list(dist, 'values of {!r} dict'.format(attr), v) @@ -326,7 +354,8 @@ def check_packages(dist, attr, value): if not re.match(r'\w+(\.\w+)*', pkgname): distutils.log.warn( "WARNING: %r not a valid package name; please use only " - ".-separated package names in setup.py", pkgname + ".-separated package names in setup.py", + pkgname, ) @@ -386,10 +415,11 @@ class Distribution(_Distribution): """ _DISTUTILS_UNSUPPORTED_METADATA = { - 'long_description_content_type': None, + 'long_description_content_type': lambda: None, 'project_urls': dict, 'provides_extras': ordered_set.OrderedSet, - 'license_files': ordered_set.OrderedSet, + 'license_file': lambda: None, + 'license_files': lambda: None, } _patched_dist = None @@ -420,27 +450,32 @@ class Distribution(_Distribution): self.setup_requires = attrs.pop('setup_requires', []) for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'): vars(self).setdefault(ep.name, None) - _Distribution.__init__(self, { - k: v for k, v in attrs.items() - if k not in self._DISTUTILS_UNSUPPORTED_METADATA - }) - - # Fill-in missing metadata fields not supported by distutils. - # Note some fields may have been set by other tools (e.g. pbr) - # above; they are taken preferrentially to setup() arguments - for option, default in self._DISTUTILS_UNSUPPORTED_METADATA.items(): - for source in self.metadata.__dict__, attrs: - if option in source: - value = source[option] - break - else: - value = default() if default else None - setattr(self.metadata, option, value) + _Distribution.__init__( + self, + { + k: v + for k, v in attrs.items() + if k not in self._DISTUTILS_UNSUPPORTED_METADATA + }, + ) + + self._set_metadata_defaults(attrs) self.metadata.version = self._normalize_version( - self._validate_version(self.metadata.version)) + self._validate_version(self.metadata.version) + ) self._finalize_requires() + def _set_metadata_defaults(self, attrs): + """ + Fill-in missing metadata fields not supported by distutils. + Some fields may have been set by other tools (e.g. pbr). + Those fields (vars(self.metadata)) take precedence to + supplied attrs. + """ + for option, default in self._DISTUTILS_UNSUPPORTED_METADATA.items(): + vars(self.metadata).setdefault(option, attrs.get(option, default())) + @staticmethod def _normalize_version(version): if isinstance(version, setuptools.sic) or version is None: @@ -548,7 +583,42 @@ class Distribution(_Distribution): req.marker = None return req - def _parse_config_files(self, filenames=None): + def _finalize_license_files(self): + """Compute names of all license files which should be included.""" + license_files: Optional[List[str]] = self.metadata.license_files + patterns: List[str] = license_files if license_files else [] + + license_file: Optional[str] = self.metadata.license_file + if license_file and license_file not in patterns: + patterns.append(license_file) + + if license_files is None and license_file is None: + # Default patterns match the ones wheel uses + # See https://wheel.readthedocs.io/en/stable/user_guide.html + # -> 'Including license files in the generated wheel file' + patterns = ('LICEN[CS]E*', 'COPYING*', 'NOTICE*', 'AUTHORS*') + + self.metadata.license_files = list( + unique_everseen(self._expand_patterns(patterns)) + ) + + @staticmethod + def _expand_patterns(patterns): + """ + >>> list(Distribution._expand_patterns(['LICENSE'])) + ['LICENSE'] + >>> list(Distribution._expand_patterns(['setup.cfg', 'LIC*'])) + ['setup.cfg', 'LICENSE'] + """ + return ( + path + for pattern in patterns + for path in sorted(iglob(pattern)) + if not path.endswith('~') and os.path.isfile(path) + ) + + # FIXME: 'Distribution._parse_config_files' is too complex (14) + def _parse_config_files(self, filenames=None): # noqa: C901 """ Adapted from distutils.dist.Distribution.parse_config_files, this method provides the same functionality in subtly-improved @@ -557,14 +627,25 @@ class Distribution(_Distribution): from configparser import ConfigParser # Ignore install directory options if we have a venv - if sys.prefix != sys.base_prefix: - ignore_options = [ - 'install-base', 'install-platbase', 'install-lib', - 'install-platlib', 'install-purelib', 'install-headers', - 'install-scripts', 'install-data', 'prefix', 'exec-prefix', - 'home', 'user', 'root'] - else: - ignore_options = [] + ignore_options = ( + [] + if sys.prefix == sys.base_prefix + else [ + 'install-base', + 'install-platbase', + 'install-lib', + 'install-platlib', + 'install-purelib', + 'install-headers', + 'install-scripts', + 'install-data', + 'prefix', + 'exec-prefix', + 'home', + 'user', + 'root', + ] + ) ignore_options = frozenset(ignore_options) @@ -575,6 +656,7 @@ class Distribution(_Distribution): self.announce("Distribution.parse_config_files():") parser = ConfigParser() + parser.optionxform = str for filename in filenames: with io.open(filename, encoding='utf-8') as reader: if DEBUG: @@ -585,32 +667,82 @@ class Distribution(_Distribution): opt_dict = self.get_option_dict(section) for opt in options: - if opt != '__name__' and opt not in ignore_options: - val = parser.get(section, opt) - opt = opt.replace('-', '_') - opt_dict[opt] = (filename, val) + if opt == '__name__' or opt in ignore_options: + continue + + val = parser.get(section, opt) + opt = self.warn_dash_deprecation(opt, section) + opt = self.make_option_lowercase(opt, section) + opt_dict[opt] = (filename, val) # Make the ConfigParser forget everything (so we retain # the original filenames that options come from) parser.__init__() + if 'global' not in self.command_options: + return + # If there was a "global" section in the config file, use it # to set Distribution options. - if 'global' in self.command_options: - for (opt, (src, val)) in self.command_options['global'].items(): - alias = self.negative_opt.get(opt) - try: - if alias: - setattr(self, alias, not strtobool(val)) - elif opt in ('verbose', 'dry_run'): # ugh! - setattr(self, opt, strtobool(val)) - else: - setattr(self, opt, val) - except ValueError as e: - raise DistutilsOptionError(e) from e + for (opt, (src, val)) in self.command_options['global'].items(): + alias = self.negative_opt.get(opt) + if alias: + val = not strtobool(val) + elif opt in ('verbose', 'dry_run'): # ugh! + val = strtobool(val) + + try: + setattr(self, alias or opt, val) + except ValueError as e: + raise DistutilsOptionError(e) from e + + def warn_dash_deprecation(self, opt, section): + if section in ( + 'options.extras_require', + 'options.data_files', + ): + return opt + + underscore_opt = opt.replace('-', '_') + commands = distutils.command.__all__ + self._setuptools_commands() + if ( + not section.startswith('options') + and section != 'metadata' + and section not in commands + ): + return underscore_opt + + if '-' in opt: + warnings.warn( + "Usage of dash-separated '%s' will not be supported in future " + "versions. Please use the underscore name '%s' instead" + % (opt, underscore_opt) + ) + return underscore_opt - def _set_command_options(self, command_obj, option_dict=None): + def _setuptools_commands(self): + try: + dist = pkg_resources.get_distribution('setuptools') + return list(dist.get_entry_map('distutils.commands')) + except pkg_resources.DistributionNotFound: + # during bootstrapping, distribution doesn't exist + return [] + + def make_option_lowercase(self, opt, section): + if section != 'metadata' or opt.islower(): + return opt + + lowercase_opt = opt.lower() + warnings.warn( + "Usage of uppercase key '%s' in '%s' will be deprecated in future " + "versions. Please use lowercase '%s' instead" + % (opt, section, lowercase_opt) + ) + return lowercase_opt + + # FIXME: 'Distribution._set_command_options' is too complex (14) + def _set_command_options(self, command_obj, option_dict=None): # noqa: C901 """ Set the options for 'command_obj' from 'option_dict'. Basically this means copying elements of a dictionary ('option_dict') to @@ -630,11 +762,9 @@ class Distribution(_Distribution): self.announce(" setting options for '%s' command:" % command_name) for (option, (source, value)) in option_dict.items(): if DEBUG: - self.announce(" %s = %s (from %s)" % (option, value, - source)) + self.announce(" %s = %s (from %s)" % (option, value, source)) try: - bool_opts = [translate_longopt(o) - for o in command_obj.boolean_options] + bool_opts = [translate_longopt(o) for o in command_obj.boolean_options] except AttributeError: bool_opts = [] try: @@ -653,7 +783,8 @@ class Distribution(_Distribution): else: raise DistutilsOptionError( "error in %s: command '%s' has no such option '%s'" - % (source, command_name, option)) + % (source, command_name, option) + ) except ValueError as e: raise DistutilsOptionError(e) from e @@ -664,9 +795,11 @@ class Distribution(_Distribution): """ self._parse_config_files(filenames=filenames) - parse_configuration(self, self.command_options, - ignore_option_errors=ignore_option_errors) + parse_configuration( + self, self.command_options, ignore_option_errors=ignore_option_errors + ) self._finalize_requires() + self._finalize_license_files() def fetch_build_eggs(self, requires): """Resolve pre-setup requirements""" @@ -690,10 +823,27 @@ class Distribution(_Distribution): def by_order(hook): return getattr(hook, 'order', 0) - eps = map(lambda e: e.load(), pkg_resources.iter_entry_points(group)) - for ep in sorted(eps, key=by_order): + + defined = pkg_resources.iter_entry_points(group) + filtered = itertools.filterfalse(self._removed, defined) + loaded = map(lambda e: e.load(), filtered) + for ep in sorted(loaded, key=by_order): ep(self) + @staticmethod + def _removed(ep): + """ + When removing an entry point, if metadata is loaded + from an older version of Setuptools, that removed + entry point will attempt to be loaded and will fail. + See #2765 for more details. + """ + removed = { + # removed 2021-09-05 + '2to3_doctests', + } + return ep.name in removed + def _finalize_setup_keywords(self): for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'): value = getattr(self, ep.name, None) @@ -701,16 +851,6 @@ class Distribution(_Distribution): ep.require(installer=self.fetch_build_egg) ep.load()(self, ep.name, value) - def _finalize_2to3_doctests(self): - if getattr(self, 'convert_2to3_doctests', None): - # XXX may convert to set here when we can rely on set being builtin - self.convert_2to3_doctests = [ - os.path.abspath(p) - for p in self.convert_2to3_doctests - ] - else: - self.convert_2to3_doctests = [] - def get_egg_cache_dir(self): egg_cache_dir = os.path.join(os.curdir, '.eggs') if not os.path.exists(egg_cache_dir): @@ -718,10 +858,14 @@ class Distribution(_Distribution): windows_support.hide_file(egg_cache_dir) readme_txt_filename = os.path.join(egg_cache_dir, 'README.txt') with open(readme_txt_filename, 'w') as f: - f.write('This directory contains eggs that were downloaded ' - 'by setuptools to build, test, and run plug-ins.\n\n') - f.write('This directory caches those eggs to prevent ' - 'repeated downloads.\n\n') + f.write( + 'This directory contains eggs that were downloaded ' + 'by setuptools to build, test, and run plug-ins.\n\n' + ) + f.write( + 'This directory caches those eggs to prevent ' + 'repeated downloads.\n\n' + ) f.write('However, it is safe to delete this directory.\n\n') return egg_cache_dir @@ -729,6 +873,7 @@ class Distribution(_Distribution): def fetch_build_egg(self, req): """Fetch an egg needed for building""" from setuptools.installer import fetch_build_egg + return fetch_build_egg(self, req) def get_command_class(self, command): @@ -788,19 +933,18 @@ class Distribution(_Distribution): pfx = package + '.' if self.packages: self.packages = [ - p for p in self.packages - if p != package and not p.startswith(pfx) + p for p in self.packages if p != package and not p.startswith(pfx) ] if self.py_modules: self.py_modules = [ - p for p in self.py_modules - if p != package and not p.startswith(pfx) + p for p in self.py_modules if p != package and not p.startswith(pfx) ] if self.ext_modules: self.ext_modules = [ - p for p in self.ext_modules + p + for p in self.ext_modules if p.name != package and not p.name.startswith(pfx) ] @@ -822,9 +966,7 @@ class Distribution(_Distribution): try: old = getattr(self, name) except AttributeError as e: - raise DistutilsSetupError( - "%s: No such distribution setting" % name - ) from e + raise DistutilsSetupError("%s: No such distribution setting" % name) from e if old is not None and not isinstance(old, sequence): raise DistutilsSetupError( name + ": this setting cannot be changed via include/exclude" @@ -836,15 +978,11 @@ class Distribution(_Distribution): """Handle 'include()' for list/tuple attrs without a special handler""" if not isinstance(value, sequence): - raise DistutilsSetupError( - "%s: setting must be a list (%r)" % (name, value) - ) + raise DistutilsSetupError("%s: setting must be a list (%r)" % (name, value)) try: old = getattr(self, name) except AttributeError as e: - raise DistutilsSetupError( - "%s: No such distribution setting" % name - ) from e + raise DistutilsSetupError("%s: No such distribution setting" % name) from e if old is None: setattr(self, name, value) elif not isinstance(old, sequence): @@ -897,6 +1035,7 @@ class Distribution(_Distribution): src, alias = aliases[command] del aliases[command] # ensure each alias can expand only once! import shlex + args[:1] = shlex.split(alias, True) command = args[0] @@ -996,12 +1135,14 @@ class Distribution(_Distribution): line_buffering = sys.stdout.line_buffering sys.stdout = io.TextIOWrapper( - sys.stdout.detach(), 'utf-8', errors, newline, line_buffering) + sys.stdout.detach(), 'utf-8', errors, newline, line_buffering + ) try: return _Distribution.handle_display_options(self, option_order) finally: sys.stdout = io.TextIOWrapper( - sys.stdout.detach(), encoding, errors, newline, line_buffering) + sys.stdout.detach(), encoding, errors, newline, line_buffering + ) class DistDeprecationWarning(SetuptoolsDeprecationWarning): diff --git a/setuptools/extern/__init__.py b/setuptools/extern/__init__.py index b7f30dc2..baca1afa 100644 --- a/setuptools/extern/__init__.py +++ b/setuptools/extern/__init__.py @@ -1,3 +1,4 @@ +import importlib.util import sys @@ -20,17 +21,10 @@ class VendorImporter: yield self.vendor_pkg + '.' yield '' - def find_module(self, fullname, path=None): - """ - Return self when fullname starts with root_name and the - target module is one vendored through this importer. - """ + def _module_matches_namespace(self, fullname): + """Figure out if the target module is vendored.""" root, base, target = fullname.partition(self.root_name + '.') - if root: - return - if not any(map(target.startswith, self.vendored_names)): - return - return self + return not root and any(map(target.startswith, self.vendored_names)) def load_module(self, fullname): """ @@ -54,6 +48,19 @@ class VendorImporter: "distribution.".format(**locals()) ) + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + + def find_spec(self, fullname, path=None, target=None): + """Return a module spec for vendored names.""" + return ( + importlib.util.spec_from_loader(fullname, self) + if self._module_matches_namespace(fullname) else None + ) + def install(self): """ Install this importer into sys.meta_path if not already present. @@ -62,5 +69,5 @@ class VendorImporter: sys.meta_path.append(self) -names = 'packaging', 'pyparsing', 'ordered_set', +names = 'packaging', 'pyparsing', 'ordered_set', 'more_itertools', VendorImporter(__name__, names, 'setuptools._vendor').install() diff --git a/setuptools/glob.py b/setuptools/glob.py index 9d7cbc5d..87062b81 100644 --- a/setuptools/glob.py +++ b/setuptools/glob.py @@ -47,6 +47,8 @@ def iglob(pathname, recursive=False): def _iglob(pathname, recursive): dirname, basename = os.path.split(pathname) + glob_in_dir = glob2 if recursive and _isrecursive(basename) else glob1 + if not has_magic(pathname): if basename: if os.path.lexists(pathname): @@ -56,13 +58,9 @@ def _iglob(pathname, recursive): if os.path.isdir(dirname): yield pathname return + if not dirname: - if recursive and _isrecursive(basename): - for x in glob2(dirname, basename): - yield x - else: - for x in glob1(dirname, basename): - yield x + yield from glob_in_dir(dirname, basename) return # `os.path.split()` returns the argument itself as a dirname if it is a # drive or UNC path. Prevent an infinite recursion if a drive or UNC path @@ -71,12 +69,7 @@ def _iglob(pathname, recursive): dirs = _iglob(dirname, recursive) else: dirs = [dirname] - if has_magic(basename): - if recursive and _isrecursive(basename): - glob_in_dir = glob2 - else: - glob_in_dir = glob1 - else: + if not has_magic(basename): glob_in_dir = glob0 for dirname in dirs: for name in glob_in_dir(dirname, basename): diff --git a/setuptools/gui-arm64.exe b/setuptools/gui-arm64.exe Binary files differnew file mode 100644 index 00000000..5730f11d --- /dev/null +++ b/setuptools/gui-arm64.exe diff --git a/setuptools/installer.py b/setuptools/installer.py index e630b874..57e2b587 100644 --- a/setuptools/installer.py +++ b/setuptools/installer.py @@ -7,7 +7,6 @@ from distutils import log from distutils.errors import DistutilsError import pkg_resources -from setuptools.command.easy_install import easy_install from setuptools.wheel import Wheel @@ -19,54 +18,11 @@ def _fixup_find_links(find_links): return find_links -def _legacy_fetch_build_egg(dist, req): - """Fetch an egg needed for building. - - Legacy path using EasyInstall. - """ - tmp_dist = dist.__class__({'script_args': ['easy_install']}) - opts = tmp_dist.get_option_dict('easy_install') - opts.clear() - opts.update( - (k, v) - for k, v in dist.get_option_dict('easy_install').items() - if k in ( - # don't use any other settings - 'find_links', 'site_dirs', 'index_url', - 'optimize', 'site_dirs', 'allow_hosts', - )) - if dist.dependency_links: - links = dist.dependency_links[:] - if 'find_links' in opts: - links = _fixup_find_links(opts['find_links'][1]) + links - opts['find_links'] = ('setup', links) - install_dir = dist.get_egg_cache_dir() - cmd = easy_install( - tmp_dist, args=["x"], install_dir=install_dir, - exclude_scripts=True, - always_copy=False, build_directory=None, editable=False, - upgrade=False, multi_version=True, no_report=True, user=False - ) - cmd.ensure_finalized() - return cmd.easy_install(req) - - -def fetch_build_egg(dist, req): +def fetch_build_egg(dist, req): # noqa: C901 # is too complex (16) # FIXME """Fetch an egg needed for building. Use pip/wheel to fetch/build a wheel.""" - # Check pip is available. - try: - pkg_resources.get_distribution('pip') - except pkg_resources.DistributionNotFound: - dist.announce( - 'WARNING: The pip package is not available, falling back ' - 'to EasyInstall for handling setup_requires/test_requires; ' - 'this is deprecated and will be removed in a future version.', - log.WARN - ) - return _legacy_fetch_build_egg(dist, req) - # Warn if wheel is not. + # Warn if wheel is not available try: pkg_resources.get_distribution('wheel') except pkg_resources.DistributionNotFound: @@ -80,20 +36,17 @@ def fetch_build_egg(dist, req): if 'allow_hosts' in opts: raise DistutilsError('the `allow-hosts` option is not supported ' 'when using pip to install requirements.') - if 'PIP_QUIET' in os.environ or 'PIP_VERBOSE' in os.environ: - quiet = False - else: - quiet = True + quiet = 'PIP_QUIET' not in os.environ and 'PIP_VERBOSE' not in os.environ if 'PIP_INDEX_URL' in os.environ: index_url = None elif 'index_url' in opts: index_url = opts['index_url'][1] else: index_url = None - if 'find_links' in opts: - find_links = _fixup_find_links(opts['find_links'][1])[:] - else: - find_links = [] + find_links = ( + _fixup_find_links(opts['find_links'][1])[:] if 'find_links' in opts + else [] + ) if dist.dependency_links: find_links.extend(dist.dependency_links) eggs_dir = os.path.realpath(dist.get_egg_cache_dir()) @@ -112,16 +65,12 @@ def fetch_build_egg(dist, req): cmd.append('--quiet') if index_url is not None: cmd.extend(('--index-url', index_url)) - if find_links is not None: - for link in find_links: - cmd.extend(('--find-links', link)) + for link in find_links or []: + cmd.extend(('--find-links', link)) # If requirement is a PEP 508 direct URL, directly pass # the URL to pip, as `req @ url` does not work on the # command line. - if req.url: - cmd.append(req.url) - else: - cmd.append(str(req)) + cmd.append(req.url or str(req)) try: subprocess.check_call(cmd) except subprocess.CalledProcessError as e: diff --git a/setuptools/lib2to3_ex.py b/setuptools/lib2to3_ex.py deleted file mode 100644 index c176abf6..00000000 --- a/setuptools/lib2to3_ex.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -Customized Mixin2to3 support: - - - adds support for converting doctests -""" - -import warnings -from distutils.util import Mixin2to3 as _Mixin2to3 -from distutils import log -from lib2to3.refactor import RefactoringTool, get_fixers_from_package - -import setuptools -from ._deprecation_warning import SetuptoolsDeprecationWarning - - -class DistutilsRefactoringTool(RefactoringTool): - def log_error(self, msg, *args, **kw): - log.error(msg, *args) - - def log_message(self, msg, *args): - log.info(msg, *args) - - def log_debug(self, msg, *args): - log.debug(msg, *args) - - -class Mixin2to3(_Mixin2to3): - def run_2to3(self, files, doctests=False): - # See of the distribution option has been set, otherwise check the - # setuptools default. - if self.distribution.use_2to3 is not True: - return - if not files: - return - - warnings.warn( - "2to3 support is deprecated. If the project still " - "requires Python 2 support, please migrate to " - "a single-codebase solution or employ an " - "independent conversion process.", - SetuptoolsDeprecationWarning) - log.info("Fixing " + " ".join(files)) - self.__build_fixer_names() - self.__exclude_fixers() - if doctests: - if setuptools.run_2to3_on_doctests: - r = DistutilsRefactoringTool(self.fixer_names) - r.refactor(files, write=True, doctests_only=True) - else: - _Mixin2to3.run_2to3(self, files) - - def __build_fixer_names(self): - if self.fixer_names: - return - self.fixer_names = [] - for p in setuptools.lib2to3_fixer_packages: - self.fixer_names.extend(get_fixers_from_package(p)) - if self.distribution.use_2to3_fixers is not None: - for p in self.distribution.use_2to3_fixers: - self.fixer_names.extend(get_fixers_from_package(p)) - - def __exclude_fixers(self): - excluded_fixers = getattr(self, 'exclude_fixers', []) - if self.distribution.use_2to3_exclude_fixers is not None: - excluded_fixers.extend(self.distribution.use_2to3_exclude_fixers) - for fixer_name in excluded_fixers: - if fixer_name in self.fixer_names: - self.fixer_names.remove(fixer_name) diff --git a/setuptools/msvc.py b/setuptools/msvc.py index 1ead72b4..281ea1c2 100644 --- a/setuptools/msvc.py +++ b/setuptools/msvc.py @@ -24,11 +24,13 @@ from io import open from os import listdir, pathsep from os.path import join, isfile, isdir, dirname import sys +import contextlib import platform import itertools import subprocess import distutils.errors from setuptools.extern.packaging.version import LegacyVersion +from setuptools.extern.more_itertools import unique_everseen from .monkey import get_unpatched @@ -192,7 +194,9 @@ def _msvc14_find_vc2017(): join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), "-latest", "-prerelease", + "-requiresAny", "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-requires", "Microsoft.VisualStudio.Workload.WDExpress", "-property", "installationPath", "-products", "*", ]).decode(encoding="mbcs", errors="strict").strip() @@ -724,28 +728,23 @@ class SystemInfo: ms = self.ri.microsoft vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs) vs_vers = [] - for hkey in self.ri.HKEYS: - for key in vckeys: - try: - bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ) - except (OSError, IOError): - continue - with bkey: - subkeys, values, _ = winreg.QueryInfoKey(bkey) - for i in range(values): - try: - ver = float(winreg.EnumValue(bkey, i)[0]) - if ver not in vs_vers: - vs_vers.append(ver) - except ValueError: - pass - for i in range(subkeys): - try: - ver = float(winreg.EnumKey(bkey, i)) - if ver not in vs_vers: - vs_vers.append(ver) - except ValueError: - pass + for hkey, key in itertools.product(self.ri.HKEYS, vckeys): + try: + bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ) + except (OSError, IOError): + continue + with bkey: + subkeys, values, _ = winreg.QueryInfoKey(bkey) + for i in range(values): + with contextlib.suppress(ValueError): + ver = float(winreg.EnumValue(bkey, i)[0]) + if ver not in vs_vers: + vs_vers.append(ver) + for i in range(subkeys): + with contextlib.suppress(ValueError): + ver = float(winreg.EnumKey(bkey, i)) + if ver not in vs_vers: + vs_vers.append(ver) return sorted(vs_vers) def find_programdata_vs_vers(self): @@ -925,8 +924,8 @@ class SystemInfo: """ return self._use_last_dir_name(join(self.WindowsSdkDir, 'lib')) - @property - def WindowsSdkDir(self): + @property # noqa: C901 + def WindowsSdkDir(self): # noqa: C901 # is too complex (12) # FIXME """ Microsoft Windows SDK directory. @@ -1802,29 +1801,5 @@ class EnvironmentInfo: if not extant_paths: msg = "%s environment variable is empty" % name.upper() raise distutils.errors.DistutilsPlatformError(msg) - unique_paths = self._unique_everseen(extant_paths) + unique_paths = unique_everseen(extant_paths) return pathsep.join(unique_paths) - - # from Python docs - @staticmethod - def _unique_everseen(iterable, key=None): - """ - List unique elements, preserving order. - Remember all elements ever seen. - - _unique_everseen('AAAABBBCCDAABBB') --> A B C D - - _unique_everseen('ABBCcAD', str.lower) --> A B C D - """ - seen = set() - seen_add = seen.add - if key is None: - for element in itertools.filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element diff --git a/setuptools/package_index.py b/setuptools/package_index.py index bc0ba7a6..c9254289 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -23,11 +23,12 @@ from pkg_resources import ( Environment, find_distributions, safe_name, safe_version, to_filename, Requirement, DEVELOP_DIST, EGG_DIST, ) -from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError from fnmatch import translate from setuptools.wheel import Wheel +from setuptools.extern.more_itertools import unique_everseen + EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) @@ -172,25 +173,6 @@ def interpret_distro_name( ) -# From Python 2.7 docs -def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen = set() - seen_add = seen.add - if key is None: - for element in itertools.filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element - - def unique_values(func): """ Wrap a function returning an iterable such that the resulting iterable @@ -299,17 +281,10 @@ class PackageIndex(Environment): self.package_pages = {} self.allows = re.compile('|'.join(map(translate, hosts))).match self.to_scan = [] - use_ssl = ( - verify_ssl - and ssl_support.is_available - and (ca_bundle or ssl_support.find_ca_bundle()) - ) - if use_ssl: - self.opener = ssl_support.opener_for(ca_bundle) - else: - self.opener = urllib.request.urlopen + self.opener = urllib.request.urlopen - def process_url(self, url, retrieve=False): + # FIXME: 'PackageIndex.process_url' is too complex (14) + def process_url(self, url, retrieve=False): # noqa: C901 """Evaluate a URL as a possible download, and maybe retrieve it""" if url in self.scanned_urls and not retrieve: return @@ -417,49 +392,53 @@ class PackageIndex(Environment): dist.precedence = SOURCE_DIST self.add(dist) + def _scan(self, link): + # Process a URL to see if it's for a package page + NO_MATCH_SENTINEL = None, None + if not link.startswith(self.index_url): + return NO_MATCH_SENTINEL + + parts = list(map( + urllib.parse.unquote, link[len(self.index_url):].split('/') + )) + if len(parts) != 2 or '#' in parts[1]: + return NO_MATCH_SENTINEL + + # it's a package page, sanitize and index it + pkg = safe_name(parts[0]) + ver = safe_version(parts[1]) + self.package_pages.setdefault(pkg.lower(), {})[link] = True + return to_filename(pkg), to_filename(ver) + def process_index(self, url, page): """Process the contents of a PyPI page""" - def scan(link): - # Process a URL to see if it's for a package page - if link.startswith(self.index_url): - parts = list(map( - urllib.parse.unquote, link[len(self.index_url):].split('/') - )) - if len(parts) == 2 and '#' not in parts[1]: - # it's a package page, sanitize and index it - pkg = safe_name(parts[0]) - ver = safe_version(parts[1]) - self.package_pages.setdefault(pkg.lower(), {})[link] = True - return to_filename(pkg), to_filename(ver) - return None, None - # process an index page into the package-page index for match in HREF.finditer(page): try: - scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) + self._scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) except ValueError: pass - pkg, ver = scan(url) # ensure this page is in the page index - if pkg: - # process individual package page - for new_url in find_external_links(url, page): - # Process the found URL - base, frag = egg_info_for_url(new_url) - if base.endswith('.py') and not frag: - if ver: - new_url += '#egg=%s-%s' % (pkg, ver) - else: - self.need_version_info(url) - self.scan_url(new_url) - - return PYPI_MD5.sub( - lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page - ) - else: + pkg, ver = self._scan(url) # ensure this page is in the page index + if not pkg: return "" # no sense double-scanning non-package pages + # process individual package page + for new_url in find_external_links(url, page): + # Process the found URL + base, frag = egg_info_for_url(new_url) + if base.endswith('.py') and not frag: + if ver: + new_url += '#egg=%s-%s' % (pkg, ver) + else: + self.need_version_info(url) + self.scan_url(new_url) + + return PYPI_MD5.sub( + lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page + ) + def need_version_info(self, url): self.scan_all( "Page at %s links to .py file(s) without version info; an index " @@ -580,7 +559,7 @@ class PackageIndex(Environment): spec = parse_requirement_arg(spec) return getattr(self.fetch_distribution(spec, tmpdir), 'location', None) - def fetch_distribution( + def fetch_distribution( # noqa: C901 # is too complex (14) # FIXME self, requirement, tmpdir, force_scan=False, source=False, develop_ok=False, local_index=None): """Obtain a distribution suitable for fulfilling `requirement` @@ -751,7 +730,8 @@ class PackageIndex(Environment): def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op - def open_url(self, url, warning=None): + # FIXME: + def open_url(self, url, warning=None): # noqa: C901 # is too complex (12) if url.startswith('file:'): return local_open(url) try: diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 91b960d8..034fc80d 100644 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -26,7 +26,10 @@ _open = open __all__ = [ - "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup", + "AbstractSandbox", + "DirectorySandbox", + "SandboxViolation", + "run_setup", ] @@ -106,6 +109,7 @@ class UnpickleableException(Exception): except Exception: # get UnpickleableException inside the sandbox from setuptools.sandbox import UnpickleableException as cls + return cls.dump(cls, cls(repr(exc))) @@ -154,7 +158,8 @@ def save_modules(): sys.modules.update(saved) # remove any modules imported since del_modules = ( - mod_name for mod_name in sys.modules + mod_name + for mod_name in sys.modules if mod_name not in saved # exclude any encodings modules. See #285 and not mod_name.startswith('encodings.') @@ -265,7 +270,8 @@ class AbstractSandbox: def __init__(self): self._attrs = [ - name for name in dir(_os) + name + for name in dir(_os) if not name.startswith('_') and hasattr(self, name) ] @@ -320,9 +326,25 @@ class AbstractSandbox: _file = _mk_single_path_wrapper('file', _file) _open = _mk_single_path_wrapper('open', _open) for name in [ - "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir", - "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat", - "startfile", "mkfifo", "mknod", "pathconf", "access" + "stat", + "listdir", + "chdir", + "open", + "chmod", + "chown", + "mkdir", + "remove", + "unlink", + "rmdir", + "utime", + "lchown", + "chroot", + "lstat", + "startfile", + "mkfifo", + "mknod", + "pathconf", + "access", ]: if hasattr(_os, name): locals()[name] = _mk_single_path_wrapper(name) @@ -373,7 +395,7 @@ class AbstractSandbox: """Called for path pairs like rename, link, and symlink operations""" return ( self._remap_input(operation + '-from', src, *args, **kw), - self._remap_input(operation + '-to', dst, *args, **kw) + self._remap_input(operation + '-to', dst, *args, **kw), ) @@ -386,28 +408,38 @@ else: class DirectorySandbox(AbstractSandbox): """Restrict operations to a single subdirectory - pseudo-chroot""" - write_ops = dict.fromkeys([ - "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir", - "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam", - ]) + write_ops = dict.fromkeys( + [ + "open", + "chmod", + "chown", + "mkdir", + "remove", + "unlink", + "rmdir", + "utime", + "lchown", + "chroot", + "mkfifo", + "mknod", + "tempnam", + ] + ) - _exception_patterns = [ - # Allow lib2to3 to attempt to save a pickled grammar object (#121) - r'.*lib2to3.*\.pickle$', - ] + _exception_patterns = [] "exempt writing to paths that match the pattern" def __init__(self, sandbox, exceptions=_EXCEPTIONS): self._sandbox = os.path.normcase(os.path.realpath(sandbox)) self._prefix = os.path.join(self._sandbox, '') self._exceptions = [ - os.path.normcase(os.path.realpath(path)) - for path in exceptions + os.path.normcase(os.path.realpath(path)) for path in exceptions ] AbstractSandbox.__init__(self) def _violation(self, operation, *args, **kw): from setuptools.sandbox import SandboxViolation + raise SandboxViolation(operation, args, kw) if _file: @@ -440,12 +472,10 @@ class DirectorySandbox(AbstractSandbox): def _exempted(self, filepath): start_matches = ( - filepath.startswith(exception) - for exception in self._exceptions + filepath.startswith(exception) for exception in self._exceptions ) pattern_matches = ( - re.match(pattern, filepath) - for pattern in self._exception_patterns + re.match(pattern, filepath) for pattern in self._exception_patterns ) candidates = itertools.chain(start_matches, pattern_matches) return any(candidates) @@ -470,16 +500,19 @@ class DirectorySandbox(AbstractSandbox): WRITE_FLAGS = functools.reduce( - operator.or_, [ - getattr(_os, a, 0) for a in - "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()] + operator.or_, + [ + getattr(_os, a, 0) + for a in "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split() + ], ) class SandboxViolation(DistutilsError): """A setup script attempted to modify the filesystem outside the sandbox""" - tmpl = textwrap.dedent(""" + tmpl = textwrap.dedent( + """ SandboxViolation: {cmd}{args!r} {kwargs} The package setup script has attempted to modify files on your system @@ -489,7 +522,8 @@ class SandboxViolation(DistutilsError): support alternate installation locations even if you run its setup script by hand. Please inform the package's author and the EasyInstall maintainers to find out if a fix or workaround is available. - """).lstrip() + """ + ).lstrip() def __str__(self): cmd, args, kwargs = self.args diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py deleted file mode 100644 index eac5e656..00000000 --- a/setuptools/ssl_support.py +++ /dev/null @@ -1,266 +0,0 @@ -import os -import socket -import atexit -import re -import functools -import urllib.request -import http.client - - -from pkg_resources import ResolutionError, ExtractionError - -try: - import ssl -except ImportError: - ssl = None - -__all__ = [ - 'VerifyingHTTPSHandler', 'find_ca_bundle', 'is_available', 'cert_paths', - 'opener_for' -] - -cert_paths = """ -/etc/pki/tls/certs/ca-bundle.crt -/etc/ssl/certs/ca-certificates.crt -/usr/share/ssl/certs/ca-bundle.crt -/usr/local/share/certs/ca-root.crt -/etc/ssl/cert.pem -/System/Library/OpenSSL/certs/cert.pem -/usr/local/share/certs/ca-root-nss.crt -/etc/ssl/ca-bundle.pem -""".strip().split() - -try: - HTTPSHandler = urllib.request.HTTPSHandler - HTTPSConnection = http.client.HTTPSConnection -except AttributeError: - HTTPSHandler = HTTPSConnection = object - -is_available = ssl is not None and object not in ( - HTTPSHandler, HTTPSConnection) - - -try: - from ssl import CertificateError, match_hostname -except ImportError: - try: - from backports.ssl_match_hostname import CertificateError - from backports.ssl_match_hostname import match_hostname - except ImportError: - CertificateError = None - match_hostname = None - -if not CertificateError: - - class CertificateError(ValueError): - pass - - -if not match_hostname: - - def _dnsname_match(dn, hostname, max_wildcards=1): - """Matching according to RFC 6125, section 6.4.3 - - https://tools.ietf.org/html/rfc6125#section-6.4.3 - """ - pats = [] - if not dn: - return False - - # Ported from python3-syntax: - # leftmost, *remainder = dn.split(r'.') - parts = dn.split(r'.') - leftmost = parts[0] - remainder = parts[1:] - - wildcards = leftmost.count('*') - if wildcards > max_wildcards: - # Issue #17980: avoid denials of service by refusing more - # than one wildcard per fragment. A survey of established - # policy among SSL implementations showed it to be a - # reasonable choice. - raise CertificateError( - "too many wildcards in certificate DNS name: " + repr(dn)) - - # speed up common case w/o wildcards - if not wildcards: - return dn.lower() == hostname.lower() - - # RFC 6125, section 6.4.3, subitem 1. - # The client SHOULD NOT attempt to match a - # presented identifier in which the wildcard - # character comprises a label other than the - # left-most label. - if leftmost == '*': - # When '*' is a fragment by itself, it matches a non-empty dotless - # fragment. - pats.append('[^.]+') - elif leftmost.startswith('xn--') or hostname.startswith('xn--'): - # RFC 6125, section 6.4.3, subitem 3. - # The client SHOULD NOT attempt to match a presented identifier - # where the wildcard character is embedded within an A-label or - # U-label of an internationalized domain name. - pats.append(re.escape(leftmost)) - else: - # Otherwise, '*' matches any dotless string, e.g. www* - pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) - - # add the remaining fragments, ignore any wildcards - for frag in remainder: - pats.append(re.escape(frag)) - - pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) - return pat.match(hostname) - - def match_hostname(cert, hostname): - """Verify that *cert* (in decoded format as returned by - SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 - rules are followed, but IP addresses are not accepted for *hostname*. - - CertificateError is raised on failure. On success, the function - returns nothing. - """ - if not cert: - raise ValueError("empty or no certificate") - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: - if key == 'DNS': - if _dnsname_match(value, hostname): - return - dnsnames.append(value) - if not dnsnames: - # The subject is only checked when there is no dNSName entry - # in subjectAltName - for sub in cert.get('subject', ()): - for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. - if key == 'commonName': - if _dnsname_match(value, hostname): - return - dnsnames.append(value) - if len(dnsnames) > 1: - raise CertificateError( - "hostname %r doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) - elif len(dnsnames) == 1: - raise CertificateError( - "hostname %r doesn't match %r" - % (hostname, dnsnames[0])) - else: - raise CertificateError( - "no appropriate commonName or " - "subjectAltName fields were found") - - -class VerifyingHTTPSHandler(HTTPSHandler): - """Simple verifying handler: no auth, subclasses, timeouts, etc.""" - - def __init__(self, ca_bundle): - self.ca_bundle = ca_bundle - HTTPSHandler.__init__(self) - - def https_open(self, req): - return self.do_open( - lambda host, **kw: VerifyingHTTPSConn(host, self.ca_bundle, **kw), - req - ) - - -class VerifyingHTTPSConn(HTTPSConnection): - """Simple verifying connection: no auth, subclasses, timeouts, etc.""" - - def __init__(self, host, ca_bundle, **kw): - HTTPSConnection.__init__(self, host, **kw) - self.ca_bundle = ca_bundle - - def connect(self): - sock = socket.create_connection( - (self.host, self.port), getattr(self, 'source_address', None) - ) - - # Handle the socket if a (proxy) tunnel is present - if hasattr(self, '_tunnel') and getattr(self, '_tunnel_host', None): - self.sock = sock - self._tunnel() - # http://bugs.python.org/issue7776: Python>=3.4.1 and >=2.7.7 - # change self.host to mean the proxy server host when tunneling is - # being used. Adapt, since we are interested in the destination - # host for the match_hostname() comparison. - actual_host = self._tunnel_host - else: - actual_host = self.host - - if hasattr(ssl, 'create_default_context'): - ctx = ssl.create_default_context(cafile=self.ca_bundle) - self.sock = ctx.wrap_socket(sock, server_hostname=actual_host) - else: - # This is for python < 2.7.9 and < 3.4? - self.sock = ssl.wrap_socket( - sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_bundle - ) - try: - match_hostname(self.sock.getpeercert(), actual_host) - except CertificateError: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - raise - - -def opener_for(ca_bundle=None): - """Get a urlopen() replacement that uses ca_bundle for verification""" - return urllib.request.build_opener( - VerifyingHTTPSHandler(ca_bundle or find_ca_bundle()) - ).open - - -# from jaraco.functools -def once(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - if not hasattr(func, 'always_returns'): - func.always_returns = func(*args, **kwargs) - return func.always_returns - return wrapper - - -@once -def get_win_certfile(): - try: - import wincertstore - except ImportError: - return None - - class CertFile(wincertstore.CertFile): - def __init__(self): - super(CertFile, self).__init__() - atexit.register(self.close) - - def close(self): - try: - super(CertFile, self).close() - except OSError: - pass - - _wincerts = CertFile() - _wincerts.addstore('CA') - _wincerts.addstore('ROOT') - return _wincerts.name - - -def find_ca_bundle(): - """Return an existing CA bundle path, or None""" - extant_cert_paths = filter(os.path.isfile, cert_paths) - return ( - get_win_certfile() - or next(extant_cert_paths, None) - or _certifi_where() - ) - - -def _certifi_where(): - try: - return __import__('certifi').where() - except (ImportError, ResolutionError, ExtractionError): - pass diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index a7a2112f..564adf2b 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -3,11 +3,8 @@ import locale import pytest -__all__ = ['fail_on_ascii', 'ack_2to3'] +__all__ = ['fail_on_ascii'] is_ascii = locale.getpreferredencoding() == 'ANSI_X3.4-1968' fail_on_ascii = pytest.mark.xfail(is_ascii, reason="Test fails in this locale") - - -ack_2to3 = pytest.mark.filterwarnings('ignore:2to3 support is deprecated') diff --git a/setuptools/tests/environment.py b/setuptools/tests/environment.py index bd3119ef..c0274c33 100644 --- a/setuptools/tests/environment.py +++ b/setuptools/tests/environment.py @@ -29,7 +29,7 @@ def run_setup_py(cmd, pypath=None, path=None, if pypath is not None: env["PYTHONPATH"] = pypath - # overide the execution path if needed + # override the execution path if needed if path is not None: env["PATH"] = path if not env.get("PATH", ""): diff --git a/setuptools/tests/files.py b/setuptools/tests/files.py deleted file mode 100644 index 71194b9d..00000000 --- a/setuptools/tests/files.py +++ /dev/null @@ -1,38 +0,0 @@ -import os - - -def build_files(file_defs, prefix=""): - """ - Build a set of files/directories, as described by the - file_defs dictionary. - - Each key/value pair in the dictionary is interpreted as - a filename/contents - pair. If the contents value is a dictionary, a directory - is created, and the - dictionary interpreted as the files within it, recursively. - - For example: - - {"README.txt": "A README file", - "foo": { - "__init__.py": "", - "bar": { - "__init__.py": "", - }, - "baz.py": "# Some code", - } - } - """ - for name, contents in file_defs.items(): - full_name = os.path.join(prefix, name) - if isinstance(contents, dict): - os.makedirs(full_name, exist_ok=True) - build_files(contents, prefix=full_name) - else: - if isinstance(contents, bytes): - with open(full_name, 'wb') as f: - f.write(contents) - else: - with open(full_name, 'w') as f: - f.write(contents) diff --git a/setuptools/tests/fixtures.py b/setuptools/tests/fixtures.py index e8cb7f52..a5a172e0 100644 --- a/setuptools/tests/fixtures.py +++ b/setuptools/tests/fixtures.py @@ -1,3 +1,8 @@ +import contextlib +import sys +import shutil +import subprocess + import pytest from . import contexts @@ -21,3 +26,49 @@ def user_override(monkeypatch): def tmpdir_cwd(tmpdir): with tmpdir.as_cwd() as orig: yield orig + + +@pytest.fixture +def tmp_src(request, tmp_path): + """Make a copy of the source dir under `$tmp/src`. + + This fixture is useful whenever it's necessary to run `setup.py` + or `pip install` against the source directory when there's no + control over the number of simultaneous invocations. Such + concurrent runs create and delete directories with the same names + under the target directory and so they influence each other's runs + when they are not being executed sequentially. + """ + tmp_src_path = tmp_path / 'src' + shutil.copytree(request.config.rootdir, tmp_src_path) + return tmp_src_path + + +@pytest.fixture(autouse=True, scope="session") +def workaround_xdist_376(request): + """ + Workaround pytest-dev/pytest-xdist#376 + + ``pytest-xdist`` tends to inject '' into ``sys.path``, + which may break certain isolation expectations. + Remove the entry so the import + machinery behaves the same irrespective of xdist. + """ + if not request.config.pluginmanager.has_plugin('xdist'): + return + + with contextlib.suppress(ValueError): + sys.path.remove('') + + +@pytest.fixture +def sample_project(tmp_path): + """ + Clone the 'sampleproject' and return a path to it. + """ + cmd = ['git', 'clone', 'https://github.com/pypa/sampleproject'] + try: + subprocess.check_call(cmd, cwd=str(tmp_path)) + except Exception: + pytest.skip("Unable to clone sampleproject") + return tmp_path / 'sampleproject' diff --git a/setuptools/tests/requirements.txt b/setuptools/tests/requirements.txt index d0d07f70..b2d84a94 100644 --- a/setuptools/tests/requirements.txt +++ b/setuptools/tests/requirements.txt @@ -11,3 +11,4 @@ paver; python_version>="3.6" futures; python_version=="2.7" pip>=19.1 # For proper file:// URLs support. jaraco.envs +sphinx diff --git a/setuptools/tests/server.py b/setuptools/tests/server.py index 7e213230..6717c053 100644 --- a/setuptools/tests/server.py +++ b/setuptools/tests/server.py @@ -65,7 +65,7 @@ class MockServer(http.server.HTTPServer, threading.Thread): http.server.HTTPServer.__init__( self, server_address, RequestHandlerClass) threading.Thread.__init__(self) - self.setDaemon(True) + self.daemon = True self.requests = [] def run(self): diff --git a/setuptools/tests/test_bdist_deprecations.py b/setuptools/tests/test_bdist_deprecations.py index 704164aa..28482fd0 100644 --- a/setuptools/tests/test_bdist_deprecations.py +++ b/setuptools/tests/test_bdist_deprecations.py @@ -1,6 +1,7 @@ """develop tests """ import mock +import sys import pytest @@ -8,14 +9,17 @@ from setuptools.dist import Distribution from setuptools import SetuptoolsDeprecationWarning -@mock.patch("distutils.command.bdist_wininst.bdist_wininst") -def test_bdist_wininst_warning(distutils_cmd): - dist = Distribution(dict( - script_name='setup.py', - script_args=['bdist_wininst'], - name='foo', - py_modules=['hi'], - )) +@pytest.mark.skipif(sys.platform == 'win32', reason='non-Windows only') +@mock.patch('distutils.command.bdist_rpm.bdist_rpm') +def test_bdist_rpm_warning(distutils_cmd): + dist = Distribution( + dict( + script_name='setup.py', + script_args=['bdist_rpm'], + name='foo', + py_modules=['hi'], + ) + ) dist.parse_command_line() with pytest.warns(SetuptoolsDeprecationWarning): dist.run_commands() diff --git a/setuptools/tests/test_bdist_egg.py b/setuptools/tests/test_bdist_egg.py index 8760ea30..fb5b90b1 100644 --- a/setuptools/tests/test_bdist_egg.py +++ b/setuptools/tests/test_bdist_egg.py @@ -7,7 +7,6 @@ import zipfile import pytest from setuptools.dist import Distribution -from setuptools import SetuptoolsDeprecationWarning from . import contexts @@ -65,17 +64,3 @@ class Test: names = list(zi.filename for zi in zip.filelist) assert 'hi.pyc' in names assert 'hi.py' not in names - - def test_eggsecutable_warning(self, setup_context, user_override): - dist = Distribution(dict( - script_name='setup.py', - script_args=['bdist_egg'], - name='foo', - py_modules=['hi'], - entry_points={ - 'setuptools.installation': - ['eggsecutable = my_package.some_module:main_func']}, - )) - dist.parse_command_line() - with pytest.warns(SetuptoolsDeprecationWarning): - dist.run_commands() diff --git a/setuptools/tests/test_build_ext.py b/setuptools/tests/test_build_ext.py index 838fdb42..3177a2cd 100644 --- a/setuptools/tests/test_build_ext.py +++ b/setuptools/tests/test_build_ext.py @@ -1,16 +1,21 @@ +import os import sys import distutils.command.build_ext as orig from distutils.sysconfig import get_config_var +from jaraco import path + from setuptools.command.build_ext import build_ext, get_abi3_suffix from setuptools.dist import Distribution from setuptools.extension import Extension from . import environment -from .files import build_files from .textwrap import DALS +IS_PYPY = '__pypy__' in sys.builtin_module_names + + class TestBuildExt: def test_get_ext_filename(self): """ @@ -46,6 +51,38 @@ class TestBuildExt: else: assert 'abi3' in res + def test_ext_suffix_override(self): + """ + SETUPTOOLS_EXT_SUFFIX variable always overrides + default extension options. + """ + dist = Distribution() + cmd = build_ext(dist) + cmd.ext_map['for_abi3'] = ext = Extension( + 'for_abi3', + ['s.c'], + # Override shouldn't affect abi3 modules + py_limited_api=True, + ) + # Mock value needed to pass tests + ext._links_to_dynamic = False + + if not IS_PYPY: + expect = cmd.get_ext_filename('for_abi3') + else: + # PyPy builds do not use ABI3 tag, so they will + # also get the overridden suffix. + expect = 'for_abi3.test-suffix' + + try: + os.environ['SETUPTOOLS_EXT_SUFFIX'] = '.test-suffix' + res = cmd.get_ext_filename('normal') + assert 'normal.test-suffix' == res + res = cmd.get_ext_filename('for_abi3') + assert expect == res + finally: + del os.environ['SETUPTOOLS_EXT_SUFFIX'] + def test_build_ext_config_handling(tmpdir_cwd): files = { @@ -103,10 +140,10 @@ def test_build_ext_config_handling(tmpdir_cwd): 'setup.cfg': DALS( """ [build] - build-base = foo_build + build_base = foo_build """), } - build_files(files) + path.build(files) code, output = environment.run_setup_py( cmd=['build'], data_stream=(0, 2), ) diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 6d3a997e..0f4a1a73 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -3,15 +3,16 @@ import shutil import tarfile import importlib from concurrent import futures +import re import pytest +from jaraco import path -from .files import build_files from .textwrap import DALS class BuildBackendBase: - def __init__(self, cwd=None, env={}, backend_name='setuptools.build_meta'): + def __init__(self, cwd='.', env={}, backend_name='setuptools.build_meta'): self.cwd = cwd self.env = env self.backend_name = backend_name @@ -126,11 +127,11 @@ class TestBuildMetaBackend: backend_name = 'setuptools.build_meta' def get_build_backend(self): - return BuildBackend(cwd='.', backend_name=self.backend_name) + return BuildBackend(backend_name=self.backend_name) @pytest.fixture(params=defns) def build_backend(self, tmpdir, request): - build_files(request.param, prefix=str(tmpdir)) + path.build(request.param, prefix=str(tmpdir)) with tmpdir.as_cwd(): yield self.get_build_backend() @@ -166,11 +167,11 @@ class TestBuildMetaBackend: 'pyproject.toml': DALS(""" [build-system] requires = ["setuptools", "wheel"] - build-backend = "setuptools.build_meta + build-backend = "setuptools.build_meta" """), } - build_files(files) + path.build(files) dist_dir = os.path.abspath('preexisting-' + build_type) @@ -259,10 +260,10 @@ class TestBuildMetaBackend: 'pyproject.toml': DALS(""" [build-system] requires = ["setuptools", "wheel"] - build-backend = "setuptools.build_meta + build-backend = "setuptools.build_meta" """), } - build_files(files) + path.build(files) build_backend = self.get_build_backend() targz_path = build_backend.build_sdist("temp") with tarfile.open(os.path.join("temp", targz_path)) as tar: @@ -271,7 +272,7 @@ class TestBuildMetaBackend: def test_build_sdist_setup_py_exists(self, tmpdir_cwd): # If build_sdist is called from a script other than setup.py, # ensure setup.py is included - build_files(defns[0]) + path.build(defns[0]) build_backend = self.get_build_backend() targz_path = build_backend.build_sdist("temp") @@ -293,7 +294,7 @@ class TestBuildMetaBackend: """) } - build_files(files) + path.build(files) build_backend = self.get_build_backend() targz_path = build_backend.build_sdist("temp") @@ -315,7 +316,7 @@ class TestBuildMetaBackend: """) } - build_files(files) + path.build(files) build_backend = self.get_build_backend() build_backend.build_sdist("temp") @@ -335,9 +336,9 @@ class TestBuildMetaBackend: } def test_build_sdist_relative_path_import(self, tmpdir_cwd): - build_files(self._relative_path_import_files) + path.build(self._relative_path_import_files) build_backend = self.get_build_backend() - with pytest.raises(ImportError): + with pytest.raises(ImportError, match="^No module named 'hello'$"): build_backend.build_sdist("temp") @pytest.mark.parametrize('setup_literal, requirements', [ @@ -374,7 +375,7 @@ class TestBuildMetaBackend: """), } - build_files(files) + path.build(files) build_backend = self.get_build_backend() @@ -409,7 +410,7 @@ class TestBuildMetaBackend: """), } - build_files(files) + path.build(files) build_backend = self.get_build_backend() @@ -437,11 +438,21 @@ class TestBuildMetaBackend: } def test_sys_argv_passthrough(self, tmpdir_cwd): - build_files(self._sys_argv_0_passthrough) + path.build(self._sys_argv_0_passthrough) build_backend = self.get_build_backend() with pytest.raises(AssertionError): build_backend.build_sdist("temp") + @pytest.mark.parametrize('build_hook', ('build_sdist', 'build_wheel')) + def test_build_with_empty_setuppy(self, build_backend, build_hook): + files = {'setup.py': ''} + path.build(files) + + with pytest.raises( + ValueError, + match=re.escape('No distribution was found.')): + getattr(build_backend, build_hook)("temp") + class TestBuildMetaLegacyBackend(TestBuildMetaBackend): backend_name = 'setuptools.build_meta:__legacy__' @@ -449,13 +460,13 @@ class TestBuildMetaLegacyBackend(TestBuildMetaBackend): # build_meta_legacy-specific tests def test_build_sdist_relative_path_import(self, tmpdir_cwd): # This must fail in build_meta, but must pass in build_meta_legacy - build_files(self._relative_path_import_files) + path.build(self._relative_path_import_files) build_backend = self.get_build_backend() build_backend.build_sdist("temp") def test_sys_argv_passthrough(self, tmpdir_cwd): - build_files(self._sys_argv_0_passthrough) + path.build(self._sys_argv_0_passthrough) build_backend = self.get_build_backend() build_backend.build_sdist("temp") diff --git a/setuptools/tests/test_config.py b/setuptools/tests/test_config.py index 1dee1271..005742e4 100644 --- a/setuptools/tests/test_config.py +++ b/setuptools/tests/test_config.py @@ -1,3 +1,6 @@ +import types +import sys + import contextlib import configparser @@ -7,6 +10,7 @@ from distutils.errors import DistutilsOptionError, DistutilsFileError from mock import patch from setuptools.dist import Distribution, _Distribution from setuptools.config import ConfigHandler, read_configuration +from distutils.core import Command from .textwrap import DALS @@ -26,14 +30,11 @@ def make_package_dir(name, base_dir, ns=False): def fake_env( - tmpdir, setup_cfg, setup_py=None, - encoding='ascii', package_path='fake_package'): + tmpdir, setup_cfg, setup_py=None, encoding='ascii', package_path='fake_package' +): if setup_py is None: - setup_py = ( - 'from setuptools import setup\n' - 'setup()\n' - ) + setup_py = 'from setuptools import setup\n' 'setup()\n' tmpdir.join('setup.py').write(setup_py) config = tmpdir.join('setup.cfg') @@ -74,7 +75,6 @@ def test_parsers_implemented(): class TestConfigurationReader: - def test_basic(self, tmpdir): _, config = fake_env( tmpdir, @@ -83,7 +83,7 @@ class TestConfigurationReader: 'keywords = one, two\n' '\n' '[options]\n' - 'scripts = bin/a.py, bin/b.py\n' + 'scripts = bin/a.py, bin/b.py\n', ) config_dict = read_configuration('%s' % config) assert config_dict['metadata']['version'] == '10.1.1' @@ -97,15 +97,12 @@ class TestConfigurationReader: def test_ignore_errors(self, tmpdir): _, config = fake_env( tmpdir, - '[metadata]\n' - 'version = attr: none.VERSION\n' - 'keywords = one, two\n' + '[metadata]\n' 'version = attr: none.VERSION\n' 'keywords = one, two\n', ) with pytest.raises(ImportError): read_configuration('%s' % config) - config_dict = read_configuration( - '%s' % config, ignore_option_errors=True) + config_dict = read_configuration('%s' % config, ignore_option_errors=True) assert config_dict['metadata']['keywords'] == ['one', 'two'] assert 'version' not in config_dict['metadata'] @@ -114,7 +111,6 @@ class TestConfigurationReader: class TestMetadata: - def test_basic(self, tmpdir): fake_env( @@ -129,7 +125,7 @@ class TestMetadata: 'provides = package, package.sub\n' 'license = otherlic\n' 'download_url = http://test.test.com/test/\n' - 'maintainer_email = test@test.com\n' + 'maintainer_email = test@test.com\n', ) tmpdir.join('README').write('readme contents\nline2') @@ -156,12 +152,14 @@ class TestMetadata: def test_license_cfg(self, tmpdir): fake_env( tmpdir, - DALS(""" + DALS( + """ [metadata] name=foo version=0.0.1 license=Apache 2.0 - """) + """ + ), ) with get_dist(tmpdir) as dist: @@ -175,9 +173,7 @@ class TestMetadata: fake_env( tmpdir, - '[metadata]\n' - 'long_description = file: README.rst, CHANGES.rst\n' - '\n' + '[metadata]\n' 'long_description = file: README.rst, CHANGES.rst\n' '\n', ) tmpdir.join('README.rst').write('readme contents\nline2') @@ -185,17 +181,12 @@ class TestMetadata: with get_dist(tmpdir) as dist: assert dist.metadata.long_description == ( - 'readme contents\nline2\n' - 'changelog contents\nand stuff' + 'readme contents\nline2\n' 'changelog contents\nand stuff' ) def test_file_sandboxed(self, tmpdir): - fake_env( - tmpdir, - '[metadata]\n' - 'long_description = file: ../../README\n' - ) + fake_env(tmpdir, '[metadata]\n' 'long_description = file: ../../README\n') with get_dist(tmpdir, parse=False) as dist: with pytest.raises(DistutilsOptionError): @@ -206,13 +197,13 @@ class TestMetadata: fake_env( tmpdir, '[metadata]\n' - 'author-email = test@test.com\n' - 'home-page = http://test.test.com/test/\n' + 'author_email = test@test.com\n' + 'home_page = http://test.test.com/test/\n' 'summary = Short summary\n' 'platform = a, b\n' 'classifier =\n' ' Framework :: Django\n' - ' Programming Language :: Python :: 3.5\n' + ' Programming Language :: Python :: 3.5\n', ) with get_dist(tmpdir) as dist: @@ -237,7 +228,7 @@ class TestMetadata: ' two\n' 'classifiers =\n' ' Framework :: Django\n' - ' Programming Language :: Python :: 3.5\n' + ' Programming Language :: Python :: 3.5\n', ) with get_dist(tmpdir) as dist: metadata = dist.metadata @@ -254,7 +245,7 @@ class TestMetadata: '[metadata]\n' 'project_urls =\n' ' Link One = https://example.com/one/\n' - ' Link Two = https://example.com/two/\n' + ' Link Two = https://example.com/two/\n', ) with get_dist(tmpdir) as dist: metadata = dist.metadata @@ -266,9 +257,7 @@ class TestMetadata: def test_version(self, tmpdir): package_dir, config = fake_env( - tmpdir, - '[metadata]\n' - 'version = attr: fake_package.VERSION\n' + tmpdir, '[metadata]\n' 'version = attr: fake_package.VERSION\n' ) sub_a = package_dir.mkdir('subpkg_a') @@ -278,37 +267,28 @@ class TestMetadata: sub_b = package_dir.mkdir('subpkg_b') sub_b.join('__init__.py').write('') sub_b.join('mod.py').write( - 'import third_party_module\n' - 'VERSION = (2016, 11, 26)' + 'import third_party_module\n' 'VERSION = (2016, 11, 26)' ) with get_dist(tmpdir) as dist: assert dist.metadata.version == '1.2.3' - config.write( - '[metadata]\n' - 'version = attr: fake_package.get_version\n' - ) + config.write('[metadata]\n' 'version = attr: fake_package.get_version\n') with get_dist(tmpdir) as dist: assert dist.metadata.version == '3.4.5.dev' - config.write( - '[metadata]\n' - 'version = attr: fake_package.VERSION_MAJOR\n' - ) + config.write('[metadata]\n' 'version = attr: fake_package.VERSION_MAJOR\n') with get_dist(tmpdir) as dist: assert dist.metadata.version == '1' config.write( - '[metadata]\n' - 'version = attr: fake_package.subpkg_a.mod.VERSION\n' + '[metadata]\n' 'version = attr: fake_package.subpkg_a.mod.VERSION\n' ) with get_dist(tmpdir) as dist: assert dist.metadata.version == '2016.11.26' config.write( - '[metadata]\n' - 'version = attr: fake_package.subpkg_b.mod.VERSION\n' + '[metadata]\n' 'version = attr: fake_package.subpkg_b.mod.VERSION\n' ) with get_dist(tmpdir) as dist: assert dist.metadata.version == '2016.11.26' @@ -316,9 +296,7 @@ class TestMetadata: def test_version_file(self, tmpdir): _, config = fake_env( - tmpdir, - '[metadata]\n' - 'version = file: fake_package/version.txt\n' + tmpdir, '[metadata]\n' 'version = file: fake_package/version.txt\n' ) tmpdir.join('fake_package', 'version.txt').write('1.2.3\n') @@ -339,7 +317,7 @@ class TestMetadata: '[options]\n' 'package_dir =\n' ' = src\n', - package_path='src/fake_package_simple' + package_path='src/fake_package_simple', ) with get_dist(tmpdir) as dist: @@ -354,7 +332,7 @@ class TestMetadata: '[options]\n' 'package_dir =\n' ' fake_package_rename = fake_dir\n', - package_path='fake_dir' + package_path='fake_dir', ) with get_dist(tmpdir) as dist: @@ -369,7 +347,7 @@ class TestMetadata: '[options]\n' 'package_dir =\n' ' fake_package_complex = src/fake_dir\n', - package_path='src/fake_dir' + package_path='src/fake_dir', ) with get_dist(tmpdir) as dist: @@ -377,39 +355,28 @@ class TestMetadata: def test_unknown_meta_item(self, tmpdir): - fake_env( - tmpdir, - '[metadata]\n' - 'name = fake_name\n' - 'unknown = some\n' - ) + fake_env(tmpdir, '[metadata]\n' 'name = fake_name\n' 'unknown = some\n') with get_dist(tmpdir, parse=False) as dist: dist.parse_config_files() # Skip unknown. def test_usupported_section(self, tmpdir): - fake_env( - tmpdir, - '[metadata.some]\n' - 'key = val\n' - ) + fake_env(tmpdir, '[metadata.some]\n' 'key = val\n') with get_dist(tmpdir, parse=False) as dist: with pytest.raises(DistutilsOptionError): dist.parse_config_files() def test_classifiers(self, tmpdir): - expected = set([ - 'Framework :: Django', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - ]) + expected = set( + [ + 'Framework :: Django', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + ] + ) # From file. - _, config = fake_env( - tmpdir, - '[metadata]\n' - 'classifiers = file: classifiers\n' - ) + _, config = fake_env(tmpdir, '[metadata]\n' 'classifiers = file: classifiers\n') tmpdir.join('classifiers').write( 'Framework :: Django\n' @@ -437,7 +404,7 @@ class TestMetadata: '[metadata]\n' 'version = 10.1.1\n' 'description = Some description\n' - 'requires = some, requirement\n' + 'requires = some, requirement\n', ) with pytest.deprecated_call(): @@ -449,41 +416,26 @@ class TestMetadata: assert metadata.requires == ['some', 'requirement'] def test_interpolation(self, tmpdir): - fake_env( - tmpdir, - '[metadata]\n' - 'description = %(message)s\n' - ) + fake_env(tmpdir, '[metadata]\n' 'description = %(message)s\n') with pytest.raises(configparser.InterpolationMissingOptionError): with get_dist(tmpdir): pass def test_non_ascii_1(self, tmpdir): - fake_env( - tmpdir, - '[metadata]\n' - 'description = éàïôñ\n', - encoding='utf-8' - ) + fake_env(tmpdir, '[metadata]\n' 'description = éàïôñ\n', encoding='utf-8') with get_dist(tmpdir): pass def test_non_ascii_3(self, tmpdir): - fake_env( - tmpdir, - '\n' - '# -*- coding: invalid\n' - ) + fake_env(tmpdir, '\n' '# -*- coding: invalid\n') with get_dist(tmpdir): pass def test_non_ascii_4(self, tmpdir): fake_env( tmpdir, - '# -*- coding: utf-8\n' - '[metadata]\n' - 'description = éàïôñ\n', - encoding='utf-8' + '# -*- coding: utf-8\n' '[metadata]\n' 'description = éàïôñ\n', + encoding='utf-8', ) with get_dist(tmpdir) as dist: assert dist.metadata.description == 'éàïôñ' @@ -497,29 +449,63 @@ class TestMetadata: '# vim: set fileencoding=iso-8859-15 :\n' '[metadata]\n' 'description = éàïôñ\n', - encoding='iso-8859-15' + encoding='iso-8859-15', ) with pytest.raises(UnicodeDecodeError): with get_dist(tmpdir): pass + def test_warn_dash_deprecation(self, tmpdir): + # warn_dash_deprecation() is a method in setuptools.dist + # remove this test and the method when no longer needed + fake_env( + tmpdir, + '[metadata]\n' + 'author-email = test@test.com\n' + 'maintainer_email = foo@foo.com\n', + ) + msg = ( + "Usage of dash-separated 'author-email' will not be supported " + "in future versions. " + "Please use the underscore name 'author_email' instead" + ) + with pytest.warns(UserWarning, match=msg): + with get_dist(tmpdir) as dist: + metadata = dist.metadata + + assert metadata.author_email == 'test@test.com' + assert metadata.maintainer_email == 'foo@foo.com' -class TestOptions: + def test_make_option_lowercase(self, tmpdir): + # remove this test and the method make_option_lowercase() in setuptools.dist + # when no longer needed + fake_env( + tmpdir, '[metadata]\n' 'Name = foo\n' 'description = Some description\n' + ) + msg = ( + "Usage of uppercase key 'Name' in 'metadata' will be deprecated in " + "future versions. " + "Please use lowercase 'name' instead" + ) + with pytest.warns(UserWarning, match=msg): + with get_dist(tmpdir) as dist: + metadata = dist.metadata + + assert metadata.name == 'foo' + assert metadata.description == 'Some description' + +class TestOptions: def test_basic(self, tmpdir): fake_env( tmpdir, '[options]\n' 'zip_safe = True\n' - 'use_2to3 = 1\n' 'include_package_data = yes\n' 'package_dir = b=c, =src\n' 'packages = pack_a, pack_b.subpack\n' 'namespace_packages = pack1, pack2\n' - 'use_2to3_fixers = your.fixers, or.here\n' - 'use_2to3_exclude_fixers = one.here, two.there\n' - 'convert_2to3_doctests = src/tests/one.txt, src/two.txt\n' 'scripts = bin/one.py, bin/two.py\n' 'eager_resources = bin/one.py, bin/two.py\n' 'install_requires = docutils>=0.3; pack ==1.1, ==1.3; hey\n' @@ -528,34 +514,24 @@ class TestOptions: 'dependency_links = http://some.com/here/1, ' 'http://some.com/there/2\n' 'python_requires = >=1.0, !=2.8\n' - 'py_modules = module1, module2\n' + 'py_modules = module1, module2\n', ) with get_dist(tmpdir) as dist: assert dist.zip_safe - assert dist.use_2to3 assert dist.include_package_data assert dist.package_dir == {'': 'src', 'b': 'c'} assert dist.packages == ['pack_a', 'pack_b.subpack'] assert dist.namespace_packages == ['pack1', 'pack2'] - assert dist.use_2to3_fixers == ['your.fixers', 'or.here'] - assert dist.use_2to3_exclude_fixers == ['one.here', 'two.there'] - assert dist.convert_2to3_doctests == ([ - 'src/tests/one.txt', 'src/two.txt']) assert dist.scripts == ['bin/one.py', 'bin/two.py'] - assert dist.dependency_links == ([ - 'http://some.com/here/1', - 'http://some.com/there/2' - ]) - assert dist.install_requires == ([ - 'docutils>=0.3', - 'pack==1.1,==1.3', - 'hey' - ]) - assert dist.setup_requires == ([ - 'docutils>=0.3', - 'spack ==1.1, ==1.3', - 'there' - ]) + assert dist.dependency_links == ( + ['http://some.com/here/1', 'http://some.com/there/2'] + ) + assert dist.install_requires == ( + ['docutils>=0.3', 'pack==1.1,==1.3', 'hey'] + ) + assert dist.setup_requires == ( + ['docutils>=0.3', 'spack ==1.1, ==1.3', 'there'] + ) assert dist.tests_require == ['mock==0.7.2', 'pytest'] assert dist.python_requires == '>=1.0, !=2.8' assert dist.py_modules == ['module1', 'module2'] @@ -573,15 +549,6 @@ class TestOptions: 'namespace_packages = \n' ' pack1\n' ' pack2\n' - 'use_2to3_fixers = \n' - ' your.fixers\n' - ' or.here\n' - 'use_2to3_exclude_fixers = \n' - ' one.here\n' - ' two.there\n' - 'convert_2to3_doctests = \n' - ' src/tests/one.txt\n' - ' src/two.txt\n' 'scripts = \n' ' bin/one.py\n' ' bin/two.py\n' @@ -601,39 +568,26 @@ class TestOptions: ' there\n' 'dependency_links = \n' ' http://some.com/here/1\n' - ' http://some.com/there/2\n' + ' http://some.com/there/2\n', ) with get_dist(tmpdir) as dist: assert dist.package_dir == {'': 'src', 'b': 'c'} assert dist.packages == ['pack_a', 'pack_b.subpack'] assert dist.namespace_packages == ['pack1', 'pack2'] - assert dist.use_2to3_fixers == ['your.fixers', 'or.here'] - assert dist.use_2to3_exclude_fixers == ['one.here', 'two.there'] - assert dist.convert_2to3_doctests == ( - ['src/tests/one.txt', 'src/two.txt']) assert dist.scripts == ['bin/one.py', 'bin/two.py'] - assert dist.dependency_links == ([ - 'http://some.com/here/1', - 'http://some.com/there/2' - ]) - assert dist.install_requires == ([ - 'docutils>=0.3', - 'pack==1.1,==1.3', - 'hey' - ]) - assert dist.setup_requires == ([ - 'docutils>=0.3', - 'spack ==1.1, ==1.3', - 'there' - ]) + assert dist.dependency_links == ( + ['http://some.com/here/1', 'http://some.com/there/2'] + ) + assert dist.install_requires == ( + ['docutils>=0.3', 'pack==1.1,==1.3', 'hey'] + ) + assert dist.setup_requires == ( + ['docutils>=0.3', 'spack ==1.1, ==1.3', 'there'] + ) assert dist.tests_require == ['mock==0.7.2', 'pytest'] def test_package_dir_fail(self, tmpdir): - fake_env( - tmpdir, - '[options]\n' - 'package_dir = a b\n' - ) + fake_env(tmpdir, '[options]\n' 'package_dir = a b\n') with get_dist(tmpdir, parse=False) as dist: with pytest.raises(DistutilsOptionError): dist.parse_config_files() @@ -647,7 +601,7 @@ class TestOptions: '\n' '[options.exclude_package_data]\n' '* = fake1.txt, fake2.txt\n' - 'hello = *.dat\n' + 'hello = *.dat\n', ) with get_dist(tmpdir) as dist: @@ -661,29 +615,21 @@ class TestOptions: } def test_packages(self, tmpdir): - fake_env( - tmpdir, - '[options]\n' - 'packages = find:\n' - ) + fake_env(tmpdir, '[options]\n' 'packages = find:\n') with get_dist(tmpdir) as dist: assert dist.packages == ['fake_package'] def test_find_directive(self, tmpdir): - dir_package, config = fake_env( - tmpdir, - '[options]\n' - 'packages = find:\n' - ) + dir_package, config = fake_env(tmpdir, '[options]\n' 'packages = find:\n') dir_sub_one, _ = make_package_dir('sub_one', dir_package) dir_sub_two, _ = make_package_dir('sub_two', dir_package) with get_dist(tmpdir) as dist: - assert set(dist.packages) == set([ - 'fake_package', 'fake_package.sub_two', 'fake_package.sub_one' - ]) + assert set(dist.packages) == set( + ['fake_package', 'fake_package.sub_two', 'fake_package.sub_one'] + ) config.write( '[options]\n' @@ -707,14 +653,11 @@ class TestOptions: ' fake_package.sub_one\n' ) with get_dist(tmpdir) as dist: - assert set(dist.packages) == set( - ['fake_package', 'fake_package.sub_two']) + assert set(dist.packages) == set(['fake_package', 'fake_package.sub_two']) def test_find_namespace_directive(self, tmpdir): dir_package, config = fake_env( - tmpdir, - '[options]\n' - 'packages = find_namespace:\n' + tmpdir, '[options]\n' 'packages = find_namespace:\n' ) dir_sub_one, _ = make_package_dir('sub_one', dir_package) @@ -722,7 +665,9 @@ class TestOptions: with get_dist(tmpdir) as dist: assert set(dist.packages) == { - 'fake_package', 'fake_package.sub_two', 'fake_package.sub_one' + 'fake_package', + 'fake_package.sub_two', + 'fake_package.sub_one', } config.write( @@ -747,9 +692,7 @@ class TestOptions: ' fake_package.sub_one\n' ) with get_dist(tmpdir) as dist: - assert set(dist.packages) == { - 'fake_package', 'fake_package.sub_two' - } + assert set(dist.packages) == {'fake_package', 'fake_package.sub_two'} def test_extras_require(self, tmpdir): fake_env( @@ -758,23 +701,29 @@ class TestOptions: 'pdf = ReportLab>=1.2; RXP\n' 'rest = \n' ' docutils>=0.3\n' - ' pack ==1.1, ==1.3\n' + ' pack ==1.1, ==1.3\n', ) with get_dist(tmpdir) as dist: assert dist.extras_require == { 'pdf': ['ReportLab>=1.2', 'RXP'], - 'rest': ['docutils>=0.3', 'pack==1.1,==1.3'] + 'rest': ['docutils>=0.3', 'pack==1.1,==1.3'], } assert dist.metadata.provides_extras == set(['pdf', 'rest']) + def test_dash_preserved_extras_require(self, tmpdir): + fake_env(tmpdir, '[options.extras_require]\n' 'foo-a = foo\n' 'foo_b = test\n') + + with get_dist(tmpdir) as dist: + assert dist.extras_require == {'foo-a': ['foo'], 'foo_b': ['test']} + def test_entry_points(self, tmpdir): _, config = fake_env( tmpdir, '[options.entry_points]\n' 'group1 = point1 = pack.module:func, ' '.point2 = pack.module2:func_rest [rest]\n' - 'group2 = point3 = pack.module:func2\n' + 'group2 = point3 = pack.module:func2\n', ) with get_dist(tmpdir) as dist: @@ -783,7 +732,7 @@ class TestOptions: 'point1 = pack.module:func', '.point2 = pack.module2:func_rest [rest]', ], - 'group2': ['point3 = pack.module:func2'] + 'group2': ['point3 = pack.module:func2'], } expected = ( @@ -794,14 +743,29 @@ class TestOptions: tmpdir.join('entry_points').write(expected) # From file. - config.write( - '[options]\n' - 'entry_points = file: entry_points\n' - ) + config.write('[options]\n' 'entry_points = file: entry_points\n') with get_dist(tmpdir) as dist: assert dist.entry_points == expected + def test_case_sensitive_entry_points(self, tmpdir): + _, config = fake_env( + tmpdir, + '[options.entry_points]\n' + 'GROUP1 = point1 = pack.module:func, ' + '.point2 = pack.module2:func_rest [rest]\n' + 'group2 = point3 = pack.module:func2\n', + ) + + with get_dist(tmpdir) as dist: + assert dist.entry_points == { + 'GROUP1': [ + 'point1 = pack.module:func', + '.point2 = pack.module2:func_rest [rest]', + ], + 'group2': ['point3 = pack.module:func2'], + } + def test_data_files(self, tmpdir): fake_env( tmpdir, @@ -809,7 +773,7 @@ class TestOptions: 'cfg =\n' ' a/b.conf\n' ' c/d.conf\n' - 'data = e/f.dat, g/h.dat\n' + 'data = e/f.dat, g/h.dat\n', ) with get_dist(tmpdir) as dist: @@ -819,13 +783,50 @@ class TestOptions: ] assert sorted(dist.data_files) == sorted(expected) + def test_data_files_globby(self, tmpdir): + fake_env( + tmpdir, + '[options.data_files]\n' + 'cfg =\n' + ' a/b.conf\n' + ' c/d.conf\n' + 'data = *.dat\n' + 'icons = \n' + ' *.ico\n' + 'audio = \n' + ' *.wav\n' + ' sounds.db\n' + ) + + # Create dummy files for glob()'s sake: + tmpdir.join('a.dat').write('') + tmpdir.join('b.dat').write('') + tmpdir.join('c.dat').write('') + tmpdir.join('a.ico').write('') + tmpdir.join('b.ico').write('') + tmpdir.join('c.ico').write('') + tmpdir.join('beep.wav').write('') + tmpdir.join('boop.wav').write('') + tmpdir.join('sounds.db').write('') + + with get_dist(tmpdir) as dist: + expected = [ + ('cfg', ['a/b.conf', 'c/d.conf']), + ('data', ['a.dat', 'b.dat', 'c.dat']), + ('icons', ['a.ico', 'b.ico', 'c.ico']), + ('audio', ['beep.wav', 'boop.wav', 'sounds.db']), + ] + assert sorted(dist.data_files) == sorted(expected) + def test_python_requires_simple(self, tmpdir): fake_env( tmpdir, - DALS(""" + DALS( + """ [options] python_requires=>=2.7 - """), + """ + ), ) with get_dist(tmpdir) as dist: dist.parse_config_files() @@ -833,10 +834,12 @@ class TestOptions: def test_python_requires_compound(self, tmpdir): fake_env( tmpdir, - DALS(""" + DALS( + """ [options] python_requires=>=2.7,!=3.0.* - """), + """ + ), ) with get_dist(tmpdir) as dist: dist.parse_config_files() @@ -844,15 +847,35 @@ class TestOptions: def test_python_requires_invalid(self, tmpdir): fake_env( tmpdir, - DALS(""" + DALS( + """ [options] python_requires=invalid - """), + """ + ), ) with pytest.raises(Exception): with get_dist(tmpdir) as dist: dist.parse_config_files() + def test_cmdclass(self, tmpdir): + class CustomCmd(Command): + pass + + m = types.ModuleType('custom_build', 'test package') + + m.__dict__['CustomCmd'] = CustomCmd + + sys.modules['custom_build'] = m + + fake_env( + tmpdir, + '[options]\n' 'cmdclass =\n' ' customcmd = custom_build.CustomCmd\n', + ) + + with get_dist(tmpdir) as dist: + assert dist.cmdclass == {'customcmd': CustomCmd} + saved_dist_init = _Distribution.__init__ @@ -871,24 +894,23 @@ class TestExternalSetters: def _fake_distribution_init(self, dist, attrs): saved_dist_init(dist, attrs) # see self._DISTUTUILS_UNSUPPORTED_METADATA - setattr(dist.metadata, 'long_description_content_type', - 'text/something') + setattr(dist.metadata, 'long_description_content_type', 'text/something') # Test overwrite setup() args - setattr(dist.metadata, 'project_urls', { - 'Link One': 'https://example.com/one/', - 'Link Two': 'https://example.com/two/', - }) + setattr( + dist.metadata, + 'project_urls', + { + 'Link One': 'https://example.com/one/', + 'Link Two': 'https://example.com/two/', + }, + ) return None @patch.object(_Distribution, '__init__', autospec=True) def test_external_setters(self, mock_parent_init, tmpdir): mock_parent_init.side_effect = self._fake_distribution_init - dist = Distribution(attrs={ - 'project_urls': { - 'will_be': 'ignored' - } - }) + dist = Distribution(attrs={'project_urls': {'will_be': 'ignored'}}) assert dist.metadata.long_description_content_type == 'text/something' assert dist.metadata.project_urls == { diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index 2766da2f..70c5794c 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -2,11 +2,11 @@ """ import os -import site import sys -import io import subprocess import platform +import pathlib +import textwrap from setuptools.command import test @@ -14,7 +14,6 @@ import pytest from setuptools.command.develop import develop from setuptools.dist import Distribution -from setuptools.tests import ack_2to3 from . import contexts from . import namespaces @@ -23,7 +22,6 @@ from setuptools import setup setup(name='foo', packages=['foo'], - use_2to3=True, ) """ @@ -60,43 +58,6 @@ class TestDevelop: in_virtualenv = hasattr(sys, 'real_prefix') in_venv = hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix - @pytest.mark.skipif( - in_virtualenv or in_venv, - reason="Cannot run when invoked in a virtualenv or venv") - @ack_2to3 - def test_2to3_user_mode(self, test_env): - settings = dict( - name='foo', - packages=['foo'], - use_2to3=True, - version='0.0', - ) - dist = Distribution(settings) - dist.script_name = 'setup.py' - cmd = develop(dist) - cmd.user = 1 - cmd.ensure_finalized() - cmd.install_dir = site.USER_SITE - cmd.user = 1 - with contexts.quiet(): - cmd.run() - - # let's see if we got our egg link at the right place - content = os.listdir(site.USER_SITE) - content.sort() - assert content == ['easy-install.pth', 'foo.egg-link'] - - # Check that we are using the right code. - fn = os.path.join(site.USER_SITE, 'foo.egg-link') - with io.open(fn) as egg_link_file: - path = egg_link_file.read().split()[0].strip() - fn = os.path.join(path, 'foo', '__init__.py') - with io.open(fn) as init_file: - init = init_file.read().strip() - - expected = 'print("foo")' - assert init == expected - def test_console_scripts(self, tmpdir): """ Test that console scripts are installed and that they reference @@ -104,7 +65,8 @@ class TestDevelop: """ pytest.skip( "TODO: needs a fixture to cause 'develop' " - "to be invoked without mutating environment.") + "to be invoked without mutating environment." + ) settings = dict( name='foo', packages=['foo'], @@ -130,6 +92,7 @@ class TestResolver: of what _resolve_setup_path is intending to do. Come up with more meaningful cases that look like real-world scenarios. """ + def test_resolve_setup_path_cwd(self): assert develop._resolve_setup_path('.', '.', '.') == '.' @@ -141,7 +104,6 @@ class TestResolver: class TestNamespaces: - @staticmethod def install_develop(src_dir, target): @@ -149,7 +111,8 @@ class TestNamespaces: sys.executable, 'setup.py', 'develop', - '--install-dir', str(target), + '--install-dir', + str(target), ] with src_dir.as_cwd(): with test.test.paths_on_pythonpath([str(target)]): @@ -180,14 +143,16 @@ class TestNamespaces: 'pip', 'install', str(pkg_A), - '-t', str(target), + '-t', + str(target), ] subprocess.check_call(install_cmd) self.install_develop(pkg_B, target) namespaces.make_site_dir(target) try_import = [ sys.executable, - '-c', 'import myns.pkgA; import myns.pkgB', + '-c', + 'import myns.pkgA; import myns.pkgB', ] with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(try_import) @@ -195,7 +160,65 @@ class TestNamespaces: # additionally ensure that pkg_resources import works pkg_resources_imp = [ sys.executable, - '-c', 'import pkg_resources', + '-c', + 'import pkg_resources', ] with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(pkg_resources_imp) + + @staticmethod + def install_workaround(site_packages): + site_packages.mkdir(parents=True) + sc = site_packages / 'sitecustomize.py' + sc.write_text( + textwrap.dedent( + """ + import site + import pathlib + here = pathlib.Path(__file__).parent + site.addsitedir(str(here)) + """ + ).lstrip() + ) + + @pytest.mark.xfail( + platform.python_implementation() == 'PyPy', + reason="Workaround fails on PyPy (why?)", + ) + def test_editable_prefix(self, tmp_path, sample_project): + """ + Editable install to a prefix should be discoverable. + """ + prefix = tmp_path / 'prefix' + prefix.mkdir() + + # figure out where pip will likely install the package + site_packages = prefix / next( + pathlib.Path(path).relative_to(sys.prefix) + for path in sys.path + if 'site-packages' in path and path.startswith(sys.prefix) + ) + + # install the workaround + self.install_workaround(site_packages) + + env = dict(os.environ, PYTHONPATH=str(site_packages)) + cmd = [ + sys.executable, + '-m', + 'pip', + 'install', + '--editable', + str(sample_project), + '--prefix', + str(prefix), + '--no-build-isolation', + ] + subprocess.check_call(cmd, env=env) + + # now run 'sample' with the prefix on the PYTHONPATH + bin = 'Scripts' if platform.system() == 'Windows' else 'bin' + exe = prefix / bin / 'sample' + if sys.version_info < (3, 7) and platform.system() == 'Windows': + exe = str(exe) + subprocess.check_call([exe], env=env) diff --git a/setuptools/tests/test_dist.py b/setuptools/tests/test_dist.py index cb47fb58..c4279f0b 100644 --- a/setuptools/tests/test_dist.py +++ b/setuptools/tests/test_dist.py @@ -9,6 +9,9 @@ from setuptools.dist import ( _get_unpatched, check_package_data, DistDeprecationWarning, + check_specifier, + rfc822_escape, + rfc822_unescape, ) from setuptools import sic from setuptools import Distribution @@ -81,11 +84,8 @@ def __read_test_cases(): test_cases = [ ('Metadata version 1.0', params()), - ('Metadata version 1.1: Provides', params( - provides=['package'], - )), - ('Metadata version 1.1: Obsoletes', params( - obsoletes=['foo'], + ('Metadata Version 1.0: Short long description', params( + long_description='Short long description', )), ('Metadata version 1.1: Classifiers', params( classifiers=[ @@ -110,6 +110,10 @@ def __read_test_cases(): ('Metadata Version 2.1: Long Description Content Type', params( long_description_content_type='text/x-rst; charset=UTF-8', )), + ('License', params(license='MIT', )), + ('License multiline', params( + license='This is a long license \nover multiple lines', + )), pytest.param( 'Metadata Version 2.1: Provides Extra', params(provides_extras=['foo', 'bar']), @@ -161,6 +165,7 @@ def test_read_metadata(name, attrs): ('metadata_version', dist_class.get_metadata_version), ('provides', dist_class.get_provides), ('description', dist_class.get_description), + ('long_description', dist_class.get_long_description), ('download_url', dist_class.get_download_url), ('keywords', dist_class.get_keywords), ('platforms', dist_class.get_platforms), @@ -246,8 +251,8 @@ def test_maintainer_author(name, attrs, tmpdir): with io.open(str(fn.join('PKG-INFO')), 'r', encoding='utf-8') as f: raw_pkg_lines = f.readlines() - # Drop blank lines - pkg_lines = list(filter(None, raw_pkg_lines)) + # Drop blank lines and strip lines from default description + pkg_lines = list(filter(None, raw_pkg_lines[:-2])) pkg_lines_set = set(pkg_lines) @@ -323,3 +328,49 @@ def test_check_package_data(package_data, expected_message): with pytest.raises( DistutilsSetupError, match=re.escape(expected_message)): check_package_data(None, str('package_data'), package_data) + + +def test_check_specifier(): + # valid specifier value + attrs = {'name': 'foo', 'python_requires': '>=3.0, !=3.1'} + dist = Distribution(attrs) + check_specifier(dist, attrs, attrs['python_requires']) + + # invalid specifier value + attrs = {'name': 'foo', 'python_requires': ['>=3.0', '!=3.1']} + with pytest.raises(DistutilsSetupError): + dist = Distribution(attrs) + + +@pytest.mark.parametrize( + 'content, result', + ( + pytest.param( + "Just a single line", + None, + id="single_line", + ), + pytest.param( + "Multiline\nText\nwithout\nextra indents\n", + None, + id="multiline", + ), + pytest.param( + "Multiline\n With\n\nadditional\n indentation", + None, + id="multiline_with_indentation", + ), + pytest.param( + " Leading whitespace", + "Leading whitespace", + id="remove_leading_whitespace", + ), + pytest.param( + " Leading whitespace\nIn\n Multiline comment", + "Leading whitespace\nIn\n Multiline comment", + id="remove_leading_whitespace_multiline", + ), + ) +) +def test_rfc822_unescape(content, result): + assert (result or content) == rfc822_unescape(rfc822_escape(content)) diff --git a/setuptools/tests/test_distutils_adoption.py b/setuptools/tests/test_distutils_adoption.py index a53773df..0e89921c 100644 --- a/setuptools/tests/test_distutils_adoption.py +++ b/setuptools/tests/test_distutils_adoption.py @@ -21,10 +21,10 @@ class VirtualEnv(jaraco.envs.VirtualEnv): @pytest.fixture -def venv(tmpdir): +def venv(tmp_path, tmp_src): env = VirtualEnv() - env.root = path.Path(tmpdir) - env.req = os.getcwd() + env.root = path.Path(tmp_path / 'venv') + env.req = str(tmp_src) return env.create() diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index dc00e697..6840d03b 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -15,8 +15,11 @@ import zipfile import mock import time import re +import subprocess +import pathlib import pytest +from jaraco import path from setuptools import sandbox from setuptools.sandbox import run_setup @@ -25,7 +28,6 @@ from setuptools.command.easy_install import ( EasyInstallDeprecationWarning, ScriptWriter, PthDistributions, WindowsScriptWriter, ) -from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution from pkg_resources import normalize_path, working_set from pkg_resources import Distribution as PRDistribution @@ -34,10 +36,19 @@ from setuptools.tests import fail_on_ascii import pkg_resources from . import contexts -from .files import build_files from .textwrap import DALS +@pytest.fixture(autouse=True) +def pip_disable_index(monkeypatch): + """ + Important: Disable the default index for pip to avoid + querying packages in the index and potentially resolving + and installing packages there. + """ + monkeypatch.setenv('PIP_NO_INDEX', 'true') + + class FakeDist: def get_entry_map(self, group): if group != 'console_scripts': @@ -445,22 +456,22 @@ class TestSetupRequires: """ monkeypatch.setenv(str('PIP_RETRIES'), str('0')) monkeypatch.setenv(str('PIP_TIMEOUT'), str('0')) + monkeypatch.setenv('PIP_NO_INDEX', 'false') with contexts.quiet(): # create an sdist that has a build-time dependency. with TestSetupRequires.create_sdist() as dist_file: with contexts.tempdir() as temp_install_dir: with contexts.environment(PYTHONPATH=temp_install_dir): - ei_params = [ + cmd = [ + sys.executable, + '-m', 'setup', + 'easy_install', '--index-url', mock_index.url, '--exclude-scripts', '--install-dir', temp_install_dir, dist_file, ] - with sandbox.save_argv(['easy_install']): - # attempt to install the dist. It should - # fail because it doesn't exist. - with pytest.raises(SystemExit): - easy_install_pkg.main(ei_params) + subprocess.Popen(cmd).wait() # there should have been one requests to the server assert [r.path for r in mock_index.requests] == ['/does-not-exist/'] @@ -618,6 +629,7 @@ class TestSetupRequires: def test_setup_requires_honors_pip_env(self, mock_index, monkeypatch): monkeypatch.setenv(str('PIP_RETRIES'), str('0')) monkeypatch.setenv(str('PIP_TIMEOUT'), str('0')) + monkeypatch.setenv('PIP_NO_INDEX', 'false') monkeypatch.setenv(str('PIP_INDEX_URL'), mock_index.url) with contexts.save_pkg_resources_state(): with contexts.tempdir() as temp_dir: @@ -648,7 +660,7 @@ class TestSetupRequires: dep_url = path_to_url(dep_sdist, authority='localhost') test_pkg = create_setup_requires_package( temp_dir, - # Ignored (overriden by setup_attrs) + # Ignored (overridden by setup_attrs) 'python-xlib', '0.19', setup_attrs=dict( setup_requires='dependency @ %s' % dep_url)) @@ -658,26 +670,24 @@ class TestSetupRequires: def test_setup_requires_with_allow_hosts(self, mock_index): ''' The `allow-hosts` option in not supported anymore. ''' + files = { + 'test_pkg': { + 'setup.py': DALS(''' + from setuptools import setup + setup(setup_requires='python-xlib') + '''), + 'setup.cfg': DALS(''' + [easy_install] + allow_hosts = * + '''), + } + } with contexts.save_pkg_resources_state(): with contexts.tempdir() as temp_dir: - test_pkg = os.path.join(temp_dir, 'test_pkg') - test_setup_py = os.path.join(test_pkg, 'setup.py') - test_setup_cfg = os.path.join(test_pkg, 'setup.cfg') - os.mkdir(test_pkg) - with open(test_setup_py, 'w') as fp: - fp.write(DALS( - ''' - from setuptools import setup - setup(setup_requires='python-xlib') - ''')) - with open(test_setup_cfg, 'w') as fp: - fp.write(DALS( - ''' - [easy_install] - allow_hosts = * - ''')) + path.build(files, prefix=temp_dir) + setup_py = str(pathlib.Path(temp_dir, 'test_pkg', 'setup.py')) with pytest.raises(distutils.errors.DistutilsError): - run_setup(test_setup_py, [str('--version')]) + run_setup(setup_py, [str('--version')]) assert len(mock_index.requests) == 0 def test_setup_requires_with_python_requires(self, monkeypatch, tmpdir): @@ -720,7 +730,7 @@ class TestSetupRequires: with contexts.save_pkg_resources_state(): test_pkg = create_setup_requires_package( str(tmpdir), - 'python-xlib', '0.19', # Ignored (overriden by setup_attrs). + 'python-xlib', '0.19', # Ignored (overridden by setup_attrs). setup_attrs=dict( setup_requires='dep', dependency_links=[index_url])) test_setup_py = os.path.join(test_pkg, 'setup.py') @@ -730,10 +740,10 @@ class TestSetupRequires: assert eggs == ['dep 1.0'] @pytest.mark.parametrize( - 'use_legacy_installer,with_dependency_links_in_setup_py', - itertools.product((False, True), (False, True))) + 'with_dependency_links_in_setup_py', + (False, True)) def test_setup_requires_with_find_links_in_setup_cfg( - self, monkeypatch, use_legacy_installer, + self, monkeypatch, with_dependency_links_in_setup_py): monkeypatch.setenv(str('PIP_RETRIES'), str('0')) monkeypatch.setenv(str('PIP_TIMEOUT'), str('0')) @@ -755,11 +765,9 @@ class TestSetupRequires: fp.write(DALS( ''' from setuptools import installer, setup - if {use_legacy_installer}: - installer.fetch_build_egg = installer._legacy_fetch_build_egg setup(setup_requires='python-xlib==42', dependency_links={dependency_links!r}) - ''').format(use_legacy_installer=use_legacy_installer, # noqa + ''').format( dependency_links=dependency_links)) with open(test_setup_cfg, 'w') as fp: fp.write(DALS( @@ -785,7 +793,7 @@ class TestSetupRequires: # Create source tree for `dep`. dep_pkg = os.path.join(temp_dir, 'dep') os.mkdir(dep_pkg) - build_files({ + path.build({ 'setup.py': DALS(""" import setuptools diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index 1047468b..ee07b5a1 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -5,16 +5,17 @@ import glob import re import stat import time +from typing import List, Tuple + +import pytest +from jaraco import path from setuptools.command.egg_info import ( egg_info, manifest_maker, EggInfoDeprecationWarning, get_pkg_info_revision, ) from setuptools.dist import Distribution -import pytest - from . import environment -from .files import build_files from .textwrap import DALS from . import contexts @@ -37,7 +38,7 @@ class TestEggInfo: """) def _create_project(self): - build_files({ + path.build({ 'setup.py': self.setup_script, 'hello.py': DALS(""" def run(): @@ -45,6 +46,11 @@ class TestEggInfo: """) }) + @staticmethod + def _extract_mv_version(pkg_info_lines: List[str]) -> Tuple[int, int]: + version_str = pkg_info_lines[0].split(' ')[1] + return tuple(map(int, version_str.split('.')[:2])) + @pytest.fixture def env(self): with contexts.tempdir(prefix='setuptools-test.') as env_dir: @@ -56,7 +62,7 @@ class TestEggInfo: for dirname in subs ) list(map(os.mkdir, env.paths.values())) - build_files({ + path.build({ env.paths['home']: { '.pydistutils.cfg': DALS(""" [egg_info] @@ -106,7 +112,7 @@ class TestEggInfo: the file should remain unchanged. """ setup_cfg = os.path.join(env.paths['home'], 'setup.cfg') - build_files({ + path.build({ setup_cfg: DALS(""" [egg_info] tag_build = @@ -159,8 +165,10 @@ class TestEggInfo: setup() """) - build_files({'setup.py': setup_script, - 'setup.cfg': setup_config}) + path.build({ + 'setup.py': setup_script, + 'setup.cfg': setup_config, + }) # This command should fail with a ValueError, but because it's # currently configured to use a subprocess, the actual traceback @@ -193,7 +201,7 @@ class TestEggInfo: def test_manifest_template_is_read(self, tmpdir_cwd, env): self._create_project() - build_files({ + path.build({ 'MANIFEST.in': DALS(""" recursive-include docs *.rst """), @@ -216,8 +224,10 @@ class TestEggInfo: ''' ) % ('' if use_setup_cfg else requires) setup_config = requires if use_setup_cfg else '' - build_files({'setup.py': setup_script, - 'setup.cfg': setup_config}) + path.build({ + 'setup.py': setup_script, + 'setup.cfg': setup_config, + }) mismatch_marker = "python_version<'{this_ver}'".format( this_ver=sys.version_info[0], @@ -533,7 +543,7 @@ class TestEggInfo: 'setup.cfg': DALS(""" """), 'LICENSE': "Test license" - }, False), # no license_file attribute + }, True), # no license_file attribute, LICENSE auto-included ({ 'setup.cfg': DALS(""" [metadata] @@ -541,12 +551,20 @@ class TestEggInfo: """), 'MANIFEST.in': "exclude LICENSE", 'LICENSE': "Test license" - }, False) # license file is manually excluded + }, True), # manifest is overwritten by license_file + pytest.param({ + 'setup.cfg': DALS(""" + [metadata] + license_file = LICEN[CS]E* + """), + 'LICENSE': "Test license", + }, True, + id="glob_pattern"), ]) def test_setup_cfg_license_file( self, tmpdir_cwd, env, files, license_in_sources): self._create_project() - build_files(files) + path.build(files) environment.run_setup_py( cmd=['egg_info'], @@ -621,7 +639,7 @@ class TestEggInfo: 'setup.cfg': DALS(""" """), 'LICENSE': "Test license" - }, [], ['LICENSE']), # no license_files attribute + }, ['LICENSE'], []), # no license_files attribute, LICENSE auto-included ({ 'setup.cfg': DALS(""" [metadata] @@ -629,7 +647,7 @@ class TestEggInfo: """), 'MANIFEST.in': "exclude LICENSE", 'LICENSE': "Test license" - }, [], ['LICENSE']), # license file is manually excluded + }, ['LICENSE'], []), # manifest is overwritten by license_files ({ 'setup.cfg': DALS(""" [metadata] @@ -640,12 +658,53 @@ class TestEggInfo: 'MANIFEST.in': "exclude LICENSE-XYZ", 'LICENSE-ABC': "ABC license", 'LICENSE-XYZ': "XYZ license" - }, ['LICENSE-ABC'], ['LICENSE-XYZ']) # subset is manually excluded + # manifest is overwritten by license_files + }, ['LICENSE-ABC', 'LICENSE-XYZ'], []), + pytest.param({ + 'setup.cfg': "", + 'LICENSE-ABC': "ABC license", + 'COPYING-ABC': "ABC copying", + 'NOTICE-ABC': "ABC notice", + 'AUTHORS-ABC': "ABC authors", + 'LICENCE-XYZ': "XYZ license", + 'LICENSE': "License", + 'INVALID-LICENSE': "Invalid license", + }, [ + 'LICENSE-ABC', + 'COPYING-ABC', + 'NOTICE-ABC', + 'AUTHORS-ABC', + 'LICENCE-XYZ', + 'LICENSE', + ], ['INVALID-LICENSE'], + # ('LICEN[CS]E*', 'COPYING*', 'NOTICE*', 'AUTHORS*') + id="default_glob_patterns"), + pytest.param({ + 'setup.cfg': DALS(""" + [metadata] + license_files = + LICENSE* + """), + 'LICENSE-ABC': "ABC license", + 'NOTICE-XYZ': "XYZ notice", + }, ['LICENSE-ABC'], ['NOTICE-XYZ'], + id="no_default_glob_patterns"), + pytest.param({ + 'setup.cfg': DALS(""" + [metadata] + license_files = + LICENSE-ABC + LICENSE* + """), + 'LICENSE-ABC': "ABC license", + }, ['LICENSE-ABC'], [], + id="files_only_added_once", + ), ]) def test_setup_cfg_license_files( self, tmpdir_cwd, env, files, incl_licenses, excl_licenses): self._create_project() - build_files(files) + path.build(files) environment.run_setup_py( cmd=['egg_info'], @@ -744,13 +803,34 @@ class TestEggInfo: 'LICENSE-ABC': "ABC license", 'LICENSE-PQR': "PQR license", 'LICENSE-XYZ': "XYZ license" - # manually excluded - }, ['LICENSE-XYZ'], ['LICENSE-ABC', 'LICENSE-PQR']) + # manifest is overwritten + }, ['LICENSE-ABC', 'LICENSE-PQR', 'LICENSE-XYZ'], []), + pytest.param({ + 'setup.cfg': DALS(""" + [metadata] + license_file = LICENSE* + """), + 'LICENSE-ABC': "ABC license", + 'NOTICE-XYZ': "XYZ notice", + }, ['LICENSE-ABC'], ['NOTICE-XYZ'], + id="no_default_glob_patterns"), + pytest.param({ + 'setup.cfg': DALS(""" + [metadata] + license_file = LICENSE* + license_files = + NOTICE* + """), + 'LICENSE-ABC': "ABC license", + 'NOTICE-ABC': "ABC notice", + 'AUTHORS-ABC': "ABC authors", + }, ['LICENSE-ABC', 'NOTICE-ABC'], ['AUTHORS-ABC'], + id="combined_glob_patterrns"), ]) def test_setup_cfg_license_file_license_files( self, tmpdir_cwd, env, files, incl_licenses, excl_licenses): self._create_project() - build_files(files) + path.build(files) environment.run_setup_py( cmd=['egg_info'], @@ -767,6 +847,52 @@ class TestEggInfo: for lf in excl_licenses: assert sources_lines.count(lf) == 0 + def test_license_file_attr_pkg_info(self, tmpdir_cwd, env): + """All matched license files should have a corresponding License-File.""" + self._create_project() + path.build({ + "setup.cfg": DALS(""" + [metadata] + license_files = + NOTICE* + LICENSE* + """), + "LICENSE-ABC": "ABC license", + "LICENSE-XYZ": "XYZ license", + "NOTICE": "included", + "IGNORE": "not include", + }) + + environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]) + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + license_file_lines = [ + line for line in pkg_info_lines if line.startswith('License-File:')] + + # Only 'NOTICE', LICENSE-ABC', and 'LICENSE-XYZ' should have been matched + # Also assert that order from license_files is keeped + assert "License-File: NOTICE" == license_file_lines[0] + assert "License-File: LICENSE-ABC" in license_file_lines[1:] + assert "License-File: LICENSE-XYZ" in license_file_lines[1:] + + def test_metadata_version(self, tmpdir_cwd, env): + """Make sure latest metadata version is used by default.""" + self._setup_script_with_requires("") + code, data = environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + # Update metadata version if changed + assert self._extract_mv_version(pkg_info_lines) == (2, 1) + def test_long_description_content_type(self, tmpdir_cwd, env): # Test that specifying a `long_description_content_type` keyword arg to # the `setup` function results in writing a `Description-Content-Type` @@ -793,6 +919,29 @@ class TestEggInfo: assert expected_line in pkg_info_lines assert 'Metadata-Version: 2.1' in pkg_info_lines + def test_long_description(self, tmpdir_cwd, env): + # Test that specifying `long_description` and `long_description_content_type` + # keyword args to the `setup` function results in writing + # the description in the message payload of the `PKG-INFO` file + # in the `<distribution>.egg-info` directory. + self._setup_script_with_requires( + "long_description='This is a long description\\nover multiple lines'," + "long_description_content_type='text/markdown'," + ) + code, data = environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + assert 'Metadata-Version: 2.1' in pkg_info_lines + assert '' == pkg_info_lines[-1] # last line should be empty + long_desc_lines = pkg_info_lines[pkg_info_lines.index(''):] + assert 'This is a long description' in long_desc_lines + assert 'over multiple lines' in long_desc_lines + def test_project_urls(self, tmpdir_cwd, env): # Test that specifying a `project_urls` dict to the `setup` # function results in writing multiple `Project-URL` lines to @@ -822,7 +971,40 @@ class TestEggInfo: assert expected_line in pkg_info_lines expected_line = 'Project-URL: Link Two, https://example.com/two/' assert expected_line in pkg_info_lines - assert 'Metadata-Version: 1.2' in pkg_info_lines + assert self._extract_mv_version(pkg_info_lines) >= (1, 2) + + def test_license(self, tmpdir_cwd, env): + """Test single line license.""" + self._setup_script_with_requires( + "license='MIT'," + ) + code, data = environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + assert 'License: MIT' in pkg_info_lines + + def test_license_escape(self, tmpdir_cwd, env): + """Test license is escaped correctly if longer than one line.""" + self._setup_script_with_requires( + "license='This is a long license text \\nover multiple lines'," + ) + code, data = environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + + assert 'License: This is a long license text ' in pkg_info_lines + assert ' over multiple lines' in pkg_info_lines + assert 'text \n over multiple' in '\n'.join(pkg_info_lines) def test_python_requires_egg_info(self, tmpdir_cwd, env): self._setup_script_with_requires( @@ -840,7 +1022,7 @@ class TestEggInfo: with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: pkg_info_lines = pkginfo_file.read().split('\n') assert 'Requires-Python: >=2.7.12' in pkg_info_lines - assert 'Metadata-Version: 1.2' in pkg_info_lines + assert self._extract_mv_version(pkg_info_lines) >= (1, 2) def test_manifest_maker_warning_suppression(self): fixtures = [ @@ -886,7 +1068,7 @@ class TestEggInfo: def test_egg_info_tag_only_once(self, tmpdir_cwd, env): self._create_project() - build_files({ + path.build({ 'setup.cfg': DALS(""" [egg_info] tag_build = dev diff --git a/setuptools/tests/test_glob.py b/setuptools/tests/test_glob.py index a0728c5d..e99587f5 100644 --- a/setuptools/tests/test_glob.py +++ b/setuptools/tests/test_glob.py @@ -1,9 +1,8 @@ import pytest +from jaraco import path from setuptools.glob import glob -from .files import build_files - @pytest.mark.parametrize('tree, pattern, matches', ( ('', b'', []), @@ -31,5 +30,5 @@ from .files import build_files )) def test_glob(monkeypatch, tmpdir, tree, pattern, matches): monkeypatch.chdir(tmpdir) - build_files({name: '' for name in tree.split()}) + path.build({name: '' for name in tree.split()}) assert list(sorted(glob(pattern))) == list(sorted(matches)) diff --git a/setuptools/tests/test_namespaces.py b/setuptools/tests/test_namespaces.py index 6c8c522d..270f90c9 100644 --- a/setuptools/tests/test_namespaces.py +++ b/setuptools/tests/test_namespaces.py @@ -62,8 +62,9 @@ class TestNamespaces: target.mkdir() install_cmd = [ sys.executable, - '-m', 'easy_install', - '-d', str(target), + '-m', 'pip', + 'install', + '-t', str(target), str(pkg), ] with test.test.paths_on_pythonpath([str(target)]): diff --git a/setuptools/tests/test_setopt.py b/setuptools/tests/test_setopt.py index 0163f9af..36008632 100644 --- a/setuptools/tests/test_setopt.py +++ b/setuptools/tests/test_setopt.py @@ -28,3 +28,14 @@ class TestEdit: parser = self.parse_config(str(config)) assert parser.get('names', 'jaraco') == 'джарако' assert parser.get('names', 'other') == 'yes' + + def test_case_retained(self, tmpdir): + """ + When editing a file, case of keys should be retained. + """ + config = tmpdir.join('setup.cfg') + self.write_text(str(config), '[names]\nFoO=bAr') + setopt.edit_config(str(config), dict(names=dict(oTher='yes'))) + actual = config.read_text(encoding='ascii') + assert 'FoO' in actual + assert 'oTher' in actual diff --git a/setuptools/tests/test_sphinx_upload_docs.py b/setuptools/tests/test_sphinx_upload_docs.py new file mode 100644 index 00000000..cc5b8293 --- /dev/null +++ b/setuptools/tests/test_sphinx_upload_docs.py @@ -0,0 +1,38 @@ +import pytest + +from jaraco import path + +from setuptools.command.upload_docs import upload_docs +from setuptools.dist import Distribution + + +@pytest.fixture +def sphinx_doc_sample_project(tmpdir_cwd): + path.build({ + 'setup.py': 'from setuptools import setup; setup()', + 'build': { + 'docs': { + 'conf.py': 'project="test"', + 'index.rst': ".. toctree::\ + :maxdepth: 2\ + :caption: Contents:", + }, + }, + }) + + +@pytest.mark.usefixtures('sphinx_doc_sample_project') +class TestSphinxUploadDocs: + def test_sphinx_doc(self): + params = dict( + name='foo', + packages=['test'], + ) + dist = Distribution(params) + + cmd = upload_docs(dist) + + cmd.initialize_options() + assert cmd.upload_dir is None + assert cmd.has_sphinx() is True + cmd.finalize_options() diff --git a/setuptools/tests/test_test.py b/setuptools/tests/test_test.py index 180562e2..6bce8e20 100644 --- a/setuptools/tests/test_test.py +++ b/setuptools/tests/test_test.py @@ -1,4 +1,3 @@ -import mock from distutils import log import os @@ -6,12 +5,12 @@ import pytest from setuptools.command.test import test from setuptools.dist import Distribution -from setuptools.tests import ack_2to3 from .textwrap import DALS -SETUP_PY = DALS(""" +SETUP_PY = DALS( + """ from setuptools import setup setup(name='foo', @@ -19,9 +18,11 @@ SETUP_PY = DALS(""" namespace_packages=['name'], test_suite='name.space.tests.test_suite', ) - """) + """ +) -NS_INIT = DALS(""" +NS_INIT = DALS( + """ # -*- coding: Latin-1 -*- # Söme Arbiträry Ünicode to test Distribute Issüé 310 try: @@ -29,17 +30,20 @@ NS_INIT = DALS(""" except ImportError: from pkgutil import extend_path __path__ = extend_path(__path__, __name__) - """) + """ +) -TEST_PY = DALS(""" +TEST_PY = DALS( + """ import unittest class TestTest(unittest.TestCase): def test_test(self): - print "Foo" # Should fail under Python 3 unless 2to3 is used + print "Foo" # Should fail under Python 3 test_suite = unittest.makeSuite(TestTest) - """) + """ +) @pytest.fixture @@ -70,25 +74,6 @@ def quiet_log(): log.set_verbosity(0) -@pytest.mark.usefixtures('sample_test', 'quiet_log') -@ack_2to3 -def test_test(capfd): - params = dict( - name='foo', - packages=['name', 'name.space', 'name.space.tests'], - namespace_packages=['name'], - test_suite='name.space.tests.test_suite', - use_2to3=True, - ) - dist = Distribution(params) - dist.script_name = 'setup.py' - cmd = test(dist) - cmd.ensure_finalized() - cmd.run() - out, err = capfd.readouterr() - assert out == 'Foo\n' - - @pytest.mark.usefixtures('tmpdir_cwd', 'quiet_log') def test_tests_are_run_once(capfd): params = dict( @@ -104,13 +89,16 @@ def test_tests_are_run_once(capfd): with open('dummy/__init__.py', 'wt'): pass with open('dummy/test_dummy.py', 'wt') as f: - f.write(DALS( - """ + f.write( + DALS( + """ import unittest class TestTest(unittest.TestCase): def test_test(self): print('Foo') - """)) + """ + ) + ) dist = Distribution(params) dist.script_name = 'setup.py' cmd = test(dist) @@ -118,54 +106,3 @@ def test_tests_are_run_once(capfd): cmd.run() out, err = capfd.readouterr() assert out == 'Foo\n' - - -@pytest.mark.usefixtures('sample_test') -@ack_2to3 -def test_warns_deprecation(capfd): - params = dict( - name='foo', - packages=['name', 'name.space', 'name.space.tests'], - namespace_packages=['name'], - test_suite='name.space.tests.test_suite', - use_2to3=True - ) - dist = Distribution(params) - dist.script_name = 'setup.py' - cmd = test(dist) - cmd.ensure_finalized() - cmd.announce = mock.Mock() - cmd.run() - capfd.readouterr() - msg = ( - "WARNING: Testing via this command is deprecated and will be " - "removed in a future version. Users looking for a generic test " - "entry point independent of test runner are encouraged to use " - "tox." - ) - cmd.announce.assert_any_call(msg, log.WARN) - - -@pytest.mark.usefixtures('sample_test') -@ack_2to3 -def test_deprecation_stderr(capfd): - params = dict( - name='foo', - packages=['name', 'name.space', 'name.space.tests'], - namespace_packages=['name'], - test_suite='name.space.tests.test_suite', - use_2to3=True - ) - dist = Distribution(params) - dist.script_name = 'setup.py' - cmd = test(dist) - cmd.ensure_finalized() - cmd.run() - out, err = capfd.readouterr() - msg = ( - "WARNING: Testing via this command is deprecated and will be " - "removed in a future version. Users looking for a generic test " - "entry point independent of test runner are encouraged to use " - "tox.\n" - ) - assert msg in err diff --git a/setuptools/tests/test_upload_docs.py b/setuptools/tests/test_upload_docs.py index a26e32a6..55978aad 100644 --- a/setuptools/tests/test_upload_docs.py +++ b/setuptools/tests/test_upload_docs.py @@ -3,6 +3,7 @@ import zipfile import contextlib import pytest +from jaraco import path from setuptools.command.upload_docs import upload_docs from setuptools.dist import Distribution @@ -10,28 +11,20 @@ from setuptools.dist import Distribution from .textwrap import DALS from . import contexts -SETUP_PY = DALS( - """ - from setuptools import setup - - setup(name='foo') - """) - @pytest.fixture def sample_project(tmpdir_cwd): - # setup.py - with open('setup.py', 'wt') as f: - f.write(SETUP_PY) - - os.mkdir('build') - - # A test document. - with open('build/index.html', 'w') as f: - f.write("Hello world.") - - # An empty folder. - os.mkdir('build/empty') + path.build({ + 'setup.py': DALS(""" + from setuptools import setup + + setup(name='foo') + """), + 'build': { + 'index.html': 'Hello world.', + 'empty': {}, + } + }) @pytest.mark.usefixtures('sample_project') diff --git a/setuptools/tests/test_virtualenv.py b/setuptools/tests/test_virtualenv.py index 21dea5bb..462e20c7 100644 --- a/setuptools/tests/test_virtualenv.py +++ b/setuptools/tests/test_virtualenv.py @@ -1,6 +1,7 @@ import glob import os import sys +import itertools import pathlib @@ -40,14 +41,12 @@ def bare_virtualenv(): yield venv -SOURCE_DIR = os.path.join(os.path.dirname(__file__), '../..') - - -def test_clean_env_install(bare_virtualenv): +def test_clean_env_install(bare_virtualenv, tmp_src): """ Check setuptools can be installed in a clean environment. """ - bare_virtualenv.run(['python', 'setup.py', 'install'], cd=SOURCE_DIR) + cmd = [bare_virtualenv.python, 'setup.py', 'install'] + bare_virtualenv.run(cmd, cd=tmp_src) def _get_pip_versions(): @@ -68,24 +67,38 @@ def _get_pip_versions(): # No network, disable most of these tests network = False + def mark(param, *marks): + if not isinstance(param, type(pytest.param(''))): + param = pytest.param(param) + return param._replace(marks=param.marks + marks) + + def skip_network(param): + return param if network else mark(param, pytest.mark.skip(reason="no network")) + + issue2599 = pytest.mark.skipif( + sys.version_info > (3, 10), + reason="pypa/setuptools#2599", + ) + network_versions = [ - 'pip==9.0.3', - 'pip==10.0.1', - 'pip==18.1', - 'pip==19.0.1', - 'https://github.com/pypa/pip/archive/master.zip', + mark('pip==9.0.3', issue2599), + mark('pip==10.0.1', issue2599), + mark('pip==18.1', issue2599), + mark('pip==19.3.1', pytest.mark.xfail(reason='pypa/pip#6599')), + 'pip==20.0.2', + 'https://github.com/pypa/pip/archive/main.zip', ] - versions = [None] + [ - pytest.param(v, **({} if network else {'marks': pytest.mark.skip})) - for v in network_versions - ] + versions = itertools.chain( + [None], + map(skip_network, network_versions) + ) - return versions + return list(versions) @pytest.mark.parametrize('pip_version', _get_pip_versions()) -def test_pip_upgrade_from_source(pip_version, virtualenv): +def test_pip_upgrade_from_source(pip_version, tmp_src, virtualenv): """ Check pip can upgrade setuptools from source. """ @@ -104,7 +117,7 @@ def test_pip_upgrade_from_source(pip_version, virtualenv): virtualenv.run(' && '.join(( 'python setup.py -q sdist -d {dist}', 'python setup.py -q bdist_wheel -d {dist}', - )).format(dist=dist_dir), cd=SOURCE_DIR) + )).format(dist=dist_dir), cd=tmp_src) sdist = glob.glob(os.path.join(dist_dir, '*.zip'))[0] wheel = glob.glob(os.path.join(dist_dir, '*.whl'))[0] # Then update from wheel. @@ -113,12 +126,12 @@ def test_pip_upgrade_from_source(pip_version, virtualenv): virtualenv.run('pip install --no-cache-dir --upgrade ' + sdist) -def _check_test_command_install_requirements(virtualenv, tmpdir): +def _check_test_command_install_requirements(virtualenv, tmpdir, cwd): """ Check the test command will install all required dependencies. """ # Install setuptools. - virtualenv.run('python setup.py develop', cd=SOURCE_DIR) + virtualenv.run('python setup.py develop', cd=cwd) def sdist(distname, version): dist_path = tmpdir.join('%s-%s.tar.gz' % (distname, version)) @@ -175,22 +188,21 @@ def _check_test_command_install_requirements(virtualenv, tmpdir): assert tmpdir.join('success').check() -def test_test_command_install_requirements(virtualenv, tmpdir): +def test_test_command_install_requirements(virtualenv, tmpdir, request): # Ensure pip/wheel packages are installed. virtualenv.run( "python -c \"__import__('pkg_resources').require(['pip', 'wheel'])\"") - _check_test_command_install_requirements(virtualenv, tmpdir) - - -def test_test_command_install_requirements_when_using_easy_install( - bare_virtualenv, tmpdir): - _check_test_command_install_requirements(bare_virtualenv, tmpdir) + # uninstall setuptools so that 'setup.py develop' works + virtualenv.run("python -m pip uninstall -y setuptools") + # disable index URL so bits and bobs aren't requested from PyPI + virtualenv.env['PIP_NO_INDEX'] = '1' + _check_test_command_install_requirements(virtualenv, tmpdir, request.config.rootdir) -def test_no_missing_dependencies(bare_virtualenv): +def test_no_missing_dependencies(bare_virtualenv, request): """ Quick and dirty test to ensure all external dependencies are vendored. """ for command in ('upload',): # sorted(distutils.command.__all__): - bare_virtualenv.run( - ['python', 'setup.py', command, '-h'], cd=SOURCE_DIR) + cmd = [bare_virtualenv.python, 'setup.py', command, '-h'] + bare_virtualenv.run(cmd, cd=request.config.rootdir) diff --git a/setuptools/tests/test_wheel.py b/setuptools/tests/test_wheel.py index e56eac14..7345b135 100644 --- a/setuptools/tests/test_wheel.py +++ b/setuptools/tests/test_wheel.py @@ -15,6 +15,7 @@ import sys import zipfile import pytest +from jaraco import path from pkg_resources import Distribution, PathMetadata, PY_MAJOR from setuptools.extern.packaging.utils import canonicalize_name @@ -22,7 +23,6 @@ from setuptools.extern.packaging.tags import parse_tag from setuptools.wheel import Wheel from .contexts import tempdir -from .files import build_files from .textwrap import DALS @@ -91,7 +91,7 @@ def build_wheel(extra_file_defs=None, **kwargs): if extra_file_defs: file_defs.update(extra_file_defs) with tempdir() as source_dir: - build_files(file_defs, source_dir) + path.build(file_defs, source_dir) subprocess.check_call((sys.executable, 'setup.py', '-q', 'bdist_wheel'), cwd=source_dir) yield glob.glob(os.path.join(source_dir, 'dist', '*.whl'))[0] diff --git a/setuptools/tests/test_windows_wrappers.py b/setuptools/tests/test_windows_wrappers.py index fa647de8..8ac9bd07 100644 --- a/setuptools/tests/test_windows_wrappers.py +++ b/setuptools/tests/test_windows_wrappers.py @@ -13,6 +13,7 @@ are to wrap. """ import sys +import platform import textwrap import subprocess @@ -51,10 +52,20 @@ class WrapperTester: f.write(w) +def win_launcher_exe(prefix): + """ A simple routine to select launcher script based on platform.""" + assert prefix in ('cli', 'gui') + if platform.machine() == "ARM64": + return "{}-arm64.exe".format(prefix) + else: + return "{}-32.exe".format(prefix) + + class TestCLI(WrapperTester): script_name = 'foo-script.py' - wrapper_source = 'cli-32.exe' wrapper_name = 'foo.exe' + wrapper_source = win_launcher_exe('cli') + script_tmpl = textwrap.dedent(""" #!%(python_exe)s import sys @@ -155,7 +166,7 @@ class TestGUI(WrapperTester): ----------------------- """ script_name = 'bar-script.pyw' - wrapper_source = 'gui-32.exe' + wrapper_source = win_launcher_exe('gui') wrapper_name = 'bar.exe' script_tmpl = textwrap.dedent(""" @@ -167,7 +178,7 @@ class TestGUI(WrapperTester): """).strip() def test_basic(self, tmpdir): - """Test the GUI version with the simple scipt, bar-script.py""" + """Test the GUI version with the simple script, bar-script.py""" self.create_script(tmpdir) cmd = [ diff --git a/skeleton.md b/skeleton.md deleted file mode 100644 index ec421c25..00000000 --- a/skeleton.md +++ /dev/null @@ -1,144 +0,0 @@ -# Overview - -This project is merged with [skeleton](https://github.com/jaraco/skeleton). What is skeleton? It's the scaffolding of a Python project jaraco [introduced in his blog](https://blog.jaraco.com/a-project-skeleton-for-python-projects/). It seeks to provide a means to re-use techniques and inherit advances when managing projects for distribution. - -## An SCM-Managed Approach - -While maintaining dozens of projects in PyPI, jaraco derives best practices for project distribution and publishes them in the [skeleton repo](https://github.com/jaraco/skeleton), a Git repo capturing the evolution and culmination of these best practices. - -It's intended to be used by a new or existing project to adopt these practices and honed and proven techniques. Adopters are encouraged to use the project directly and maintain a small deviation from the technique, make their own fork for more substantial changes unique to their environment or preferences, or simply adopt the skeleton once and abandon it thereafter. - -The primary advantage to using an SCM for maintaining these techniques is that those tools help facilitate the merge between the template and its adopting projects. - -Another advantage to using an SCM-managed approach is that tools like GitHub recognize that a change in the skeleton is the _same change_ across all projects that merge with that skeleton. Without the ancestry, with a traditional copy/paste approach, a [commit like this](https://github.com/jaraco/skeleton/commit/12eed1326e1bc26ce256e7b3f8cd8d3a5beab2d5) would produce notifications in the upstream project issue for each and every application, but because it's centralized, GitHub provides just the one notification when the change is added to the skeleton. - -# Usage - -## new projects - -To use skeleton for a new project, simply pull the skeleton into a new project: - -``` -$ git init my-new-project -$ cd my-new-project -$ git pull gh://jaraco/skeleton -``` - -Now customize the project to suit your individual project needs. - -## existing projects - -If you have an existing project, you can still incorporate the skeleton by merging it into the codebase. - -``` -$ git merge skeleton --allow-unrelated-histories -``` - -The `--allow-unrelated-histories` is necessary because the history from the skeleton was previously unrelated to the existing codebase. Resolve any merge conflicts and commit to the master, and now the project is based on the shared skeleton. - -## Updating - -Whenever a change is needed or desired for the general technique for packaging, it can be made in the skeleton project and then merged into each of the derived projects as needed, recommended before each release. As a result, features and best practices for packaging are centrally maintained and readily trickle into a whole suite of packages. This technique lowers the amount of tedious work necessary to create or maintain a project, and coupled with other techniques like continuous integration and deployment, lowers the cost of creating and maintaining refined Python projects to just a few, familiar Git operations. - -For example, here's a session of the [path project](https://pypi.org/project/path) pulling non-conflicting changes from the skeleton: - -<img src="https://raw.githubusercontent.com/jaraco/skeleton/gh-pages/docs/refresh.svg"> - -Thereafter, the target project can make whatever customizations it deems relevant to the scaffolding. The project may even at some point decide that the divergence is too great to merit renewed merging with the original skeleton. This approach applies maximal guidance while creating minimal constraints. - -# Features - -The features/techniques employed by the skeleton include: - -- PEP 517/518-based build relying on Setuptools as the build tool -- Setuptools declarative configuration using setup.cfg -- tox for running tests -- A README.rst as reStructuredText with some popular badges, but with Read the Docs and AppVeyor badges commented out -- A CHANGES.rst file intended for publishing release notes about the project -- Use of [Black](https://black.readthedocs.io/en/stable/) for code formatting (disabled on unsupported Python 3.5 and earlier) -- Integrated type checking through [mypy](https://github.com/python/mypy/). - -## Packaging Conventions - -A pyproject.toml is included to enable PEP 517 and PEP 518 compatibility and declares the requirements necessary to build the project on Setuptools (a minimum version compatible with setup.cfg declarative config). - -The setup.cfg file implements the following features: - -- Assumes universal wheel for release -- Advertises the project's LICENSE file (MIT by default) -- Reads the README.rst file into the long description -- Some common Trove classifiers -- Includes all packages discovered in the repo -- Data files in the package are also included (not just Python files) -- Declares the required Python versions -- Declares install requirements (empty by default) -- Declares setup requirements for legacy environments -- Supplies two 'extras': - - testing: requirements for running tests - - docs: requirements for building docs - - these extras split the declaration into "upstream" (requirements as declared by the skeleton) and "local" (those specific to the local project); these markers help avoid merge conflicts -- Placeholder for defining entry points - -Additionally, the setup.py file declares `use_scm_version` which relies on [setuptools_scm](https://pypi.org/project/setuptools_scm) to do two things: - -- derive the project version from SCM tags -- ensure that all files committed to the repo are automatically included in releases - -## Running Tests - -The skeleton assumes the developer has [tox](https://pypi.org/project/tox) installed. The developer is expected to run `tox` to run tests on the current Python version using [pytest](https://pypi.org/project/pytest). - -Other environments (invoked with `tox -e {name}`) supplied include: - - - a `docs` environment to build the documentation - - a `release` environment to publish the package to PyPI - -A pytest.ini is included to define common options around running tests. In particular: - -- rely on default test discovery in the current directory -- avoid recursing into common directories not containing tests -- run doctests on modules and invoke Flake8 tests -- in doctests, allow Unicode literals and regular literals to match, allowing for doctests to run on Python 2 and 3. Also enable ELLIPSES, a default that would be undone by supplying the prior option. -- filters out known warnings caused by libraries/functionality included by the skeleton - -Relies on a .flake8 file to correct some default behaviors: - -- disable mutually incompatible rules W503 and W504 -- support for Black format - -## Continuous Integration - -The project is pre-configured to run Continuous Integration tests. - -### Github Actions - -[Github Actions](https://docs.github.com/en/free-pro-team@latest/actions) are the preferred provider as they provide free, fast, multi-platform services with straightforward configuration. Configured in `.github/workflows`. - -Features include: -- test against multiple Python versions -- run on late (and updated) platform versions -- automated releases of tagged commits - -### Continuous Deployments - -In addition to running tests, an additional publish stage is configured to automatically release tagged commits to PyPI using [API tokens](https://pypi.org/help/#apitoken). The release process expects an authorized token to be configured with each Github project (or org) `PYPI_TOKEN` [secret](https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets). Example: - -``` -pip-run -q jaraco.develop -- -m jaraco.develop.add-github-secrets -``` - -## Building Documentation - -Documentation is automatically built by [Read the Docs](https://readthedocs.org) when the project is registered with it, by way of the .readthedocs.yml file. To test the docs build manually, a tox env may be invoked as `tox -e docs`. Both techniques rely on the dependencies declared in `setup.cfg/options.extras_require.docs`. - -In addition to building the Sphinx docs scaffolded in `docs/`, the docs build a `history.html` file that first injects release dates and hyperlinks into the CHANGES.rst before incorporating it as history in the docs. - -## Cutting releases - -By default, tagged commits are released through the continuous integration deploy stage. - -Releases may also be cut manually by invoking the tox environment `release` with the PyPI token set as the TWINE_PASSWORD: - -``` -TWINE_PASSWORD={token} tox -e release -``` diff --git a/tools/finalize.py b/tools/finalize.py index 35294281..516a2fb5 100644 --- a/tools/finalize.py +++ b/tools/finalize.py @@ -46,6 +46,18 @@ def update_changelog(): '--yes', ] subprocess.check_call(cmd) + _repair_changelog() + + +def _repair_changelog(): + """ + Workaround for #2666 + """ + changelog_fn = pathlib.Path('CHANGES.rst') + changelog = changelog_fn.read_text() + fixed = re.sub(r'^(v[0-9.]+)v[0-9.]+$', r'\1', changelog, flags=re.M) + changelog_fn.write_text(fixed) + subprocess.check_output(['git', 'add', changelog_fn]) def bump_version(): diff --git a/tools/tox_pip.py b/tools/tox_pip.py deleted file mode 100644 index be2ff1d0..00000000 --- a/tools/tox_pip.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -import subprocess -import sys -import re - - -def remove_setuptools(): - """ - Remove setuptools from the current environment. - """ - print("Removing setuptools") - cmd = [sys.executable, '-m', 'pip', 'uninstall', '-y', 'setuptools'] - # set cwd to something other than '.' to avoid detecting - # '.' as the installed package. - subprocess.check_call(cmd, cwd=os.environ['TOX_WORK_DIR']) - - -def bootstrap(): - print("Running bootstrap") - cmd = [sys.executable, '-m', 'bootstrap'] - subprocess.check_call(cmd) - - -def is_install_self(args): - """ - Do the args represent an install of .? - """ - def strip_extras(arg): - match = re.match(r'(.*)?\[.*\]$', arg) - return match.group(1) if match else arg - - return ( - 'install' in args - and any( - arg in ['.', os.getcwd()] - for arg in map(strip_extras, args) - ) - ) - - -def pip(*args): - cmd = [sys.executable, '-m', 'pip'] + list(args) - return subprocess.check_call(cmd) - - -def test_dependencies(): - from ConfigParser import ConfigParser - - def clean(dep): - spec, _, _ = dep.partition('#') - return spec.strip() - - parser = ConfigParser() - parser.read('setup.cfg') - raw = parser.get('options.extras_require', 'tests').split('\n') - return filter(None, map(clean, raw)) - - -def run(args): - os.environ['PIP_USE_PEP517'] = 'true' - - if is_install_self(args): - remove_setuptools() - bootstrap() - - pip(*args) - - -if __name__ == '__main__': - run(sys.argv[1:]) diff --git a/towncrier_template.rst b/towncrier_template.rst index 31098b7c..7f507342 100644 --- a/towncrier_template.rst +++ b/towncrier_template.rst @@ -1,3 +1,7 @@ +{% if top_line %} +{{ top_line }} +{{ top_underline * ((top_line)|length)}} +{% endif %} {% for section, _ in sections.items() %} {% set underline = underlines[0] %}{% if section %}{{section}} {{ underline * section|length }} @@ -7,40 +7,23 @@ toxworkdir={env:TOX_WORK_DIR:.tox} [testenv] deps = + # workaround for sphinx-doc/sphinx#9562 + # TODO: remove after Sphinx>4.1.2 is available. + sphinx@git+https://github.com/sphinx-doc/sphinx; python_version>="3.10" commands = pytest {posargs} usedevelop = True extras = testing -install_command = {[helpers]pip} install {opts} {packages} -list_dependencies_command = {[helpers]pip} freeze --all -setenv = - COVERAGE_FILE={toxworkdir}/.coverage.{envname} passenv = windir # required for test_pkg_resources -[testenv:coverage] -description=Combine coverage data and create report -deps=coverage -skip_install=True -changedir={toxworkdir} -setenv=COVERAGE_FILE=.coverage -commands=coverage erase - coverage combine - coverage {posargs:xml} - -[testenv:codecov] -description=[Only run on CI]: Upload coverage data to codecov -deps=codecov -skip_install=True -commands=codecov -X gcov --file {toxworkdir}/coverage.xml - [testenv:docs] extras = docs testing changedir = docs commands = - python -m sphinx -W . {toxinidir}/build/html + python -m sphinx -W --keep-going . {toxinidir}/build/html [testenv:finalize] skip_install = True @@ -54,25 +37,19 @@ commands = [testenv:release] skip_install = True deps = - pep517>=0.5 - twine[keyring]>=1.13 - path + build + twine>=3 jaraco.develop>=7.1 - jaraco.tidelift passenv = TWINE_PASSWORD GITHUB_TOKEN - TIDELIFT_TOKEN setenv = TWINE_USERNAME = {env:TWINE_USERNAME:__token__} commands = python -m bootstrap - python -c "import path; path.Path('dist').rmtree_p()" - python -m pep517.build . + python -c "import shutil; shutil.rmtree('dist', ignore_errors=True)" + # unset tag_build and tag_date pypa/setuptools#2500 + python setup.py egg_info -Db "" saveopts + python -m build python -m twine upload dist/* python -m jaraco.develop.create-github-release - python -m jaraco.tidelift.publish-release-notes - -[helpers] -# Custom pip behavior -pip = python {toxinidir}/tools/tox_pip.py |