diff options
83 files changed, 6636 insertions, 1944 deletions
@@ -8,7 +8,6 @@ distribute.egg-info setuptools.egg-info .coverage .tox -CHANGES (links).txt *.egg *.py[cod] *.swp @@ -8,7 +8,6 @@ distribute.egg-info setuptools.egg-info .coverage .tox -CHANGES (links).txt *.egg *.py[cod] *.swp @@ -1,56 +1,44 @@ -1010d08fd8dfd2f1496843b557b5369a0beba82a 0.6 -4d114c5f2a3ecb4a0fa552075dbbb221b19e291b 0.6.1 -41415244ee90664042d277d0b1f0f59c04ddd0e4 0.6.2 -e033bf2d3d05f4a7130f5f8f5de152c4db9ff32e 0.6.3 -e06c416e911c61771708f5afbf3f35db0e12ba71 0.6.4 -2df182df8a0224d429402de3cddccdb97af6ea21 0.6.5 -f1fb564d6d67a6340ff33df2f5a74b89753f159d 0.6.6 -71f08668d050589b92ecd164a4f5a91f3484313b 0.6.7 -445547a5729ed5517cf1a9baad595420a8831ef8 0.6.8 -669ed9388b17ec461380cc41760a9a7384fb5284 0.6.9 -669ed9388b17ec461380cc41760a9a7384fb5284 0.6.9 -ac7d9b14ac43fecb8b65de548b25773553facaee 0.6.9 -0fd5c506037880409308f2b79c6e901d21e7fe92 0.6.10 -0fd5c506037880409308f2b79c6e901d21e7fe92 0.6.10 -f18396c6e1875476279d8bbffd8e6dadcc695136 0.6.10 -e00987890c0b386f09d0f6b73d8558b72f6367f1 0.6.11 -48a97bc89e2f65fc9b78b358d7dc89ba9ec9524a 0.6.12 -dae247400d0ca1fdfaf38db275622c9bec550b08 0.6.13 -2b9d9977ea75b8eb3766bab808ef31f192d2b1bc 0.6.14 -51a9d1a1f31a4be3107d06cf088aff8e182dc633 0.6.15 -3f1ff138e947bfc1c9bcfe0037030b7bfb4ab3a5 0.6.16 -9c40f23d0bda3f3f169686e27a422f853fa4d0fa 0.6.17 -9c40f23d0bda3f3f169686e27a422f853fa4d0fa 0.6.17 -4bbc01e4709ea7425cf0c186bbaf1d928cfa2a65 0.6.17 -4bbc01e4709ea7425cf0c186bbaf1d928cfa2a65 0.6.17 -0502d5117d8304ab21084912758ed28812a5a8f1 0.6.17 -74108d7f07343556a8db94e8122221a43243f586 0.6.18 -611910892a0421633d72677979f94a25ef590d54 0.6.19 -a7cf5ae137f1646adf86ce5d6b5d8b7bd6eab69f 0.6.20 -c4a375336d552129aef174486018ed09c212d684 0.6.20 -de44acab3cfce1f5bc811d6c0fa1a88ca0e9533f 0.6.21 -1a1ab844f03e10528ae693ad3cb45064e08f49e5 0.6.23 -1a1ab844f03e10528ae693ad3cb45064e08f49e5 0.6.23 -9406c5dac8429216f1a264e6f692fdc534476acd 0.6.23 -7fd7b6e30a0effa082baed1c4103a0efa56be98c 0.6.24 -6124053afb5c98f11e146ae62049b4c232d50dc5 0.6.25 -b69f072c000237435e17b8bbb304ba6f957283eb 0.6.26 -469c3b948e41ef28752b3cdf3c7fb9618355ebf5 0.6.27 -fc379e63586ad3c6838e1bda216548ba8270b8f0 0.6.28 -4f82563d0f5d1af1fb215c0ac87f38b16bb5c42d 0.6.29 -7464fc916fa4d8308e34e45a1198512fe04c97b4 0.6.30 -17bc972d67edd96c7748061910172e1200a73efe 0.6.31 -b1a7f86b315a1f8c20036d718d6dc641bb84cac6 0.6.32 -6acac3919ae9a7dba2cbecbe3d4b31ece25d5f09 0.6.33 -23c310bf4ae8e4616e37027f08891702f5a33bc9 0.6.34 -2abe1117543be0edbafb10c7c159d1bcb1cb1b87 0.6.35 -c813a29e831f266d427d4a4bce3da97f475a8eee 0.6.36 -be6f65eea9c10ce78b6698d8c220b6e5de577292 0.6.37 -2b26ec8909bff210f47c5f8fc620bc505e1610b5 0.6.37 -f0d502a83f6c83ba38ad21c15a849c2daf389ec7 0.6.38 -d737b2039c5f92af8000f78bbc80b6a5183caa97 0.6.39 +7e9441311eb21dd1fbc32cfbad58168e46c5450e 0.6 +26f429772565f69d1f6d21adf57c3d8c40197129 0.6.1 +6f46749a7454be6e044a54cd73c51318b74bdee8 0.6.2 +34b80fb58862d18f8f957f98a883ed4a72d06f8e 0.6.3 +fb04abddb50d82a9005c9082c94d5eb983be1d79 0.6.4 +8ae0bd250b4a0d58cbaf16b4354ad60f73f24a01 0.6.5 +88847883dfed39829d3a5ed292ad540723ad31cc 0.6.6 +fcbef325349ada38f6c674eb92db82664cf6437c 0.6.7 +3af7f2b8270b9bb34fb65f08ee567bfe8e2a6a5a 0.6.8 +669725d03fd1e345ea47590e9b14cb19742b96a2 0.6.9 +eff3ca9c2d8d39e24c221816c52a37f964535336 0.6.10 +88710e34b91c98c9348749722cce3acd574d177d 0.6.11 +5ce754773a43ac21f7bd13872f45c75e27b593f8 0.6.12 +de36566d35e51bee7cfc86ffa694795e52f4147c 0.6.13 +e5f3f0ffe9e1a243d49a06f26c79dd160f521483 0.6.14 +dc03a300ec7a89ad773047172d43e52b34e7cd1e 0.6.15 +e620fb4ee8ba17debadb614fb583c6dfac229dea 0.6.16 +21df276275b5a47c6a994927d69ad3d90cf62b5d 0.6.17 +e9264ca4ba8c24239c36a8426a0394f7c7d5dd83 0.6.18 +aed31b1fa47ed1f39e55c75b76bbbdb80775b7f1 0.6.19 +c6e6273587816c3e486ef7739e53c864a0145251 0.6.20 +7afdf4c84a713fe151e6163ab25d45e8727ce653 0.6.21 +105066342777cd1319a95d7ae0271a2ea1ac33fe 0.6.23 +7b5ef4e6c80e82541dffb5a9a130d81550d5a835 0.6.24 +9c014a80f32e532371826ed1dc3236975f37f371 0.6.25 +ff8c4d6c8e5d2093750a58a3d43b76556570007c 0.6.26 +2a5c42ed097a195e398b97261c40cd66c8da8913 0.6.27 +4ed34b38851f90278cfe2bff75784f7e32883725 0.6.28 +acecfa2cfb6fca207dd2f4e025c695def3bb6b40 0.6.29 +e950f50addff150859f5990b9df2a33c691b6354 0.6.30 +06dae3faee2de50ff17b90719df410b2ebc5b71e 0.6.31 +1f4f79258ed5b418f680a55d3006f41aa6a56d2b 0.6.32 +89f57bf1406a5e745470af35446902c21ac9b6f6 0.6.33 +3c8f9fc13862124cf20ef2ff2140254fb272bb94 0.6.34 +7c3f8b9eb7cfa17481c835d5caaa918d337c7a83 0.6.35 +192094c0d1e2e5d2cb5c718f84a36c9de04b314b 0.6.36 +66d4e3b8899166e4c04189ee1831c649b7ff38bf 0.6.37 +398d58aa8bba33778c30ce72055a27d4b425809c 0.6.38 +f457fc2a3ebe609d8ca7a869eb65b7506ecf49ef 0.6.39 9b2e2aa06e058c63e06c5e42a7f279ddae2dfb7d 0.7b1 -0a783fa0dceb95b5fc743e47c2d89c1523d0afb7 0.6.40 +9089a40343981baa593b9bb5953f9088e9507099 0.6.40 ad107e9b4beea24516ac4e1e854696e586fe279d 0.6.41 f30167716b659f96c5e0b7ea3d5be2bcff8c0eac 0.6.42 8951daac6c1bc7b24c7fb054fd369f2c5b88cdb3 0.7b2 @@ -231,3 +219,35 @@ c811801ffa1de758cf01fbf6a86e4c04ff0c0935 18.8 fbf06fa35f93a43f044b1645a7e4ff470edb462c 18.8.1 cc41477ecf92f221c113736fac2830bf8079d40c 19.0 834782ce49154e9744e499e00eb392c347f9e034 19.1 +0a2a3d89416e1642cf6f41d22dbc07b3d3c15a4d 19.1.1 +5d24cf9d1ced76c406ab3c4a94c25d1fe79b94bc 19.2 +66fa131a0d77a1b0e6f89ccb76b254cfb07d3da3 19.3b1 +32bba9bf8cce8350b560a7591c9ef5884a194211 19.3 +f47f3671508b015e9bb735603d3a0a6ec6a77b01 19.4 +0bda3291ac725750b899b4ba3e4b6765e7645daa 19.4.1 +0a68cbab72580a6f8d3bf9c45206669eefcd256b 19.5 +34121bf49b1a7ac77da7f7c75105c8a920218dd7 19.6b1 +3c2332e4ec72717bf17321473e5c3ad6e5778903 19.6 +35d9179d04390aada66eceae9ceb7b9274f67646 19.6.1 +d2782cbb2f15ca6831ab9426fbf8d4d6ca60db8a 19.6.2 +c6e619ce910d1650cc2433f94e5594964085f973 19.7 +2a60daeff0cdb039b20b2058aaad7dae7bcd2c1c 20.0 +06c9d3ffae80d7f5786c0a454d040d253d47fc03 20.1 +919a40f1843131249f98104c73f3aee3fc835e67 20.1.1 +74c4ffbe1f399345eb4f6a64785cfff54f7e6e7e 20.2 +1aacb05fbdfe06cee904e7a138a4aa6df7b88a63 20.2.1 +48aa5271ef1cd5379cf91a1c958e490692b978e7 20.2.2 +9c55a3a1268a33b4a57b96b2b9fa2cd0701780ee 20.3 +3e87e975a95c780eec497ef9e5a742f7adfb77ec 20.3.1 +06692c64fb9b5843331a918ab7093f151412ec8e 20.4 +f8174392e9e9c6a21ea5df0f22cb4ca885c799ca 20.5 +114f3dbc8a73dacbce2ebe08bb70ca76ab18390e v20.6.0 +a3d4006688fe5e754d0e709a52a00b8191819979 v20.6.1 +2831509712601a78fddf46e51d6f41ae0f92bd0e v20.6.2 +8b46dc41cb234c435b950a879214a6dee54c9dd2 v20.6.3 +7258be20fe93bbf936dc1a81ce71c04c5880663e v20.6.4 +7e0ab283db4e6f780777f7f06af475f044631fa1 v20.6.5 +57d63b38e85515d06e06d3cea62e35e6c54b5093 v20.6.6 +57d63b38e85515d06e06d3cea62e35e6c54b5093 v20.6.6 +b04dbdd161d7f68903a53e1dbd1fa5b5fde73f94 v20.6.6 +0804d30b6ead64e0e324aefd67439b84df2d1c01 v20.6.7 diff --git a/.travis.yml b/.travis.yml index 54d9c395..2611f02a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,10 @@ python: - 3.4 - 3.5 - pypy + - pypy3 +matrix: + allow_failures: + - python: pypy3 env: - "" - LC_ALL=C LC_CTYPE=C @@ -21,5 +25,15 @@ script: - python setup.py test --addopts='-rs' - # test the bootstrap script - - python ez_setup.py +before_deploy: + - export SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES=1 +deploy: + provider: pypi + on: + tags: true + all_branches: true + user: jaraco + password: + secure: tfWrsQMH2bHrWjqnP+08IX1WlkbW94Q30f4d7lCyhWS1FIf/jBDx4jrEILNfMxQ1NCwuBRje5sihj1Ow0BFf0vVrkaeff2IdvnNDEGFduMejaEQJL3s3QrLfpiAvUbtqwyWaHfAdGfk48PovDKTx0ZTvXZKYGXZhxGCYSlG2CE6Y6RDvnEl6Tk8e+LqUohkcSOwxrRwUoyxSnUaavdGohXxDT8MJlfWOXgr2u+KsRrriZqp3l6Fdsnk4IGvy6pXpy42L1HYQyyVu9XyJilR2JTbC6eCp5f8p26093m1Qas49+t6vYb0VLqQe12dO+Jm3v4uztSS5pPQzS7PFyjEYd2Rdb6ijsdbsy1074S4q7G9Sz+T3RsPUwYEJ07lzez8cxP64dtj5j94RL8m35A1Fb1OE8hHN+4c1yLG1gudfXbem+fUhi2eqhJrzQo5vsvDv1xS5x5GIS5ZHgKHCsWcW1Tv+dsFkrhaup3uU6VkOuc9UN+7VPsGEY7NvquGpTm8O1CnGJRzuJg6nbYRGj8ORwDpI0KmrExx6akV92P72fMC/I5TCgbSQSZn370H3Jj40gz1SM30WAli9M+wFHFd4ddMVY65yxj0NLmrP+m1tvnWdKtNh/RHuoW92d9/UFtiA5IhMf1/3djfsjBq6S9NT1uaLkVkTttqrPYJ7hOql8+g= + distributions: release + python: 3.5 diff --git a/CHANGES.txt b/CHANGES.rst index 4e71bd72..dee622ac 100644 --- a/CHANGES.txt +++ b/CHANGES.rst @@ -2,8 +2,210 @@ CHANGES ======= +v20.6.7 +------- +* Issue #523: Disabled support for environment markers + introduced in v20.5. + +v20.6.6 +------- + +* Issue #503: Restore support for PEP 345 environment + markers by updating to Packaging 16.6. + +v20.6.0 +------- + +* New release process that relies on + `bumpversion <https://github.com/peritus/bumpversion>`_ + and Travis CI for continuous deployment. +* Project versioning semantics now follow + `semver <https://semver.org>`_ precisely. + The 'v' prefix on version numbers now also allows + version numbers to be referenced in the changelog, + e.g. https://pythonhosted.org/setuptools/history.html#v20-6-0. + +20.5 +---- + +* BB Pull Request #185: Add support for environment markers + in requirements in install_requires, setup_requires, + tests_require as well as adding a test for the existing + extra_requires machinery. + +20.4 +---- + +* Issue #422: Moved hosting to + `Github <https://github.com/pypa/setuptools>`_ + from `Bitbucket <https://bitbucket.org/pypa/setuptools>`_. + Issues have been migrated, though all issues and comments + are attributed to bb-migration. So if you have a particular + issue or issues to which you've been subscribed, you will + want to "watch" the equivalent issue in Github. + The Bitbucket project will be retained for the indefinite + future, but Github now hosts the canonical project repository. + +20.3.1 +------ + +* Issue #519: Remove import hook when reloading the + ``pkg_resources`` module. +* BB Pull Request #184: Update documentation in ``pkg_resources`` + around new ``Requirement`` implementation. + +20.3 +---- + +* BB Pull Request #179: ``pkg_resources.Requirement`` objects are + now a subclass of ``packaging.requirements.Requirement``, + allowing any environment markers and url (if any) to be + affiliated with the requirement +* BB Pull Request #179: Restore use of RequirementParseError + exception unintentionally dropped in 20.2. + +20.2.2 +------ + +* Issue #502: Correct regression in parsing of multiple + version specifiers separated by commas and spaces. + +20.2.1 +------ + +* Issue #499: Restore compatiblity for legacy versions + by bumping to packaging 16.4. + +20.2 +---- + +* Changelog now includes release dates and links to PEPs. +* BB Pull Request #173: Replace dual PEP 345 _markerlib implementation + and PEP 426 implementation of environment marker support from + packaging 16.1 and PEP 508. Fixes Issue #122. + See also BB Pull Request #175, BB Pull Request #168, and + BB Pull Request #164. Additionally: + + - ``Requirement.parse`` no longer retains the order of extras. + - ``parse_requirements`` now requires that all versions be + PEP-440 compliant, as revealed in #499. Packages released + with invalid local versions should be re-released using + the proper local version syntax, e.g. ``mypkg-1.0+myorg.1``. + +20.1.1 +------ + +* Update ``upload_docs`` command to also honor keyring + for password resolution. + +20.1 +---- + +* Added support for using passwords from keyring in the upload + command. See `the upload docs + <http://pythonhosted.org/setuptools/setuptools.html#upload-upload-source-and-or-egg-distributions-to-pypi>`_ + for details. + +20.0 +---- + +* Issue #118: Once again omit the package metadata (egg-info) + from the list of outputs in ``--record``. This version of setuptools + can no longer be used to upgrade pip earlier than 6.0. + +19.7 ---- + +* `Off-project PR <https://github.com/jaraco/setuptools/pull/32>`_: + For FreeBSD, also honor root certificates from ca_root_nss. + +19.6.2 +------ + +* Issue #491: Correct regression incurred in 19.4 where + a double-namespace package installed using pip would + cause a TypeError. + +19.6.1 +------ + +* Restore compatibility for PyPy 3 compatibility lost in + 19.4.1 addressing Issue #487. +* ``setuptools.launch`` shim now loads scripts in a new + namespace, avoiding getting relative imports from + the setuptools package on Python 2. + +19.6 +---- + +* Added a new entry script ``setuptools.launch``, + implementing the shim found in + ``pip.util.setuptools_build``. Use this command to launch + distutils-only packages under setuptools in the same way that + pip does, causing the setuptools monkeypatching of distutils + to be invoked prior to invoking a script. Useful for debugging + or otherwise installing a distutils-only package under + setuptools when pip isn't available or otherwise does not + expose the desired functionality. For example:: + + $ python -m setuptools.launch setup.py develop + +* Issue #488: Fix dual manifestation of Extension class in + extension packages installed as dependencies when Cython + is present. + +19.5 +---- + +* Issue #486: Correct TypeError when getfilesystemencoding + returns None. +* Issue #139: Clarified the license as MIT. +* BB Pull Request #169: Removed special handling of command + spec in scripts for Jython. + +19.4.1 +------ + +* Issue #487: Use direct invocation of ``importlib.machinery`` + in ``pkg_resources`` to avoid missing detection on relevant + platforms. + +19.4 +---- + +* Issue #341: Correct error in path handling of package data + files in ``build_py`` command when package is empty. +* Distribute #323, Issue #141, Issue #207, and + BB Pull Request #167: Another implementation of + ``pkg_resources.WorkingSet`` and ``pkg_resources.Distribution`` + that supports replacing an extant package with a new one, + allowing for setup_requires dependencies to supersede installed + packages for the session. + +19.3 +---- + +* Issue #229: Implement new technique for readily incorporating + dependencies conditionally from vendored copies or primary + locations. Adds a new dependency on six. + +19.2 +---- + +* BB Pull Request #163: Add get_command_list method to Distribution. +* BB Pull Request #162: Add missing whitespace to multiline string + literals. + +19.1.1 +------ + +* Issue #476: Cast version to string (using default encoding) + to avoid creating Unicode types on Python 2 clients. +* Issue #477: In Powershell downloader, use explicit rendering + of strings, rather than rely on ``repr``, which can be + incorrect (especially on Python 2). + 19.1 ---- @@ -13,14 +215,12 @@ CHANGES than hard-coding a particular value. * Issue #475: Fix incorrect usage in _translate_metadata2. ----- 19.0 ---- * Issue #442: Use RawConfigParser for parsing .pypirc file. Interpolated values are no longer honored in .pypirc files. ------- 18.8.1 ------ @@ -29,7 +229,6 @@ CHANGES with setuptools hidden. Fixes regression introduced in Setuptools 12.0. ----- 18.8 ---- @@ -41,39 +240,34 @@ CHANGES * Issue #472: Remove deprecated use of 'U' in mode parameter when opening files. ------- 18.7.1 ------ * Issue #469: Refactored logic for Issue #419 fix to re-use metadata loading from Provider. ----- 18.7 ---- * Update dependency on certify. -* Pull Request #160: Improve detection of gui script in +* BB Pull Request #160: Improve detection of gui script in ``easy_install._adjust_header``. * Made ``test.test_args`` a non-data property; alternate fix - for the issue reported in Pull Request #155. + for the issue reported in BB Pull Request #155. * Issue #453: In ``ez_setup`` bootstrap module, unload all ``pkg_resources`` modules following download. -* Pull Request #158: Honor `PEP-488 - <https://www.python.org/dev/peps/pep-0488/>`_ when excluding +* BB Pull Request #158: Honor PEP-488 when excluding files for namespace packages. -* Issue #419 and Pull Request #144: Add experimental support for +* Issue #419 and BB Pull Request #144: Add experimental support for reading the version info from distutils-installed metadata rather than using the version in the filename. ------- 18.6.1 ------ * Issue #464: Correct regression in invocation of superclass on old-style class on Python 2. ----- 18.6 ---- @@ -81,7 +275,6 @@ CHANGES omit the version number of the package, allowing any version of the package to be used. ----- 18.5 ---- @@ -93,27 +286,23 @@ CHANGES * `Fix dictionary mutation during iteration <https://github.com/jaraco/setuptools/pull/29>`_. ----- 18.4 ---- * Issue #446: Test command now always invokes unittest, even if no test suite is supplied. ------- 18.3.2 ------ * Correct another regression in setuptools.findall where the fix for Python #12885 was lost. ------- 18.3.1 ------ * Issue #425: Correct regression in setuptools.findall. ----- 18.3 ---- @@ -134,25 +323,21 @@ CHANGES * Refactor setuptools.findall in preparation for re-submission back to distutils. ----- 18.2 ---- * Issue #412: More efficient directory search in ``find_packages``. ----- 18.1 ---- * Upgrade to vendored packaging 15.3. ------- 18.0.1 ------ * Issue #401: Fix failure in test suite. ----- 18.0 ---- @@ -175,10 +360,9 @@ CHANGES However, for systems with this build of setuptools, Cython will be downloaded on demand. * Issue #396: Fixed test failure on OS X. -* Pull Request #136: Remove excessive quoting from shebang headers +* BB Pull Request #136: Remove excessive quoting from shebang headers for Jython. ------- 17.1.1 ------ @@ -186,14 +370,12 @@ CHANGES deprecated imp module (`ref <https://bitbucket.org/pypa/setuptools/commits/f572ec9563d647fa8d4ffc534f2af8070ea07a8b#comment-1881283>`_). ----- 17.1 ---- * Issue #380: Add support for range operators on environment marker evaluation. ----- 17.0 ---- @@ -202,16 +384,16 @@ CHANGES the name. Removes unintended functionality and brings behavior into parity with pip. ----- 16.0 ---- -* Pull Request #130: Better error messages for errors in +* BB Pull Request #130: Better error messages for errors in parsed requirements. -* Pull Request #133: Removed ``setuptools.tests`` from the +* BB Pull Request #133: Removed ``setuptools.tests`` from the installed packages. +* BB Pull Request #129: Address deprecation warning due to usage + of imp module. ----- 15.2 ---- @@ -219,18 +401,16 @@ CHANGES ``pkg_resources._initialize_master_working_set``, allowing for imperative re-initialization of the master working set. ----- 15.1 ---- * Updated to Packaging 15.1 to address Packaging #28. * Fix ``setuptools.sandbox._execfile()`` with Python 3.1. ----- 15.0 ---- -* Pull Request #126: DistributionNotFound message now lists the package or +* BB Pull Request #126: DistributionNotFound message now lists the package or packages that required it. E.g.:: pkg_resources.DistributionNotFound: The 'colorama>=0.3.1' distribution was not found and is required by smlib.log. @@ -240,7 +420,6 @@ CHANGES has since been changed, but older versions of buildout may experience problems. See Buildout #242 for details. ------- 14.3.1 ------ @@ -249,7 +428,6 @@ CHANGES * Issue #364: Replace deprecated usage with recommended usage of ``EntryPoint.load``. ----- 14.3 ---- @@ -257,7 +435,6 @@ CHANGES for creating the directory to avoid the subsequent warning if the directory is group writable. ----- 14.2 ---- @@ -265,27 +442,24 @@ CHANGES None for pyversion or platform can be compared against Distributions defining those attributes. ------- 14.1.1 ------ * Issue #360: Removed undesirable behavior from test runs, preventing write tests and installation to system site packages. ----- 14.1 ---- -* Pull Request #125: Add ``__ne__`` to Requirement class. +* BB Pull Request #125: Add ``__ne__`` to Requirement class. * Various refactoring of easy_install. ----- 14.0 ---- * Bootstrap script now accepts ``--to-dir`` to customize save directory or allow for re-use of existing repository of setuptools versions. See - Pull Request #112 for background. + BB Pull Request #112 for background. * Issue #285: ``easy_install`` no longer will default to installing packages to the "user site packages" directory if it is itself installed there. Instead, the user must pass ``--user`` in all cases to install @@ -295,38 +469,33 @@ CHANGES using the "install-dir" and "scripts-dir" parameters to easy_install through an appropriate distutils config file. ------- 13.0.2 ------ * Issue #359: Include pytest.ini in the sdist so invocation of py.test on the sdist honors the pytest configuration. ------- 13.0.1 ------ Re-release of 13.0. Intermittent connectivity issues caused the release process to fail and PyPI uploads no longer accept files for 13.0. ----- 13.0 ---- -* Issue #356: Back out Pull Request #119 as it requires Setuptools 10 or later +* Issue #356: Back out BB Pull Request #119 as it requires Setuptools 10 or later as the source during an upgrade. * Removed build_py class from setup.py. According to 892f439d216e, this functionality was added to support upgrades from old Distribute versions, 0.6.5 and 0.6.6. ----- 12.4 ---- -* Pull Request #119: Restore writing of ``setup_requires`` to metadata +* BB Pull Request #119: Restore writing of ``setup_requires`` to metadata (previously added in 8.4 and removed in 9.0). ----- 12.3 ---- @@ -336,7 +505,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Issue #354. Added documentation on building setuptools documentation. ----- 12.2 ---- @@ -347,40 +515,34 @@ process to fail and PyPI uploads no longer accept files for 13.0. remains deprecated for use by individual packages. * Simplified implementation of ``ez_setup.use_setuptools``. ----- 12.1 ---- -* Pull Request #118: Soften warning for non-normalized versions in +* BB Pull Request #118: Soften warning for non-normalized versions in Distribution. ------- 12.0.5 ------ * Issue #339: Correct Attribute reference in ``cant_write_to_target``. * Issue #336: Deprecated ``ez_setup.use_setuptools``. ------- 12.0.4 ------ * Issue #335: Fix script header generation on Windows. ------- 12.0.3 ------ * Fixed incorrect class attribute in ``install_scripts``. Tests would be nice. ------- 12.0.2 ------ * Issue #331: Fixed ``install_scripts`` command on Windows systems corrupting the header. ------- 12.0.1 ------ @@ -388,7 +550,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. compatibility. For the future, tools should construct a CommandSpec explicitly. ----- 12.0 ---- @@ -399,14 +560,12 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Deprecated ``easy_install.ScriptWriter.get_writer``, replaced by ``.best()`` with slightly different semantics (no force_windows flag). ------- 11.3.1 ------ * Issue #327: Formalize and restore support for any printable character in an entry point name. ----- 11.3 ---- @@ -420,13 +579,11 @@ process to fail and PyPI uploads no longer accept files for 13.0. getattr(ep, "resolve", lambda: ep.load(require=False))() ----- 11.2 ---- * Pip #2326: Report deprecation warning at stacklevel 2 for easier diagnosis. ----- 11.1 ---- @@ -437,20 +594,17 @@ process to fail and PyPI uploads no longer accept files for 13.0. a VersionConflict when no dependent package context is known. New unit tests now capture the expected interface. ----- 11.0 ---- * Interop #3: Upgrade to Packaging 15.0; updates to PEP 440 so that >1.7 does not exclude 1.7.1 but does exclude 1.7.0 and 1.7.0.post1. ------- 10.2.1 ------ * Issue #323: Fix regression in entry point name parsing. ----- 10.2 ---- @@ -460,7 +614,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Substantial refactoring of all unit tests. Tests are now much leaner and re-use a lot of fixtures and contexts for better clarity of purpose. ----- 10.1 ---- @@ -469,13 +622,11 @@ process to fail and PyPI uploads no longer accept files for 13.0. so that systems relying on that interface do not fail (namely, Ubuntu 12.04 and similar Debian releases). ------- 10.0.1 ------ * Issue #319: Fixed issue installing pure distutils packages. ----- 10.0 ---- @@ -488,53 +639,45 @@ process to fail and PyPI uploads no longer accept files for 13.0. upgrade (or downgrade) itself even when its own metadata and implementation change. ---- 9.1 --- * Prefer vendored packaging library `as recommended <https://github.com/jaraco/setuptools/commit/170657b68f4b92e7e1bf82f5e19a831f5744af67#commitcomment-9109448>`_. ------ 9.0.1 ----- * Issue #312: Restored presence of pkg_resources API tests (doctest) to sdist. ---- 9.0 --- * Issue #314: Disabled support for ``setup_requires`` metadata to avoid issue where Setuptools was unable to upgrade over earlier versions. ---- 8.4 --- -* Pull Request #106: Now write ``setup_requires`` metadata. +* BB Pull Request #106: Now write ``setup_requires`` metadata. ---- 8.3 --- * Issue #311: Decoupled pkg_resources from setuptools once again. ``pkg_resources`` is now a package instead of a module. ------ 8.2.1 ----- * Issue #306: Suppress warnings about Version format except in select scenarios (such as installation). ---- 8.2 --- -* Pull Request #85: Search egg-base when adding egg-info to manifest. +* BB Pull Request #85: Search egg-base when adding egg-info to manifest. ---- 8.1 --- @@ -544,7 +687,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. ``pkg_resources.PEP440Warning``, instead of RuntimeWarning. * Disabled warnings on empty versions. ------ 8.0.4 ----- @@ -554,31 +696,27 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Issue #296: Add warning when a version is parsed as legacy. This warning will make it easier for developers to recognize deprecated version numbers. ------ 8.0.3 ----- * Issue #296: Restored support for ``__hash__`` on parse_version results. ------ 8.0.2 ----- * Issue #296: Restored support for ``__getitem__`` and sort operations on parse_version result. ------ 8.0.1 ----- * Issue #296: Restore support for iteration over parse_version result, but deprecated that usage with a warning. Fixes failure with buildout. ---- 8.0 --- -* Implement `PEP 440 <http://legacy.python.org/dev/peps/pep-0440/>`_ within +* Implement PEP 440 within pkg_resources and setuptools. This change deprecates some version numbers such that they will no longer be installable without using the ``===`` escape hatch. See `the changes to test_resources @@ -587,7 +725,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. supported. Setuptools now "vendors" the `packaging <https://github.com/pypa/packaging>`_ library. ---- 7.0 --- @@ -604,28 +741,24 @@ process to fail and PyPI uploads no longer accept files for 13.0. adapted to ignore ``.eggs``. The files will need to be manually moved or will be retrieved again. Most use cases will require no attention. ---- 6.1 --- * Issue #268: When resolving package versions, a VersionConflict now reports which package previously required the conflicting version. ------ 6.0.2 ----- * Issue #262: Fixed regression in pip install due to egg-info directories being omitted. Re-opens Issue #118. ------ 6.0.1 ----- * Issue #259: Fixed regression with namespace package handling on ``single version, externally managed`` installs. ---- 6.0 --- @@ -646,9 +779,9 @@ process to fail and PyPI uploads no longer accept files for 13.0. Any users producing distributions with filenames that match those above case-insensitively, but not case-sensitively, should rename those files in their repository for better portability. -* Pull Request #72: When using ``single_version_externally_managed``, the +* BB Pull Request #72: When using ``single_version_externally_managed``, the exclusion list now includes Python 3.2 ``__pycache__`` entries. -* Pull Request #76 and Pull Request #78: lines in top_level.txt are now +* BB Pull Request #76 and BB Pull Request #78: lines in top_level.txt are now ordered deterministically. * Issue #118: The egg-info directory is now no longer included in the list of outputs. @@ -656,7 +789,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. recognize the specially-packaged compiler package for easy extension module support on Python 2.6, 2.7, and 3.2. ---- 5.8 --- @@ -664,7 +796,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. Python 3, supporting environments where builtins have been patched to make Python 3 look more like Python 2. ---- 5.7 --- @@ -675,20 +806,17 @@ process to fail and PyPI uploads no longer accept files for 13.0. notes and detailed in Issue #154 was likely not an increase over the status quo, but rather only an increase over not storing the zip info at all. ---- 5.6 --- * Issue #242: Use absolute imports in svn_utils to avoid issues if the installing package adds an xml module to the path. ------ 5.5.1 ----- * Issue #239: Fix typo in 5.5 such that fix did not take. ---- 5.5 --- @@ -696,20 +824,17 @@ process to fail and PyPI uploads no longer accept files for 13.0. Distribution objects and validates the syntax just like install_requires and tests_require directives. ------ 5.4.2 ----- * Issue #236: Corrected regression in execfile implementation for Python 2.6. ------ 5.4.1 ----- * Python #7776: (ssl_support) Correct usage of host for validation when tunneling for HTTPS. ---- 5.4 --- @@ -720,7 +845,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. in startup time by enabling this feature. This feature is not enabled by default because it causes a substantial increase in memory usage. ---- 5.3 --- @@ -729,7 +853,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Prune revision control directories (e.g .svn) from base path as well as sub-directories. ---- 5.2 --- @@ -741,7 +864,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. * During install_egg_info, the generated lines for namespace package .pth files are now processed even during a dry run. ---- 5.1 --- @@ -749,20 +871,17 @@ process to fail and PyPI uploads no longer accept files for 13.0. building on the work in Issue #168. Special thanks to Jurko Gospodnetic and PJE. ------ 5.0.2 ----- * Issue #220: Restored script templates. ------ 5.0.1 ----- * Renamed script templates to end with .tmpl now that they no longer need to be processed by 2to3. Fixes spurious syntax errors during build/install. ---- 5.0 --- @@ -770,7 +889,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Incidentally, script templates were updated not to include the triple-quote escaping. -------------------------- 3.7.1 and 3.8.1 and 4.0.1 ------------------------- @@ -778,48 +896,41 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Issue #218: Setuptools 3.8.1 superseded 4.0.1, and 4.x was removed from the available versions to install. ---- 4.0 --- * Issue #210: ``setup.py develop`` now copies scripts in binary mode rather than text mode, matching the behavior of the ``install`` command. ---- 3.8 --- * Extend Issue #197 workaround to include all Python 3 versions prior to 3.2.2. ---- 3.7 --- * Issue #193: Improved handling of Unicode filenames when building manifests. ---- 3.6 --- * Issue #203: Honor proxy settings for Powershell downloader in the bootstrap routine. ------ 3.5.2 ----- * Issue #168: More robust handling of replaced zip files and stale caches. Fixes ZipImportError complaining about a 'bad local header'. ------ 3.5.1 ----- * Issue #199: Restored ``install._install`` for compatibility with earlier NumPy versions. ---- 3.5 --- @@ -830,32 +941,27 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Issue #192: Preferred bootstrap location is now https://bootstrap.pypa.io/ez_setup.py (mirrored from former location). ------ 3.4.4 ----- * Issue #184: Correct failure where find_package over-matched packages when directory traversal isn't short-circuited. ------ 3.4.3 ----- * Issue #183: Really fix test command with Python 3.1. ------ 3.4.2 ----- * Issue #183: Fix additional regression in test command on Python 3.1. ------ 3.4.1 ----- * Issue #180: Fix regression in test command not caught by py.test-run tests. ---- 3.4 --- @@ -866,41 +972,35 @@ process to fail and PyPI uploads no longer accept files for 13.0. now installs naturally on IronPython. Behavior on CPython should be unchanged. ---- 3.3 --- * Add ``include`` parameter to ``setuptools.find_packages()``. ---- 3.2 --- -* Pull Request #39: Add support for C++ targets from Cython ``.pyx`` files. +* BB Pull Request #39: Add support for C++ targets from Cython ``.pyx`` files. * Issue #162: Update dependency on certifi to 1.0.1. * Issue #164: Update dependency on wincertstore to 0.2. ---- 3.1 --- * Issue #161: Restore Features functionality to allow backward compatibility (for Features) until the uses of that functionality is sufficiently removed. ------ 3.0.2 ----- * Correct typo in previous bugfix. ------ 3.0.1 ----- * Issue #157: Restore support for Python 2.6 in bootstrap script where ``zipfile.ZipFile`` does not yet have support for context managers. ---- 3.0 --- @@ -923,12 +1023,11 @@ process to fail and PyPI uploads no longer accept files for 13.0. security vulnerabilities presented by use of tar archives in ez_setup.py. It also leverages the security features added to ZipFile.extract in Python 2.7.4. * Issue #65: Removed deprecated Features functionality. -* Pull Request #28: Remove backport of ``_bytecode_filenames`` which is +* BB Pull Request #28: Remove backport of ``_bytecode_filenames`` which is available in Python 2.6 and later, but also has better compatibility with Python 3 environments. * Issue #156: Fix spelling of __PYVENV_LAUNCHER__ variable. ---- 2.2 --- @@ -937,21 +1036,18 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Issue #128: Fixed issue where only the first dependency link was honored in a distribution where multiple dependency links were supplied. ------ 2.1.2 ----- * Issue #144: Read long_description using codecs module to avoid errors installing on systems where LANG=C. ------ 2.1.1 ----- * Issue #139: Fix regression in re_finder for CVS repos (and maybe Git repos as well). ---- 2.1 --- @@ -959,7 +1055,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. in a zip-imported file. * Issue #131: Fix RuntimeError when constructing an egg fetcher. ------ 2.0.2 ----- @@ -967,13 +1062,11 @@ process to fail and PyPI uploads no longer accept files for 13.0. not containing parser module. * Fix NameError in ``sdist:re_finder``. ------ 2.0.1 ----- * Issue #124: Fixed error in list detection in upload_docs. ---- 2.0 --- @@ -986,14 +1079,12 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Removed ``pkg_resources.ImpWrapper``. Clients that expected this class should use ``pkgutil.ImpImporter`` instead. ------ 1.4.2 ----- * Issue #116: Correct TypeError when reading a local package index on Python 3. ------ 1.4.1 ----- @@ -1023,28 +1114,24 @@ process to fail and PyPI uploads no longer accept files for 13.0. for legacy SVN releases and support for SVN without the subprocess command would simple go away as support for the older SVNs does. ---- 1.4 --- * Issue #27: ``easy_install`` will now use credentials from .pypirc if present for connecting to the package index. -* Pull Request #21: Omit unwanted newlines in ``package_index._encode_auth`` +* BB Pull Request #21: Omit unwanted newlines in ``package_index._encode_auth`` when the username/password pair length indicates wrapping. ------ 1.3.2 ----- * Issue #99: Fix filename encoding issues in SVN support. ------ 1.3.1 ----- * Remove exuberant warning in SVN support when SVN is not used. ---- 1.3 --- @@ -1055,7 +1142,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. implementation if present. * Correct NameError in ``ssl_support`` module (``socket.error``). ---- 1.2 --- @@ -1068,7 +1154,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Setuptools "natural" launcher support, introduced in 1.0, is now officially supported. ------ 1.1.7 ----- @@ -1079,39 +1164,33 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Distribute #363 and Issue #55: Skip an sdist test that fails on locales other than UTF-8. ------ 1.1.6 ----- * Distribute #349: ``sandbox.execfile`` now opens the target file in binary mode, thus honoring a BOM in the file when compiled. ------ 1.1.5 ----- * Issue #69: Second attempt at fix (logic was reversed). ------ 1.1.4 ----- * Issue #77: Fix error in upload command (Python 2.4). ------ 1.1.3 ----- * Fix NameError in previous patch. ------ 1.1.2 ----- * Issue #69: Correct issue where 404 errors are returned for URLs with fragments in them (such as #egg=). ------ 1.1.1 ----- @@ -1119,7 +1198,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. environments where a trusted SSL connection cannot be validated. * Issue #76: Fix AttributeError in upload command with Python 2.4. ---- 1.1 --- @@ -1127,7 +1205,6 @@ process to fail and PyPI uploads no longer accept files for 13.0. condition when a host is blocked via ``--allow-hosts``. * Issue #72: Restored Python 2.4 compatibility in ``ez_setup.py``. ---- 1.0 --- @@ -1164,13 +1241,11 @@ not all users will find 1.0 a drop-in replacement for 0.9. * Removed ``--ignore-conflicts-at-my-risk`` and ``--delete-conflicting`` options to easy_install. These options have been deprecated since 0.6a11. ------ 0.9.8 ----- * Issue #53: Fix NameErrors in `_vcs_split_rev_from_url`. ------ 0.9.7 ----- @@ -1180,79 +1255,67 @@ not all users will find 1.0 a drop-in replacement for 0.9. referenced by bookmark. * Add underscore-separated keys to environment markers (markerlib). ------ 0.9.6 ----- * Issue #44: Test failure on Python 2.4 when MD5 hash doesn't have a `.name` attribute. ------ 0.9.5 ----- * Python #17980: Fix security vulnerability in SSL certificate validation. ------ 0.9.4 ----- * Issue #43: Fix issue (introduced in 0.9.1) with version resolution when upgrading over other releases of Setuptools. ------ 0.9.3 ----- * Issue #42: Fix new ``AttributeError`` introduced in last fix. ------ 0.9.2 ----- * Issue #42: Fix regression where blank checksums would trigger an ``AttributeError``. ------ 0.9.1 ----- * Distribute #386: Allow other positional and keyword arguments to os.open. * Corrected dependency on certifi mis-referenced in 0.9. ---- 0.9 --- * `package_index` now validates hashes other than MD5 in download links. ---- 0.8 --- * Code base now runs on Python 2.4 - Python 3.3 without Python 2to3 conversion. ------ 0.7.8 ----- * Distribute #375: Yet another fix for yet another regression. ------ 0.7.7 ----- * Distribute #375: Repair AttributeError created in last release (redo). * Issue #30: Added test for get_cache_path. ------ 0.7.6 ----- * Distribute #375: Repair AttributeError created in last release. ------ 0.7.5 ----- @@ -1262,33 +1325,28 @@ not all users will find 1.0 a drop-in replacement for 0.9. ``SETUPTOOLS_DISABLE_VERSIONED_EASY_INSTALL_SCRIPT`` in addition to the now deprecated ``DISTRIBUTE_DISABLE_VERSIONED_EASY_INSTALL_SCRIPT``. ------ 0.7.4 ----- * Issue #20: Fix comparison of parsed SVN version on Python 3. ------ 0.7.3 ----- * Issue #1: Disable installation of Windows-specific files on non-Windows systems. * Use new sysconfig module with Python 2.7 or >=3.2. ------ 0.7.2 ----- * Issue #14: Use markerlib when the `parser` module is not available. * Issue #10: ``ez_setup.py`` now uses HTTPS to download setuptools from PyPI. ------ 0.7.1 ----- * Fix NameError (Issue #3) again - broken in bad merge. ---- 0.7 --- @@ -1306,13 +1364,11 @@ Added several features that were slated for setuptools 0.6c12: * Added support for SSL certificate validation when installing packages from an HTTPS service. ------ 0.7b4 ----- * Issue #3: Fixed NameError in SSL support. ------- 0.6.49 ------ @@ -1320,21 +1376,18 @@ Added several features that were slated for setuptools 0.6c12: to avoid errors when the cache path does not yet exist. Fixes the error reported in Distribute #375. ------- 0.6.48 ------ * Correct AttributeError in ``ResourceManager.get_cache_path`` introduced in 0.6.46 (redo). ------- 0.6.47 ------ * Correct AttributeError in ``ResourceManager.get_cache_path`` introduced in 0.6.46. ------- 0.6.46 ------ @@ -1342,27 +1395,23 @@ Added several features that were slated for setuptools 0.6c12: customized egg cache location specifies a directory that's group- or world-writable. ------- 0.6.45 ------ * Distribute #379: ``distribute_setup.py`` now traps VersionConflict as well, restoring ability to upgrade from an older setuptools version. ------- 0.6.44 ------ * ``distribute_setup.py`` has been updated to allow Setuptools 0.7 to satisfy use_setuptools. ------- 0.6.43 ------ * Distribute #378: Restore support for Python 2.4 Syntax (regression in 0.6.42). ------- 0.6.42 ------ @@ -1370,7 +1419,6 @@ Added several features that were slated for setuptools 0.6c12: * Distribute #337: Moved site.py to setuptools/site-patch.py (graft of very old patch from setuptools trunk which inspired PR #31). ------- 0.6.41 ------ @@ -1379,14 +1427,12 @@ Added several features that were slated for setuptools 0.6c12: * Added a new function ``easy_install.get_win_launcher`` which may be used by third-party libraries such as buildout to get a suitable script launcher. ------- 0.6.40 ------ * Distribute #376: brought back cli.exe and gui.exe that were deleted in the previous release. ------- 0.6.39 ------ @@ -1398,13 +1444,11 @@ Added several features that were slated for setuptools 0.6c12: check the contents of the file against the zip contents during each invocation of get_resource_filename. ------- 0.6.38 ------ * Distribute #371: The launcher manifest file is now installed properly. ------- 0.6.37 ------ @@ -1414,17 +1458,15 @@ Added several features that were slated for setuptools 0.6c12: in `this Microsoft article <http://technet.microsoft.com/en-us/library/cc709628%28WS.10%29.aspx>`_. ------- 0.6.36 ------ -* Pull Request #35: In Buildout #64, it was reported that +* BB Pull Request #35: In Buildout #64, it was reported that under Python 3, installation of distutils scripts could attempt to copy the ``__pycache__`` directory as a file, causing an error, apparently only under Windows. Easy_install now skips all directories when processing metadata scripts. ------- 0.6.35 ------ @@ -1436,13 +1478,11 @@ how it parses version numbers. 0.6. Updated the documentation to match more closely with the version parsing as intended in setuptools 0.6. ------- 0.6.34 ------ * Distribute #341: 0.6.33 fails to build under Python 2.4. ------- 0.6.33 ------ @@ -1455,7 +1495,6 @@ how it parses version numbers. for details. * Distribute #341: Fix a ResourceWarning. ------- 0.6.32 ------ @@ -1464,7 +1503,6 @@ how it parses version numbers. * Distribute #335: Backed out `setup_requires` superceding installed requirements until regression can be addressed. ------- 0.6.31 ------ @@ -1489,18 +1527,16 @@ how it parses version numbers. would have been in had that egg been on the path when pkg_resources was first imported. ------- 0.6.30 ------ * Distribute #328: Clean up temporary directories in distribute_setup.py. * Fix fatal bug in distribute_setup.py. ------- 0.6.29 ------ -* Pull Request #14: Honor file permissions in zip files. +* BB Pull Request #14: Honor file permissions in zip files. * Distribute #327: Merged pull request #24 to fix a dependency problem with pip. * Merged pull request #23 to fix https://github.com/pypa/virtualenv/issues/301. * If Sphinx is installed, the `upload_docs` command now runs `build_sphinx` @@ -1528,7 +1564,6 @@ how it parses version numbers. * `distribute_setup.py` now allows a `--download-base` argument for retrieving distribute from a specified location. ------- 0.6.28 ------ @@ -1538,7 +1573,6 @@ how it parses version numbers. * Distribute #283: Fix and disable scanning of `*.pyc` / `*.pyo` files on Python 3.3. ------- 0.6.27 ------ @@ -1549,7 +1583,6 @@ how it parses version numbers. * Distribute #231: Don't fiddle with system python when used with buildout (bootstrap.py) ------- 0.6.26 ------ @@ -1558,7 +1591,6 @@ how it parses version numbers. installation of a source distribution; now fulfillment of setup_requires dependencies will honor the parameters passed to easy_install. ------- 0.6.25 ------ @@ -1574,13 +1606,11 @@ how it parses version numbers. 449. * Distribute #273: Legacy script launchers now install with Python2/3 support. ------- 0.6.24 ------ * Distribute #249: Added options to exclude 2to3 fixers ------- 0.6.23 ------ @@ -1596,13 +1626,11 @@ how it parses version numbers. * Distribute #227: easy_install now passes its arguments to setup.py bdist_egg * Distribute #225: Fixed a NameError on Python 2.5, 2.4 ------- 0.6.21 ------ * Distribute #225: FIxed a regression on py2.4 ------- 0.6.20 ------ @@ -1610,19 +1638,16 @@ how it parses version numbers. * Distribute #212: Fix issue where easy_instal fails on Python 3 on windows installer. * Distribute #213: Fix typo in documentation. ------- 0.6.19 ------ * Distribute #206: AttributeError: 'HTTPMessage' object has no attribute 'getheaders' ------- 0.6.18 ------ * Distribute #210: Fixed a regression introduced by Distribute #204 fix. ------- 0.6.17 ------ @@ -1635,7 +1660,6 @@ how it parses version numbers. * Distribute #205: Sandboxing doesn't preserve working_set. Leads to setup_requires problems. ------- 0.6.16 ------ @@ -1645,7 +1669,6 @@ how it parses version numbers. * Distribute #195: Cython build support. * Distribute #200: Issues with recognizing 64-bit packages on Windows. ------- 0.6.15 ------ @@ -1653,7 +1676,6 @@ how it parses version numbers. * Several issues under Python 3 has been solved. * Distribute #146: Fixed missing DLL files after easy_install of windows exe package. ------- 0.6.14 ------ @@ -1663,7 +1685,6 @@ how it parses version numbers. Thanks to David and Zooko. * Distribute #174: Fixed the edit mode when its used with setuptools itself ------- 0.6.13 ------ @@ -1672,13 +1693,11 @@ how it parses version numbers. * Distribute #163: scan index links before external links, and don't use the md5 when comparing two distributions ------- 0.6.12 ------ * Distribute #149: Fixed various failures on 2.3/2.4 ------- 0.6.11 ------ @@ -1695,7 +1714,6 @@ how it parses version numbers. * Distribute #138: cant_write_to_target error when setup_requires is used. * Distribute #147: respect the sys.dont_write_bytecode flag ------- 0.6.10 ------ @@ -1703,7 +1721,6 @@ how it parses version numbers. zc.buildout uses the exception message to get the name of the distribution. ------ 0.6.9 ----- @@ -1730,14 +1747,12 @@ how it parses version numbers. * Distribute #100: making sure there's no SandboxViolation when the setup script patches setuptools. ------ 0.6.8 ----- * Added "check_packages" in dist. (added in Setuptools 0.6c11) * Fixed the DONT_PATCH_SETUPTOOLS state. ------ 0.6.7 ----- @@ -1762,21 +1777,19 @@ how it parses version numbers. * Distribute #74: no_fake should be True by default. * Distribute #72: avoid a bootstrapping issue with easy_install -U ------ 0.6.6 ----- * Unified the bootstrap file so it works on both py2.x and py3k without 2to3 (patch by Holger Krekel) ------ 0.6.5 ----- * Distribute #65: cli.exe and gui.exe are now generated at build time, depending on the platform in use. -* Distribute #67: Fixed doc typo (PEP 381/382) +* Distribute #67: Fixed doc typo (PEP 381/PEP 382). * Distribute no longer shadows setuptools if we require a 0.7-series setuptools. And an error is raised when installing a 0.7 setuptools with @@ -1788,7 +1801,6 @@ how it parses version numbers. * Fixed a hole in sandboxing allowing builtin file to write outside of the sandbox. ------ 0.6.4 ----- @@ -1800,7 +1812,6 @@ how it parses version numbers. * Fixed a bootstrap bug on the use_setuptools() API. ------ 0.6.3 ----- @@ -1814,7 +1825,6 @@ bootstrapping * Fixed a bug in sorting that caused bootstrap to fail on Python 3. ------ 0.6.2 ----- @@ -1846,7 +1856,6 @@ bootstrapping * Make sure setuptools is patched when running through easy_install This closes Old Setuptools #40. ------ 0.6.1 ----- @@ -1874,7 +1883,6 @@ bootstrapping and --root or --prefix is provided, but is not in the same location. This closes Distribute #10. ---- 0.6 --- @@ -1917,7 +1925,6 @@ easy_install * Immediately close all file handles. This closes Distribute #3. ------ 0.6c9 ----- @@ -1958,7 +1965,6 @@ easy_install gracefully under Google App Engine (with an ``ImportError`` loading the C-based module, instead of getting a ``NameError``). ------ 0.6c7 ----- @@ -1969,7 +1975,6 @@ easy_install ``--root`` or ``--single-version-externally-managed``, due to the parent package not having the child package as an attribute. ------ 0.6c6 ----- @@ -1993,7 +1998,6 @@ easy_install * Fix ``find_packages()`` treating ``ez_setup`` and directories with ``.`` in their names as packages. ------ 0.6c5 ----- @@ -2003,7 +2007,6 @@ easy_install * Fix uploaded ``bdist_wininst`` packages being described as suitable for "any" version by Python 2.5, even if a ``--target-version`` was specified. ------ 0.6c4 ----- @@ -2033,13 +2036,11 @@ easy_install listed a namespace package ``foo.bar`` without explicitly listing ``foo`` as a namespace package. ------ 0.6c3 ----- * Fixed breakages caused by Subversion 1.4's new "working copy" format ------ 0.6c2 ----- @@ -2050,7 +2051,6 @@ easy_install * Running ``setup.py develop`` on a setuptools-using project will now install setuptools if needed, instead of only downloading the egg. ------ 0.6c1 ----- @@ -2074,7 +2074,6 @@ easy_install the version was overridden on the command line that built the source distribution.) ------ 0.6b4 ----- @@ -2087,7 +2086,6 @@ easy_install * Fixed redundant warnings about missing ``README`` file(s); it should now appear only if you are actually a source distribution. ------ 0.6b3 ----- @@ -2098,7 +2096,6 @@ easy_install ``include_package_data`` and ``package_data`` are used to refer to the same files. ------ 0.6b1 ----- @@ -2106,7 +2103,6 @@ easy_install the name of a ``.py`` loader/wrapper. (Python's import machinery ignores this suffix when searching for an extension module.) ------- 0.6a11 ------ @@ -2140,13 +2136,11 @@ easy_install it. Previously, the file could be left open and the actual error would be masked by problems trying to remove the open file on Windows systems. ------- 0.6a10 ------ * Fixed the ``develop`` command ignoring ``--find-links``. ------ 0.6a9 ----- @@ -2198,7 +2192,6 @@ easy_install back into an ``.egg`` file or directory and install it as such. ------ 0.6a8 ----- @@ -2226,13 +2219,11 @@ easy_install metadata cache to pretend that the egg has valid version information, until it has a chance to make it actually be so (via the ``egg_info`` command). ------ 0.6a5 ----- * Fixed missing gui/cli .exe files in distribution. Fixed bugs in tests. ------ 0.6a3 ----- @@ -2240,7 +2231,6 @@ easy_install on Windows and other platforms. (The special handling is only for Windows; other platforms are treated the same as for ``console_scripts``.) ------ 0.6a2 ----- @@ -2249,7 +2239,6 @@ easy_install scripts get an ``.exe`` wrapper so you can just type their name. On other platforms, the scripts are written without a file extension. ------ 0.6a1 ----- @@ -2295,7 +2284,6 @@ easy_install or documented, and never would have worked without EasyInstall - which it pre-dated and was never compatible with. ------- 0.5a12 ------ @@ -2303,14 +2291,12 @@ easy_install ``python -m``, and marks them as unsafe for zipping, since Python 2.4 can't handle ``-m`` on zipped modules. ------- 0.5a11 ------ * Fix breakage of the "develop" command that was caused by the addition of ``--always-unzip`` to the ``easy_install`` command. ------ 0.5a9 ----- @@ -2345,7 +2331,6 @@ easy_install * Fixed the swapped ``-d`` and ``-b`` options of ``bdist_egg``. ------ 0.5a8 ----- @@ -2373,7 +2358,6 @@ easy_install * Added a "setopt" command that sets a single option in a specified distutils configuration file. ------ 0.5a7 ----- @@ -2381,7 +2365,6 @@ easy_install fix for "upload" and a temporary workaround for lack of .egg support in PyPI. ------ 0.5a6 ----- @@ -2400,7 +2383,6 @@ easy_install revisions compare *lower* than the version specified in setup.py (e.g. by using ``--tag-build=dev``). ------ 0.5a5 ----- @@ -2428,7 +2410,6 @@ easy_install accordingly. ``easy_install.py`` is still installed as a script, but not as a module. ------ 0.5a4 ----- @@ -2444,7 +2425,6 @@ easy_install * Setup scripts using setuptools now always install using ``easy_install`` internally, for ease of uninstallation and upgrading. ------ 0.5a1 ----- @@ -2459,7 +2439,6 @@ easy_install from setuptools import setup # etc... ------ 0.4a2 ----- @@ -2487,7 +2466,6 @@ easy_install their ``command_consumes_arguments`` attribute to ``True`` in order to receive an ``args`` option containing the rest of the command line. ------ 0.3a2 ----- @@ -2497,8 +2475,8 @@ easy_install * Misc. bug fixes ------ 0.3a1 ----- * Initial release. + diff --git a/MANIFEST.in b/MANIFEST.in index 8a4523a9..dfea2049 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ recursive-include tests *.py recursive-include setuptools/tests *.html recursive-include docs *.py *.txt *.conf *.css *.css_t Makefile indexsidebar.html recursive-include _markerlib *.py +recursive-include setuptools/_vendor * recursive-include pkg_resources *.py *.txt include *.py include *.txt diff --git a/Makefile b/Makefile deleted file mode 100644 index 37dd26eb..00000000 --- a/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -empty: - exit 1 - -update-vendored: - rm -rf pkg_resources/_vendor/packaging - pip install -r pkg_resources/_vendor/vendored.txt -t pkg_resources/_vendor/ - rm -rf pkg_resources/_vendor/*.{egg,dist}-info @@ -18,7 +18,7 @@ basic routine, so below are some examples to get you started. Setuptools requires Python 2.6 or later. To install setuptools on Python 2.4 or Python 2.5, use the `bootstrap script for Setuptools 1.x -<https://bitbucket.org/pypa/setuptools/raw/bootstrap-py24/ez_setup.py>`_. +<https://raw.githubusercontent.com/pypa/setuptools/bootstrap-py24/ez_setup.py>`_. The link provided to ez_setup.py is a bookmark to bootstrap script for the latest known stable release. @@ -176,7 +176,7 @@ them there, so this reference list can be updated. If you have working, *tested* patches to correct problems or add features, you may submit them to the `setuptools bug tracker`_. -.. _setuptools bug tracker: https://bitbucket.org/pypa/setuptools/issues +.. _setuptools bug tracker: https://github.com/pypa/setuptools/issues .. _The Internal Structure of Python Eggs: https://pythonhosted.org/setuptools/formats.html .. _The setuptools Developer's Guide: https://pythonhosted.org/setuptools/setuptools.html .. _The pkg_resources API reference: https://pythonhosted.org/setuptools/pkg_resources.html diff --git a/_markerlib/__init__.py b/_markerlib/__init__.py deleted file mode 100644 index e2b237b1..00000000 --- a/_markerlib/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -try: - import ast - from _markerlib.markers import default_environment, compile, interpret -except ImportError: - if 'ast' in globals(): - raise - def default_environment(): - return {} - def compile(marker): - def marker_fn(environment=None, override=None): - # 'empty markers are True' heuristic won't install extra deps. - return not marker.strip() - marker_fn.__doc__ = marker - return marker_fn - def interpret(marker, environment=None, override=None): - return compile(marker)() diff --git a/_markerlib/markers.py b/_markerlib/markers.py deleted file mode 100644 index fa837061..00000000 --- a/_markerlib/markers.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- -"""Interpret PEP 345 environment markers. - -EXPR [in|==|!=|not in] EXPR [or|and] ... - -where EXPR belongs to any of those: - - python_version = '%s.%s' % (sys.version_info[0], sys.version_info[1]) - python_full_version = sys.version.split()[0] - os.name = os.name - sys.platform = sys.platform - platform.version = platform.version() - platform.machine = platform.machine() - platform.python_implementation = platform.python_implementation() - a free string, like '2.6', or 'win32' -""" - -__all__ = ['default_environment', 'compile', 'interpret'] - -import ast -import os -import platform -import sys -import weakref - -_builtin_compile = compile - -try: - from platform import python_implementation -except ImportError: - if os.name == "java": - # Jython 2.5 has ast module, but not platform.python_implementation() function. - def python_implementation(): - return "Jython" - else: - raise - - -# restricted set of variables -_VARS = {'sys.platform': sys.platform, - 'python_version': '%s.%s' % sys.version_info[:2], - # FIXME parsing sys.platform is not reliable, but there is no other - # way to get e.g. 2.7.2+, and the PEP is defined with sys.version - 'python_full_version': sys.version.split(' ', 1)[0], - 'os.name': os.name, - 'platform.version': platform.version(), - 'platform.machine': platform.machine(), - 'platform.python_implementation': python_implementation(), - 'extra': None # wheel extension - } - -for var in list(_VARS.keys()): - if '.' in var: - _VARS[var.replace('.', '_')] = _VARS[var] - -def default_environment(): - """Return copy of default PEP 385 globals dictionary.""" - return dict(_VARS) - -class ASTWhitelist(ast.NodeTransformer): - def __init__(self, statement): - self.statement = statement # for error messages - - ALLOWED = (ast.Compare, ast.BoolOp, ast.Attribute, ast.Name, ast.Load, ast.Str) - # Bool operations - ALLOWED += (ast.And, ast.Or) - # Comparison operations - ALLOWED += (ast.Eq, ast.Gt, ast.GtE, ast.In, ast.Is, ast.IsNot, ast.Lt, ast.LtE, ast.NotEq, ast.NotIn) - - def visit(self, node): - """Ensure statement only contains allowed nodes.""" - if not isinstance(node, self.ALLOWED): - raise SyntaxError('Not allowed in environment markers.\n%s\n%s' % - (self.statement, - (' ' * node.col_offset) + '^')) - return ast.NodeTransformer.visit(self, node) - - def visit_Attribute(self, node): - """Flatten one level of attribute access.""" - new_node = ast.Name("%s.%s" % (node.value.id, node.attr), node.ctx) - return ast.copy_location(new_node, node) - -def parse_marker(marker): - tree = ast.parse(marker, mode='eval') - new_tree = ASTWhitelist(marker).generic_visit(tree) - return new_tree - -def compile_marker(parsed_marker): - return _builtin_compile(parsed_marker, '<environment marker>', 'eval', - dont_inherit=True) - -_cache = weakref.WeakValueDictionary() - -def compile(marker): - """Return compiled marker as a function accepting an environment dict.""" - try: - return _cache[marker] - except KeyError: - pass - if not marker.strip(): - def marker_fn(environment=None, override=None): - """""" - return True - else: - compiled_marker = compile_marker(parse_marker(marker)) - def marker_fn(environment=None, override=None): - """override updates environment""" - if override is None: - override = {} - if environment is None: - environment = default_environment() - environment.update(override) - return eval(compiled_marker, environment) - marker_fn.__doc__ = marker - _cache[marker] = marker_fn - return _cache[marker] - -def interpret(marker, environment=None): - return compile(marker)(environment) diff --git a/docs/conf.py b/docs/conf.py index 846e4b3e..8bba3e94 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -196,7 +196,7 @@ latex_documents = [ #latex_use_modindex = True link_files = { - 'CHANGES.txt': dict( + 'CHANGES.rst': dict( using=dict( BB='https://bitbucket.org', GH='https://github.com', @@ -204,11 +204,11 @@ link_files = { replace=[ dict( pattern=r"(Issue )?#(?P<issue>\d+)", - url='{BB}/pypa/setuptools/issue/{issue}', + url='{GH}/pypa/setuptools/issues/{issue}', ), dict( - pattern=r"Pull Request ?#(?P<pull_request>\d+)", - url='{BB}/pypa/setuptools/pull-request/{pull_request}', + pattern=r"BB Pull Request ?#(?P<bb_pull_request>\d+)", + url='{BB}/pypa/setuptools/pull-request/{bb_pull_request}', ), dict( pattern=r"Distribute #(?P<distribute>\d+)", @@ -246,6 +246,14 @@ link_files = { pattern=r"[Pp]ackaging (?P<packaging_ver>\d+(\.\d+)+)", url='{GH}/pypa/packaging/blob/{packaging_ver}/CHANGELOG.rst', ), + dict( + pattern=r"PEP[- ](?P<pep_number>\d+)", + url='https://www.python.org/dev/peps/pep-{pep_number:0>4}/', + ), + dict( + pattern=r"^(?m)((?P<scm_version>v?\d+(\.\d+){1,2}))\n[-=]+\n", + with_scm="{text}\n{rev[timestamp]:%d %b %Y}\n", + ), ], ), } diff --git a/docs/developer-guide.txt b/docs/developer-guide.txt index ae33649b..7cd3c6d2 100644 --- a/docs/developer-guide.txt +++ b/docs/developer-guide.txt @@ -23,10 +23,10 @@ quality of contribution. Project Management ------------------ -Setuptools is maintained primarily in Bitbucket at `this home -<https://bitbucket.org/pypa/setuptools>`_. Setuptools is maintained under the +Setuptools is maintained primarily in Github at `this home +<https://github.com/pypa/setuptools>`_. Setuptools is maintained under the Python Packaging Authority (PyPA) with several core contributors. All bugs -for Setuptools are filed and the canonical source is maintained in Bitbucket. +for Setuptools are filed and the canonical source is maintained in Github. User support and discussions are done through the issue tracker (for specific) issues, through the distutils-sig mailing list, or on IRC (Freenode) at @@ -44,7 +44,7 @@ describing the motivation behind making changes. First search to see if a ticket already exists for your issue. If not, create one. Try to think from the perspective of the reader. Explain what behavior you expected, what you got instead, and what factors might have contributed to the unexpected -behavior. In Bitbucket, surround a block of code or traceback with the triple +behavior. In Github, surround a block of code or traceback with the triple backtick "\`\`\`" so that it is formatted nicely. Filing a ticket provides a forum for justification, discussion, and @@ -61,17 +61,17 @@ jump to the in-depth discussion about any subject referenced. Source Code ----------- -Grab the code at Bitbucket:: +Grab the code at Github:: - $ hg clone https://bitbucket.org/pypa/setuptools + $ git checkout https://github.com/pypa/setuptools If you want to contribute changes, we recommend you fork the repository on -Bitbucket, commit the changes to your repository, and then make a pull request -on Bitbucket. If you make some changes, don't forget to: +Github, commit the changes to your repository, and then make a pull request +on Github. If you make some changes, don't forget to: -- add a note in CHANGES.txt +- add a note in CHANGES.rst -Please commit all changes in the 'default' branch against the latest available +Please commit all changes in the 'master' branch against the latest available commit or for bug-fixes, against an earlier commit or release in which the bug occurred. @@ -79,12 +79,11 @@ If you find yourself working on more than one issue at a time, Setuptools generally prefers Git-style branches, so use Mercurial bookmarks or Git branches or multiple forks to maintain separate efforts. -Setuptools also maintains an unofficial `Git mirror in Github -<https://github.com/jaraco/setuptools>`_. Contributors are welcome to submit -pull requests here, but because they are not integrated with the Bitbucket -Issue tracker, linking pull requests to tickets is more difficult. The -Continuous Integration tests that validate every release are run from this -mirror. +The Continuous Integration tests that validate every release are run +from this repository. + +For posterity, the old `Bitbucket mirror +<https://bitbucket.org/pypa/setuptools>`_ is available. ------- Testing @@ -104,10 +103,7 @@ Under continuous integration, additional tests may be run. See the Semantic Versioning ------------------- -Setuptools follows ``semver`` with some exceptions: - -- Uses two-segment version when three segment version ends in zero -- Omits 'v' prefix for tags. +Setuptools follows ``semver``. .. explain value of reflecting meaning in versions. diff --git a/docs/history.txt b/docs/history.txt index 268137cd..8e217503 100644 --- a/docs/history.txt +++ b/docs/history.txt @@ -5,4 +5,4 @@ History ******* -.. include:: ../CHANGES (links).txt +.. include:: ../CHANGES (links).rst diff --git a/docs/pkg_resources.txt b/docs/pkg_resources.txt index 3d40a1a2..7b979ec3 100644 --- a/docs/pkg_resources.txt +++ b/docs/pkg_resources.txt @@ -590,20 +590,7 @@ Requirements Parsing parse multiple specifiers from a string or iterable of strings, use ``parse_requirements()`` instead.) - The syntax of a requirement specifier can be defined in EBNF as follows:: - - requirement ::= project_name extras? versionspec? - versionspec ::= comparison version (',' comparison version)* - comparison ::= '<' | '<=' | '!=' | '==' | '>=' | '>' | '~=' | '===' - extras ::= '[' extralist? ']' - extralist ::= identifier (',' identifier)* - project_name ::= identifier - identifier ::= [-A-Za-z0-9_]+ - version ::= [-A-Za-z0-9_.]+ - - Tokens can be separated by whitespace, and a requirement can be continued - over multiple lines using a backslash (``\\``). Line-end comments (using - ``#``) are also allowed. + The syntax of a requirement specifier is defined in full in PEP 508. Some examples of valid requirement specifiers:: @@ -611,6 +598,7 @@ Requirements Parsing Fizzy [foo, bar] PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1 SomethingWhoseVersionIDontCareAbout + SomethingWithMarker[foo]>1.0;python_version<"2.7" The project name is the only required portion of a requirement string, and if it's the only thing supplied, the requirement will accept any version @@ -631,6 +619,11 @@ Requirements Parsing ``pkg_resources.require('Report-O-Rama[PDF]')`` to add the necessary distributions to sys.path at runtime. + The "markers" in a requirement are used to specify when a requirement + should be installed -- the requirement will be installed if the marker + evaluates as true in the current environment. For example, specifying + ``argparse;python_version<"2.7"`` will not install in an Python 2.7 or 3.3 + environment, but will in a Python 2.6 environment. ``Requirement`` Methods and Attributes -------------------------------------- @@ -680,6 +673,12 @@ Requirements Parsing order. The `op` in each tuple is a comparison operator, represented as a string. The `version` is the (unparsed) version number. +``marker`` + An instance of ``packaging.markers.Marker`` that allows evaluation + against the current environment. May be None if no marker specified. + +``url`` + The location to download the requirement from if specified. Entry Points ============ diff --git a/docs/releases.txt b/docs/releases.txt index a9742c20..3f29334a 100644 --- a/docs/releases.txt +++ b/docs/releases.txt @@ -3,19 +3,14 @@ Release Process =============== In order to allow for rapid, predictable releases, Setuptools uses a -mechanical technique for releases. The release script, ``release.py`` in the -repository, defines the details of the releases, and is executed by the -`jaraco.packaging <https://bitbucket.org/jaraco/jaraco.packaging>`_ release -module. The script does some checks (some interactive) and fully automates -the release process. +mechanical technique for releases, enacted by Travis following a +successful build of a tagged release per +`PyPI deployment <https://docs.travis-ci.com/user/deployment/pypi>`_. -A Setuptools release manager must have maintainer access on PyPI to the -project and administrative access to the Bitbucket project. - -To make a release, run the following from a Mercurial checkout at the -revision slated for release:: - - python -m jaraco.packaging.release +To cut a release, install and run ``bumpversion {part}`` where ``part`` +is major, minor, or patch based on the scope of the changes in the +release. Then, push the commits to the master branch. If tests pass, +the release will be uploaded to PyPI. Bootstrap Bookmark ------------------ @@ -23,7 +18,7 @@ Bootstrap Bookmark Setuptools has a bootstrap script (ez_setup.py) which is hosted in the repository and must be updated with each release (to bump the default version). The "published" version of the script is the one indicated by the ``bootstrap`` -bookmark (Mercurial) or branch (Git). +branch. Therefore, the latest bootstrap script can be retrieved by checking out the repository at that bookmark. It's also possible to get the bootstrap script for @@ -57,7 +52,5 @@ corrected quickly, in many cases before other users have yet to encounter them. Release Managers ---------------- -Jason R. Coombs is the primary release manager. Additionally, the following -people have access to create releases: - -- Matthew Iversen (Ivoz) +Additionally, anyone with push access to the master branch has access to cut +releases. diff --git a/docs/setuptools.txt b/docs/setuptools.txt index d6a62de8..eeeab937 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -258,10 +258,9 @@ unless you need the associated ``setuptools`` feature. ``include_package_data`` If set to ``True``, this tells ``setuptools`` to automatically include any - data files it finds inside your package directories, that are either under - CVS or Subversion control, or which are specified by your ``MANIFEST.in`` - file. For more information, see the section below on `Including Data - Files`_. + data files it finds inside your package directories that are specified by + your ``MANIFEST.in`` file. For more information, see the section below on + `Including Data Files`_. ``exclude_package_data`` A dictionary mapping package names to lists of glob patterns that should @@ -785,17 +784,15 @@ e.g.:: ) This tells setuptools to install any data files it finds in your packages. -The data files must be under CVS or Subversion control, or else they must be -specified via the distutils' ``MANIFEST.in`` file. (They can also be tracked -by another revision control system, using an appropriate plugin. See the -section below on `Adding Support for Other Revision Control Systems`_ for -information on how to write such plugins.) - -If the data files are not under version control, or are not in a supported -version control system, or if you want finer-grained control over what files -are included (for example, if you have documentation files in your package -directories and want to exclude them from installation), then you can also use -the ``package_data`` keyword, e.g.:: +The data files must be specified via the distutils' ``MANIFEST.in`` file. +(They can also be tracked by a revision control system, using an appropriate +plugin. See the section below on `Adding Support for Revision Control +Systems`_ for information on how to write such plugins.) + +If you want finer-grained control over what files are included (for example, +if you have documentation files in your package directories and want to exclude +them from installation), then you can also use the ``package_data`` keyword, +e.g.:: from setuptools import setup, find_packages setup( @@ -853,8 +850,7 @@ converts slashes to appropriate platform-specific separators at build time. Python 2.4; there is `some documentation for the feature`__ available on the python.org website. If using the setuptools-specific ``include_package_data`` argument, files specified by ``package_data`` will *not* be automatically -added to the manifest unless they are tracked by a supported version control -system, or are listed in the MANIFEST.in file.) +added to the manifest unless they are listed in the MANIFEST.in file.) __ http://docs.python.org/dist/node11.html @@ -887,8 +883,7 @@ included as a result of using ``include_package_data``. In summary, the three options allow you to: ``include_package_data`` - Accept all data files and directories matched by ``MANIFEST.in`` or found - in source control. + Accept all data files and directories matched by ``MANIFEST.in``. ``package_data`` Specify additional patterns to match files and directories that may or may @@ -1231,15 +1226,14 @@ Your Project's Dependencies target audience isn't able to compile packages (e.g. most Windows users) and your package or some of its dependencies include C code. -Subversion or CVS Users and Co-Developers +Revision Control System Users and Co-Developers Users and co-developers who are tracking your in-development code using - CVS, Subversion, or some other revision control system should probably read - this manual's sections regarding such development. Alternately, you may - wish to create a quick-reference guide containing the tips from this manual - that apply to your particular situation. For example, if you recommend - that people use ``setup.py develop`` when tracking your in-development - code, you should let them know that this needs to be run after every update - or commit. + a revision control system should probably read this manual's sections + regarding such development. Alternately, you may wish to create a + quick-reference guide containing the tips from this manual that apply to + your particular situation. For example, if you recommend that people use + ``setup.py develop`` when tracking your in-development code, you should let + them know that this needs to be run after every update or commit. Similarly, if you remove modules or data files from your project, you should remind them to run ``setup.py clean --all`` and delete any obsolete @@ -1468,18 +1462,11 @@ Generating Source Distributions ------------------------------- ``setuptools`` enhances the distutils' default algorithm for source file -selection, so that all files managed by CVS or Subversion in your project tree -are included in any source distribution you build. This is a big improvement -over having to manually write a ``MANIFEST.in`` file and try to keep it in -sync with your project. So, if you are using CVS or Subversion, and your -source distributions only need to include files that you're tracking in -revision control, don't create a ``MANIFEST.in`` file for your project. -(And, if you already have one, you might consider deleting it the next time -you would otherwise have to change it.) - -(NOTE: other revision control systems besides CVS and Subversion can be -supported using plugins; see the section below on `Adding Support for Other -Revision Control Systems`_ for information on how to write such plugins.) +selection with pluggable endpoints for looking up files to include. If you are +using a revision control system, and your source distributions only need to +include files that you're tracking in revision control, use a corresponding +plugin instead of writing a ``MANIFEST.in`` file. See the section below on +`Adding Support for Revision Control Systems`_ for information on plugins. If you need to include automatically generated files, or files that are kept in an unsupported revision control system, you'll need to create a ``MANIFEST.in`` @@ -1501,12 +1488,6 @@ the options that the distutils' more complex ``sdist`` process requires. For all practical purposes, you'll probably use only the ``--formats`` option, if you use any option at all. -(By the way, if you're using some other revision control system, you might -consider creating and publishing a `revision control plugin for setuptools`_.) - - -.. _revision control plugin for setuptools: `Adding Support for Other Revision Control Systems`_ - Making your package available for EasyInstall --------------------------------------------- @@ -1687,9 +1668,10 @@ Of course, for this to work, your source distributions must include the C code generated by Pyrex, as well as your original ``.pyx`` files. This means that you will probably want to include current ``.c`` files in your revision control system, rebuilding them whenever you check changes in for the ``.pyx`` -source files. This will ensure that people tracking your project in CVS or -Subversion will be able to build it even if they don't have Pyrex installed, -and that your source releases will be similarly usable with or without Pyrex. +source files. This will ensure that people tracking your project in a revision +control system will be able to build it even if they don't have Pyrex +installed, and that your source releases will be similarly usable with or +without Pyrex. ----------------- @@ -2328,54 +2310,25 @@ available: ``upload`` - Upload source and/or egg distributions to PyPI =========================================================== -PyPI now supports uploading project files for redistribution; uploaded files -are easily found by EasyInstall, even if you don't have download links on your -project's home page. - -Although Python 2.5 will support uploading all types of distributions to PyPI, -setuptools only supports source distributions and eggs. (This is partly -because PyPI's upload support is currently broken for various other file -types.) To upload files, you must include the ``upload`` command *after* the -``sdist`` or ``bdist_egg`` commands on the setup command line. For example:: +The ``upload`` command is implemented and `documented +<https://docs.python.org/3.1/distutils/uploading.html>`_ +in distutils. - setup.py bdist_egg upload # create an egg and upload it - setup.py sdist upload # create a source distro and upload it - setup.py sdist bdist_egg upload # create and upload both +Setuptools augments the ``upload`` command with support +for `keyring <https://pypi.python.org/pypi/keyring>`_, +allowing the password to be stored in a secure +location and not in plaintext in the .pypirc file. To use +keyring, first install keyring and set the password for +the relevant repository, e.g.:: -Note that to upload files for a project, the corresponding version must already -be registered with PyPI, using the distutils ``register`` command. It's -usually a good idea to include the ``register`` command at the start of the -command line, so that any registration problems can be found and fixed before -building and uploading the distributions, e.g.:: + python -m keyring set <repository> <username> + Password for '<username>' in '<repository>': ******** - setup.py register sdist bdist_egg upload +Then, in .pypirc, set the repository configuration as normal, +but omit the password. Thereafter, uploads will use the +password from the keyring. -This will update PyPI's listing for your project's current version. - -Note, by the way, that the metadata in your ``setup()`` call determines what -will be listed in PyPI for your package. Try to fill out as much of it as -possible, as it will save you a lot of trouble manually adding and updating -your PyPI listings. Just put it in ``setup.py`` and use the ``register`` -command to keep PyPI up to date. - -The ``upload`` command has a few options worth noting: - -``--sign, -s`` - Sign each uploaded file using GPG (GNU Privacy Guard). The ``gpg`` program - must be available for execution on the system ``PATH``. - -``--identity=NAME, -i NAME`` - Specify the identity or key name for GPG to use when signing. The value of - this option will be passed through the ``--local-user`` option of the - ``gpg`` program. - -``--show-response`` - Display the full response text from server; this is useful for debugging - PyPI problems. - -``--repository=URL, -r URL`` - The URL of the repository to upload to. Defaults to - https://pypi.python.org/pypi (i.e., the main PyPI installation). +New in 20.1: Added keyring support. .. _upload_docs: @@ -2598,15 +2551,21 @@ the ``cmd`` object's ``write_file()``, ``delete_file()``, and those methods' docstrings for more details. -Adding Support for Other Revision Control Systems +Adding Support for Revision Control Systems ------------------------------------------------- -If you would like to create a plugin for ``setuptools`` to find files in -source control systems, you can do so by adding an -entry point to the ``setuptools.file_finders`` group. The entry point should -be a function accepting a single directory name, and should yield -all the filenames within that directory (and any subdirectories thereof) that -are under revision control. +If the files you want to include in the source distribution are tracked using +Git, Mercurial or SVN, you can use the following packages to achieve that: + +- Git and Mercurial: `setuptools_scm <https://pypi.python.org/pypi/setuptools_scm>`_ +- SVN: `setuptools_svn <https://pypi.python.org/pypi/setuptools_svn>`_ + +If you would like to create a plugin for ``setuptools`` to find files tracked +by another revision control system, you can do so by adding an entry point to +the ``setuptools.file_finders`` group. The entry point should be a function +accepting a single directory name, and should yield all the filenames within +that directory (and any subdirectories thereof) that are under revision +control. For example, if you were going to create a plugin for a revision control system called "foobar", you would write a function something like this: @@ -2699,5 +2658,5 @@ confirmed via the list are actual bugs, and which you have reduced to a minimal set of steps to reproduce. .. _distutils-sig mailing list: http://mail.python.org/pipermail/distutils-sig/ -.. _setuptools bug tracker: https://bitbucket.org/pypa/setuptools/ +.. _setuptools bug tracker: https://github.com/pypa/setuptools/ diff --git a/ez_setup.py b/ez_setup.py index 1320ac92..d6f4b78c 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -192,6 +192,8 @@ def _conflict_bail(VC_err, version): def _unload_pkg_resources(): + sys.meta_path = [importer for importer in sys.meta_path if + importer.__class__.__module__ != 'pkg_resources.extern'] del_modules = [ name for name in sys.modules if name.startswith('pkg_resources') @@ -225,8 +227,8 @@ def download_file_powershell(url, target): ps_cmd = ( "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " "[System.Net.CredentialCache]::DefaultCredentials; " - "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" - % vars() + '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")' + % locals() ) cmd = [ 'powershell', @@ -354,7 +356,7 @@ def _resolve_version(version): reader = codecs.getreader(charset) doc = json.load(reader(resp)) - return doc['info']['version'] + return str(doc['info']['version']) def _build_install_args(options): diff --git a/pavement.py b/pavement.py new file mode 100644 index 00000000..8d7574e2 --- /dev/null +++ b/pavement.py @@ -0,0 +1,28 @@ +import re + +from paver.easy import task, path as Path +import pip + +def remove_all(paths): + for path in paths: + path.rmtree() if path.isdir() else path.remove() + +@task +def update_vendored(): + vendor = Path('pkg_resources/_vendor') + remove_all(vendor.glob('packaging*')) + remove_all(vendor.glob('six*')) + remove_all(vendor.glob('pyparsing*')) + install_args = [ + 'install', + '-r', str(vendor/'vendored.txt'), + '-t', str(vendor), + ] + pip.main(install_args) + packaging = vendor / 'packaging' + for file in packaging.glob('*.py'): + text = file.text() + text = re.sub(r' (pyparsing|six)', r' pkg_resources.extern.\1', text) + file.write_text(text) + remove_all(vendor.glob('*.dist-info')) + remove_all(vendor.glob('*.egg-info')) diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index b55e4127..eb84f4ba 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -28,8 +28,6 @@ import warnings import stat import functools import pkgutil -import token -import symbol import operator import platform import collections @@ -37,7 +35,6 @@ import plistlib import email.parser import tempfile import textwrap -import itertools from pkgutil import get_importer try: @@ -46,23 +43,8 @@ except ImportError: # Python 3.2 compatibility import imp as _imp -PY3 = sys.version_info > (3,) -PY2 = not PY3 - -if PY3: - from urllib.parse import urlparse, urlunparse - -if PY2: - from urlparse import urlparse, urlunparse - filter = itertools.ifilter - map = itertools.imap - -if PY3: - string_types = str, -else: - string_types = str, eval('unicode') - -iteritems = (lambda i: i.items()) if PY3 else lambda i: i.iteritems() +from pkg_resources.extern import six +from pkg_resources.extern.six.moves import urllib, map, filter # capture these to bypass sandboxing from os import utime @@ -76,26 +58,18 @@ except ImportError: from os import open as os_open from os.path import isdir, split -# Avoid try/except due to potential problems with delayed import mechanisms. -if sys.version_info >= (3, 3) and sys.implementation.name == "cpython": - import importlib.machinery as importlib_machinery -else: - importlib_machinery = None - try: - import parser + import importlib.machinery as importlib_machinery + # access attribute to force import under delayed import mechanisms. + importlib_machinery.__name__ except ImportError: - pass + importlib_machinery = None -try: - import pkg_resources._vendor.packaging.version - import pkg_resources._vendor.packaging.specifiers - packaging = pkg_resources._vendor.packaging -except ImportError: - # fallback to naturally-installed version; allows system packagers to - # omit vendored packages. - import packaging.version - import packaging.specifiers +from pkg_resources.extern import packaging +__import__('pkg_resources.extern.packaging.version') +__import__('pkg_resources.extern.packaging.specifiers') +__import__('pkg_resources.extern.packaging.requirements') +__import__('pkg_resources.extern.packaging.markers') if (3, 0) < sys.version_info < (3, 3): @@ -555,7 +529,7 @@ run_main = run_script def get_distribution(dist): """Return a current distribution object for a Requirement or string""" - if isinstance(dist, string_types): + if isinstance(dist, six.string_types): dist = Requirement.parse(dist) if isinstance(dist, Requirement): dist = get_provider(dist) @@ -774,7 +748,7 @@ class WorkingSet(object): will be called. """ if insert: - dist.insert_on(self.entries, entry) + dist.insert_on(self.entries, entry, replace=replace) if entry is None: entry = dist.location @@ -828,6 +802,12 @@ class WorkingSet(object): if req in processed: # Ignore cyclic or redundant dependencies continue + # If the req has a marker, evaluate it -- skipping the req if + # it evaluates to False. + # https://github.com/pypa/setuptools/issues/523 + _issue_523_bypass = True + if not _issue_523_bypass and req.marker and not req.marker.evaluate(): + continue dist = best.get(req.key) if dist is None: # Find the best distribution and add it to the map @@ -1195,22 +1175,23 @@ class ResourceManager: old_exc = sys.exc_info()[1] cache_path = self.extraction_path or get_default_cache() - err = ExtractionError("""Can't extract file(s) to egg cache + tmpl = textwrap.dedent(""" + Can't extract file(s) to egg cache -The following error occurred while trying to extract file(s) to the Python egg -cache: + The following error occurred while trying to extract file(s) to the Python egg + cache: - %s + {old_exc} -The Python egg cache directory is currently set to: + The Python egg cache directory is currently set to: - %s + {cache_path} -Perhaps your account does not have write access to this directory? You can -change the cache directory by setting the PYTHON_EGG_CACHE environment -variable to point to an accessible directory. -""" % (old_exc, cache_path) - ) + Perhaps your account does not have write access to this directory? You can + change the cache directory by setting the PYTHON_EGG_CACHE environment + variable to point to an accessible directory. + """).lstrip() + err = ExtractionError(tmpl.format(**locals())) err.manager = self err.cache_path = cache_path err.original_error = old_exc @@ -1405,202 +1386,34 @@ def to_filename(name): return name.replace('-','_') -class MarkerEvaluation(object): - values = { - 'os_name': lambda: os.name, - 'sys_platform': lambda: sys.platform, - 'python_full_version': platform.python_version, - 'python_version': lambda: platform.python_version()[:3], - 'platform_version': platform.version, - 'platform_machine': platform.machine, - 'platform_python_implementation': platform.python_implementation, - 'python_implementation': platform.python_implementation, - } - - @classmethod - def is_invalid_marker(cls, text): - """ - Validate text as a PEP 426 environment marker; return an exception - if invalid or False otherwise. - """ - try: - cls.evaluate_marker(text) - except SyntaxError as e: - return cls.normalize_exception(e) - return False - - @staticmethod - def normalize_exception(exc): - """ - Given a SyntaxError from a marker evaluation, normalize the error - message: - - Remove indications of filename and line number. - - Replace platform-specific error messages with standard error - messages. - """ - subs = { - 'unexpected EOF while parsing': 'invalid syntax', - 'parenthesis is never closed': 'invalid syntax', - } - exc.filename = None - exc.lineno = None - exc.msg = subs.get(exc.msg, exc.msg) - return exc - - @classmethod - def and_test(cls, nodelist): - # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! - items = [ - cls.interpret(nodelist[i]) - for i in range(1, len(nodelist), 2) - ] - return functools.reduce(operator.and_, items) - - @classmethod - def test(cls, nodelist): - # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! - items = [ - cls.interpret(nodelist[i]) - for i in range(1, len(nodelist), 2) - ] - return functools.reduce(operator.or_, items) - - @classmethod - def atom(cls, nodelist): - t = nodelist[1][0] - if t == token.LPAR: - if nodelist[2][0] == token.RPAR: - raise SyntaxError("Empty parentheses") - return cls.interpret(nodelist[2]) - msg = "Language feature not supported in environment markers" - raise SyntaxError(msg) - - @classmethod - def comparison(cls, nodelist): - if len(nodelist) > 4: - msg = "Chained comparison not allowed in environment markers" - raise SyntaxError(msg) - comp = nodelist[2][1] - cop = comp[1] - if comp[0] == token.NAME: - if len(nodelist[2]) == 3: - if cop == 'not': - cop = 'not in' - else: - cop = 'is not' - try: - cop = cls.get_op(cop) - except KeyError: - msg = repr(cop) + " operator not allowed in environment markers" - raise SyntaxError(msg) - return cop(cls.evaluate(nodelist[1]), cls.evaluate(nodelist[3])) - - @classmethod - def get_op(cls, op): - ops = { - symbol.test: cls.test, - symbol.and_test: cls.and_test, - symbol.atom: cls.atom, - symbol.comparison: cls.comparison, - 'not in': lambda x, y: x not in y, - 'in': lambda x, y: x in y, - '==': operator.eq, - '!=': operator.ne, - '<': operator.lt, - '>': operator.gt, - '<=': operator.le, - '>=': operator.ge, - } - if hasattr(symbol, 'or_test'): - ops[symbol.or_test] = cls.test - return ops[op] - - @classmethod - def evaluate_marker(cls, text, extra=None): - """ - Evaluate a PEP 426 environment marker on CPython 2.4+. - Return a boolean indicating the marker result in this environment. - Raise SyntaxError if marker is invalid. - - This implementation uses the 'parser' module, which is not implemented - on - Jython and has been superseded by the 'ast' module in Python 2.6 and - later. - """ - return cls.interpret(parser.expr(text).totuple(1)[1]) - - @staticmethod - def _translate_metadata2(env): - """ - Markerlib implements Metadata 1.2 (PEP 345) environment markers. - Translate the variables to Metadata 2.0 (PEP 426). - """ - return dict( - (key.replace('.', '_'), value) - for key, value in env.items() - ) - - @classmethod - def _markerlib_evaluate(cls, text): - """ - Evaluate a PEP 426 environment marker using markerlib. - Return a boolean indicating the marker result in this environment. - Raise SyntaxError if marker is invalid. - """ - import _markerlib - - env = cls._translate_metadata2(_markerlib.default_environment()) - try: - result = _markerlib.interpret(text, env) - except NameError as e: - raise SyntaxError(e.args[0]) - return result - - if 'parser' not in globals(): - # Fall back to less-complete _markerlib implementation if 'parser' module - # is not available. - evaluate_marker = _markerlib_evaluate +def invalid_marker(text): + """ + Validate text as a PEP 508 environment marker; return an exception + if invalid or False otherwise. + """ + try: + evaluate_marker(text) + except SyntaxError as e: + e.filename = None + e.lineno = None + return e + return False - @classmethod - def interpret(cls, nodelist): - while len(nodelist)==2: nodelist = nodelist[1] - try: - op = cls.get_op(nodelist[0]) - except KeyError: - raise SyntaxError("Comparison or logical expression expected") - return op(nodelist) - @classmethod - def evaluate(cls, nodelist): - while len(nodelist)==2: nodelist = nodelist[1] - kind = nodelist[0] - name = nodelist[1] - if kind==token.NAME: - try: - op = cls.values[name] - except KeyError: - raise SyntaxError("Unknown name %r" % name) - return op() - if kind==token.STRING: - s = nodelist[1] - if not cls._safe_string(s): - raise SyntaxError( - "Only plain strings allowed in environment markers") - return s[1:-1] - msg = "Language feature not supported in environment markers" - raise SyntaxError(msg) +def evaluate_marker(text, extra=None): + """ + Evaluate a PEP 508 environment marker. + Return a boolean indicating the marker result in this environment. + Raise SyntaxError if marker is invalid. - @staticmethod - def _safe_string(cand): - return ( - cand[:1] in "'\"" and - not cand.startswith('"""') and - not cand.startswith("'''") and - '\\' not in cand - ) + This implementation uses the 'pyparsing' module. + """ + try: + marker = packaging.markers.Marker(text) + return marker.evaluate() + except packaging.markers.InvalidMarker as e: + raise SyntaxError(e) -invalid_marker = MarkerEvaluation.is_invalid_marker -evaluate_marker = MarkerEvaluation.evaluate_marker class NullProvider: """Try to implement resources and metadata for arbitrary PEP 302 loaders""" @@ -1746,10 +1559,13 @@ class DefaultProvider(EggProvider): with open(path, 'rb') as stream: return stream.read() -register_loader_type(type(None), DefaultProvider) + @classmethod + def _register(cls): + loader_cls = getattr(importlib_machinery, 'SourceFileLoader', + type(None)) + register_loader_type(loader_cls, cls) -if importlib_machinery is not None: - register_loader_type(importlib_machinery.SourceFileLoader, DefaultProvider) +DefaultProvider._register() class EmptyProvider(NullProvider): @@ -2155,7 +1971,7 @@ def find_on_path(importer, path_item, only=False): break register_finder(pkgutil.ImpImporter, find_on_path) -if importlib_machinery is not None: +if hasattr(importlib_machinery, 'FileFinder'): register_finder(importlib_machinery.FileFinder, find_on_path) _declare_state('dict', _namespace_handlers={}) @@ -2201,11 +2017,29 @@ def _handle_ns(packageName, path_item): path = module.__path__ path.append(subpath) loader.load_module(packageName) - for path_item in path: - if path_item not in module.__path__: - module.__path__.append(path_item) + _rebuild_mod_path(path, packageName, module) return subpath + +def _rebuild_mod_path(orig_path, package_name, module): + """ + Rebuild module.__path__ ensuring that all entries are ordered + corresponding to their sys.path order + """ + sys_path = [_normalize_cached(p) for p in sys.path] + def position_in_sys_path(path): + """ + Return the ordinal of the path based on its position in sys.path + """ + path_parts = path.split(os.sep) + module_parts = package_name.count('.') + 1 + parts = path_parts[:-module_parts] + return sys_path.index(_normalize_cached(os.sep.join(parts))) + + orig_path.sort(key=position_in_sys_path) + module.__path__[:] = [_normalize_cached(p) for p in orig_path] + + def declare_namespace(packageName): """Declare that package 'packageName' is a namespace package""" @@ -2264,7 +2098,7 @@ def file_ns_handler(importer, path_item, packageName, module): register_namespace_handler(pkgutil.ImpImporter, file_ns_handler) register_namespace_handler(zipimport.zipimporter, file_ns_handler) -if importlib_machinery is not None: +if hasattr(importlib_machinery, 'FileFinder'): register_namespace_handler(importlib_machinery.FileFinder, file_ns_handler) @@ -2303,7 +2137,7 @@ def _set_parent_ns(packageName): def yield_lines(strs): """Yield non-empty/non-comment lines of a string or sequence""" - if isinstance(strs, string_types): + if isinstance(strs, six.string_types): for s in strs.splitlines(): s = s.strip() # skip blank lines/comments @@ -2314,18 +2148,6 @@ def yield_lines(strs): for s in yield_lines(ss): yield s -# whitespace and comment -LINE_END = re.compile(r"\s*(#.*)?$").match -# line continuation -CONTINUE = re.compile(r"\s*\\\s*(#.*)?$").match -# Distribution or extra -DISTRO = re.compile(r"\s*((\w|[-.])+)").match -# ver. info -VERSION = re.compile(r"\s*(<=?|>=?|===?|!=|~=)\s*((\w|[-.*_!+])+)").match -# comma between items -COMMA = re.compile(r"\s*,").match -OBRACKET = re.compile(r"\s*\[").match -CBRACKET = re.compile(r"\s*\]").match MODULE = re.compile(r"\w+(\.\w+)*$").match EGG_NAME = re.compile( r""" @@ -2470,9 +2292,9 @@ class EntryPoint(object): def _remove_md5_fragment(location): if not location: return '' - parsed = urlparse(location) + parsed = urllib.parse.urlparse(location) if parsed[-1].startswith('md5='): - return urlunparse(parsed[:-1] + ('',)) + return urllib.parse.urlunparse(parsed[:-1] + ('',)) return location @@ -2658,7 +2480,7 @@ class Distribution(object): """Ensure distribution is importable on `path` (default=sys.path)""" if path is None: path = sys.path - self.insert_on(path) + self.insert_on(path, replace=True) if path is sys.path: fixup_namespace_packages(self.location) for pkg in self._get_metadata('namespace_packages.txt'): @@ -2735,7 +2557,7 @@ class Distribution(object): """Return the EntryPoint object for `group`+`name`, or ``None``""" return self.get_entry_map(group).get(name) - def insert_on(self, path, loc = None): + def insert_on(self, path, loc=None, replace=False): """Insert self.location in path before its nearest parent directory""" loc = loc or self.location @@ -2759,7 +2581,10 @@ class Distribution(object): else: if path is sys.path: self.check_version_conflict() - path.append(loc) + if replace: + path.insert(0, loc) + else: + path.append(loc) return # p is the spot where we found or inserted loc; now remove duplicates @@ -2861,34 +2686,18 @@ class DistInfoDistribution(Distribution): self.__dep_map = self._compute_dependencies() return self.__dep_map - def _preparse_requirement(self, requires_dist): - """Convert 'Foobar (1); baz' to ('Foobar ==1', 'baz') - Split environment marker, add == prefix to version specifiers as - necessary, and remove parenthesis. - """ - parts = requires_dist.split(';', 1) + [''] - distvers = parts[0].strip() - mark = parts[1].strip() - distvers = re.sub(self.EQEQ, r"\1==\2\3", distvers) - distvers = distvers.replace('(', '').replace(')', '') - return (distvers, mark) - def _compute_dependencies(self): """Recompute this distribution's dependencies.""" - from _markerlib import compile as compile_marker dm = self.__dep_map = {None: []} reqs = [] # Including any condition expressions for req in self._parsed_pkg_info.get_all('Requires-Dist') or []: - distvers, mark = self._preparse_requirement(req) - parsed = next(parse_requirements(distvers)) - parsed.marker_fn = compile_marker(mark) - reqs.append(parsed) + reqs.extend(parse_requirements(req)) def reqs_for_extra(extra): for req in reqs: - if req.marker_fn(override={'extra':extra}): + if not req.marker or req.marker.evaluate({'extra': extra}): yield req common = frozenset(reqs_for_extra(None)) @@ -2934,85 +2743,38 @@ def parse_requirements(strs): # create a steppable iterator, so we can handle \-continuations lines = iter(yield_lines(strs)) - def scan_list(ITEM, TERMINATOR, line, p, groups, item_name): - - items = [] - - while not TERMINATOR(line, p): - if CONTINUE(line, p): - try: - line = next(lines) - p = 0 - except StopIteration: - msg = "\\ must not appear on the last nonblank line" - raise RequirementParseError(msg) - - match = ITEM(line, p) - if not match: - msg = "Expected " + item_name + " in" - raise RequirementParseError(msg, line, "at", line[p:]) - - items.append(match.group(*groups)) - p = match.end() - - match = COMMA(line, p) - if match: - # skip the comma - p = match.end() - elif not TERMINATOR(line, p): - msg = "Expected ',' or end-of-list in" - raise RequirementParseError(msg, line, "at", line[p:]) - - match = TERMINATOR(line, p) - # skip the terminator, if any - if match: - p = match.end() - return line, p, items - for line in lines: - match = DISTRO(line) - if not match: - raise RequirementParseError("Missing distribution spec", line) - project_name = match.group(1) - p = match.end() - extras = [] - - match = OBRACKET(line, p) - if match: - p = match.end() - line, p, extras = scan_list( - DISTRO, CBRACKET, line, p, (1,), "'extra' name" - ) - - line, p, specs = scan_list(VERSION, LINE_END, line, p, (1, 2), - "version spec") - specs = [(op, val) for op, val in specs] - yield Requirement(project_name, specs, extras) - - -class Requirement: - def __init__(self, project_name, specs, extras): + # Drop comments -- a hash without a space may be in a URL. + if ' #' in line: + line = line[:line.find(' #')] + # If there is a line continuation, drop it, and append the next line. + if line.endswith('\\'): + line = line[:-2].strip() + line += next(lines) + yield Requirement(line) + + +class Requirement(packaging.requirements.Requirement): + def __init__(self, requirement_string): """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!""" - self.unsafe_name, project_name = project_name, safe_name(project_name) + try: + super(Requirement, self).__init__(requirement_string) + except packaging.requirements.InvalidRequirement as e: + raise RequirementParseError(str(e)) + self.unsafe_name = self.name + project_name = safe_name(self.name) self.project_name, self.key = project_name, project_name.lower() - self.specifier = packaging.specifiers.SpecifierSet( - ",".join(["".join([x, y]) for x, y in specs]) - ) - self.specs = specs - self.extras = tuple(map(safe_extra, extras)) + self.specs = [ + (spec.operator, spec.version) for spec in self.specifier] + self.extras = tuple(map(safe_extra, self.extras)) self.hashCmp = ( self.key, self.specifier, frozenset(self.extras), + str(self.marker) if self.marker else None, ) self.__hash = hash(self.hashCmp) - def __str__(self): - extras = ','.join(self.extras) - if extras: - extras = '[%s]' % extras - return '%s%s%s' % (self.project_name, extras, self.specifier) - def __eq__(self, other): return ( isinstance(other, Requirement) and diff --git a/pkg_resources/_vendor/packaging/__about__.py b/pkg_resources/_vendor/packaging/__about__.py index eadb794e..baa90755 100644 --- a/pkg_resources/_vendor/packaging/__about__.py +++ b/pkg_resources/_vendor/packaging/__about__.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function __all__ = [ @@ -22,10 +12,10 @@ __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "15.3" +__version__ = "16.6" -__author__ = "Donald Stufft" +__author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" -__license__ = "Apache License, Version 2.0" -__copyright__ = "Copyright 2014 %s" % __author__ +__license__ = "BSD or Apache License, Version 2.0" +__copyright__ = "Copyright 2014-2016 %s" % __author__ diff --git a/pkg_resources/_vendor/packaging/__init__.py b/pkg_resources/_vendor/packaging/__init__.py index c39a8eab..5ee62202 100644 --- a/pkg_resources/_vendor/packaging/__init__.py +++ b/pkg_resources/_vendor/packaging/__init__.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function from .__about__ import ( diff --git a/pkg_resources/_vendor/packaging/_compat.py b/pkg_resources/_vendor/packaging/_compat.py index 5c396cea..210bb80b 100644 --- a/pkg_resources/_vendor/packaging/_compat.py +++ b/pkg_resources/_vendor/packaging/_compat.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function import sys diff --git a/pkg_resources/_vendor/packaging/_structures.py b/pkg_resources/_vendor/packaging/_structures.py index 0ae9bb52..ccc27861 100644 --- a/pkg_resources/_vendor/packaging/_structures.py +++ b/pkg_resources/_vendor/packaging/_structures.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function diff --git a/pkg_resources/_vendor/packaging/markers.py b/pkg_resources/_vendor/packaging/markers.py new file mode 100644 index 00000000..bac852df --- /dev/null +++ b/pkg_resources/_vendor/packaging/markers.py @@ -0,0 +1,278 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import operator +import os +import platform +import sys + +from pkg_resources.extern.pyparsing import ParseException, ParseResults, stringStart, stringEnd +from pkg_resources.extern.pyparsing import ZeroOrMore, Group, Forward, QuotedString +from pkg_resources.extern.pyparsing import Literal as L # noqa + +from ._compat import string_types +from .specifiers import Specifier, InvalidSpecifier + + +__all__ = [ + "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", + "Marker", "default_environment", +] + + +class InvalidMarker(ValueError): + """ + An invalid marker was found, users should refer to PEP 508. + """ + + +class UndefinedComparison(ValueError): + """ + An invalid operation was attempted on a value that doesn't support it. + """ + + +class UndefinedEnvironmentName(ValueError): + """ + A name was attempted to be used that does not exist inside of the + environment. + """ + + +class Node(object): + + def __init__(self, value): + self.value = value + + def __str__(self): + return str(self.value) + + def __repr__(self): + return "<{0}({1!r})>".format(self.__class__.__name__, str(self)) + + +class Variable(Node): + pass + + +class Value(Node): + pass + + +VARIABLE = ( + L("implementation_version") | + L("platform_python_implementation") | + L("implementation_name") | + L("python_full_version") | + L("platform_release") | + L("platform_version") | + L("platform_machine") | + L("platform_system") | + L("python_version") | + L("sys_platform") | + L("os_name") | + L("os.name") | # PEP-345 + L("sys.platform") | # PEP-345 + L("platform.version") | # PEP-345 + L("platform.machine") | # PEP-345 + L("platform.python_implementation") | # PEP-345 + L("extra") +) +VARIABLE.setParseAction(lambda s, l, t: Variable(t[0].replace('.', '_'))) + +VERSION_CMP = ( + L("===") | + L("==") | + L(">=") | + L("<=") | + L("!=") | + L("~=") | + L(">") | + L("<") +) + +MARKER_OP = VERSION_CMP | L("not in") | L("in") + +MARKER_VALUE = QuotedString("'") | QuotedString('"') +MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) + +BOOLOP = L("and") | L("or") + +MARKER_VAR = VARIABLE | MARKER_VALUE + +MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) +MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) + +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() + +MARKER_EXPR = Forward() +MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) +MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) + +MARKER = stringStart + MARKER_EXPR + stringEnd + + +def _coerce_parse_result(results): + if isinstance(results, ParseResults): + return [_coerce_parse_result(i) for i in results] + else: + return results + + +def _format_marker(marker, first=True): + assert isinstance(marker, (list, tuple, string_types)) + + # Sometimes we have a structure like [[...]] which is a single item list + # where the single item is itself it's own list. In that case we want skip + # the rest of this function so that we don't get extraneous () on the + # outside. + if (isinstance(marker, list) and len(marker) == 1 and + isinstance(marker[0], (list, tuple))): + return _format_marker(marker[0]) + + if isinstance(marker, list): + inner = (_format_marker(m, first=False) for m in marker) + if first: + return " ".join(inner) + else: + return "(" + " ".join(inner) + ")" + elif isinstance(marker, tuple): + return '{0} {1} "{2}"'.format(*marker) + else: + return marker + + +_operators = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +def _eval_op(lhs, op, rhs): + try: + spec = Specifier("".join([op, rhs])) + except InvalidSpecifier: + pass + else: + return spec.contains(lhs) + + oper = _operators.get(op) + if oper is None: + raise UndefinedComparison( + "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs) + ) + + return oper(lhs, rhs) + + +_undefined = object() + + +def _get_env(environment, name): + value = environment.get(name, _undefined) + + if value is _undefined: + raise UndefinedEnvironmentName( + "{0!r} does not exist in evaluation environment.".format(name) + ) + + return value + + +def _evaluate_markers(markers, environment): + groups = [[]] + + for marker in markers: + assert isinstance(marker, (list, tuple, string_types)) + + if isinstance(marker, list): + groups[-1].append(_evaluate_markers(marker, environment)) + elif isinstance(marker, tuple): + lhs, op, rhs = marker + + if isinstance(lhs, Variable): + lhs_value = _get_env(environment, lhs.value) + rhs_value = rhs.value + else: + lhs_value = lhs.value + rhs_value = _get_env(environment, rhs.value) + + groups[-1].append(_eval_op(lhs_value, op, rhs_value)) + else: + assert marker in ["and", "or"] + if marker == "or": + groups.append([]) + + return any(all(item) for item in groups) + + +def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + +def default_environment(): + if hasattr(sys, 'implementation'): + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + else: + iver = '0' + implementation_name = '' + + return { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "platform_python_implementation": platform.python_implementation(), + "python_version": platform.python_version()[:3], + "sys_platform": sys.platform, + } + + +class Marker(object): + + def __init__(self, marker): + try: + self._markers = _coerce_parse_result(MARKER.parseString(marker)) + except ParseException as e: + err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( + marker, marker[e.loc:e.loc + 8]) + raise InvalidMarker(err_str) + + def __str__(self): + return _format_marker(self._markers) + + def __repr__(self): + return "<Marker({0!r})>".format(str(self)) + + def evaluate(self, environment=None): + """Evaluate a marker. + + Return the boolean from evaluating the given marker against the + environment. environment is an optional argument to override all or + part of the determined environment. + + The environment is determined from the current Python process. + """ + current_environment = default_environment() + if environment is not None: + current_environment.update(environment) + + return _evaluate_markers(self._markers, current_environment) diff --git a/pkg_resources/_vendor/packaging/requirements.py b/pkg_resources/_vendor/packaging/requirements.py new file mode 100644 index 00000000..0c8c4a38 --- /dev/null +++ b/pkg_resources/_vendor/packaging/requirements.py @@ -0,0 +1,127 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import string +import re + +from pkg_resources.extern.pyparsing import stringStart, stringEnd, originalTextFor, ParseException +from pkg_resources.extern.pyparsing import ZeroOrMore, Word, Optional, Regex, Combine +from pkg_resources.extern.pyparsing import Literal as L # noqa +from pkg_resources.extern.six.moves.urllib import parse as urlparse + +from .markers import MARKER_EXPR, Marker +from .specifiers import LegacySpecifier, Specifier, SpecifierSet + + +class InvalidRequirement(ValueError): + """ + An invalid requirement was found, users should refer to PEP 508. + """ + + +ALPHANUM = Word(string.ascii_letters + string.digits) + +LBRACKET = L("[").suppress() +RBRACKET = L("]").suppress() +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() +COMMA = L(",").suppress() +SEMICOLON = L(";").suppress() +AT = L("@").suppress() + +PUNCTUATION = Word("-_.") +IDENTIFIER_END = ALPHANUM | (ZeroOrMore(PUNCTUATION) + ALPHANUM) +IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) + +NAME = IDENTIFIER("name") +EXTRA = IDENTIFIER + +URI = Regex(r'[^ ]+')("url") +URL = (AT + URI) + +EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) +EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") + +VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) +VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) + +VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY +VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), + joinString=",", adjacent=False)("_raw_spec") +_VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') + +VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") +VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) + +MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") +MARKER_EXPR.setParseAction( + lambda s, l, t: Marker(s[t._original_start:t._original_end]) +) +MARKER_SEPERATOR = SEMICOLON +MARKER = MARKER_SEPERATOR + MARKER_EXPR + +VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) +URL_AND_MARKER = URL + Optional(MARKER) + +NAMED_REQUIREMENT = \ + NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) + +REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd + + +class Requirement(object): + """Parse a requirement. + + Parse a given requirement string into its parts, such as name, specifier, + URL, and extras. Raises InvalidRequirement on a badly-formed requirement + string. + """ + + # TODO: Can we test whether something is contained within a requirement? + # If so how do we do that? Do we need to test against the _name_ of + # the thing as well as the version? What about the markers? + # TODO: Can we normalize the name and extra name? + + def __init__(self, requirement_string): + try: + req = REQUIREMENT.parseString(requirement_string) + except ParseException as e: + raise InvalidRequirement( + "Invalid requirement, parse error at \"{0!r}\"".format( + requirement_string[e.loc:e.loc + 8])) + + self.name = req.name + if req.url: + parsed_url = urlparse.urlparse(req.url) + if not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc): + raise InvalidRequirement("Invalid URL given") + self.url = req.url + else: + self.url = None + self.extras = set(req.extras.asList() if req.extras else []) + self.specifier = SpecifierSet(req.specifier) + self.marker = req.marker if req.marker else None + + def __str__(self): + parts = [self.name] + + if self.extras: + parts.append("[{0}]".format(",".join(sorted(self.extras)))) + + if self.specifier: + parts.append(str(self.specifier)) + + if self.url: + parts.append("@ {0}".format(self.url)) + + if self.marker: + parts.append("; {0}".format(self.marker)) + + return "".join(parts) + + def __repr__(self): + return "<Requirement({0!r})>".format(str(self)) diff --git a/pkg_resources/_vendor/packaging/specifiers.py b/pkg_resources/_vendor/packaging/specifiers.py index 891664f0..7f5a76cf 100644 --- a/pkg_resources/_vendor/packaging/specifiers.py +++ b/pkg_resources/_vendor/packaging/specifiers.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function import abc @@ -204,8 +194,8 @@ class _IndividualSpecifier(BaseSpecifier): # If our version is a prerelease, and we were not set to allow # prereleases, then we'll store it for later incase nothing # else matches this specifier. - if (parsed_version.is_prerelease - and not (prereleases or self.prereleases)): + if (parsed_version.is_prerelease and not + (prereleases or self.prereleases)): found_prereleases.append(version) # Either this is not a prerelease, or we should have been # accepting prereleases from the begining. @@ -223,23 +213,23 @@ class _IndividualSpecifier(BaseSpecifier): class LegacySpecifier(_IndividualSpecifier): - _regex = re.compile( + _regex_str = ( r""" - ^ - \s* (?P<operator>(==|!=|<=|>=|<|>)) \s* (?P<version> - [^\s]* # We just match everything, except for whitespace since this - # is a "legacy" specifier and the version string can be just - # about anything. + [^,;\s)]* # Since this is a "legacy" specifier, and the version + # string can be just about anything, we match everything + # except for whitespace, a semi-colon for marker support, + # a closing paren since versions can be enclosed in + # them, and a comma since it's a version separator. ) - \s* - $ - """, - re.VERBOSE | re.IGNORECASE, + """ ) + _regex = re.compile( + r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _operators = { "==": "equal", "!=": "not_equal", @@ -284,10 +274,8 @@ def _require_version_compare(fn): class Specifier(_IndividualSpecifier): - _regex = re.compile( + _regex_str = ( r""" - ^ - \s* (?P<operator>(~=|==|!=|<=|>=|<|>|===)) (?P<version> (?: @@ -378,12 +366,12 @@ class Specifier(_IndividualSpecifier): (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release ) ) - \s* - $ - """, - re.VERBOSE | re.IGNORECASE, + """ ) + _regex = re.compile( + r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _operators = { "~=": "compatible", "==": "equal", @@ -409,8 +397,8 @@ class Specifier(_IndividualSpecifier): prefix = ".".join( list( itertools.takewhile( - lambda x: (not x.startswith("post") - and not x.startswith("dev")), + lambda x: (not x.startswith("post") and not + x.startswith("dev")), _version_split(spec), ) )[:-1] @@ -419,13 +407,15 @@ class Specifier(_IndividualSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return (self._get_operator(">=")(prospective, spec) - and self._get_operator("==")(prospective, prefix)) + return (self._get_operator(">=")(prospective, spec) and + self._get_operator("==")(prospective, prefix)) @_require_version_compare def _compare_equal(self, prospective, spec): # We need special logic to handle prefix matching if spec.endswith(".*"): + # In the case of prefix matching we want to ignore local segment. + prospective = Version(prospective.public) # Split the spec out by dots, and pretend that there is an implicit # dot in between a release segment and a pre-release segment. spec = _version_split(spec[:-2]) # Remove the trailing .* @@ -577,8 +567,8 @@ def _pad_version(left, right): right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) # Get the rest of our versions - left_split.append(left[len(left_split):]) - right_split.append(left[len(right_split):]) + left_split.append(left[len(left_split[0]):]) + right_split.append(right[len(right_split[0]):]) # Insert our padding left_split.insert( diff --git a/pkg_resources/_vendor/packaging/utils.py b/pkg_resources/_vendor/packaging/utils.py new file mode 100644 index 00000000..942387ce --- /dev/null +++ b/pkg_resources/_vendor/packaging/utils.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import re + + +_canonicalize_regex = re.compile(r"[-_.]+") + + +def canonicalize_name(name): + # This is taken from PEP 503. + return _canonicalize_regex.sub("-", name).lower() diff --git a/pkg_resources/_vendor/packaging/version.py b/pkg_resources/_vendor/packaging/version.py index 4ba574b9..83b5ee8c 100644 --- a/pkg_resources/_vendor/packaging/version.py +++ b/pkg_resources/_vendor/packaging/version.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function import collections diff --git a/pkg_resources/_vendor/pyparsing.py b/pkg_resources/_vendor/pyparsing.py new file mode 100644 index 00000000..3e02dbee --- /dev/null +++ b/pkg_resources/_vendor/pyparsing.py @@ -0,0 +1,3805 @@ +# module pyparsing.py
+#
+# Copyright (c) 2003-2015 Paul T. McGuire
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__doc__ = \
+"""
+pyparsing module - Classes and methods to define and execute parsing grammars
+
+The pyparsing module is an alternative approach to creating and executing simple grammars,
+vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you
+don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
+provides a library of classes that you use to construct the grammar directly in Python.
+
+Here is a program to parse "Hello, World!" (or any greeting of the form C{"<salutation>, <addressee>!"})::
+
+ from pyparsing import Word, alphas
+
+ # define grammar of a greeting
+ greet = Word( alphas ) + "," + Word( alphas ) + "!"
+
+ hello = "Hello, World!"
+ print (hello, "->", greet.parseString( hello ))
+
+The program outputs the following::
+
+ Hello, World! -> ['Hello', ',', 'World', '!']
+
+The Python representation of the grammar is quite readable, owing to the self-explanatory
+class names, and the use of '+', '|' and '^' operators.
+
+The parsed results returned from C{parseString()} can be accessed as a nested list, a dictionary, or an
+object with named attributes.
+
+The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
+ - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.)
+ - quoted strings
+ - embedded comments
+"""
+
+__version__ = "2.0.6"
+__versionTime__ = "9 Nov 2015 19:03"
+__author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"
+
+import string
+from weakref import ref as wkref
+import copy
+import sys
+import warnings
+import re
+import sre_constants
+import collections
+import pprint
+import functools
+import itertools
+
+#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
+
+__all__ = [
+'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty',
+'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal',
+'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or',
+'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException',
+'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException',
+'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', 'Upcase',
+'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
+'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
+'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
+'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums',
+'htmlComment', 'javaStyleComment', 'keepOriginalText', 'line', 'lineEnd', 'lineStart', 'lineno',
+'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
+'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
+'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity',
+'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
+'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
+'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass',
+]
+
+PY_3 = sys.version.startswith('3')
+if PY_3:
+ _MAX_INT = sys.maxsize
+ basestring = str
+ unichr = chr
+ _ustr = str
+
+ # build list of single arg builtins, that can be used as parse actions
+ singleArgBuiltins = [sum, len, sorted, reversed, list, tuple, set, any, all, min, max]
+
+else:
+ _MAX_INT = sys.maxint
+ range = xrange
+
+ def _ustr(obj):
+ """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
+ str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
+ then < returns the unicode object | encodes it with the default encoding | ... >.
+ """
+ if isinstance(obj,unicode):
+ return obj
+
+ try:
+ # If this works, then _ustr(obj) has the same behaviour as str(obj), so
+ # it won't break any existing code.
+ return str(obj)
+
+ except UnicodeEncodeError:
+ # The Python docs (http://docs.python.org/ref/customization.html#l2h-182)
+ # state that "The return value must be a string object". However, does a
+ # unicode object (being a subclass of basestring) count as a "string
+ # object"?
+ # If so, then return a unicode object:
+ return unicode(obj)
+ # Else encode it... but how? There are many choices... :)
+ # Replace unprintables with escape codes?
+ #return unicode(obj).encode(sys.getdefaultencoding(), 'backslashreplace_errors')
+ # Replace unprintables with question marks?
+ #return unicode(obj).encode(sys.getdefaultencoding(), 'replace')
+ # ...
+
+ # build list of single arg builtins, tolerant of Python version, that can be used as parse actions
+ singleArgBuiltins = []
+ import __builtin__
+ for fname in "sum len sorted reversed list tuple set any all min max".split():
+ try:
+ singleArgBuiltins.append(getattr(__builtin__,fname))
+ except AttributeError:
+ continue
+
+_generatorType = type((y for y in range(1)))
+
+def _xml_escape(data):
+ """Escape &, <, >, ", ', etc. in a string of data."""
+
+ # ampersand must be replaced first
+ from_symbols = '&><"\''
+ to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split())
+ for from_,to_ in zip(from_symbols, to_symbols):
+ data = data.replace(from_, to_)
+ return data
+
+class _Constants(object):
+ pass
+
+alphas = string.ascii_lowercase + string.ascii_uppercase
+nums = "0123456789"
+hexnums = nums + "ABCDEFabcdef"
+alphanums = alphas + nums
+_bslash = chr(92)
+printables = "".join(c for c in string.printable if c not in string.whitespace)
+
+class ParseBaseException(Exception):
+ """base exception class for all parsing runtime exceptions"""
+ # Performance tuning: we construct a *lot* of these, so keep this
+ # constructor as small and fast as possible
+ def __init__( self, pstr, loc=0, msg=None, elem=None ):
+ self.loc = loc
+ if msg is None:
+ self.msg = pstr
+ self.pstr = ""
+ else:
+ self.msg = msg
+ self.pstr = pstr
+ self.parserElement = elem
+
+ def __getattr__( self, aname ):
+ """supported attributes by name are:
+ - lineno - returns the line number of the exception text
+ - col - returns the column number of the exception text
+ - line - returns the line containing the exception text
+ """
+ if( aname == "lineno" ):
+ return lineno( self.loc, self.pstr )
+ elif( aname in ("col", "column") ):
+ return col( self.loc, self.pstr )
+ elif( aname == "line" ):
+ return line( self.loc, self.pstr )
+ else:
+ raise AttributeError(aname)
+
+ def __str__( self ):
+ return "%s (at char %d), (line:%d, col:%d)" % \
+ ( self.msg, self.loc, self.lineno, self.column )
+ def __repr__( self ):
+ return _ustr(self)
+ def markInputline( self, markerString = ">!<" ):
+ """Extracts the exception line from the input string, and marks
+ the location of the exception with a special symbol.
+ """
+ line_str = self.line
+ line_column = self.column - 1
+ if markerString:
+ line_str = "".join((line_str[:line_column],
+ markerString, line_str[line_column:]))
+ return line_str.strip()
+ def __dir__(self):
+ return "loc msg pstr parserElement lineno col line " \
+ "markInputline __str__ __repr__".split()
+
+class ParseException(ParseBaseException):
+ """exception thrown when parse expressions don't match class;
+ supported attributes by name are:
+ - lineno - returns the line number of the exception text
+ - col - returns the column number of the exception text
+ - line - returns the line containing the exception text
+ """
+ pass
+
+class ParseFatalException(ParseBaseException):
+ """user-throwable exception thrown when inconsistent parse content
+ is found; stops all parsing immediately"""
+ pass
+
+class ParseSyntaxException(ParseFatalException):
+ """just like C{L{ParseFatalException}}, but thrown internally when an
+ C{L{ErrorStop<And._ErrorStop>}} ('-' operator) indicates that parsing is to stop immediately because
+ an unbacktrackable syntax error has been found"""
+ def __init__(self, pe):
+ super(ParseSyntaxException, self).__init__(
+ pe.pstr, pe.loc, pe.msg, pe.parserElement)
+
+#~ class ReparseException(ParseBaseException):
+ #~ """Experimental class - parse actions can raise this exception to cause
+ #~ pyparsing to reparse the input string:
+ #~ - with a modified input string, and/or
+ #~ - with a modified start location
+ #~ Set the values of the ReparseException in the constructor, and raise the
+ #~ exception in a parse action to cause pyparsing to use the new string/location.
+ #~ Setting the values as None causes no change to be made.
+ #~ """
+ #~ def __init_( self, newstring, restartLoc ):
+ #~ self.newParseText = newstring
+ #~ self.reparseLoc = restartLoc
+
+class RecursiveGrammarException(Exception):
+ """exception thrown by C{validate()} if the grammar could be improperly recursive"""
+ def __init__( self, parseElementList ):
+ self.parseElementTrace = parseElementList
+
+ def __str__( self ):
+ return "RecursiveGrammarException: %s" % self.parseElementTrace
+
+class _ParseResultsWithOffset(object):
+ def __init__(self,p1,p2):
+ self.tup = (p1,p2)
+ def __getitem__(self,i):
+ return self.tup[i]
+ def __repr__(self):
+ return repr(self.tup)
+ def setOffset(self,i):
+ self.tup = (self.tup[0],i)
+
+class ParseResults(object):
+ """Structured parse results, to provide multiple means of access to the parsed data:
+ - as a list (C{len(results)})
+ - by list index (C{results[0], results[1]}, etc.)
+ - by attribute (C{results.<resultsName>})
+ """
+ def __new__(cls, toklist, name=None, asList=True, modal=True ):
+ if isinstance(toklist, cls):
+ return toklist
+ retobj = object.__new__(cls)
+ retobj.__doinit = True
+ return retobj
+
+ # Performance tuning: we construct a *lot* of these, so keep this
+ # constructor as small and fast as possible
+ def __init__( self, toklist, name=None, asList=True, modal=True, isinstance=isinstance ):
+ if self.__doinit:
+ self.__doinit = False
+ self.__name = None
+ self.__parent = None
+ self.__accumNames = {}
+ if isinstance(toklist, list):
+ self.__toklist = toklist[:]
+ elif isinstance(toklist, _generatorType):
+ self.__toklist = list(toklist)
+ else:
+ self.__toklist = [toklist]
+ self.__tokdict = dict()
+
+ if name is not None and name:
+ if not modal:
+ self.__accumNames[name] = 0
+ if isinstance(name,int):
+ name = _ustr(name) # will always return a str, but use _ustr for consistency
+ self.__name = name
+ if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None,'',[])):
+ if isinstance(toklist,basestring):
+ toklist = [ toklist ]
+ if asList:
+ if isinstance(toklist,ParseResults):
+ self[name] = _ParseResultsWithOffset(toklist.copy(),0)
+ else:
+ self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0)
+ self[name].__name = name
+ else:
+ try:
+ self[name] = toklist[0]
+ except (KeyError,TypeError,IndexError):
+ self[name] = toklist
+
+ def __getitem__( self, i ):
+ if isinstance( i, (int,slice) ):
+ return self.__toklist[i]
+ else:
+ if i not in self.__accumNames:
+ return self.__tokdict[i][-1][0]
+ else:
+ return ParseResults([ v[0] for v in self.__tokdict[i] ])
+
+ def __setitem__( self, k, v, isinstance=isinstance ):
+ if isinstance(v,_ParseResultsWithOffset):
+ self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
+ sub = v[0]
+ elif isinstance(k,int):
+ self.__toklist[k] = v
+ sub = v
+ else:
+ self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)]
+ sub = v
+ if isinstance(sub,ParseResults):
+ sub.__parent = wkref(self)
+
+ def __delitem__( self, i ):
+ if isinstance(i,(int,slice)):
+ mylen = len( self.__toklist )
+ del self.__toklist[i]
+
+ # convert int to slice
+ if isinstance(i, int):
+ if i < 0:
+ i += mylen
+ i = slice(i, i+1)
+ # get removed indices
+ removed = list(range(*i.indices(mylen)))
+ removed.reverse()
+ # fixup indices in token dictionary
+ #~ for name in self.__tokdict:
+ #~ occurrences = self.__tokdict[name]
+ #~ for j in removed:
+ #~ for k, (value, position) in enumerate(occurrences):
+ #~ occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
+ for name,occurrences in self.__tokdict.items():
+ for j in removed:
+ for k, (value, position) in enumerate(occurrences):
+ occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
+ else:
+ del self.__tokdict[i]
+
+ def __contains__( self, k ):
+ return k in self.__tokdict
+
+ def __len__( self ): return len( self.__toklist )
+ def __bool__(self): return len( self.__toklist ) > 0
+ __nonzero__ = __bool__
+ def __iter__( self ): return iter( self.__toklist )
+ def __reversed__( self ): return iter( self.__toklist[::-1] )
+ def iterkeys( self ):
+ """Returns all named result keys."""
+ if hasattr(self.__tokdict, "iterkeys"):
+ return self.__tokdict.iterkeys()
+ else:
+ return iter(self.__tokdict)
+
+ def itervalues( self ):
+ """Returns all named result values."""
+ return (self[k] for k in self.iterkeys())
+
+ def iteritems( self ):
+ return ((k, self[k]) for k in self.iterkeys())
+
+ if PY_3:
+ keys = iterkeys
+ values = itervalues
+ items = iteritems
+ else:
+ def keys( self ):
+ """Returns all named result keys."""
+ return list(self.iterkeys())
+
+ def values( self ):
+ """Returns all named result values."""
+ return list(self.itervalues())
+
+ def items( self ):
+ """Returns all named result keys and values as a list of tuples."""
+ return list(self.iteritems())
+
+ def haskeys( self ):
+ """Since keys() returns an iterator, this method is helpful in bypassing
+ code that looks for the existence of any defined results names."""
+ return bool(self.__tokdict)
+
+ def pop( self, *args, **kwargs):
+ """Removes and returns item at specified index (default=last).
+ Supports both list and dict semantics for pop(). If passed no
+ argument or an integer argument, it will use list semantics
+ and pop tokens from the list of parsed tokens. If passed a
+ non-integer argument (most likely a string), it will use dict
+ semantics and pop the corresponding value from any defined
+ results names. A second default return value argument is
+ supported, just as in dict.pop()."""
+ if not args:
+ args = [-1]
+ for k,v in kwargs.items():
+ if k == 'default':
+ args = (args[0], v)
+ else:
+ raise TypeError("pop() got an unexpected keyword argument '%s'" % k)
+ if (isinstance(args[0], int) or
+ len(args) == 1 or
+ args[0] in self):
+ index = args[0]
+ ret = self[index]
+ del self[index]
+ return ret
+ else:
+ defaultvalue = args[1]
+ return defaultvalue
+
+ def get(self, key, defaultValue=None):
+ """Returns named result matching the given key, or if there is no
+ such name, then returns the given C{defaultValue} or C{None} if no
+ C{defaultValue} is specified."""
+ if key in self:
+ return self[key]
+ else:
+ return defaultValue
+
+ def insert( self, index, insStr ):
+ """Inserts new element at location index in the list of parsed tokens."""
+ self.__toklist.insert(index, insStr)
+ # fixup indices in token dictionary
+ #~ for name in self.__tokdict:
+ #~ occurrences = self.__tokdict[name]
+ #~ for k, (value, position) in enumerate(occurrences):
+ #~ occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
+ for name,occurrences in self.__tokdict.items():
+ for k, (value, position) in enumerate(occurrences):
+ occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
+
+ def append( self, item ):
+ """Add single element to end of ParseResults list of elements."""
+ self.__toklist.append(item)
+
+ def extend( self, itemseq ):
+ """Add sequence of elements to end of ParseResults list of elements."""
+ if isinstance(itemseq, ParseResults):
+ self += itemseq
+ else:
+ self.__toklist.extend(itemseq)
+
+ def clear( self ):
+ """Clear all elements and results names."""
+ del self.__toklist[:]
+ self.__tokdict.clear()
+
+ def __getattr__( self, name ):
+ try:
+ return self[name]
+ except KeyError:
+ return ""
+
+ if name in self.__tokdict:
+ if name not in self.__accumNames:
+ return self.__tokdict[name][-1][0]
+ else:
+ return ParseResults([ v[0] for v in self.__tokdict[name] ])
+ else:
+ return ""
+
+ def __add__( self, other ):
+ ret = self.copy()
+ ret += other
+ return ret
+
+ def __iadd__( self, other ):
+ if other.__tokdict:
+ offset = len(self.__toklist)
+ addoffset = lambda a: offset if a<0 else a+offset
+ otheritems = other.__tokdict.items()
+ otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) )
+ for (k,vlist) in otheritems for v in vlist]
+ for k,v in otherdictitems:
+ self[k] = v
+ if isinstance(v[0],ParseResults):
+ v[0].__parent = wkref(self)
+
+ self.__toklist += other.__toklist
+ self.__accumNames.update( other.__accumNames )
+ return self
+
+ def __radd__(self, other):
+ if isinstance(other,int) and other == 0:
+ return self.copy()
+
+ def __repr__( self ):
+ return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
+
+ def __str__( self ):
+ return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']'
+
+ def _asStringList( self, sep='' ):
+ out = []
+ for item in self.__toklist:
+ if out and sep:
+ out.append(sep)
+ if isinstance( item, ParseResults ):
+ out += item._asStringList()
+ else:
+ out.append( _ustr(item) )
+ return out
+
+ def asList( self ):
+ """Returns the parse results as a nested list of matching tokens, all converted to strings."""
+ return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist]
+
+ def asDict( self ):
+ """Returns the named parse results as dictionary."""
+ if PY_3:
+ return dict( self.items() )
+ else:
+ return dict( self.iteritems() )
+
+ def copy( self ):
+ """Returns a new copy of a C{ParseResults} object."""
+ ret = ParseResults( self.__toklist )
+ ret.__tokdict = self.__tokdict.copy()
+ ret.__parent = self.__parent
+ ret.__accumNames.update( self.__accumNames )
+ ret.__name = self.__name
+ return ret
+
+ def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
+ """Returns the parse results as XML. Tags are created for tokens and lists that have defined results names."""
+ nl = "\n"
+ out = []
+ namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items()
+ for v in vlist)
+ nextLevelIndent = indent + " "
+
+ # collapse out indents if formatting is not desired
+ if not formatted:
+ indent = ""
+ nextLevelIndent = ""
+ nl = ""
+
+ selfTag = None
+ if doctag is not None:
+ selfTag = doctag
+ else:
+ if self.__name:
+ selfTag = self.__name
+
+ if not selfTag:
+ if namedItemsOnly:
+ return ""
+ else:
+ selfTag = "ITEM"
+
+ out += [ nl, indent, "<", selfTag, ">" ]
+
+ for i,res in enumerate(self.__toklist):
+ if isinstance(res,ParseResults):
+ if i in namedItems:
+ out += [ res.asXML(namedItems[i],
+ namedItemsOnly and doctag is None,
+ nextLevelIndent,
+ formatted)]
+ else:
+ out += [ res.asXML(None,
+ namedItemsOnly and doctag is None,
+ nextLevelIndent,
+ formatted)]
+ else:
+ # individual token, see if there is a name for it
+ resTag = None
+ if i in namedItems:
+ resTag = namedItems[i]
+ if not resTag:
+ if namedItemsOnly:
+ continue
+ else:
+ resTag = "ITEM"
+ xmlBodyText = _xml_escape(_ustr(res))
+ out += [ nl, nextLevelIndent, "<", resTag, ">",
+ xmlBodyText,
+ "</", resTag, ">" ]
+
+ out += [ nl, indent, "</", selfTag, ">" ]
+ return "".join(out)
+
+ def __lookup(self,sub):
+ for k,vlist in self.__tokdict.items():
+ for v,loc in vlist:
+ if sub is v:
+ return k
+ return None
+
+ def getName(self):
+ """Returns the results name for this token expression."""
+ if self.__name:
+ return self.__name
+ elif self.__parent:
+ par = self.__parent()
+ if par:
+ return par.__lookup(self)
+ else:
+ return None
+ elif (len(self) == 1 and
+ len(self.__tokdict) == 1 and
+ self.__tokdict.values()[0][0][1] in (0,-1)):
+ return self.__tokdict.keys()[0]
+ else:
+ return None
+
+ def dump(self,indent='',depth=0):
+ """Diagnostic method for listing out the contents of a C{ParseResults}.
+ Accepts an optional C{indent} argument so that this string can be embedded
+ in a nested display of other data."""
+ out = []
+ NL = '\n'
+ out.append( indent+_ustr(self.asList()) )
+ if self.haskeys():
+ items = sorted(self.items())
+ for k,v in items:
+ if out:
+ out.append(NL)
+ out.append( "%s%s- %s: " % (indent,(' '*depth), k) )
+ if isinstance(v,ParseResults):
+ if v:
+ out.append( v.dump(indent,depth+1) )
+ else:
+ out.append(_ustr(v))
+ else:
+ out.append(_ustr(v))
+ elif any(isinstance(vv,ParseResults) for vv in self):
+ v = self
+ for i,vv in enumerate(v):
+ if isinstance(vv,ParseResults):
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) ))
+ else:
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv)))
+
+ return "".join(out)
+
+ def pprint(self, *args, **kwargs):
+ """Pretty-printer for parsed results as a list, using the C{pprint} module.
+ Accepts additional positional or keyword args as defined for the
+ C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint})"""
+ pprint.pprint(self.asList(), *args, **kwargs)
+
+ # add support for pickle protocol
+ def __getstate__(self):
+ return ( self.__toklist,
+ ( self.__tokdict.copy(),
+ self.__parent is not None and self.__parent() or None,
+ self.__accumNames,
+ self.__name ) )
+
+ def __setstate__(self,state):
+ self.__toklist = state[0]
+ (self.__tokdict,
+ par,
+ inAccumNames,
+ self.__name) = state[1]
+ self.__accumNames = {}
+ self.__accumNames.update(inAccumNames)
+ if par is not None:
+ self.__parent = wkref(par)
+ else:
+ self.__parent = None
+
+ def __dir__(self):
+ return dir(super(ParseResults,self)) + list(self.keys())
+
+collections.MutableMapping.register(ParseResults)
+
+def col (loc,strg):
+ """Returns current column within a string, counting newlines as line separators.
+ The first column is number 1.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ s = strg
+ return 1 if loc<len(s) and s[loc] == '\n' else loc - s.rfind("\n", 0, loc)
+
+def lineno(loc,strg):
+ """Returns current line number within a string, counting newlines as line separators.
+ The first line is number 1.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ return strg.count("\n",0,loc) + 1
+
+def line( loc, strg ):
+ """Returns the line of text containing loc within a string, counting newlines as line separators.
+ """
+ lastCR = strg.rfind("\n", 0, loc)
+ nextCR = strg.find("\n", loc)
+ if nextCR >= 0:
+ return strg[lastCR+1:nextCR]
+ else:
+ return strg[lastCR+1:]
+
+def _defaultStartDebugAction( instring, loc, expr ):
+ print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )))
+
+def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
+ print ("Matched " + _ustr(expr) + " -> " + str(toks.asList()))
+
+def _defaultExceptionDebugAction( instring, loc, expr, exc ):
+ print ("Exception raised:" + _ustr(exc))
+
+def nullDebugAction(*args):
+ """'Do-nothing' debug action, to suppress debugging output during parsing."""
+ pass
+
+# Only works on Python 3.x - nonlocal is toxic to Python 2 installs
+#~ 'decorator to trim function calls to match the arity of the target'
+#~ def _trim_arity(func, maxargs=3):
+ #~ if func in singleArgBuiltins:
+ #~ return lambda s,l,t: func(t)
+ #~ limit = 0
+ #~ foundArity = False
+ #~ def wrapper(*args):
+ #~ nonlocal limit,foundArity
+ #~ while 1:
+ #~ try:
+ #~ ret = func(*args[limit:])
+ #~ foundArity = True
+ #~ return ret
+ #~ except TypeError:
+ #~ if limit == maxargs or foundArity:
+ #~ raise
+ #~ limit += 1
+ #~ continue
+ #~ return wrapper
+
+# this version is Python 2.x-3.x cross-compatible
+'decorator to trim function calls to match the arity of the target'
+def _trim_arity(func, maxargs=2):
+ if func in singleArgBuiltins:
+ return lambda s,l,t: func(t)
+ limit = [0]
+ foundArity = [False]
+ def wrapper(*args):
+ while 1:
+ try:
+ ret = func(*args[limit[0]:])
+ foundArity[0] = True
+ return ret
+ except TypeError:
+ if limit[0] <= maxargs and not foundArity[0]:
+ limit[0] += 1
+ continue
+ raise
+ return wrapper
+
+class ParserElement(object):
+ """Abstract base level parser element class."""
+ DEFAULT_WHITE_CHARS = " \n\t\r"
+ verbose_stacktrace = False
+
+ @staticmethod
+ def setDefaultWhitespaceChars( chars ):
+ """Overrides the default whitespace chars
+ """
+ ParserElement.DEFAULT_WHITE_CHARS = chars
+
+ @staticmethod
+ def inlineLiteralsUsing(cls):
+ """
+ Set class to be used for inclusion of string literals into a parser.
+ """
+ ParserElement.literalStringClass = cls
+
+ def __init__( self, savelist=False ):
+ self.parseAction = list()
+ self.failAction = None
+ #~ self.name = "<unknown>" # don't define self.name, let subclasses try/except upcall
+ self.strRepr = None
+ self.resultsName = None
+ self.saveAsList = savelist
+ self.skipWhitespace = True
+ self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+ self.copyDefaultWhiteChars = True
+ self.mayReturnEmpty = False # used when checking for left-recursion
+ self.keepTabs = False
+ self.ignoreExprs = list()
+ self.debug = False
+ self.streamlined = False
+ self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index
+ self.errmsg = ""
+ self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all)
+ self.debugActions = ( None, None, None ) #custom debug actions
+ self.re = None
+ self.callPreparse = True # used to avoid redundant calls to preParse
+ self.callDuringTry = False
+
+ def copy( self ):
+ """Make a copy of this C{ParserElement}. Useful for defining different parse actions
+ for the same parsing pattern, using copies of the original parse element."""
+ cpy = copy.copy( self )
+ cpy.parseAction = self.parseAction[:]
+ cpy.ignoreExprs = self.ignoreExprs[:]
+ if self.copyDefaultWhiteChars:
+ cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+ return cpy
+
+ def setName( self, name ):
+ """Define name for this expression, for use in debugging."""
+ self.name = name
+ self.errmsg = "Expected " + self.name
+ if hasattr(self,"exception"):
+ self.exception.msg = self.errmsg
+ return self
+
+ def setResultsName( self, name, listAllMatches=False ):
+ """Define name for referencing matching tokens as a nested attribute
+ of the returned parse results.
+ NOTE: this returns a *copy* of the original C{ParserElement} object;
+ this is so that the client can define a basic element, such as an
+ integer, and reference it in multiple places with different names.
+
+ You can also set results names using the abbreviated syntax,
+ C{expr("name")} in place of C{expr.setResultsName("name")} -
+ see L{I{__call__}<__call__>}.
+ """
+ newself = self.copy()
+ if name.endswith("*"):
+ name = name[:-1]
+ listAllMatches=True
+ newself.resultsName = name
+ newself.modalResults = not listAllMatches
+ return newself
+
+ def setBreak(self,breakFlag = True):
+ """Method to invoke the Python pdb debugger when this element is
+ about to be parsed. Set C{breakFlag} to True to enable, False to
+ disable.
+ """
+ if breakFlag:
+ _parseMethod = self._parse
+ def breaker(instring, loc, doActions=True, callPreParse=True):
+ import pdb
+ pdb.set_trace()
+ return _parseMethod( instring, loc, doActions, callPreParse )
+ breaker._originalParseMethod = _parseMethod
+ self._parse = breaker
+ else:
+ if hasattr(self._parse,"_originalParseMethod"):
+ self._parse = self._parse._originalParseMethod
+ return self
+
+ def setParseAction( self, *fns, **kwargs ):
+ """Define action to perform when successfully matching parse element definition.
+ Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)},
+ C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
+ - s = the original string being parsed (see note below)
+ - loc = the location of the matching substring
+ - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object
+ If the functions in fns modify the tokens, they can return them as the return
+ value from fn, and the modified list of tokens will replace the original.
+ Otherwise, fn does not need to return any value.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{parseString}<parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ self.parseAction = list(map(_trim_arity, list(fns)))
+ self.callDuringTry = kwargs.get("callDuringTry", False)
+ return self
+
+ def addParseAction( self, *fns, **kwargs ):
+ """Add parse action to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}."""
+ self.parseAction += list(map(_trim_arity, list(fns)))
+ self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+ return self
+
+ def addCondition(self, *fns, **kwargs):
+ """Add a boolean predicate function to expression's list of parse actions. See
+ L{I{setParseAction}<setParseAction>}. Optional keyword argument C{message} can
+ be used to define a custom message to be used in the raised exception."""
+ msg = kwargs.get("message") or "failed user-defined condition"
+ for fn in fns:
+ def pa(s,l,t):
+ if not bool(_trim_arity(fn)(s,l,t)):
+ raise ParseException(s,l,msg)
+ return t
+ self.parseAction.append(pa)
+ self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+ return self
+
+ def setFailAction( self, fn ):
+ """Define action to perform if parsing fails at this expression.
+ Fail acton fn is a callable function that takes the arguments
+ C{fn(s,loc,expr,err)} where:
+ - s = string being parsed
+ - loc = location where expression match was attempted and failed
+ - expr = the parse expression that failed
+ - err = the exception thrown
+ The function returns no value. It may throw C{L{ParseFatalException}}
+ if it is desired to stop parsing immediately."""
+ self.failAction = fn
+ return self
+
+ def _skipIgnorables( self, instring, loc ):
+ exprsFound = True
+ while exprsFound:
+ exprsFound = False
+ for e in self.ignoreExprs:
+ try:
+ while 1:
+ loc,dummy = e._parse( instring, loc )
+ exprsFound = True
+ except ParseException:
+ pass
+ return loc
+
+ def preParse( self, instring, loc ):
+ if self.ignoreExprs:
+ loc = self._skipIgnorables( instring, loc )
+
+ if self.skipWhitespace:
+ wt = self.whiteChars
+ instrlen = len(instring)
+ while loc < instrlen and instring[loc] in wt:
+ loc += 1
+
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ return loc, []
+
+ def postParse( self, instring, loc, tokenlist ):
+ return tokenlist
+
+ #~ @profile
+ def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ):
+ debugging = ( self.debug ) #and doActions )
+
+ if debugging or self.failAction:
+ #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
+ if (self.debugActions[0] ):
+ self.debugActions[0]( instring, loc, self )
+ if callPreParse and self.callPreparse:
+ preloc = self.preParse( instring, loc )
+ else:
+ preloc = loc
+ tokensStart = preloc
+ try:
+ try:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+ except IndexError:
+ raise ParseException( instring, len(instring), self.errmsg, self )
+ except ParseBaseException as err:
+ #~ print ("Exception raised:", err)
+ if self.debugActions[2]:
+ self.debugActions[2]( instring, tokensStart, self, err )
+ if self.failAction:
+ self.failAction( instring, tokensStart, self, err )
+ raise
+ else:
+ if callPreParse and self.callPreparse:
+ preloc = self.preParse( instring, loc )
+ else:
+ preloc = loc
+ tokensStart = preloc
+ if self.mayIndexError or loc >= len(instring):
+ try:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+ except IndexError:
+ raise ParseException( instring, len(instring), self.errmsg, self )
+ else:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+
+ tokens = self.postParse( instring, loc, tokens )
+
+ retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults )
+ if self.parseAction and (doActions or self.callDuringTry):
+ if debugging:
+ try:
+ for fn in self.parseAction:
+ tokens = fn( instring, tokensStart, retTokens )
+ if tokens is not None:
+ retTokens = ParseResults( tokens,
+ self.resultsName,
+ asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+ modal=self.modalResults )
+ except ParseBaseException as err:
+ #~ print "Exception raised in user parse action:", err
+ if (self.debugActions[2] ):
+ self.debugActions[2]( instring, tokensStart, self, err )
+ raise
+ else:
+ for fn in self.parseAction:
+ tokens = fn( instring, tokensStart, retTokens )
+ if tokens is not None:
+ retTokens = ParseResults( tokens,
+ self.resultsName,
+ asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+ modal=self.modalResults )
+
+ if debugging:
+ #~ print ("Matched",self,"->",retTokens.asList())
+ if (self.debugActions[1] ):
+ self.debugActions[1]( instring, tokensStart, loc, self, retTokens )
+
+ return loc, retTokens
+
+ def tryParse( self, instring, loc ):
+ try:
+ return self._parse( instring, loc, doActions=False )[0]
+ except ParseFatalException:
+ raise ParseException( instring, loc, self.errmsg, self)
+
+ # this method gets repeatedly called during backtracking with the same arguments -
+ # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
+ def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
+ lookup = (self,instring,loc,callPreParse,doActions)
+ if lookup in ParserElement._exprArgCache:
+ value = ParserElement._exprArgCache[ lookup ]
+ if isinstance(value, Exception):
+ raise value
+ return (value[0],value[1].copy())
+ else:
+ try:
+ value = self._parseNoCache( instring, loc, doActions, callPreParse )
+ ParserElement._exprArgCache[ lookup ] = (value[0],value[1].copy())
+ return value
+ except ParseBaseException as pe:
+ pe.__traceback__ = None
+ ParserElement._exprArgCache[ lookup ] = pe
+ raise
+
+ _parse = _parseNoCache
+
+ # argument cache for optimizing repeated calls when backtracking through recursive expressions
+ _exprArgCache = {}
+ @staticmethod
+ def resetCache():
+ ParserElement._exprArgCache.clear()
+
+ _packratEnabled = False
+ @staticmethod
+ def enablePackrat():
+ """Enables "packrat" parsing, which adds memoizing to the parsing logic.
+ Repeated parse attempts at the same string location (which happens
+ often in many complex grammars) can immediately return a cached value,
+ instead of re-executing parsing/validating code. Memoizing is done of
+ both valid results and parsing exceptions.
+
+ This speedup may break existing programs that use parse actions that
+ have side-effects. For this reason, packrat parsing is disabled when
+ you first import pyparsing. To activate the packrat feature, your
+ program must call the class method C{ParserElement.enablePackrat()}. If
+ your program uses C{psyco} to "compile as you go", you must call
+ C{enablePackrat} before calling C{psyco.full()}. If you do not do this,
+ Python will crash. For best results, call C{enablePackrat()} immediately
+ after importing pyparsing.
+ """
+ if not ParserElement._packratEnabled:
+ ParserElement._packratEnabled = True
+ ParserElement._parse = ParserElement._parseCache
+
+ def parseString( self, instring, parseAll=False ):
+ """Execute the parse expression with the given string.
+ This is the main interface to the client code, once the complete
+ expression has been built.
+
+ If you want the grammar to require that the entire input string be
+ successfully parsed, then set C{parseAll} to True (equivalent to ending
+ the grammar with C{L{StringEnd()}}).
+
+ Note: C{parseString} implicitly calls C{expandtabs()} on the input string,
+ in order to report proper column numbers in parse actions.
+ If the input string contains tabs and
+ the grammar uses parse actions that use the C{loc} argument to index into the
+ string being parsed, you can ensure you have a consistent view of the input
+ string by:
+ - calling C{parseWithTabs} on your grammar before calling C{parseString}
+ (see L{I{parseWithTabs}<parseWithTabs>})
+ - define your parse action using the full C{(s,loc,toks)} signature, and
+ reference the input string using the parse action's C{s} argument
+ - explictly expand the tabs in your input string before calling
+ C{parseString}
+ """
+ ParserElement.resetCache()
+ if not self.streamlined:
+ self.streamline()
+ #~ self.saveAsList = True
+ for e in self.ignoreExprs:
+ e.streamline()
+ if not self.keepTabs:
+ instring = instring.expandtabs()
+ try:
+ loc, tokens = self._parse( instring, 0 )
+ if parseAll:
+ loc = self.preParse( instring, loc )
+ se = Empty() + StringEnd()
+ se._parse( instring, loc )
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+ else:
+ return tokens
+
+ def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ):
+ """Scan the input string for expression matches. Each match will return the
+ matching tokens, start location, and end location. May be called with optional
+ C{maxMatches} argument, to clip scanning after 'n' matches are found. If
+ C{overlap} is specified, then overlapping matches will be reported.
+
+ Note that the start and end locations are reported relative to the string
+ being parsed. See L{I{parseString}<parseString>} for more information on parsing
+ strings with embedded tabs."""
+ if not self.streamlined:
+ self.streamline()
+ for e in self.ignoreExprs:
+ e.streamline()
+
+ if not self.keepTabs:
+ instring = _ustr(instring).expandtabs()
+ instrlen = len(instring)
+ loc = 0
+ preparseFn = self.preParse
+ parseFn = self._parse
+ ParserElement.resetCache()
+ matches = 0
+ try:
+ while loc <= instrlen and matches < maxMatches:
+ try:
+ preloc = preparseFn( instring, loc )
+ nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )
+ except ParseException:
+ loc = preloc+1
+ else:
+ if nextLoc > loc:
+ matches += 1
+ yield tokens, preloc, nextLoc
+ if overlap:
+ nextloc = preparseFn( instring, loc )
+ if nextloc > loc:
+ loc = nextLoc
+ else:
+ loc += 1
+ else:
+ loc = nextLoc
+ else:
+ loc = preloc+1
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def transformString( self, instring ):
+ """Extension to C{L{scanString}}, to modify matching text with modified tokens that may
+ be returned from a parse action. To use C{transformString}, define a grammar and
+ attach a parse action to it that modifies the returned token list.
+ Invoking C{transformString()} on a target string will then scan for matches,
+ and replace the matched text patterns according to the logic in the parse
+ action. C{transformString()} returns the resulting transformed string."""
+ out = []
+ lastE = 0
+ # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
+ # keep string locs straight between transformString and scanString
+ self.keepTabs = True
+ try:
+ for t,s,e in self.scanString( instring ):
+ out.append( instring[lastE:s] )
+ if t:
+ if isinstance(t,ParseResults):
+ out += t.asList()
+ elif isinstance(t,list):
+ out += t
+ else:
+ out.append(t)
+ lastE = e
+ out.append(instring[lastE:])
+ out = [o for o in out if o]
+ return "".join(map(_ustr,_flatten(out)))
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def searchString( self, instring, maxMatches=_MAX_INT ):
+ """Another extension to C{L{scanString}}, simplifying the access to the tokens found
+ to match the given parse expression. May be called with optional
+ C{maxMatches} argument, to clip searching after 'n' matches are found.
+ """
+ try:
+ return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def __add__(self, other ):
+ """Implementation of + operator - returns C{L{And}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return And( [ self, other ] )
+
+ def __radd__(self, other ):
+ """Implementation of + operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other + self
+
+ def __sub__(self, other):
+ """Implementation of - operator, returns C{L{And}} with error stop"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return And( [ self, And._ErrorStop(), other ] )
+
+ def __rsub__(self, other ):
+ """Implementation of - operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other - self
+
+ def __mul__(self,other):
+ """Implementation of * operator, allows use of C{expr * 3} in place of
+ C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer
+ tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples
+ may also include C{None} as in:
+ - C{expr*(n,None)} or C{expr*(n,)} is equivalent
+ to C{expr*n + L{ZeroOrMore}(expr)}
+ (read as "at least n instances of C{expr}")
+ - C{expr*(None,n)} is equivalent to C{expr*(0,n)}
+ (read as "0 to n instances of C{expr}")
+ - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)}
+ - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)}
+
+ Note that C{expr*(None,n)} does not raise an exception if
+ more than n exprs exist in the input stream; that is,
+ C{expr*(None,n)} does not enforce a maximum number of expr
+ occurrences. If this behavior is desired, then write
+ C{expr*(None,n) + ~expr}
+
+ """
+ if isinstance(other,int):
+ minElements, optElements = other,0
+ elif isinstance(other,tuple):
+ other = (other + (None, None))[:2]
+ if other[0] is None:
+ other = (0, other[1])
+ if isinstance(other[0],int) and other[1] is None:
+ if other[0] == 0:
+ return ZeroOrMore(self)
+ if other[0] == 1:
+ return OneOrMore(self)
+ else:
+ return self*other[0] + ZeroOrMore(self)
+ elif isinstance(other[0],int) and isinstance(other[1],int):
+ minElements, optElements = other
+ optElements -= minElements
+ else:
+ raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1]))
+ else:
+ raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other))
+
+ if minElements < 0:
+ raise ValueError("cannot multiply ParserElement by negative value")
+ if optElements < 0:
+ raise ValueError("second tuple value must be greater or equal to first tuple value")
+ if minElements == optElements == 0:
+ raise ValueError("cannot multiply ParserElement by 0 or (0,0)")
+
+ if (optElements):
+ def makeOptionalList(n):
+ if n>1:
+ return Optional(self + makeOptionalList(n-1))
+ else:
+ return Optional(self)
+ if minElements:
+ if minElements == 1:
+ ret = self + makeOptionalList(optElements)
+ else:
+ ret = And([self]*minElements) + makeOptionalList(optElements)
+ else:
+ ret = makeOptionalList(optElements)
+ else:
+ if minElements == 1:
+ ret = self
+ else:
+ ret = And([self]*minElements)
+ return ret
+
+ def __rmul__(self, other):
+ return self.__mul__(other)
+
+ def __or__(self, other ):
+ """Implementation of | operator - returns C{L{MatchFirst}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return MatchFirst( [ self, other ] )
+
+ def __ror__(self, other ):
+ """Implementation of | operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other | self
+
+ def __xor__(self, other ):
+ """Implementation of ^ operator - returns C{L{Or}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return Or( [ self, other ] )
+
+ def __rxor__(self, other ):
+ """Implementation of ^ operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other ^ self
+
+ def __and__(self, other ):
+ """Implementation of & operator - returns C{L{Each}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return Each( [ self, other ] )
+
+ def __rand__(self, other ):
+ """Implementation of & operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other & self
+
+ def __invert__( self ):
+ """Implementation of ~ operator - returns C{L{NotAny}}"""
+ return NotAny( self )
+
+ def __call__(self, name=None):
+ """Shortcut for C{L{setResultsName}}, with C{listAllMatches=default}::
+ userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
+ could be written as::
+ userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
+
+ If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be
+ passed as C{True}.
+
+ If C{name} is omitted, same as calling C{L{copy}}.
+ """
+ if name is not None:
+ return self.setResultsName(name)
+ else:
+ return self.copy()
+
+ def suppress( self ):
+ """Suppresses the output of this C{ParserElement}; useful to keep punctuation from
+ cluttering up returned output.
+ """
+ return Suppress( self )
+
+ def leaveWhitespace( self ):
+ """Disables the skipping of whitespace before matching the characters in the
+ C{ParserElement}'s defined pattern. This is normally only used internally by
+ the pyparsing module, but may be needed in some whitespace-sensitive grammars.
+ """
+ self.skipWhitespace = False
+ return self
+
+ def setWhitespaceChars( self, chars ):
+ """Overrides the default whitespace chars
+ """
+ self.skipWhitespace = True
+ self.whiteChars = chars
+ self.copyDefaultWhiteChars = False
+ return self
+
+ def parseWithTabs( self ):
+ """Overrides default behavior to expand C{<TAB>}s to spaces before parsing the input string.
+ Must be called before C{parseString} when the input grammar contains elements that
+ match C{<TAB>} characters."""
+ self.keepTabs = True
+ return self
+
+ def ignore( self, other ):
+ """Define expression to be ignored (e.g., comments) while doing pattern
+ matching; may be called repeatedly, to define multiple comment or other
+ ignorable patterns.
+ """
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ self.ignoreExprs.append( other.copy() )
+ else:
+ self.ignoreExprs.append( Suppress( other.copy() ) )
+ return self
+
+ def setDebugActions( self, startAction, successAction, exceptionAction ):
+ """Enable display of debugging messages while doing pattern matching."""
+ self.debugActions = (startAction or _defaultStartDebugAction,
+ successAction or _defaultSuccessDebugAction,
+ exceptionAction or _defaultExceptionDebugAction)
+ self.debug = True
+ return self
+
+ def setDebug( self, flag=True ):
+ """Enable display of debugging messages while doing pattern matching.
+ Set C{flag} to True to enable, False to disable."""
+ if flag:
+ self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
+ else:
+ self.debug = False
+ return self
+
+ def __str__( self ):
+ return self.name
+
+ def __repr__( self ):
+ return _ustr(self)
+
+ def streamline( self ):
+ self.streamlined = True
+ self.strRepr = None
+ return self
+
+ def checkRecursion( self, parseElementList ):
+ pass
+
+ def validate( self, validateTrace=[] ):
+ """Check defined expressions for valid structure, check for infinite recursive definitions."""
+ self.checkRecursion( [] )
+
+ def parseFile( self, file_or_filename, parseAll=False ):
+ """Execute the parse expression on the given file or filename.
+ If a filename is specified (instead of a file object),
+ the entire file is opened, read, and closed before parsing.
+ """
+ try:
+ file_contents = file_or_filename.read()
+ except AttributeError:
+ f = open(file_or_filename, "r")
+ file_contents = f.read()
+ f.close()
+ try:
+ return self.parseString(file_contents, parseAll)
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def __eq__(self,other):
+ if isinstance(other, ParserElement):
+ return self is other or self.__dict__ == other.__dict__
+ elif isinstance(other, basestring):
+ try:
+ self.parseString(_ustr(other), parseAll=True)
+ return True
+ except ParseBaseException:
+ return False
+ else:
+ return super(ParserElement,self)==other
+
+ def __ne__(self,other):
+ return not (self == other)
+
+ def __hash__(self):
+ return hash(id(self))
+
+ def __req__(self,other):
+ return self == other
+
+ def __rne__(self,other):
+ return not (self == other)
+
+ def runTests(self, tests, parseAll=False):
+ """Execute the parse expression on a series of test strings, showing each
+ test, the parsed results or where the parse failed. Quick and easy way to
+ run a parse expression against a list of sample strings.
+
+ Parameters:
+ - tests - a list of separate test strings, or a multiline string of test strings
+ - parseAll - (default=False) - flag to pass to C{L{parseString}} when running tests
+ """
+ if isinstance(tests, basestring):
+ tests = map(str.strip, tests.splitlines())
+ for t in tests:
+ out = [t]
+ try:
+ out.append(self.parseString(t, parseAll=parseAll).dump())
+ except ParseException as pe:
+ if '\n' in t:
+ out.append(line(pe.loc, t))
+ out.append(' '*(col(pe.loc,t)-1) + '^')
+ else:
+ out.append(' '*pe.loc + '^')
+ out.append(str(pe))
+ out.append('')
+ print('\n'.join(out))
+
+
+class Token(ParserElement):
+ """Abstract C{ParserElement} subclass, for defining atomic matching patterns."""
+ def __init__( self ):
+ super(Token,self).__init__( savelist=False )
+
+
+class Empty(Token):
+ """An empty token, will always match."""
+ def __init__( self ):
+ super(Empty,self).__init__()
+ self.name = "Empty"
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+
+
+class NoMatch(Token):
+ """A token that will never match."""
+ def __init__( self ):
+ super(NoMatch,self).__init__()
+ self.name = "NoMatch"
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+ self.errmsg = "Unmatchable token"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Literal(Token):
+ """Token to exactly match a specified string."""
+ def __init__( self, matchString ):
+ super(Literal,self).__init__()
+ self.match = matchString
+ self.matchLen = len(matchString)
+ try:
+ self.firstMatchChar = matchString[0]
+ except IndexError:
+ warnings.warn("null string passed to Literal; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+ self.__class__ = Empty
+ self.name = '"%s"' % _ustr(self.match)
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = False
+ self.mayIndexError = False
+
+ # Performance tuning: this routine gets called a *lot*
+ # if this is a single character match string and the first character matches,
+ # short-circuit as quickly as possible, and avoid calling startswith
+ #~ @profile
+ def parseImpl( self, instring, loc, doActions=True ):
+ if (instring[loc] == self.firstMatchChar and
+ (self.matchLen==1 or instring.startswith(self.match,loc)) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+_L = Literal
+ParserElement.literalStringClass = Literal
+
+class Keyword(Token):
+ """Token to exactly match a specified string as a keyword, that is, it must be
+ immediately followed by a non-keyword character. Compare with C{L{Literal}}::
+ Literal("if") will match the leading C{'if'} in C{'ifAndOnlyIf'}.
+ Keyword("if") will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'}
+ Accepts two optional constructor arguments in addition to the keyword string:
+ C{identChars} is a string of characters that would be valid identifier characters,
+ defaulting to all alphanumerics + "_" and "$"; C{caseless} allows case-insensitive
+ matching, default is C{False}.
+ """
+ DEFAULT_KEYWORD_CHARS = alphanums+"_$"
+
+ def __init__( self, matchString, identChars=DEFAULT_KEYWORD_CHARS, caseless=False ):
+ super(Keyword,self).__init__()
+ self.match = matchString
+ self.matchLen = len(matchString)
+ try:
+ self.firstMatchChar = matchString[0]
+ except IndexError:
+ warnings.warn("null string passed to Keyword; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+ self.name = '"%s"' % self.match
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = False
+ self.mayIndexError = False
+ self.caseless = caseless
+ if caseless:
+ self.caselessmatch = matchString.upper()
+ identChars = identChars.upper()
+ self.identChars = set(identChars)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.caseless:
+ if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and
+ (loc == 0 or instring[loc-1].upper() not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ else:
+ if (instring[loc] == self.firstMatchChar and
+ (self.matchLen==1 or instring.startswith(self.match,loc)) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
+ (loc == 0 or instring[loc-1] not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ def copy(self):
+ c = super(Keyword,self).copy()
+ c.identChars = Keyword.DEFAULT_KEYWORD_CHARS
+ return c
+
+ @staticmethod
+ def setDefaultKeywordChars( chars ):
+ """Overrides the default Keyword chars
+ """
+ Keyword.DEFAULT_KEYWORD_CHARS = chars
+
+class CaselessLiteral(Literal):
+ """Token to match a specified string, ignoring case of letters.
+ Note: the matched results will always be in the case of the given
+ match string, NOT the case of the input text.
+ """
+ def __init__( self, matchString ):
+ super(CaselessLiteral,self).__init__( matchString.upper() )
+ # Preserve the defining literal.
+ self.returnString = matchString
+ self.name = "'%s'" % self.returnString
+ self.errmsg = "Expected " + self.name
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if instring[ loc:loc+self.matchLen ].upper() == self.match:
+ return loc+self.matchLen, self.returnString
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class CaselessKeyword(Keyword):
+ def __init__( self, matchString, identChars=Keyword.DEFAULT_KEYWORD_CHARS ):
+ super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class Word(Token):
+ """Token for matching words composed of allowed character sets.
+ Defined with string containing all allowed initial characters,
+ an optional string containing allowed body characters (if omitted,
+ defaults to the initial character set), and an optional minimum,
+ maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction. An optional
+ C{exclude} parameter can list characters that might be found in
+ the input C{bodyChars} string; useful to define a word of all printables
+ except for one or two characters, for instance.
+ """
+ def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ):
+ super(Word,self).__init__()
+ if excludeChars:
+ initChars = ''.join(c for c in initChars if c not in excludeChars)
+ if bodyChars:
+ bodyChars = ''.join(c for c in bodyChars if c not in excludeChars)
+ self.initCharsOrig = initChars
+ self.initChars = set(initChars)
+ if bodyChars :
+ self.bodyCharsOrig = bodyChars
+ self.bodyChars = set(bodyChars)
+ else:
+ self.bodyCharsOrig = initChars
+ self.bodyChars = set(initChars)
+
+ self.maxSpecified = max > 0
+
+ if min < 1:
+ raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted")
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.asKeyword = asKeyword
+
+ if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0):
+ if self.bodyCharsOrig == self.initCharsOrig:
+ self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig)
+ elif len(self.initCharsOrig) == 1:
+ self.reString = "%s[%s]*" % \
+ (re.escape(self.initCharsOrig),
+ _escapeRegexRangeChars(self.bodyCharsOrig),)
+ else:
+ self.reString = "[%s][%s]*" % \
+ (_escapeRegexRangeChars(self.initCharsOrig),
+ _escapeRegexRangeChars(self.bodyCharsOrig),)
+ if self.asKeyword:
+ self.reString = r"\b"+self.reString+r"\b"
+ try:
+ self.re = re.compile( self.reString )
+ except:
+ self.re = None
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.re:
+ result = self.re.match(instring,loc)
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ return loc, result.group()
+
+ if not(instring[ loc ] in self.initChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ start = loc
+ loc += 1
+ instrlen = len(instring)
+ bodychars = self.bodyChars
+ maxloc = start + self.maxLen
+ maxloc = min( maxloc, instrlen )
+ while loc < maxloc and instring[loc] in bodychars:
+ loc += 1
+
+ throwException = False
+ if loc - start < self.minLen:
+ throwException = True
+ if self.maxSpecified and loc < instrlen and instring[loc] in bodychars:
+ throwException = True
+ if self.asKeyword:
+ if (start>0 and instring[start-1] in bodychars) or (loc<instrlen and instring[loc] in bodychars):
+ throwException = True
+
+ if throwException:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+ def __str__( self ):
+ try:
+ return super(Word,self).__str__()
+ except:
+ pass
+
+
+ if self.strRepr is None:
+
+ def charsAsStr(s):
+ if len(s)>4:
+ return s[:4]+"..."
+ else:
+ return s
+
+ if ( self.initCharsOrig != self.bodyCharsOrig ):
+ self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) )
+ else:
+ self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig)
+
+ return self.strRepr
+
+
+class Regex(Token):
+ """Token for matching strings that match a given regular expression.
+ Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
+ """
+ compiledREtype = type(re.compile("[A-Z]"))
+ def __init__( self, pattern, flags=0):
+ """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags."""
+ super(Regex,self).__init__()
+
+ if isinstance(pattern, basestring):
+ if len(pattern) == 0:
+ warnings.warn("null string passed to Regex; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+
+ self.pattern = pattern
+ self.flags = flags
+
+ try:
+ self.re = re.compile(self.pattern, self.flags)
+ self.reString = self.pattern
+ except sre_constants.error:
+ warnings.warn("invalid pattern (%s) passed to Regex" % pattern,
+ SyntaxWarning, stacklevel=2)
+ raise
+
+ elif isinstance(pattern, Regex.compiledREtype):
+ self.re = pattern
+ self.pattern = \
+ self.reString = str(pattern)
+ self.flags = flags
+
+ else:
+ raise ValueError("Regex may only be constructed with a string or a compiled RE object")
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ result = self.re.match(instring,loc)
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ d = result.groupdict()
+ ret = ParseResults(result.group())
+ if d:
+ for k in d:
+ ret[k] = d[k]
+ return loc,ret
+
+ def __str__( self ):
+ try:
+ return super(Regex,self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "Re:(%s)" % repr(self.pattern)
+
+ return self.strRepr
+
+
+class QuotedString(Token):
+ """Token for matching strings that are delimited by quoting characters.
+ """
+ def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None):
+ """
+ Defined with the following parameters:
+ - quoteChar - string of one or more characters defining the quote delimiting string
+ - escChar - character to escape quotes, typically backslash (default=None)
+ - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=None)
+ - multiline - boolean indicating whether quotes can span multiple lines (default=C{False})
+ - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True})
+ - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar)
+ """
+ super(QuotedString,self).__init__()
+
+ # remove white space from quote chars - wont work anyway
+ quoteChar = quoteChar.strip()
+ if len(quoteChar) == 0:
+ warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+ raise SyntaxError()
+
+ if endQuoteChar is None:
+ endQuoteChar = quoteChar
+ else:
+ endQuoteChar = endQuoteChar.strip()
+ if len(endQuoteChar) == 0:
+ warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+ raise SyntaxError()
+
+ self.quoteChar = quoteChar
+ self.quoteCharLen = len(quoteChar)
+ self.firstQuoteChar = quoteChar[0]
+ self.endQuoteChar = endQuoteChar
+ self.endQuoteCharLen = len(endQuoteChar)
+ self.escChar = escChar
+ self.escQuote = escQuote
+ self.unquoteResults = unquoteResults
+
+ if multiline:
+ self.flags = re.MULTILINE | re.DOTALL
+ self.pattern = r'%s(?:[^%s%s]' % \
+ ( re.escape(self.quoteChar),
+ _escapeRegexRangeChars(self.endQuoteChar[0]),
+ (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+ else:
+ self.flags = 0
+ self.pattern = r'%s(?:[^%s\n\r%s]' % \
+ ( re.escape(self.quoteChar),
+ _escapeRegexRangeChars(self.endQuoteChar[0]),
+ (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+ if len(self.endQuoteChar) > 1:
+ self.pattern += (
+ '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
+ _escapeRegexRangeChars(self.endQuoteChar[i]))
+ for i in range(len(self.endQuoteChar)-1,0,-1)) + ')'
+ )
+ if escQuote:
+ self.pattern += (r'|(?:%s)' % re.escape(escQuote))
+ if escChar:
+ self.pattern += (r'|(?:%s.)' % re.escape(escChar))
+ self.escCharReplacePattern = re.escape(self.escChar)+"(.)"
+ self.pattern += (r')*%s' % re.escape(self.endQuoteChar))
+
+ try:
+ self.re = re.compile(self.pattern, self.flags)
+ self.reString = self.pattern
+ except sre_constants.error:
+ warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern,
+ SyntaxWarning, stacklevel=2)
+ raise
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ ret = result.group()
+
+ if self.unquoteResults:
+
+ # strip off quotes
+ ret = ret[self.quoteCharLen:-self.endQuoteCharLen]
+
+ if isinstance(ret,basestring):
+ # replace escaped characters
+ if self.escChar:
+ ret = re.sub(self.escCharReplacePattern,"\g<1>",ret)
+
+ # replace escaped quotes
+ if self.escQuote:
+ ret = ret.replace(self.escQuote, self.endQuoteChar)
+
+ return loc, ret
+
+ def __str__( self ):
+ try:
+ return super(QuotedString,self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar)
+
+ return self.strRepr
+
+
+class CharsNotIn(Token):
+ """Token for matching words composed of characters *not* in a given set.
+ Defined with string containing all disallowed characters, and an optional
+ minimum, maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction.
+ """
+ def __init__( self, notChars, min=1, max=0, exact=0 ):
+ super(CharsNotIn,self).__init__()
+ self.skipWhitespace = False
+ self.notChars = notChars
+
+ if min < 1:
+ raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted")
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = ( self.minLen == 0 )
+ self.mayIndexError = False
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if instring[loc] in self.notChars:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ start = loc
+ loc += 1
+ notchars = self.notChars
+ maxlen = min( start+self.maxLen, len(instring) )
+ while loc < maxlen and \
+ (instring[loc] not in notchars):
+ loc += 1
+
+ if loc - start < self.minLen:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+ def __str__( self ):
+ try:
+ return super(CharsNotIn, self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None:
+ if len(self.notChars) > 4:
+ self.strRepr = "!W:(%s...)" % self.notChars[:4]
+ else:
+ self.strRepr = "!W:(%s)" % self.notChars
+
+ return self.strRepr
+
+class White(Token):
+ """Special matching class for matching whitespace. Normally, whitespace is ignored
+ by pyparsing grammars. This class is included when some whitespace structures
+ are significant. Define with a string containing the whitespace characters to be
+ matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments,
+ as defined for the C{L{Word}} class."""
+ whiteStrs = {
+ " " : "<SPC>",
+ "\t": "<TAB>",
+ "\n": "<LF>",
+ "\r": "<CR>",
+ "\f": "<FF>",
+ }
+ def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
+ super(White,self).__init__()
+ self.matchWhite = ws
+ self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) )
+ #~ self.leaveWhitespace()
+ self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite))
+ self.mayReturnEmpty = True
+ self.errmsg = "Expected " + self.name
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if not(instring[ loc ] in self.matchWhite):
+ raise ParseException(instring, loc, self.errmsg, self)
+ start = loc
+ loc += 1
+ maxloc = start + self.maxLen
+ maxloc = min( maxloc, len(instring) )
+ while loc < maxloc and instring[loc] in self.matchWhite:
+ loc += 1
+
+ if loc - start < self.minLen:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+
+class _PositionToken(Token):
+ def __init__( self ):
+ super(_PositionToken,self).__init__()
+ self.name=self.__class__.__name__
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+
+class GoToColumn(_PositionToken):
+ """Token to advance to a specific column of input text; useful for tabular report scraping."""
+ def __init__( self, colno ):
+ super(GoToColumn,self).__init__()
+ self.col = colno
+
+ def preParse( self, instring, loc ):
+ if col(loc,instring) != self.col:
+ instrlen = len(instring)
+ if self.ignoreExprs:
+ loc = self._skipIgnorables( instring, loc )
+ while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col :
+ loc += 1
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ thiscol = col( loc, instring )
+ if thiscol > self.col:
+ raise ParseException( instring, loc, "Text not in expected column", self )
+ newloc = loc + self.col - thiscol
+ ret = instring[ loc: newloc ]
+ return newloc, ret
+
+class LineStart(_PositionToken):
+ """Matches if current position is at the beginning of a line within the parse string"""
+ def __init__( self ):
+ super(LineStart,self).__init__()
+ self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
+ self.errmsg = "Expected start of line"
+
+ def preParse( self, instring, loc ):
+ preloc = super(LineStart,self).preParse(instring,loc)
+ if instring[preloc] == "\n":
+ loc += 1
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if not( loc==0 or
+ (loc == self.preParse( instring, 0 )) or
+ (instring[loc-1] == "\n") ): #col(loc, instring) != 1:
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class LineEnd(_PositionToken):
+ """Matches if current position is at the end of a line within the parse string"""
+ def __init__( self ):
+ super(LineEnd,self).__init__()
+ self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
+ self.errmsg = "Expected end of line"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc<len(instring):
+ if instring[loc] == "\n":
+ return loc+1, "\n"
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+ elif loc == len(instring):
+ return loc+1, []
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class StringStart(_PositionToken):
+ """Matches if current position is at the beginning of the parse string"""
+ def __init__( self ):
+ super(StringStart,self).__init__()
+ self.errmsg = "Expected start of text"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc != 0:
+ # see if entire string up to here is just whitespace and ignoreables
+ if loc != self.preParse( instring, 0 ):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class StringEnd(_PositionToken):
+ """Matches if current position is at the end of the parse string"""
+ def __init__( self ):
+ super(StringEnd,self).__init__()
+ self.errmsg = "Expected end of text"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc < len(instring):
+ raise ParseException(instring, loc, self.errmsg, self)
+ elif loc == len(instring):
+ return loc+1, []
+ elif loc > len(instring):
+ return loc, []
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class WordStart(_PositionToken):
+ """Matches if the current position is at the beginning of a Word, and
+ is not preceded by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of
+ the string being parsed, or at the beginning of a line.
+ """
+ def __init__(self, wordChars = printables):
+ super(WordStart,self).__init__()
+ self.wordChars = set(wordChars)
+ self.errmsg = "Not at the start of a word"
+
+ def parseImpl(self, instring, loc, doActions=True ):
+ if loc != 0:
+ if (instring[loc-1] in self.wordChars or
+ instring[loc] not in self.wordChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class WordEnd(_PositionToken):
+ """Matches if the current position is at the end of a Word, and
+ is not followed by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of
+ the string being parsed, or at the end of a line.
+ """
+ def __init__(self, wordChars = printables):
+ super(WordEnd,self).__init__()
+ self.wordChars = set(wordChars)
+ self.skipWhitespace = False
+ self.errmsg = "Not at the end of a word"
+
+ def parseImpl(self, instring, loc, doActions=True ):
+ instrlen = len(instring)
+ if instrlen>0 and loc<instrlen:
+ if (instring[loc] in self.wordChars or
+ instring[loc-1] not in self.wordChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+
+class ParseExpression(ParserElement):
+ """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
+ def __init__( self, exprs, savelist = False ):
+ super(ParseExpression,self).__init__(savelist)
+ if isinstance( exprs, _generatorType ):
+ exprs = list(exprs)
+
+ if isinstance( exprs, basestring ):
+ self.exprs = [ Literal( exprs ) ]
+ elif isinstance( exprs, collections.Sequence ):
+ # if sequence of strings provided, wrap with Literal
+ if all(isinstance(expr, basestring) for expr in exprs):
+ exprs = map(Literal, exprs)
+ self.exprs = list(exprs)
+ else:
+ try:
+ self.exprs = list( exprs )
+ except TypeError:
+ self.exprs = [ exprs ]
+ self.callPreparse = False
+
+ def __getitem__( self, i ):
+ return self.exprs[i]
+
+ def append( self, other ):
+ self.exprs.append( other )
+ self.strRepr = None
+ return self
+
+ def leaveWhitespace( self ):
+ """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on
+ all contained expressions."""
+ self.skipWhitespace = False
+ self.exprs = [ e.copy() for e in self.exprs ]
+ for e in self.exprs:
+ e.leaveWhitespace()
+ return self
+
+ def ignore( self, other ):
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ super( ParseExpression, self).ignore( other )
+ for e in self.exprs:
+ e.ignore( self.ignoreExprs[-1] )
+ else:
+ super( ParseExpression, self).ignore( other )
+ for e in self.exprs:
+ e.ignore( self.ignoreExprs[-1] )
+ return self
+
+ def __str__( self ):
+ try:
+ return super(ParseExpression,self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.exprs) )
+ return self.strRepr
+
+ def streamline( self ):
+ super(ParseExpression,self).streamline()
+
+ for e in self.exprs:
+ e.streamline()
+
+ # collapse nested And's of the form And( And( And( a,b), c), d) to And( a,b,c,d )
+ # but only if there are no parse actions or resultsNames on the nested And's
+ # (likewise for Or's and MatchFirst's)
+ if ( len(self.exprs) == 2 ):
+ other = self.exprs[0]
+ if ( isinstance( other, self.__class__ ) and
+ not(other.parseAction) and
+ other.resultsName is None and
+ not other.debug ):
+ self.exprs = other.exprs[:] + [ self.exprs[1] ]
+ self.strRepr = None
+ self.mayReturnEmpty |= other.mayReturnEmpty
+ self.mayIndexError |= other.mayIndexError
+
+ other = self.exprs[-1]
+ if ( isinstance( other, self.__class__ ) and
+ not(other.parseAction) and
+ other.resultsName is None and
+ not other.debug ):
+ self.exprs = self.exprs[:-1] + other.exprs[:]
+ self.strRepr = None
+ self.mayReturnEmpty |= other.mayReturnEmpty
+ self.mayIndexError |= other.mayIndexError
+
+ self.errmsg = "Expected " + str(self)
+
+ return self
+
+ def setResultsName( self, name, listAllMatches=False ):
+ ret = super(ParseExpression,self).setResultsName(name,listAllMatches)
+ return ret
+
+ def validate( self, validateTrace=[] ):
+ tmp = validateTrace[:]+[self]
+ for e in self.exprs:
+ e.validate(tmp)
+ self.checkRecursion( [] )
+
+ def copy(self):
+ ret = super(ParseExpression,self).copy()
+ ret.exprs = [e.copy() for e in self.exprs]
+ return ret
+
+class And(ParseExpression):
+ """Requires all given C{ParseExpression}s to be found in the given order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'+'} operator.
+ """
+
+ class _ErrorStop(Empty):
+ def __init__(self, *args, **kwargs):
+ super(And._ErrorStop,self).__init__(*args, **kwargs)
+ self.name = '-'
+ self.leaveWhitespace()
+
+ def __init__( self, exprs, savelist = True ):
+ super(And,self).__init__(exprs, savelist)
+ self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+ self.setWhitespaceChars( self.exprs[0].whiteChars )
+ self.skipWhitespace = self.exprs[0].skipWhitespace
+ self.callPreparse = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ # pass False as last arg to _parse for first element, since we already
+ # pre-parsed the string as part of our And pre-parsing
+ loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False )
+ errorStop = False
+ for e in self.exprs[1:]:
+ if isinstance(e, And._ErrorStop):
+ errorStop = True
+ continue
+ if errorStop:
+ try:
+ loc, exprtokens = e._parse( instring, loc, doActions )
+ except ParseSyntaxException:
+ raise
+ except ParseBaseException as pe:
+ pe.__traceback__ = None
+ raise ParseSyntaxException(pe)
+ except IndexError:
+ raise ParseSyntaxException( ParseException(instring, len(instring), self.errmsg, self) )
+ else:
+ loc, exprtokens = e._parse( instring, loc, doActions )
+ if exprtokens or exprtokens.haskeys():
+ resultlist += exprtokens
+ return loc, resultlist
+
+ def __iadd__(self, other ):
+ if isinstance( other, basestring ):
+ other = Literal( other )
+ return self.append( other ) #And( [ self, other ] )
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+ if not e.mayReturnEmpty:
+ break
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+
+class Or(ParseExpression):
+ """Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the expression that matches the longest string will be used.
+ May be constructed using the C{'^'} operator.
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(Or,self).__init__(exprs, savelist)
+ if self.exprs:
+ self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+ else:
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ maxExcLoc = -1
+ maxException = None
+ matches = []
+ for e in self.exprs:
+ try:
+ loc2 = e.tryParse( instring, loc )
+ except ParseException as err:
+ err.__traceback__ = None
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+ except IndexError:
+ if len(instring) > maxExcLoc:
+ maxException = ParseException(instring,len(instring),e.errmsg,self)
+ maxExcLoc = len(instring)
+ else:
+ # save match among all matches, to retry longest to shortest
+ matches.append((loc2, e))
+
+ if matches:
+ matches.sort(key=lambda x: -x[0])
+ for _,e in matches:
+ try:
+ return e._parse( instring, loc, doActions )
+ except ParseException as err:
+ err.__traceback__ = None
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+
+ if maxException is not None:
+ maxException.msg = self.errmsg
+ raise maxException
+ else:
+ raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+
+ def __ixor__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ return self.append( other ) #Or( [ self, other ] )
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " ^ ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class MatchFirst(ParseExpression):
+ """Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the first one listed is the one that will match.
+ May be constructed using the C{'|'} operator.
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(MatchFirst,self).__init__(exprs, savelist)
+ if self.exprs:
+ self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+ else:
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ maxExcLoc = -1
+ maxException = None
+ for e in self.exprs:
+ try:
+ ret = e._parse( instring, loc, doActions )
+ return ret
+ except ParseException as err:
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+ except IndexError:
+ if len(instring) > maxExcLoc:
+ maxException = ParseException(instring,len(instring),e.errmsg,self)
+ maxExcLoc = len(instring)
+
+ # only got here if no expression matched, raise exception for match that made it the furthest
+ else:
+ if maxException is not None:
+ maxException.msg = self.errmsg
+ raise maxException
+ else:
+ raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+ def __ior__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ return self.append( other ) #MatchFirst( [ self, other ] )
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " | ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class Each(ParseExpression):
+ """Requires all given C{ParseExpression}s to be found, but in any order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'&'} operator.
+ """
+ def __init__( self, exprs, savelist = True ):
+ super(Each,self).__init__(exprs, savelist)
+ self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+ self.skipWhitespace = True
+ self.initExprGroups = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.initExprGroups:
+ self.opt1map = dict((id(e.expr),e) for e in self.exprs if isinstance(e,Optional))
+ opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ]
+ opt2 = [ e for e in self.exprs if e.mayReturnEmpty and not isinstance(e,Optional)]
+ self.optionals = opt1 + opt2
+ self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ]
+ self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ]
+ self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ]
+ self.required += self.multirequired
+ self.initExprGroups = False
+ tmpLoc = loc
+ tmpReqd = self.required[:]
+ tmpOpt = self.optionals[:]
+ matchOrder = []
+
+ keepMatching = True
+ while keepMatching:
+ tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired
+ failed = []
+ for e in tmpExprs:
+ try:
+ tmpLoc = e.tryParse( instring, tmpLoc )
+ except ParseException:
+ failed.append(e)
+ else:
+ matchOrder.append(self.opt1map.get(id(e),e))
+ if e in tmpReqd:
+ tmpReqd.remove(e)
+ elif e in tmpOpt:
+ tmpOpt.remove(e)
+ if len(failed) == len(tmpExprs):
+ keepMatching = False
+
+ if tmpReqd:
+ missing = ", ".join(_ustr(e) for e in tmpReqd)
+ raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )
+
+ # add any unmatched Optionals, in case they have default values defined
+ matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt]
+
+ resultlist = []
+ for e in matchOrder:
+ loc,results = e._parse(instring,loc,doActions)
+ resultlist.append(results)
+
+ finalResults = ParseResults([])
+ for r in resultlist:
+ dups = {}
+ for k in r.keys():
+ if k in finalResults:
+ tmp = ParseResults(finalResults[k])
+ tmp += ParseResults(r[k])
+ dups[k] = tmp
+ finalResults += ParseResults(r)
+ for k,v in dups.items():
+ finalResults[k] = v
+ return loc, finalResults
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " & ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class ParseElementEnhance(ParserElement):
+ """Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens."""
+ def __init__( self, expr, savelist=False ):
+ super(ParseElementEnhance,self).__init__(savelist)
+ if isinstance( expr, basestring ):
+ expr = Literal(expr)
+ self.expr = expr
+ self.strRepr = None
+ if expr is not None:
+ self.mayIndexError = expr.mayIndexError
+ self.mayReturnEmpty = expr.mayReturnEmpty
+ self.setWhitespaceChars( expr.whiteChars )
+ self.skipWhitespace = expr.skipWhitespace
+ self.saveAsList = expr.saveAsList
+ self.callPreparse = expr.callPreparse
+ self.ignoreExprs.extend(expr.ignoreExprs)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.expr is not None:
+ return self.expr._parse( instring, loc, doActions, callPreParse=False )
+ else:
+ raise ParseException("",loc,self.errmsg,self)
+
+ def leaveWhitespace( self ):
+ self.skipWhitespace = False
+ self.expr = self.expr.copy()
+ if self.expr is not None:
+ self.expr.leaveWhitespace()
+ return self
+
+ def ignore( self, other ):
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ super( ParseElementEnhance, self).ignore( other )
+ if self.expr is not None:
+ self.expr.ignore( self.ignoreExprs[-1] )
+ else:
+ super( ParseElementEnhance, self).ignore( other )
+ if self.expr is not None:
+ self.expr.ignore( self.ignoreExprs[-1] )
+ return self
+
+ def streamline( self ):
+ super(ParseElementEnhance,self).streamline()
+ if self.expr is not None:
+ self.expr.streamline()
+ return self
+
+ def checkRecursion( self, parseElementList ):
+ if self in parseElementList:
+ raise RecursiveGrammarException( parseElementList+[self] )
+ subRecCheckList = parseElementList[:] + [ self ]
+ if self.expr is not None:
+ self.expr.checkRecursion( subRecCheckList )
+
+ def validate( self, validateTrace=[] ):
+ tmp = validateTrace[:]+[self]
+ if self.expr is not None:
+ self.expr.validate(tmp)
+ self.checkRecursion( [] )
+
+ def __str__( self ):
+ try:
+ return super(ParseElementEnhance,self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None and self.expr is not None:
+ self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) )
+ return self.strRepr
+
+
+class FollowedBy(ParseElementEnhance):
+ """Lookahead matching of the given parse expression. C{FollowedBy}
+ does *not* advance the parsing position within the input string, it only
+ verifies that the specified parse expression matches at the current
+ position. C{FollowedBy} always returns a null token list."""
+ def __init__( self, expr ):
+ super(FollowedBy,self).__init__(expr)
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ self.expr.tryParse( instring, loc )
+ return loc, []
+
+
+class NotAny(ParseElementEnhance):
+ """Lookahead to disallow matching with the given parse expression. C{NotAny}
+ does *not* advance the parsing position within the input string, it only
+ verifies that the specified parse expression does *not* match at the current
+ position. Also, C{NotAny} does *not* skip over leading whitespace. C{NotAny}
+ always returns a null token list. May be constructed using the '~' operator."""
+ def __init__( self, expr ):
+ super(NotAny,self).__init__(expr)
+ #~ self.leaveWhitespace()
+ self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs
+ self.mayReturnEmpty = True
+ self.errmsg = "Found unwanted token, "+_ustr(self.expr)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ try:
+ self.expr.tryParse( instring, loc )
+ except (ParseException,IndexError):
+ pass
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "~{" + _ustr(self.expr) + "}"
+
+ return self.strRepr
+
+
+class ZeroOrMore(ParseElementEnhance):
+ """Optional repetition of zero or more of the given expression."""
+ def __init__( self, expr ):
+ super(ZeroOrMore,self).__init__(expr)
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ tokens = []
+ try:
+ loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+ hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
+ while 1:
+ if hasIgnoreExprs:
+ preloc = self._skipIgnorables( instring, loc )
+ else:
+ preloc = loc
+ loc, tmptokens = self.expr._parse( instring, preloc, doActions )
+ if tmptokens or tmptokens.haskeys():
+ tokens += tmptokens
+ except (ParseException,IndexError):
+ pass
+
+ return loc, tokens
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "[" + _ustr(self.expr) + "]..."
+
+ return self.strRepr
+
+ def setResultsName( self, name, listAllMatches=False ):
+ ret = super(ZeroOrMore,self).setResultsName(name,listAllMatches)
+ ret.saveAsList = True
+ return ret
+
+
+class OneOrMore(ParseElementEnhance):
+ """Repetition of one or more of the given expression."""
+ def parseImpl( self, instring, loc, doActions=True ):
+ # must be at least one
+ loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+ try:
+ hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
+ while 1:
+ if hasIgnoreExprs:
+ preloc = self._skipIgnorables( instring, loc )
+ else:
+ preloc = loc
+ loc, tmptokens = self.expr._parse( instring, preloc, doActions )
+ if tmptokens or tmptokens.haskeys():
+ tokens += tmptokens
+ except (ParseException,IndexError):
+ pass
+
+ return loc, tokens
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + _ustr(self.expr) + "}..."
+
+ return self.strRepr
+
+ def setResultsName( self, name, listAllMatches=False ):
+ ret = super(OneOrMore,self).setResultsName(name,listAllMatches)
+ ret.saveAsList = True
+ return ret
+
+class _NullToken(object):
+ def __bool__(self):
+ return False
+ __nonzero__ = __bool__
+ def __str__(self):
+ return ""
+
+_optionalNotMatched = _NullToken()
+class Optional(ParseElementEnhance):
+ """Optional matching of the given expression.
+ A default return string can also be specified, if the optional expression
+ is not found.
+ """
+ def __init__( self, expr, default=_optionalNotMatched ):
+ super(Optional,self).__init__( expr, savelist=False )
+ self.defaultValue = default
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ try:
+ loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+ except (ParseException,IndexError):
+ if self.defaultValue is not _optionalNotMatched:
+ if self.expr.resultsName:
+ tokens = ParseResults([ self.defaultValue ])
+ tokens[self.expr.resultsName] = self.defaultValue
+ else:
+ tokens = [ self.defaultValue ]
+ else:
+ tokens = []
+ return loc, tokens
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "[" + _ustr(self.expr) + "]"
+
+ return self.strRepr
+
+
+class SkipTo(ParseElementEnhance):
+ """Token for skipping over all undefined text until the matched expression is found.
+ If C{include} is set to true, the matched expression is also parsed (the skipped text
+ and matched expression are returned as a 2-element list). The C{ignore}
+ argument is used to define grammars (typically quoted strings and comments) that
+ might contain false matches.
+ """
+ def __init__( self, other, include=False, ignore=None, failOn=None ):
+ super( SkipTo, self ).__init__( other )
+ self.ignoreExpr = ignore
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+ self.includeMatch = include
+ self.asList = False
+ if failOn is not None and isinstance(failOn, basestring):
+ self.failOn = Literal(failOn)
+ else:
+ self.failOn = failOn
+ self.errmsg = "No match found for "+_ustr(self.expr)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ startLoc = loc
+ instrlen = len(instring)
+ expr = self.expr
+ failParse = False
+ while loc <= instrlen:
+ try:
+ if self.failOn:
+ try:
+ self.failOn.tryParse(instring, loc)
+ except ParseBaseException:
+ pass
+ else:
+ failParse = True
+ raise ParseException(instring, loc, "Found expression " + str(self.failOn))
+ failParse = False
+ if self.ignoreExpr is not None:
+ while 1:
+ try:
+ loc = self.ignoreExpr.tryParse(instring,loc)
+ # print("found ignoreExpr, advance to", loc)
+ except ParseBaseException:
+ break
+ expr._parse( instring, loc, doActions=False, callPreParse=False )
+ skipText = instring[startLoc:loc]
+ if self.includeMatch:
+ loc,mat = expr._parse(instring,loc,doActions,callPreParse=False)
+ if mat:
+ skipRes = ParseResults( skipText )
+ skipRes += mat
+ return loc, [ skipRes ]
+ else:
+ return loc, [ skipText ]
+ else:
+ return loc, [ skipText ]
+ except (ParseException,IndexError):
+ if failParse:
+ raise
+ else:
+ loc += 1
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class Forward(ParseElementEnhance):
+ """Forward declaration of an expression to be defined later -
+ used for recursive grammars, such as algebraic infix notation.
+ When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator.
+
+ Note: take care when assigning to C{Forward} not to overlook precedence of operators.
+ Specifically, '|' has a lower precedence than '<<', so that::
+ fwdExpr << a | b | c
+ will actually be evaluated as::
+ (fwdExpr << a) | b | c
+ thereby leaving b and c out as parseable alternatives. It is recommended that you
+ explicitly group the values inserted into the C{Forward}::
+ fwdExpr << (a | b | c)
+ Converting to use the '<<=' operator instead will avoid this problem.
+ """
+ def __init__( self, other=None ):
+ super(Forward,self).__init__( other, savelist=False )
+
+ def __lshift__( self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass(other)
+ self.expr = other
+ self.mayReturnEmpty = other.mayReturnEmpty
+ self.strRepr = None
+ self.mayIndexError = self.expr.mayIndexError
+ self.mayReturnEmpty = self.expr.mayReturnEmpty
+ self.setWhitespaceChars( self.expr.whiteChars )
+ self.skipWhitespace = self.expr.skipWhitespace
+ self.saveAsList = self.expr.saveAsList
+ self.ignoreExprs.extend(self.expr.ignoreExprs)
+ return self
+
+ def __ilshift__(self, other):
+ return self << other
+
+ def leaveWhitespace( self ):
+ self.skipWhitespace = False
+ return self
+
+ def streamline( self ):
+ if not self.streamlined:
+ self.streamlined = True
+ if self.expr is not None:
+ self.expr.streamline()
+ return self
+
+ def validate( self, validateTrace=[] ):
+ if self not in validateTrace:
+ tmp = validateTrace[:]+[self]
+ if self.expr is not None:
+ self.expr.validate(tmp)
+ self.checkRecursion([])
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ self._revertClass = self.__class__
+ self.__class__ = _ForwardNoRecurse
+ try:
+ if self.expr is not None:
+ retString = _ustr(self.expr)
+ else:
+ retString = "None"
+ finally:
+ self.__class__ = self._revertClass
+ return self.__class__.__name__ + ": " + retString
+
+ def copy(self):
+ if self.expr is not None:
+ return super(Forward,self).copy()
+ else:
+ ret = Forward()
+ ret <<= self
+ return ret
+
+class _ForwardNoRecurse(Forward):
+ def __str__( self ):
+ return "..."
+
+class TokenConverter(ParseElementEnhance):
+ """Abstract subclass of C{ParseExpression}, for converting parsed results."""
+ def __init__( self, expr, savelist=False ):
+ super(TokenConverter,self).__init__( expr )#, savelist )
+ self.saveAsList = False
+
+class Upcase(TokenConverter):
+ """Converter to upper case all matching tokens."""
+ def __init__(self, *args):
+ super(Upcase,self).__init__(*args)
+ warnings.warn("Upcase class is deprecated, use upcaseTokens parse action instead",
+ DeprecationWarning,stacklevel=2)
+
+ def postParse( self, instring, loc, tokenlist ):
+ return list(map( str.upper, tokenlist ))
+
+
+class Combine(TokenConverter):
+ """Converter to concatenate all matching tokens to a single string.
+ By default, the matching patterns must also be contiguous in the input string;
+ this can be disabled by specifying C{'adjacent=False'} in the constructor.
+ """
+ def __init__( self, expr, joinString="", adjacent=True ):
+ super(Combine,self).__init__( expr )
+ # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
+ if adjacent:
+ self.leaveWhitespace()
+ self.adjacent = adjacent
+ self.skipWhitespace = True
+ self.joinString = joinString
+ self.callPreparse = True
+
+ def ignore( self, other ):
+ if self.adjacent:
+ ParserElement.ignore(self, other)
+ else:
+ super( Combine, self).ignore( other )
+ return self
+
+ def postParse( self, instring, loc, tokenlist ):
+ retToks = tokenlist.copy()
+ del retToks[:]
+ retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)
+
+ if self.resultsName and retToks.haskeys():
+ return [ retToks ]
+ else:
+ return retToks
+
+class Group(TokenConverter):
+ """Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions."""
+ def __init__( self, expr ):
+ super(Group,self).__init__( expr )
+ self.saveAsList = True
+
+ def postParse( self, instring, loc, tokenlist ):
+ return [ tokenlist ]
+
+class Dict(TokenConverter):
+ """Converter to return a repetitive expression as a list, but also as a dictionary.
+ Each element can also be referenced using the first token in the expression as its key.
+ Useful for tabular report scraping when the first column can be used as a item key.
+ """
+ def __init__( self, expr ):
+ super(Dict,self).__init__( expr )
+ self.saveAsList = True
+
+ def postParse( self, instring, loc, tokenlist ):
+ for i,tok in enumerate(tokenlist):
+ if len(tok) == 0:
+ continue
+ ikey = tok[0]
+ if isinstance(ikey,int):
+ ikey = _ustr(tok[0]).strip()
+ if len(tok)==1:
+ tokenlist[ikey] = _ParseResultsWithOffset("",i)
+ elif len(tok)==2 and not isinstance(tok[1],ParseResults):
+ tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i)
+ else:
+ dictvalue = tok.copy() #ParseResults(i)
+ del dictvalue[0]
+ if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()):
+ tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
+ else:
+ tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)
+
+ if self.resultsName:
+ return [ tokenlist ]
+ else:
+ return tokenlist
+
+
+class Suppress(TokenConverter):
+ """Converter for ignoring the results of a parsed expression."""
+ def postParse( self, instring, loc, tokenlist ):
+ return []
+
+ def suppress( self ):
+ return self
+
+
+class OnlyOnce(object):
+ """Wrapper for parse actions, to ensure they are only called once."""
+ def __init__(self, methodCall):
+ self.callable = _trim_arity(methodCall)
+ self.called = False
+ def __call__(self,s,l,t):
+ if not self.called:
+ results = self.callable(s,l,t)
+ self.called = True
+ return results
+ raise ParseException(s,l,"")
+ def reset(self):
+ self.called = False
+
+def traceParseAction(f):
+ """Decorator for debugging parse actions."""
+ f = _trim_arity(f)
+ def z(*paArgs):
+ thisFunc = f.func_name
+ s,l,t = paArgs[-3:]
+ if len(paArgs)>3:
+ thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
+ sys.stderr.write( ">>entering %s(line: '%s', %d, %s)\n" % (thisFunc,line(l,s),l,t) )
+ try:
+ ret = f(*paArgs)
+ except Exception as exc:
+ sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) )
+ raise
+ sys.stderr.write( "<<leaving %s (ret: %s)\n" % (thisFunc,ret) )
+ return ret
+ try:
+ z.__name__ = f.__name__
+ except AttributeError:
+ pass
+ return z
+
+#
+# global helpers
+#
+def delimitedList( expr, delim=",", combine=False ):
+ """Helper to define a delimited list of expressions - the delimiter defaults to ','.
+ By default, the list elements and delimiters can have intervening whitespace, and
+ comments, but this can be overridden by passing C{combine=True} in the constructor.
+ If C{combine} is set to C{True}, the matching tokens are returned as a single token
+ string, with the delimiters included; otherwise, the matching tokens are returned
+ as a list of tokens, with the delimiters suppressed.
+ """
+ dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
+ if combine:
+ return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName)
+ else:
+ return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)
+
+def countedArray( expr, intExpr=None ):
+ """Helper to define a counted list of expressions.
+ This helper defines a pattern of the form::
+ integer expr expr expr...
+ where the leading integer tells how many expr expressions follow.
+ The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
+ """
+ arrayExpr = Forward()
+ def countFieldParseAction(s,l,t):
+ n = t[0]
+ arrayExpr << (n and Group(And([expr]*n)) or Group(empty))
+ return []
+ if intExpr is None:
+ intExpr = Word(nums).setParseAction(lambda t:int(t[0]))
+ else:
+ intExpr = intExpr.copy()
+ intExpr.setName("arrayLen")
+ intExpr.addParseAction(countFieldParseAction, callDuringTry=True)
+ return ( intExpr + arrayExpr )
+
+def _flatten(L):
+ ret = []
+ for i in L:
+ if isinstance(i,list):
+ ret.extend(_flatten(i))
+ else:
+ ret.append(i)
+ return ret
+
+def matchPreviousLiteral(expr):
+ """Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousLiteral(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches a
+ previous literal, will also match the leading C{"1:1"} in C{"1:10"}.
+ If this is not desired, use C{matchPreviousExpr}.
+ Do *not* use with packrat parsing enabled.
+ """
+ rep = Forward()
+ def copyTokenToRepeater(s,l,t):
+ if t:
+ if len(t) == 1:
+ rep << t[0]
+ else:
+ # flatten t tokens
+ tflat = _flatten(t.asList())
+ rep << And( [ Literal(tt) for tt in tflat ] )
+ else:
+ rep << Empty()
+ expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ return rep
+
+def matchPreviousExpr(expr):
+ """Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousExpr(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches by
+ expressions, will *not* match the leading C{"1:1"} in C{"1:10"};
+ the expressions are evaluated first, and then compared, so
+ C{"1"} is compared with C{"10"}.
+ Do *not* use with packrat parsing enabled.
+ """
+ rep = Forward()
+ e2 = expr.copy()
+ rep <<= e2
+ def copyTokenToRepeater(s,l,t):
+ matchTokens = _flatten(t.asList())
+ def mustMatchTheseTokens(s,l,t):
+ theseTokens = _flatten(t.asList())
+ if theseTokens != matchTokens:
+ raise ParseException("",0,"")
+ rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
+ expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ return rep
+
+def _escapeRegexRangeChars(s):
+ #~ escape these chars: ^-]
+ for c in r"\^-]":
+ s = s.replace(c,_bslash+c)
+ s = s.replace("\n",r"\n")
+ s = s.replace("\t",r"\t")
+ return _ustr(s)
+
+def oneOf( strs, caseless=False, useRegex=True ):
+ """Helper to quickly define a set of alternative Literals, and makes sure to do
+ longest-first testing when there is a conflict, regardless of the input order,
+ but returns a C{L{MatchFirst}} for best performance.
+
+ Parameters:
+ - strs - a string of space-delimited literals, or a list of string literals
+ - caseless - (default=False) - treat all literals as caseless
+ - useRegex - (default=True) - as an optimization, will generate a Regex
+ object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or
+ if creating a C{Regex} raises an exception)
+ """
+ if caseless:
+ isequal = ( lambda a,b: a.upper() == b.upper() )
+ masks = ( lambda a,b: b.upper().startswith(a.upper()) )
+ parseElementClass = CaselessLiteral
+ else:
+ isequal = ( lambda a,b: a == b )
+ masks = ( lambda a,b: b.startswith(a) )
+ parseElementClass = Literal
+
+ symbols = []
+ if isinstance(strs,basestring):
+ symbols = strs.split()
+ elif isinstance(strs, collections.Sequence):
+ symbols = list(strs[:])
+ elif isinstance(strs, _generatorType):
+ symbols = list(strs)
+ else:
+ warnings.warn("Invalid argument to oneOf, expected string or list",
+ SyntaxWarning, stacklevel=2)
+ if not symbols:
+ return NoMatch()
+
+ i = 0
+ while i < len(symbols)-1:
+ cur = symbols[i]
+ for j,other in enumerate(symbols[i+1:]):
+ if ( isequal(other, cur) ):
+ del symbols[i+j+1]
+ break
+ elif ( masks(cur, other) ):
+ del symbols[i+j+1]
+ symbols.insert(i,other)
+ cur = other
+ break
+ else:
+ i += 1
+
+ if not caseless and useRegex:
+ #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
+ try:
+ if len(symbols)==len("".join(symbols)):
+ return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) )
+ else:
+ return Regex( "|".join(re.escape(sym) for sym in symbols) )
+ except:
+ warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
+ SyntaxWarning, stacklevel=2)
+
+
+ # last resort, just use MatchFirst
+ return MatchFirst( [ parseElementClass(sym) for sym in symbols ] )
+
+def dictOf( key, value ):
+ """Helper to easily and clearly define a dictionary by specifying the respective patterns
+ for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens
+ in the proper order. The key pattern can include delimiting markers or punctuation,
+ as long as they are suppressed, thereby leaving the significant key text. The value
+ pattern can include named results, so that the C{Dict} results can include named token
+ fields.
+ """
+ return Dict( ZeroOrMore( Group ( key + value ) ) )
+
+def originalTextFor(expr, asString=True):
+ """Helper to return the original, untokenized text for a given expression. Useful to
+ restore the parsed fields of an HTML start tag into the raw tag text itself, or to
+ revert separate tokens with intervening whitespace back to the original matching
+ input text. Simpler to use than the parse action C{L{keepOriginalText}}, and does not
+ require the inspect module to chase up the call stack. By default, returns a
+ string containing the original parsed text.
+
+ If the optional C{asString} argument is passed as C{False}, then the return value is a
+ C{L{ParseResults}} containing any results names that were originally matched, and a
+ single token containing the original matched text from the input string. So if
+ the expression passed to C{L{originalTextFor}} contains expressions with defined
+ results names, you must set C{asString} to C{False} if you want to preserve those
+ results name values."""
+ locMarker = Empty().setParseAction(lambda s,loc,t: loc)
+ endlocMarker = locMarker.copy()
+ endlocMarker.callPreparse = False
+ matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end")
+ if asString:
+ extractText = lambda s,l,t: s[t._original_start:t._original_end]
+ else:
+ def extractText(s,l,t):
+ del t[:]
+ t.insert(0, s[t._original_start:t._original_end])
+ del t["_original_start"]
+ del t["_original_end"]
+ matchExpr.setParseAction(extractText)
+ return matchExpr
+
+def ungroup(expr):
+ """Helper to undo pyparsing's default grouping of And expressions, even
+ if all but one are non-empty."""
+ return TokenConverter(expr).setParseAction(lambda t:t[0])
+
+def locatedExpr(expr):
+ """Helper to decorate a returned token with its starting and ending locations in the input string.
+ This helper adds the following results names:
+ - locn_start = location where matched expression begins
+ - locn_end = location where matched expression ends
+ - value = the actual parsed results
+
+ Be careful if the input text contains C{<TAB>} characters, you may want to call
+ C{L{ParserElement.parseWithTabs}}
+ """
+ locator = Empty().setParseAction(lambda s,l,t: l)
+ return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end"))
+
+
+# convenience constants for positional expressions
+empty = Empty().setName("empty")
+lineStart = LineStart().setName("lineStart")
+lineEnd = LineEnd().setName("lineEnd")
+stringStart = StringStart().setName("stringStart")
+stringEnd = StringEnd().setName("stringEnd")
+
+_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
+_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16)))
+_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8)))
+_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1) | Regex(r"\w", re.UNICODE)
+_charRange = Group(_singleChar + Suppress("-") + _singleChar)
+_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
+
+def srange(s):
+ r"""Helper to easily define string ranges for use in Word construction. Borrows
+ syntax from regexp '[]' string range definitions::
+ srange("[0-9]") -> "0123456789"
+ srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz"
+ srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
+ The input string must be enclosed in []'s, and the returned string is the expanded
+ character set joined into a single string.
+ The values enclosed in the []'s may be::
+ a single character
+ an escaped character with a leading backslash (such as \- or \])
+ an escaped hex character with a leading '\x' (\x21, which is a '!' character)
+ (\0x## is also supported for backwards compatibility)
+ an escaped octal character with a leading '\0' (\041, which is a '!' character)
+ a range of any of the above, separated by a dash ('a-z', etc.)
+ any combination of the above ('aeiouy', 'a-zA-Z0-9_$', etc.)
+ """
+ _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1))
+ try:
+ return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body)
+ except:
+ return ""
+
+def matchOnlyAtCol(n):
+ """Helper method for defining parse actions that require matching at a specific
+ column in the input text.
+ """
+ def verifyCol(strg,locn,toks):
+ if col(locn,strg) != n:
+ raise ParseException(strg,locn,"matched token not at column %d" % n)
+ return verifyCol
+
+def replaceWith(replStr):
+ """Helper method for common parse actions that simply return a literal value. Especially
+ useful when used with C{L{transformString<ParserElement.transformString>}()}.
+ """
+ #def _replFunc(*args):
+ # return [replStr]
+ #return _replFunc
+ return functools.partial(next, itertools.repeat([replStr]))
+
+def removeQuotes(s,l,t):
+ """Helper parse action for removing quotation marks from parsed quoted strings.
+ To use, add this parse action to quoted string using::
+ quotedString.setParseAction( removeQuotes )
+ """
+ return t[0][1:-1]
+
+def upcaseTokens(s,l,t):
+ """Helper parse action to convert tokens to upper case."""
+ return [ tt.upper() for tt in map(_ustr,t) ]
+
+def downcaseTokens(s,l,t):
+ """Helper parse action to convert tokens to lower case."""
+ return [ tt.lower() for tt in map(_ustr,t) ]
+
+def keepOriginalText(s,startLoc,t):
+ """DEPRECATED - use new helper method C{L{originalTextFor}}.
+ Helper parse action to preserve original parsed text,
+ overriding any nested parse actions."""
+ try:
+ endloc = getTokensEndLoc()
+ except ParseException:
+ raise ParseFatalException("incorrect usage of keepOriginalText - may only be called as a parse action")
+ del t[:]
+ t += ParseResults(s[startLoc:endloc])
+ return t
+
+def getTokensEndLoc():
+ """Method to be called from within a parse action to determine the end
+ location of the parsed tokens."""
+ import inspect
+ fstack = inspect.stack()
+ try:
+ # search up the stack (through intervening argument normalizers) for correct calling routine
+ for f in fstack[2:]:
+ if f[3] == "_parseNoCache":
+ endloc = f[0].f_locals["loc"]
+ return endloc
+ else:
+ raise ParseFatalException("incorrect usage of getTokensEndLoc - may only be called from within a parse action")
+ finally:
+ del fstack
+
+def _makeTags(tagStr, xml):
+ """Internal helper to construct opening and closing tag expressions, given a tag name"""
+ if isinstance(tagStr,basestring):
+ resname = tagStr
+ tagStr = Keyword(tagStr, caseless=not xml)
+ else:
+ resname = tagStr.name
+
+ tagAttrName = Word(alphas,alphanums+"_-:")
+ if (xml):
+ tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes )
+ openTag = Suppress("<") + tagStr("tag") + \
+ Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
+ Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+ else:
+ printablesLessRAbrack = "".join(c for c in printables if c not in ">")
+ tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
+ openTag = Suppress("<") + tagStr("tag") + \
+ Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
+ Optional( Suppress("=") + tagAttrValue ) ))) + \
+ Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+ closeTag = Combine(_L("</") + tagStr + ">")
+
+ openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % tagStr)
+ closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % tagStr)
+ openTag.tag = resname
+ closeTag.tag = resname
+ return openTag, closeTag
+
+def makeHTMLTags(tagStr):
+ """Helper to construct opening and closing tag expressions for HTML, given a tag name"""
+ return _makeTags( tagStr, False )
+
+def makeXMLTags(tagStr):
+ """Helper to construct opening and closing tag expressions for XML, given a tag name"""
+ return _makeTags( tagStr, True )
+
+def withAttribute(*args,**attrDict):
+ """Helper to create a validating parse action to be used with start tags created
+ with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag
+ with a required attribute value, to avoid false matches on common tags such as
+ C{<TD>} or C{<DIV>}.
+
+ Call C{withAttribute} with a series of attribute names and values. Specify the list
+ of filter attributes names and values as:
+ - keyword arguments, as in C{(align="right")}, or
+ - as an explicit dict with C{**} operator, when an attribute name is also a Python
+ reserved word, as in C{**{"class":"Customer", "align":"right"}}
+ - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
+ For attribute names with a namespace prefix, you must use the second form. Attribute
+ names are matched insensitive to upper/lower case.
+
+ If just testing for C{class} (with or without a namespace), use C{L{withClass}}.
+
+ To verify that the attribute exists, but without specifying a value, pass
+ C{withAttribute.ANY_VALUE} as the value.
+ """
+ if args:
+ attrs = args[:]
+ else:
+ attrs = attrDict.items()
+ attrs = [(k,v) for k,v in attrs]
+ def pa(s,l,tokens):
+ for attrName,attrValue in attrs:
+ if attrName not in tokens:
+ raise ParseException(s,l,"no matching attribute " + attrName)
+ if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue:
+ raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" %
+ (attrName, tokens[attrName], attrValue))
+ return pa
+withAttribute.ANY_VALUE = object()
+
+def withClass(classname, namespace=''):
+ """Simplified version of C{L{withAttribute}} when matching on a div class - made
+ difficult because C{class} is a reserved word in Python.
+ """
+ classattr = "%s:class" % namespace if namespace else "class"
+ return withAttribute(**{classattr : classname})
+
+opAssoc = _Constants()
+opAssoc.LEFT = object()
+opAssoc.RIGHT = object()
+
+def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ):
+ """Helper method for constructing grammars of expressions made up of
+ operators working in a precedence hierarchy. Operators may be unary or
+ binary, left- or right-associative. Parse actions can also be attached
+ to operator expressions.
+
+ Parameters:
+ - baseExpr - expression representing the most basic element for the nested
+ - opList - list of tuples, one for each operator precedence level in the
+ expression grammar; each tuple is of the form
+ (opExpr, numTerms, rightLeftAssoc, parseAction), where:
+ - opExpr is the pyparsing expression for the operator;
+ may also be a string, which will be converted to a Literal;
+ if numTerms is 3, opExpr is a tuple of two expressions, for the
+ two operators separating the 3 terms
+ - numTerms is the number of terms for this operator (must
+ be 1, 2, or 3)
+ - rightLeftAssoc is the indicator whether the operator is
+ right or left associative, using the pyparsing-defined
+ constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
+ - parseAction is the parse action to be associated with
+ expressions matching this operator expression (the
+ parse action tuple member may be omitted)
+ - lpar - expression for matching left-parentheses (default=Suppress('('))
+ - rpar - expression for matching right-parentheses (default=Suppress(')'))
+ """
+ ret = Forward()
+ lastExpr = baseExpr | ( lpar + ret + rpar )
+ for i,operDef in enumerate(opList):
+ opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
+ if arity == 3:
+ if opExpr is None or len(opExpr) != 2:
+ raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions")
+ opExpr1, opExpr2 = opExpr
+ thisExpr = Forward()#.setName("expr%d" % i)
+ if rightLeftAssoc == opAssoc.LEFT:
+ if arity == 1:
+ matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) )
+ elif arity == 2:
+ if opExpr is not None:
+ matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) )
+ else:
+ matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) )
+ elif arity == 3:
+ matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \
+ Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr )
+ else:
+ raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+ elif rightLeftAssoc == opAssoc.RIGHT:
+ if arity == 1:
+ # try to avoid LR with this extra test
+ if not isinstance(opExpr, Optional):
+ opExpr = Optional(opExpr)
+ matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr )
+ elif arity == 2:
+ if opExpr is not None:
+ matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) )
+ else:
+ matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) )
+ elif arity == 3:
+ matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \
+ Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr )
+ else:
+ raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+ else:
+ raise ValueError("operator must indicate right or left associativity")
+ if pa:
+ matchExpr.setParseAction( pa )
+ thisExpr <<= ( matchExpr | lastExpr )
+ lastExpr = thisExpr
+ ret <<= lastExpr
+ return ret
+operatorPrecedence = infixNotation
+
+dblQuotedString = Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*"').setName("string enclosed in double quotes")
+sglQuotedString = Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*'").setName("string enclosed in single quotes")
+quotedString = Regex(r'''(?:"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*")|(?:'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*')''').setName("quotedString using single or double quotes")
+unicodeString = Combine(_L('u') + quotedString.copy())
+
+def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()):
+ """Helper method for defining nested lists enclosed in opening and closing
+ delimiters ("(" and ")" are the default).
+
+ Parameters:
+ - opener - opening character for a nested list (default="("); can also be a pyparsing expression
+ - closer - closing character for a nested list (default=")"); can also be a pyparsing expression
+ - content - expression for items within the nested lists (default=None)
+ - ignoreExpr - expression for ignoring opening and closing delimiters (default=quotedString)
+
+ If an expression is not provided for the content argument, the nested
+ expression will capture all whitespace-delimited content between delimiters
+ as a list of separate values.
+
+ Use the C{ignoreExpr} argument to define expressions that may contain
+ opening or closing characters that should not be treated as opening
+ or closing characters for nesting, such as quotedString or a comment
+ expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}.
+ The default is L{quotedString}, but if no expressions are to be ignored,
+ then pass C{None} for this argument.
+ """
+ if opener == closer:
+ raise ValueError("opening and closing strings cannot be the same")
+ if content is None:
+ if isinstance(opener,basestring) and isinstance(closer,basestring):
+ if len(opener) == 1 and len(closer)==1:
+ if ignoreExpr is not None:
+ content = (Combine(OneOrMore(~ignoreExpr +
+ CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ if ignoreExpr is not None:
+ content = (Combine(OneOrMore(~ignoreExpr +
+ ~Literal(opener) + ~Literal(closer) +
+ CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) +
+ CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ raise ValueError("opening and closing arguments must be strings if no content expression is given")
+ ret = Forward()
+ if ignoreExpr is not None:
+ ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
+ else:
+ ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) )
+ return ret
+
+def indentedBlock(blockStatementExpr, indentStack, indent=True):
+ """Helper method for defining space-delimited indentation blocks, such as
+ those used to define block statements in Python source code.
+
+ Parameters:
+ - blockStatementExpr - expression defining syntax of statement that
+ is repeated within the indented block
+ - indentStack - list created by caller to manage indentation stack
+ (multiple statementWithIndentedBlock expressions within a single grammar
+ should share a common indentStack)
+ - indent - boolean indicating whether block must be indented beyond the
+ the current level; set to False for block of left-most statements
+ (default=True)
+
+ A valid block must contain at least one C{blockStatement}.
+ """
+ def checkPeerIndent(s,l,t):
+ if l >= len(s): return
+ curCol = col(l,s)
+ if curCol != indentStack[-1]:
+ if curCol > indentStack[-1]:
+ raise ParseFatalException(s,l,"illegal nesting")
+ raise ParseException(s,l,"not a peer entry")
+
+ def checkSubIndent(s,l,t):
+ curCol = col(l,s)
+ if curCol > indentStack[-1]:
+ indentStack.append( curCol )
+ else:
+ raise ParseException(s,l,"not a subentry")
+
+ def checkUnindent(s,l,t):
+ if l >= len(s): return
+ curCol = col(l,s)
+ if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]):
+ raise ParseException(s,l,"not an unindent")
+ indentStack.pop()
+
+ NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress())
+ INDENT = Empty() + Empty().setParseAction(checkSubIndent)
+ PEER = Empty().setParseAction(checkPeerIndent)
+ UNDENT = Empty().setParseAction(checkUnindent)
+ if indent:
+ smExpr = Group( Optional(NL) +
+ #~ FollowedBy(blockStatementExpr) +
+ INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT)
+ else:
+ smExpr = Group( Optional(NL) +
+ (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )
+ blockStatementExpr.ignore(_bslash + LineEnd())
+ return smExpr
+
+alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
+punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
+
+anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:"))
+commonHTMLEntity = Combine(_L("&") + oneOf("gt lt amp nbsp quot").setResultsName("entity") +";").streamline()
+_htmlEntityMap = dict(zip("gt lt amp nbsp quot".split(),'><& "'))
+replaceHTMLEntity = lambda t : t.entity in _htmlEntityMap and _htmlEntityMap[t.entity] or None
+
+# it's easy to get these comment structures wrong - they're very common, so may as well make them available
+cStyleComment = Regex(r"/\*(?:[^*]*\*+)+?/").setName("C style comment")
+
+htmlComment = Regex(r"<!--[\s\S]*?-->")
+restOfLine = Regex(r".*").leaveWhitespace()
+dblSlashComment = Regex(r"\/\/(\\\n|.)*").setName("// comment")
+cppStyleComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))").setName("C++ style comment")
+
+javaStyleComment = cppStyleComment
+pythonStyleComment = Regex(r"#.*").setName("Python style comment")
+_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') +
+ Optional( Word(" \t") +
+ ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
+commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList")
+
+
+if __name__ == "__main__":
+
+ selectToken = CaselessLiteral( "select" )
+ fromToken = CaselessLiteral( "from" )
+
+ ident = Word( alphas, alphanums + "_$" )
+ columnName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
+ columnNameList = Group( delimitedList( columnName ) ).setName("columns")
+ tableName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
+ tableNameList = Group( delimitedList( tableName ) ).setName("tables")
+ simpleSQL = ( selectToken + \
+ ( '*' | columnNameList ).setResultsName( "columns" ) + \
+ fromToken + \
+ tableNameList.setResultsName( "tables" ) )
+
+ simpleSQL.runTests("""\
+ SELECT * from XYZZY, ABC
+ select * from SYS.XYZZY
+ Select A from Sys.dual
+ Select AA,BB,CC from Sys.dual
+ Select A, B, C from Sys.dual
+ Select A, B, C from Sys.dual
+ Xelect A, B, C from Sys.dual
+ Select A, B, C frox Sys.dual
+ Select
+ Select ^^^ frox Sys.dual
+ Select A, B, C from Sys.dual, Table2""")
+
diff --git a/pkg_resources/_vendor/six.py b/pkg_resources/_vendor/six.py new file mode 100644 index 00000000..190c0239 --- /dev/null +++ b/pkg_resources/_vendor/six.py @@ -0,0 +1,868 @@ +"""Utilities for writing code that runs on Python 2 and 3""" + +# Copyright (c) 2010-2015 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson <benjamin@python.org>" +__version__ = "1.10.0" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") + + +if PY3: + def b(s): + return s.encode("latin-1") + + def u(s): + return s + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" +else: + def b(s): + return s + # Workaround for standalone backslash + + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb=None): + raise tp, value, tb +""") + + +if sys.version_info[:2] == (3, 2): + exec_("""def raise_from(value, from_value): + if from_value is None: + raise value + raise value from from_value +""") +elif sys.version_info[:2] > (3, 2): + exec_("""def raise_from(value, from_value): + raise value from from_value +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + +_add_doc(reraise, """Reraise an exception.""") + +if sys.version_info[0:2] < (3, 4): + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + def wrapper(f): + f = functools.wraps(wrapped, assigned, updated)(f) + f.__wrapped__ = wrapped + return f + return wrapper +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) diff --git a/pkg_resources/_vendor/vendored.txt b/pkg_resources/_vendor/vendored.txt index 4cf7664e..1d03759f 100644 --- a/pkg_resources/_vendor/vendored.txt +++ b/pkg_resources/_vendor/vendored.txt @@ -1 +1,3 @@ -packaging==15.3 +packaging==16.6 +pyparsing==2.0.6 +six==1.10.0 diff --git a/pkg_resources/api_tests.txt b/pkg_resources/api_tests.txt index d28db0f5..4fbd3d23 100644 --- a/pkg_resources/api_tests.txt +++ b/pkg_resources/api_tests.txt @@ -338,88 +338,64 @@ Environment Markers >>> import os >>> print(im("sys_platform")) - Comparison or logical expression expected + Invalid marker: 'sys_platform', parse error at '' >>> print(im("sys_platform==")) - invalid syntax + Invalid marker: 'sys_platform==', parse error at '' >>> print(im("sys_platform=='win32'")) False >>> print(im("sys=='x'")) - Unknown name 'sys' + Invalid marker: "sys=='x'", parse error at "sys=='x'" >>> print(im("(extra)")) - Comparison or logical expression expected + Invalid marker: '(extra)', parse error at ')' >>> print(im("(extra")) - invalid syntax + Invalid marker: '(extra', parse error at '' >>> print(im("os.open('foo')=='y'")) - Language feature not supported in environment markers + Invalid marker: "os.open('foo')=='y'", parse error at 'os.open(' >>> print(im("'x'=='y' and os.open('foo')=='y'")) # no short-circuit! - Language feature not supported in environment markers + Invalid marker: "'x'=='y' and os.open('foo')=='y'", parse error at 'and os.o' >>> print(im("'x'=='x' or os.open('foo')=='y'")) # no short-circuit! - Language feature not supported in environment markers + Invalid marker: "'x'=='x' or os.open('foo')=='y'", parse error at 'or os.op' >>> print(im("'x' < 'y' < 'z'")) - Chained comparison not allowed in environment markers + Invalid marker: "'x' < 'y' < 'z'", parse error at "< 'z'" >>> print(im("r'x'=='x'")) - Only plain strings allowed in environment markers + Invalid marker: "r'x'=='x'", parse error at "r'x'=='x" >>> print(im("'''x'''=='x'")) - Only plain strings allowed in environment markers + Invalid marker: "'''x'''=='x'", parse error at "'x'''=='" >>> print(im('"""x"""=="x"')) - Only plain strings allowed in environment markers + Invalid marker: '"""x"""=="x"', parse error at '"x"""=="' - >>> print(im(r"'x\n'=='x'")) - Only plain strings allowed in environment markers + >>> print(im(r"x\n=='x'")) + Invalid marker: "x\\n=='x'", parse error at "x\\n=='x'" >>> print(im("os.open=='y'")) - Language feature not supported in environment markers - - >>> em('"x"=="x"') - True - - >>> em('"x"=="y"') - False - - >>> em('"x"=="y" and "x"=="x"') - False - - >>> em('"x"=="y" or "x"=="x"') - True - - >>> em('"x"=="y" and "x"=="q" or "z"=="z"') - True - - >>> em('"x"=="y" and ("x"=="q" or "z"=="z")') - False - - >>> em('"x"=="y" and "z"=="z" or "x"=="q"') - False - - >>> em('"x"=="x" and "z"=="z" or "x"=="q"') - True + Invalid marker: "os.open=='y'", parse error at 'os.open=' >>> em("sys_platform=='win32'") == (sys.platform=='win32') True - >>> em("'x' in 'yx'") - True - - >>> em("'yx' in 'x'") - False - >>> em("python_version >= '2.6'") True >>> em("python_version > '2.5'") True + >>> im("implementation_name=='cpython'") + False + >>> im("platform_python_implementation=='CPython'") False + + >>> im("implementation_version=='3.5.1'") + False diff --git a/pkg_resources/extern/__init__.py b/pkg_resources/extern/__init__.py new file mode 100644 index 00000000..6758d36f --- /dev/null +++ b/pkg_resources/extern/__init__.py @@ -0,0 +1,71 @@ +import sys + + +class VendorImporter: + """ + A PEP 302 meta path importer for finding optionally-vendored + or otherwise naturally-installed packages from root_name. + """ + def __init__(self, root_name, vendored_names=(), vendor_pkg=None): + self.root_name = root_name + self.vendored_names = set(vendored_names) + self.vendor_pkg = vendor_pkg or root_name.replace('extern', '_vendor') + + @property + def search_path(self): + """ + Search first the vendor package then as a natural package. + """ + yield self.vendor_pkg + '.' + yield '' + + def find_module(self, fullname, path=None): + """ + Return self when fullname starts with root_name and the + target module is one vendored through this importer. + """ + root, base, target = fullname.partition(self.root_name + '.') + if root: + return + if not any(map(target.startswith, self.vendored_names)): + return + return self + + def load_module(self, fullname): + """ + Iterate over the search path to locate and load fullname. + """ + root, base, target = fullname.partition(self.root_name + '.') + for prefix in self.search_path: + try: + extant = prefix + target + __import__(extant) + mod = sys.modules[extant] + sys.modules[fullname] = mod + # mysterious hack: + # Remove the reference to the extant package/module + # on later Python versions to cause relative imports + # in the vendor package to resolve the same modules + # as those going through this importer. + if sys.version_info > (3, 3): + del sys.modules[extant] + return mod + except ImportError: + pass + else: + raise ImportError( + "The '{target}' package is required; " + "normally this is bundled with this package so if you get " + "this warning, consult the packager of your " + "distribution.".format(**locals()) + ) + + def install(self): + """ + Install this importer into sys.meta_path if not already present. + """ + if self not in sys.meta_path: + sys.meta_path.append(self) + +names = 'packaging', 'pyparsing', 'six' +VendorImporter(__name__, names).install() diff --git a/pkg_resources/tests/test_markers.py b/pkg_resources/tests/test_markers.py index d8844e74..8d451de3 100644 --- a/pkg_resources/tests/test_markers.py +++ b/pkg_resources/tests/test_markers.py @@ -1,16 +1,10 @@ try: - import unittest.mock as mock + import unittest.mock as mock except ImportError: - import mock + import mock from pkg_resources import evaluate_marker - -@mock.patch.dict('pkg_resources.MarkerEvaluation.values', - python_full_version=mock.Mock(return_value='2.7.10')) -def test_lexicographic_ordering(): - """ - Although one might like 2.7.10 to be greater than 2.7.3, - the marker spec only supports lexicographic ordering. - """ - assert evaluate_marker("python_full_version > '2.7.3'") is False +@mock.patch('platform.python_version', return_value='2.7.10') +def test_ordering(python_version_mock): + assert evaluate_marker("python_full_version > '2.7.3'") is True diff --git a/pkg_resources/tests/test_pkg_resources.py b/pkg_resources/tests/test_pkg_resources.py index 31eee635..8b276ffc 100644 --- a/pkg_resources/tests/test_pkg_resources.py +++ b/pkg_resources/tests/test_pkg_resources.py @@ -12,6 +12,8 @@ import stat import distutils.dist import distutils.command.install_egg_info +from pkg_resources.extern.six.moves import map + import pytest import pkg_resources diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index a55478a2..98b8dcd7 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -1,29 +1,19 @@ +from __future__ import unicode_literals + import os import sys -import tempfile -import shutil import string +from pkg_resources.extern.six.moves import map + import pytest +from pkg_resources.extern import packaging import pkg_resources from pkg_resources import (parse_requirements, VersionConflict, parse_version, Distribution, EntryPoint, Requirement, safe_version, safe_name, WorkingSet) -packaging = pkg_resources.packaging - - -def safe_repr(obj, short=False): - """ copied from Python2.7""" - try: - result = repr(obj) - except Exception: - result = object.__repr__(obj) - if not short or len(result) < pkg_resources._MAX_LENGTH: - return result - return result[:pkg_resources._MAX_LENGTH] + ' [truncated]...' - class Metadata(pkg_resources.EmptyProvider): """Mock object to return metadata as if from an on-disk distribution""" @@ -159,7 +149,7 @@ class TestDistro: for i in range(3): targets = list(ws.resolve(parse_requirements("Foo"), ad)) assert targets == [Foo] - list(map(ws.add,targets)) + list(map(ws.add, targets)) with pytest.raises(VersionConflict): ws.resolve(parse_requirements("Foo==0.9"), ad) ws = WorkingSet([]) # reset @@ -181,6 +171,16 @@ class TestDistro: msg = 'Foo 0.9 is installed but Foo==1.2 is required' assert vc.value.report() == msg + @pytest.mark.xfail(reason="Functionality disabled; see #523") + def test_environment_markers(self): + """ + Environment markers are evaluated at resolution time. + """ + ad = pkg_resources.Environment([]) + ws = WorkingSet([]) + res = ws.resolve(parse_requirements("Foo;python_version<'2'"), ad) + assert list(res) == [] + def testDistroDependsOptions(self): d = self.distRequires(""" Twisted>=1.5 @@ -245,9 +245,8 @@ class TestWorkingSet: with pytest.raises(VersionConflict) as vc: ws.resolve(parse_requirements("Foo\nBar\n")) - msg = "Baz 1.0 is installed but Baz==2.0 is required by {'Bar'}" - if pkg_resources.PY2: - msg = msg.replace("{'Bar'}", "set(['Bar'])") + msg = "Baz 1.0 is installed but Baz==2.0 is required by " + msg += repr(set(['Bar'])) assert vc.value.report() == msg @@ -313,7 +312,7 @@ class TestEntryPoints: def checkSubMap(self, m): assert len(m) == len(self.submap_expect) - for key, ep in pkg_resources.iteritems(self.submap_expect): + for key, ep in self.submap_expect.items(): assert repr(m.get(key)) == repr(ep) submap_expect = dict( @@ -353,22 +352,22 @@ class TestRequirements: r = Requirement.parse("Twisted>=1.2") assert str(r) == "Twisted>=1.2" assert repr(r) == "Requirement.parse('Twisted>=1.2')" - assert r == Requirement("Twisted", [('>=','1.2')], ()) - assert r == Requirement("twisTed", [('>=','1.2')], ()) - assert r != Requirement("Twisted", [('>=','2.0')], ()) - assert r != Requirement("Zope", [('>=','1.2')], ()) - assert r != Requirement("Zope", [('>=','3.0')], ()) - assert r != Requirement.parse("Twisted[extras]>=1.2") + assert r == Requirement("Twisted>=1.2") + assert r == Requirement("twisTed>=1.2") + assert r != Requirement("Twisted>=2.0") + assert r != Requirement("Zope>=1.2") + assert r != Requirement("Zope>=3.0") + assert r != Requirement("Twisted[extras]>=1.2") def testOrdering(self): - r1 = Requirement("Twisted", [('==','1.2c1'),('>=','1.2')], ()) - r2 = Requirement("Twisted", [('>=','1.2'),('==','1.2c1')], ()) + r1 = Requirement("Twisted==1.2c1,>=1.2") + r2 = Requirement("Twisted>=1.2,==1.2c1") assert r1 == r2 assert str(r1) == str(r2) assert str(r2) == "Twisted==1.2c1,>=1.2" def testBasicContains(self): - r = Requirement("Twisted", [('>=','1.2')], ()) + r = Requirement("Twisted>=1.2") foo_dist = Distribution.from_filename("FooPkg-1.3_1.egg") twist11 = Distribution.from_filename("Twisted-1.1.egg") twist12 = Distribution.from_filename("Twisted-1.2.egg") @@ -384,8 +383,8 @@ class TestRequirements: r1 = Requirement.parse("Twisted[foo,bar]>=1.2") r2 = Requirement.parse("Twisted[bar,FOO]>=1.2") assert r1 == r2 - assert r1.extras == ("foo","bar") - assert r2.extras == ("bar","foo") # extras are normalized + assert set(r1.extras) == set(("foo", "bar")) + assert set(r2.extras) == set(("foo", "bar")) assert hash(r1) == hash(r2) assert ( hash(r1) @@ -394,6 +393,7 @@ class TestRequirements: "twisted", packaging.specifiers.SpecifierSet(">=1.2"), frozenset(["foo","bar"]), + None )) ) @@ -485,17 +485,17 @@ class TestParsing: assert ( list(parse_requirements('Twis-Ted>=1.2-1')) == - [Requirement('Twis-Ted',[('>=','1.2-1')], ())] + [Requirement('Twis-Ted>=1.2-1')] ) assert ( list(parse_requirements('Twisted >=1.2, \ # more\n<2.0')) == - [Requirement('Twisted',[('>=','1.2'),('<','2.0')], ())] + [Requirement('Twisted>=1.2,<2.0')] ) assert ( Requirement.parse("FooBar==1.99a3") == - Requirement("FooBar", [('==','1.99a3')], ()) + Requirement("FooBar==1.99a3") ) with pytest.raises(ValueError): Requirement.parse(">=2.3") @@ -508,6 +508,35 @@ class TestParsing: with pytest.raises(ValueError): Requirement.parse("#") + def test_requirements_with_markers(self): + assert ( + Requirement.parse("foobar;os_name=='a'") + == + Requirement.parse("foobar;os_name=='a'") + ) + assert ( + Requirement.parse("name==1.1;python_version=='2.7'") + != + Requirement.parse("name==1.1;python_version=='3.3'") + ) + assert ( + Requirement.parse("name==1.0;python_version=='2.7'") + != + Requirement.parse("name==1.2;python_version=='2.7'") + ) + assert ( + Requirement.parse("name[foo]==1.0;python_version=='3.3'") + != + Requirement.parse("name[foo,bar]==1.0;python_version=='3.3'") + ) + + def test_local_version(self): + req, = parse_requirements('foo==1.0.org1') + + def test_spaces_between_multiple_versions(self): + req, = parse_requirements('foo>=1.0, <3') + req, = parse_requirements('foo >= 1.0, < 3') + def testVersionEquality(self): def c(s1,s2): p1, p2 = parse_version(s1),parse_version(s2) @@ -610,21 +639,44 @@ class TestParsing: class TestNamespaces: - def setup_method(self, method): - self._ns_pkgs = pkg_resources._namespace_packages.copy() - self._tmpdir = tempfile.mkdtemp(prefix="tests-setuptools-") - os.makedirs(os.path.join(self._tmpdir, "site-pkgs")) - self._prev_sys_path = sys.path[:] - sys.path.append(os.path.join(self._tmpdir, "site-pkgs")) - - def teardown_method(self, method): - shutil.rmtree(self._tmpdir) - pkg_resources._namespace_packages = self._ns_pkgs.copy() - sys.path = self._prev_sys_path[:] - - @pytest.mark.skipif(os.path.islink(tempfile.gettempdir()), - reason="Test fails when /tmp is a symlink. See #231") - def test_two_levels_deep(self): + ns_str = "__import__('pkg_resources').declare_namespace(__name__)\n" + + @pytest.yield_fixture + def symlinked_tmpdir(self, tmpdir): + """ + Where available, return the tempdir as a symlink, + which as revealed in #231 is more fragile than + a natural tempdir. + """ + if not hasattr(os, 'symlink'): + yield str(tmpdir) + return + + link_name = str(tmpdir) + '-linked' + os.symlink(str(tmpdir), link_name) + try: + yield type(tmpdir)(link_name) + finally: + os.unlink(link_name) + + @pytest.yield_fixture(autouse=True) + def patched_path(self, tmpdir): + """ + Patch sys.path to include the 'site-pkgs' dir. Also + restore pkg_resources._namespace_packages to its + former state. + """ + saved_ns_pkgs = pkg_resources._namespace_packages.copy() + saved_sys_path = sys.path[:] + site_pkgs = tmpdir.mkdir('site-pkgs') + sys.path.append(str(site_pkgs)) + try: + yield + finally: + pkg_resources._namespace_packages = saved_ns_pkgs + sys.path = saved_sys_path + + def test_two_levels_deep(self, symlinked_tmpdir): """ Test nested namespace packages Create namespace packages in the following tree : @@ -633,19 +685,16 @@ class TestNamespaces: Check both are in the _namespace_packages dict and that their __path__ is correct """ - sys.path.append(os.path.join(self._tmpdir, "site-pkgs2")) - os.makedirs(os.path.join(self._tmpdir, "site-pkgs", "pkg1", "pkg2")) - os.makedirs(os.path.join(self._tmpdir, "site-pkgs2", "pkg1", "pkg2")) - ns_str = "__import__('pkg_resources').declare_namespace(__name__)\n" - for site in ["site-pkgs", "site-pkgs2"]: - pkg1_init = open(os.path.join(self._tmpdir, site, - "pkg1", "__init__.py"), "w") - pkg1_init.write(ns_str) - pkg1_init.close() - pkg2_init = open(os.path.join(self._tmpdir, site, - "pkg1", "pkg2", "__init__.py"), "w") - pkg2_init.write(ns_str) - pkg2_init.close() + real_tmpdir = symlinked_tmpdir.realpath() + tmpdir = symlinked_tmpdir + sys.path.append(str(tmpdir / 'site-pkgs2')) + site_dirs = tmpdir / 'site-pkgs', tmpdir / 'site-pkgs2' + for site in site_dirs: + pkg1 = site / 'pkg1' + pkg2 = pkg1 / 'pkg2' + pkg2.ensure_dir() + (pkg1 / '__init__.py').write_text(self.ns_str, encoding='utf-8') + (pkg2 / '__init__.py').write_text(self.ns_str, encoding='utf-8') import pkg1 assert "pkg1" in pkg_resources._namespace_packages # attempt to import pkg2 from site-pkgs2 @@ -655,7 +704,44 @@ class TestNamespaces: assert pkg_resources._namespace_packages["pkg1"] == ["pkg1.pkg2"] # check the __path__ attribute contains both paths expected = [ - os.path.join(self._tmpdir, "site-pkgs", "pkg1", "pkg2"), - os.path.join(self._tmpdir, "site-pkgs2", "pkg1", "pkg2"), + str(real_tmpdir / "site-pkgs" / "pkg1" / "pkg2"), + str(real_tmpdir / "site-pkgs2" / "pkg1" / "pkg2"), ] assert pkg1.pkg2.__path__ == expected + + def test_path_order(self, symlinked_tmpdir): + """ + Test that if multiple versions of the same namespace package subpackage + are on different sys.path entries, that only the one earliest on + sys.path is imported, and that the namespace package's __path__ is in + the correct order. + + Regression test for https://github.com/pypa/setuptools/issues/207 + """ + + tmpdir = symlinked_tmpdir + site_dirs = ( + tmpdir / "site-pkgs", + tmpdir / "site-pkgs2", + tmpdir / "site-pkgs3", + ) + + vers_str = "__version__ = %r" + + for number, site in enumerate(site_dirs, 1): + if number > 1: + sys.path.append(str(site)) + nspkg = site / 'nspkg' + subpkg = nspkg / 'subpkg' + subpkg.ensure_dir() + (nspkg / '__init__.py').write_text(self.ns_str, encoding='utf-8') + (subpkg / '__init__.py').write_text(vers_str % number, encoding='utf-8') + + import nspkg.subpkg + import nspkg + expected = [ + str(site.realpath() / 'nspkg') + for site in site_dirs + ] + assert nspkg.__path__ == expected + assert nspkg.subpkg.__version__ == 1 @@ -1,3 +1,3 @@ [pytest] -addopts=--doctest-modules --ignore release.py --ignore setuptools/lib2to3_ex.py --ignore tests/manual_test.py --ignore tests/shlib_test --doctest-glob=pkg_resources/api_tests.txt --ignore scripts/upload-old-releases-as-zip.py -norecursedirs=dist build *.egg +addopts=--doctest-modules --ignore release.py --ignore setuptools/lib2to3_ex.py --ignore tests/manual_test.py --ignore tests/shlib_test --doctest-glob=pkg_resources/api_tests.txt --ignore scripts/upload-old-releases-as-zip.py --ignore pavement.py +norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern diff --git a/scripts/upload-old-releases-as-zip.py b/scripts/upload-old-releases-as-zip.py deleted file mode 100644 index 38cfcd55..00000000 --- a/scripts/upload-old-releases-as-zip.py +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# declare and require dependencies -__requires__ = [ - 'twine', -]; __import__('pkg_resources') - -import errno -import glob -import hashlib -import json -import os -import shutil -import tarfile -import codecs -import urllib.request -import urllib.parse -import urllib.error -from distutils.version import LooseVersion - -from twine.commands import upload - - -OK = '\033[92m' -FAIL = '\033[91m' -END = '\033[0m' -DISTRIBUTION = "setuptools" - - -class SetuptoolsOldReleasesWithoutZip: - """docstring for SetuptoolsOldReleases""" - - def __init__(self): - self.dirpath = './dist' - os.makedirs(self.dirpath, exist_ok=True) - print("Downloading %s releases..." % DISTRIBUTION) - print("All releases will be downloaded to %s" % self.dirpath) - self.data_json_setuptools = self.get_json_data(DISTRIBUTION) - self.valid_releases_numbers = sorted([ - release - for release in self.data_json_setuptools['releases'] - # This condition is motivated by 13.0 release, which - # comes as "13.0": [], in the json - if self.data_json_setuptools['releases'][release] - ], key=LooseVersion) - self.total_downloaded_ok = 0 - - def get_json_data(self, package_name): - """ - "releases": { - "0.7.2": [ - { - "has_sig": false, - "upload_time": "2013-06-09T16:10:00", - "comment_text": "", - "python_version": "source", - "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.2.tar.gz", # NOQA - "md5_digest": "de44cd90f8a1c713d6c2bff67bbca65d", - "downloads": 159014, - "filename": "setuptools-0.7.2.tar.gz", - "packagetype": "sdist", - "size": 633077 - } - ], - "0.7.3": [ - { - "has_sig": false, - "upload_time": "2013-06-18T21:08:56", - "comment_text": "", - "python_version": "source", - "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.3.tar.gz", # NOQA - "md5_digest": "c854adacbf9067d330a847f06f7a8eba", - "downloads": 30594, - "filename": "setuptools-0.7.3.tar.gz", - "packagetype": "sdist", - "size": 751152 - } - ], - "12.3": [ - { - "has_sig": false, - "upload_time": "2015-02-26T19:15:51", - "comment_text": "", - "python_version": "3.4", - "url": "https://pypi.python.org/packages/3.4/s/setuptools/setuptools-12.3-py2.py3-none-any.whl", # NOQA - "md5_digest": "31f51a38497a70efadf5ce8d4c2211ab", - "downloads": 288451, - "filename": "setuptools-12.3-py2.py3-none-any.whl", - "packagetype": "bdist_wheel", - "size": 501904 - }, - { - "has_sig": false, - "upload_time": "2015-02-26T19:15:43", - "comment_text": "", - "python_version": "source", - "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.tar.gz", # NOQA - "md5_digest": "67614b6d560fa4f240e99cd553ec7f32", - "downloads": 110109, - "filename": "setuptools-12.3.tar.gz", - "packagetype": "sdist", - "size": 635025 - }, - { - "has_sig": false, - "upload_time": "2015-02-26T19:15:47", - "comment_text": "", - "python_version": "source", - "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.zip", # NOQA - "md5_digest": "abc799e7db6e7281535bf342bfc41a12", - "downloads": 67539, - "filename": "setuptools-12.3.zip", - "packagetype": "sdist", - "size": 678783 - } - ], - """ - url = "https://pypi.python.org/pypi/%s/json" % (package_name,) - resp = urllib.request.urlopen(urllib.request.Request(url)) - charset = resp.info().get_content_charset() - reader = codecs.getreader(charset)(resp) - data = json.load(reader) - - # Mainly for debug. - json_filename = "%s/%s.json" % (self.dirpath, DISTRIBUTION) - with open(json_filename, 'w') as outfile: - json.dump( - data, - outfile, - sort_keys=True, - indent=4, - separators=(',', ': '), - ) - - return data - - def get_setuptools_releases_without_zip_counterpart(self): - # Get set(all_valid_releases) - set(releases_with_zip), so now we have - # the releases without zip. - return set(self.valid_releases_numbers) - set([ - release - for release in self.valid_releases_numbers - for same_version_release_dict in self.data_json_setuptools['releases'][release] # NOQA - if 'zip' in same_version_release_dict['filename'] - ]) - - def download_setuptools_releases_without_zip_counterpart(self): - try: - releases_without_zip = self.get_setuptools_releases_without_zip_counterpart() # NOQA - failed_md5_releases = [] - # This is a "strange" loop, going through all releases and - # testing only the release I need to download, but I thought it - # would be mouch more readable than trying to iterate through - # releases I need and get into traverse hell values inside dicts - # inside dicts of the json to get the distribution's url to - # download. - for release in self.valid_releases_numbers: - if release in releases_without_zip: - for same_version_release_dict in self.data_json_setuptools['releases'][release]: # NOQA - if 'tar.gz' in same_version_release_dict['filename']: - print("Downloading %s..." % release) - local_file = '%s/%s' % ( - self.dirpath, - same_version_release_dict["filename"] - ) - urllib.request.urlretrieve( - same_version_release_dict["url"], - local_file - ) - targz = open(local_file, 'rb').read() - hexdigest = hashlib.md5(targz).hexdigest() - if (hexdigest != same_version_release_dict['md5_digest']): # NOQA - print(FAIL + "FAIL: md5 for %s didn't match!" % release + END) # NOQA - failed_md5_releases.append(release) - else: - self.total_downloaded_ok += 1 - print('Total releases without zip: %s' % len(releases_without_zip)) - print('Total downloaded: %s' % self.total_downloaded_ok) - if failed_md5_releases: - msg = FAIL + ( - "FAIL: these releases %s failed the md5 check!" % - ','.join(failed_md5_releases) - ) + END - raise Exception(msg) - elif self.total_downloaded_ok != len(releases_without_zip): - msg = FAIL + ( - "FAIL: Unknown error occured. Please check the logs." - ) + END - raise Exception(msg) - else: - print(OK + "All releases downloaded and md5 checked." + END) - - except OSError as e: - if e.errno != errno.EEXIST: - raise e - - def convert_targz_to_zip(self): - print("Converting the tar.gz to zip...") - files = glob.glob('%s/*.tar.gz' % self.dirpath) - total_converted = 0 - for targz in sorted(files, key=LooseVersion): - # Extract and remove tar. - tar = tarfile.open(targz) - tar.extractall(path=self.dirpath) - tar.close() - os.remove(targz) - - # Zip the extracted tar. - setuptools_folder_path = targz.replace('.tar.gz', '') - setuptools_folder_name = setuptools_folder_path.split("/")[-1] - print(setuptools_folder_name) - shutil.make_archive( - setuptools_folder_path, - 'zip', - self.dirpath, - setuptools_folder_name - ) - # Exclude extracted tar folder. - shutil.rmtree(setuptools_folder_path.replace('.zip', '')) - total_converted += 1 - print('Total converted: %s' % total_converted) - if self.total_downloaded_ok != total_converted: - msg = FAIL + ( - "FAIL: Total number of downloaded releases is different" - " from converted ones. Please check the logs." - ) + END - raise Exception(msg) - print("Done with the tar.gz->zip. Check folder %s." % main.dirpath) - - def upload_zips_to_pypi(self): - print('Uploading to pypi...') - zips = sorted(glob.glob('%s/*.zip' % self.dirpath), key=LooseVersion) - print("simulated upload of", zips); return - upload.upload(dists=zips) - - -if __name__ == '__main__': - main = SetuptoolsOldReleasesWithoutZip() - main.download_setuptools_releases_without_zip_counterpart() - main.convert_targz_to_zip() - main.upload_zips_to_pypi() @@ -1,5 +1,15 @@ +[bumpversion] +current_version = 20.6.7 +commit = True +tag = True + +[egg_info] +tag_build = .post +tag_date = 1 + [aliases] -release = sdist bdist_wheel upload build_sphinx upload_docs +clean_egg_info = egg_info -RDb '' +release = clean_egg_info sdist bdist_wheel build_sphinx test = pytest [build_sphinx] @@ -14,4 +24,7 @@ upload-dir = docs/build/html formats = gztar zip [wheel] -universal=1 +universal = 1 + +[bumpversion:file:setup.py] + @@ -43,7 +43,7 @@ def _gen_console_scripts(): console_scripts = list(_gen_console_scripts()) -readme_file = io.open('README.txt', encoding='utf-8') +readme_file = io.open('README.rst', encoding='utf-8') with readme_file: long_description = readme_file.read() @@ -54,14 +54,17 @@ force_windows_specific_files = ( os.environ.get("SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES") not in (None, "", "0") ) -if sys.platform == 'win32' or force_windows_specific_files: +if (sys.platform == 'win32' or (os.name == 'java' and os._name == 'nt')) \ + or force_windows_specific_files: package_data.setdefault('setuptools', []).extend(['*.exe']) package_data.setdefault('setuptools.command', []).extend(['*.xml']) needs_pytest = set(['ptr', 'pytest', 'test']).intersection(sys.argv) pytest_runner = ['pytest-runner'] if needs_pytest else [] -needs_sphinx = set(['build_sphinx', 'upload_docs']).intersection(sys.argv) -sphinx = ['sphinx', 'rst.linker'] if needs_sphinx else [] +needs_sphinx = set(['build_sphinx', 'upload_docs', 'release']).intersection(sys.argv) +sphinx = ['sphinx', 'rst.linker>=1.5'] if needs_sphinx else [] +needs_wheel = set(['release', 'bdist_wheel']).intersection(sys.argv) +wheel = ['wheel'] if needs_wheel else [] setup_params = dict( name="setuptools", @@ -70,10 +73,9 @@ setup_params = dict( "Python packages", author="Python Packaging Authority", author_email="distutils-sig@python.org", - license="PSF or ZPL", long_description=long_description, keywords="CPAN PyPI distutils eggs package management", - url="https://bitbucket.org/pypa/setuptools", + url="https://github.com/pypa/setuptools", src_root=src_root, packages=setuptools.find_packages(exclude=['*.tests']), package_data=package_data, @@ -129,8 +131,7 @@ setup_params = dict( classifiers=textwrap.dedent(""" Development Status :: 5 - Production/Stable Intended Audience :: Developers - License :: OSI Approved :: Python Software Foundation License - License :: OSI Approved :: Zope Public License + License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python :: 2.6 Programming Language :: Python :: 2.7 @@ -158,7 +159,7 @@ setup_params = dict( ] + (['mock'] if sys.version_info[:2] < (3, 3) else []), setup_requires=[ 'setuptools_scm>=1.9', - ] + sphinx + pytest_runner, + ] + sphinx + pytest_runner + wheel, ) if __name__ == '__main__': diff --git a/setuptools/__init__.py b/setuptools/__init__.py index b0a5401f..b8cec4c3 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -8,11 +8,12 @@ from distutils.core import Command as _Command from distutils.util import convert_path from fnmatch import fnmatchcase +from setuptools.extern.six.moves import filterfalse, map + import pkg_resources from setuptools.extension import Extension from setuptools.dist import Distribution, Feature, _get_unpatched from setuptools.depends import Require -from setuptools.compat import filterfalse __all__ = [ 'setup', 'Distribution', 'Feature', 'Command', 'Extension', 'Require', diff --git a/setuptools/command/__init__.py b/setuptools/command/__init__.py index f6dbc39c..3fb2f6df 100644 --- a/setuptools/command/__init__.py +++ b/setuptools/command/__init__.py @@ -2,7 +2,7 @@ __all__ = [ 'alias', 'bdist_egg', 'bdist_rpm', 'build_ext', 'build_py', 'develop', 'easy_install', 'egg_info', 'install', 'install_lib', 'rotate', 'saveopts', 'sdist', 'setopt', 'test', 'install_egg_info', 'install_scripts', - 'register', 'bdist_wininst', 'upload_docs', + 'register', 'bdist_wininst', 'upload_docs', 'upload', ] from distutils.command.bdist import bdist diff --git a/setuptools/command/alias.py b/setuptools/command/alias.py index 452a9244..4532b1cc 100755 --- a/setuptools/command/alias.py +++ b/setuptools/command/alias.py @@ -1,5 +1,7 @@ from distutils.errors import DistutilsOptionError +from setuptools.extern.six.moves import map + from setuptools.command.setopt import edit_config, option_base, config_file diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 87dce882..9cebd7fa 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -11,9 +11,10 @@ import os import marshal import textwrap +from setuptools.extern import six + from pkg_resources import get_build_platform, Distribution, ensure_directory from pkg_resources import EntryPoint -from setuptools.compat import basestring from setuptools.extension import Library from setuptools import Command @@ -413,7 +414,7 @@ def iter_symbols(code): for name in code.co_names: yield name for const in code.co_consts: - if isinstance(const, basestring): + if isinstance(const, six.string_types): yield const elif isinstance(const, CodeType): for name in iter_symbols(const): diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 8a50f032..8623c777 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -9,6 +9,7 @@ import distutils.errors import collections import itertools +from setuptools.extern.six.moves import map try: from setuptools.lib2to3_ex import Mixin2to3 @@ -59,9 +60,10 @@ class build_py(orig.build_py, Mixin2to3): self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0)) def __getattr__(self, attr): - if attr == 'data_files': # lazily compute data files - self.data_files = files = self._get_data_files() - return files + "lazily compute data files" + if attr == 'data_files': + self.data_files = self._get_data_files() + return self.data_files return orig.build_py.__getattr__(self, attr) def build_module(self, module, module_file, package): @@ -74,23 +76,21 @@ class build_py(orig.build_py, Mixin2to3): def _get_data_files(self): """Generate list of '(package,src_dir,build_dir,filenames)' tuples""" self.analyze_manifest() - data = [] - for package in self.packages or (): - # Locate package source directory - src_dir = self.get_package_dir(package) + return list(map(self._get_pkg_data_files, self.packages or ())) - # Compute package build directory - build_dir = os.path.join(*([self.build_lib] + package.split('.'))) + def _get_pkg_data_files(self, package): + # Locate package source directory + src_dir = self.get_package_dir(package) - # Length of path to strip from found files - plen = len(src_dir) + 1 + # Compute package build directory + build_dir = os.path.join(*([self.build_lib] + package.split('.'))) - # Strip directory from globbed filenames - filenames = [ - file[plen:] for file in self.find_data_files(package, src_dir) - ] - data.append((package, src_dir, build_dir, filenames)) - return data + # Strip directory from globbed filenames + filenames = [ + os.path.relpath(file, src_dir) + for file in self.find_data_files(package, src_dir) + ] + return package, src_dir, build_dir, filenames def find_data_files(self, package, src_dir): """Return filenames for package's data files in 'src_dir'""" diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py index 07b66ccb..11b5df10 100755 --- a/setuptools/command/develop.py +++ b/setuptools/command/develop.py @@ -5,9 +5,10 @@ import os import glob import io +from setuptools.extern import six + from pkg_resources import Distribution, PathMetadata, normalize_path from setuptools.command.easy_install import easy_install -from setuptools.compat import PY3 import setuptools @@ -87,7 +88,7 @@ class develop(easy_install): " installation directory", p, normalize_path(os.curdir)) def install_for_development(self): - if PY3 and getattr(self.distribution, 'use_2to3', False): + if six.PY3 and getattr(self.distribution, 'use_2to3', False): # If we run 2to3 we can not do this inplace: # Ensure metadata is up-to-date diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 9ca3b515..ea5cb028 100755 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -40,6 +40,9 @@ import subprocess import shlex import io +from setuptools.extern import six +from setuptools.extern.six.moves import configparser, map + from setuptools import Command from setuptools.sandbox import run_setup from setuptools.py31compat import get_path, get_config_vars @@ -48,8 +51,6 @@ from setuptools.archive_util import unpack_archive from setuptools.package_index import PackageIndex from setuptools.package_index import URL_SCHEME from setuptools.command import bdist_egg, egg_info -from setuptools.compat import (iteritems, maxsize, basestring, unicode, - reraise, PY2, PY3) from pkg_resources import ( yield_lines, normalize_path, resource_string, ensure_directory, get_distribution, find_distributions, Environment, Requirement, @@ -82,13 +83,13 @@ def samefile(p1, p2): return norm_p1 == norm_p2 -if PY2: +if six.PY2: def _to_ascii(s): return s def isascii(s): try: - unicode(s, 'ascii') + six.text_type(s, 'ascii') return True except UnicodeError: return False @@ -320,7 +321,7 @@ class easy_install(Command): self.local_index = Environment(self.shadow_path + sys.path) if self.find_links is not None: - if isinstance(self.find_links, basestring): + if isinstance(self.find_links, six.string_types): self.find_links = self.find_links.split() else: self.find_links = [] @@ -413,7 +414,7 @@ class easy_install(Command): try: pid = os.getpid() except: - pid = random.randint(0, maxsize) + pid = random.randint(0, sys.maxsize) return os.path.join(self.install_dir, "test-easy-install-%s" % pid) def warn_deprecated_options(self): @@ -709,10 +710,7 @@ class easy_install(Command): elif requirement is None or dist not in requirement: # if we wound up with a different version, resolve what we've got distreq = dist.as_requirement() - requirement = requirement or distreq - requirement = Requirement( - distreq.project_name, distreq.specs, requirement.extras - ) + requirement = Requirement(str(distreq)) log.info("Processing dependencies for %s", requirement) try: distros = WorkingSet([]).resolve( @@ -782,7 +780,7 @@ class easy_install(Command): There are a couple of template scripts in the package. This function loads one of them and prepares it for use. """ - # See https://bitbucket.org/pypa/setuptools/issue/134 for info + # See https://github.com/pypa/setuptools/issues/134 for info # on script file naming and downstream issues with SVR4 name = 'script.tmpl' if dev_path: @@ -1238,17 +1236,14 @@ class easy_install(Command): sitepy = os.path.join(self.install_dir, "site.py") source = resource_string("setuptools", "site-patch.py") + source = source.decode('utf-8') current = "" if os.path.exists(sitepy): log.debug("Checking existing site.py in %s", self.install_dir) - f = open(sitepy, 'rb') - current = f.read() - # we want str, not bytes - if PY3: - current = current.decode() + with io.open(sitepy) as strm: + current = strm.read() - f.close() if not current.startswith('def __boot():'): raise DistutilsError( "%s is not a setuptools-generated site.py; please" @@ -1259,9 +1254,8 @@ class easy_install(Command): log.info("Creating %s", sitepy) if not self.dry_run: ensure_directory(sitepy) - f = open(sitepy, 'wb') - f.write(source) - f.close() + with io.open(sitepy, 'w', encoding='utf-8') as strm: + strm.write(source) self.byte_compile([sitepy]) self.sitepy_installed = True @@ -1271,7 +1265,7 @@ class easy_install(Command): if not self.user: return home = convert_path(os.path.expanduser("~")) - for name, path in iteritems(self.config_vars): + for name, path in six.iteritems(self.config_vars): if path.startswith(home) and not os.path.isdir(path): self.debug_print("os.makedirs('%s', 0o700)" % path) os.makedirs(path, 0o700) @@ -1415,9 +1409,6 @@ def extract_wininst_cfg(dist_filename): return None f.seek(prepended - 12) - from setuptools.compat import StringIO, configparser - import struct - tag, cfglen, bmlen = struct.unpack("<iii", f.read(12)) if tag not in (0x1234567A, 0x1234567B): return None # not a valid tag @@ -1432,7 +1423,7 @@ def extract_wininst_cfg(dist_filename): # Now the config is in bytes, but for RawConfigParser, it should # be text, so decode it. config = config.decode(sys.getfilesystemencoding()) - cfg.readfp(StringIO(config)) + cfg.readfp(six.StringIO(config)) except configparser.Error: return None if not cfg.has_section('metadata') or not cfg.has_section('Setup'): @@ -1467,7 +1458,7 @@ def get_exe_prefixes(exe_filename): continue if parts[0].upper() in ('PURELIB', 'PLATLIB'): contents = z.read(name) - if PY3: + if six.PY3: contents = contents.decode() for pth in yield_lines(contents): pth = pth.strip().replace('\\', '/') @@ -1643,7 +1634,7 @@ def auto_chmod(func, arg, exc): chmod(arg, stat.S_IWRITE) return func(arg) et, ev, _ = sys.exc_info() - reraise(et, (ev[0], ev[1] + (" %s %s" % (func, arg)))) + six.reraise(et, (ev[0], ev[1] + (" %s %s" % (func, arg)))) def update_dist_caches(dist_path, fix_zipimporter_caches): @@ -1771,7 +1762,7 @@ def _update_zipimporter_cache(normalized_path, cache, updater=None): # * Does not support the dict.pop() method, forcing us to use the # get/del patterns instead. For more detailed information see the # following links: - # https://bitbucket.org/pypa/setuptools/issue/202/more-robust-zipimporter-cache-invalidation#comment-10495960 + # https://github.com/pypa/setuptools/issues/202#issuecomment-202913420 # https://bitbucket.org/pypy/pypy/src/dd07756a34a41f674c0cacfbc8ae1d4cc9ea2ae4/pypy/module/zipimport/interp_zipimport.py#cl-99 old_entry = cache[p] del cache[p] @@ -1878,17 +1869,6 @@ def chmod(path, mode): log.debug("chmod failed: %s", e) -def fix_jython_executable(executable, options): - warnings.warn("Use JythonCommandSpec", DeprecationWarning, stacklevel=2) - - if not JythonCommandSpec.relevant(): - return executable - - cmd = CommandSpec.best().from_param(executable) - cmd.install_options(options) - return cmd.as_header().lstrip('#!').rstrip('\n') - - class CommandSpec(list): """ A command spec for a #! header, specified as a list of arguments akin to @@ -1903,7 +1883,7 @@ class CommandSpec(list): """ Choose the best CommandSpec class based on environmental conditions. """ - return cls if not JythonCommandSpec.relevant() else JythonCommandSpec + return cls @classmethod def _sys_executable(cls): @@ -1970,36 +1950,6 @@ class WindowsCommandSpec(CommandSpec): split_args = dict(posix=False) -class JythonCommandSpec(CommandSpec): - @classmethod - def relevant(cls): - return ( - sys.platform.startswith('java') - and - __import__('java').lang.System.getProperty('os.name') != 'Linux' - ) - - def as_header(self): - """ - Workaround Jython's sys.executable being a .sh (an invalid - shebang line interpreter) - """ - if not is_sh(self[0]): - return super(JythonCommandSpec, self).as_header() - - if self.options: - # Can't apply the workaround, leave it broken - log.warn( - "WARNING: Unable to adapt shebang line for Jython," - " the following script is NOT executable\n" - " see http://bugs.jython.org/issue1112 for" - " more information.") - return super(JythonCommandSpec, self).as_header() - - items = ['/usr/bin/env'] + self + list(self.options) - return self._render(items) - - class ScriptWriter(object): """ Encapsulates behavior around writing entry point scripts for console and @@ -2076,7 +2026,10 @@ class ScriptWriter(object): """ Select the best ScriptWriter for this environment. """ - return WindowsScriptWriter.best() if sys.platform == 'win32' else cls + if sys.platform == 'win32' or (os.name == 'java' and os._name == 'nt'): + return WindowsScriptWriter.best() + else: + return cls @classmethod def _get_script_args(cls, type_, name, header, script_text): @@ -2209,7 +2162,7 @@ def get_win_launcher(type): def load_launcher_manifest(name): manifest = pkg_resources.resource_string(__name__, 'launcher manifest.xml') - if PY2: + if six.PY2: return manifest % vars() else: return manifest.decode('utf-8') % vars() diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 1301bd84..d1bd9b04 100755 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -14,7 +14,8 @@ import io import warnings import time -from setuptools.compat import basestring, PY3, StringIO +from setuptools.extern import six +from setuptools.extern.six.moves import map from setuptools import Command from setuptools.command.sdist import sdist @@ -26,7 +27,7 @@ from pkg_resources import ( safe_version, yield_lines, EntryPoint, iter_entry_points, to_filename) import setuptools.unicode_utils as unicode_utils -from pkg_resources import packaging +from pkg_resources.extern import packaging try: from setuptools_svn import svn_utils @@ -148,7 +149,7 @@ class egg_info(Command): to the file. """ log.info("writing %s to %s", what, filename) - if PY3: + if six.PY3: data = data.encode("utf-8") if not self.dry_run: f = open(filename, 'wb') @@ -412,7 +413,7 @@ def _write_requirements(stream, reqs): def write_requirements(cmd, basename, filename): dist = cmd.distribution - data = StringIO() + data = six.StringIO() _write_requirements(data, dist.install_requires) extras_require = dist.extras_require or {} for extra in sorted(extras_require): @@ -452,12 +453,12 @@ def write_arg(cmd, basename, filename, force=False): def write_entries(cmd, basename, filename): ep = cmd.distribution.entry_points - if isinstance(ep, basestring) or ep is None: + if isinstance(ep, six.string_types) or ep is None: data = ep elif ep is not None: data = [] for section, contents in sorted(ep.items()): - if not isinstance(contents, basestring): + if not isinstance(contents, six.string_types): contents = EntryPoint.parse_group(section, contents) contents = '\n'.join(sorted(map(str, contents.values()))) data.append('[%s]\n%s\n\n' % (section, contents)) diff --git a/setuptools/command/install.py b/setuptools/command/install.py index d2bca2ec..31a5ddb5 100644 --- a/setuptools/command/install.py +++ b/setuptools/command/install.py @@ -8,7 +8,7 @@ import distutils.command.install as orig import setuptools # Prior to numpy 1.9, NumPy relies on the '_install' name, so provide it for -# now. See https://bitbucket.org/pypa/setuptools/issue/199/ +# now. See https://github.com/pypa/setuptools/issues/199/ _install = orig.install diff --git a/setuptools/command/install_egg_info.py b/setuptools/command/install_egg_info.py index fd0f118b..60b615d2 100755 --- a/setuptools/command/install_egg_info.py +++ b/setuptools/command/install_egg_info.py @@ -1,6 +1,8 @@ from distutils import log, dir_util import os +from setuptools.extern.six.moves import map + from setuptools import Command from setuptools.archive_util import unpack_archive import pkg_resources @@ -27,7 +29,7 @@ class install_egg_info(Command): ).egg_name() + '.egg-info' self.source = ei_cmd.egg_info self.target = os.path.join(self.install_dir, basename) - self.outputs = [self.target] + self.outputs = [] def run(self): self.run_command('egg_info') diff --git a/setuptools/command/rotate.py b/setuptools/command/rotate.py index 1b073620..804f962a 100755 --- a/setuptools/command/rotate.py +++ b/setuptools/command/rotate.py @@ -3,8 +3,9 @@ from distutils import log from distutils.errors import DistutilsOptionError import os +from setuptools.extern import six + from setuptools import Command -from setuptools.compat import basestring class rotate(Command): @@ -36,7 +37,7 @@ class rotate(Command): self.keep = int(self.keep) except ValueError: raise DistutilsOptionError("--keep must be an integer") - if isinstance(self.match, basestring): + if isinstance(self.match, six.string_types): self.match = [ convert_path(p.strip()) for p in self.match.split(',') ] diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 71196512..6640d4e3 100755 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -5,7 +5,8 @@ import os import sys import io -from setuptools.compat import PY3 +from setuptools.extern import six + from setuptools.utils import cs_path_exists import pkg_resources @@ -181,7 +182,7 @@ class sdist(orig.sdist): manifest = open(self.manifest, 'rbU') for line in manifest: # The manifest must contain UTF-8. See #303. - if PY3: + if six.PY3: try: line = line.decode('UTF-8') except UnicodeDecodeError: diff --git a/setuptools/command/setopt.py b/setuptools/command/setopt.py index 74c7cad8..7f332be5 100755 --- a/setuptools/command/setopt.py +++ b/setuptools/command/setopt.py @@ -4,6 +4,8 @@ from distutils.errors import DistutilsOptionError import distutils import os +from setuptools.extern.six.moves import configparser + from setuptools import Command @@ -37,8 +39,6 @@ def edit_config(filename, settings, dry_run=False): while a dictionary lists settings to be changed or deleted in that section. A setting of ``None`` means to delete that setting. """ - from setuptools.compat import configparser - log.debug("Reading configuration from %s", filename) opts = configparser.RawConfigParser() opts.read([filename]) diff --git a/setuptools/command/test.py b/setuptools/command/test.py index c26f5fc9..371e913b 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -2,11 +2,13 @@ from distutils.errors import DistutilsOptionError from unittest import TestLoader import sys +from setuptools.extern import six +from setuptools.extern.six.moves import map + from pkg_resources import (resource_listdir, resource_exists, normalize_path, working_set, _namespace_packages, add_activation_listener, require, EntryPoint) from setuptools import Command -from setuptools.compat import PY3 from setuptools.py31compat import unittest_main @@ -100,7 +102,7 @@ class test(Command): yield self.test_suite def with_project_on_sys_path(self, func): - with_2to3 = PY3 and getattr(self.distribution, 'use_2to3', False) + with_2to3 = six.PY3 and getattr(self.distribution, 'use_2to3', False) if with_2to3: # If we run 2to3 we can not do this inplace: @@ -160,7 +162,7 @@ class test(Command): # Purge modules under test from sys.modules. The test loader will # re-import them from the build location. Required when 2to3 is used # with namespace packages. - if PY3 and getattr(self.distribution, 'use_2to3', False): + if six.PY3 and getattr(self.distribution, 'use_2to3', False): module = self.test_suite.split('.')[0] if module in _namespace_packages: del_modules = [] diff --git a/setuptools/command/upload.py b/setuptools/command/upload.py new file mode 100644 index 00000000..08c20ba8 --- /dev/null +++ b/setuptools/command/upload.py @@ -0,0 +1,23 @@ +from distutils.command import upload as orig + + +class upload(orig.upload): + """ + Override default upload behavior to look up password + in the keyring if available. + """ + + def finalize_options(self): + orig.upload.finalize_options(self) + self.password or self._load_password_from_keyring() + + def _load_password_from_keyring(self): + """ + Attempt to load password from keyring. Suppress Exceptions. + """ + try: + keyring = __import__('keyring') + self.password = keyring.get_password(self.repository, + self.username) + except Exception: + pass diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 001ee936..f887b47e 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -8,25 +8,26 @@ PyPI's pythonhosted.org). from base64 import standard_b64encode from distutils import log from distutils.errors import DistutilsOptionError -from distutils.command.upload import upload import os import socket import zipfile import tempfile -import sys import shutil -from setuptools.compat import httplib, urlparse, unicode, iteritems, PY3 +from setuptools.extern import six +from setuptools.extern.six.moves import http_client, urllib + from pkg_resources import iter_entry_points +from .upload import upload -errors = 'surrogateescape' if PY3 else 'strict' +errors = 'surrogateescape' if six.PY3 else 'strict' # This is not just a replacement for byte literals # but works as a general purpose encoder def b(s, encoding='utf-8'): - if isinstance(s, unicode): + if isinstance(s, six.text_type): return s.encode(encoding, errors) return s @@ -113,7 +114,7 @@ class upload_docs(upload): # set up the authentication credentials = b(self.username + ':' + self.password) credentials = standard_b64encode(credentials) - if PY3: + if six.PY3: credentials = credentials.decode('ascii') auth = "Basic " + credentials @@ -122,7 +123,7 @@ class upload_docs(upload): sep_boundary = b('\n--') + b(boundary) end_boundary = sep_boundary + b('--') body = [] - for key, values in iteritems(data): + for key, values in six.iteritems(data): title = '\nContent-Disposition: form-data; name="%s"' % key # handle multiple entries for the same name if not isinstance(values, list): @@ -150,12 +151,12 @@ class upload_docs(upload): # We can't use urllib2 since we need to send the Basic # auth right with the first request schema, netloc, url, params, query, fragments = \ - urlparse(self.repository) + urllib.parse.urlparse(self.repository) assert not params and not query and not fragments if schema == 'http': - conn = httplib.HTTPConnection(netloc) + conn = http_client.HTTPConnection(netloc) elif schema == 'https': - conn = httplib.HTTPSConnection(netloc) + conn = http_client.HTTPSConnection(netloc) else: raise AssertionError("unsupported schema " + schema) diff --git a/setuptools/compat.py b/setuptools/compat.py deleted file mode 100644 index f0175a5d..00000000 --- a/setuptools/compat.py +++ /dev/null @@ -1,70 +0,0 @@ -import sys -import itertools - -PY3 = sys.version_info >= (3,) -PY2 = not PY3 - -if PY2: - basestring = basestring - import __builtin__ as builtins - import ConfigParser as configparser - from StringIO import StringIO - BytesIO = StringIO - func_code = lambda o: o.func_code - func_globals = lambda o: o.func_globals - im_func = lambda o: o.im_func - from htmlentitydefs import name2codepoint - import httplib - from BaseHTTPServer import HTTPServer - from SimpleHTTPServer import SimpleHTTPRequestHandler - from BaseHTTPServer import BaseHTTPRequestHandler - iteritems = lambda o: o.iteritems() - long_type = long - maxsize = sys.maxint - unichr = unichr - unicode = unicode - bytes = str - from urllib import url2pathname, splittag, pathname2url - import urllib2 - from urllib2 import urlopen, HTTPError, URLError, unquote, splituser - from urlparse import urlparse, urlunparse, urljoin, urlsplit, urlunsplit - filterfalse = itertools.ifilterfalse - filter = itertools.ifilter - map = itertools.imap - - exec("""def reraise(tp, value, tb=None): - raise tp, value, tb""") - -if PY3: - basestring = str - import builtins - import configparser - from io import StringIO, BytesIO - func_code = lambda o: o.__code__ - func_globals = lambda o: o.__globals__ - im_func = lambda o: o.__func__ - from html.entities import name2codepoint - import http.client as httplib - from http.server import HTTPServer, SimpleHTTPRequestHandler - from http.server import BaseHTTPRequestHandler - iteritems = lambda o: o.items() - long_type = int - maxsize = sys.maxsize - unichr = chr - unicode = str - bytes = bytes - from urllib.error import HTTPError, URLError - import urllib.request as urllib2 - from urllib.request import urlopen, url2pathname, pathname2url - from urllib.parse import ( - urlparse, urlunparse, unquote, splituser, urljoin, urlsplit, - urlunsplit, splittag, - ) - filterfalse = itertools.filterfalse - filter = filter - map = map - - def reraise(tp, value, tb=None): - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value diff --git a/setuptools/depends.py b/setuptools/depends.py index e87ef3f3..9f7c9a35 100644 --- a/setuptools/depends.py +++ b/setuptools/depends.py @@ -3,7 +3,8 @@ import imp import marshal from imp import PKG_DIRECTORY, PY_COMPILED, PY_SOURCE, PY_FROZEN from distutils.version import StrictVersion -from setuptools import compat + +from setuptools.extern import six __all__ = [ 'Require', 'find_module', 'get_module_constant', 'extract_constant' @@ -99,7 +100,8 @@ def _iter_code(code): ptr += 3 if op==EXTENDED_ARG: - extended_arg = arg * compat.long_type(65536) + long_type = six.integer_types[-1] + extended_arg = arg * long_type(65536) continue else: diff --git a/setuptools/dist.py b/setuptools/dist.py index b0c52838..086e0a58 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -13,13 +13,14 @@ from distutils.core import Distribution as _Distribution from distutils.errors import (DistutilsOptionError, DistutilsPlatformError, DistutilsSetupError) +from setuptools.extern import six +from setuptools.extern.six.moves import map +from pkg_resources.extern import packaging + from setuptools.depends import Require -from setuptools.compat import basestring, PY2 from setuptools import windows_support import pkg_resources -packaging = pkg_resources.packaging - def _get_unpatched(cls): """Protect against re-patching the distutils if reloaded @@ -138,7 +139,7 @@ def check_entry_points(dist, attr, value): raise DistutilsSetupError(e) def check_test_suite(dist, attr, value): - if not isinstance(value,basestring): + if not isinstance(value, six.string_types): raise DistutilsSetupError("test_suite must be a string") def check_package_data(dist, attr, value): @@ -160,7 +161,7 @@ def check_packages(dist, attr, value): for pkgname in value: if not re.match(r'\w+(\.\w+)*', pkgname): distutils.log.warn( - "WARNING: %r not a valid package name; please use only" + "WARNING: %r not a valid package name; please use only " ".-separated package names in setup.py", pkgname ) @@ -439,6 +440,14 @@ class Distribution(_Distribution): self.cmdclass[ep.name] = cmdclass return _Distribution.print_commands(self) + def get_command_list(self): + for ep in pkg_resources.iter_entry_points('distutils.commands'): + if ep.name not in self.cmdclass: + # don't require extras as the commands won't be invoked + cmdclass = ep.resolve() + self.cmdclass[ep.name] = cmdclass + return _Distribution.get_command_list(self) + def _set_feature(self,name,status): """Set feature's inclusion status""" setattr(self,self._feature_attrname(name),status) @@ -674,7 +683,7 @@ class Distribution(_Distribution): """ import sys - if PY2 or self.help_commands: + if six.PY2 or self.help_commands: return _Distribution.handle_display_options(self, option_order) # Stdout may be StringIO (e.g. in tests) @@ -711,7 +720,7 @@ class Feature: """ **deprecated** -- The `Feature` facility was never completely implemented or supported, `has reported issues - <https://bitbucket.org/pypa/setuptools/issue/58>`_ and will be removed in + <https://github.com/pypa/setuptools/issues/58>`_ and will be removed in a future version. A subset of the distribution that can be excluded if unneeded/wanted @@ -768,7 +777,7 @@ class Feature: def warn_deprecated(): warnings.warn( "Features are deprecated and will be removed in a future " - "version. See http://bitbucket.org/pypa/setuptools/65.", + "version. See https://github.com/pypa/setuptools/issues/65.", DeprecationWarning, stacklevel=3, ) @@ -817,7 +826,7 @@ class Feature: if not self.available: raise DistutilsPlatformError( - self.description+" is required," + self.description+" is required, " "but is not available on this platform" ) diff --git a/setuptools/extension.py b/setuptools/extension.py index 35eb7c7c..d10609b6 100644 --- a/setuptools/extension.py +++ b/setuptools/extension.py @@ -5,6 +5,8 @@ import distutils.core import distutils.errors import distutils.extension +from setuptools.extern.six.moves import map + from .dist import _get_unpatched from . import msvc9_support diff --git a/setuptools/extern/__init__.py b/setuptools/extern/__init__.py new file mode 100644 index 00000000..6859aa5b --- /dev/null +++ b/setuptools/extern/__init__.py @@ -0,0 +1,5 @@ +from pkg_resources.extern import VendorImporter + + +names = 'six', +VendorImporter(__name__, names, 'pkg_resources._vendor').install() diff --git a/setuptools/launch.py b/setuptools/launch.py new file mode 100644 index 00000000..06e15e1e --- /dev/null +++ b/setuptools/launch.py @@ -0,0 +1,35 @@ +""" +Launch the Python script on the command line after +setuptools is bootstrapped via import. +""" + +# Note that setuptools gets imported implicitly by the +# invocation of this script using python -m setuptools.launch + +import tokenize +import sys + + +def run(): + """ + Run the script in sys.argv[1] as if it had + been invoked naturally. + """ + __builtins__ + script_name = sys.argv[1] + namespace = dict( + __file__ = script_name, + __name__ = '__main__', + __doc__ = None, + ) + sys.argv[:] = sys.argv[1:] + + open_ = getattr(tokenize, 'open', open) + script = open_(script_name).read() + norm_script = script.replace('\\r\\n', '\\n') + code = compile(norm_script, script_name, 'exec') + exec(code, namespace) + + +if __name__ == '__main__': + run() diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 2c565e88..c53343e4 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -9,6 +9,14 @@ import hashlib import itertools from functools import wraps +try: + from urllib.parse import splituser +except ImportError: + from urllib2 import splituser + +from setuptools.extern import six +from setuptools.extern.six.moves import urllib, http_client, configparser, map + from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, require, Environment, find_distributions, safe_name, safe_version, @@ -17,12 +25,6 @@ from pkg_resources import ( from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError -from setuptools.compat import ( - urllib2, httplib, StringIO, HTTPError, urlparse, urlunparse, unquote, - splituser, url2pathname, name2codepoint, unichr, urljoin, urlsplit, - urlunsplit, configparser, filter, map, -) -from setuptools.compat import filterfalse from fnmatch import translate from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers @@ -69,10 +71,11 @@ def parse_bdist_wininst(name): def egg_info_for_url(url): - scheme, server, path, parameters, query, fragment = urlparse(url) - base = unquote(path.split('/')[-1]) + parts = urllib.parse.urlparse(url) + scheme, server, path, parameters, query, fragment = parts + base = urllib.parse.unquote(path.split('/')[-1]) if server=='sourceforge.net' and base=='download': # XXX Yuck - base = unquote(path.split('/')[-2]) + base = urllib.parse.unquote(path.split('/')[-2]) if '#' in base: base, fragment = base.split('#',1) return base,fragment @@ -158,7 +161,7 @@ def unique_everseen(iterable, key=None): seen = set() seen_add = seen.add if key is None: - for element in filterfalse(seen.__contains__, iterable): + for element in six.moves.filterfalse(seen.__contains__, iterable): seen_add(element) yield element else: @@ -190,14 +193,14 @@ def find_external_links(url, page): rels = set(map(str.strip, rel.lower().split(','))) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - yield urljoin(url, htmldecode(match.group(1))) + yield urllib.parse.urljoin(url, htmldecode(match.group(1))) for tag in ("<th>Home Page", "<th>Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - yield urljoin(url, htmldecode(match.group(1))) + yield urllib.parse.urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s setuptools/%s" % ( sys.version[:3], require('setuptools')[0].version @@ -240,7 +243,7 @@ class HashChecker(ContentChecker): @classmethod def from_url(cls, url): "Construct a (possibly null) ContentChecker from a URL" - fragment = urlparse(url)[-1] + fragment = urllib.parse.urlparse(url)[-1] if not fragment: return ContentChecker() match = cls.pattern.search(fragment) @@ -275,7 +278,7 @@ class PackageIndex(Environment): self.to_scan = [] if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()): self.opener = ssl_support.opener_for(ca_bundle) - else: self.opener = urllib2.urlopen + else: self.opener = urllib.request.urlopen def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -312,7 +315,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. - if isinstance(f, HTTPError): + if isinstance(f, urllib.error.HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' else: @@ -320,7 +323,7 @@ class PackageIndex(Environment): page = page.decode(charset, "ignore") f.close() for match in HREF.finditer(page): - link = urljoin(base, htmldecode(match.group(1))) + link = urllib.parse.urljoin(base, htmldecode(match.group(1))) self.process_url(link) if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) @@ -343,7 +346,7 @@ class PackageIndex(Environment): def url_ok(self, url, fatal=False): s = URL_SCHEME(url) - if (s and s.group(1).lower()=='file') or self.allows(urlparse(url)[1]): + if (s and s.group(1).lower()=='file') or self.allows(urllib.parse.urlparse(url)[1]): return True msg = ("\nNote: Bypassing %s (disallowed host; see " "http://bit.ly/1dg9ijs for details).\n") @@ -384,7 +387,7 @@ class PackageIndex(Environment): # Process a URL to see if it's for a package page if link.startswith(self.index_url): parts = list(map( - unquote, link[len(self.index_url):].split('/') + urllib.parse.unquote, link[len(self.index_url):].split('/') )) if len(parts)==2 and '#' not in parts[1]: # it's a package page, sanitize and index it @@ -397,7 +400,7 @@ class PackageIndex(Environment): # process an index page into the package-page index for match in HREF.finditer(page): try: - scan(urljoin(url, htmldecode(match.group(1)))) + scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) except ValueError: pass @@ -673,7 +676,7 @@ class PackageIndex(Environment): try: checker = HashChecker.from_url(url) fp = self.open_url(strip_fragment(url)) - if isinstance(fp, HTTPError): + if isinstance(fp, urllib.error.HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) @@ -709,21 +712,21 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url, self.opener) - except (ValueError, httplib.InvalidURL) as v: + except (ValueError, http_client.InvalidURL) as v: msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib2.HTTPError as v: + except urllib.error.HTTPError as v: return v - except urllib2.URLError as v: + except urllib.error.URLError as v: if warning: self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) - except httplib.BadStatusLine as v: + except http_client.BadStatusLine as v: if warning: self.warn(warning, v.line) else: @@ -732,7 +735,7 @@ class PackageIndex(Environment): 'down, %s' % (url, v.line) ) - except httplib.HTTPException as v: + except http_client.HTTPException as v: if warning: self.warn(warning, v) else: @@ -763,7 +766,7 @@ class PackageIndex(Environment): elif scheme.startswith('hg+'): return self._download_hg(url, filename) elif scheme=='file': - return url2pathname(urlparse(url)[2]) + return urllib.request.url2pathname(urllib.parse.urlparse(url)[2]) else: self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) @@ -797,7 +800,7 @@ class PackageIndex(Environment): url = url.split('#',1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: - scheme, netloc, path, p, q, f = urlparse(url) + scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: netloc, path = path[2:].split('/',1) auth, host = splituser(netloc) @@ -808,14 +811,15 @@ class PackageIndex(Environment): else: creds = " --username="+auth netloc = host - url = urlunparse((scheme, netloc, url, p, q, f)) + parts = scheme, netloc, url, p, q, f + url = urllib.parse.urlunparse(parts) self.info("Doing subversion checkout from %s to %s", url, filename) os.system("svn checkout%s -q %s %s" % (creds, url, filename)) return filename @staticmethod def _vcs_split_rev_from_url(url, pop_prefix=False): - scheme, netloc, path, query, frag = urlsplit(url) + scheme, netloc, path, query, frag = urllib.parse.urlsplit(url) scheme = scheme.split('+', 1)[-1] @@ -827,7 +831,7 @@ class PackageIndex(Environment): path, rev = path.rsplit('@', 1) # Also, discard fragment - url = urlunsplit((scheme, netloc, path, query, '')) + url = urllib.parse.urlunsplit((scheme, netloc, path, query, '')) return url, rev @@ -879,7 +883,7 @@ entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub def uchr(c): if not isinstance(c, int): return c - if c>255: return unichr(c) + if c>255: return six.unichr(c) return chr(c) def decode_entity(match): @@ -889,7 +893,7 @@ def decode_entity(match): elif what.startswith('#'): what = int(what[1:]) else: - what = name2codepoint.get(what, match.group(0)) + what = six.moves.html_entities.name2codepoint.get(what, match.group(0)) return uchr(what) def htmldecode(text): @@ -920,7 +924,7 @@ def _encode_auth(auth): >>> chr(10) in str(_encode_auth(long_auth)) False """ - auth_s = unquote(auth) + auth_s = urllib.parse.unquote(auth) # convert to bytes auth_bytes = auth_s.encode() # use the legacy interface for Python 2.3 support @@ -984,15 +988,15 @@ class PyPIConfig(configparser.RawConfigParser): return cred -def open_with_auth(url, opener=urllib2.urlopen): +def open_with_auth(url, opener=urllib.request.urlopen): """Open a urllib2 request, handling HTTP authentication""" - scheme, netloc, path, params, query, frag = urlparse(url) + scheme, netloc, path, params, query, frag = urllib.parse.urlparse(url) # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): - raise httplib.InvalidURL("nonnumeric port: ''") + raise http_client.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): auth, host = splituser(netloc) @@ -1008,11 +1012,12 @@ def open_with_auth(url, opener=urllib2.urlopen): if auth: auth = "Basic " + _encode_auth(auth) - new_url = urlunparse((scheme,host,path,params,query,frag)) - request = urllib2.Request(new_url) + parts = scheme, host, path, params, query, frag + new_url = urllib.parse.urlunparse(parts) + request = urllib.request.Request(new_url) request.add_header("Authorization", auth) else: - request = urllib2.Request(url) + request = urllib.request.Request(url) request.add_header('User-Agent', user_agent) fp = opener(request) @@ -1020,9 +1025,10 @@ def open_with_auth(url, opener=urllib2.urlopen): if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work - s2, h2, path2, param2, query2, frag2 = urlparse(fp.url) + s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) if s2==scheme and h2==host: - fp.url = urlunparse((s2,netloc,path2,param2,query2,frag2)) + parts = s2, netloc, path2, param2, query2, frag2 + fp.url = urllib.parse.urlunparse(parts) return fp @@ -1035,10 +1041,10 @@ def fix_sf_url(url): def local_open(url): """Read a local path, with special support for directories""" - scheme, server, path, param, query, frag = urlparse(url) - filename = url2pathname(path) + scheme, server, path, param, query, frag = urllib.parse.urlparse(url) + filename = urllib.request.url2pathname(path) if os.path.isfile(filename): - return urllib2.urlopen(url) + return urllib.request.urlopen(url) elif path.endswith('/') and os.path.isdir(filename): files = [] for f in os.listdir(filename): @@ -1059,4 +1065,5 @@ def local_open(url): status, message, body = 404, "Path not found", "Not found" headers = {'content-type': 'text/html'} - return HTTPError(url, status, message, headers, StringIO(body)) + body_stream = six.StringIO(body) + return urllib.error.HTTPError(url, status, message, headers, body_stream) diff --git a/setuptools/py26compat.py b/setuptools/py26compat.py index 738b0cc4..e52bd85b 100644 --- a/setuptools/py26compat.py +++ b/setuptools/py26compat.py @@ -4,7 +4,10 @@ Compatibility Support for Python 2.6 and earlier import sys -from setuptools.compat import splittag +try: + from urllib.parse import splittag +except ImportError: + from urllib import splittag def strip_fragment(url): """ diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 85de85ff..23e296b1 100755 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -8,6 +8,9 @@ import re import contextlib import pickle +from setuptools.extern import six +from setuptools.extern.six.moves import builtins, map + import pkg_resources if sys.platform.startswith('java'): @@ -22,9 +25,6 @@ _open = open from distutils.errors import DistutilsError from pkg_resources import working_set -from setuptools import compat -from setuptools.compat import builtins - __all__ = [ "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup", ] @@ -138,7 +138,7 @@ class ExceptionSaver: return type, exc = map(pickle.loads, self._saved) - compat.reraise(type, exc, self._tb) + six.reraise(type, exc, self._tb) @contextlib.contextmanager @@ -207,8 +207,12 @@ def _needs_hiding(mod_name): True >>> _needs_hiding('distutils') True + >>> _needs_hiding('os') + False + >>> _needs_hiding('Cython') + True """ - pattern = re.compile('(setuptools|pkg_resources|distutils)(\.|$)') + pattern = re.compile('(setuptools|pkg_resources|distutils|Cython)(\.|$)') return bool(pattern.match(mod_name)) diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py index cc7db067..657197cf 100644 --- a/setuptools/ssl_support.py +++ b/setuptools/ssl_support.py @@ -3,9 +3,10 @@ import socket import atexit import re +from setuptools.extern.six.moves import urllib, http_client, map + import pkg_resources from pkg_resources import ResolutionError, ExtractionError -from setuptools.compat import urllib2 try: import ssl @@ -24,20 +25,15 @@ cert_paths = """ /usr/local/share/certs/ca-root.crt /etc/ssl/cert.pem /System/Library/OpenSSL/certs/cert.pem +/usr/local/share/certs/ca-root-nss.crt """.strip().split() -HTTPSHandler = HTTPSConnection = object - -for what, where in ( - ('HTTPSHandler', ['urllib2','urllib.request']), - ('HTTPSConnection', ['httplib', 'http.client']), -): - for module in where: - try: - exec("from %s import %s" % (module, what)) - except ImportError: - pass +try: + HTTPSHandler = urllib.request.HTTPSHandler + HTTPSConnection = http_client.HTTPSConnection +except AttributeError: + HTTPSHandler = HTTPSConnection = object is_available = ssl is not None and object not in (HTTPSHandler, HTTPSConnection) @@ -198,7 +194,7 @@ class VerifyingHTTPSConn(HTTPSConnection): def opener_for(ca_bundle=None): """Get a urlopen() replacement that uses ca_bundle for verification""" - return urllib2.build_opener( + return urllib.request.build_opener( VerifyingHTTPSHandler(ca_bundle or find_ca_bundle()) ).open @@ -223,6 +219,12 @@ def get_win_certfile(): self.addcerts(certs) atexit.register(self.close) + def close(self): + try: + super(MyCertFile, self).close() + except OSError: + pass + _wincerts = MyCertFile(stores=['CA', 'ROOT']) return _wincerts.name diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index f985a6e4..32447356 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -7,8 +7,8 @@ from distutils.errors import DistutilsOptionError, DistutilsPlatformError from distutils.errors import DistutilsSetupError from distutils.core import Extension from distutils.version import LooseVersion -from setuptools.compat import func_code +from setuptools.extern import six import pytest import setuptools.dist @@ -52,7 +52,7 @@ class TestDepends: x = "test" y = z - fc = func_code(f1) + fc = six.get_function_code(f1) # unrecognized name assert dep.extract_constant(fc,'q', -1) is None diff --git a/setuptools/tests/contexts.py b/setuptools/tests/contexts.py index 1d29284b..ae28c7c3 100644 --- a/setuptools/tests/contexts.py +++ b/setuptools/tests/contexts.py @@ -5,7 +5,8 @@ import sys import contextlib import site -from ..compat import StringIO +from setuptools.extern import six +import pkg_resources @contextlib.contextmanager @@ -57,8 +58,8 @@ def quiet(): old_stdout = sys.stdout old_stderr = sys.stderr - new_stdout = sys.stdout = StringIO() - new_stderr = sys.stderr = StringIO() + new_stdout = sys.stdout = six.StringIO() + new_stderr = sys.stderr = six.StringIO() try: yield new_stdout, new_stderr finally: @@ -78,6 +79,18 @@ def save_user_site_setting(): @contextlib.contextmanager +def save_pkg_resources_state(): + pr_state = pkg_resources.__getstate__() + # also save sys.path + sys_path = sys.path[:] + try: + yield pr_state, sys_path + finally: + sys.path[:] = sys_path + pkg_resources.__setstate__(pr_state) + + +@contextlib.contextmanager def suppress_exceptions(*excs): try: yield diff --git a/setuptools/tests/files.py b/setuptools/tests/files.py new file mode 100644 index 00000000..4364241b --- /dev/null +++ b/setuptools/tests/files.py @@ -0,0 +1,32 @@ +import os + + +def build_files(file_defs, prefix=""): + """ + Build a set of files/directories, as described by the file_defs dictionary. + + Each key/value pair in the dictionary is interpreted as a filename/contents + pair. If the contents value is a dictionary, a directory is created, and the + dictionary interpreted as the files within it, recursively. + + For example: + + {"README.txt": "A README file", + "foo": { + "__init__.py": "", + "bar": { + "__init__.py": "", + }, + "baz.py": "# Some code", + } + } + """ + for name, contents in file_defs.items(): + full_name = os.path.join(prefix, name) + if isinstance(contents, dict): + if not os.path.exists(full_name): + os.makedirs(full_name) + build_files(contents, prefix=full_name) + else: + with open(full_name, 'w') as f: + f.write(contents) diff --git a/setuptools/tests/server.py b/setuptools/tests/server.py index 6b214279..6a687937 100644 --- a/setuptools/tests/server.py +++ b/setuptools/tests/server.py @@ -3,10 +3,11 @@ import time import threading -from setuptools.compat import BaseHTTPRequestHandler -from setuptools.compat import HTTPServer, SimpleHTTPRequestHandler -class IndexServer(HTTPServer): +from setuptools.extern.six.moves import BaseHTTPServer, SimpleHTTPServer + + +class IndexServer(BaseHTTPServer.HTTPServer): """Basic single-threaded http server simulating a package index You can use this server in unittest like this:: @@ -18,8 +19,9 @@ class IndexServer(HTTPServer): s.stop() """ def __init__(self, server_address=('', 0), - RequestHandlerClass=SimpleHTTPRequestHandler): - HTTPServer.__init__(self, server_address, RequestHandlerClass) + RequestHandlerClass=SimpleHTTPServer.SimpleHTTPRequestHandler): + BaseHTTPServer.HTTPServer.__init__(self, server_address, + RequestHandlerClass) self._run = True def start(self): @@ -40,19 +42,20 @@ class IndexServer(HTTPServer): port = self.server_port return 'http://127.0.0.1:%s/setuptools/tests/indexes/' % port -class RequestRecorder(BaseHTTPRequestHandler): +class RequestRecorder(BaseHTTPServer.BaseHTTPRequestHandler): def do_GET(self): requests = vars(self.server).setdefault('requests', []) requests.append(self) self.send_response(200, 'OK') -class MockServer(HTTPServer, threading.Thread): +class MockServer(BaseHTTPServer.HTTPServer, threading.Thread): """ A simple HTTP Server that records the requests made to it. """ def __init__(self, server_address=('', 0), RequestHandlerClass=RequestRecorder): - HTTPServer.__init__(self, server_address, RequestHandlerClass) + BaseHTTPServer.HTTPServer.__init__(self, server_address, + RequestHandlerClass) threading.Thread.__init__(self) self.setDaemon(True) self.requests = [] diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index ab5da00e..1b844499 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -5,12 +5,13 @@ import site import sys import io +from setuptools.extern import six + import pytest from setuptools.command.develop import develop from setuptools.dist import Distribution from . import contexts -from setuptools.compat import PY3 SETUP_PY = """\ @@ -85,7 +86,7 @@ class TestDevelop: with io.open(fn) as init_file: init = init_file.read().strip() - expected = 'print("foo")' if PY3 else 'print "foo"' + expected = 'print("foo")' if six.PY3 else 'print "foo"' assert init == expected def test_console_scripts(self, tmpdir): diff --git a/setuptools/tests/test_dist_info.py b/setuptools/tests/test_dist_info.py index 6d0ab587..9f226a55 100644 --- a/setuptools/tests/test_dist_info.py +++ b/setuptools/tests/test_dist_info.py @@ -4,6 +4,8 @@ import os import shutil import tempfile +from setuptools.extern.six.moves import map + import pytest import pkg_resources @@ -26,14 +28,15 @@ class TestDistInfo: assert versioned.version == '2.718' # from filename assert unversioned.version == '0.3' # from METADATA - @pytest.mark.importorskip('ast') def test_conditional_dependencies(self): specs = 'splort==4', 'quux>=1.1' requires = list(map(pkg_resources.Requirement.parse, specs)) for d in pkg_resources.find_distributions(self.tmpdir): assert d.requires() == requires[:1] - assert d.requires(extras=('baz',)) == requires + assert d.requires(extras=('baz',)) == [ + requires[0], + pkg_resources.Requirement.parse('quux>=1.1;extra=="baz"')] assert d.extras == ['baz'] metadata_template = DALS(""" diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index 00e16b63..55b8b05a 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -14,6 +14,10 @@ import tarfile import logging import itertools import distutils.errors +import io + +from setuptools.extern.six.moves import urllib +import time import pytest try: @@ -22,8 +26,6 @@ except ImportError: import mock from setuptools import sandbox -from setuptools import compat -from setuptools.compat import StringIO, BytesIO, urlparse from setuptools.sandbox import run_setup import setuptools.command.easy_install as ei from setuptools.command.easy_install import PthDistributions @@ -35,7 +37,7 @@ import setuptools.tests.server import pkg_resources from .py26compat import tarfile_open -from . import contexts, is_ascii +from . import contexts from .textwrap import DALS @@ -56,17 +58,13 @@ SETUP_PY = DALS(""" class TestEasyInstallTest: - def test_install_site_py(self): + def test_install_site_py(self, tmpdir): dist = Distribution() cmd = ei.easy_install(dist) cmd.sitepy_installed = False - cmd.install_dir = tempfile.mkdtemp() - try: - cmd.install_site_py() - sitepy = os.path.join(cmd.install_dir, 'site.py') - assert os.path.exists(sitepy) - finally: - shutil.rmtree(cmd.install_dir) + cmd.install_dir = str(tmpdir) + cmd.install_site_py() + assert (tmpdir / 'site.py').exists() def test_get_script_args(self): header = ei.CommandSpec.best().from_environment().as_header() @@ -272,7 +270,7 @@ class TestSetupRequires: p_index = setuptools.tests.server.MockServer() p_index.start() netloc = 1 - p_index_loc = urlparse(p_index.url)[netloc] + p_index_loc = urllib.parse.urlparse(p_index.url)[netloc] if p_index_loc.endswith(':0'): # Some platforms (Jython) don't find a port to which to bind, # so skip this test for them. @@ -308,32 +306,32 @@ class TestSetupRequires: """ with contexts.tempdir() as dir: dist_path = os.path.join(dir, 'setuptools-test-fetcher-1.0.tar.gz') - script = DALS(""" - import setuptools - setuptools.setup( - name="setuptools-test-fetcher", - version="1.0", - setup_requires = ['does-not-exist'], - ) - """) - make_trivial_sdist(dist_path, script) + make_sdist(dist_path, [ + ('setup.py', DALS(""" + import setuptools + setuptools.setup( + name="setuptools-test-fetcher", + version="1.0", + setup_requires = ['does-not-exist'], + ) + """))]) yield dist_path def test_setup_requires_overrides_version_conflict(self): """ - Regression test for issue #323. + Regression test for distribution issue 323: + https://bitbucket.org/tarek/distribute/issues/323 Ensures that a distribution's setup_requires requirements can still be installed and used locally even if a conflicting version of that requirement is already on the path. """ - pr_state = pkg_resources.__getstate__() fake_dist = PRDistribution('does-not-matter', project_name='foobar', version='0.0') working_set.add(fake_dist) - try: + with contexts.save_pkg_resources_state(): with contexts.tempdir() as temp_dir: test_pkg = create_setup_requires_package(temp_dir) test_setup_py = os.path.join(test_pkg, 'setup.py') @@ -345,19 +343,144 @@ class TestSetupRequires: lines = stdout.readlines() assert len(lines) > 0 assert lines[-1].strip(), 'test_pkg' - finally: - pkg_resources.__setstate__(pr_state) + def test_setup_requires_override_nspkg(self): + """ + Like ``test_setup_requires_overrides_version_conflict`` but where the + ``setup_requires`` package is part of a namespace package that has + *already* been imported. + """ + + with contexts.save_pkg_resources_state(): + with contexts.tempdir() as temp_dir: + foobar_1_archive = os.path.join(temp_dir, 'foo.bar-0.1.tar.gz') + make_nspkg_sdist(foobar_1_archive, 'foo.bar', '0.1') + # Now actually go ahead an extract to the temp dir and add the + # extracted path to sys.path so foo.bar v0.1 is importable + foobar_1_dir = os.path.join(temp_dir, 'foo.bar-0.1') + os.mkdir(foobar_1_dir) + with tarfile_open(foobar_1_archive) as tf: + tf.extractall(foobar_1_dir) + sys.path.insert(1, foobar_1_dir) + + dist = PRDistribution(foobar_1_dir, project_name='foo.bar', + version='0.1') + working_set.add(dist) + + template = DALS("""\ + import foo # Even with foo imported first the + # setup_requires package should override + import setuptools + setuptools.setup(**%r) + + if not (hasattr(foo, '__path__') and + len(foo.__path__) == 2): + print('FAIL') + + if 'foo.bar-0.2' not in foo.__path__[0]: + print('FAIL') + """) + + test_pkg = create_setup_requires_package( + temp_dir, 'foo.bar', '0.2', make_nspkg_sdist, template) + + test_setup_py = os.path.join(test_pkg, 'setup.py') + + with contexts.quiet() as (stdout, stderr): + try: + # Don't even need to install the package, just + # running the setup.py at all is sufficient + run_setup(test_setup_py, ['--name']) + except pkg_resources.VersionConflict: + self.fail('Installing setup.py requirements ' + 'caused a VersionConflict') + + assert 'FAIL' not in stdout.getvalue() + lines = stdout.readlines() + assert len(lines) > 0 + assert lines[-1].strip() == 'test_pkg' + + +def make_trivial_sdist(dist_path, distname, version): + """ + Create a simple sdist tarball at dist_path, containing just a simple + setup.py. + """ + + make_sdist(dist_path, [ + ('setup.py', + DALS("""\ + import setuptools + setuptools.setup( + name=%r, + version=%r + ) + """ % (distname, version)))]) + + +def make_nspkg_sdist(dist_path, distname, version): + """ + Make an sdist tarball with distname and version which also contains one + package with the same name as distname. The top-level package is + designated a namespace package). + """ + + parts = distname.split('.') + nspackage = parts[0] + + packages = ['.'.join(parts[:idx]) for idx in range(1, len(parts) + 1)] + + setup_py = DALS("""\ + import setuptools + setuptools.setup( + name=%r, + version=%r, + packages=%r, + namespace_packages=[%r] + ) + """ % (distname, version, packages, nspackage)) + + init = "__import__('pkg_resources').declare_namespace(__name__)" + + files = [('setup.py', setup_py), + (os.path.join(nspackage, '__init__.py'), init)] + for package in packages[1:]: + filename = os.path.join(*(package.split('.') + ['__init__.py'])) + files.append((filename, '')) + + make_sdist(dist_path, files) + + +def make_sdist(dist_path, files): + """ + Create a simple sdist tarball at dist_path, containing the files + listed in ``files`` as ``(filename, content)`` tuples. + """ -def create_setup_requires_package(path): + with tarfile_open(dist_path, 'w:gz') as dist: + for filename, content in files: + file_bytes = io.BytesIO(content.encode('utf-8')) + file_info = tarfile.TarInfo(name=filename) + file_info.size = len(file_bytes.getvalue()) + file_info.mtime = int(time.time()) + dist.addfile(file_info, fileobj=file_bytes) + + +def create_setup_requires_package(path, distname='foobar', version='0.1', + make_package=make_trivial_sdist, + setup_py_template=None): """Creates a source tree under path for a trivial test package that has a single requirement in setup_requires--a tarball for that requirement is also created and added to the dependency_links argument. + + ``distname`` and ``version`` refer to the name/version of the package that + the test package requires via ``setup_requires``. The name of the test + package itself is just 'test_pkg'. """ test_setup_attrs = { 'name': 'test_pkg', 'version': '0.0', - 'setup_requires': ['foobar==0.1'], + 'setup_requires': ['%s==%s' % (distname, version)], 'dependency_links': [os.path.abspath(path)] } @@ -365,22 +488,17 @@ def create_setup_requires_package(path): test_setup_py = os.path.join(test_pkg, 'setup.py') os.mkdir(test_pkg) - with open(test_setup_py, 'w') as f: - f.write(DALS(""" + if setup_py_template is None: + setup_py_template = DALS("""\ import setuptools setuptools.setup(**%r) - """ % test_setup_attrs)) + """) - foobar_path = os.path.join(path, 'foobar-0.1.tar.gz') - make_trivial_sdist( - foobar_path, - DALS(""" - import setuptools - setuptools.setup( - name='foobar', - version='0.1' - ) - """)) + with open(test_setup_py, 'w') as f: + f.write(setup_py_template % test_setup_attrs) + + foobar_path = os.path.join(path, '%s-%s.tar.gz' % (distname, version)) + make_package(foobar_path, distname, version) return test_pkg @@ -391,12 +509,7 @@ def make_trivial_sdist(dist_path, setup_py): """ setup_py_file = tarfile.TarInfo(name='setup.py') - try: - # Python 3 (StringIO gets converted to io module) - MemFile = BytesIO - except AttributeError: - MemFile = StringIO - setup_py_bytes = MemFile(setup_py.encode('utf-8')) + setup_py_bytes = io.BytesIO(setup_py.encode('utf-8')) setup_py_file.size = len(setup_py_bytes.getvalue()) with tarfile_open(dist_path, 'w:gz') as dist: dist.addfile(setup_py_file, fileobj=setup_py_bytes) @@ -430,46 +543,6 @@ class TestScriptHeader: expected = '#!"%s"\n' % self.exe_with_spaces assert actual == expected - @pytest.mark.xfail( - compat.PY3 and is_ascii, - reason="Test fails in this locale on Python 3" - ) - @mock.patch.dict(sys.modules, java=mock.Mock(lang=mock.Mock(System= - mock.Mock(getProperty=mock.Mock(return_value=""))))) - @mock.patch('sys.platform', 'java1.5.0_13') - def test_get_script_header_jython_workaround(self, tmpdir): - # Create a mock sys.executable that uses a shebang line - header = DALS(""" - #!/usr/bin/python - # -*- coding: utf-8 -*- - """) - exe = tmpdir / 'exe.py' - with exe.open('w') as f: - f.write(header) - exe = str(exe) - - header = ei.ScriptWriter.get_script_header('#!/usr/local/bin/python', - executable=exe) - assert header == '#!/usr/bin/env %s\n' % exe - - expect_out = 'stdout' if sys.version_info < (2,7) else 'stderr' - - with contexts.quiet() as (stdout, stderr): - # When options are included, generate a broken shebang line - # with a warning emitted - candidate = ei.ScriptWriter.get_script_header('#!/usr/bin/python -x', - executable=exe) - assert candidate == '#!%s -x\n' % exe - output = locals()[expect_out] - assert 'Unable to adapt shebang line' in output.getvalue() - - with contexts.quiet() as (stdout, stderr): - candidate = ei.ScriptWriter.get_script_header('#!/usr/bin/python', - executable=self.non_ascii_exe) - assert candidate == '#!%s -x\n' % self.non_ascii_exe - output = locals()[expect_out] - assert 'Unable to adapt shebang line' in output.getvalue() - class TestCommandSpec: def test_custom_launch_command(self): diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index 645c379c..d37567b4 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -1,9 +1,13 @@ import os +import glob import stat +from setuptools.extern.six.moves import map + import pytest from . import environment +from .files import build_files from .textwrap import DALS from . import contexts @@ -26,14 +30,13 @@ class TestEggInfo(object): """) def _create_project(self): - with open('setup.py', 'w') as f: - f.write(self.setup_script) - - with open('hello.py', 'w') as f: - f.write(DALS(""" + build_files({ + 'setup.py': self.setup_script, + 'hello.py': DALS(""" def run(): print('hello') - """)) + """) + }) @pytest.yield_fixture def env(self): @@ -46,36 +49,20 @@ class TestEggInfo(object): for dirname in subs ) list(map(os.mkdir, env.paths.values())) - config = os.path.join(env.paths['home'], '.pydistutils.cfg') - with open(config, 'w') as f: - f.write(DALS(""" + build_files({ + env.paths['home']: { + '.pydistutils.cfg': DALS(""" [egg_info] egg-base = %(egg-base)s - """ % env.paths)) + """ % env.paths) + } + }) yield env def test_egg_base_installed_egg_info(self, tmpdir_cwd, env): self._create_project() - environ = os.environ.copy().update( - HOME=env.paths['home'], - ) - cmd = [ - 'install', - '--home', env.paths['home'], - '--install-lib', env.paths['lib'], - '--install-scripts', env.paths['scripts'], - '--install-data', env.paths['data'], - ] - code, data = environment.run_setup_py( - cmd=cmd, - pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), - data_stream=1, - env=environ, - ) - if code: - raise AssertionError(data) - + self._run_install_command(tmpdir_cwd, env) actual = self._find_egg_info_files(env.paths['lib']) expected = [ @@ -88,9 +75,98 @@ class TestEggInfo(object): ] assert sorted(actual) == expected + def test_manifest_template_is_read(self, tmpdir_cwd, env): + self._create_project() + build_files({ + 'MANIFEST.in': DALS(""" + recursive-include docs *.rst + """), + 'docs': { + 'usage.rst': "Run 'hi'", + } + }) + self._run_install_command(tmpdir_cwd, env) + egg_info_dir = self._find_egg_info_files(env.paths['lib']).base + sources_txt = os.path.join(egg_info_dir, 'SOURCES.txt') + assert 'docs/usage.rst' in open(sources_txt).read().split('\n') + + def _setup_script_with_requires(self, requires_line): + setup_script = DALS(""" + from setuptools import setup + + setup( + name='foo', + %s + zip_safe=False, + ) + """ % requires_line) + build_files({ + 'setup.py': setup_script, + }) + + @pytest.mark.xfail(reason="Functionality disabled; see #523") + def test_install_requires_with_markers(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """install_requires=["barbazquux;python_version<'2'"],""") + self._run_install_command(tmpdir_cwd, env) + egg_info_dir = self._find_egg_info_files(env.paths['lib']).base + requires_txt = os.path.join(egg_info_dir, 'requires.txt') + assert "barbazquux;python_version<'2'" in open( + requires_txt).read().split('\n') + assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + + @pytest.mark.xfail(reason="Functionality disabled; see #523") + def test_setup_requires_with_markers(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """setup_requires=["barbazquux;python_version<'2'"],""") + self._run_install_command(tmpdir_cwd, env) + assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + + @pytest.mark.xfail(reason="Functionality disabled; see #523") + def test_tests_require_with_markers(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """tests_require=["barbazquux;python_version<'2'"],""") + self._run_install_command( + tmpdir_cwd, env, cmd=['test'], output="Ran 0 tests in") + assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + + def test_extra_requires_with_markers(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """extra_requires={":python_version<'2'": ["barbazquux"]},""") + self._run_install_command(tmpdir_cwd, env) + assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + + def _run_install_command(self, tmpdir_cwd, env, cmd=None, output=None): + environ = os.environ.copy().update( + HOME=env.paths['home'], + ) + if cmd is None: + cmd = [ + 'install', + '--home', env.paths['home'], + '--install-lib', env.paths['lib'], + '--install-scripts', env.paths['scripts'], + '--install-data', env.paths['data'], + ] + code, data = environment.run_setup_py( + cmd=cmd, + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + env=environ, + ) + if code: + raise AssertionError(data) + if output: + assert output in data + def _find_egg_info_files(self, root): + class DirList(list): + def __init__(self, files, base): + super(DirList, self).__init__(files) + self.base = base + results = ( - filenames + DirList(filenames, dirpath) for dirpath, dirnames, filenames in os.walk(root) if os.path.basename(dirpath) == 'EGG-INFO' ) diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py index 07f06db2..04772ba5 100644 --- a/setuptools/tests/test_integration.py +++ b/setuptools/tests/test_integration.py @@ -7,12 +7,12 @@ import glob import os import sys +from setuptools.extern.six.moves import urllib import pytest from setuptools.command.easy_install import easy_install from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution -from setuptools.compat import urlopen def setup_module(module): @@ -26,7 +26,7 @@ def setup_module(module): pass try: - urlopen('https://pypi.python.org/pypi') + urllib.request.urlopen('https://pypi.python.org/pypi') except Exception as exc: pytest.skip(str(exc)) diff --git a/setuptools/tests/test_markerlib.py b/setuptools/tests/test_markerlib.py deleted file mode 100644 index 8197b49d..00000000 --- a/setuptools/tests/test_markerlib.py +++ /dev/null @@ -1,63 +0,0 @@ -import os - -import pytest - - -class TestMarkerlib: - - @pytest.mark.importorskip('ast') - def test_markers(self): - from _markerlib import interpret, default_environment, compile - - os_name = os.name - - assert interpret("") - - assert interpret("os.name != 'buuuu'") - assert interpret("os_name != 'buuuu'") - assert interpret("python_version > '1.0'") - assert interpret("python_version < '5.0'") - assert interpret("python_version <= '5.0'") - assert interpret("python_version >= '1.0'") - assert interpret("'%s' in os.name" % os_name) - assert interpret("'%s' in os_name" % os_name) - assert interpret("'buuuu' not in os.name") - - assert not interpret("os.name == 'buuuu'") - assert not interpret("os_name == 'buuuu'") - assert not interpret("python_version < '1.0'") - assert not interpret("python_version > '5.0'") - assert not interpret("python_version >= '5.0'") - assert not interpret("python_version <= '1.0'") - assert not interpret("'%s' not in os.name" % os_name) - assert not interpret("'buuuu' in os.name and python_version >= '5.0'") - assert not interpret("'buuuu' in os_name and python_version >= '5.0'") - - environment = default_environment() - environment['extra'] = 'test' - assert interpret("extra == 'test'", environment) - assert not interpret("extra == 'doc'", environment) - - def raises_nameError(): - try: - interpret("python.version == '42'") - except NameError: - pass - else: - raise Exception("Expected NameError") - - raises_nameError() - - def raises_syntaxError(): - try: - interpret("(x for x in (4,))") - except SyntaxError: - pass - else: - raise Exception("Expected SyntaxError") - - raises_syntaxError() - - statement = "python_version == '5'" - assert compile(statement).__doc__ == statement - diff --git a/setuptools/tests/test_packageindex.py b/setuptools/tests/test_packageindex.py index 746860d5..6a76b5fc 100644 --- a/setuptools/tests/test_packageindex.py +++ b/setuptools/tests/test_packageindex.py @@ -4,9 +4,10 @@ import sys import os import distutils.errors -from setuptools.compat import httplib, HTTPError, unicode, pathname2url -from .textwrap import DALS +from setuptools.extern import six +from setuptools.extern.six.moves import urllib, http_client +from .textwrap import DALS import pkg_resources import setuptools.package_index from setuptools.tests.server import IndexServer @@ -22,7 +23,7 @@ class TestPackageIndex: except Exception as v: assert url in str(v) else: - assert isinstance(v, HTTPError) + assert isinstance(v, urllib.error.HTTPError) def test_bad_url_typo(self): # issue 16 @@ -38,7 +39,7 @@ class TestPackageIndex: except Exception as v: assert url in str(v) else: - assert isinstance(v, HTTPError) + assert isinstance(v, urllib.error.HTTPError) def test_bad_url_bad_status_line(self): index = setuptools.package_index.PackageIndex( @@ -46,7 +47,7 @@ class TestPackageIndex: ) def _urlopen(*args): - raise httplib.BadStatusLine('line') + raise http_client.BadStatusLine('line') index.opener = _urlopen url = 'http://example.com' @@ -70,7 +71,7 @@ class TestPackageIndex: try: index.open_url(url) except distutils.errors.DistutilsError as error: - msg = unicode(error) + msg = six.text_type(error) assert 'nonnumeric port' in msg or 'getaddrinfo failed' in msg or 'Name or service not known' in msg return raise RuntimeError("Did not raise") @@ -167,7 +168,7 @@ class TestPackageIndex: index_file = tmpdir / 'index.html' with index_file.open('w') as f: f.write('<div>content</div>') - url = 'file:' + pathname2url(str(tmpdir)) + '/' + url = 'file:' + urllib.request.pathname2url(str(tmpdir)) + '/' res = setuptools.package_index.local_open(url) assert 'content' in res.read() diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py index 8ec9a4cb..d2a1f1bb 100644 --- a/setuptools/tests/test_sdist.py +++ b/setuptools/tests/test_sdist.py @@ -9,17 +9,19 @@ import unicodedata import contextlib import io +from setuptools.extern import six +from setuptools.extern.six.moves import map + import pytest import pkg_resources -from setuptools.compat import StringIO, unicode, PY3, PY2 from setuptools.command.sdist import sdist from setuptools.command.egg_info import manifest_maker from setuptools.dist import Distribution from setuptools.tests import fail_on_ascii -py3_only = pytest.mark.xfail(PY2, reason="Test runs on Python 3 only") +py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only") SETUP_ATTRS = { @@ -37,7 +39,7 @@ setup(**%r) """ % SETUP_ATTRS -if PY3: +if six.PY3: LATIN1_FILENAME = 'smörbröd.py'.encode('latin-1') else: LATIN1_FILENAME = 'sm\xf6rbr\xf6d.py' @@ -47,7 +49,7 @@ else: @contextlib.contextmanager def quiet(): old_stdout, old_stderr = sys.stdout, sys.stderr - sys.stdout, sys.stderr = StringIO(), StringIO() + sys.stdout, sys.stderr = six.StringIO(), six.StringIO() try: yield finally: @@ -56,14 +58,14 @@ def quiet(): # Fake byte literals for Python <= 2.5 def b(s, encoding='utf-8'): - if PY3: + if six.PY3: return s.encode(encoding) return s # Convert to POSIX path def posix(path): - if PY3 and not isinstance(path, str): + if six.PY3 and not isinstance(path, str): return path.replace(os.sep.encode('ascii'), b('/')) else: return path.replace(os.sep, '/') @@ -71,7 +73,7 @@ def posix(path): # HFS Plus uses decomposed UTF-8 def decompose(path): - if isinstance(path, unicode): + if isinstance(path, six.text_type): return unicodedata.normalize('NFD', path) try: path = path.decode('utf-8') @@ -184,7 +186,7 @@ class TestSdistTest: u_contents = contents.decode('UTF-8') # The manifest should contain the UTF-8 filename - if PY2: + if six.PY2: fs_enc = sys.getfilesystemencoding() filename = filename.decode(fs_enc) @@ -289,7 +291,7 @@ class TestSdistTest: cmd.read_manifest() # The filelist should contain the UTF-8 filename - if PY3: + if six.PY3: filename = filename.decode('utf-8') assert filename in cmd.filelist.files @@ -342,7 +344,7 @@ class TestSdistTest: if sys.platform == 'darwin': filename = decompose(filename) - if PY3: + if six.PY3: fs_enc = sys.getfilesystemencoding() if sys.platform == 'win32': @@ -374,7 +376,7 @@ class TestSdistTest: with quiet(): cmd.run() - if PY3: + if six.PY3: # not all windows systems have a default FS encoding of cp1252 if sys.platform == 'win32': # Latin-1 is similar to Windows-1252 however diff --git a/setuptools/tests/test_setuptools.py b/setuptools/tests/test_setuptools.py index 8aca593a..e59800d2 100644 --- a/setuptools/tests/test_setuptools.py +++ b/setuptools/tests/test_setuptools.py @@ -23,7 +23,7 @@ def test_findall(example_source): def test_findall_curdir(example_source): with example_source.as_cwd(): found = list(setuptools.findall()) - expected = ['readme.txt', 'foo/bar.py'] + expected = ['readme.txt', os.path.join('foo', 'bar.py')] assert found == expected diff --git a/setuptools/tests/test_test.py b/setuptools/tests/test_test.py index a66294c9..4155a5b1 100644 --- a/setuptools/tests/test_test.py +++ b/setuptools/tests/test_test.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import os import site +from distutils.errors import DistutilsError import pytest diff --git a/setuptools/tests/test_unicode_utils.py b/setuptools/tests/test_unicode_utils.py new file mode 100644 index 00000000..a24a9bd5 --- /dev/null +++ b/setuptools/tests/test_unicode_utils.py @@ -0,0 +1,10 @@ +from setuptools import unicode_utils + + +def test_filesys_decode_fs_encoding_is_None(monkeypatch): + """ + Test filesys_decode does not raise TypeError when + getfilesystemencoding returns None. + """ + monkeypatch.setattr('sys.getfilesystemencoding', lambda: None) + unicode_utils.filesys_decode(b'test') diff --git a/setuptools/unicode_utils.py b/setuptools/unicode_utils.py index d2de941a..ffab3e24 100644 --- a/setuptools/unicode_utils.py +++ b/setuptools/unicode_utils.py @@ -1,11 +1,11 @@ import unicodedata import sys -from setuptools.compat import unicode as decoded_string +from setuptools.extern import six # HFS Plus uses decomposed UTF-8 def decompose(path): - if isinstance(path, decoded_string): + if isinstance(path, six.text_type): return unicodedata.normalize('NFD', path) try: path = path.decode('utf-8') @@ -22,11 +22,13 @@ def filesys_decode(path): NONE when no expected encoding works """ - fs_enc = sys.getfilesystemencoding() - if isinstance(path, decoded_string): + if isinstance(path, six.text_type): return path - for enc in (fs_enc, "utf-8"): + fs_enc = sys.getfilesystemencoding() or 'utf-8' + candidates = fs_enc, 'utf-8' + + for enc in candidates: try: return path.decode(enc) except UnicodeDecodeError: diff --git a/tests/manual_test.py b/tests/manual_test.py index e6489b1c..808fa55a 100644 --- a/tests/manual_test.py +++ b/tests/manual_test.py @@ -7,7 +7,8 @@ import tempfile import subprocess from distutils.command.install import INSTALL_SCHEMES from string import Template -from setuptools.compat import urlopen + +from six.moves import urllib def _system_call(*args): assert subprocess.call(args) == 0 @@ -42,10 +43,8 @@ PYVER = sys.version.split()[0][:3] _VARS = {'base': '.', 'py_version_short': PYVER} -if sys.platform == 'win32': - PURELIB = INSTALL_SCHEMES['nt']['purelib'] -else: - PURELIB = INSTALL_SCHEMES['unix_prefix']['purelib'] +scheme = 'nt' if sys.platform == 'win32' else 'unix_prefix' +PURELIB = INSTALL_SCHEMES[scheme]['purelib'] @tempdir @@ -76,7 +75,7 @@ def test_full(): f.write(SIMPLE_BUILDOUT) with open('bootstrap.py', 'w') as f: - f.write(urlopen(BOOTSTRAP).read()) + f.write(urllib.request.urlopen(BOOTSTRAP).read()) _system_call('bin/python', 'bootstrap.py') _system_call('bin/buildout', '-q') |