diff options
| author | Jason R. Coombs <jaraco@jaraco.com> | 2016-05-18 19:57:23 -0400 |
|---|---|---|
| committer | Jason R. Coombs <jaraco@jaraco.com> | 2016-05-18 19:57:23 -0400 |
| commit | c98abcb08a80ea84a6350f8901a4a3ea709c15f0 (patch) | |
| tree | 97972a5ebb507f2ea73681f1e6b1cde3542b542e | |
| parent | 1b66f61563f7835d7b5a29dc5413bd482ef4910d (diff) | |
| parent | 41d0279e41eaa5bc948bd7d6b815212e8d99e1fc (diff) | |
| download | python-setuptools-git-c98abcb08a80ea84a6350f8901a4a3ea709c15f0.tar.gz | |
Merge with master to fix failing tests
54 files changed, 5243 insertions, 1510 deletions
@@ -8,7 +8,6 @@ distribute.egg-info setuptools.egg-info .coverage .tox -CHANGES (links).txt *.egg *.py[cod] *.swp @@ -8,7 +8,6 @@ distribute.egg-info setuptools.egg-info .coverage .tox -CHANGES (links).txt *.egg *.py[cod] *.swp @@ -1,56 +1,44 @@ -1010d08fd8dfd2f1496843b557b5369a0beba82a 0.6 -4d114c5f2a3ecb4a0fa552075dbbb221b19e291b 0.6.1 -41415244ee90664042d277d0b1f0f59c04ddd0e4 0.6.2 -e033bf2d3d05f4a7130f5f8f5de152c4db9ff32e 0.6.3 -e06c416e911c61771708f5afbf3f35db0e12ba71 0.6.4 -2df182df8a0224d429402de3cddccdb97af6ea21 0.6.5 -f1fb564d6d67a6340ff33df2f5a74b89753f159d 0.6.6 -71f08668d050589b92ecd164a4f5a91f3484313b 0.6.7 -445547a5729ed5517cf1a9baad595420a8831ef8 0.6.8 -669ed9388b17ec461380cc41760a9a7384fb5284 0.6.9 -669ed9388b17ec461380cc41760a9a7384fb5284 0.6.9 -ac7d9b14ac43fecb8b65de548b25773553facaee 0.6.9 -0fd5c506037880409308f2b79c6e901d21e7fe92 0.6.10 -0fd5c506037880409308f2b79c6e901d21e7fe92 0.6.10 -f18396c6e1875476279d8bbffd8e6dadcc695136 0.6.10 -e00987890c0b386f09d0f6b73d8558b72f6367f1 0.6.11 -48a97bc89e2f65fc9b78b358d7dc89ba9ec9524a 0.6.12 -dae247400d0ca1fdfaf38db275622c9bec550b08 0.6.13 -2b9d9977ea75b8eb3766bab808ef31f192d2b1bc 0.6.14 -51a9d1a1f31a4be3107d06cf088aff8e182dc633 0.6.15 -3f1ff138e947bfc1c9bcfe0037030b7bfb4ab3a5 0.6.16 -9c40f23d0bda3f3f169686e27a422f853fa4d0fa 0.6.17 -9c40f23d0bda3f3f169686e27a422f853fa4d0fa 0.6.17 -4bbc01e4709ea7425cf0c186bbaf1d928cfa2a65 0.6.17 -4bbc01e4709ea7425cf0c186bbaf1d928cfa2a65 0.6.17 -0502d5117d8304ab21084912758ed28812a5a8f1 0.6.17 -74108d7f07343556a8db94e8122221a43243f586 0.6.18 -611910892a0421633d72677979f94a25ef590d54 0.6.19 -a7cf5ae137f1646adf86ce5d6b5d8b7bd6eab69f 0.6.20 -c4a375336d552129aef174486018ed09c212d684 0.6.20 -de44acab3cfce1f5bc811d6c0fa1a88ca0e9533f 0.6.21 -1a1ab844f03e10528ae693ad3cb45064e08f49e5 0.6.23 -1a1ab844f03e10528ae693ad3cb45064e08f49e5 0.6.23 -9406c5dac8429216f1a264e6f692fdc534476acd 0.6.23 -7fd7b6e30a0effa082baed1c4103a0efa56be98c 0.6.24 -6124053afb5c98f11e146ae62049b4c232d50dc5 0.6.25 -b69f072c000237435e17b8bbb304ba6f957283eb 0.6.26 -469c3b948e41ef28752b3cdf3c7fb9618355ebf5 0.6.27 -fc379e63586ad3c6838e1bda216548ba8270b8f0 0.6.28 -4f82563d0f5d1af1fb215c0ac87f38b16bb5c42d 0.6.29 -7464fc916fa4d8308e34e45a1198512fe04c97b4 0.6.30 -17bc972d67edd96c7748061910172e1200a73efe 0.6.31 -b1a7f86b315a1f8c20036d718d6dc641bb84cac6 0.6.32 -6acac3919ae9a7dba2cbecbe3d4b31ece25d5f09 0.6.33 -23c310bf4ae8e4616e37027f08891702f5a33bc9 0.6.34 -2abe1117543be0edbafb10c7c159d1bcb1cb1b87 0.6.35 -c813a29e831f266d427d4a4bce3da97f475a8eee 0.6.36 -be6f65eea9c10ce78b6698d8c220b6e5de577292 0.6.37 -2b26ec8909bff210f47c5f8fc620bc505e1610b5 0.6.37 -f0d502a83f6c83ba38ad21c15a849c2daf389ec7 0.6.38 -d737b2039c5f92af8000f78bbc80b6a5183caa97 0.6.39 +7e9441311eb21dd1fbc32cfbad58168e46c5450e 0.6 +26f429772565f69d1f6d21adf57c3d8c40197129 0.6.1 +6f46749a7454be6e044a54cd73c51318b74bdee8 0.6.2 +34b80fb58862d18f8f957f98a883ed4a72d06f8e 0.6.3 +fb04abddb50d82a9005c9082c94d5eb983be1d79 0.6.4 +8ae0bd250b4a0d58cbaf16b4354ad60f73f24a01 0.6.5 +88847883dfed39829d3a5ed292ad540723ad31cc 0.6.6 +fcbef325349ada38f6c674eb92db82664cf6437c 0.6.7 +3af7f2b8270b9bb34fb65f08ee567bfe8e2a6a5a 0.6.8 +669725d03fd1e345ea47590e9b14cb19742b96a2 0.6.9 +eff3ca9c2d8d39e24c221816c52a37f964535336 0.6.10 +88710e34b91c98c9348749722cce3acd574d177d 0.6.11 +5ce754773a43ac21f7bd13872f45c75e27b593f8 0.6.12 +de36566d35e51bee7cfc86ffa694795e52f4147c 0.6.13 +e5f3f0ffe9e1a243d49a06f26c79dd160f521483 0.6.14 +dc03a300ec7a89ad773047172d43e52b34e7cd1e 0.6.15 +e620fb4ee8ba17debadb614fb583c6dfac229dea 0.6.16 +21df276275b5a47c6a994927d69ad3d90cf62b5d 0.6.17 +e9264ca4ba8c24239c36a8426a0394f7c7d5dd83 0.6.18 +aed31b1fa47ed1f39e55c75b76bbbdb80775b7f1 0.6.19 +c6e6273587816c3e486ef7739e53c864a0145251 0.6.20 +7afdf4c84a713fe151e6163ab25d45e8727ce653 0.6.21 +105066342777cd1319a95d7ae0271a2ea1ac33fe 0.6.23 +7b5ef4e6c80e82541dffb5a9a130d81550d5a835 0.6.24 +9c014a80f32e532371826ed1dc3236975f37f371 0.6.25 +ff8c4d6c8e5d2093750a58a3d43b76556570007c 0.6.26 +2a5c42ed097a195e398b97261c40cd66c8da8913 0.6.27 +4ed34b38851f90278cfe2bff75784f7e32883725 0.6.28 +acecfa2cfb6fca207dd2f4e025c695def3bb6b40 0.6.29 +e950f50addff150859f5990b9df2a33c691b6354 0.6.30 +06dae3faee2de50ff17b90719df410b2ebc5b71e 0.6.31 +1f4f79258ed5b418f680a55d3006f41aa6a56d2b 0.6.32 +89f57bf1406a5e745470af35446902c21ac9b6f6 0.6.33 +3c8f9fc13862124cf20ef2ff2140254fb272bb94 0.6.34 +7c3f8b9eb7cfa17481c835d5caaa918d337c7a83 0.6.35 +192094c0d1e2e5d2cb5c718f84a36c9de04b314b 0.6.36 +66d4e3b8899166e4c04189ee1831c649b7ff38bf 0.6.37 +398d58aa8bba33778c30ce72055a27d4b425809c 0.6.38 +f457fc2a3ebe609d8ca7a869eb65b7506ecf49ef 0.6.39 9b2e2aa06e058c63e06c5e42a7f279ddae2dfb7d 0.7b1 -0a783fa0dceb95b5fc743e47c2d89c1523d0afb7 0.6.40 +9089a40343981baa593b9bb5953f9088e9507099 0.6.40 ad107e9b4beea24516ac4e1e854696e586fe279d 0.6.41 f30167716b659f96c5e0b7ea3d5be2bcff8c0eac 0.6.42 8951daac6c1bc7b24c7fb054fd369f2c5b88cdb3 0.7b2 @@ -246,3 +234,29 @@ c6e619ce910d1650cc2433f94e5594964085f973 19.7 2a60daeff0cdb039b20b2058aaad7dae7bcd2c1c 20.0 06c9d3ffae80d7f5786c0a454d040d253d47fc03 20.1 919a40f1843131249f98104c73f3aee3fc835e67 20.1.1 +74c4ffbe1f399345eb4f6a64785cfff54f7e6e7e 20.2 +1aacb05fbdfe06cee904e7a138a4aa6df7b88a63 20.2.1 +48aa5271ef1cd5379cf91a1c958e490692b978e7 20.2.2 +9c55a3a1268a33b4a57b96b2b9fa2cd0701780ee 20.3 +3e87e975a95c780eec497ef9e5a742f7adfb77ec 20.3.1 +06692c64fb9b5843331a918ab7093f151412ec8e 20.4 +f8174392e9e9c6a21ea5df0f22cb4ca885c799ca 20.5 +114f3dbc8a73dacbce2ebe08bb70ca76ab18390e v20.6.0 +a3d4006688fe5e754d0e709a52a00b8191819979 v20.6.1 +2831509712601a78fddf46e51d6f41ae0f92bd0e v20.6.2 +8b46dc41cb234c435b950a879214a6dee54c9dd2 v20.6.3 +7258be20fe93bbf936dc1a81ce71c04c5880663e v20.6.4 +7e0ab283db4e6f780777f7f06af475f044631fa1 v20.6.5 +57d63b38e85515d06e06d3cea62e35e6c54b5093 v20.6.6 +57d63b38e85515d06e06d3cea62e35e6c54b5093 v20.6.6 +b04dbdd161d7f68903a53e1dbd1fa5b5fde73f94 v20.6.6 +0804d30b6ead64e0e324aefd67439b84df2d1c01 v20.6.7 +a00910db03ec15865e4c8506820d4ad1df3e26f3 v20.6.8 +0262ab29fc2417b502a55f49b7fd43528fbd3df4 v20.7.0 +7f56b6f40de39456c78507a14c288709712881cb v20.8.0 +8cf9340669ae26e2b31f68b9c3f885ab7bdd65ce v20.8.1 +8bf8aaa139bb6a36fcd243214d6730a214ae08f5 v20.9.0 +c72faa468919fd2f226c97e94d4e64a6506860e5 v20.10.0 +3b5fdd077c7d83d02c4979ad69cc0bf199b47587 v20.10.1 +ddd3f81eb9e0860bf95c380c50a72c52a215231f v21.0.0 +018e4a727cf691d6404cd24ffb25e8eebea2fad4 v20.6.8 diff --git a/.travis.yml b/.travis.yml index ae14639a..feeb039f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,6 @@ python: - 3.5 - pypy - pypy3 -matrix: - allow_failures: - - python: pypy3 env: - "" - LC_ALL=C LC_CTYPE=C @@ -25,5 +22,16 @@ script: - python setup.py test --addopts='-rs' - # test the bootstrap script - - python ez_setup.py +before_deploy: + - export SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES=1 +deploy: + provider: pypi + on: + tags: true + all_branches: true + python: 3.5 + condition: $LC_ALL != "C" + user: jaraco + password: + secure: tfWrsQMH2bHrWjqnP+08IX1WlkbW94Q30f4d7lCyhWS1FIf/jBDx4jrEILNfMxQ1NCwuBRje5sihj1Ow0BFf0vVrkaeff2IdvnNDEGFduMejaEQJL3s3QrLfpiAvUbtqwyWaHfAdGfk48PovDKTx0ZTvXZKYGXZhxGCYSlG2CE6Y6RDvnEl6Tk8e+LqUohkcSOwxrRwUoyxSnUaavdGohXxDT8MJlfWOXgr2u+KsRrriZqp3l6Fdsnk4IGvy6pXpy42L1HYQyyVu9XyJilR2JTbC6eCp5f8p26093m1Qas49+t6vYb0VLqQe12dO+Jm3v4uztSS5pPQzS7PFyjEYd2Rdb6ijsdbsy1074S4q7G9Sz+T3RsPUwYEJ07lzez8cxP64dtj5j94RL8m35A1Fb1OE8hHN+4c1yLG1gudfXbem+fUhi2eqhJrzQo5vsvDv1xS5x5GIS5ZHgKHCsWcW1Tv+dsFkrhaup3uU6VkOuc9UN+7VPsGEY7NvquGpTm8O1CnGJRzuJg6nbYRGj8ORwDpI0KmrExx6akV92P72fMC/I5TCgbSQSZn370H3Jj40gz1SM30WAli9M+wFHFd4ddMVY65yxj0NLmrP+m1tvnWdKtNh/RHuoW92d9/UFtiA5IhMf1/3djfsjBq6S9NT1uaLkVkTttqrPYJ7hOql8+g= + distributions: release diff --git a/CHANGES.txt b/CHANGES.rst index 9209a1db..eeb9a02c 100644 --- a/CHANGES.txt +++ b/CHANGES.rst @@ -8,6 +8,157 @@ Next * Pull Request #174: Add more aggressive support for Windows SDK in msvc9compiler patch. +v21.0.0 +------- + +* Removed ez_setup.py from Setuptools sdist. The + bootstrap script will be maintained in its own + branch and should be generally be retrieved from + its canonical location at + https://bootstrap.pypa.io/ez_setup.py. + +v20.10.0 +-------- + +* #553: egg_info section is now generated in a + deterministic order, matching the order generated + by earlier versions of Python. Except on Python 2.6, + order is preserved when existing settings are present. +* #556: Update to Packaging 16.7, restoring support + for deprecated ``python_implmentation`` marker. +* #555: Upload command now prompts for a password + when uploading to PyPI (or other repository) if no + password is present in .pypirc or in the keyring. + +v20.9.0 +------- + +* #548: Update certify version to 2016.2.28 +* #545: Safely handle deletion of non-zip eggs in rotate + command. + +v20.8.1 +------- + +* Issue #544: Fix issue with extra environment marker + processing in WorkingSet due to refactor in v20.7.0. + +v20.8.0 +------- + +* Issue #543: Re-release so that latest release doesn't + cause déjà vu with distribute and setuptools 0.7 in + older environments. + +v20.7.0 +------- + +* Refactored extra enviroment marker processing + in WorkingSet. +* Issue #533: Fixed intermittent test failures. +* Issue #536: In msvc9_support, trap additional exceptions + that might occur when importing + ``distutils.msvc9compiler`` in mingw environments. +* Issue #537: Provide better context when package + metadata fails to decode in UTF-8. + +v20.6.8 +------- + +* Issue #523: Restored support for environment markers, + now honoring 'extra' environment markers. + +v20.6.7 +------- + +* Issue #523: Disabled support for environment markers + introduced in v20.5. + +v20.6.6 +------- + +* Issue #503: Restore support for PEP 345 environment + markers by updating to Packaging 16.6. + +v20.6.0 +------- + +* New release process that relies on + `bumpversion <https://github.com/peritus/bumpversion>`_ + and Travis CI for continuous deployment. +* Project versioning semantics now follow + `semver <https://semver.org>`_ precisely. + The 'v' prefix on version numbers now also allows + version numbers to be referenced in the changelog, + e.g. https://pythonhosted.org/setuptools/history.html#v20-6-0. + +20.5 +---- + +* BB Pull Request #185: Add support for environment markers + in requirements in install_requires, setup_requires, + tests_require as well as adding a test for the existing + extra_requires machinery. + +20.4 +---- + +* Issue #422: Moved hosting to + `Github <https://github.com/pypa/setuptools>`_ + from `Bitbucket <https://bitbucket.org/pypa/setuptools>`_. + Issues have been migrated, though all issues and comments + are attributed to bb-migration. So if you have a particular + issue or issues to which you've been subscribed, you will + want to "watch" the equivalent issue in Github. + The Bitbucket project will be retained for the indefinite + future, but Github now hosts the canonical project repository. + +20.3.1 +------ + +* Issue #519: Remove import hook when reloading the + ``pkg_resources`` module. +* BB Pull Request #184: Update documentation in ``pkg_resources`` + around new ``Requirement`` implementation. + +20.3 +---- + +* BB Pull Request #179: ``pkg_resources.Requirement`` objects are + now a subclass of ``packaging.requirements.Requirement``, + allowing any environment markers and url (if any) to be + affiliated with the requirement +* BB Pull Request #179: Restore use of RequirementParseError + exception unintentionally dropped in 20.2. + +20.2.2 +------ + +* Issue #502: Correct regression in parsing of multiple + version specifiers separated by commas and spaces. + +20.2.1 +------ + +* Issue #499: Restore compatiblity for legacy versions + by bumping to packaging 16.4. + +20.2 +---- + +* Changelog now includes release dates and links to PEPs. +* BB Pull Request #173: Replace dual PEP 345 _markerlib implementation + and PEP 426 implementation of environment marker support from + packaging 16.1 and PEP 508. Fixes Issue #122. + See also BB Pull Request #175, BB Pull Request #168, and + BB Pull Request #164. Additionally: + + - ``Requirement.parse`` no longer retains the order of extras. + - ``parse_requirements`` now requires that all versions be + PEP-440 compliant, as revealed in #499. Packages released + with invalid local versions should be re-released using + the proper local version syntax, e.g. ``mypkg-1.0+myorg.1``. + 20.1.1 ------ @@ -76,7 +227,7 @@ Next * Issue #486: Correct TypeError when getfilesystemencoding returns None. * Issue #139: Clarified the license as MIT. -* Pull Request #169: Removed special handling of command +* BB Pull Request #169: Removed special handling of command spec in scripts for Jython. 19.4.1 @@ -92,7 +243,7 @@ Next * Issue #341: Correct error in path handling of package data files in ``build_py`` command when package is empty. * Distribute #323, Issue #141, Issue #207, and - Pull Request #167: Another implementation of + BB Pull Request #167: Another implementation of ``pkg_resources.WorkingSet`` and ``pkg_resources.Distribution`` that supports replacing an extant package with a new one, allowing for setup_requires dependencies to supersede installed @@ -108,8 +259,8 @@ Next 19.2 ---- -* Pull Request #163: Add get_command_list method to Distribution. -* Pull Request #162: Add missing whitespace to multiline string +* BB Pull Request #163: Add get_command_list method to Distribution. +* BB Pull Request #162: Add missing whitespace to multiline string literals. 19.1.1 @@ -165,16 +316,15 @@ Next ---- * Update dependency on certify. -* Pull Request #160: Improve detection of gui script in +* BB Pull Request #160: Improve detection of gui script in ``easy_install._adjust_header``. * Made ``test.test_args`` a non-data property; alternate fix - for the issue reported in Pull Request #155. + for the issue reported in BB Pull Request #155. * Issue #453: In ``ez_setup`` bootstrap module, unload all ``pkg_resources`` modules following download. -* Pull Request #158: Honor `PEP-488 - <https://www.python.org/dev/peps/pep-0488/>`_ when excluding +* BB Pull Request #158: Honor PEP-488 when excluding files for namespace packages. -* Issue #419 and Pull Request #144: Add experimental support for +* Issue #419 and BB Pull Request #144: Add experimental support for reading the version info from distutils-installed metadata rather than using the version in the filename. @@ -276,7 +426,7 @@ Next However, for systems with this build of setuptools, Cython will be downloaded on demand. * Issue #396: Fixed test failure on OS X. -* Pull Request #136: Remove excessive quoting from shebang headers +* BB Pull Request #136: Remove excessive quoting from shebang headers for Jython. 17.1.1 @@ -303,11 +453,11 @@ Next 16.0 ---- -* Pull Request #130: Better error messages for errors in +* BB Pull Request #130: Better error messages for errors in parsed requirements. -* Pull Request #133: Removed ``setuptools.tests`` from the +* BB Pull Request #133: Removed ``setuptools.tests`` from the installed packages. -* Pull Request #129: Address deprecation warning due to usage +* BB Pull Request #129: Address deprecation warning due to usage of imp module. 15.2 @@ -326,7 +476,7 @@ Next 15.0 ---- -* Pull Request #126: DistributionNotFound message now lists the package or +* BB Pull Request #126: DistributionNotFound message now lists the package or packages that required it. E.g.:: pkg_resources.DistributionNotFound: The 'colorama>=0.3.1' distribution was not found and is required by smlib.log. @@ -367,7 +517,7 @@ Next 14.1 ---- -* Pull Request #125: Add ``__ne__`` to Requirement class. +* BB Pull Request #125: Add ``__ne__`` to Requirement class. * Various refactoring of easy_install. 14.0 @@ -375,7 +525,7 @@ Next * Bootstrap script now accepts ``--to-dir`` to customize save directory or allow for re-use of existing repository of setuptools versions. See - Pull Request #112 for background. + BB Pull Request #112 for background. * Issue #285: ``easy_install`` no longer will default to installing packages to the "user site packages" directory if it is itself installed there. Instead, the user must pass ``--user`` in all cases to install @@ -400,7 +550,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. 13.0 ---- -* Issue #356: Back out Pull Request #119 as it requires Setuptools 10 or later +* Issue #356: Back out BB Pull Request #119 as it requires Setuptools 10 or later as the source during an upgrade. * Removed build_py class from setup.py. According to 892f439d216e, this functionality was added to support upgrades from old Distribute versions, @@ -409,7 +559,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. 12.4 ---- -* Pull Request #119: Restore writing of ``setup_requires`` to metadata +* BB Pull Request #119: Restore writing of ``setup_requires`` to metadata (previously added in 8.4 and removed in 9.0). 12.3 @@ -434,7 +584,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. 12.1 ---- -* Pull Request #118: Soften warning for non-normalized versions in +* BB Pull Request #118: Soften warning for non-normalized versions in Distribution. 12.0.5 @@ -575,7 +725,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. 8.4 --- -* Pull Request #106: Now write ``setup_requires`` metadata. +* BB Pull Request #106: Now write ``setup_requires`` metadata. 8.3 --- @@ -592,7 +742,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. 8.2 --- -* Pull Request #85: Search egg-base when adding egg-info to manifest. +* BB Pull Request #85: Search egg-base when adding egg-info to manifest. 8.1 --- @@ -632,7 +782,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. 8.0 --- -* Implement `PEP 440 <http://legacy.python.org/dev/peps/pep-0440/>`_ within +* Implement PEP 440 within pkg_resources and setuptools. This change deprecates some version numbers such that they will no longer be installable without using the ``===`` escape hatch. See `the changes to test_resources @@ -695,9 +845,9 @@ process to fail and PyPI uploads no longer accept files for 13.0. Any users producing distributions with filenames that match those above case-insensitively, but not case-sensitively, should rename those files in their repository for better portability. -* Pull Request #72: When using ``single_version_externally_managed``, the +* BB Pull Request #72: When using ``single_version_externally_managed``, the exclusion list now includes Python 3.2 ``__pycache__`` entries. -* Pull Request #76 and Pull Request #78: lines in top_level.txt are now +* BB Pull Request #76 and BB Pull Request #78: lines in top_level.txt are now ordered deterministically. * Issue #118: The egg-info directory is now no longer included in the list of outputs. @@ -896,7 +1046,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. 3.2 --- -* Pull Request #39: Add support for C++ targets from Cython ``.pyx`` files. +* BB Pull Request #39: Add support for C++ targets from Cython ``.pyx`` files. * Issue #162: Update dependency on certifi to 1.0.1. * Issue #164: Update dependency on wincertstore to 0.2. @@ -939,7 +1089,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. security vulnerabilities presented by use of tar archives in ez_setup.py. It also leverages the security features added to ZipFile.extract in Python 2.7.4. * Issue #65: Removed deprecated Features functionality. -* Pull Request #28: Remove backport of ``_bytecode_filenames`` which is +* BB Pull Request #28: Remove backport of ``_bytecode_filenames`` which is available in Python 2.6 and later, but also has better compatibility with Python 3 environments. * Issue #156: Fix spelling of __PYVENV_LAUNCHER__ variable. @@ -1035,7 +1185,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Issue #27: ``easy_install`` will now use credentials from .pypirc if present for connecting to the package index. -* Pull Request #21: Omit unwanted newlines in ``package_index._encode_auth`` +* BB Pull Request #21: Omit unwanted newlines in ``package_index._encode_auth`` when the username/password pair length indicates wrapping. 1.3.2 @@ -1377,7 +1527,7 @@ Added several features that were slated for setuptools 0.6c12: 0.6.36 ------ -* Pull Request #35: In Buildout #64, it was reported that +* BB Pull Request #35: In Buildout #64, it was reported that under Python 3, installation of distutils scripts could attempt to copy the ``__pycache__`` directory as a file, causing an error, apparently only under Windows. Easy_install now skips all directories when processing @@ -1452,7 +1602,7 @@ how it parses version numbers. 0.6.29 ------ -* Pull Request #14: Honor file permissions in zip files. +* BB Pull Request #14: Honor file permissions in zip files. * Distribute #327: Merged pull request #24 to fix a dependency problem with pip. * Merged pull request #23 to fix https://github.com/pypa/virtualenv/issues/301. * If Sphinx is installed, the `upload_docs` command now runs `build_sphinx` @@ -1705,7 +1855,7 @@ how it parses version numbers. * Distribute #65: cli.exe and gui.exe are now generated at build time, depending on the platform in use. -* Distribute #67: Fixed doc typo (PEP 381/382) +* Distribute #67: Fixed doc typo (PEP 381/PEP 382). * Distribute no longer shadows setuptools if we require a 0.7-series setuptools. And an error is raised when installing a 0.7 setuptools with diff --git a/MANIFEST.in b/MANIFEST.in index dfea2049..cfd1b001 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,11 +2,10 @@ recursive-include setuptools *.py *.exe *.xml recursive-include tests *.py recursive-include setuptools/tests *.html recursive-include docs *.py *.txt *.conf *.css *.css_t Makefile indexsidebar.html -recursive-include _markerlib *.py recursive-include setuptools/_vendor * recursive-include pkg_resources *.py *.txt include *.py -include *.txt +include *.rst include MANIFEST.in include launcher.c include msvc-build-launcher.cmd diff --git a/Makefile b/Makefile deleted file mode 100644 index 353b987e..00000000 --- a/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -empty: - exit 1 - -update-vendored: - rm -rf pkg_resources/_vendor/packaging* - rm -rf pkg_resources/_vendor/six* - python3 -m pip install -r pkg_resources/_vendor/vendored.txt -t pkg_resources/_vendor/ - rm -rf pkg_resources/_vendor/*.{egg,dist}-info @@ -18,7 +18,7 @@ basic routine, so below are some examples to get you started. Setuptools requires Python 2.6 or later. To install setuptools on Python 2.4 or Python 2.5, use the `bootstrap script for Setuptools 1.x -<https://bitbucket.org/pypa/setuptools/raw/bootstrap-py24/ez_setup.py>`_. +<https://raw.githubusercontent.com/pypa/setuptools/bootstrap-py24/ez_setup.py>`_. The link provided to ez_setup.py is a bookmark to bootstrap script for the latest known stable release. @@ -176,7 +176,7 @@ them there, so this reference list can be updated. If you have working, *tested* patches to correct problems or add features, you may submit them to the `setuptools bug tracker`_. -.. _setuptools bug tracker: https://bitbucket.org/pypa/setuptools/issues +.. _setuptools bug tracker: https://github.com/pypa/setuptools/issues .. _The Internal Structure of Python Eggs: https://pythonhosted.org/setuptools/formats.html .. _The setuptools Developer's Guide: https://pythonhosted.org/setuptools/setuptools.html .. _The pkg_resources API reference: https://pythonhosted.org/setuptools/pkg_resources.html diff --git a/_markerlib/__init__.py b/_markerlib/__init__.py deleted file mode 100644 index e2b237b1..00000000 --- a/_markerlib/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -try: - import ast - from _markerlib.markers import default_environment, compile, interpret -except ImportError: - if 'ast' in globals(): - raise - def default_environment(): - return {} - def compile(marker): - def marker_fn(environment=None, override=None): - # 'empty markers are True' heuristic won't install extra deps. - return not marker.strip() - marker_fn.__doc__ = marker - return marker_fn - def interpret(marker, environment=None, override=None): - return compile(marker)() diff --git a/_markerlib/markers.py b/_markerlib/markers.py deleted file mode 100644 index fa837061..00000000 --- a/_markerlib/markers.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- -"""Interpret PEP 345 environment markers. - -EXPR [in|==|!=|not in] EXPR [or|and] ... - -where EXPR belongs to any of those: - - python_version = '%s.%s' % (sys.version_info[0], sys.version_info[1]) - python_full_version = sys.version.split()[0] - os.name = os.name - sys.platform = sys.platform - platform.version = platform.version() - platform.machine = platform.machine() - platform.python_implementation = platform.python_implementation() - a free string, like '2.6', or 'win32' -""" - -__all__ = ['default_environment', 'compile', 'interpret'] - -import ast -import os -import platform -import sys -import weakref - -_builtin_compile = compile - -try: - from platform import python_implementation -except ImportError: - if os.name == "java": - # Jython 2.5 has ast module, but not platform.python_implementation() function. - def python_implementation(): - return "Jython" - else: - raise - - -# restricted set of variables -_VARS = {'sys.platform': sys.platform, - 'python_version': '%s.%s' % sys.version_info[:2], - # FIXME parsing sys.platform is not reliable, but there is no other - # way to get e.g. 2.7.2+, and the PEP is defined with sys.version - 'python_full_version': sys.version.split(' ', 1)[0], - 'os.name': os.name, - 'platform.version': platform.version(), - 'platform.machine': platform.machine(), - 'platform.python_implementation': python_implementation(), - 'extra': None # wheel extension - } - -for var in list(_VARS.keys()): - if '.' in var: - _VARS[var.replace('.', '_')] = _VARS[var] - -def default_environment(): - """Return copy of default PEP 385 globals dictionary.""" - return dict(_VARS) - -class ASTWhitelist(ast.NodeTransformer): - def __init__(self, statement): - self.statement = statement # for error messages - - ALLOWED = (ast.Compare, ast.BoolOp, ast.Attribute, ast.Name, ast.Load, ast.Str) - # Bool operations - ALLOWED += (ast.And, ast.Or) - # Comparison operations - ALLOWED += (ast.Eq, ast.Gt, ast.GtE, ast.In, ast.Is, ast.IsNot, ast.Lt, ast.LtE, ast.NotEq, ast.NotIn) - - def visit(self, node): - """Ensure statement only contains allowed nodes.""" - if not isinstance(node, self.ALLOWED): - raise SyntaxError('Not allowed in environment markers.\n%s\n%s' % - (self.statement, - (' ' * node.col_offset) + '^')) - return ast.NodeTransformer.visit(self, node) - - def visit_Attribute(self, node): - """Flatten one level of attribute access.""" - new_node = ast.Name("%s.%s" % (node.value.id, node.attr), node.ctx) - return ast.copy_location(new_node, node) - -def parse_marker(marker): - tree = ast.parse(marker, mode='eval') - new_tree = ASTWhitelist(marker).generic_visit(tree) - return new_tree - -def compile_marker(parsed_marker): - return _builtin_compile(parsed_marker, '<environment marker>', 'eval', - dont_inherit=True) - -_cache = weakref.WeakValueDictionary() - -def compile(marker): - """Return compiled marker as a function accepting an environment dict.""" - try: - return _cache[marker] - except KeyError: - pass - if not marker.strip(): - def marker_fn(environment=None, override=None): - """""" - return True - else: - compiled_marker = compile_marker(parse_marker(marker)) - def marker_fn(environment=None, override=None): - """override updates environment""" - if override is None: - override = {} - if environment is None: - environment = default_environment() - environment.update(override) - return eval(compiled_marker, environment) - marker_fn.__doc__ = marker - _cache[marker] = marker_fn - return _cache[marker] - -def interpret(marker, environment=None): - return compile(marker)(environment) diff --git a/docs/conf.py b/docs/conf.py index c2a63873..604e7138 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -200,56 +200,64 @@ latex_documents = [ #latex_use_modindex = True link_files = { - 'CHANGES.txt': dict( - using=dict( - BB='https://bitbucket.org', - GH='https://github.com', - ), - replace=[ - dict( - pattern=r"(Issue )?#(?P<issue>\d+)", - url='{BB}/pypa/setuptools/issue/{issue}', - ), - dict( - pattern=r"Pull Request ?#(?P<pull_request>\d+)", - url='{BB}/pypa/setuptools/pull-request/{pull_request}', - ), - dict( - pattern=r"Distribute #(?P<distribute>\d+)", - url='{BB}/tarek/distribute/issue/{distribute}', - ), - dict( - pattern=r"Buildout #(?P<buildout>\d+)", - url='{GH}/buildout/buildout/issues/{buildout}', - ), - dict( - pattern=r"Old Setuptools #(?P<old_setuptools>\d+)", - url='http://bugs.python.org/setuptools/issue{old_setuptools}', - ), - dict( - pattern=r"Jython #(?P<jython>\d+)", - url='http://bugs.jython.org/issue{jython}', - ), - dict( - pattern=r"Python #(?P<python>\d+)", - url='http://bugs.python.org/issue{python}', - ), - dict( - pattern=r"Interop #(?P<interop>\d+)", - url='{GH}/pypa/interoperability-peps/issues/{interop}', - ), - dict( - pattern=r"Pip #(?P<pip>\d+)", - url='{GH}/pypa/pip/issues/{pip}', - ), - dict( - pattern=r"Packaging #(?P<packaging>\d+)", - url='{GH}/pypa/packaging/issues/{packaging}', - ), - dict( - pattern=r"[Pp]ackaging (?P<packaging_ver>\d+(\.\d+)+)", - url='{GH}/pypa/packaging/blob/{packaging_ver}/CHANGELOG.rst', - ), - ], - ), + 'CHANGES.rst': dict( + using=dict( + BB='https://bitbucket.org', + GH='https://github.com', + ), + replace=[ + dict( + pattern=r"(Issue )?#(?P<issue>\d+)", + url='{GH}/pypa/setuptools/issues/{issue}', + ), + dict( + pattern=r"BB Pull Request ?#(?P<bb_pull_request>\d+)", + url='{BB}/pypa/setuptools/pull-request/{bb_pull_request}', + ), + dict( + pattern=r"Distribute #(?P<distribute>\d+)", + url='{BB}/tarek/distribute/issue/{distribute}', + ), + dict( + pattern=r"Buildout #(?P<buildout>\d+)", + url='{GH}/buildout/buildout/issues/{buildout}', + ), + dict( + pattern=r"Old Setuptools #(?P<old_setuptools>\d+)", + url='http://bugs.python.org/setuptools/issue{old_setuptools}', + ), + dict( + pattern=r"Jython #(?P<jython>\d+)", + url='http://bugs.jython.org/issue{jython}', + ), + dict( + pattern=r"Python #(?P<python>\d+)", + url='http://bugs.python.org/issue{python}', + ), + dict( + pattern=r"Interop #(?P<interop>\d+)", + url='{GH}/pypa/interoperability-peps/issues/{interop}', + ), + dict( + pattern=r"Pip #(?P<pip>\d+)", + url='{GH}/pypa/pip/issues/{pip}', + ), + dict( + pattern=r"Packaging #(?P<packaging>\d+)", + url='{GH}/pypa/packaging/issues/{packaging}', + ), + dict( + pattern=r"[Pp]ackaging (?P<packaging_ver>\d+(\.\d+)+)", + url='{GH}/pypa/packaging/blob/{packaging_ver}/CHANGELOG.rst', + ), + dict( + pattern=r"PEP[- ](?P<pep_number>\d+)", + url='https://www.python.org/dev/peps/pep-{pep_number:0>4}/', + ), + dict( + pattern=r"^(?m)((?P<scm_version>v?\d+(\.\d+){1,2}))\n[-=]+\n", + with_scm="{text}\n{rev[timestamp]:%d %b %Y}\n", + ), + ], + ), } diff --git a/docs/developer-guide.txt b/docs/developer-guide.txt index ae33649b..7cd3c6d2 100644 --- a/docs/developer-guide.txt +++ b/docs/developer-guide.txt @@ -23,10 +23,10 @@ quality of contribution. Project Management ------------------ -Setuptools is maintained primarily in Bitbucket at `this home -<https://bitbucket.org/pypa/setuptools>`_. Setuptools is maintained under the +Setuptools is maintained primarily in Github at `this home +<https://github.com/pypa/setuptools>`_. Setuptools is maintained under the Python Packaging Authority (PyPA) with several core contributors. All bugs -for Setuptools are filed and the canonical source is maintained in Bitbucket. +for Setuptools are filed and the canonical source is maintained in Github. User support and discussions are done through the issue tracker (for specific) issues, through the distutils-sig mailing list, or on IRC (Freenode) at @@ -44,7 +44,7 @@ describing the motivation behind making changes. First search to see if a ticket already exists for your issue. If not, create one. Try to think from the perspective of the reader. Explain what behavior you expected, what you got instead, and what factors might have contributed to the unexpected -behavior. In Bitbucket, surround a block of code or traceback with the triple +behavior. In Github, surround a block of code or traceback with the triple backtick "\`\`\`" so that it is formatted nicely. Filing a ticket provides a forum for justification, discussion, and @@ -61,17 +61,17 @@ jump to the in-depth discussion about any subject referenced. Source Code ----------- -Grab the code at Bitbucket:: +Grab the code at Github:: - $ hg clone https://bitbucket.org/pypa/setuptools + $ git checkout https://github.com/pypa/setuptools If you want to contribute changes, we recommend you fork the repository on -Bitbucket, commit the changes to your repository, and then make a pull request -on Bitbucket. If you make some changes, don't forget to: +Github, commit the changes to your repository, and then make a pull request +on Github. If you make some changes, don't forget to: -- add a note in CHANGES.txt +- add a note in CHANGES.rst -Please commit all changes in the 'default' branch against the latest available +Please commit all changes in the 'master' branch against the latest available commit or for bug-fixes, against an earlier commit or release in which the bug occurred. @@ -79,12 +79,11 @@ If you find yourself working on more than one issue at a time, Setuptools generally prefers Git-style branches, so use Mercurial bookmarks or Git branches or multiple forks to maintain separate efforts. -Setuptools also maintains an unofficial `Git mirror in Github -<https://github.com/jaraco/setuptools>`_. Contributors are welcome to submit -pull requests here, but because they are not integrated with the Bitbucket -Issue tracker, linking pull requests to tickets is more difficult. The -Continuous Integration tests that validate every release are run from this -mirror. +The Continuous Integration tests that validate every release are run +from this repository. + +For posterity, the old `Bitbucket mirror +<https://bitbucket.org/pypa/setuptools>`_ is available. ------- Testing @@ -104,10 +103,7 @@ Under continuous integration, additional tests may be run. See the Semantic Versioning ------------------- -Setuptools follows ``semver`` with some exceptions: - -- Uses two-segment version when three segment version ends in zero -- Omits 'v' prefix for tags. +Setuptools follows ``semver``. .. explain value of reflecting meaning in versions. diff --git a/docs/history.txt b/docs/history.txt index 268137cd..8e217503 100644 --- a/docs/history.txt +++ b/docs/history.txt @@ -5,4 +5,4 @@ History ******* -.. include:: ../CHANGES (links).txt +.. include:: ../CHANGES (links).rst diff --git a/docs/pkg_resources.txt b/docs/pkg_resources.txt index 3d40a1a2..7b979ec3 100644 --- a/docs/pkg_resources.txt +++ b/docs/pkg_resources.txt @@ -590,20 +590,7 @@ Requirements Parsing parse multiple specifiers from a string or iterable of strings, use ``parse_requirements()`` instead.) - The syntax of a requirement specifier can be defined in EBNF as follows:: - - requirement ::= project_name extras? versionspec? - versionspec ::= comparison version (',' comparison version)* - comparison ::= '<' | '<=' | '!=' | '==' | '>=' | '>' | '~=' | '===' - extras ::= '[' extralist? ']' - extralist ::= identifier (',' identifier)* - project_name ::= identifier - identifier ::= [-A-Za-z0-9_]+ - version ::= [-A-Za-z0-9_.]+ - - Tokens can be separated by whitespace, and a requirement can be continued - over multiple lines using a backslash (``\\``). Line-end comments (using - ``#``) are also allowed. + The syntax of a requirement specifier is defined in full in PEP 508. Some examples of valid requirement specifiers:: @@ -611,6 +598,7 @@ Requirements Parsing Fizzy [foo, bar] PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1 SomethingWhoseVersionIDontCareAbout + SomethingWithMarker[foo]>1.0;python_version<"2.7" The project name is the only required portion of a requirement string, and if it's the only thing supplied, the requirement will accept any version @@ -631,6 +619,11 @@ Requirements Parsing ``pkg_resources.require('Report-O-Rama[PDF]')`` to add the necessary distributions to sys.path at runtime. + The "markers" in a requirement are used to specify when a requirement + should be installed -- the requirement will be installed if the marker + evaluates as true in the current environment. For example, specifying + ``argparse;python_version<"2.7"`` will not install in an Python 2.7 or 3.3 + environment, but will in a Python 2.6 environment. ``Requirement`` Methods and Attributes -------------------------------------- @@ -680,6 +673,12 @@ Requirements Parsing order. The `op` in each tuple is a comparison operator, represented as a string. The `version` is the (unparsed) version number. +``marker`` + An instance of ``packaging.markers.Marker`` that allows evaluation + against the current environment. May be None if no marker specified. + +``url`` + The location to download the requirement from if specified. Entry Points ============ diff --git a/docs/releases.txt b/docs/releases.txt index a9742c20..c84ddd75 100644 --- a/docs/releases.txt +++ b/docs/releases.txt @@ -3,35 +3,27 @@ Release Process =============== In order to allow for rapid, predictable releases, Setuptools uses a -mechanical technique for releases. The release script, ``release.py`` in the -repository, defines the details of the releases, and is executed by the -`jaraco.packaging <https://bitbucket.org/jaraco/jaraco.packaging>`_ release -module. The script does some checks (some interactive) and fully automates -the release process. +mechanical technique for releases, enacted by Travis following a +successful build of a tagged release per +`PyPI deployment <https://docs.travis-ci.com/user/deployment/pypi>`_. -A Setuptools release manager must have maintainer access on PyPI to the -project and administrative access to the Bitbucket project. +To cut a release, install and run ``bumpversion {part}`` where ``part`` +is major, minor, or patch based on the scope of the changes in the +release. Then, push the commits to the master branch. If tests pass, +the release will be uploaded to PyPI (from the Python 3.5 tests). -To make a release, run the following from a Mercurial checkout at the -revision slated for release:: - - python -m jaraco.packaging.release - -Bootstrap Bookmark ------------------- +Bootstrap Branch +---------------- -Setuptools has a bootstrap script (ez_setup.py) which is hosted in the -repository and must be updated with each release (to bump the default version). -The "published" version of the script is the one indicated by the ``bootstrap`` -bookmark (Mercurial) or branch (Git). +Setuptools has a bootstrap script (ez_setup.py), which is hosted in the +repository in the ``bootstrap`` branch. -Therefore, the latest bootstrap script can be retrieved by checking out the -repository at that bookmark. It's also possible to get the bootstrap script for -any particular release by grabbing the script from that tagged release. +Therefore, the latest bootstrap script can be retrieved by checking out +that branch. The officially-published location of the bootstrap script is hosted on Python infrastructure (#python-infra on freenode) at https://bootstrap.pypa.io and -is updated every fifteen minutes from the bootstrap script. Sometimes, +is updated every fifteen minutes from the bootstrap branch. Sometimes, especially when the bootstrap script is rolled back, this process doesn't work as expected and requires manual intervention. @@ -57,7 +49,5 @@ corrected quickly, in many cases before other users have yet to encounter them. Release Managers ---------------- -Jason R. Coombs is the primary release manager. Additionally, the following -people have access to create releases: - -- Matthew Iversen (Ivoz) +Additionally, anyone with push access to the master branch has access to cut +releases. diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 610a0e61..57818281 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -258,10 +258,9 @@ unless you need the associated ``setuptools`` feature. ``include_package_data`` If set to ``True``, this tells ``setuptools`` to automatically include any - data files it finds inside your package directories, that are either under - CVS or Subversion control, or which are specified by your ``MANIFEST.in`` - file. For more information, see the section below on `Including Data - Files`_. + data files it finds inside your package directories that are specified by + your ``MANIFEST.in`` file. For more information, see the section below on + `Including Data Files`_. ``exclude_package_data`` A dictionary mapping package names to lists of glob patterns that should @@ -785,17 +784,15 @@ e.g.:: ) This tells setuptools to install any data files it finds in your packages. -The data files must be under CVS or Subversion control, or else they must be -specified via the distutils' ``MANIFEST.in`` file. (They can also be tracked -by another revision control system, using an appropriate plugin. See the -section below on `Adding Support for Other Revision Control Systems`_ for -information on how to write such plugins.) - -If the data files are not under version control, or are not in a supported -version control system, or if you want finer-grained control over what files -are included (for example, if you have documentation files in your package -directories and want to exclude them from installation), then you can also use -the ``package_data`` keyword, e.g.:: +The data files must be specified via the distutils' ``MANIFEST.in`` file. +(They can also be tracked by a revision control system, using an appropriate +plugin. See the section below on `Adding Support for Revision Control +Systems`_ for information on how to write such plugins.) + +If you want finer-grained control over what files are included (for example, +if you have documentation files in your package directories and want to exclude +them from installation), then you can also use the ``package_data`` keyword, +e.g.:: from setuptools import setup, find_packages setup( @@ -853,8 +850,7 @@ converts slashes to appropriate platform-specific separators at build time. Python 2.4; there is `some documentation for the feature`__ available on the python.org website. If using the setuptools-specific ``include_package_data`` argument, files specified by ``package_data`` will *not* be automatically -added to the manifest unless they are tracked by a supported version control -system, or are listed in the MANIFEST.in file.) +added to the manifest unless they are listed in the MANIFEST.in file.) __ http://docs.python.org/dist/node11.html @@ -887,8 +883,7 @@ included as a result of using ``include_package_data``. In summary, the three options allow you to: ``include_package_data`` - Accept all data files and directories matched by ``MANIFEST.in`` or found - in source control. + Accept all data files and directories matched by ``MANIFEST.in``. ``package_data`` Specify additional patterns to match files and directories that may or may @@ -1231,15 +1226,14 @@ Your Project's Dependencies target audience isn't able to compile packages (e.g. most Windows users) and your package or some of its dependencies include C code. -Subversion or CVS Users and Co-Developers +Revision Control System Users and Co-Developers Users and co-developers who are tracking your in-development code using - CVS, Subversion, or some other revision control system should probably read - this manual's sections regarding such development. Alternately, you may - wish to create a quick-reference guide containing the tips from this manual - that apply to your particular situation. For example, if you recommend - that people use ``setup.py develop`` when tracking your in-development - code, you should let them know that this needs to be run after every update - or commit. + a revision control system should probably read this manual's sections + regarding such development. Alternately, you may wish to create a + quick-reference guide containing the tips from this manual that apply to + your particular situation. For example, if you recommend that people use + ``setup.py develop`` when tracking your in-development code, you should let + them know that this needs to be run after every update or commit. Similarly, if you remove modules or data files from your project, you should remind them to run ``setup.py clean --all`` and delete any obsolete @@ -1276,7 +1270,8 @@ Creating System Packages Setting the ``zip_safe`` flag ----------------------------- -For maximum performance, Python packages are best installed as zip files. +For some use cases (such as bundling as part of a larger application), Python +packages may be run directly from a zip file. Not all packages, however, are capable of running in compressed form, because they may expect to be able to access either source code or data files as normal operating system files. So, ``setuptools`` can install your project @@ -1468,18 +1463,11 @@ Generating Source Distributions ------------------------------- ``setuptools`` enhances the distutils' default algorithm for source file -selection, so that all files managed by CVS or Subversion in your project tree -are included in any source distribution you build. This is a big improvement -over having to manually write a ``MANIFEST.in`` file and try to keep it in -sync with your project. So, if you are using CVS or Subversion, and your -source distributions only need to include files that you're tracking in -revision control, don't create a ``MANIFEST.in`` file for your project. -(And, if you already have one, you might consider deleting it the next time -you would otherwise have to change it.) - -(NOTE: other revision control systems besides CVS and Subversion can be -supported using plugins; see the section below on `Adding Support for Other -Revision Control Systems`_ for information on how to write such plugins.) +selection with pluggable endpoints for looking up files to include. If you are +using a revision control system, and your source distributions only need to +include files that you're tracking in revision control, use a corresponding +plugin instead of writing a ``MANIFEST.in`` file. See the section below on +`Adding Support for Revision Control Systems`_ for information on plugins. If you need to include automatically generated files, or files that are kept in an unsupported revision control system, you'll need to create a ``MANIFEST.in`` @@ -1501,12 +1489,6 @@ the options that the distutils' more complex ``sdist`` process requires. For all practical purposes, you'll probably use only the ``--formats`` option, if you use any option at all. -(By the way, if you're using some other revision control system, you might -consider creating and publishing a `revision control plugin for setuptools`_.) - - -.. _revision control plugin for setuptools: `Adding Support for Other Revision Control Systems`_ - Making your package available for EasyInstall --------------------------------------------- @@ -1687,9 +1669,10 @@ Of course, for this to work, your source distributions must include the C code generated by Pyrex, as well as your original ``.pyx`` files. This means that you will probably want to include current ``.c`` files in your revision control system, rebuilding them whenever you check changes in for the ``.pyx`` -source files. This will ensure that people tracking your project in CVS or -Subversion will be able to build it even if they don't have Pyrex installed, -and that your source releases will be similarly usable with or without Pyrex. +source files. This will ensure that people tracking your project in a revision +control system will be able to build it even if they don't have Pyrex +installed, and that your source releases will be similarly usable with or +without Pyrex. ----------------- @@ -2569,15 +2552,21 @@ the ``cmd`` object's ``write_file()``, ``delete_file()``, and those methods' docstrings for more details. -Adding Support for Other Revision Control Systems +Adding Support for Revision Control Systems ------------------------------------------------- -If you would like to create a plugin for ``setuptools`` to find files in -source control systems, you can do so by adding an -entry point to the ``setuptools.file_finders`` group. The entry point should -be a function accepting a single directory name, and should yield -all the filenames within that directory (and any subdirectories thereof) that -are under revision control. +If the files you want to include in the source distribution are tracked using +Git, Mercurial or SVN, you can use the following packages to achieve that: + +- Git and Mercurial: `setuptools_scm <https://pypi.python.org/pypi/setuptools_scm>`_ +- SVN: `setuptools_svn <https://pypi.python.org/pypi/setuptools_svn>`_ + +If you would like to create a plugin for ``setuptools`` to find files tracked +by another revision control system, you can do so by adding an entry point to +the ``setuptools.file_finders`` group. The entry point should be a function +accepting a single directory name, and should yield all the filenames within +that directory (and any subdirectories thereof) that are under revision +control. For example, if you were going to create a plugin for a revision control system called "foobar", you would write a function something like this: @@ -2670,5 +2659,5 @@ confirmed via the list are actual bugs, and which you have reduced to a minimal set of steps to reproduce. .. _distutils-sig mailing list: http://mail.python.org/pipermail/distutils-sig/ -.. _setuptools bug tracker: https://bitbucket.org/pypa/setuptools/ +.. _setuptools bug tracker: https://github.com/pypa/setuptools/ diff --git a/ez_setup.py b/ez_setup.py deleted file mode 100644 index 9715bdc7..00000000 --- a/ez_setup.py +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env python - -""" -Setuptools bootstrapping installer. - -Run this script to install or upgrade setuptools. -""" - -import os -import shutil -import sys -import tempfile -import zipfile -import optparse -import subprocess -import platform -import textwrap -import contextlib -import json -import codecs - -from distutils import log - -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - -try: - from site import USER_SITE -except ImportError: - USER_SITE = None - -LATEST = object() -DEFAULT_VERSION = LATEST -DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" -DEFAULT_SAVE_DIR = os.curdir - - -def _python_cmd(*args): - """ - Execute a command. - - Return True if the command succeeded. - """ - args = (sys.executable,) + args - return subprocess.call(args) == 0 - - -def _install(archive_filename, install_args=()): - """Install Setuptools.""" - with archive_context(archive_filename): - # installing - log.warn('Installing Setuptools') - if not _python_cmd('setup.py', 'install', *install_args): - log.warn('Something went wrong during the installation.') - log.warn('See the error message above.') - # exitcode will be 2 - return 2 - - -def _build_egg(egg, archive_filename, to_dir): - """Build Setuptools egg.""" - with archive_context(archive_filename): - # building an egg - log.warn('Building a Setuptools egg in %s', to_dir) - _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) - # returning the result - log.warn(egg) - if not os.path.exists(egg): - raise IOError('Could not build the egg.') - - -class ContextualZipFile(zipfile.ZipFile): - - """Supplement ZipFile class to support context manager for Python 2.6.""" - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def __new__(cls, *args, **kwargs): - """Construct a ZipFile or ContextualZipFile as appropriate.""" - if hasattr(zipfile.ZipFile, '__exit__'): - return zipfile.ZipFile(*args, **kwargs) - return super(ContextualZipFile, cls).__new__(cls) - - -@contextlib.contextmanager -def archive_context(filename): - """ - Unzip filename to a temporary directory, set to the cwd. - - The unzipped target is cleaned up after. - """ - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - with ContextualZipFile(filename) as archive: - archive.extractall() - - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) - yield - - finally: - os.chdir(old_wd) - shutil.rmtree(tmpdir) - - -def _do_download(version, download_base, to_dir, download_delay): - """Download Setuptools.""" - egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' - % (version, sys.version_info[0], sys.version_info[1])) - if not os.path.exists(egg): - archive = download_setuptools(version, download_base, - to_dir, download_delay) - _build_egg(egg, archive, to_dir) - sys.path.insert(0, egg) - - # Remove previously-imported pkg_resources if present (see - # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). - if 'pkg_resources' in sys.modules: - _unload_pkg_resources() - - import setuptools - setuptools.bootstrap_install_from = egg - - -def use_setuptools( - version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=DEFAULT_SAVE_DIR, download_delay=15): - """ - Ensure that a setuptools version is installed. - - Return None. Raise SystemExit if the requested version - or later cannot be installed. - """ - version = _resolve_version(version) - to_dir = os.path.abspath(to_dir) - - # prior to importing, capture the module state for - # representative modules. - rep_modules = 'pkg_resources', 'setuptools' - imported = set(sys.modules).intersection(rep_modules) - - try: - import pkg_resources - pkg_resources.require("setuptools>=" + version) - # a suitable version is already installed - return - except ImportError: - # pkg_resources not available; setuptools is not installed; download - pass - except pkg_resources.DistributionNotFound: - # no version of setuptools was found; allow download - pass - except pkg_resources.VersionConflict as VC_err: - if imported: - _conflict_bail(VC_err, version) - - # otherwise, unload pkg_resources to allow the downloaded version to - # take precedence. - del pkg_resources - _unload_pkg_resources() - - return _do_download(version, download_base, to_dir, download_delay) - - -def _conflict_bail(VC_err, version): - """ - Setuptools was imported prior to invocation, so it is - unsafe to unload it. Bail out. - """ - conflict_tmpl = textwrap.dedent(""" - The required version of setuptools (>={version}) is not available, - and can't be installed while this script is running. Please - install a more recent version first, using - 'easy_install -U setuptools'. - - (Currently using {VC_err.args[0]!r}) - """) - msg = conflict_tmpl.format(**locals()) - sys.stderr.write(msg) - sys.exit(2) - - -def _unload_pkg_resources(): - del_modules = [ - name for name in sys.modules - if name.startswith('pkg_resources') - ] - for mod_name in del_modules: - del sys.modules[mod_name] - - -def _clean_check(cmd, target): - """ - Run the command to download target. - - If the command fails, clean up before re-raising the error. - """ - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - if os.access(target, os.F_OK): - os.unlink(target) - raise - - -def download_file_powershell(url, target): - """ - Download the file at url to target using Powershell. - - Powershell will validate trust. - Raise an exception if the command cannot complete. - """ - target = os.path.abspath(target) - ps_cmd = ( - "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " - "[System.Net.CredentialCache]::DefaultCredentials; " - '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")' - % locals() - ) - cmd = [ - 'powershell', - '-Command', - ps_cmd, - ] - _clean_check(cmd, target) - - -def has_powershell(): - """Determine if Powershell is available.""" - if platform.system() != 'Windows': - return False - cmd = ['powershell', '-Command', 'echo test'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True -download_file_powershell.viable = has_powershell - - -def download_file_curl(url, target): - cmd = ['curl', url, '--silent', '--output', target] - _clean_check(cmd, target) - - -def has_curl(): - cmd = ['curl', '--version'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True -download_file_curl.viable = has_curl - - -def download_file_wget(url, target): - cmd = ['wget', url, '--quiet', '--output-document', target] - _clean_check(cmd, target) - - -def has_wget(): - cmd = ['wget', '--version'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True -download_file_wget.viable = has_wget - - -def download_file_insecure(url, target): - """Use Python to download the file, without connection authentication.""" - src = urlopen(url) - try: - # Read all the data in one block. - data = src.read() - finally: - src.close() - - # Write all the data in one block to avoid creating a partial file. - with open(target, "wb") as dst: - dst.write(data) -download_file_insecure.viable = lambda: True - - -def get_best_downloader(): - downloaders = ( - download_file_powershell, - download_file_curl, - download_file_wget, - download_file_insecure, - ) - viable_downloaders = (dl for dl in downloaders if dl.viable()) - return next(viable_downloaders, None) - - -def download_setuptools( - version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=DEFAULT_SAVE_DIR, delay=15, - downloader_factory=get_best_downloader): - """ - Download setuptools from a specified location and return its filename. - - `version` should be a valid setuptools version number that is available - as an sdist for download under the `download_base` URL (which should end - with a '/'). `to_dir` is the directory where the egg will be downloaded. - `delay` is the number of seconds to pause before an actual download - attempt. - - ``downloader_factory`` should be a function taking no arguments and - returning a function for downloading a URL to a target. - """ - version = _resolve_version(version) - # making sure we use the absolute path - to_dir = os.path.abspath(to_dir) - zip_name = "setuptools-%s.zip" % version - url = download_base + zip_name - saveto = os.path.join(to_dir, zip_name) - if not os.path.exists(saveto): # Avoid repeated downloads - log.warn("Downloading %s", url) - downloader = downloader_factory() - downloader(url, saveto) - return os.path.realpath(saveto) - - -def _resolve_version(version): - """ - Resolve LATEST version - """ - if version is not LATEST: - return version - - resp = urlopen('https://pypi.python.org/pypi/setuptools/json') - with contextlib.closing(resp): - try: - charset = resp.info().get_content_charset() - except Exception: - # Python 2 compat; assume UTF-8 - charset = 'UTF-8' - reader = codecs.getreader(charset) - doc = json.load(reader(resp)) - - return str(doc['info']['version']) - - -def _build_install_args(options): - """ - Build the arguments to 'python setup.py install' on the setuptools package. - - Returns list of command line arguments. - """ - return ['--user'] if options.user_install else [] - - -def _parse_args(): - """Parse the command line for options.""" - parser = optparse.OptionParser() - parser.add_option( - '--user', dest='user_install', action='store_true', default=False, - help='install in user site package (requires Python 2.6 or later)') - parser.add_option( - '--download-base', dest='download_base', metavar="URL", - default=DEFAULT_URL, - help='alternative URL from where to download the setuptools package') - parser.add_option( - '--insecure', dest='downloader_factory', action='store_const', - const=lambda: download_file_insecure, default=get_best_downloader, - help='Use internal, non-validating downloader' - ) - parser.add_option( - '--version', help="Specify which version to download", - default=DEFAULT_VERSION, - ) - parser.add_option( - '--to-dir', - help="Directory to save (and re-use) package", - default=DEFAULT_SAVE_DIR, - ) - options, args = parser.parse_args() - # positional arguments are ignored - return options - - -def _download_args(options): - """Return args for download_setuptools function from cmdline args.""" - return dict( - version=options.version, - download_base=options.download_base, - downloader_factory=options.downloader_factory, - to_dir=options.to_dir, - ) - - -def main(): - """Install or upgrade setuptools and EasyInstall.""" - options = _parse_args() - archive = download_setuptools(**_download_args(options)) - return _install(archive, _build_install_args(options)) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/pavement.py b/pavement.py new file mode 100644 index 00000000..303e9bac --- /dev/null +++ b/pavement.py @@ -0,0 +1,28 @@ +import re + +from paver.easy import task, path as Path +import pip + +def remove_all(paths): + for path in paths: + path.rmtree() if path.isdir() else path.remove() + +@task +def update_vendored(): + vendor = Path('pkg_resources/_vendor') + remove_all(vendor.glob('packaging*')) + remove_all(vendor.glob('six*')) + remove_all(vendor.glob('pyparsing*')) + install_args = [ + 'install', + '-r', str(vendor/'vendored.txt'), + '-t', str(vendor), + ] + pip.main(install_args) + packaging = vendor / 'packaging' + for file in packaging.glob('*.py'): + text = file.text() + text = re.sub(r' (pyparsing|six)', r' pkg_resources.extern.\1', text) + file.write_text(text) + remove_all(vendor.glob('*.dist-info')) + remove_all(vendor.glob('*.egg-info')) diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index d04cd347..2eab8230 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -28,8 +28,6 @@ import warnings import stat import functools import pkgutil -import token -import symbol import operator import platform import collections @@ -67,14 +65,11 @@ try: except ImportError: importlib_machinery = None -try: - import parser -except ImportError: - pass - from pkg_resources.extern import packaging __import__('pkg_resources.extern.packaging.version') __import__('pkg_resources.extern.packaging.specifiers') +__import__('pkg_resources.extern.packaging.requirements') +__import__('pkg_resources.extern.packaging.markers') if (3, 0) < sys.version_info < (3, 3): @@ -797,6 +792,8 @@ class WorkingSet(object): best = {} to_activate = [] + req_extras = _ReqExtras() + # Mapping of requirement to set of distributions that required it; # useful for reporting info about conflicts. required_by = collections.defaultdict(set) @@ -807,6 +804,10 @@ class WorkingSet(object): if req in processed: # Ignore cyclic or redundant dependencies continue + + if not req_extras.markers_pass(req): + continue + dist = best.get(req.key) if dist is None: # Find the best distribution and add it to the map @@ -839,6 +840,7 @@ class WorkingSet(object): # Register the new requirements needed by req for new_requirement in new_requirements: required_by[new_requirement].add(req.project_name) + req_extras[new_requirement] = req.extras processed[req] = True @@ -971,6 +973,26 @@ class WorkingSet(object): self.callbacks = callbacks[:] +class _ReqExtras(dict): + """ + Map each requirement to the extras that demanded it. + """ + + def markers_pass(self, req): + """ + Evaluate markers for req against each extra that + demanded it. + + Return False if the req has a marker and fails + evaluation. Otherwise, return True. + """ + extra_evals = ( + req.marker.evaluate({'extra': extra}) + for extra in self.get(req, ()) + (None,) + ) + return not req.marker or any(extra_evals) + + class Environment(object): """Searchable snapshot of distributions on a search path""" @@ -1385,202 +1407,34 @@ def to_filename(name): return name.replace('-','_') -class MarkerEvaluation(object): - values = { - 'os_name': lambda: os.name, - 'sys_platform': lambda: sys.platform, - 'python_full_version': platform.python_version, - 'python_version': lambda: platform.python_version()[:3], - 'platform_version': platform.version, - 'platform_machine': platform.machine, - 'platform_python_implementation': platform.python_implementation, - 'python_implementation': platform.python_implementation, - } - - @classmethod - def is_invalid_marker(cls, text): - """ - Validate text as a PEP 426 environment marker; return an exception - if invalid or False otherwise. - """ - try: - cls.evaluate_marker(text) - except SyntaxError as e: - return cls.normalize_exception(e) - return False - - @staticmethod - def normalize_exception(exc): - """ - Given a SyntaxError from a marker evaluation, normalize the error - message: - - Remove indications of filename and line number. - - Replace platform-specific error messages with standard error - messages. - """ - subs = { - 'unexpected EOF while parsing': 'invalid syntax', - 'parenthesis is never closed': 'invalid syntax', - } - exc.filename = None - exc.lineno = None - exc.msg = subs.get(exc.msg, exc.msg) - return exc - - @classmethod - def and_test(cls, nodelist): - # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! - items = [ - cls.interpret(nodelist[i]) - for i in range(1, len(nodelist), 2) - ] - return functools.reduce(operator.and_, items) - - @classmethod - def test(cls, nodelist): - # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! - items = [ - cls.interpret(nodelist[i]) - for i in range(1, len(nodelist), 2) - ] - return functools.reduce(operator.or_, items) - - @classmethod - def atom(cls, nodelist): - t = nodelist[1][0] - if t == token.LPAR: - if nodelist[2][0] == token.RPAR: - raise SyntaxError("Empty parentheses") - return cls.interpret(nodelist[2]) - msg = "Language feature not supported in environment markers" - raise SyntaxError(msg) - - @classmethod - def comparison(cls, nodelist): - if len(nodelist) > 4: - msg = "Chained comparison not allowed in environment markers" - raise SyntaxError(msg) - comp = nodelist[2][1] - cop = comp[1] - if comp[0] == token.NAME: - if len(nodelist[2]) == 3: - if cop == 'not': - cop = 'not in' - else: - cop = 'is not' - try: - cop = cls.get_op(cop) - except KeyError: - msg = repr(cop) + " operator not allowed in environment markers" - raise SyntaxError(msg) - return cop(cls.evaluate(nodelist[1]), cls.evaluate(nodelist[3])) - - @classmethod - def get_op(cls, op): - ops = { - symbol.test: cls.test, - symbol.and_test: cls.and_test, - symbol.atom: cls.atom, - symbol.comparison: cls.comparison, - 'not in': lambda x, y: x not in y, - 'in': lambda x, y: x in y, - '==': operator.eq, - '!=': operator.ne, - '<': operator.lt, - '>': operator.gt, - '<=': operator.le, - '>=': operator.ge, - } - if hasattr(symbol, 'or_test'): - ops[symbol.or_test] = cls.test - return ops[op] - - @classmethod - def evaluate_marker(cls, text, extra=None): - """ - Evaluate a PEP 426 environment marker on CPython 2.4+. - Return a boolean indicating the marker result in this environment. - Raise SyntaxError if marker is invalid. - - This implementation uses the 'parser' module, which is not implemented - on - Jython and has been superseded by the 'ast' module in Python 2.6 and - later. - """ - return cls.interpret(parser.expr(text).totuple(1)[1]) - - @staticmethod - def _translate_metadata2(env): - """ - Markerlib implements Metadata 1.2 (PEP 345) environment markers. - Translate the variables to Metadata 2.0 (PEP 426). - """ - return dict( - (key.replace('.', '_'), value) - for key, value in env.items() - ) - - @classmethod - def _markerlib_evaluate(cls, text): - """ - Evaluate a PEP 426 environment marker using markerlib. - Return a boolean indicating the marker result in this environment. - Raise SyntaxError if marker is invalid. - """ - import _markerlib - - env = cls._translate_metadata2(_markerlib.default_environment()) - try: - result = _markerlib.interpret(text, env) - except NameError as e: - raise SyntaxError(e.args[0]) - return result - - if 'parser' not in globals(): - # Fall back to less-complete _markerlib implementation if 'parser' module - # is not available. - evaluate_marker = _markerlib_evaluate +def invalid_marker(text): + """ + Validate text as a PEP 508 environment marker; return an exception + if invalid or False otherwise. + """ + try: + evaluate_marker(text) + except SyntaxError as e: + e.filename = None + e.lineno = None + return e + return False - @classmethod - def interpret(cls, nodelist): - while len(nodelist)==2: nodelist = nodelist[1] - try: - op = cls.get_op(nodelist[0]) - except KeyError: - raise SyntaxError("Comparison or logical expression expected") - return op(nodelist) - @classmethod - def evaluate(cls, nodelist): - while len(nodelist)==2: nodelist = nodelist[1] - kind = nodelist[0] - name = nodelist[1] - if kind==token.NAME: - try: - op = cls.values[name] - except KeyError: - raise SyntaxError("Unknown name %r" % name) - return op() - if kind==token.STRING: - s = nodelist[1] - if not cls._safe_string(s): - raise SyntaxError( - "Only plain strings allowed in environment markers") - return s[1:-1] - msg = "Language feature not supported in environment markers" - raise SyntaxError(msg) +def evaluate_marker(text, extra=None): + """ + Evaluate a PEP 508 environment marker. + Return a boolean indicating the marker result in this environment. + Raise SyntaxError if marker is invalid. - @staticmethod - def _safe_string(cand): - return ( - cand[:1] in "'\"" and - not cand.startswith('"""') and - not cand.startswith("'''") and - '\\' not in cand - ) + This implementation uses the 'pyparsing' module. + """ + try: + marker = packaging.markers.Marker(text) + return marker.evaluate() + except packaging.markers.InvalidMarker as e: + raise SyntaxError(e) -invalid_marker = MarkerEvaluation.is_invalid_marker -evaluate_marker = MarkerEvaluation.evaluate_marker class NullProvider: """Try to implement resources and metadata for arbitrary PEP 302 loaders""" @@ -2005,7 +1859,13 @@ class FileMetadata(EmptyProvider): def get_metadata(self, name): if name=='PKG-INFO': with io.open(self.path, encoding='utf-8') as f: - metadata = f.read() + try: + metadata = f.read() + except UnicodeDecodeError as exc: + # add path context to error message + tmpl = " in {self.path}" + exc.reason += tmpl.format(self=self) + raise return metadata raise KeyError("No metadata except PKG-INFO is available") @@ -2194,12 +2054,13 @@ def _rebuild_mod_path(orig_path, package_name, module): corresponding to their sys.path order """ sys_path = [_normalize_cached(p) for p in sys.path] - def position_in_sys_path(p): + def position_in_sys_path(path): """ Return the ordinal of the path based on its position in sys.path """ - parts = p.split(os.sep) - parts = parts[:-(package_name.count('.') + 1)] + path_parts = path.split(os.sep) + module_parts = package_name.count('.') + 1 + parts = path_parts[:-module_parts] return sys_path.index(_normalize_cached(os.sep.join(parts))) orig_path.sort(key=position_in_sys_path) @@ -2314,18 +2175,6 @@ def yield_lines(strs): for s in yield_lines(ss): yield s -# whitespace and comment -LINE_END = re.compile(r"\s*(#.*)?$").match -# line continuation -CONTINUE = re.compile(r"\s*\\\s*(#.*)?$").match -# Distribution or extra -DISTRO = re.compile(r"\s*((\w|[-.])+)").match -# ver. info -VERSION = re.compile(r"\s*(<=?|>=?|===?|!=|~=)\s*((\w|[-.*_!+])+)").match -# comma between items -COMMA = re.compile(r"\s*,").match -OBRACKET = re.compile(r"\s*\[").match -CBRACKET = re.compile(r"\s*\]").match MODULE = re.compile(r"\w+(\.\w+)*$").match EGG_NAME = re.compile( r""" @@ -2864,34 +2713,18 @@ class DistInfoDistribution(Distribution): self.__dep_map = self._compute_dependencies() return self.__dep_map - def _preparse_requirement(self, requires_dist): - """Convert 'Foobar (1); baz' to ('Foobar ==1', 'baz') - Split environment marker, add == prefix to version specifiers as - necessary, and remove parenthesis. - """ - parts = requires_dist.split(';', 1) + [''] - distvers = parts[0].strip() - mark = parts[1].strip() - distvers = re.sub(self.EQEQ, r"\1==\2\3", distvers) - distvers = distvers.replace('(', '').replace(')', '') - return (distvers, mark) - def _compute_dependencies(self): """Recompute this distribution's dependencies.""" - from _markerlib import compile as compile_marker dm = self.__dep_map = {None: []} reqs = [] # Including any condition expressions for req in self._parsed_pkg_info.get_all('Requires-Dist') or []: - distvers, mark = self._preparse_requirement(req) - parsed = next(parse_requirements(distvers)) - parsed.marker_fn = compile_marker(mark) - reqs.append(parsed) + reqs.extend(parse_requirements(req)) def reqs_for_extra(extra): for req in reqs: - if req.marker_fn(override={'extra':extra}): + if not req.marker or req.marker.evaluate({'extra': extra}): yield req common = frozenset(reqs_for_extra(None)) @@ -2937,85 +2770,38 @@ def parse_requirements(strs): # create a steppable iterator, so we can handle \-continuations lines = iter(yield_lines(strs)) - def scan_list(ITEM, TERMINATOR, line, p, groups, item_name): - - items = [] - - while not TERMINATOR(line, p): - if CONTINUE(line, p): - try: - line = next(lines) - p = 0 - except StopIteration: - msg = "\\ must not appear on the last nonblank line" - raise RequirementParseError(msg) - - match = ITEM(line, p) - if not match: - msg = "Expected " + item_name + " in" - raise RequirementParseError(msg, line, "at", line[p:]) - - items.append(match.group(*groups)) - p = match.end() - - match = COMMA(line, p) - if match: - # skip the comma - p = match.end() - elif not TERMINATOR(line, p): - msg = "Expected ',' or end-of-list in" - raise RequirementParseError(msg, line, "at", line[p:]) - - match = TERMINATOR(line, p) - # skip the terminator, if any - if match: - p = match.end() - return line, p, items - for line in lines: - match = DISTRO(line) - if not match: - raise RequirementParseError("Missing distribution spec", line) - project_name = match.group(1) - p = match.end() - extras = [] - - match = OBRACKET(line, p) - if match: - p = match.end() - line, p, extras = scan_list( - DISTRO, CBRACKET, line, p, (1,), "'extra' name" - ) - - line, p, specs = scan_list(VERSION, LINE_END, line, p, (1, 2), - "version spec") - specs = [(op, val) for op, val in specs] - yield Requirement(project_name, specs, extras) - - -class Requirement: - def __init__(self, project_name, specs, extras): + # Drop comments -- a hash without a space may be in a URL. + if ' #' in line: + line = line[:line.find(' #')] + # If there is a line continuation, drop it, and append the next line. + if line.endswith('\\'): + line = line[:-2].strip() + line += next(lines) + yield Requirement(line) + + +class Requirement(packaging.requirements.Requirement): + def __init__(self, requirement_string): """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!""" - self.unsafe_name, project_name = project_name, safe_name(project_name) + try: + super(Requirement, self).__init__(requirement_string) + except packaging.requirements.InvalidRequirement as e: + raise RequirementParseError(str(e)) + self.unsafe_name = self.name + project_name = safe_name(self.name) self.project_name, self.key = project_name, project_name.lower() - self.specifier = packaging.specifiers.SpecifierSet( - ",".join(["".join([x, y]) for x, y in specs]) - ) - self.specs = specs - self.extras = tuple(map(safe_extra, extras)) + self.specs = [ + (spec.operator, spec.version) for spec in self.specifier] + self.extras = tuple(map(safe_extra, self.extras)) self.hashCmp = ( self.key, self.specifier, frozenset(self.extras), + str(self.marker) if self.marker else None, ) self.__hash = hash(self.hashCmp) - def __str__(self): - extras = ','.join(self.extras) - if extras: - extras = '[%s]' % extras - return '%s%s%s' % (self.project_name, extras, self.specifier) - def __eq__(self, other): return ( isinstance(other, Requirement) and diff --git a/pkg_resources/_vendor/packaging/__about__.py b/pkg_resources/_vendor/packaging/__about__.py index eadb794e..c21a758b 100644 --- a/pkg_resources/_vendor/packaging/__about__.py +++ b/pkg_resources/_vendor/packaging/__about__.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function __all__ = [ @@ -22,10 +12,10 @@ __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "15.3" +__version__ = "16.7" -__author__ = "Donald Stufft" +__author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" -__license__ = "Apache License, Version 2.0" -__copyright__ = "Copyright 2014 %s" % __author__ +__license__ = "BSD or Apache License, Version 2.0" +__copyright__ = "Copyright 2014-2016 %s" % __author__ diff --git a/pkg_resources/_vendor/packaging/__init__.py b/pkg_resources/_vendor/packaging/__init__.py index c39a8eab..5ee62202 100644 --- a/pkg_resources/_vendor/packaging/__init__.py +++ b/pkg_resources/_vendor/packaging/__init__.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function from .__about__ import ( diff --git a/pkg_resources/_vendor/packaging/_compat.py b/pkg_resources/_vendor/packaging/_compat.py index 5c396cea..210bb80b 100644 --- a/pkg_resources/_vendor/packaging/_compat.py +++ b/pkg_resources/_vendor/packaging/_compat.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function import sys diff --git a/pkg_resources/_vendor/packaging/_structures.py b/pkg_resources/_vendor/packaging/_structures.py index 0ae9bb52..ccc27861 100644 --- a/pkg_resources/_vendor/packaging/_structures.py +++ b/pkg_resources/_vendor/packaging/_structures.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function diff --git a/pkg_resources/_vendor/packaging/markers.py b/pkg_resources/_vendor/packaging/markers.py new file mode 100644 index 00000000..c5d29cd9 --- /dev/null +++ b/pkg_resources/_vendor/packaging/markers.py @@ -0,0 +1,287 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import operator +import os +import platform +import sys + +from pkg_resources.extern.pyparsing import ParseException, ParseResults, stringStart, stringEnd +from pkg_resources.extern.pyparsing import ZeroOrMore, Group, Forward, QuotedString +from pkg_resources.extern.pyparsing import Literal as L # noqa + +from ._compat import string_types +from .specifiers import Specifier, InvalidSpecifier + + +__all__ = [ + "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", + "Marker", "default_environment", +] + + +class InvalidMarker(ValueError): + """ + An invalid marker was found, users should refer to PEP 508. + """ + + +class UndefinedComparison(ValueError): + """ + An invalid operation was attempted on a value that doesn't support it. + """ + + +class UndefinedEnvironmentName(ValueError): + """ + A name was attempted to be used that does not exist inside of the + environment. + """ + + +class Node(object): + + def __init__(self, value): + self.value = value + + def __str__(self): + return str(self.value) + + def __repr__(self): + return "<{0}({1!r})>".format(self.__class__.__name__, str(self)) + + +class Variable(Node): + pass + + +class Value(Node): + pass + + +VARIABLE = ( + L("implementation_version") | + L("platform_python_implementation") | + L("implementation_name") | + L("python_full_version") | + L("platform_release") | + L("platform_version") | + L("platform_machine") | + L("platform_system") | + L("python_version") | + L("sys_platform") | + L("os_name") | + L("os.name") | # PEP-345 + L("sys.platform") | # PEP-345 + L("platform.version") | # PEP-345 + L("platform.machine") | # PEP-345 + L("platform.python_implementation") | # PEP-345 + L("python_implementation") | # undocumented setuptools legacy + L("extra") +) +ALIASES = { + 'os.name': 'os_name', + 'sys.platform': 'sys_platform', + 'platform.version': 'platform_version', + 'platform.machine': 'platform_machine', + 'platform.python_implementation': 'platform_python_implementation', + 'python_implementation': 'platform_python_implementation' +} +VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) + +VERSION_CMP = ( + L("===") | + L("==") | + L(">=") | + L("<=") | + L("!=") | + L("~=") | + L(">") | + L("<") +) + +MARKER_OP = VERSION_CMP | L("not in") | L("in") + +MARKER_VALUE = QuotedString("'") | QuotedString('"') +MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) + +BOOLOP = L("and") | L("or") + +MARKER_VAR = VARIABLE | MARKER_VALUE + +MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) +MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) + +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() + +MARKER_EXPR = Forward() +MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) +MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) + +MARKER = stringStart + MARKER_EXPR + stringEnd + + +def _coerce_parse_result(results): + if isinstance(results, ParseResults): + return [_coerce_parse_result(i) for i in results] + else: + return results + + +def _format_marker(marker, first=True): + assert isinstance(marker, (list, tuple, string_types)) + + # Sometimes we have a structure like [[...]] which is a single item list + # where the single item is itself it's own list. In that case we want skip + # the rest of this function so that we don't get extraneous () on the + # outside. + if (isinstance(marker, list) and len(marker) == 1 and + isinstance(marker[0], (list, tuple))): + return _format_marker(marker[0]) + + if isinstance(marker, list): + inner = (_format_marker(m, first=False) for m in marker) + if first: + return " ".join(inner) + else: + return "(" + " ".join(inner) + ")" + elif isinstance(marker, tuple): + return '{0} {1} "{2}"'.format(*marker) + else: + return marker + + +_operators = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +def _eval_op(lhs, op, rhs): + try: + spec = Specifier("".join([op, rhs])) + except InvalidSpecifier: + pass + else: + return spec.contains(lhs) + + oper = _operators.get(op) + if oper is None: + raise UndefinedComparison( + "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs) + ) + + return oper(lhs, rhs) + + +_undefined = object() + + +def _get_env(environment, name): + value = environment.get(name, _undefined) + + if value is _undefined: + raise UndefinedEnvironmentName( + "{0!r} does not exist in evaluation environment.".format(name) + ) + + return value + + +def _evaluate_markers(markers, environment): + groups = [[]] + + for marker in markers: + assert isinstance(marker, (list, tuple, string_types)) + + if isinstance(marker, list): + groups[-1].append(_evaluate_markers(marker, environment)) + elif isinstance(marker, tuple): + lhs, op, rhs = marker + + if isinstance(lhs, Variable): + lhs_value = _get_env(environment, lhs.value) + rhs_value = rhs.value + else: + lhs_value = lhs.value + rhs_value = _get_env(environment, rhs.value) + + groups[-1].append(_eval_op(lhs_value, op, rhs_value)) + else: + assert marker in ["and", "or"] + if marker == "or": + groups.append([]) + + return any(all(item) for item in groups) + + +def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + +def default_environment(): + if hasattr(sys, 'implementation'): + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + else: + iver = '0' + implementation_name = '' + + return { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "platform_python_implementation": platform.python_implementation(), + "python_version": platform.python_version()[:3], + "sys_platform": sys.platform, + } + + +class Marker(object): + + def __init__(self, marker): + try: + self._markers = _coerce_parse_result(MARKER.parseString(marker)) + except ParseException as e: + err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( + marker, marker[e.loc:e.loc + 8]) + raise InvalidMarker(err_str) + + def __str__(self): + return _format_marker(self._markers) + + def __repr__(self): + return "<Marker({0!r})>".format(str(self)) + + def evaluate(self, environment=None): + """Evaluate a marker. + + Return the boolean from evaluating the given marker against the + environment. environment is an optional argument to override all or + part of the determined environment. + + The environment is determined from the current Python process. + """ + current_environment = default_environment() + if environment is not None: + current_environment.update(environment) + + return _evaluate_markers(self._markers, current_environment) diff --git a/pkg_resources/_vendor/packaging/requirements.py b/pkg_resources/_vendor/packaging/requirements.py new file mode 100644 index 00000000..0c8c4a38 --- /dev/null +++ b/pkg_resources/_vendor/packaging/requirements.py @@ -0,0 +1,127 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import string +import re + +from pkg_resources.extern.pyparsing import stringStart, stringEnd, originalTextFor, ParseException +from pkg_resources.extern.pyparsing import ZeroOrMore, Word, Optional, Regex, Combine +from pkg_resources.extern.pyparsing import Literal as L # noqa +from pkg_resources.extern.six.moves.urllib import parse as urlparse + +from .markers import MARKER_EXPR, Marker +from .specifiers import LegacySpecifier, Specifier, SpecifierSet + + +class InvalidRequirement(ValueError): + """ + An invalid requirement was found, users should refer to PEP 508. + """ + + +ALPHANUM = Word(string.ascii_letters + string.digits) + +LBRACKET = L("[").suppress() +RBRACKET = L("]").suppress() +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() +COMMA = L(",").suppress() +SEMICOLON = L(";").suppress() +AT = L("@").suppress() + +PUNCTUATION = Word("-_.") +IDENTIFIER_END = ALPHANUM | (ZeroOrMore(PUNCTUATION) + ALPHANUM) +IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) + +NAME = IDENTIFIER("name") +EXTRA = IDENTIFIER + +URI = Regex(r'[^ ]+')("url") +URL = (AT + URI) + +EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) +EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") + +VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) +VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) + +VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY +VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), + joinString=",", adjacent=False)("_raw_spec") +_VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') + +VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") +VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) + +MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") +MARKER_EXPR.setParseAction( + lambda s, l, t: Marker(s[t._original_start:t._original_end]) +) +MARKER_SEPERATOR = SEMICOLON +MARKER = MARKER_SEPERATOR + MARKER_EXPR + +VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) +URL_AND_MARKER = URL + Optional(MARKER) + +NAMED_REQUIREMENT = \ + NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) + +REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd + + +class Requirement(object): + """Parse a requirement. + + Parse a given requirement string into its parts, such as name, specifier, + URL, and extras. Raises InvalidRequirement on a badly-formed requirement + string. + """ + + # TODO: Can we test whether something is contained within a requirement? + # If so how do we do that? Do we need to test against the _name_ of + # the thing as well as the version? What about the markers? + # TODO: Can we normalize the name and extra name? + + def __init__(self, requirement_string): + try: + req = REQUIREMENT.parseString(requirement_string) + except ParseException as e: + raise InvalidRequirement( + "Invalid requirement, parse error at \"{0!r}\"".format( + requirement_string[e.loc:e.loc + 8])) + + self.name = req.name + if req.url: + parsed_url = urlparse.urlparse(req.url) + if not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc): + raise InvalidRequirement("Invalid URL given") + self.url = req.url + else: + self.url = None + self.extras = set(req.extras.asList() if req.extras else []) + self.specifier = SpecifierSet(req.specifier) + self.marker = req.marker if req.marker else None + + def __str__(self): + parts = [self.name] + + if self.extras: + parts.append("[{0}]".format(",".join(sorted(self.extras)))) + + if self.specifier: + parts.append(str(self.specifier)) + + if self.url: + parts.append("@ {0}".format(self.url)) + + if self.marker: + parts.append("; {0}".format(self.marker)) + + return "".join(parts) + + def __repr__(self): + return "<Requirement({0!r})>".format(str(self)) diff --git a/pkg_resources/_vendor/packaging/specifiers.py b/pkg_resources/_vendor/packaging/specifiers.py index 891664f0..7f5a76cf 100644 --- a/pkg_resources/_vendor/packaging/specifiers.py +++ b/pkg_resources/_vendor/packaging/specifiers.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function import abc @@ -204,8 +194,8 @@ class _IndividualSpecifier(BaseSpecifier): # If our version is a prerelease, and we were not set to allow # prereleases, then we'll store it for later incase nothing # else matches this specifier. - if (parsed_version.is_prerelease - and not (prereleases or self.prereleases)): + if (parsed_version.is_prerelease and not + (prereleases or self.prereleases)): found_prereleases.append(version) # Either this is not a prerelease, or we should have been # accepting prereleases from the begining. @@ -223,23 +213,23 @@ class _IndividualSpecifier(BaseSpecifier): class LegacySpecifier(_IndividualSpecifier): - _regex = re.compile( + _regex_str = ( r""" - ^ - \s* (?P<operator>(==|!=|<=|>=|<|>)) \s* (?P<version> - [^\s]* # We just match everything, except for whitespace since this - # is a "legacy" specifier and the version string can be just - # about anything. + [^,;\s)]* # Since this is a "legacy" specifier, and the version + # string can be just about anything, we match everything + # except for whitespace, a semi-colon for marker support, + # a closing paren since versions can be enclosed in + # them, and a comma since it's a version separator. ) - \s* - $ - """, - re.VERBOSE | re.IGNORECASE, + """ ) + _regex = re.compile( + r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _operators = { "==": "equal", "!=": "not_equal", @@ -284,10 +274,8 @@ def _require_version_compare(fn): class Specifier(_IndividualSpecifier): - _regex = re.compile( + _regex_str = ( r""" - ^ - \s* (?P<operator>(~=|==|!=|<=|>=|<|>|===)) (?P<version> (?: @@ -378,12 +366,12 @@ class Specifier(_IndividualSpecifier): (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release ) ) - \s* - $ - """, - re.VERBOSE | re.IGNORECASE, + """ ) + _regex = re.compile( + r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _operators = { "~=": "compatible", "==": "equal", @@ -409,8 +397,8 @@ class Specifier(_IndividualSpecifier): prefix = ".".join( list( itertools.takewhile( - lambda x: (not x.startswith("post") - and not x.startswith("dev")), + lambda x: (not x.startswith("post") and not + x.startswith("dev")), _version_split(spec), ) )[:-1] @@ -419,13 +407,15 @@ class Specifier(_IndividualSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return (self._get_operator(">=")(prospective, spec) - and self._get_operator("==")(prospective, prefix)) + return (self._get_operator(">=")(prospective, spec) and + self._get_operator("==")(prospective, prefix)) @_require_version_compare def _compare_equal(self, prospective, spec): # We need special logic to handle prefix matching if spec.endswith(".*"): + # In the case of prefix matching we want to ignore local segment. + prospective = Version(prospective.public) # Split the spec out by dots, and pretend that there is an implicit # dot in between a release segment and a pre-release segment. spec = _version_split(spec[:-2]) # Remove the trailing .* @@ -577,8 +567,8 @@ def _pad_version(left, right): right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) # Get the rest of our versions - left_split.append(left[len(left_split):]) - right_split.append(left[len(right_split):]) + left_split.append(left[len(left_split[0]):]) + right_split.append(right[len(right_split[0]):]) # Insert our padding left_split.insert( diff --git a/pkg_resources/_vendor/packaging/utils.py b/pkg_resources/_vendor/packaging/utils.py new file mode 100644 index 00000000..942387ce --- /dev/null +++ b/pkg_resources/_vendor/packaging/utils.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import re + + +_canonicalize_regex = re.compile(r"[-_.]+") + + +def canonicalize_name(name): + # This is taken from PEP 503. + return _canonicalize_regex.sub("-", name).lower() diff --git a/pkg_resources/_vendor/packaging/version.py b/pkg_resources/_vendor/packaging/version.py index 4ba574b9..83b5ee8c 100644 --- a/pkg_resources/_vendor/packaging/version.py +++ b/pkg_resources/_vendor/packaging/version.py @@ -1,16 +1,6 @@ -# Copyright 2014 Donald Stufft -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. from __future__ import absolute_import, division, print_function import collections diff --git a/pkg_resources/_vendor/pyparsing.py b/pkg_resources/_vendor/pyparsing.py new file mode 100644 index 00000000..3e02dbee --- /dev/null +++ b/pkg_resources/_vendor/pyparsing.py @@ -0,0 +1,3805 @@ +# module pyparsing.py
+#
+# Copyright (c) 2003-2015 Paul T. McGuire
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__doc__ = \
+"""
+pyparsing module - Classes and methods to define and execute parsing grammars
+
+The pyparsing module is an alternative approach to creating and executing simple grammars,
+vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you
+don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
+provides a library of classes that you use to construct the grammar directly in Python.
+
+Here is a program to parse "Hello, World!" (or any greeting of the form C{"<salutation>, <addressee>!"})::
+
+ from pyparsing import Word, alphas
+
+ # define grammar of a greeting
+ greet = Word( alphas ) + "," + Word( alphas ) + "!"
+
+ hello = "Hello, World!"
+ print (hello, "->", greet.parseString( hello ))
+
+The program outputs the following::
+
+ Hello, World! -> ['Hello', ',', 'World', '!']
+
+The Python representation of the grammar is quite readable, owing to the self-explanatory
+class names, and the use of '+', '|' and '^' operators.
+
+The parsed results returned from C{parseString()} can be accessed as a nested list, a dictionary, or an
+object with named attributes.
+
+The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
+ - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.)
+ - quoted strings
+ - embedded comments
+"""
+
+__version__ = "2.0.6"
+__versionTime__ = "9 Nov 2015 19:03"
+__author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"
+
+import string
+from weakref import ref as wkref
+import copy
+import sys
+import warnings
+import re
+import sre_constants
+import collections
+import pprint
+import functools
+import itertools
+
+#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
+
+__all__ = [
+'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty',
+'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal',
+'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or',
+'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException',
+'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException',
+'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', 'Upcase',
+'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
+'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
+'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
+'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums',
+'htmlComment', 'javaStyleComment', 'keepOriginalText', 'line', 'lineEnd', 'lineStart', 'lineno',
+'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
+'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
+'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity',
+'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
+'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
+'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass',
+]
+
+PY_3 = sys.version.startswith('3')
+if PY_3:
+ _MAX_INT = sys.maxsize
+ basestring = str
+ unichr = chr
+ _ustr = str
+
+ # build list of single arg builtins, that can be used as parse actions
+ singleArgBuiltins = [sum, len, sorted, reversed, list, tuple, set, any, all, min, max]
+
+else:
+ _MAX_INT = sys.maxint
+ range = xrange
+
+ def _ustr(obj):
+ """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
+ str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
+ then < returns the unicode object | encodes it with the default encoding | ... >.
+ """
+ if isinstance(obj,unicode):
+ return obj
+
+ try:
+ # If this works, then _ustr(obj) has the same behaviour as str(obj), so
+ # it won't break any existing code.
+ return str(obj)
+
+ except UnicodeEncodeError:
+ # The Python docs (http://docs.python.org/ref/customization.html#l2h-182)
+ # state that "The return value must be a string object". However, does a
+ # unicode object (being a subclass of basestring) count as a "string
+ # object"?
+ # If so, then return a unicode object:
+ return unicode(obj)
+ # Else encode it... but how? There are many choices... :)
+ # Replace unprintables with escape codes?
+ #return unicode(obj).encode(sys.getdefaultencoding(), 'backslashreplace_errors')
+ # Replace unprintables with question marks?
+ #return unicode(obj).encode(sys.getdefaultencoding(), 'replace')
+ # ...
+
+ # build list of single arg builtins, tolerant of Python version, that can be used as parse actions
+ singleArgBuiltins = []
+ import __builtin__
+ for fname in "sum len sorted reversed list tuple set any all min max".split():
+ try:
+ singleArgBuiltins.append(getattr(__builtin__,fname))
+ except AttributeError:
+ continue
+
+_generatorType = type((y for y in range(1)))
+
+def _xml_escape(data):
+ """Escape &, <, >, ", ', etc. in a string of data."""
+
+ # ampersand must be replaced first
+ from_symbols = '&><"\''
+ to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split())
+ for from_,to_ in zip(from_symbols, to_symbols):
+ data = data.replace(from_, to_)
+ return data
+
+class _Constants(object):
+ pass
+
+alphas = string.ascii_lowercase + string.ascii_uppercase
+nums = "0123456789"
+hexnums = nums + "ABCDEFabcdef"
+alphanums = alphas + nums
+_bslash = chr(92)
+printables = "".join(c for c in string.printable if c not in string.whitespace)
+
+class ParseBaseException(Exception):
+ """base exception class for all parsing runtime exceptions"""
+ # Performance tuning: we construct a *lot* of these, so keep this
+ # constructor as small and fast as possible
+ def __init__( self, pstr, loc=0, msg=None, elem=None ):
+ self.loc = loc
+ if msg is None:
+ self.msg = pstr
+ self.pstr = ""
+ else:
+ self.msg = msg
+ self.pstr = pstr
+ self.parserElement = elem
+
+ def __getattr__( self, aname ):
+ """supported attributes by name are:
+ - lineno - returns the line number of the exception text
+ - col - returns the column number of the exception text
+ - line - returns the line containing the exception text
+ """
+ if( aname == "lineno" ):
+ return lineno( self.loc, self.pstr )
+ elif( aname in ("col", "column") ):
+ return col( self.loc, self.pstr )
+ elif( aname == "line" ):
+ return line( self.loc, self.pstr )
+ else:
+ raise AttributeError(aname)
+
+ def __str__( self ):
+ return "%s (at char %d), (line:%d, col:%d)" % \
+ ( self.msg, self.loc, self.lineno, self.column )
+ def __repr__( self ):
+ return _ustr(self)
+ def markInputline( self, markerString = ">!<" ):
+ """Extracts the exception line from the input string, and marks
+ the location of the exception with a special symbol.
+ """
+ line_str = self.line
+ line_column = self.column - 1
+ if markerString:
+ line_str = "".join((line_str[:line_column],
+ markerString, line_str[line_column:]))
+ return line_str.strip()
+ def __dir__(self):
+ return "loc msg pstr parserElement lineno col line " \
+ "markInputline __str__ __repr__".split()
+
+class ParseException(ParseBaseException):
+ """exception thrown when parse expressions don't match class;
+ supported attributes by name are:
+ - lineno - returns the line number of the exception text
+ - col - returns the column number of the exception text
+ - line - returns the line containing the exception text
+ """
+ pass
+
+class ParseFatalException(ParseBaseException):
+ """user-throwable exception thrown when inconsistent parse content
+ is found; stops all parsing immediately"""
+ pass
+
+class ParseSyntaxException(ParseFatalException):
+ """just like C{L{ParseFatalException}}, but thrown internally when an
+ C{L{ErrorStop<And._ErrorStop>}} ('-' operator) indicates that parsing is to stop immediately because
+ an unbacktrackable syntax error has been found"""
+ def __init__(self, pe):
+ super(ParseSyntaxException, self).__init__(
+ pe.pstr, pe.loc, pe.msg, pe.parserElement)
+
+#~ class ReparseException(ParseBaseException):
+ #~ """Experimental class - parse actions can raise this exception to cause
+ #~ pyparsing to reparse the input string:
+ #~ - with a modified input string, and/or
+ #~ - with a modified start location
+ #~ Set the values of the ReparseException in the constructor, and raise the
+ #~ exception in a parse action to cause pyparsing to use the new string/location.
+ #~ Setting the values as None causes no change to be made.
+ #~ """
+ #~ def __init_( self, newstring, restartLoc ):
+ #~ self.newParseText = newstring
+ #~ self.reparseLoc = restartLoc
+
+class RecursiveGrammarException(Exception):
+ """exception thrown by C{validate()} if the grammar could be improperly recursive"""
+ def __init__( self, parseElementList ):
+ self.parseElementTrace = parseElementList
+
+ def __str__( self ):
+ return "RecursiveGrammarException: %s" % self.parseElementTrace
+
+class _ParseResultsWithOffset(object):
+ def __init__(self,p1,p2):
+ self.tup = (p1,p2)
+ def __getitem__(self,i):
+ return self.tup[i]
+ def __repr__(self):
+ return repr(self.tup)
+ def setOffset(self,i):
+ self.tup = (self.tup[0],i)
+
+class ParseResults(object):
+ """Structured parse results, to provide multiple means of access to the parsed data:
+ - as a list (C{len(results)})
+ - by list index (C{results[0], results[1]}, etc.)
+ - by attribute (C{results.<resultsName>})
+ """
+ def __new__(cls, toklist, name=None, asList=True, modal=True ):
+ if isinstance(toklist, cls):
+ return toklist
+ retobj = object.__new__(cls)
+ retobj.__doinit = True
+ return retobj
+
+ # Performance tuning: we construct a *lot* of these, so keep this
+ # constructor as small and fast as possible
+ def __init__( self, toklist, name=None, asList=True, modal=True, isinstance=isinstance ):
+ if self.__doinit:
+ self.__doinit = False
+ self.__name = None
+ self.__parent = None
+ self.__accumNames = {}
+ if isinstance(toklist, list):
+ self.__toklist = toklist[:]
+ elif isinstance(toklist, _generatorType):
+ self.__toklist = list(toklist)
+ else:
+ self.__toklist = [toklist]
+ self.__tokdict = dict()
+
+ if name is not None and name:
+ if not modal:
+ self.__accumNames[name] = 0
+ if isinstance(name,int):
+ name = _ustr(name) # will always return a str, but use _ustr for consistency
+ self.__name = name
+ if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None,'',[])):
+ if isinstance(toklist,basestring):
+ toklist = [ toklist ]
+ if asList:
+ if isinstance(toklist,ParseResults):
+ self[name] = _ParseResultsWithOffset(toklist.copy(),0)
+ else:
+ self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0)
+ self[name].__name = name
+ else:
+ try:
+ self[name] = toklist[0]
+ except (KeyError,TypeError,IndexError):
+ self[name] = toklist
+
+ def __getitem__( self, i ):
+ if isinstance( i, (int,slice) ):
+ return self.__toklist[i]
+ else:
+ if i not in self.__accumNames:
+ return self.__tokdict[i][-1][0]
+ else:
+ return ParseResults([ v[0] for v in self.__tokdict[i] ])
+
+ def __setitem__( self, k, v, isinstance=isinstance ):
+ if isinstance(v,_ParseResultsWithOffset):
+ self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
+ sub = v[0]
+ elif isinstance(k,int):
+ self.__toklist[k] = v
+ sub = v
+ else:
+ self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)]
+ sub = v
+ if isinstance(sub,ParseResults):
+ sub.__parent = wkref(self)
+
+ def __delitem__( self, i ):
+ if isinstance(i,(int,slice)):
+ mylen = len( self.__toklist )
+ del self.__toklist[i]
+
+ # convert int to slice
+ if isinstance(i, int):
+ if i < 0:
+ i += mylen
+ i = slice(i, i+1)
+ # get removed indices
+ removed = list(range(*i.indices(mylen)))
+ removed.reverse()
+ # fixup indices in token dictionary
+ #~ for name in self.__tokdict:
+ #~ occurrences = self.__tokdict[name]
+ #~ for j in removed:
+ #~ for k, (value, position) in enumerate(occurrences):
+ #~ occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
+ for name,occurrences in self.__tokdict.items():
+ for j in removed:
+ for k, (value, position) in enumerate(occurrences):
+ occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
+ else:
+ del self.__tokdict[i]
+
+ def __contains__( self, k ):
+ return k in self.__tokdict
+
+ def __len__( self ): return len( self.__toklist )
+ def __bool__(self): return len( self.__toklist ) > 0
+ __nonzero__ = __bool__
+ def __iter__( self ): return iter( self.__toklist )
+ def __reversed__( self ): return iter( self.__toklist[::-1] )
+ def iterkeys( self ):
+ """Returns all named result keys."""
+ if hasattr(self.__tokdict, "iterkeys"):
+ return self.__tokdict.iterkeys()
+ else:
+ return iter(self.__tokdict)
+
+ def itervalues( self ):
+ """Returns all named result values."""
+ return (self[k] for k in self.iterkeys())
+
+ def iteritems( self ):
+ return ((k, self[k]) for k in self.iterkeys())
+
+ if PY_3:
+ keys = iterkeys
+ values = itervalues
+ items = iteritems
+ else:
+ def keys( self ):
+ """Returns all named result keys."""
+ return list(self.iterkeys())
+
+ def values( self ):
+ """Returns all named result values."""
+ return list(self.itervalues())
+
+ def items( self ):
+ """Returns all named result keys and values as a list of tuples."""
+ return list(self.iteritems())
+
+ def haskeys( self ):
+ """Since keys() returns an iterator, this method is helpful in bypassing
+ code that looks for the existence of any defined results names."""
+ return bool(self.__tokdict)
+
+ def pop( self, *args, **kwargs):
+ """Removes and returns item at specified index (default=last).
+ Supports both list and dict semantics for pop(). If passed no
+ argument or an integer argument, it will use list semantics
+ and pop tokens from the list of parsed tokens. If passed a
+ non-integer argument (most likely a string), it will use dict
+ semantics and pop the corresponding value from any defined
+ results names. A second default return value argument is
+ supported, just as in dict.pop()."""
+ if not args:
+ args = [-1]
+ for k,v in kwargs.items():
+ if k == 'default':
+ args = (args[0], v)
+ else:
+ raise TypeError("pop() got an unexpected keyword argument '%s'" % k)
+ if (isinstance(args[0], int) or
+ len(args) == 1 or
+ args[0] in self):
+ index = args[0]
+ ret = self[index]
+ del self[index]
+ return ret
+ else:
+ defaultvalue = args[1]
+ return defaultvalue
+
+ def get(self, key, defaultValue=None):
+ """Returns named result matching the given key, or if there is no
+ such name, then returns the given C{defaultValue} or C{None} if no
+ C{defaultValue} is specified."""
+ if key in self:
+ return self[key]
+ else:
+ return defaultValue
+
+ def insert( self, index, insStr ):
+ """Inserts new element at location index in the list of parsed tokens."""
+ self.__toklist.insert(index, insStr)
+ # fixup indices in token dictionary
+ #~ for name in self.__tokdict:
+ #~ occurrences = self.__tokdict[name]
+ #~ for k, (value, position) in enumerate(occurrences):
+ #~ occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
+ for name,occurrences in self.__tokdict.items():
+ for k, (value, position) in enumerate(occurrences):
+ occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
+
+ def append( self, item ):
+ """Add single element to end of ParseResults list of elements."""
+ self.__toklist.append(item)
+
+ def extend( self, itemseq ):
+ """Add sequence of elements to end of ParseResults list of elements."""
+ if isinstance(itemseq, ParseResults):
+ self += itemseq
+ else:
+ self.__toklist.extend(itemseq)
+
+ def clear( self ):
+ """Clear all elements and results names."""
+ del self.__toklist[:]
+ self.__tokdict.clear()
+
+ def __getattr__( self, name ):
+ try:
+ return self[name]
+ except KeyError:
+ return ""
+
+ if name in self.__tokdict:
+ if name not in self.__accumNames:
+ return self.__tokdict[name][-1][0]
+ else:
+ return ParseResults([ v[0] for v in self.__tokdict[name] ])
+ else:
+ return ""
+
+ def __add__( self, other ):
+ ret = self.copy()
+ ret += other
+ return ret
+
+ def __iadd__( self, other ):
+ if other.__tokdict:
+ offset = len(self.__toklist)
+ addoffset = lambda a: offset if a<0 else a+offset
+ otheritems = other.__tokdict.items()
+ otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) )
+ for (k,vlist) in otheritems for v in vlist]
+ for k,v in otherdictitems:
+ self[k] = v
+ if isinstance(v[0],ParseResults):
+ v[0].__parent = wkref(self)
+
+ self.__toklist += other.__toklist
+ self.__accumNames.update( other.__accumNames )
+ return self
+
+ def __radd__(self, other):
+ if isinstance(other,int) and other == 0:
+ return self.copy()
+
+ def __repr__( self ):
+ return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
+
+ def __str__( self ):
+ return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']'
+
+ def _asStringList( self, sep='' ):
+ out = []
+ for item in self.__toklist:
+ if out and sep:
+ out.append(sep)
+ if isinstance( item, ParseResults ):
+ out += item._asStringList()
+ else:
+ out.append( _ustr(item) )
+ return out
+
+ def asList( self ):
+ """Returns the parse results as a nested list of matching tokens, all converted to strings."""
+ return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist]
+
+ def asDict( self ):
+ """Returns the named parse results as dictionary."""
+ if PY_3:
+ return dict( self.items() )
+ else:
+ return dict( self.iteritems() )
+
+ def copy( self ):
+ """Returns a new copy of a C{ParseResults} object."""
+ ret = ParseResults( self.__toklist )
+ ret.__tokdict = self.__tokdict.copy()
+ ret.__parent = self.__parent
+ ret.__accumNames.update( self.__accumNames )
+ ret.__name = self.__name
+ return ret
+
+ def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
+ """Returns the parse results as XML. Tags are created for tokens and lists that have defined results names."""
+ nl = "\n"
+ out = []
+ namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items()
+ for v in vlist)
+ nextLevelIndent = indent + " "
+
+ # collapse out indents if formatting is not desired
+ if not formatted:
+ indent = ""
+ nextLevelIndent = ""
+ nl = ""
+
+ selfTag = None
+ if doctag is not None:
+ selfTag = doctag
+ else:
+ if self.__name:
+ selfTag = self.__name
+
+ if not selfTag:
+ if namedItemsOnly:
+ return ""
+ else:
+ selfTag = "ITEM"
+
+ out += [ nl, indent, "<", selfTag, ">" ]
+
+ for i,res in enumerate(self.__toklist):
+ if isinstance(res,ParseResults):
+ if i in namedItems:
+ out += [ res.asXML(namedItems[i],
+ namedItemsOnly and doctag is None,
+ nextLevelIndent,
+ formatted)]
+ else:
+ out += [ res.asXML(None,
+ namedItemsOnly and doctag is None,
+ nextLevelIndent,
+ formatted)]
+ else:
+ # individual token, see if there is a name for it
+ resTag = None
+ if i in namedItems:
+ resTag = namedItems[i]
+ if not resTag:
+ if namedItemsOnly:
+ continue
+ else:
+ resTag = "ITEM"
+ xmlBodyText = _xml_escape(_ustr(res))
+ out += [ nl, nextLevelIndent, "<", resTag, ">",
+ xmlBodyText,
+ "</", resTag, ">" ]
+
+ out += [ nl, indent, "</", selfTag, ">" ]
+ return "".join(out)
+
+ def __lookup(self,sub):
+ for k,vlist in self.__tokdict.items():
+ for v,loc in vlist:
+ if sub is v:
+ return k
+ return None
+
+ def getName(self):
+ """Returns the results name for this token expression."""
+ if self.__name:
+ return self.__name
+ elif self.__parent:
+ par = self.__parent()
+ if par:
+ return par.__lookup(self)
+ else:
+ return None
+ elif (len(self) == 1 and
+ len(self.__tokdict) == 1 and
+ self.__tokdict.values()[0][0][1] in (0,-1)):
+ return self.__tokdict.keys()[0]
+ else:
+ return None
+
+ def dump(self,indent='',depth=0):
+ """Diagnostic method for listing out the contents of a C{ParseResults}.
+ Accepts an optional C{indent} argument so that this string can be embedded
+ in a nested display of other data."""
+ out = []
+ NL = '\n'
+ out.append( indent+_ustr(self.asList()) )
+ if self.haskeys():
+ items = sorted(self.items())
+ for k,v in items:
+ if out:
+ out.append(NL)
+ out.append( "%s%s- %s: " % (indent,(' '*depth), k) )
+ if isinstance(v,ParseResults):
+ if v:
+ out.append( v.dump(indent,depth+1) )
+ else:
+ out.append(_ustr(v))
+ else:
+ out.append(_ustr(v))
+ elif any(isinstance(vv,ParseResults) for vv in self):
+ v = self
+ for i,vv in enumerate(v):
+ if isinstance(vv,ParseResults):
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) ))
+ else:
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv)))
+
+ return "".join(out)
+
+ def pprint(self, *args, **kwargs):
+ """Pretty-printer for parsed results as a list, using the C{pprint} module.
+ Accepts additional positional or keyword args as defined for the
+ C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint})"""
+ pprint.pprint(self.asList(), *args, **kwargs)
+
+ # add support for pickle protocol
+ def __getstate__(self):
+ return ( self.__toklist,
+ ( self.__tokdict.copy(),
+ self.__parent is not None and self.__parent() or None,
+ self.__accumNames,
+ self.__name ) )
+
+ def __setstate__(self,state):
+ self.__toklist = state[0]
+ (self.__tokdict,
+ par,
+ inAccumNames,
+ self.__name) = state[1]
+ self.__accumNames = {}
+ self.__accumNames.update(inAccumNames)
+ if par is not None:
+ self.__parent = wkref(par)
+ else:
+ self.__parent = None
+
+ def __dir__(self):
+ return dir(super(ParseResults,self)) + list(self.keys())
+
+collections.MutableMapping.register(ParseResults)
+
+def col (loc,strg):
+ """Returns current column within a string, counting newlines as line separators.
+ The first column is number 1.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ s = strg
+ return 1 if loc<len(s) and s[loc] == '\n' else loc - s.rfind("\n", 0, loc)
+
+def lineno(loc,strg):
+ """Returns current line number within a string, counting newlines as line separators.
+ The first line is number 1.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ return strg.count("\n",0,loc) + 1
+
+def line( loc, strg ):
+ """Returns the line of text containing loc within a string, counting newlines as line separators.
+ """
+ lastCR = strg.rfind("\n", 0, loc)
+ nextCR = strg.find("\n", loc)
+ if nextCR >= 0:
+ return strg[lastCR+1:nextCR]
+ else:
+ return strg[lastCR+1:]
+
+def _defaultStartDebugAction( instring, loc, expr ):
+ print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )))
+
+def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
+ print ("Matched " + _ustr(expr) + " -> " + str(toks.asList()))
+
+def _defaultExceptionDebugAction( instring, loc, expr, exc ):
+ print ("Exception raised:" + _ustr(exc))
+
+def nullDebugAction(*args):
+ """'Do-nothing' debug action, to suppress debugging output during parsing."""
+ pass
+
+# Only works on Python 3.x - nonlocal is toxic to Python 2 installs
+#~ 'decorator to trim function calls to match the arity of the target'
+#~ def _trim_arity(func, maxargs=3):
+ #~ if func in singleArgBuiltins:
+ #~ return lambda s,l,t: func(t)
+ #~ limit = 0
+ #~ foundArity = False
+ #~ def wrapper(*args):
+ #~ nonlocal limit,foundArity
+ #~ while 1:
+ #~ try:
+ #~ ret = func(*args[limit:])
+ #~ foundArity = True
+ #~ return ret
+ #~ except TypeError:
+ #~ if limit == maxargs or foundArity:
+ #~ raise
+ #~ limit += 1
+ #~ continue
+ #~ return wrapper
+
+# this version is Python 2.x-3.x cross-compatible
+'decorator to trim function calls to match the arity of the target'
+def _trim_arity(func, maxargs=2):
+ if func in singleArgBuiltins:
+ return lambda s,l,t: func(t)
+ limit = [0]
+ foundArity = [False]
+ def wrapper(*args):
+ while 1:
+ try:
+ ret = func(*args[limit[0]:])
+ foundArity[0] = True
+ return ret
+ except TypeError:
+ if limit[0] <= maxargs and not foundArity[0]:
+ limit[0] += 1
+ continue
+ raise
+ return wrapper
+
+class ParserElement(object):
+ """Abstract base level parser element class."""
+ DEFAULT_WHITE_CHARS = " \n\t\r"
+ verbose_stacktrace = False
+
+ @staticmethod
+ def setDefaultWhitespaceChars( chars ):
+ """Overrides the default whitespace chars
+ """
+ ParserElement.DEFAULT_WHITE_CHARS = chars
+
+ @staticmethod
+ def inlineLiteralsUsing(cls):
+ """
+ Set class to be used for inclusion of string literals into a parser.
+ """
+ ParserElement.literalStringClass = cls
+
+ def __init__( self, savelist=False ):
+ self.parseAction = list()
+ self.failAction = None
+ #~ self.name = "<unknown>" # don't define self.name, let subclasses try/except upcall
+ self.strRepr = None
+ self.resultsName = None
+ self.saveAsList = savelist
+ self.skipWhitespace = True
+ self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+ self.copyDefaultWhiteChars = True
+ self.mayReturnEmpty = False # used when checking for left-recursion
+ self.keepTabs = False
+ self.ignoreExprs = list()
+ self.debug = False
+ self.streamlined = False
+ self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index
+ self.errmsg = ""
+ self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all)
+ self.debugActions = ( None, None, None ) #custom debug actions
+ self.re = None
+ self.callPreparse = True # used to avoid redundant calls to preParse
+ self.callDuringTry = False
+
+ def copy( self ):
+ """Make a copy of this C{ParserElement}. Useful for defining different parse actions
+ for the same parsing pattern, using copies of the original parse element."""
+ cpy = copy.copy( self )
+ cpy.parseAction = self.parseAction[:]
+ cpy.ignoreExprs = self.ignoreExprs[:]
+ if self.copyDefaultWhiteChars:
+ cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+ return cpy
+
+ def setName( self, name ):
+ """Define name for this expression, for use in debugging."""
+ self.name = name
+ self.errmsg = "Expected " + self.name
+ if hasattr(self,"exception"):
+ self.exception.msg = self.errmsg
+ return self
+
+ def setResultsName( self, name, listAllMatches=False ):
+ """Define name for referencing matching tokens as a nested attribute
+ of the returned parse results.
+ NOTE: this returns a *copy* of the original C{ParserElement} object;
+ this is so that the client can define a basic element, such as an
+ integer, and reference it in multiple places with different names.
+
+ You can also set results names using the abbreviated syntax,
+ C{expr("name")} in place of C{expr.setResultsName("name")} -
+ see L{I{__call__}<__call__>}.
+ """
+ newself = self.copy()
+ if name.endswith("*"):
+ name = name[:-1]
+ listAllMatches=True
+ newself.resultsName = name
+ newself.modalResults = not listAllMatches
+ return newself
+
+ def setBreak(self,breakFlag = True):
+ """Method to invoke the Python pdb debugger when this element is
+ about to be parsed. Set C{breakFlag} to True to enable, False to
+ disable.
+ """
+ if breakFlag:
+ _parseMethod = self._parse
+ def breaker(instring, loc, doActions=True, callPreParse=True):
+ import pdb
+ pdb.set_trace()
+ return _parseMethod( instring, loc, doActions, callPreParse )
+ breaker._originalParseMethod = _parseMethod
+ self._parse = breaker
+ else:
+ if hasattr(self._parse,"_originalParseMethod"):
+ self._parse = self._parse._originalParseMethod
+ return self
+
+ def setParseAction( self, *fns, **kwargs ):
+ """Define action to perform when successfully matching parse element definition.
+ Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)},
+ C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
+ - s = the original string being parsed (see note below)
+ - loc = the location of the matching substring
+ - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object
+ If the functions in fns modify the tokens, they can return them as the return
+ value from fn, and the modified list of tokens will replace the original.
+ Otherwise, fn does not need to return any value.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{parseString}<parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ self.parseAction = list(map(_trim_arity, list(fns)))
+ self.callDuringTry = kwargs.get("callDuringTry", False)
+ return self
+
+ def addParseAction( self, *fns, **kwargs ):
+ """Add parse action to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}."""
+ self.parseAction += list(map(_trim_arity, list(fns)))
+ self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+ return self
+
+ def addCondition(self, *fns, **kwargs):
+ """Add a boolean predicate function to expression's list of parse actions. See
+ L{I{setParseAction}<setParseAction>}. Optional keyword argument C{message} can
+ be used to define a custom message to be used in the raised exception."""
+ msg = kwargs.get("message") or "failed user-defined condition"
+ for fn in fns:
+ def pa(s,l,t):
+ if not bool(_trim_arity(fn)(s,l,t)):
+ raise ParseException(s,l,msg)
+ return t
+ self.parseAction.append(pa)
+ self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+ return self
+
+ def setFailAction( self, fn ):
+ """Define action to perform if parsing fails at this expression.
+ Fail acton fn is a callable function that takes the arguments
+ C{fn(s,loc,expr,err)} where:
+ - s = string being parsed
+ - loc = location where expression match was attempted and failed
+ - expr = the parse expression that failed
+ - err = the exception thrown
+ The function returns no value. It may throw C{L{ParseFatalException}}
+ if it is desired to stop parsing immediately."""
+ self.failAction = fn
+ return self
+
+ def _skipIgnorables( self, instring, loc ):
+ exprsFound = True
+ while exprsFound:
+ exprsFound = False
+ for e in self.ignoreExprs:
+ try:
+ while 1:
+ loc,dummy = e._parse( instring, loc )
+ exprsFound = True
+ except ParseException:
+ pass
+ return loc
+
+ def preParse( self, instring, loc ):
+ if self.ignoreExprs:
+ loc = self._skipIgnorables( instring, loc )
+
+ if self.skipWhitespace:
+ wt = self.whiteChars
+ instrlen = len(instring)
+ while loc < instrlen and instring[loc] in wt:
+ loc += 1
+
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ return loc, []
+
+ def postParse( self, instring, loc, tokenlist ):
+ return tokenlist
+
+ #~ @profile
+ def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ):
+ debugging = ( self.debug ) #and doActions )
+
+ if debugging or self.failAction:
+ #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
+ if (self.debugActions[0] ):
+ self.debugActions[0]( instring, loc, self )
+ if callPreParse and self.callPreparse:
+ preloc = self.preParse( instring, loc )
+ else:
+ preloc = loc
+ tokensStart = preloc
+ try:
+ try:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+ except IndexError:
+ raise ParseException( instring, len(instring), self.errmsg, self )
+ except ParseBaseException as err:
+ #~ print ("Exception raised:", err)
+ if self.debugActions[2]:
+ self.debugActions[2]( instring, tokensStart, self, err )
+ if self.failAction:
+ self.failAction( instring, tokensStart, self, err )
+ raise
+ else:
+ if callPreParse and self.callPreparse:
+ preloc = self.preParse( instring, loc )
+ else:
+ preloc = loc
+ tokensStart = preloc
+ if self.mayIndexError or loc >= len(instring):
+ try:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+ except IndexError:
+ raise ParseException( instring, len(instring), self.errmsg, self )
+ else:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+
+ tokens = self.postParse( instring, loc, tokens )
+
+ retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults )
+ if self.parseAction and (doActions or self.callDuringTry):
+ if debugging:
+ try:
+ for fn in self.parseAction:
+ tokens = fn( instring, tokensStart, retTokens )
+ if tokens is not None:
+ retTokens = ParseResults( tokens,
+ self.resultsName,
+ asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+ modal=self.modalResults )
+ except ParseBaseException as err:
+ #~ print "Exception raised in user parse action:", err
+ if (self.debugActions[2] ):
+ self.debugActions[2]( instring, tokensStart, self, err )
+ raise
+ else:
+ for fn in self.parseAction:
+ tokens = fn( instring, tokensStart, retTokens )
+ if tokens is not None:
+ retTokens = ParseResults( tokens,
+ self.resultsName,
+ asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+ modal=self.modalResults )
+
+ if debugging:
+ #~ print ("Matched",self,"->",retTokens.asList())
+ if (self.debugActions[1] ):
+ self.debugActions[1]( instring, tokensStart, loc, self, retTokens )
+
+ return loc, retTokens
+
+ def tryParse( self, instring, loc ):
+ try:
+ return self._parse( instring, loc, doActions=False )[0]
+ except ParseFatalException:
+ raise ParseException( instring, loc, self.errmsg, self)
+
+ # this method gets repeatedly called during backtracking with the same arguments -
+ # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
+ def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
+ lookup = (self,instring,loc,callPreParse,doActions)
+ if lookup in ParserElement._exprArgCache:
+ value = ParserElement._exprArgCache[ lookup ]
+ if isinstance(value, Exception):
+ raise value
+ return (value[0],value[1].copy())
+ else:
+ try:
+ value = self._parseNoCache( instring, loc, doActions, callPreParse )
+ ParserElement._exprArgCache[ lookup ] = (value[0],value[1].copy())
+ return value
+ except ParseBaseException as pe:
+ pe.__traceback__ = None
+ ParserElement._exprArgCache[ lookup ] = pe
+ raise
+
+ _parse = _parseNoCache
+
+ # argument cache for optimizing repeated calls when backtracking through recursive expressions
+ _exprArgCache = {}
+ @staticmethod
+ def resetCache():
+ ParserElement._exprArgCache.clear()
+
+ _packratEnabled = False
+ @staticmethod
+ def enablePackrat():
+ """Enables "packrat" parsing, which adds memoizing to the parsing logic.
+ Repeated parse attempts at the same string location (which happens
+ often in many complex grammars) can immediately return a cached value,
+ instead of re-executing parsing/validating code. Memoizing is done of
+ both valid results and parsing exceptions.
+
+ This speedup may break existing programs that use parse actions that
+ have side-effects. For this reason, packrat parsing is disabled when
+ you first import pyparsing. To activate the packrat feature, your
+ program must call the class method C{ParserElement.enablePackrat()}. If
+ your program uses C{psyco} to "compile as you go", you must call
+ C{enablePackrat} before calling C{psyco.full()}. If you do not do this,
+ Python will crash. For best results, call C{enablePackrat()} immediately
+ after importing pyparsing.
+ """
+ if not ParserElement._packratEnabled:
+ ParserElement._packratEnabled = True
+ ParserElement._parse = ParserElement._parseCache
+
+ def parseString( self, instring, parseAll=False ):
+ """Execute the parse expression with the given string.
+ This is the main interface to the client code, once the complete
+ expression has been built.
+
+ If you want the grammar to require that the entire input string be
+ successfully parsed, then set C{parseAll} to True (equivalent to ending
+ the grammar with C{L{StringEnd()}}).
+
+ Note: C{parseString} implicitly calls C{expandtabs()} on the input string,
+ in order to report proper column numbers in parse actions.
+ If the input string contains tabs and
+ the grammar uses parse actions that use the C{loc} argument to index into the
+ string being parsed, you can ensure you have a consistent view of the input
+ string by:
+ - calling C{parseWithTabs} on your grammar before calling C{parseString}
+ (see L{I{parseWithTabs}<parseWithTabs>})
+ - define your parse action using the full C{(s,loc,toks)} signature, and
+ reference the input string using the parse action's C{s} argument
+ - explictly expand the tabs in your input string before calling
+ C{parseString}
+ """
+ ParserElement.resetCache()
+ if not self.streamlined:
+ self.streamline()
+ #~ self.saveAsList = True
+ for e in self.ignoreExprs:
+ e.streamline()
+ if not self.keepTabs:
+ instring = instring.expandtabs()
+ try:
+ loc, tokens = self._parse( instring, 0 )
+ if parseAll:
+ loc = self.preParse( instring, loc )
+ se = Empty() + StringEnd()
+ se._parse( instring, loc )
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+ else:
+ return tokens
+
+ def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ):
+ """Scan the input string for expression matches. Each match will return the
+ matching tokens, start location, and end location. May be called with optional
+ C{maxMatches} argument, to clip scanning after 'n' matches are found. If
+ C{overlap} is specified, then overlapping matches will be reported.
+
+ Note that the start and end locations are reported relative to the string
+ being parsed. See L{I{parseString}<parseString>} for more information on parsing
+ strings with embedded tabs."""
+ if not self.streamlined:
+ self.streamline()
+ for e in self.ignoreExprs:
+ e.streamline()
+
+ if not self.keepTabs:
+ instring = _ustr(instring).expandtabs()
+ instrlen = len(instring)
+ loc = 0
+ preparseFn = self.preParse
+ parseFn = self._parse
+ ParserElement.resetCache()
+ matches = 0
+ try:
+ while loc <= instrlen and matches < maxMatches:
+ try:
+ preloc = preparseFn( instring, loc )
+ nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )
+ except ParseException:
+ loc = preloc+1
+ else:
+ if nextLoc > loc:
+ matches += 1
+ yield tokens, preloc, nextLoc
+ if overlap:
+ nextloc = preparseFn( instring, loc )
+ if nextloc > loc:
+ loc = nextLoc
+ else:
+ loc += 1
+ else:
+ loc = nextLoc
+ else:
+ loc = preloc+1
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def transformString( self, instring ):
+ """Extension to C{L{scanString}}, to modify matching text with modified tokens that may
+ be returned from a parse action. To use C{transformString}, define a grammar and
+ attach a parse action to it that modifies the returned token list.
+ Invoking C{transformString()} on a target string will then scan for matches,
+ and replace the matched text patterns according to the logic in the parse
+ action. C{transformString()} returns the resulting transformed string."""
+ out = []
+ lastE = 0
+ # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
+ # keep string locs straight between transformString and scanString
+ self.keepTabs = True
+ try:
+ for t,s,e in self.scanString( instring ):
+ out.append( instring[lastE:s] )
+ if t:
+ if isinstance(t,ParseResults):
+ out += t.asList()
+ elif isinstance(t,list):
+ out += t
+ else:
+ out.append(t)
+ lastE = e
+ out.append(instring[lastE:])
+ out = [o for o in out if o]
+ return "".join(map(_ustr,_flatten(out)))
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def searchString( self, instring, maxMatches=_MAX_INT ):
+ """Another extension to C{L{scanString}}, simplifying the access to the tokens found
+ to match the given parse expression. May be called with optional
+ C{maxMatches} argument, to clip searching after 'n' matches are found.
+ """
+ try:
+ return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def __add__(self, other ):
+ """Implementation of + operator - returns C{L{And}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return And( [ self, other ] )
+
+ def __radd__(self, other ):
+ """Implementation of + operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other + self
+
+ def __sub__(self, other):
+ """Implementation of - operator, returns C{L{And}} with error stop"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return And( [ self, And._ErrorStop(), other ] )
+
+ def __rsub__(self, other ):
+ """Implementation of - operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other - self
+
+ def __mul__(self,other):
+ """Implementation of * operator, allows use of C{expr * 3} in place of
+ C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer
+ tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples
+ may also include C{None} as in:
+ - C{expr*(n,None)} or C{expr*(n,)} is equivalent
+ to C{expr*n + L{ZeroOrMore}(expr)}
+ (read as "at least n instances of C{expr}")
+ - C{expr*(None,n)} is equivalent to C{expr*(0,n)}
+ (read as "0 to n instances of C{expr}")
+ - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)}
+ - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)}
+
+ Note that C{expr*(None,n)} does not raise an exception if
+ more than n exprs exist in the input stream; that is,
+ C{expr*(None,n)} does not enforce a maximum number of expr
+ occurrences. If this behavior is desired, then write
+ C{expr*(None,n) + ~expr}
+
+ """
+ if isinstance(other,int):
+ minElements, optElements = other,0
+ elif isinstance(other,tuple):
+ other = (other + (None, None))[:2]
+ if other[0] is None:
+ other = (0, other[1])
+ if isinstance(other[0],int) and other[1] is None:
+ if other[0] == 0:
+ return ZeroOrMore(self)
+ if other[0] == 1:
+ return OneOrMore(self)
+ else:
+ return self*other[0] + ZeroOrMore(self)
+ elif isinstance(other[0],int) and isinstance(other[1],int):
+ minElements, optElements = other
+ optElements -= minElements
+ else:
+ raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1]))
+ else:
+ raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other))
+
+ if minElements < 0:
+ raise ValueError("cannot multiply ParserElement by negative value")
+ if optElements < 0:
+ raise ValueError("second tuple value must be greater or equal to first tuple value")
+ if minElements == optElements == 0:
+ raise ValueError("cannot multiply ParserElement by 0 or (0,0)")
+
+ if (optElements):
+ def makeOptionalList(n):
+ if n>1:
+ return Optional(self + makeOptionalList(n-1))
+ else:
+ return Optional(self)
+ if minElements:
+ if minElements == 1:
+ ret = self + makeOptionalList(optElements)
+ else:
+ ret = And([self]*minElements) + makeOptionalList(optElements)
+ else:
+ ret = makeOptionalList(optElements)
+ else:
+ if minElements == 1:
+ ret = self
+ else:
+ ret = And([self]*minElements)
+ return ret
+
+ def __rmul__(self, other):
+ return self.__mul__(other)
+
+ def __or__(self, other ):
+ """Implementation of | operator - returns C{L{MatchFirst}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return MatchFirst( [ self, other ] )
+
+ def __ror__(self, other ):
+ """Implementation of | operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other | self
+
+ def __xor__(self, other ):
+ """Implementation of ^ operator - returns C{L{Or}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return Or( [ self, other ] )
+
+ def __rxor__(self, other ):
+ """Implementation of ^ operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other ^ self
+
+ def __and__(self, other ):
+ """Implementation of & operator - returns C{L{Each}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return Each( [ self, other ] )
+
+ def __rand__(self, other ):
+ """Implementation of & operator when left operand is not a C{L{ParserElement}}"""
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other & self
+
+ def __invert__( self ):
+ """Implementation of ~ operator - returns C{L{NotAny}}"""
+ return NotAny( self )
+
+ def __call__(self, name=None):
+ """Shortcut for C{L{setResultsName}}, with C{listAllMatches=default}::
+ userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
+ could be written as::
+ userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
+
+ If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be
+ passed as C{True}.
+
+ If C{name} is omitted, same as calling C{L{copy}}.
+ """
+ if name is not None:
+ return self.setResultsName(name)
+ else:
+ return self.copy()
+
+ def suppress( self ):
+ """Suppresses the output of this C{ParserElement}; useful to keep punctuation from
+ cluttering up returned output.
+ """
+ return Suppress( self )
+
+ def leaveWhitespace( self ):
+ """Disables the skipping of whitespace before matching the characters in the
+ C{ParserElement}'s defined pattern. This is normally only used internally by
+ the pyparsing module, but may be needed in some whitespace-sensitive grammars.
+ """
+ self.skipWhitespace = False
+ return self
+
+ def setWhitespaceChars( self, chars ):
+ """Overrides the default whitespace chars
+ """
+ self.skipWhitespace = True
+ self.whiteChars = chars
+ self.copyDefaultWhiteChars = False
+ return self
+
+ def parseWithTabs( self ):
+ """Overrides default behavior to expand C{<TAB>}s to spaces before parsing the input string.
+ Must be called before C{parseString} when the input grammar contains elements that
+ match C{<TAB>} characters."""
+ self.keepTabs = True
+ return self
+
+ def ignore( self, other ):
+ """Define expression to be ignored (e.g., comments) while doing pattern
+ matching; may be called repeatedly, to define multiple comment or other
+ ignorable patterns.
+ """
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ self.ignoreExprs.append( other.copy() )
+ else:
+ self.ignoreExprs.append( Suppress( other.copy() ) )
+ return self
+
+ def setDebugActions( self, startAction, successAction, exceptionAction ):
+ """Enable display of debugging messages while doing pattern matching."""
+ self.debugActions = (startAction or _defaultStartDebugAction,
+ successAction or _defaultSuccessDebugAction,
+ exceptionAction or _defaultExceptionDebugAction)
+ self.debug = True
+ return self
+
+ def setDebug( self, flag=True ):
+ """Enable display of debugging messages while doing pattern matching.
+ Set C{flag} to True to enable, False to disable."""
+ if flag:
+ self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
+ else:
+ self.debug = False
+ return self
+
+ def __str__( self ):
+ return self.name
+
+ def __repr__( self ):
+ return _ustr(self)
+
+ def streamline( self ):
+ self.streamlined = True
+ self.strRepr = None
+ return self
+
+ def checkRecursion( self, parseElementList ):
+ pass
+
+ def validate( self, validateTrace=[] ):
+ """Check defined expressions for valid structure, check for infinite recursive definitions."""
+ self.checkRecursion( [] )
+
+ def parseFile( self, file_or_filename, parseAll=False ):
+ """Execute the parse expression on the given file or filename.
+ If a filename is specified (instead of a file object),
+ the entire file is opened, read, and closed before parsing.
+ """
+ try:
+ file_contents = file_or_filename.read()
+ except AttributeError:
+ f = open(file_or_filename, "r")
+ file_contents = f.read()
+ f.close()
+ try:
+ return self.parseString(file_contents, parseAll)
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def __eq__(self,other):
+ if isinstance(other, ParserElement):
+ return self is other or self.__dict__ == other.__dict__
+ elif isinstance(other, basestring):
+ try:
+ self.parseString(_ustr(other), parseAll=True)
+ return True
+ except ParseBaseException:
+ return False
+ else:
+ return super(ParserElement,self)==other
+
+ def __ne__(self,other):
+ return not (self == other)
+
+ def __hash__(self):
+ return hash(id(self))
+
+ def __req__(self,other):
+ return self == other
+
+ def __rne__(self,other):
+ return not (self == other)
+
+ def runTests(self, tests, parseAll=False):
+ """Execute the parse expression on a series of test strings, showing each
+ test, the parsed results or where the parse failed. Quick and easy way to
+ run a parse expression against a list of sample strings.
+
+ Parameters:
+ - tests - a list of separate test strings, or a multiline string of test strings
+ - parseAll - (default=False) - flag to pass to C{L{parseString}} when running tests
+ """
+ if isinstance(tests, basestring):
+ tests = map(str.strip, tests.splitlines())
+ for t in tests:
+ out = [t]
+ try:
+ out.append(self.parseString(t, parseAll=parseAll).dump())
+ except ParseException as pe:
+ if '\n' in t:
+ out.append(line(pe.loc, t))
+ out.append(' '*(col(pe.loc,t)-1) + '^')
+ else:
+ out.append(' '*pe.loc + '^')
+ out.append(str(pe))
+ out.append('')
+ print('\n'.join(out))
+
+
+class Token(ParserElement):
+ """Abstract C{ParserElement} subclass, for defining atomic matching patterns."""
+ def __init__( self ):
+ super(Token,self).__init__( savelist=False )
+
+
+class Empty(Token):
+ """An empty token, will always match."""
+ def __init__( self ):
+ super(Empty,self).__init__()
+ self.name = "Empty"
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+
+
+class NoMatch(Token):
+ """A token that will never match."""
+ def __init__( self ):
+ super(NoMatch,self).__init__()
+ self.name = "NoMatch"
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+ self.errmsg = "Unmatchable token"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Literal(Token):
+ """Token to exactly match a specified string."""
+ def __init__( self, matchString ):
+ super(Literal,self).__init__()
+ self.match = matchString
+ self.matchLen = len(matchString)
+ try:
+ self.firstMatchChar = matchString[0]
+ except IndexError:
+ warnings.warn("null string passed to Literal; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+ self.__class__ = Empty
+ self.name = '"%s"' % _ustr(self.match)
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = False
+ self.mayIndexError = False
+
+ # Performance tuning: this routine gets called a *lot*
+ # if this is a single character match string and the first character matches,
+ # short-circuit as quickly as possible, and avoid calling startswith
+ #~ @profile
+ def parseImpl( self, instring, loc, doActions=True ):
+ if (instring[loc] == self.firstMatchChar and
+ (self.matchLen==1 or instring.startswith(self.match,loc)) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+_L = Literal
+ParserElement.literalStringClass = Literal
+
+class Keyword(Token):
+ """Token to exactly match a specified string as a keyword, that is, it must be
+ immediately followed by a non-keyword character. Compare with C{L{Literal}}::
+ Literal("if") will match the leading C{'if'} in C{'ifAndOnlyIf'}.
+ Keyword("if") will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'}
+ Accepts two optional constructor arguments in addition to the keyword string:
+ C{identChars} is a string of characters that would be valid identifier characters,
+ defaulting to all alphanumerics + "_" and "$"; C{caseless} allows case-insensitive
+ matching, default is C{False}.
+ """
+ DEFAULT_KEYWORD_CHARS = alphanums+"_$"
+
+ def __init__( self, matchString, identChars=DEFAULT_KEYWORD_CHARS, caseless=False ):
+ super(Keyword,self).__init__()
+ self.match = matchString
+ self.matchLen = len(matchString)
+ try:
+ self.firstMatchChar = matchString[0]
+ except IndexError:
+ warnings.warn("null string passed to Keyword; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+ self.name = '"%s"' % self.match
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = False
+ self.mayIndexError = False
+ self.caseless = caseless
+ if caseless:
+ self.caselessmatch = matchString.upper()
+ identChars = identChars.upper()
+ self.identChars = set(identChars)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.caseless:
+ if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and
+ (loc == 0 or instring[loc-1].upper() not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ else:
+ if (instring[loc] == self.firstMatchChar and
+ (self.matchLen==1 or instring.startswith(self.match,loc)) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
+ (loc == 0 or instring[loc-1] not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ def copy(self):
+ c = super(Keyword,self).copy()
+ c.identChars = Keyword.DEFAULT_KEYWORD_CHARS
+ return c
+
+ @staticmethod
+ def setDefaultKeywordChars( chars ):
+ """Overrides the default Keyword chars
+ """
+ Keyword.DEFAULT_KEYWORD_CHARS = chars
+
+class CaselessLiteral(Literal):
+ """Token to match a specified string, ignoring case of letters.
+ Note: the matched results will always be in the case of the given
+ match string, NOT the case of the input text.
+ """
+ def __init__( self, matchString ):
+ super(CaselessLiteral,self).__init__( matchString.upper() )
+ # Preserve the defining literal.
+ self.returnString = matchString
+ self.name = "'%s'" % self.returnString
+ self.errmsg = "Expected " + self.name
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if instring[ loc:loc+self.matchLen ].upper() == self.match:
+ return loc+self.matchLen, self.returnString
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class CaselessKeyword(Keyword):
+ def __init__( self, matchString, identChars=Keyword.DEFAULT_KEYWORD_CHARS ):
+ super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class Word(Token):
+ """Token for matching words composed of allowed character sets.
+ Defined with string containing all allowed initial characters,
+ an optional string containing allowed body characters (if omitted,
+ defaults to the initial character set), and an optional minimum,
+ maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction. An optional
+ C{exclude} parameter can list characters that might be found in
+ the input C{bodyChars} string; useful to define a word of all printables
+ except for one or two characters, for instance.
+ """
+ def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ):
+ super(Word,self).__init__()
+ if excludeChars:
+ initChars = ''.join(c for c in initChars if c not in excludeChars)
+ if bodyChars:
+ bodyChars = ''.join(c for c in bodyChars if c not in excludeChars)
+ self.initCharsOrig = initChars
+ self.initChars = set(initChars)
+ if bodyChars :
+ self.bodyCharsOrig = bodyChars
+ self.bodyChars = set(bodyChars)
+ else:
+ self.bodyCharsOrig = initChars
+ self.bodyChars = set(initChars)
+
+ self.maxSpecified = max > 0
+
+ if min < 1:
+ raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted")
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.asKeyword = asKeyword
+
+ if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0):
+ if self.bodyCharsOrig == self.initCharsOrig:
+ self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig)
+ elif len(self.initCharsOrig) == 1:
+ self.reString = "%s[%s]*" % \
+ (re.escape(self.initCharsOrig),
+ _escapeRegexRangeChars(self.bodyCharsOrig),)
+ else:
+ self.reString = "[%s][%s]*" % \
+ (_escapeRegexRangeChars(self.initCharsOrig),
+ _escapeRegexRangeChars(self.bodyCharsOrig),)
+ if self.asKeyword:
+ self.reString = r"\b"+self.reString+r"\b"
+ try:
+ self.re = re.compile( self.reString )
+ except:
+ self.re = None
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.re:
+ result = self.re.match(instring,loc)
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ return loc, result.group()
+
+ if not(instring[ loc ] in self.initChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ start = loc
+ loc += 1
+ instrlen = len(instring)
+ bodychars = self.bodyChars
+ maxloc = start + self.maxLen
+ maxloc = min( maxloc, instrlen )
+ while loc < maxloc and instring[loc] in bodychars:
+ loc += 1
+
+ throwException = False
+ if loc - start < self.minLen:
+ throwException = True
+ if self.maxSpecified and loc < instrlen and instring[loc] in bodychars:
+ throwException = True
+ if self.asKeyword:
+ if (start>0 and instring[start-1] in bodychars) or (loc<instrlen and instring[loc] in bodychars):
+ throwException = True
+
+ if throwException:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+ def __str__( self ):
+ try:
+ return super(Word,self).__str__()
+ except:
+ pass
+
+
+ if self.strRepr is None:
+
+ def charsAsStr(s):
+ if len(s)>4:
+ return s[:4]+"..."
+ else:
+ return s
+
+ if ( self.initCharsOrig != self.bodyCharsOrig ):
+ self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) )
+ else:
+ self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig)
+
+ return self.strRepr
+
+
+class Regex(Token):
+ """Token for matching strings that match a given regular expression.
+ Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
+ """
+ compiledREtype = type(re.compile("[A-Z]"))
+ def __init__( self, pattern, flags=0):
+ """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags."""
+ super(Regex,self).__init__()
+
+ if isinstance(pattern, basestring):
+ if len(pattern) == 0:
+ warnings.warn("null string passed to Regex; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+
+ self.pattern = pattern
+ self.flags = flags
+
+ try:
+ self.re = re.compile(self.pattern, self.flags)
+ self.reString = self.pattern
+ except sre_constants.error:
+ warnings.warn("invalid pattern (%s) passed to Regex" % pattern,
+ SyntaxWarning, stacklevel=2)
+ raise
+
+ elif isinstance(pattern, Regex.compiledREtype):
+ self.re = pattern
+ self.pattern = \
+ self.reString = str(pattern)
+ self.flags = flags
+
+ else:
+ raise ValueError("Regex may only be constructed with a string or a compiled RE object")
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ result = self.re.match(instring,loc)
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ d = result.groupdict()
+ ret = ParseResults(result.group())
+ if d:
+ for k in d:
+ ret[k] = d[k]
+ return loc,ret
+
+ def __str__( self ):
+ try:
+ return super(Regex,self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "Re:(%s)" % repr(self.pattern)
+
+ return self.strRepr
+
+
+class QuotedString(Token):
+ """Token for matching strings that are delimited by quoting characters.
+ """
+ def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None):
+ """
+ Defined with the following parameters:
+ - quoteChar - string of one or more characters defining the quote delimiting string
+ - escChar - character to escape quotes, typically backslash (default=None)
+ - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=None)
+ - multiline - boolean indicating whether quotes can span multiple lines (default=C{False})
+ - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True})
+ - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar)
+ """
+ super(QuotedString,self).__init__()
+
+ # remove white space from quote chars - wont work anyway
+ quoteChar = quoteChar.strip()
+ if len(quoteChar) == 0:
+ warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+ raise SyntaxError()
+
+ if endQuoteChar is None:
+ endQuoteChar = quoteChar
+ else:
+ endQuoteChar = endQuoteChar.strip()
+ if len(endQuoteChar) == 0:
+ warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+ raise SyntaxError()
+
+ self.quoteChar = quoteChar
+ self.quoteCharLen = len(quoteChar)
+ self.firstQuoteChar = quoteChar[0]
+ self.endQuoteChar = endQuoteChar
+ self.endQuoteCharLen = len(endQuoteChar)
+ self.escChar = escChar
+ self.escQuote = escQuote
+ self.unquoteResults = unquoteResults
+
+ if multiline:
+ self.flags = re.MULTILINE | re.DOTALL
+ self.pattern = r'%s(?:[^%s%s]' % \
+ ( re.escape(self.quoteChar),
+ _escapeRegexRangeChars(self.endQuoteChar[0]),
+ (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+ else:
+ self.flags = 0
+ self.pattern = r'%s(?:[^%s\n\r%s]' % \
+ ( re.escape(self.quoteChar),
+ _escapeRegexRangeChars(self.endQuoteChar[0]),
+ (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+ if len(self.endQuoteChar) > 1:
+ self.pattern += (
+ '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
+ _escapeRegexRangeChars(self.endQuoteChar[i]))
+ for i in range(len(self.endQuoteChar)-1,0,-1)) + ')'
+ )
+ if escQuote:
+ self.pattern += (r'|(?:%s)' % re.escape(escQuote))
+ if escChar:
+ self.pattern += (r'|(?:%s.)' % re.escape(escChar))
+ self.escCharReplacePattern = re.escape(self.escChar)+"(.)"
+ self.pattern += (r')*%s' % re.escape(self.endQuoteChar))
+
+ try:
+ self.re = re.compile(self.pattern, self.flags)
+ self.reString = self.pattern
+ except sre_constants.error:
+ warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern,
+ SyntaxWarning, stacklevel=2)
+ raise
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ ret = result.group()
+
+ if self.unquoteResults:
+
+ # strip off quotes
+ ret = ret[self.quoteCharLen:-self.endQuoteCharLen]
+
+ if isinstance(ret,basestring):
+ # replace escaped characters
+ if self.escChar:
+ ret = re.sub(self.escCharReplacePattern,"\g<1>",ret)
+
+ # replace escaped quotes
+ if self.escQuote:
+ ret = ret.replace(self.escQuote, self.endQuoteChar)
+
+ return loc, ret
+
+ def __str__( self ):
+ try:
+ return super(QuotedString,self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar)
+
+ return self.strRepr
+
+
+class CharsNotIn(Token):
+ """Token for matching words composed of characters *not* in a given set.
+ Defined with string containing all disallowed characters, and an optional
+ minimum, maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction.
+ """
+ def __init__( self, notChars, min=1, max=0, exact=0 ):
+ super(CharsNotIn,self).__init__()
+ self.skipWhitespace = False
+ self.notChars = notChars
+
+ if min < 1:
+ raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted")
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = ( self.minLen == 0 )
+ self.mayIndexError = False
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if instring[loc] in self.notChars:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ start = loc
+ loc += 1
+ notchars = self.notChars
+ maxlen = min( start+self.maxLen, len(instring) )
+ while loc < maxlen and \
+ (instring[loc] not in notchars):
+ loc += 1
+
+ if loc - start < self.minLen:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+ def __str__( self ):
+ try:
+ return super(CharsNotIn, self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None:
+ if len(self.notChars) > 4:
+ self.strRepr = "!W:(%s...)" % self.notChars[:4]
+ else:
+ self.strRepr = "!W:(%s)" % self.notChars
+
+ return self.strRepr
+
+class White(Token):
+ """Special matching class for matching whitespace. Normally, whitespace is ignored
+ by pyparsing grammars. This class is included when some whitespace structures
+ are significant. Define with a string containing the whitespace characters to be
+ matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments,
+ as defined for the C{L{Word}} class."""
+ whiteStrs = {
+ " " : "<SPC>",
+ "\t": "<TAB>",
+ "\n": "<LF>",
+ "\r": "<CR>",
+ "\f": "<FF>",
+ }
+ def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
+ super(White,self).__init__()
+ self.matchWhite = ws
+ self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) )
+ #~ self.leaveWhitespace()
+ self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite))
+ self.mayReturnEmpty = True
+ self.errmsg = "Expected " + self.name
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if not(instring[ loc ] in self.matchWhite):
+ raise ParseException(instring, loc, self.errmsg, self)
+ start = loc
+ loc += 1
+ maxloc = start + self.maxLen
+ maxloc = min( maxloc, len(instring) )
+ while loc < maxloc and instring[loc] in self.matchWhite:
+ loc += 1
+
+ if loc - start < self.minLen:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+
+class _PositionToken(Token):
+ def __init__( self ):
+ super(_PositionToken,self).__init__()
+ self.name=self.__class__.__name__
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+
+class GoToColumn(_PositionToken):
+ """Token to advance to a specific column of input text; useful for tabular report scraping."""
+ def __init__( self, colno ):
+ super(GoToColumn,self).__init__()
+ self.col = colno
+
+ def preParse( self, instring, loc ):
+ if col(loc,instring) != self.col:
+ instrlen = len(instring)
+ if self.ignoreExprs:
+ loc = self._skipIgnorables( instring, loc )
+ while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col :
+ loc += 1
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ thiscol = col( loc, instring )
+ if thiscol > self.col:
+ raise ParseException( instring, loc, "Text not in expected column", self )
+ newloc = loc + self.col - thiscol
+ ret = instring[ loc: newloc ]
+ return newloc, ret
+
+class LineStart(_PositionToken):
+ """Matches if current position is at the beginning of a line within the parse string"""
+ def __init__( self ):
+ super(LineStart,self).__init__()
+ self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
+ self.errmsg = "Expected start of line"
+
+ def preParse( self, instring, loc ):
+ preloc = super(LineStart,self).preParse(instring,loc)
+ if instring[preloc] == "\n":
+ loc += 1
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if not( loc==0 or
+ (loc == self.preParse( instring, 0 )) or
+ (instring[loc-1] == "\n") ): #col(loc, instring) != 1:
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class LineEnd(_PositionToken):
+ """Matches if current position is at the end of a line within the parse string"""
+ def __init__( self ):
+ super(LineEnd,self).__init__()
+ self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
+ self.errmsg = "Expected end of line"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc<len(instring):
+ if instring[loc] == "\n":
+ return loc+1, "\n"
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+ elif loc == len(instring):
+ return loc+1, []
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class StringStart(_PositionToken):
+ """Matches if current position is at the beginning of the parse string"""
+ def __init__( self ):
+ super(StringStart,self).__init__()
+ self.errmsg = "Expected start of text"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc != 0:
+ # see if entire string up to here is just whitespace and ignoreables
+ if loc != self.preParse( instring, 0 ):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class StringEnd(_PositionToken):
+ """Matches if current position is at the end of the parse string"""
+ def __init__( self ):
+ super(StringEnd,self).__init__()
+ self.errmsg = "Expected end of text"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc < len(instring):
+ raise ParseException(instring, loc, self.errmsg, self)
+ elif loc == len(instring):
+ return loc+1, []
+ elif loc > len(instring):
+ return loc, []
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class WordStart(_PositionToken):
+ """Matches if the current position is at the beginning of a Word, and
+ is not preceded by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of
+ the string being parsed, or at the beginning of a line.
+ """
+ def __init__(self, wordChars = printables):
+ super(WordStart,self).__init__()
+ self.wordChars = set(wordChars)
+ self.errmsg = "Not at the start of a word"
+
+ def parseImpl(self, instring, loc, doActions=True ):
+ if loc != 0:
+ if (instring[loc-1] in self.wordChars or
+ instring[loc] not in self.wordChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class WordEnd(_PositionToken):
+ """Matches if the current position is at the end of a Word, and
+ is not followed by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of
+ the string being parsed, or at the end of a line.
+ """
+ def __init__(self, wordChars = printables):
+ super(WordEnd,self).__init__()
+ self.wordChars = set(wordChars)
+ self.skipWhitespace = False
+ self.errmsg = "Not at the end of a word"
+
+ def parseImpl(self, instring, loc, doActions=True ):
+ instrlen = len(instring)
+ if instrlen>0 and loc<instrlen:
+ if (instring[loc] in self.wordChars or
+ instring[loc-1] not in self.wordChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+
+class ParseExpression(ParserElement):
+ """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
+ def __init__( self, exprs, savelist = False ):
+ super(ParseExpression,self).__init__(savelist)
+ if isinstance( exprs, _generatorType ):
+ exprs = list(exprs)
+
+ if isinstance( exprs, basestring ):
+ self.exprs = [ Literal( exprs ) ]
+ elif isinstance( exprs, collections.Sequence ):
+ # if sequence of strings provided, wrap with Literal
+ if all(isinstance(expr, basestring) for expr in exprs):
+ exprs = map(Literal, exprs)
+ self.exprs = list(exprs)
+ else:
+ try:
+ self.exprs = list( exprs )
+ except TypeError:
+ self.exprs = [ exprs ]
+ self.callPreparse = False
+
+ def __getitem__( self, i ):
+ return self.exprs[i]
+
+ def append( self, other ):
+ self.exprs.append( other )
+ self.strRepr = None
+ return self
+
+ def leaveWhitespace( self ):
+ """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on
+ all contained expressions."""
+ self.skipWhitespace = False
+ self.exprs = [ e.copy() for e in self.exprs ]
+ for e in self.exprs:
+ e.leaveWhitespace()
+ return self
+
+ def ignore( self, other ):
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ super( ParseExpression, self).ignore( other )
+ for e in self.exprs:
+ e.ignore( self.ignoreExprs[-1] )
+ else:
+ super( ParseExpression, self).ignore( other )
+ for e in self.exprs:
+ e.ignore( self.ignoreExprs[-1] )
+ return self
+
+ def __str__( self ):
+ try:
+ return super(ParseExpression,self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.exprs) )
+ return self.strRepr
+
+ def streamline( self ):
+ super(ParseExpression,self).streamline()
+
+ for e in self.exprs:
+ e.streamline()
+
+ # collapse nested And's of the form And( And( And( a,b), c), d) to And( a,b,c,d )
+ # but only if there are no parse actions or resultsNames on the nested And's
+ # (likewise for Or's and MatchFirst's)
+ if ( len(self.exprs) == 2 ):
+ other = self.exprs[0]
+ if ( isinstance( other, self.__class__ ) and
+ not(other.parseAction) and
+ other.resultsName is None and
+ not other.debug ):
+ self.exprs = other.exprs[:] + [ self.exprs[1] ]
+ self.strRepr = None
+ self.mayReturnEmpty |= other.mayReturnEmpty
+ self.mayIndexError |= other.mayIndexError
+
+ other = self.exprs[-1]
+ if ( isinstance( other, self.__class__ ) and
+ not(other.parseAction) and
+ other.resultsName is None and
+ not other.debug ):
+ self.exprs = self.exprs[:-1] + other.exprs[:]
+ self.strRepr = None
+ self.mayReturnEmpty |= other.mayReturnEmpty
+ self.mayIndexError |= other.mayIndexError
+
+ self.errmsg = "Expected " + str(self)
+
+ return self
+
+ def setResultsName( self, name, listAllMatches=False ):
+ ret = super(ParseExpression,self).setResultsName(name,listAllMatches)
+ return ret
+
+ def validate( self, validateTrace=[] ):
+ tmp = validateTrace[:]+[self]
+ for e in self.exprs:
+ e.validate(tmp)
+ self.checkRecursion( [] )
+
+ def copy(self):
+ ret = super(ParseExpression,self).copy()
+ ret.exprs = [e.copy() for e in self.exprs]
+ return ret
+
+class And(ParseExpression):
+ """Requires all given C{ParseExpression}s to be found in the given order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'+'} operator.
+ """
+
+ class _ErrorStop(Empty):
+ def __init__(self, *args, **kwargs):
+ super(And._ErrorStop,self).__init__(*args, **kwargs)
+ self.name = '-'
+ self.leaveWhitespace()
+
+ def __init__( self, exprs, savelist = True ):
+ super(And,self).__init__(exprs, savelist)
+ self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+ self.setWhitespaceChars( self.exprs[0].whiteChars )
+ self.skipWhitespace = self.exprs[0].skipWhitespace
+ self.callPreparse = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ # pass False as last arg to _parse for first element, since we already
+ # pre-parsed the string as part of our And pre-parsing
+ loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False )
+ errorStop = False
+ for e in self.exprs[1:]:
+ if isinstance(e, And._ErrorStop):
+ errorStop = True
+ continue
+ if errorStop:
+ try:
+ loc, exprtokens = e._parse( instring, loc, doActions )
+ except ParseSyntaxException:
+ raise
+ except ParseBaseException as pe:
+ pe.__traceback__ = None
+ raise ParseSyntaxException(pe)
+ except IndexError:
+ raise ParseSyntaxException( ParseException(instring, len(instring), self.errmsg, self) )
+ else:
+ loc, exprtokens = e._parse( instring, loc, doActions )
+ if exprtokens or exprtokens.haskeys():
+ resultlist += exprtokens
+ return loc, resultlist
+
+ def __iadd__(self, other ):
+ if isinstance( other, basestring ):
+ other = Literal( other )
+ return self.append( other ) #And( [ self, other ] )
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+ if not e.mayReturnEmpty:
+ break
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+
+class Or(ParseExpression):
+ """Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the expression that matches the longest string will be used.
+ May be constructed using the C{'^'} operator.
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(Or,self).__init__(exprs, savelist)
+ if self.exprs:
+ self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+ else:
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ maxExcLoc = -1
+ maxException = None
+ matches = []
+ for e in self.exprs:
+ try:
+ loc2 = e.tryParse( instring, loc )
+ except ParseException as err:
+ err.__traceback__ = None
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+ except IndexError:
+ if len(instring) > maxExcLoc:
+ maxException = ParseException(instring,len(instring),e.errmsg,self)
+ maxExcLoc = len(instring)
+ else:
+ # save match among all matches, to retry longest to shortest
+ matches.append((loc2, e))
+
+ if matches:
+ matches.sort(key=lambda x: -x[0])
+ for _,e in matches:
+ try:
+ return e._parse( instring, loc, doActions )
+ except ParseException as err:
+ err.__traceback__ = None
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+
+ if maxException is not None:
+ maxException.msg = self.errmsg
+ raise maxException
+ else:
+ raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+
+ def __ixor__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ return self.append( other ) #Or( [ self, other ] )
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " ^ ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class MatchFirst(ParseExpression):
+ """Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the first one listed is the one that will match.
+ May be constructed using the C{'|'} operator.
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(MatchFirst,self).__init__(exprs, savelist)
+ if self.exprs:
+ self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+ else:
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ maxExcLoc = -1
+ maxException = None
+ for e in self.exprs:
+ try:
+ ret = e._parse( instring, loc, doActions )
+ return ret
+ except ParseException as err:
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+ except IndexError:
+ if len(instring) > maxExcLoc:
+ maxException = ParseException(instring,len(instring),e.errmsg,self)
+ maxExcLoc = len(instring)
+
+ # only got here if no expression matched, raise exception for match that made it the furthest
+ else:
+ if maxException is not None:
+ maxException.msg = self.errmsg
+ raise maxException
+ else:
+ raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+ def __ior__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass( other )
+ return self.append( other ) #MatchFirst( [ self, other ] )
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " | ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class Each(ParseExpression):
+ """Requires all given C{ParseExpression}s to be found, but in any order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'&'} operator.
+ """
+ def __init__( self, exprs, savelist = True ):
+ super(Each,self).__init__(exprs, savelist)
+ self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+ self.skipWhitespace = True
+ self.initExprGroups = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.initExprGroups:
+ self.opt1map = dict((id(e.expr),e) for e in self.exprs if isinstance(e,Optional))
+ opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ]
+ opt2 = [ e for e in self.exprs if e.mayReturnEmpty and not isinstance(e,Optional)]
+ self.optionals = opt1 + opt2
+ self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ]
+ self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ]
+ self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ]
+ self.required += self.multirequired
+ self.initExprGroups = False
+ tmpLoc = loc
+ tmpReqd = self.required[:]
+ tmpOpt = self.optionals[:]
+ matchOrder = []
+
+ keepMatching = True
+ while keepMatching:
+ tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired
+ failed = []
+ for e in tmpExprs:
+ try:
+ tmpLoc = e.tryParse( instring, tmpLoc )
+ except ParseException:
+ failed.append(e)
+ else:
+ matchOrder.append(self.opt1map.get(id(e),e))
+ if e in tmpReqd:
+ tmpReqd.remove(e)
+ elif e in tmpOpt:
+ tmpOpt.remove(e)
+ if len(failed) == len(tmpExprs):
+ keepMatching = False
+
+ if tmpReqd:
+ missing = ", ".join(_ustr(e) for e in tmpReqd)
+ raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )
+
+ # add any unmatched Optionals, in case they have default values defined
+ matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt]
+
+ resultlist = []
+ for e in matchOrder:
+ loc,results = e._parse(instring,loc,doActions)
+ resultlist.append(results)
+
+ finalResults = ParseResults([])
+ for r in resultlist:
+ dups = {}
+ for k in r.keys():
+ if k in finalResults:
+ tmp = ParseResults(finalResults[k])
+ tmp += ParseResults(r[k])
+ dups[k] = tmp
+ finalResults += ParseResults(r)
+ for k,v in dups.items():
+ finalResults[k] = v
+ return loc, finalResults
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " & ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class ParseElementEnhance(ParserElement):
+ """Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens."""
+ def __init__( self, expr, savelist=False ):
+ super(ParseElementEnhance,self).__init__(savelist)
+ if isinstance( expr, basestring ):
+ expr = Literal(expr)
+ self.expr = expr
+ self.strRepr = None
+ if expr is not None:
+ self.mayIndexError = expr.mayIndexError
+ self.mayReturnEmpty = expr.mayReturnEmpty
+ self.setWhitespaceChars( expr.whiteChars )
+ self.skipWhitespace = expr.skipWhitespace
+ self.saveAsList = expr.saveAsList
+ self.callPreparse = expr.callPreparse
+ self.ignoreExprs.extend(expr.ignoreExprs)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.expr is not None:
+ return self.expr._parse( instring, loc, doActions, callPreParse=False )
+ else:
+ raise ParseException("",loc,self.errmsg,self)
+
+ def leaveWhitespace( self ):
+ self.skipWhitespace = False
+ self.expr = self.expr.copy()
+ if self.expr is not None:
+ self.expr.leaveWhitespace()
+ return self
+
+ def ignore( self, other ):
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ super( ParseElementEnhance, self).ignore( other )
+ if self.expr is not None:
+ self.expr.ignore( self.ignoreExprs[-1] )
+ else:
+ super( ParseElementEnhance, self).ignore( other )
+ if self.expr is not None:
+ self.expr.ignore( self.ignoreExprs[-1] )
+ return self
+
+ def streamline( self ):
+ super(ParseElementEnhance,self).streamline()
+ if self.expr is not None:
+ self.expr.streamline()
+ return self
+
+ def checkRecursion( self, parseElementList ):
+ if self in parseElementList:
+ raise RecursiveGrammarException( parseElementList+[self] )
+ subRecCheckList = parseElementList[:] + [ self ]
+ if self.expr is not None:
+ self.expr.checkRecursion( subRecCheckList )
+
+ def validate( self, validateTrace=[] ):
+ tmp = validateTrace[:]+[self]
+ if self.expr is not None:
+ self.expr.validate(tmp)
+ self.checkRecursion( [] )
+
+ def __str__( self ):
+ try:
+ return super(ParseElementEnhance,self).__str__()
+ except:
+ pass
+
+ if self.strRepr is None and self.expr is not None:
+ self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) )
+ return self.strRepr
+
+
+class FollowedBy(ParseElementEnhance):
+ """Lookahead matching of the given parse expression. C{FollowedBy}
+ does *not* advance the parsing position within the input string, it only
+ verifies that the specified parse expression matches at the current
+ position. C{FollowedBy} always returns a null token list."""
+ def __init__( self, expr ):
+ super(FollowedBy,self).__init__(expr)
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ self.expr.tryParse( instring, loc )
+ return loc, []
+
+
+class NotAny(ParseElementEnhance):
+ """Lookahead to disallow matching with the given parse expression. C{NotAny}
+ does *not* advance the parsing position within the input string, it only
+ verifies that the specified parse expression does *not* match at the current
+ position. Also, C{NotAny} does *not* skip over leading whitespace. C{NotAny}
+ always returns a null token list. May be constructed using the '~' operator."""
+ def __init__( self, expr ):
+ super(NotAny,self).__init__(expr)
+ #~ self.leaveWhitespace()
+ self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs
+ self.mayReturnEmpty = True
+ self.errmsg = "Found unwanted token, "+_ustr(self.expr)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ try:
+ self.expr.tryParse( instring, loc )
+ except (ParseException,IndexError):
+ pass
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "~{" + _ustr(self.expr) + "}"
+
+ return self.strRepr
+
+
+class ZeroOrMore(ParseElementEnhance):
+ """Optional repetition of zero or more of the given expression."""
+ def __init__( self, expr ):
+ super(ZeroOrMore,self).__init__(expr)
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ tokens = []
+ try:
+ loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+ hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
+ while 1:
+ if hasIgnoreExprs:
+ preloc = self._skipIgnorables( instring, loc )
+ else:
+ preloc = loc
+ loc, tmptokens = self.expr._parse( instring, preloc, doActions )
+ if tmptokens or tmptokens.haskeys():
+ tokens += tmptokens
+ except (ParseException,IndexError):
+ pass
+
+ return loc, tokens
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "[" + _ustr(self.expr) + "]..."
+
+ return self.strRepr
+
+ def setResultsName( self, name, listAllMatches=False ):
+ ret = super(ZeroOrMore,self).setResultsName(name,listAllMatches)
+ ret.saveAsList = True
+ return ret
+
+
+class OneOrMore(ParseElementEnhance):
+ """Repetition of one or more of the given expression."""
+ def parseImpl( self, instring, loc, doActions=True ):
+ # must be at least one
+ loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+ try:
+ hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
+ while 1:
+ if hasIgnoreExprs:
+ preloc = self._skipIgnorables( instring, loc )
+ else:
+ preloc = loc
+ loc, tmptokens = self.expr._parse( instring, preloc, doActions )
+ if tmptokens or tmptokens.haskeys():
+ tokens += tmptokens
+ except (ParseException,IndexError):
+ pass
+
+ return loc, tokens
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + _ustr(self.expr) + "}..."
+
+ return self.strRepr
+
+ def setResultsName( self, name, listAllMatches=False ):
+ ret = super(OneOrMore,self).setResultsName(name,listAllMatches)
+ ret.saveAsList = True
+ return ret
+
+class _NullToken(object):
+ def __bool__(self):
+ return False
+ __nonzero__ = __bool__
+ def __str__(self):
+ return ""
+
+_optionalNotMatched = _NullToken()
+class Optional(ParseElementEnhance):
+ """Optional matching of the given expression.
+ A default return string can also be specified, if the optional expression
+ is not found.
+ """
+ def __init__( self, expr, default=_optionalNotMatched ):
+ super(Optional,self).__init__( expr, savelist=False )
+ self.defaultValue = default
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ try:
+ loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+ except (ParseException,IndexError):
+ if self.defaultValue is not _optionalNotMatched:
+ if self.expr.resultsName:
+ tokens = ParseResults([ self.defaultValue ])
+ tokens[self.expr.resultsName] = self.defaultValue
+ else:
+ tokens = [ self.defaultValue ]
+ else:
+ tokens = []
+ return loc, tokens
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "[" + _ustr(self.expr) + "]"
+
+ return self.strRepr
+
+
+class SkipTo(ParseElementEnhance):
+ """Token for skipping over all undefined text until the matched expression is found.
+ If C{include} is set to true, the matched expression is also parsed (the skipped text
+ and matched expression are returned as a 2-element list). The C{ignore}
+ argument is used to define grammars (typically quoted strings and comments) that
+ might contain false matches.
+ """
+ def __init__( self, other, include=False, ignore=None, failOn=None ):
+ super( SkipTo, self ).__init__( other )
+ self.ignoreExpr = ignore
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+ self.includeMatch = include
+ self.asList = False
+ if failOn is not None and isinstance(failOn, basestring):
+ self.failOn = Literal(failOn)
+ else:
+ self.failOn = failOn
+ self.errmsg = "No match found for "+_ustr(self.expr)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ startLoc = loc
+ instrlen = len(instring)
+ expr = self.expr
+ failParse = False
+ while loc <= instrlen:
+ try:
+ if self.failOn:
+ try:
+ self.failOn.tryParse(instring, loc)
+ except ParseBaseException:
+ pass
+ else:
+ failParse = True
+ raise ParseException(instring, loc, "Found expression " + str(self.failOn))
+ failParse = False
+ if self.ignoreExpr is not None:
+ while 1:
+ try:
+ loc = self.ignoreExpr.tryParse(instring,loc)
+ # print("found ignoreExpr, advance to", loc)
+ except ParseBaseException:
+ break
+ expr._parse( instring, loc, doActions=False, callPreParse=False )
+ skipText = instring[startLoc:loc]
+ if self.includeMatch:
+ loc,mat = expr._parse(instring,loc,doActions,callPreParse=False)
+ if mat:
+ skipRes = ParseResults( skipText )
+ skipRes += mat
+ return loc, [ skipRes ]
+ else:
+ return loc, [ skipText ]
+ else:
+ return loc, [ skipText ]
+ except (ParseException,IndexError):
+ if failParse:
+ raise
+ else:
+ loc += 1
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class Forward(ParseElementEnhance):
+ """Forward declaration of an expression to be defined later -
+ used for recursive grammars, such as algebraic infix notation.
+ When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator.
+
+ Note: take care when assigning to C{Forward} not to overlook precedence of operators.
+ Specifically, '|' has a lower precedence than '<<', so that::
+ fwdExpr << a | b | c
+ will actually be evaluated as::
+ (fwdExpr << a) | b | c
+ thereby leaving b and c out as parseable alternatives. It is recommended that you
+ explicitly group the values inserted into the C{Forward}::
+ fwdExpr << (a | b | c)
+ Converting to use the '<<=' operator instead will avoid this problem.
+ """
+ def __init__( self, other=None ):
+ super(Forward,self).__init__( other, savelist=False )
+
+ def __lshift__( self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement.literalStringClass(other)
+ self.expr = other
+ self.mayReturnEmpty = other.mayReturnEmpty
+ self.strRepr = None
+ self.mayIndexError = self.expr.mayIndexError
+ self.mayReturnEmpty = self.expr.mayReturnEmpty
+ self.setWhitespaceChars( self.expr.whiteChars )
+ self.skipWhitespace = self.expr.skipWhitespace
+ self.saveAsList = self.expr.saveAsList
+ self.ignoreExprs.extend(self.expr.ignoreExprs)
+ return self
+
+ def __ilshift__(self, other):
+ return self << other
+
+ def leaveWhitespace( self ):
+ self.skipWhitespace = False
+ return self
+
+ def streamline( self ):
+ if not self.streamlined:
+ self.streamlined = True
+ if self.expr is not None:
+ self.expr.streamline()
+ return self
+
+ def validate( self, validateTrace=[] ):
+ if self not in validateTrace:
+ tmp = validateTrace[:]+[self]
+ if self.expr is not None:
+ self.expr.validate(tmp)
+ self.checkRecursion([])
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ self._revertClass = self.__class__
+ self.__class__ = _ForwardNoRecurse
+ try:
+ if self.expr is not None:
+ retString = _ustr(self.expr)
+ else:
+ retString = "None"
+ finally:
+ self.__class__ = self._revertClass
+ return self.__class__.__name__ + ": " + retString
+
+ def copy(self):
+ if self.expr is not None:
+ return super(Forward,self).copy()
+ else:
+ ret = Forward()
+ ret <<= self
+ return ret
+
+class _ForwardNoRecurse(Forward):
+ def __str__( self ):
+ return "..."
+
+class TokenConverter(ParseElementEnhance):
+ """Abstract subclass of C{ParseExpression}, for converting parsed results."""
+ def __init__( self, expr, savelist=False ):
+ super(TokenConverter,self).__init__( expr )#, savelist )
+ self.saveAsList = False
+
+class Upcase(TokenConverter):
+ """Converter to upper case all matching tokens."""
+ def __init__(self, *args):
+ super(Upcase,self).__init__(*args)
+ warnings.warn("Upcase class is deprecated, use upcaseTokens parse action instead",
+ DeprecationWarning,stacklevel=2)
+
+ def postParse( self, instring, loc, tokenlist ):
+ return list(map( str.upper, tokenlist ))
+
+
+class Combine(TokenConverter):
+ """Converter to concatenate all matching tokens to a single string.
+ By default, the matching patterns must also be contiguous in the input string;
+ this can be disabled by specifying C{'adjacent=False'} in the constructor.
+ """
+ def __init__( self, expr, joinString="", adjacent=True ):
+ super(Combine,self).__init__( expr )
+ # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
+ if adjacent:
+ self.leaveWhitespace()
+ self.adjacent = adjacent
+ self.skipWhitespace = True
+ self.joinString = joinString
+ self.callPreparse = True
+
+ def ignore( self, other ):
+ if self.adjacent:
+ ParserElement.ignore(self, other)
+ else:
+ super( Combine, self).ignore( other )
+ return self
+
+ def postParse( self, instring, loc, tokenlist ):
+ retToks = tokenlist.copy()
+ del retToks[:]
+ retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)
+
+ if self.resultsName and retToks.haskeys():
+ return [ retToks ]
+ else:
+ return retToks
+
+class Group(TokenConverter):
+ """Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions."""
+ def __init__( self, expr ):
+ super(Group,self).__init__( expr )
+ self.saveAsList = True
+
+ def postParse( self, instring, loc, tokenlist ):
+ return [ tokenlist ]
+
+class Dict(TokenConverter):
+ """Converter to return a repetitive expression as a list, but also as a dictionary.
+ Each element can also be referenced using the first token in the expression as its key.
+ Useful for tabular report scraping when the first column can be used as a item key.
+ """
+ def __init__( self, expr ):
+ super(Dict,self).__init__( expr )
+ self.saveAsList = True
+
+ def postParse( self, instring, loc, tokenlist ):
+ for i,tok in enumerate(tokenlist):
+ if len(tok) == 0:
+ continue
+ ikey = tok[0]
+ if isinstance(ikey,int):
+ ikey = _ustr(tok[0]).strip()
+ if len(tok)==1:
+ tokenlist[ikey] = _ParseResultsWithOffset("",i)
+ elif len(tok)==2 and not isinstance(tok[1],ParseResults):
+ tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i)
+ else:
+ dictvalue = tok.copy() #ParseResults(i)
+ del dictvalue[0]
+ if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()):
+ tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
+ else:
+ tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)
+
+ if self.resultsName:
+ return [ tokenlist ]
+ else:
+ return tokenlist
+
+
+class Suppress(TokenConverter):
+ """Converter for ignoring the results of a parsed expression."""
+ def postParse( self, instring, loc, tokenlist ):
+ return []
+
+ def suppress( self ):
+ return self
+
+
+class OnlyOnce(object):
+ """Wrapper for parse actions, to ensure they are only called once."""
+ def __init__(self, methodCall):
+ self.callable = _trim_arity(methodCall)
+ self.called = False
+ def __call__(self,s,l,t):
+ if not self.called:
+ results = self.callable(s,l,t)
+ self.called = True
+ return results
+ raise ParseException(s,l,"")
+ def reset(self):
+ self.called = False
+
+def traceParseAction(f):
+ """Decorator for debugging parse actions."""
+ f = _trim_arity(f)
+ def z(*paArgs):
+ thisFunc = f.func_name
+ s,l,t = paArgs[-3:]
+ if len(paArgs)>3:
+ thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
+ sys.stderr.write( ">>entering %s(line: '%s', %d, %s)\n" % (thisFunc,line(l,s),l,t) )
+ try:
+ ret = f(*paArgs)
+ except Exception as exc:
+ sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) )
+ raise
+ sys.stderr.write( "<<leaving %s (ret: %s)\n" % (thisFunc,ret) )
+ return ret
+ try:
+ z.__name__ = f.__name__
+ except AttributeError:
+ pass
+ return z
+
+#
+# global helpers
+#
+def delimitedList( expr, delim=",", combine=False ):
+ """Helper to define a delimited list of expressions - the delimiter defaults to ','.
+ By default, the list elements and delimiters can have intervening whitespace, and
+ comments, but this can be overridden by passing C{combine=True} in the constructor.
+ If C{combine} is set to C{True}, the matching tokens are returned as a single token
+ string, with the delimiters included; otherwise, the matching tokens are returned
+ as a list of tokens, with the delimiters suppressed.
+ """
+ dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
+ if combine:
+ return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName)
+ else:
+ return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)
+
+def countedArray( expr, intExpr=None ):
+ """Helper to define a counted list of expressions.
+ This helper defines a pattern of the form::
+ integer expr expr expr...
+ where the leading integer tells how many expr expressions follow.
+ The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
+ """
+ arrayExpr = Forward()
+ def countFieldParseAction(s,l,t):
+ n = t[0]
+ arrayExpr << (n and Group(And([expr]*n)) or Group(empty))
+ return []
+ if intExpr is None:
+ intExpr = Word(nums).setParseAction(lambda t:int(t[0]))
+ else:
+ intExpr = intExpr.copy()
+ intExpr.setName("arrayLen")
+ intExpr.addParseAction(countFieldParseAction, callDuringTry=True)
+ return ( intExpr + arrayExpr )
+
+def _flatten(L):
+ ret = []
+ for i in L:
+ if isinstance(i,list):
+ ret.extend(_flatten(i))
+ else:
+ ret.append(i)
+ return ret
+
+def matchPreviousLiteral(expr):
+ """Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousLiteral(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches a
+ previous literal, will also match the leading C{"1:1"} in C{"1:10"}.
+ If this is not desired, use C{matchPreviousExpr}.
+ Do *not* use with packrat parsing enabled.
+ """
+ rep = Forward()
+ def copyTokenToRepeater(s,l,t):
+ if t:
+ if len(t) == 1:
+ rep << t[0]
+ else:
+ # flatten t tokens
+ tflat = _flatten(t.asList())
+ rep << And( [ Literal(tt) for tt in tflat ] )
+ else:
+ rep << Empty()
+ expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ return rep
+
+def matchPreviousExpr(expr):
+ """Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousExpr(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches by
+ expressions, will *not* match the leading C{"1:1"} in C{"1:10"};
+ the expressions are evaluated first, and then compared, so
+ C{"1"} is compared with C{"10"}.
+ Do *not* use with packrat parsing enabled.
+ """
+ rep = Forward()
+ e2 = expr.copy()
+ rep <<= e2
+ def copyTokenToRepeater(s,l,t):
+ matchTokens = _flatten(t.asList())
+ def mustMatchTheseTokens(s,l,t):
+ theseTokens = _flatten(t.asList())
+ if theseTokens != matchTokens:
+ raise ParseException("",0,"")
+ rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
+ expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ return rep
+
+def _escapeRegexRangeChars(s):
+ #~ escape these chars: ^-]
+ for c in r"\^-]":
+ s = s.replace(c,_bslash+c)
+ s = s.replace("\n",r"\n")
+ s = s.replace("\t",r"\t")
+ return _ustr(s)
+
+def oneOf( strs, caseless=False, useRegex=True ):
+ """Helper to quickly define a set of alternative Literals, and makes sure to do
+ longest-first testing when there is a conflict, regardless of the input order,
+ but returns a C{L{MatchFirst}} for best performance.
+
+ Parameters:
+ - strs - a string of space-delimited literals, or a list of string literals
+ - caseless - (default=False) - treat all literals as caseless
+ - useRegex - (default=True) - as an optimization, will generate a Regex
+ object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or
+ if creating a C{Regex} raises an exception)
+ """
+ if caseless:
+ isequal = ( lambda a,b: a.upper() == b.upper() )
+ masks = ( lambda a,b: b.upper().startswith(a.upper()) )
+ parseElementClass = CaselessLiteral
+ else:
+ isequal = ( lambda a,b: a == b )
+ masks = ( lambda a,b: b.startswith(a) )
+ parseElementClass = Literal
+
+ symbols = []
+ if isinstance(strs,basestring):
+ symbols = strs.split()
+ elif isinstance(strs, collections.Sequence):
+ symbols = list(strs[:])
+ elif isinstance(strs, _generatorType):
+ symbols = list(strs)
+ else:
+ warnings.warn("Invalid argument to oneOf, expected string or list",
+ SyntaxWarning, stacklevel=2)
+ if not symbols:
+ return NoMatch()
+
+ i = 0
+ while i < len(symbols)-1:
+ cur = symbols[i]
+ for j,other in enumerate(symbols[i+1:]):
+ if ( isequal(other, cur) ):
+ del symbols[i+j+1]
+ break
+ elif ( masks(cur, other) ):
+ del symbols[i+j+1]
+ symbols.insert(i,other)
+ cur = other
+ break
+ else:
+ i += 1
+
+ if not caseless and useRegex:
+ #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
+ try:
+ if len(symbols)==len("".join(symbols)):
+ return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) )
+ else:
+ return Regex( "|".join(re.escape(sym) for sym in symbols) )
+ except:
+ warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
+ SyntaxWarning, stacklevel=2)
+
+
+ # last resort, just use MatchFirst
+ return MatchFirst( [ parseElementClass(sym) for sym in symbols ] )
+
+def dictOf( key, value ):
+ """Helper to easily and clearly define a dictionary by specifying the respective patterns
+ for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens
+ in the proper order. The key pattern can include delimiting markers or punctuation,
+ as long as they are suppressed, thereby leaving the significant key text. The value
+ pattern can include named results, so that the C{Dict} results can include named token
+ fields.
+ """
+ return Dict( ZeroOrMore( Group ( key + value ) ) )
+
+def originalTextFor(expr, asString=True):
+ """Helper to return the original, untokenized text for a given expression. Useful to
+ restore the parsed fields of an HTML start tag into the raw tag text itself, or to
+ revert separate tokens with intervening whitespace back to the original matching
+ input text. Simpler to use than the parse action C{L{keepOriginalText}}, and does not
+ require the inspect module to chase up the call stack. By default, returns a
+ string containing the original parsed text.
+
+ If the optional C{asString} argument is passed as C{False}, then the return value is a
+ C{L{ParseResults}} containing any results names that were originally matched, and a
+ single token containing the original matched text from the input string. So if
+ the expression passed to C{L{originalTextFor}} contains expressions with defined
+ results names, you must set C{asString} to C{False} if you want to preserve those
+ results name values."""
+ locMarker = Empty().setParseAction(lambda s,loc,t: loc)
+ endlocMarker = locMarker.copy()
+ endlocMarker.callPreparse = False
+ matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end")
+ if asString:
+ extractText = lambda s,l,t: s[t._original_start:t._original_end]
+ else:
+ def extractText(s,l,t):
+ del t[:]
+ t.insert(0, s[t._original_start:t._original_end])
+ del t["_original_start"]
+ del t["_original_end"]
+ matchExpr.setParseAction(extractText)
+ return matchExpr
+
+def ungroup(expr):
+ """Helper to undo pyparsing's default grouping of And expressions, even
+ if all but one are non-empty."""
+ return TokenConverter(expr).setParseAction(lambda t:t[0])
+
+def locatedExpr(expr):
+ """Helper to decorate a returned token with its starting and ending locations in the input string.
+ This helper adds the following results names:
+ - locn_start = location where matched expression begins
+ - locn_end = location where matched expression ends
+ - value = the actual parsed results
+
+ Be careful if the input text contains C{<TAB>} characters, you may want to call
+ C{L{ParserElement.parseWithTabs}}
+ """
+ locator = Empty().setParseAction(lambda s,l,t: l)
+ return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end"))
+
+
+# convenience constants for positional expressions
+empty = Empty().setName("empty")
+lineStart = LineStart().setName("lineStart")
+lineEnd = LineEnd().setName("lineEnd")
+stringStart = StringStart().setName("stringStart")
+stringEnd = StringEnd().setName("stringEnd")
+
+_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
+_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16)))
+_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8)))
+_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1) | Regex(r"\w", re.UNICODE)
+_charRange = Group(_singleChar + Suppress("-") + _singleChar)
+_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
+
+def srange(s):
+ r"""Helper to easily define string ranges for use in Word construction. Borrows
+ syntax from regexp '[]' string range definitions::
+ srange("[0-9]") -> "0123456789"
+ srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz"
+ srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
+ The input string must be enclosed in []'s, and the returned string is the expanded
+ character set joined into a single string.
+ The values enclosed in the []'s may be::
+ a single character
+ an escaped character with a leading backslash (such as \- or \])
+ an escaped hex character with a leading '\x' (\x21, which is a '!' character)
+ (\0x## is also supported for backwards compatibility)
+ an escaped octal character with a leading '\0' (\041, which is a '!' character)
+ a range of any of the above, separated by a dash ('a-z', etc.)
+ any combination of the above ('aeiouy', 'a-zA-Z0-9_$', etc.)
+ """
+ _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1))
+ try:
+ return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body)
+ except:
+ return ""
+
+def matchOnlyAtCol(n):
+ """Helper method for defining parse actions that require matching at a specific
+ column in the input text.
+ """
+ def verifyCol(strg,locn,toks):
+ if col(locn,strg) != n:
+ raise ParseException(strg,locn,"matched token not at column %d" % n)
+ return verifyCol
+
+def replaceWith(replStr):
+ """Helper method for common parse actions that simply return a literal value. Especially
+ useful when used with C{L{transformString<ParserElement.transformString>}()}.
+ """
+ #def _replFunc(*args):
+ # return [replStr]
+ #return _replFunc
+ return functools.partial(next, itertools.repeat([replStr]))
+
+def removeQuotes(s,l,t):
+ """Helper parse action for removing quotation marks from parsed quoted strings.
+ To use, add this parse action to quoted string using::
+ quotedString.setParseAction( removeQuotes )
+ """
+ return t[0][1:-1]
+
+def upcaseTokens(s,l,t):
+ """Helper parse action to convert tokens to upper case."""
+ return [ tt.upper() for tt in map(_ustr,t) ]
+
+def downcaseTokens(s,l,t):
+ """Helper parse action to convert tokens to lower case."""
+ return [ tt.lower() for tt in map(_ustr,t) ]
+
+def keepOriginalText(s,startLoc,t):
+ """DEPRECATED - use new helper method C{L{originalTextFor}}.
+ Helper parse action to preserve original parsed text,
+ overriding any nested parse actions."""
+ try:
+ endloc = getTokensEndLoc()
+ except ParseException:
+ raise ParseFatalException("incorrect usage of keepOriginalText - may only be called as a parse action")
+ del t[:]
+ t += ParseResults(s[startLoc:endloc])
+ return t
+
+def getTokensEndLoc():
+ """Method to be called from within a parse action to determine the end
+ location of the parsed tokens."""
+ import inspect
+ fstack = inspect.stack()
+ try:
+ # search up the stack (through intervening argument normalizers) for correct calling routine
+ for f in fstack[2:]:
+ if f[3] == "_parseNoCache":
+ endloc = f[0].f_locals["loc"]
+ return endloc
+ else:
+ raise ParseFatalException("incorrect usage of getTokensEndLoc - may only be called from within a parse action")
+ finally:
+ del fstack
+
+def _makeTags(tagStr, xml):
+ """Internal helper to construct opening and closing tag expressions, given a tag name"""
+ if isinstance(tagStr,basestring):
+ resname = tagStr
+ tagStr = Keyword(tagStr, caseless=not xml)
+ else:
+ resname = tagStr.name
+
+ tagAttrName = Word(alphas,alphanums+"_-:")
+ if (xml):
+ tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes )
+ openTag = Suppress("<") + tagStr("tag") + \
+ Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
+ Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+ else:
+ printablesLessRAbrack = "".join(c for c in printables if c not in ">")
+ tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
+ openTag = Suppress("<") + tagStr("tag") + \
+ Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
+ Optional( Suppress("=") + tagAttrValue ) ))) + \
+ Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+ closeTag = Combine(_L("</") + tagStr + ">")
+
+ openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % tagStr)
+ closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % tagStr)
+ openTag.tag = resname
+ closeTag.tag = resname
+ return openTag, closeTag
+
+def makeHTMLTags(tagStr):
+ """Helper to construct opening and closing tag expressions for HTML, given a tag name"""
+ return _makeTags( tagStr, False )
+
+def makeXMLTags(tagStr):
+ """Helper to construct opening and closing tag expressions for XML, given a tag name"""
+ return _makeTags( tagStr, True )
+
+def withAttribute(*args,**attrDict):
+ """Helper to create a validating parse action to be used with start tags created
+ with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag
+ with a required attribute value, to avoid false matches on common tags such as
+ C{<TD>} or C{<DIV>}.
+
+ Call C{withAttribute} with a series of attribute names and values. Specify the list
+ of filter attributes names and values as:
+ - keyword arguments, as in C{(align="right")}, or
+ - as an explicit dict with C{**} operator, when an attribute name is also a Python
+ reserved word, as in C{**{"class":"Customer", "align":"right"}}
+ - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
+ For attribute names with a namespace prefix, you must use the second form. Attribute
+ names are matched insensitive to upper/lower case.
+
+ If just testing for C{class} (with or without a namespace), use C{L{withClass}}.
+
+ To verify that the attribute exists, but without specifying a value, pass
+ C{withAttribute.ANY_VALUE} as the value.
+ """
+ if args:
+ attrs = args[:]
+ else:
+ attrs = attrDict.items()
+ attrs = [(k,v) for k,v in attrs]
+ def pa(s,l,tokens):
+ for attrName,attrValue in attrs:
+ if attrName not in tokens:
+ raise ParseException(s,l,"no matching attribute " + attrName)
+ if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue:
+ raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" %
+ (attrName, tokens[attrName], attrValue))
+ return pa
+withAttribute.ANY_VALUE = object()
+
+def withClass(classname, namespace=''):
+ """Simplified version of C{L{withAttribute}} when matching on a div class - made
+ difficult because C{class} is a reserved word in Python.
+ """
+ classattr = "%s:class" % namespace if namespace else "class"
+ return withAttribute(**{classattr : classname})
+
+opAssoc = _Constants()
+opAssoc.LEFT = object()
+opAssoc.RIGHT = object()
+
+def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ):
+ """Helper method for constructing grammars of expressions made up of
+ operators working in a precedence hierarchy. Operators may be unary or
+ binary, left- or right-associative. Parse actions can also be attached
+ to operator expressions.
+
+ Parameters:
+ - baseExpr - expression representing the most basic element for the nested
+ - opList - list of tuples, one for each operator precedence level in the
+ expression grammar; each tuple is of the form
+ (opExpr, numTerms, rightLeftAssoc, parseAction), where:
+ - opExpr is the pyparsing expression for the operator;
+ may also be a string, which will be converted to a Literal;
+ if numTerms is 3, opExpr is a tuple of two expressions, for the
+ two operators separating the 3 terms
+ - numTerms is the number of terms for this operator (must
+ be 1, 2, or 3)
+ - rightLeftAssoc is the indicator whether the operator is
+ right or left associative, using the pyparsing-defined
+ constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
+ - parseAction is the parse action to be associated with
+ expressions matching this operator expression (the
+ parse action tuple member may be omitted)
+ - lpar - expression for matching left-parentheses (default=Suppress('('))
+ - rpar - expression for matching right-parentheses (default=Suppress(')'))
+ """
+ ret = Forward()
+ lastExpr = baseExpr | ( lpar + ret + rpar )
+ for i,operDef in enumerate(opList):
+ opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
+ if arity == 3:
+ if opExpr is None or len(opExpr) != 2:
+ raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions")
+ opExpr1, opExpr2 = opExpr
+ thisExpr = Forward()#.setName("expr%d" % i)
+ if rightLeftAssoc == opAssoc.LEFT:
+ if arity == 1:
+ matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) )
+ elif arity == 2:
+ if opExpr is not None:
+ matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) )
+ else:
+ matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) )
+ elif arity == 3:
+ matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \
+ Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr )
+ else:
+ raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+ elif rightLeftAssoc == opAssoc.RIGHT:
+ if arity == 1:
+ # try to avoid LR with this extra test
+ if not isinstance(opExpr, Optional):
+ opExpr = Optional(opExpr)
+ matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr )
+ elif arity == 2:
+ if opExpr is not None:
+ matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) )
+ else:
+ matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) )
+ elif arity == 3:
+ matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \
+ Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr )
+ else:
+ raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+ else:
+ raise ValueError("operator must indicate right or left associativity")
+ if pa:
+ matchExpr.setParseAction( pa )
+ thisExpr <<= ( matchExpr | lastExpr )
+ lastExpr = thisExpr
+ ret <<= lastExpr
+ return ret
+operatorPrecedence = infixNotation
+
+dblQuotedString = Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*"').setName("string enclosed in double quotes")
+sglQuotedString = Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*'").setName("string enclosed in single quotes")
+quotedString = Regex(r'''(?:"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*")|(?:'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*')''').setName("quotedString using single or double quotes")
+unicodeString = Combine(_L('u') + quotedString.copy())
+
+def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()):
+ """Helper method for defining nested lists enclosed in opening and closing
+ delimiters ("(" and ")" are the default).
+
+ Parameters:
+ - opener - opening character for a nested list (default="("); can also be a pyparsing expression
+ - closer - closing character for a nested list (default=")"); can also be a pyparsing expression
+ - content - expression for items within the nested lists (default=None)
+ - ignoreExpr - expression for ignoring opening and closing delimiters (default=quotedString)
+
+ If an expression is not provided for the content argument, the nested
+ expression will capture all whitespace-delimited content between delimiters
+ as a list of separate values.
+
+ Use the C{ignoreExpr} argument to define expressions that may contain
+ opening or closing characters that should not be treated as opening
+ or closing characters for nesting, such as quotedString or a comment
+ expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}.
+ The default is L{quotedString}, but if no expressions are to be ignored,
+ then pass C{None} for this argument.
+ """
+ if opener == closer:
+ raise ValueError("opening and closing strings cannot be the same")
+ if content is None:
+ if isinstance(opener,basestring) and isinstance(closer,basestring):
+ if len(opener) == 1 and len(closer)==1:
+ if ignoreExpr is not None:
+ content = (Combine(OneOrMore(~ignoreExpr +
+ CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ if ignoreExpr is not None:
+ content = (Combine(OneOrMore(~ignoreExpr +
+ ~Literal(opener) + ~Literal(closer) +
+ CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) +
+ CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ raise ValueError("opening and closing arguments must be strings if no content expression is given")
+ ret = Forward()
+ if ignoreExpr is not None:
+ ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
+ else:
+ ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) )
+ return ret
+
+def indentedBlock(blockStatementExpr, indentStack, indent=True):
+ """Helper method for defining space-delimited indentation blocks, such as
+ those used to define block statements in Python source code.
+
+ Parameters:
+ - blockStatementExpr - expression defining syntax of statement that
+ is repeated within the indented block
+ - indentStack - list created by caller to manage indentation stack
+ (multiple statementWithIndentedBlock expressions within a single grammar
+ should share a common indentStack)
+ - indent - boolean indicating whether block must be indented beyond the
+ the current level; set to False for block of left-most statements
+ (default=True)
+
+ A valid block must contain at least one C{blockStatement}.
+ """
+ def checkPeerIndent(s,l,t):
+ if l >= len(s): return
+ curCol = col(l,s)
+ if curCol != indentStack[-1]:
+ if curCol > indentStack[-1]:
+ raise ParseFatalException(s,l,"illegal nesting")
+ raise ParseException(s,l,"not a peer entry")
+
+ def checkSubIndent(s,l,t):
+ curCol = col(l,s)
+ if curCol > indentStack[-1]:
+ indentStack.append( curCol )
+ else:
+ raise ParseException(s,l,"not a subentry")
+
+ def checkUnindent(s,l,t):
+ if l >= len(s): return
+ curCol = col(l,s)
+ if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]):
+ raise ParseException(s,l,"not an unindent")
+ indentStack.pop()
+
+ NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress())
+ INDENT = Empty() + Empty().setParseAction(checkSubIndent)
+ PEER = Empty().setParseAction(checkPeerIndent)
+ UNDENT = Empty().setParseAction(checkUnindent)
+ if indent:
+ smExpr = Group( Optional(NL) +
+ #~ FollowedBy(blockStatementExpr) +
+ INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT)
+ else:
+ smExpr = Group( Optional(NL) +
+ (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )
+ blockStatementExpr.ignore(_bslash + LineEnd())
+ return smExpr
+
+alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
+punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
+
+anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:"))
+commonHTMLEntity = Combine(_L("&") + oneOf("gt lt amp nbsp quot").setResultsName("entity") +";").streamline()
+_htmlEntityMap = dict(zip("gt lt amp nbsp quot".split(),'><& "'))
+replaceHTMLEntity = lambda t : t.entity in _htmlEntityMap and _htmlEntityMap[t.entity] or None
+
+# it's easy to get these comment structures wrong - they're very common, so may as well make them available
+cStyleComment = Regex(r"/\*(?:[^*]*\*+)+?/").setName("C style comment")
+
+htmlComment = Regex(r"<!--[\s\S]*?-->")
+restOfLine = Regex(r".*").leaveWhitespace()
+dblSlashComment = Regex(r"\/\/(\\\n|.)*").setName("// comment")
+cppStyleComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))").setName("C++ style comment")
+
+javaStyleComment = cppStyleComment
+pythonStyleComment = Regex(r"#.*").setName("Python style comment")
+_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') +
+ Optional( Word(" \t") +
+ ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
+commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList")
+
+
+if __name__ == "__main__":
+
+ selectToken = CaselessLiteral( "select" )
+ fromToken = CaselessLiteral( "from" )
+
+ ident = Word( alphas, alphanums + "_$" )
+ columnName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
+ columnNameList = Group( delimitedList( columnName ) ).setName("columns")
+ tableName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
+ tableNameList = Group( delimitedList( tableName ) ).setName("tables")
+ simpleSQL = ( selectToken + \
+ ( '*' | columnNameList ).setResultsName( "columns" ) + \
+ fromToken + \
+ tableNameList.setResultsName( "tables" ) )
+
+ simpleSQL.runTests("""\
+ SELECT * from XYZZY, ABC
+ select * from SYS.XYZZY
+ Select A from Sys.dual
+ Select AA,BB,CC from Sys.dual
+ Select A, B, C from Sys.dual
+ Select A, B, C from Sys.dual
+ Xelect A, B, C from Sys.dual
+ Select A, B, C frox Sys.dual
+ Select
+ Select ^^^ frox Sys.dual
+ Select A, B, C from Sys.dual, Table2""")
+
diff --git a/pkg_resources/_vendor/vendored.txt b/pkg_resources/_vendor/vendored.txt index eec98267..46532c0a 100644 --- a/pkg_resources/_vendor/vendored.txt +++ b/pkg_resources/_vendor/vendored.txt @@ -1,2 +1,3 @@ -packaging==15.3 +packaging==16.7 +pyparsing==2.0.6 six==1.10.0 diff --git a/pkg_resources/api_tests.txt b/pkg_resources/api_tests.txt index d28db0f5..4fbd3d23 100644 --- a/pkg_resources/api_tests.txt +++ b/pkg_resources/api_tests.txt @@ -338,88 +338,64 @@ Environment Markers >>> import os >>> print(im("sys_platform")) - Comparison or logical expression expected + Invalid marker: 'sys_platform', parse error at '' >>> print(im("sys_platform==")) - invalid syntax + Invalid marker: 'sys_platform==', parse error at '' >>> print(im("sys_platform=='win32'")) False >>> print(im("sys=='x'")) - Unknown name 'sys' + Invalid marker: "sys=='x'", parse error at "sys=='x'" >>> print(im("(extra)")) - Comparison or logical expression expected + Invalid marker: '(extra)', parse error at ')' >>> print(im("(extra")) - invalid syntax + Invalid marker: '(extra', parse error at '' >>> print(im("os.open('foo')=='y'")) - Language feature not supported in environment markers + Invalid marker: "os.open('foo')=='y'", parse error at 'os.open(' >>> print(im("'x'=='y' and os.open('foo')=='y'")) # no short-circuit! - Language feature not supported in environment markers + Invalid marker: "'x'=='y' and os.open('foo')=='y'", parse error at 'and os.o' >>> print(im("'x'=='x' or os.open('foo')=='y'")) # no short-circuit! - Language feature not supported in environment markers + Invalid marker: "'x'=='x' or os.open('foo')=='y'", parse error at 'or os.op' >>> print(im("'x' < 'y' < 'z'")) - Chained comparison not allowed in environment markers + Invalid marker: "'x' < 'y' < 'z'", parse error at "< 'z'" >>> print(im("r'x'=='x'")) - Only plain strings allowed in environment markers + Invalid marker: "r'x'=='x'", parse error at "r'x'=='x" >>> print(im("'''x'''=='x'")) - Only plain strings allowed in environment markers + Invalid marker: "'''x'''=='x'", parse error at "'x'''=='" >>> print(im('"""x"""=="x"')) - Only plain strings allowed in environment markers + Invalid marker: '"""x"""=="x"', parse error at '"x"""=="' - >>> print(im(r"'x\n'=='x'")) - Only plain strings allowed in environment markers + >>> print(im(r"x\n=='x'")) + Invalid marker: "x\\n=='x'", parse error at "x\\n=='x'" >>> print(im("os.open=='y'")) - Language feature not supported in environment markers - - >>> em('"x"=="x"') - True - - >>> em('"x"=="y"') - False - - >>> em('"x"=="y" and "x"=="x"') - False - - >>> em('"x"=="y" or "x"=="x"') - True - - >>> em('"x"=="y" and "x"=="q" or "z"=="z"') - True - - >>> em('"x"=="y" and ("x"=="q" or "z"=="z")') - False - - >>> em('"x"=="y" and "z"=="z" or "x"=="q"') - False - - >>> em('"x"=="x" and "z"=="z" or "x"=="q"') - True + Invalid marker: "os.open=='y'", parse error at 'os.open=' >>> em("sys_platform=='win32'") == (sys.platform=='win32') True - >>> em("'x' in 'yx'") - True - - >>> em("'yx' in 'x'") - False - >>> em("python_version >= '2.6'") True >>> em("python_version > '2.5'") True + >>> im("implementation_name=='cpython'") + False + >>> im("platform_python_implementation=='CPython'") False + + >>> im("implementation_version=='3.5.1'") + False diff --git a/pkg_resources/extern/__init__.py b/pkg_resources/extern/__init__.py index 317f4b8d..6758d36f 100644 --- a/pkg_resources/extern/__init__.py +++ b/pkg_resources/extern/__init__.py @@ -67,5 +67,5 @@ class VendorImporter: if self not in sys.meta_path: sys.meta_path.append(self) -names = 'packaging', 'six' +names = 'packaging', 'pyparsing', 'six' VendorImporter(__name__, names).install() diff --git a/pkg_resources/tests/test_markers.py b/pkg_resources/tests/test_markers.py index d8844e74..8d451de3 100644 --- a/pkg_resources/tests/test_markers.py +++ b/pkg_resources/tests/test_markers.py @@ -1,16 +1,10 @@ try: - import unittest.mock as mock + import unittest.mock as mock except ImportError: - import mock + import mock from pkg_resources import evaluate_marker - -@mock.patch.dict('pkg_resources.MarkerEvaluation.values', - python_full_version=mock.Mock(return_value='2.7.10')) -def test_lexicographic_ordering(): - """ - Although one might like 2.7.10 to be greater than 2.7.3, - the marker spec only supports lexicographic ordering. - """ - assert evaluate_marker("python_full_version > '2.7.3'") is False +@mock.patch('platform.python_version', return_value='2.7.10') +def test_ordering(python_version_mock): + assert evaluate_marker("python_full_version > '2.7.3'") is True diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index f2afdf95..31847dc8 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -15,17 +15,6 @@ from pkg_resources import (parse_requirements, VersionConflict, parse_version, WorkingSet) -def safe_repr(obj, short=False): - """ copied from Python2.7""" - try: - result = repr(obj) - except Exception: - result = object.__repr__(obj) - if not short or len(result) < pkg_resources._MAX_LENGTH: - return result - return result[:pkg_resources._MAX_LENGTH] + ' [truncated]...' - - class Metadata(pkg_resources.EmptyProvider): """Mock object to return metadata as if from an on-disk distribution""" @@ -182,6 +171,100 @@ class TestDistro: msg = 'Foo 0.9 is installed but Foo==1.2 is required' assert vc.value.report() == msg + def test_environment_marker_evaluation_negative(self): + """Environment markers are evaluated at resolution time.""" + ad = pkg_resources.Environment([]) + ws = WorkingSet([]) + res = ws.resolve(parse_requirements("Foo;python_version<'2'"), ad) + assert list(res) == [] + + def test_environment_marker_evaluation_positive(self): + ad = pkg_resources.Environment([]) + ws = WorkingSet([]) + Foo = Distribution.from_filename("/foo_dir/Foo-1.2.dist-info") + ad.add(Foo) + res = ws.resolve(parse_requirements("Foo;python_version>='2'"), ad) + assert list(res) == [Foo] + + def test_environment_marker_evaluation_called(self): + """ + If one package foo requires bar without any extras, + markers should pass for bar without extras. + """ + parent_req, = parse_requirements("foo") + req, = parse_requirements("bar;python_version>='2'") + req_extras = pkg_resources._ReqExtras({req: parent_req.extras}) + assert req_extras.markers_pass(req) + + parent_req, = parse_requirements("foo[]") + req, = parse_requirements("bar;python_version>='2'") + req_extras = pkg_resources._ReqExtras({req: parent_req.extras}) + assert req_extras.markers_pass(req) + + def test_marker_evaluation_with_extras(self): + """Extras are also evaluated as markers at resolution time.""" + ad = pkg_resources.Environment([]) + ws = WorkingSet([]) + # Metadata needs to be native strings due to cStringIO behaviour in + # 2.6, so use str(). + Foo = Distribution.from_filename( + "/foo_dir/Foo-1.2.dist-info", + metadata=Metadata(("METADATA", str("Provides-Extra: baz\n" + "Requires-Dist: quux; extra=='baz'"))) + ) + ad.add(Foo) + assert list(ws.resolve(parse_requirements("Foo"), ad)) == [Foo] + quux = Distribution.from_filename("/foo_dir/quux-1.0.dist-info") + ad.add(quux) + res = list(ws.resolve(parse_requirements("Foo[baz]"), ad)) + assert res == [Foo,quux] + + def test_marker_evaluation_with_multiple_extras(self): + ad = pkg_resources.Environment([]) + ws = WorkingSet([]) + # Metadata needs to be native strings due to cStringIO behaviour in + # 2.6, so use str(). + Foo = Distribution.from_filename( + "/foo_dir/Foo-1.2.dist-info", + metadata=Metadata(("METADATA", str("Provides-Extra: baz\n" + "Requires-Dist: quux; extra=='baz'\n" + "Provides-Extra: bar\n" + "Requires-Dist: fred; extra=='bar'\n"))) + ) + ad.add(Foo) + quux = Distribution.from_filename("/foo_dir/quux-1.0.dist-info") + ad.add(quux) + fred = Distribution.from_filename("/foo_dir/fred-0.1.dist-info") + ad.add(fred) + res = list(ws.resolve(parse_requirements("Foo[baz,bar]"), ad)) + assert sorted(res) == [fred,quux,Foo] + + def test_marker_evaluation_with_extras_loop(self): + ad = pkg_resources.Environment([]) + ws = WorkingSet([]) + # Metadata needs to be native strings due to cStringIO behaviour in + # 2.6, so use str(). + a = Distribution.from_filename( + "/foo_dir/a-0.2.dist-info", + metadata=Metadata(("METADATA", str("Requires-Dist: c[a]"))) + ) + b = Distribution.from_filename( + "/foo_dir/b-0.3.dist-info", + metadata=Metadata(("METADATA", str("Requires-Dist: c[b]"))) + ) + c = Distribution.from_filename( + "/foo_dir/c-1.0.dist-info", + metadata=Metadata(("METADATA", str("Provides-Extra: a\n" + "Requires-Dist: b;extra=='a'\n" + "Provides-Extra: b\n" + "Requires-Dist: foo;extra=='b'"))) + ) + foo = Distribution.from_filename("/foo_dir/foo-0.1.dist-info") + for dist in (a, b, c, foo): + ad.add(dist) + res = list(ws.resolve(parse_requirements("a"), ad)) + assert res == [a, c, b, foo] + def testDistroDependsOptions(self): d = self.distRequires(""" Twisted>=1.5 @@ -314,7 +397,10 @@ class TestEntryPoints: def checkSubMap(self, m): assert len(m) == len(self.submap_expect) for key, ep in self.submap_expect.items(): - assert repr(m.get(key)) == repr(ep) + assert m.get(key).name == ep.name + assert m.get(key).module_name == ep.module_name + assert sorted(m.get(key).attrs) == sorted(ep.attrs) + assert sorted(m.get(key).extras) == sorted(ep.extras) submap_expect = dict( feature1=EntryPoint('feature1', 'somemodule', ['somefunction']), @@ -353,22 +439,22 @@ class TestRequirements: r = Requirement.parse("Twisted>=1.2") assert str(r) == "Twisted>=1.2" assert repr(r) == "Requirement.parse('Twisted>=1.2')" - assert r == Requirement("Twisted", [('>=','1.2')], ()) - assert r == Requirement("twisTed", [('>=','1.2')], ()) - assert r != Requirement("Twisted", [('>=','2.0')], ()) - assert r != Requirement("Zope", [('>=','1.2')], ()) - assert r != Requirement("Zope", [('>=','3.0')], ()) - assert r != Requirement.parse("Twisted[extras]>=1.2") + assert r == Requirement("Twisted>=1.2") + assert r == Requirement("twisTed>=1.2") + assert r != Requirement("Twisted>=2.0") + assert r != Requirement("Zope>=1.2") + assert r != Requirement("Zope>=3.0") + assert r != Requirement("Twisted[extras]>=1.2") def testOrdering(self): - r1 = Requirement("Twisted", [('==','1.2c1'),('>=','1.2')], ()) - r2 = Requirement("Twisted", [('>=','1.2'),('==','1.2c1')], ()) + r1 = Requirement("Twisted==1.2c1,>=1.2") + r2 = Requirement("Twisted>=1.2,==1.2c1") assert r1 == r2 assert str(r1) == str(r2) assert str(r2) == "Twisted==1.2c1,>=1.2" def testBasicContains(self): - r = Requirement("Twisted", [('>=','1.2')], ()) + r = Requirement("Twisted>=1.2") foo_dist = Distribution.from_filename("FooPkg-1.3_1.egg") twist11 = Distribution.from_filename("Twisted-1.1.egg") twist12 = Distribution.from_filename("Twisted-1.2.egg") @@ -384,8 +470,8 @@ class TestRequirements: r1 = Requirement.parse("Twisted[foo,bar]>=1.2") r2 = Requirement.parse("Twisted[bar,FOO]>=1.2") assert r1 == r2 - assert r1.extras == ("foo","bar") - assert r2.extras == ("bar","foo") # extras are normalized + assert set(r1.extras) == set(("foo", "bar")) + assert set(r2.extras) == set(("foo", "bar")) assert hash(r1) == hash(r2) assert ( hash(r1) @@ -394,6 +480,7 @@ class TestRequirements: "twisted", packaging.specifiers.SpecifierSet(">=1.2"), frozenset(["foo","bar"]), + None )) ) @@ -485,17 +572,17 @@ class TestParsing: assert ( list(parse_requirements('Twis-Ted>=1.2-1')) == - [Requirement('Twis-Ted',[('>=','1.2-1')], ())] + [Requirement('Twis-Ted>=1.2-1')] ) assert ( list(parse_requirements('Twisted >=1.2, \ # more\n<2.0')) == - [Requirement('Twisted',[('>=','1.2'),('<','2.0')], ())] + [Requirement('Twisted>=1.2,<2.0')] ) assert ( Requirement.parse("FooBar==1.99a3") == - Requirement("FooBar", [('==','1.99a3')], ()) + Requirement("FooBar==1.99a3") ) with pytest.raises(ValueError): Requirement.parse(">=2.3") @@ -508,6 +595,35 @@ class TestParsing: with pytest.raises(ValueError): Requirement.parse("#") + def test_requirements_with_markers(self): + assert ( + Requirement.parse("foobar;os_name=='a'") + == + Requirement.parse("foobar;os_name=='a'") + ) + assert ( + Requirement.parse("name==1.1;python_version=='2.7'") + != + Requirement.parse("name==1.1;python_version=='3.3'") + ) + assert ( + Requirement.parse("name==1.0;python_version=='2.7'") + != + Requirement.parse("name==1.2;python_version=='2.7'") + ) + assert ( + Requirement.parse("name[foo]==1.0;python_version=='3.3'") + != + Requirement.parse("name[foo,bar]==1.0;python_version=='3.3'") + ) + + def test_local_version(self): + req, = parse_requirements('foo==1.0.org1') + + def test_spaces_between_multiple_versions(self): + req, = parse_requirements('foo>=1.0, <3') + req, = parse_requirements('foo >= 1.0, < 3') + def testVersionEquality(self): def c(s1,s2): p1, p2 = parse_version(s1),parse_version(s2) @@ -687,7 +803,7 @@ class TestNamespaces: sys.path is imported, and that the namespace package's __path__ is in the correct order. - Regression test for https://bitbucket.org/pypa/setuptools/issues/207 + Regression test for https://github.com/pypa/setuptools/issues/207 """ tmpdir = symlinked_tmpdir @@ -1,3 +1,3 @@ [pytest] -addopts=--doctest-modules --ignore release.py --ignore setuptools/lib2to3_ex.py --ignore tests/manual_test.py --ignore tests/shlib_test --doctest-glob=pkg_resources/api_tests.txt --ignore scripts/upload-old-releases-as-zip.py +addopts=--doctest-modules --ignore release.py --ignore setuptools/lib2to3_ex.py --ignore tests/manual_test.py --ignore tests/shlib_test --doctest-glob=pkg_resources/api_tests.txt --ignore scripts/upload-old-releases-as-zip.py --ignore pavement.py norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern diff --git a/release.py b/release.py deleted file mode 100644 index dd1d6a1c..00000000 --- a/release.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Setuptools is released using 'jaraco.packaging.release'. To make a release, -install jaraco.packaging and run 'python -m jaraco.packaging.release' -""" - -import os - -import pkg_resources - -pkg_resources.require('jaraco.packaging>=2.0') -pkg_resources.require('wheel') - -files_with_versions = 'setuptools/version.py', - -# bdist_wheel must be included or pip will break -dist_commands = 'sdist', 'bdist_wheel' - -test_info = "Travis-CI tests: http://travis-ci.org/#!/jaraco/setuptools" - -os.environ["SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES"] = "1" @@ -1,8 +1,15 @@ +[bumpversion] +current_version = 21.0.0 +commit = True +tag = True + [egg_info] -tag_build = dev +tag_build = .post +tag_date = 1 [aliases] -release = egg_info -RDb '' +clean_egg_info = egg_info -RDb '' +release = clean_egg_info sdist bdist_wheel build_sphinx source = register sdist binary binary = bdist_egg upload --show-response test = pytest @@ -19,4 +26,7 @@ upload-dir = docs/build/html formats = gztar zip [wheel] -universal=1 +universal = 1 + +[bumpversion:file:setup.py] + @@ -22,11 +22,6 @@ with open(init_path) as init_file: SETUP_COMMANDS = command_ns['__all__'] -main_ns = {} -ver_path = convert_path('setuptools/version.py') -with open(ver_path) as ver_file: - exec(ver_file.read(), main_ns) - import setuptools scripts = [] @@ -48,7 +43,7 @@ def _gen_console_scripts(): console_scripts = list(_gen_console_scripts()) -readme_file = io.open('README.txt', encoding='utf-8') +readme_file = io.open('README.rst', encoding='utf-8') with readme_file: long_description = readme_file.read() @@ -66,19 +61,21 @@ if (sys.platform == 'win32' or (os.name == 'java' and os._name == 'nt')) \ needs_pytest = set(['ptr', 'pytest', 'test']).intersection(sys.argv) pytest_runner = ['pytest-runner'] if needs_pytest else [] -needs_sphinx = set(['build_sphinx', 'upload_docs']).intersection(sys.argv) -sphinx = ['sphinx', 'rst.linker'] if needs_sphinx else [] +needs_sphinx = set(['build_sphinx', 'upload_docs', 'release']).intersection(sys.argv) +sphinx = ['sphinx', 'rst.linker>=1.5'] if needs_sphinx else [] +needs_wheel = set(['release', 'bdist_wheel']).intersection(sys.argv) +wheel = ['wheel'] if needs_wheel else [] setup_params = dict( name="setuptools", - version=main_ns['__version__'], + version="21.0.0", description="Easily download, build, install, upgrade, and uninstall " "Python packages", author="Python Packaging Authority", author_email="distutils-sig@python.org", long_description=long_description, keywords="CPAN PyPI distutils eggs package management", - url="https://bitbucket.org/pypa/setuptools", + url="https://github.com/pypa/setuptools", src_root=src_root, packages=setuptools.find_packages(exclude=['*.tests']), package_data=package_data, @@ -149,10 +146,10 @@ setup_params = dict( """).strip().splitlines(), extras_require={ "ssl:sys_platform=='win32'": "wincertstore==0.2", - "certs": "certifi==2015.11.20", + "certs": "certifi==2016.2.28", }, dependency_links=[ - 'https://pypi.python.org/packages/source/c/certifi/certifi-2015.11.20.tar.gz#md5=25134646672c695c1ff1593c2dd75d08', + 'https://pypi.python.org/packages/source/c/certifi/certifi-2016.2.28.tar.gz#md5=5d672aa766e1f773c75cfeccd02d3650', 'https://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2', ], scripts=[], @@ -161,7 +158,7 @@ setup_params = dict( 'pytest>=2.8', ] + (['mock'] if sys.version_info[:2] < (3, 3) else []), setup_requires=[ - ] + sphinx + pytest_runner, + ] + sphinx + pytest_runner + wheel, ) if __name__ == '__main__': diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 46056173..ea5cb028 100755 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -710,10 +710,7 @@ class easy_install(Command): elif requirement is None or dist not in requirement: # if we wound up with a different version, resolve what we've got distreq = dist.as_requirement() - requirement = requirement or distreq - requirement = Requirement( - distreq.project_name, distreq.specs, requirement.extras - ) + requirement = Requirement(str(distreq)) log.info("Processing dependencies for %s", requirement) try: distros = WorkingSet([]).resolve( @@ -783,7 +780,7 @@ class easy_install(Command): There are a couple of template scripts in the package. This function loads one of them and prepares it for use. """ - # See https://bitbucket.org/pypa/setuptools/issue/134 for info + # See https://github.com/pypa/setuptools/issues/134 for info # on script file naming and downstream issues with SVR4 name = 'script.tmpl' if dev_path: @@ -1239,17 +1236,14 @@ class easy_install(Command): sitepy = os.path.join(self.install_dir, "site.py") source = resource_string("setuptools", "site-patch.py") + source = source.decode('utf-8') current = "" if os.path.exists(sitepy): log.debug("Checking existing site.py in %s", self.install_dir) - f = open(sitepy, 'rb') - current = f.read() - # we want str, not bytes - if six.PY3: - current = current.decode() + with io.open(sitepy) as strm: + current = strm.read() - f.close() if not current.startswith('def __boot():'): raise DistutilsError( "%s is not a setuptools-generated site.py; please" @@ -1260,9 +1254,8 @@ class easy_install(Command): log.info("Creating %s", sitepy) if not self.dry_run: ensure_directory(sitepy) - f = open(sitepy, 'wb') - f.write(source) - f.close() + with io.open(sitepy, 'w', encoding='utf-8') as strm: + strm.write(source) self.byte_compile([sitepy]) self.sitepy_installed = True @@ -1769,7 +1762,7 @@ def _update_zipimporter_cache(normalized_path, cache, updater=None): # * Does not support the dict.pop() method, forcing us to use the # get/del patterns instead. For more detailed information see the # following links: - # https://bitbucket.org/pypa/setuptools/issue/202/more-robust-zipimporter-cache-invalidation#comment-10495960 + # https://github.com/pypa/setuptools/issues/202#issuecomment-202913420 # https://bitbucket.org/pypy/pypy/src/dd07756a34a41f674c0cacfbc8ae1d4cc9ea2ae4/pypy/module/zipimport/interp_zipimport.py#cl-99 old_entry = cache[p] del cache[p] diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index d1bd9b04..8e1502a5 100755 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -13,6 +13,7 @@ import sys import io import warnings import time +import collections from setuptools.extern import six from setuptools.extern.six.moves import map @@ -66,14 +67,20 @@ class egg_info(Command): self.vtags = None def save_version_info(self, filename): - values = dict( - egg_info=dict( - tag_svn_revision=0, - tag_date=0, - tag_build=self.tags(), - ) - ) - edit_config(filename, values) + """ + Materialize the values of svn_revision and date into the + build tag. Install these keys in a deterministic order + to avoid arbitrary reordering on subsequent builds. + """ + # python 2.6 compatibility + odict = getattr(collections, 'OrderedDict', dict) + egg_info = odict() + # follow the order these keys would have been added + # when PYTHONHASHSEED=0 + egg_info['tag_build'] = self.tags() + egg_info['tag_date'] = 0 + egg_info['tag_svn_revision'] = 0 + edit_config(filename, dict(egg_info=egg_info)) def finalize_options(self): self.egg_name = safe_name(self.distribution.get_name()) diff --git a/setuptools/command/install.py b/setuptools/command/install.py index d2bca2ec..31a5ddb5 100644 --- a/setuptools/command/install.py +++ b/setuptools/command/install.py @@ -8,7 +8,7 @@ import distutils.command.install as orig import setuptools # Prior to numpy 1.9, NumPy relies on the '_install' name, so provide it for -# now. See https://bitbucket.org/pypa/setuptools/issue/199/ +# now. See https://github.com/pypa/setuptools/issues/199/ _install = orig.install diff --git a/setuptools/command/rotate.py b/setuptools/command/rotate.py index 804f962a..b89353f5 100755 --- a/setuptools/command/rotate.py +++ b/setuptools/command/rotate.py @@ -2,6 +2,7 @@ from distutils.util import convert_path from distutils import log from distutils.errors import DistutilsOptionError import os +import shutil from setuptools.extern import six @@ -59,4 +60,7 @@ class rotate(Command): for (t, f) in files: log.info("Deleting %s", f) if not self.dry_run: - os.unlink(f) + if os.path.isdir(f): + shutil.rmtree(f) + else: + os.unlink(f) diff --git a/setuptools/command/upload.py b/setuptools/command/upload.py index 08c20ba8..484baa5a 100644 --- a/setuptools/command/upload.py +++ b/setuptools/command/upload.py @@ -1,15 +1,22 @@ +import getpass from distutils.command import upload as orig class upload(orig.upload): """ - Override default upload behavior to look up password - in the keyring if available. + Override default upload behavior to obtain password + in a variety of different ways. """ def finalize_options(self): orig.upload.finalize_options(self) - self.password or self._load_password_from_keyring() + # Attempt to obtain password. Short circuit evaluation at the first + # sign of success. + self.password = ( + self.password or + self._load_password_from_keyring() or + self._prompt_for_password() + ) def _load_password_from_keyring(self): """ @@ -17,7 +24,15 @@ class upload(orig.upload): """ try: keyring = __import__('keyring') - self.password = keyring.get_password(self.repository, - self.username) + return keyring.get_password(self.repository, self.username) except Exception: pass + + def _prompt_for_password(self): + """ + Prompt for a password on the tty. Suppress Exceptions. + """ + try: + return getpass.getpass() + except (Exception, KeyboardInterrupt): + pass diff --git a/setuptools/dist.py b/setuptools/dist.py index 77855415..086e0a58 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -720,7 +720,7 @@ class Feature: """ **deprecated** -- The `Feature` facility was never completely implemented or supported, `has reported issues - <https://bitbucket.org/pypa/setuptools/issue/58>`_ and will be removed in + <https://github.com/pypa/setuptools/issues/58>`_ and will be removed in a future version. A subset of the distribution that can be excluded if unneeded/wanted @@ -777,7 +777,7 @@ class Feature: def warn_deprecated(): warnings.warn( "Features are deprecated and will be removed in a future " - "version. See http://bitbucket.org/pypa/setuptools/65.", + "version. See https://github.com/pypa/setuptools/issues/65.", DeprecationWarning, stacklevel=3, ) diff --git a/setuptools/launch.py b/setuptools/launch.py index 06e15e1e..b05cbd2c 100644 --- a/setuptools/launch.py +++ b/setuptools/launch.py @@ -11,25 +11,25 @@ import sys def run(): - """ - Run the script in sys.argv[1] as if it had - been invoked naturally. - """ - __builtins__ - script_name = sys.argv[1] - namespace = dict( - __file__ = script_name, - __name__ = '__main__', - __doc__ = None, - ) - sys.argv[:] = sys.argv[1:] + """ + Run the script in sys.argv[1] as if it had + been invoked naturally. + """ + __builtins__ + script_name = sys.argv[1] + namespace = dict( + __file__ = script_name, + __name__ = '__main__', + __doc__ = None, + ) + sys.argv[:] = sys.argv[1:] - open_ = getattr(tokenize, 'open', open) - script = open_(script_name).read() - norm_script = script.replace('\\r\\n', '\\n') - code = compile(norm_script, script_name, 'exec') - exec(code, namespace) + open_ = getattr(tokenize, 'open', open) + script = open_(script_name).read() + norm_script = script.replace('\\r\\n', '\\n') + code = compile(norm_script, script_name, 'exec') + exec(code, namespace) if __name__ == '__main__': - run() + run() diff --git a/setuptools/py26compat.py b/setuptools/py26compat.py index e52bd85b..40cbb88e 100644 --- a/setuptools/py26compat.py +++ b/setuptools/py26compat.py @@ -5,18 +5,18 @@ Compatibility Support for Python 2.6 and earlier import sys try: - from urllib.parse import splittag + from urllib.parse import splittag except ImportError: - from urllib import splittag + from urllib import splittag def strip_fragment(url): - """ - In `Python 8280 <http://bugs.python.org/issue8280>`_, Python 2.7 and - later was patched to disregard the fragment when making URL requests. - Do the same for Python 2.6 and earlier. - """ - url, fragment = splittag(url) - return url + """ + In `Python 8280 <http://bugs.python.org/issue8280>`_, Python 2.7 and + later was patched to disregard the fragment when making URL requests. + Do the same for Python 2.6 and earlier. + """ + url, fragment = splittag(url) + return url if sys.version_info >= (2,7): - strip_fragment = lambda x: x + strip_fragment = lambda x: x diff --git a/setuptools/py27compat.py b/setuptools/py27compat.py index 9d2886db..702f7d65 100644 --- a/setuptools/py27compat.py +++ b/setuptools/py27compat.py @@ -5,11 +5,11 @@ Compatibility Support for Python 2.7 and earlier import sys def get_all_headers(message, key): - """ - Given an HTTPMessage, return all headers matching a given key. - """ - return message.get_all(key) + """ + Given an HTTPMessage, return all headers matching a given key. + """ + return message.get_all(key) if sys.version_info < (3,): - def get_all_headers(message, key): - return message.getheaders(key) + def get_all_headers(message, key): + return message.getheaders(key) diff --git a/setuptools/tests/py26compat.py b/setuptools/tests/py26compat.py index c5680881..7211f275 100644 --- a/setuptools/tests/py26compat.py +++ b/setuptools/tests/py26compat.py @@ -3,10 +3,10 @@ import tarfile import contextlib def _tarfile_open_ex(*args, **kwargs): - """ - Extend result as a context manager. - """ - return contextlib.closing(tarfile.open(*args, **kwargs)) + """ + Extend result as a context manager. + """ + return contextlib.closing(tarfile.open(*args, **kwargs)) if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 2): tarfile_open = _tarfile_open_ex diff --git a/setuptools/tests/test_dist_info.py b/setuptools/tests/test_dist_info.py index abd0a763..9f226a55 100644 --- a/setuptools/tests/test_dist_info.py +++ b/setuptools/tests/test_dist_info.py @@ -28,14 +28,15 @@ class TestDistInfo: assert versioned.version == '2.718' # from filename assert unversioned.version == '0.3' # from METADATA - @pytest.mark.importorskip('ast') def test_conditional_dependencies(self): specs = 'splort==4', 'quux>=1.1' requires = list(map(pkg_resources.Requirement.parse, specs)) for d in pkg_resources.find_distributions(self.tmpdir): assert d.requires() == requires[:1] - assert d.requires(extras=('baz',)) == requires + assert d.requires(extras=('baz',)) == [ + requires[0], + pkg_resources.Requirement.parse('quux>=1.1;extra=="baz"')] assert d.extras == ['baz'] metadata_template = DALS(""" diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index 07d8a3c5..55b8b05a 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -16,7 +16,6 @@ import itertools import distutils.errors import io -from setuptools.extern import six from setuptools.extern.six.moves import urllib import time @@ -38,7 +37,7 @@ import setuptools.tests.server import pkg_resources from .py26compat import tarfile_open -from . import contexts, is_ascii +from . import contexts from .textwrap import DALS @@ -59,17 +58,13 @@ SETUP_PY = DALS(""" class TestEasyInstallTest: - def test_install_site_py(self): + def test_install_site_py(self, tmpdir): dist = Distribution() cmd = ei.easy_install(dist) cmd.sitepy_installed = False - cmd.install_dir = tempfile.mkdtemp() - try: - cmd.install_site_py() - sitepy = os.path.join(cmd.install_dir, 'site.py') - assert os.path.exists(sitepy) - finally: - shutil.rmtree(cmd.install_dir) + cmd.install_dir = str(tmpdir) + cmd.install_site_py() + assert (tmpdir / 'site.py').exists() def test_get_script_args(self): header = ei.CommandSpec.best().from_environment().as_header() diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index 7d51585b..3a0db58f 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -1,6 +1,11 @@ import os +import glob +import re import stat +import sys +from setuptools.command.egg_info import egg_info +from setuptools.dist import Distribution from setuptools.extern.six.moves import map import pytest @@ -58,6 +63,79 @@ class TestEggInfo(object): }) yield env + def test_egg_info_save_version_info_setup_empty(self, tmpdir_cwd, env): + """ + When the egg_info section is empty or not present, running + save_version_info should add the settings to the setup.cfg + in a deterministic order, consistent with the ordering found + on Python 2.6 and 2.7 with PYTHONHASHSEED=0. + """ + setup_cfg = os.path.join(env.paths['home'], 'setup.cfg') + dist = Distribution() + ei = egg_info(dist) + ei.initialize_options() + ei.save_version_info(setup_cfg) + + with open(setup_cfg, 'r') as f: + content = f.read() + + assert '[egg_info]' in content + assert 'tag_build =' in content + assert 'tag_date = 0' in content + assert 'tag_svn_revision = 0' in content + + expected_order = 'tag_build', 'tag_date', 'tag_svn_revision' + + self._validate_content_order(content, expected_order) + + @staticmethod + def _validate_content_order(content, expected): + """ + Assert that the strings in expected appear in content + in order. + """ + if sys.version_info < (2, 7): + # On Python 2.6, expect dict key order. + expected = dict.fromkeys(expected).keys() + + pattern = '.*'.join(expected) + flags = re.MULTILINE | re.DOTALL + assert re.search(pattern, content, flags) + + def test_egg_info_save_version_info_setup_defaults(self, tmpdir_cwd, env): + """ + When running save_version_info on an existing setup.cfg + with the 'default' values present from a previous run, + the file should remain unchanged, except on Python 2.6, + where the order of the keys will be changed to match the + order as found in a dictionary of those keys. + """ + setup_cfg = os.path.join(env.paths['home'], 'setup.cfg') + build_files({ + setup_cfg: DALS(""" + [egg_info] + tag_build = + tag_date = 0 + tag_svn_revision = 0 + """), + }) + dist = Distribution() + ei = egg_info(dist) + ei.initialize_options() + ei.save_version_info(setup_cfg) + + with open(setup_cfg, 'r') as f: + content = f.read() + + assert '[egg_info]' in content + assert 'tag_build =' in content + assert 'tag_date = 0' in content + assert 'tag_svn_revision = 0' in content + + expected_order = 'tag_build', 'tag_date', 'tag_svn_revision' + + self._validate_content_order(content, expected_order) + def test_egg_base_installed_egg_info(self, tmpdir_cwd, env): self._create_project() @@ -89,17 +167,61 @@ class TestEggInfo(object): sources_txt = os.path.join(egg_info_dir, 'SOURCES.txt') assert 'docs/usage.rst' in open(sources_txt).read().split('\n') - def _run_install_command(self, tmpdir_cwd, env): + def _setup_script_with_requires(self, requires_line): + setup_script = DALS(""" + from setuptools import setup + + setup( + name='foo', + %s + zip_safe=False, + ) + """ % requires_line) + build_files({ + 'setup.py': setup_script, + }) + + def test_install_requires_with_markers(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """install_requires=["barbazquux;python_version<'2'"],""") + self._run_install_command(tmpdir_cwd, env) + egg_info_dir = self._find_egg_info_files(env.paths['lib']).base + requires_txt = os.path.join(egg_info_dir, 'requires.txt') + assert "barbazquux;python_version<'2'" in open( + requires_txt).read().split('\n') + assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + + def test_setup_requires_with_markers(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """setup_requires=["barbazquux;python_version<'2'"],""") + self._run_install_command(tmpdir_cwd, env) + assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + + def test_tests_require_with_markers(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """tests_require=["barbazquux;python_version<'2'"],""") + self._run_install_command( + tmpdir_cwd, env, cmd=['test'], output="Ran 0 tests in") + assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + + def test_extra_requires_with_markers(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """extra_requires={":python_version<'2'": ["barbazquux"]},""") + self._run_install_command(tmpdir_cwd, env) + assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + + def _run_install_command(self, tmpdir_cwd, env, cmd=None, output=None): environ = os.environ.copy().update( HOME=env.paths['home'], ) - cmd = [ - 'install', - '--home', env.paths['home'], - '--install-lib', env.paths['lib'], - '--install-scripts', env.paths['scripts'], - '--install-data', env.paths['data'], - ] + if cmd is None: + cmd = [ + 'install', + '--home', env.paths['home'], + '--install-lib', env.paths['lib'], + '--install-scripts', env.paths['scripts'], + '--install-data', env.paths['data'], + ] code, data = environment.run_setup_py( cmd=cmd, pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), @@ -108,6 +230,8 @@ class TestEggInfo(object): ) if code: raise AssertionError(data) + if output: + assert output in data def _find_egg_info_files(self, root): class DirList(list): diff --git a/setuptools/tests/test_markerlib.py b/setuptools/tests/test_markerlib.py deleted file mode 100644 index 8197b49d..00000000 --- a/setuptools/tests/test_markerlib.py +++ /dev/null @@ -1,63 +0,0 @@ -import os - -import pytest - - -class TestMarkerlib: - - @pytest.mark.importorskip('ast') - def test_markers(self): - from _markerlib import interpret, default_environment, compile - - os_name = os.name - - assert interpret("") - - assert interpret("os.name != 'buuuu'") - assert interpret("os_name != 'buuuu'") - assert interpret("python_version > '1.0'") - assert interpret("python_version < '5.0'") - assert interpret("python_version <= '5.0'") - assert interpret("python_version >= '1.0'") - assert interpret("'%s' in os.name" % os_name) - assert interpret("'%s' in os_name" % os_name) - assert interpret("'buuuu' not in os.name") - - assert not interpret("os.name == 'buuuu'") - assert not interpret("os_name == 'buuuu'") - assert not interpret("python_version < '1.0'") - assert not interpret("python_version > '5.0'") - assert not interpret("python_version >= '5.0'") - assert not interpret("python_version <= '1.0'") - assert not interpret("'%s' not in os.name" % os_name) - assert not interpret("'buuuu' in os.name and python_version >= '5.0'") - assert not interpret("'buuuu' in os_name and python_version >= '5.0'") - - environment = default_environment() - environment['extra'] = 'test' - assert interpret("extra == 'test'", environment) - assert not interpret("extra == 'doc'", environment) - - def raises_nameError(): - try: - interpret("python.version == '42'") - except NameError: - pass - else: - raise Exception("Expected NameError") - - raises_nameError() - - def raises_syntaxError(): - try: - interpret("(x for x in (4,))") - except SyntaxError: - pass - else: - raise Exception("Expected SyntaxError") - - raises_syntaxError() - - statement = "python_version == '5'" - assert compile(statement).__doc__ == statement - diff --git a/setuptools/version.py b/setuptools/version.py index 4211fc44..f2b40722 100644 --- a/setuptools/version.py +++ b/setuptools/version.py @@ -1 +1,6 @@ -__version__ = '20.1.2' +import pkg_resources + +try: + __version__ = pkg_resources.require('setuptools')[0].version +except Exception: + __version__ = 'unknown' diff --git a/tests/manual_test.py b/tests/manual_test.py index af4ec09b..808fa55a 100644 --- a/tests/manual_test.py +++ b/tests/manual_test.py @@ -43,10 +43,8 @@ PYVER = sys.version.split()[0][:3] _VARS = {'base': '.', 'py_version_short': PYVER} -if sys.platform == 'win32': - PURELIB = INSTALL_SCHEMES['nt']['purelib'] -else: - PURELIB = INSTALL_SCHEMES['unix_prefix']['purelib'] +scheme = 'nt' if sys.platform == 'win32' else 'unix_prefix' +PURELIB = INSTALL_SCHEMES[scheme]['purelib'] @tempdir |