diff options
82 files changed, 6940 insertions, 2157 deletions
diff --git a/.hgignore b/.hgignore deleted file mode 100644 index ebc53b33..00000000 --- a/.hgignore +++ /dev/null @@ -1,15 +0,0 @@ -syntax: glob -bin -build -dist -include -lib -distribute.egg-info -setuptools.egg-info -.coverage -.tox -*.egg -*.py[cod] -*.swp -*~ -.git* diff --git a/.hgtags b/.hgtags deleted file mode 100644 index 86954cd8..00000000 --- a/.hgtags +++ /dev/null @@ -1,274 +0,0 @@ -7e9441311eb21dd1fbc32cfbad58168e46c5450e 0.6 -26f429772565f69d1f6d21adf57c3d8c40197129 0.6.1 -6f46749a7454be6e044a54cd73c51318b74bdee8 0.6.2 -34b80fb58862d18f8f957f98a883ed4a72d06f8e 0.6.3 -fb04abddb50d82a9005c9082c94d5eb983be1d79 0.6.4 -8ae0bd250b4a0d58cbaf16b4354ad60f73f24a01 0.6.5 -88847883dfed39829d3a5ed292ad540723ad31cc 0.6.6 -fcbef325349ada38f6c674eb92db82664cf6437c 0.6.7 -3af7f2b8270b9bb34fb65f08ee567bfe8e2a6a5a 0.6.8 -669725d03fd1e345ea47590e9b14cb19742b96a2 0.6.9 -eff3ca9c2d8d39e24c221816c52a37f964535336 0.6.10 -88710e34b91c98c9348749722cce3acd574d177d 0.6.11 -5ce754773a43ac21f7bd13872f45c75e27b593f8 0.6.12 -de36566d35e51bee7cfc86ffa694795e52f4147c 0.6.13 -e5f3f0ffe9e1a243d49a06f26c79dd160f521483 0.6.14 -dc03a300ec7a89ad773047172d43e52b34e7cd1e 0.6.15 -e620fb4ee8ba17debadb614fb583c6dfac229dea 0.6.16 -21df276275b5a47c6a994927d69ad3d90cf62b5d 0.6.17 -e9264ca4ba8c24239c36a8426a0394f7c7d5dd83 0.6.18 -aed31b1fa47ed1f39e55c75b76bbbdb80775b7f1 0.6.19 -c6e6273587816c3e486ef7739e53c864a0145251 0.6.20 -7afdf4c84a713fe151e6163ab25d45e8727ce653 0.6.21 -105066342777cd1319a95d7ae0271a2ea1ac33fe 0.6.23 -7b5ef4e6c80e82541dffb5a9a130d81550d5a835 0.6.24 -9c014a80f32e532371826ed1dc3236975f37f371 0.6.25 -ff8c4d6c8e5d2093750a58a3d43b76556570007c 0.6.26 -2a5c42ed097a195e398b97261c40cd66c8da8913 0.6.27 -4ed34b38851f90278cfe2bff75784f7e32883725 0.6.28 -acecfa2cfb6fca207dd2f4e025c695def3bb6b40 0.6.29 -e950f50addff150859f5990b9df2a33c691b6354 0.6.30 -06dae3faee2de50ff17b90719df410b2ebc5b71e 0.6.31 -1f4f79258ed5b418f680a55d3006f41aa6a56d2b 0.6.32 -89f57bf1406a5e745470af35446902c21ac9b6f6 0.6.33 -3c8f9fc13862124cf20ef2ff2140254fb272bb94 0.6.34 -7c3f8b9eb7cfa17481c835d5caaa918d337c7a83 0.6.35 -192094c0d1e2e5d2cb5c718f84a36c9de04b314b 0.6.36 -66d4e3b8899166e4c04189ee1831c649b7ff38bf 0.6.37 -398d58aa8bba33778c30ce72055a27d4b425809c 0.6.38 -f457fc2a3ebe609d8ca7a869eb65b7506ecf49ef 0.6.39 -9b2e2aa06e058c63e06c5e42a7f279ddae2dfb7d 0.7b1 -9089a40343981baa593b9bb5953f9088e9507099 0.6.40 -ad107e9b4beea24516ac4e1e854696e586fe279d 0.6.41 -f30167716b659f96c5e0b7ea3d5be2bcff8c0eac 0.6.42 -8951daac6c1bc7b24c7fb054fd369f2c5b88cdb3 0.7b2 -35086ee286732b0f63d2be18d9f26f2734586e2d 0.6.43 -63e4eb2d61204f77f9b557201a0efa187b05a611 0.7b3 -73aa98aee6bbc4a9d19a334a8ac928dece7799c6 0.6.44 -53b4ac9a748aa28893aaca42c41e5e99568667bb 0.7b4 -ddca71ae5ceb9b14512dc60ea83802c10e224cf0 0.6.45 -7f2c08e9ca22023d1499c512fccc1513813b7dc4 0.7 -024dd30ed702135f5328975042566e48cc479d7d 0.7.1 -d04c05f035e3a5636006fc34f4be7e6c77035d17 0.7.2 -d212e48e0cef689acba57ed017289c027660b23c 0.7.3 -74c6c12268059986f9cc0b535399594f1d131201 0.8b1 -85640475dda0621f20e11db0995fa07f51744a98 0.7.4 -b57e5ba934767dd498669b17551678081b3047b5 0.6.46 -dd5bbc116c53d3732d22f983e7ca6d8cfabd3b08 0.7.5 -512744f3f306aea0fdde4cfd600af8b2d6e773e7 0.8b2 -8af9839a76407eebf3610fcd3e7973f1625abaa2 0.8b3 -ee2c967017024197b38e39ced852808265387a4b 0.6.47 -48d3d26cbea68e21c96e51f01092e8fdead5cd60 0.7.6 -5b3c7981a02b4a86af1b10ae16492899b515d485 0.8b4 -cae9127e0534fc46d7ddbc11f68dc88fd9311459 0.6.48 -1506fa538fff01e70424530a32a44e070720cf3c 0.7.7 -5679393794978a1d3e1e087472b8a0fdf3d8423c 0.8b5 -26f59ec0f0f69714d28a891aaad048e3b9fcd6f7 0.8b6 -f657df1f1ed46596d236376649c99a470662b4ba 0.6.49 -236de1de68b14230036147c7c9e7c09b215b53ee 0.7.8 -979d598822bc64b05fb177a2ba221e75ee5b44d3 0.8b7 -e3d70539e79f39a97f69674ab038661961a1eb43 0.8 -3078b1e566399bf0c5590f3528df03d0c23a0777 0.9 -9e5a8f734662dd36e6fd6e4ba9031d0e2d294632 0.9.1 -37444bb32e172aaacbc0aeafdf5a778ee471723d 0.9.2 -3e9d2e89de3aa499382d6be2ec8b64d2a29f7f13 0.9.3 -1aef141fc968113e4c521d1edf6ea863c4ff7e00 0.9.4 -88e3d6788facbb2dd6467a23c4f35529a5ce20a1 0.9.5 -acc6c5d61d0f82040c237ac7ea010c0fc9e67d66 0.9.6 -19965a03c1d5231c894e0fabfaf45af1fd99f484 0.9.7 -e0a6e225ad6b28471cd42cfede6e8a334bb548fb 0.9.8 -7b91ff93a30ef78634b7bb34f4a6229a5de281ee 1.0b1 -aba16323ec9382da7bc77c633990ccb3bd58d050 1.0b2 -8a98492f0d852402c93ddbbf3f07081909a9105f 1.0b3 -c385fdf1f976fb1d2a6accc9292d8eca419180fa 1.0 -d943b67fe80dbd61326014e4acedfc488adfa1c9 1.1 -2e42e86546100c9f6845b04eb31b75c5add05f78 1.1.1 -462fe5ccd8befeb2a235e8295d6d73eb3a49cc78 1.1.2 -ddf3561d6a54087745f4bf6ea2048b86195d6fe2 1.1.3 -f94c7e4fa03077e069c1c3cef93ead735559e706 1.1.4 -d9bb58331007ee3f69d31983a180f56b15c731c3 1.1.5 -5e426bdeb46b87e299422adc419f4163b6c78d13 1.1.6 -cc9b19cd0ec64e44308a852e9b9fdc6026ea2e46 1.1.7 -4c7dc4ae2440ae3e9ba26b4a12ffca3407e7030d 1.2b1 -77921bbe3931caf40474dc36e55d3d541981c749 1.2 -19873119647deae8a68e9ed683317b9ee170a8d8 1.3 -a197b626075a8c2e393a08c42a20bd2624a41092 1.3.1 -076b472a9e3f840021e9d5509878337e6e5fcd89 1.3.2 -0d1bdb99a535a2c7ed4edd37141fb0b54348b713 1.4b1 -a13f8c18ce742bc83c794b9eea57980cb94ae18a 1.4 -9a5f26d7df8ef779cb5f40cc0389343fb4c61365 1.4.1 -274cb3beba4f22d5f461b0578b6d56e171d94f2e 1.4.2 -0bb1df93c2eaa50e95ccfce18208b0cca20ebae3 2.0 -bbdba51e1bc1779728ed351529252f73543ace65 2.0.1 -5a62ac60ba31d249db1cfcff31d85ca26421be6d 2.0.2 -c49c651997ebec3b40b71139e8a6a6a15c62c848 2.1 -b5be6c2b828cb92d27f52fccc725ce86a37e9ce0 2.1.1 -ab1c2a26e06f2a2006e8e867e4d41ccf1d6cf9b2 2.2b1 -caab085e829f29679d0e47430b2761af6b20fc76 2.1.2 -39f7ef5ef22183f3eba9e05a46068e1d9fd877b0 2.2 -faba785e9b9e05ba890d0851ef1f3287c32fcac2 3.0b1 -8e8c50925f18eafb7e66fe020aa91a85b9a4b122 3.0 -cd9e857476ac70515f7436f846b593f696ac672d 3.0.1 -bad1f30ee0dfa7a2af4f428d06f62efa39ca48db 3.0.2 -47224d55ddc6bb08c1d17a219f124d0d9c524491 3.1 -07c459bea1c58ff52e0576fc29c1865d18a83b09 3.2 -b306e681a945406833fb297ae10241e2241fc22b 3.3 -78c8cfbe3e1017d1653c48f7306b2c4b4911bf1a 4.0b1 -5cb90066d98700e6d37a01d95c4a2090e730ae02 3.4 -e39de2d3eb774b70c023a1151758213cc9ed2178 3.4.1 -369f6f90f69683702cc0b72827ccf949977808b0 3.4.2 -06a56e063c327b0606f9e9690764279d424646b2 3.4.3 -0917d575d26091a184796624743825914994bf95 3.4.4 -98f29d521c3a57bae0090d2bc5597d93db95b108 3.5 -254d8c625f4620993ce2d2b21212ba01cf307fe6 3.5.1 -572201d08eadc59210f6f0f28f9dc79f906672d3 3.5.2 -e94e768594a1405efde0b79cc60549dd8a4cda9a 3.6 -292dfca15d33e72a862d044183a6ad7c06862a19 3.7b1 -49bd27eebf212c067392796bb2d0fa6d8e583586 3.7 -2fa97c06cc013a9c82f4c1219711e72238d5b6e6 3.8 -9b422fc0b8b97cdb62f02d754283f747adef7f83 3.7.1 -40744de29b848f0e88139ba91d645c08a56855e9 3.8.1 -84d936fd18a93d16c46e68ee2e39f5733f3cd863 5.0 -871bd7b4326f48860ebe0baccdaea8fe4f8f8583 5.0.1 -95996b713722376679c3168b15ab12ea8360dd5f 5.0.2 -3a948b6d01e3449b478fcdc532c44eb3cea5ee10 5.1 -f493e6c4ffd88951871110858c141385305e0077 5.2 -1f9505cfd7524ce0c83ab31d139f47b39c56ccbe 5.3 -baae103e80c307008b156e426a07eb9f486eb4f0 5.4 -ba3b08c7bffd6123e1a7d58994f15e8051a67cb7 5.4.1 -7adcf1397f6eccb9e73eda294343de2943f7c8fb 5.4.2 -68910a89f97a508a64f9f235dc64ad43d4477ea0 5.5 -949a66af4f03521e1404deda940aa951418a13d2 5.5.1 -a1fc0220bfa3581158688789f6dfdc00672eb99b 5.6 -37ed55fd310d0cd32009dc5676121e86b404a23d 5.7 -67550a8ed9f4ef49ee5a31f433adbf5a0eaeccf9 5.8 -755cbfd3743ffb186cdf7e20be8e61dbdaa22503 6.0 -bc6655b4acf205dd9f25c702955645656077398a 6.0.1 -1ae2a75724bbba56373784f185a7f235ed0f24a4 6.0.2b1 -01271e84e5125fcc4f0f368a6e21116a5722953c 6.0.2 -7ea80190d494a766c6356fce85c844703964b6cc 6.1 -df26609c2f614f5fc9110342e4003ee8bd95cf84 7.0 -850a5c155c48b6ecfbb83b961586ea359b561522 8.0b1 -7ea0e7498e4ddbf63b6929ee83c75a9207996b08 8.0 -1af3a5f24f7dd4e51d117f701918052b7de65c99 8.1b1 -d62bf4e407b3b9b5bedcc1396a9ba46f35571902 8.0.1 -1c03d512e39d5cfd711ae3ed7e316769f427e43b 8.0.2 -6c3467488123ce70b1dd009145a02f51fb78cdcc 8.0.3 -2c467afffe9fe1e14618b576fac6b4f7c412a61e 8.0.4 -3f87370b6863e5a4e831b394ef1a58e0e97a4336 8.1 -995f6d9651312cd481ca1e5ddb271cbdd0474c57 8.2 -efbe39dae0aba9a7db399f6442758ae94e315c93 8.2.1 -cd14b2a72e51c7d13873ab6c2041f901b1a7a1cd 8.3 -0eee586a153f068142c1a0df4bc2635ed2c1a1cc 9.0b1 -921e60a0f9067311571fde9ccf2f35223159d9f6 8.4 -0d7b9b63d06ab7f68bc8edd56cb2034e6395d7fc 9.0 -fa069bf2411a150c9379d31a04d1c3836e2d3027 9.0.1 -3ed27d68d3f41bb5daa2afecfa9180d5958fe9d3 9.1 -0c4d18a747a6d39bff8e194a58af949a960d674a 10.0 -4c41e2cdd70beb0da556d71f46a67734c14f2bc2 10.0.1 -26b00011ec65b8f7b4f3d51078ec0a694701a45c 10.1 -651d41db58849d4fc50e466f4dc458d448480c4e 10.2 -1f5de53c079d577ead9d80265c9e006503b16457 10.2.1 -b4b92805bc0e9802da0b597d00df4fa42b30bc40 11.0 -6cd2b18f4be2a9c188fa505b34505b32f4a4554b 11.1 -feb5971e7827483bbdeb67613126bb79ed09e6d9 11.2 -a1a6a1ac9113b90009052ca7263174a488434099 11.3 -1116e568f534ad8f4f41328a0f5fa183eb739c90 11.3.1 -55666947c9eb7e3ba78081ad6ae004807c84aede 12.0 -747018b2e35a40cb4b1c444f150f013d02197c64 12.0.1 -a177ea34bf81662b904fe3af46f3c8719a947ef1 12.0.2 -bf8c5bcacd49bf0f9648013a40ebfc8f7c727f7b 12.0.3 -73dcfc90e3eecec6baddea19302c6b342e68e2fa 12.0.4 -01fbfc9194a2bc502edd682eebbf4d2f1bc79eee 12.0.5 -7bca8938434839dbb546b8bfccd9aab7a86d851e 12.1 -5ff5c804a8fa580cff499ba0025ff2e6a5474fd0 12.2 -8d50aac3b20793954121edb300b477cc75f3ec96 12.3 -297931cb8cac7d44d970adb927efd6cb36ac3526 12.4 -df34cc18624279faffdbc729c0a11e6ab0f46572 13.0 -ae1a5c5cf78f4f9f98c054f1c8cec6168d1d19b4 13.0.1 -e22a1d613bddf311e125eecd9c1e1cad02ab5063 13.0.2 -a3a105f795f8362f26e84e9acbc237ee2d6bcca4 14.0 -9751a1671a124e30ae344d1510b9c1dbb14f2775 14.1 -07fcc3226782b979cedaaf456c7f1c5b2fdafd2c 14.1.1 -d714fb731de779a1337d2d78cd413931f1f06193 14.2 -e3c635a7d463c7713c647d1aa560f83fd8e27ef0 14.3 -608948cef7e0ab8951691b149f5b6f0184a5635e 14.3.1 -617699fd3e44e54b6f95b80bfcf78164df37f266 15.0b1 -d2c4d84867154243993876d6248aafec1fd12679 15.0 -10fde952613b7a3f650fb1f6b6ed58cbd232fa3c 15.1 -df5dc9c7aa7521f552824dee1ed1315cfe180844 15.2 -e0825f0c7d5963c498266fe3c175220c695ae83b 16.0 -8e56240961015347fed477f00ca6a0783e81d3a2 17.0 -a37bcaaeab367f2364ed8c070659d52a4c0ae38e 17.1 -4a0d01d690ff184904293e7a3244ac24ec060a73 17.1.1 -fac98a49bd984ef5accf7177674d693277bfbaef 18.0b1 -0a49ee524b0a1d67d2a11c8c22f082b57acd7ae1 18.0 -e364795c1b09c70b6abb53770e09763b52bf807d 18.0.1 -c0395f556c35d8311fdfe2bda6846b91149819cd 18.1 -1a981f2e5031f55267dc2a28fa1b42274a1b64b2 18.2 -b59320212c8371d0be9e5e6c5f7eec392124c009 18.3 -7a705b610abb1177ca169311c4ee261f3e4f0957 18.3.1 -1e120f04bcaa2421c4df0eb6678c3019ba4a82f6 18.3.2 -6203335278be7543d31790d9fba55739469a4c6c 18.4 -31dc6d2ac0f5ab766652602fe6ca716fff7180e7 18.5 -dfe190b09908f6b953209d13573063809de451b8 18.6 -804f87045a901f1dc121cf9149143d654228dc13 18.6.1 -67d07805606aead09349d5b91d7d26c68ddad2fc 18.7 -3041e1fc409be90e885968b90faba405420fc161 18.7.1 -c811801ffa1de758cf01fbf6a86e4c04ff0c0935 18.8 -fbf06fa35f93a43f044b1645a7e4ff470edb462c 18.8.1 -cc41477ecf92f221c113736fac2830bf8079d40c 19.0 -834782ce49154e9744e499e00eb392c347f9e034 19.1 -0a2a3d89416e1642cf6f41d22dbc07b3d3c15a4d 19.1.1 -5d24cf9d1ced76c406ab3c4a94c25d1fe79b94bc 19.2 -66fa131a0d77a1b0e6f89ccb76b254cfb07d3da3 19.3b1 -32bba9bf8cce8350b560a7591c9ef5884a194211 19.3 -f47f3671508b015e9bb735603d3a0a6ec6a77b01 19.4 -0bda3291ac725750b899b4ba3e4b6765e7645daa 19.4.1 -0a68cbab72580a6f8d3bf9c45206669eefcd256b 19.5 -34121bf49b1a7ac77da7f7c75105c8a920218dd7 19.6b1 -3c2332e4ec72717bf17321473e5c3ad6e5778903 19.6 -35d9179d04390aada66eceae9ceb7b9274f67646 19.6.1 -d2782cbb2f15ca6831ab9426fbf8d4d6ca60db8a 19.6.2 -c6e619ce910d1650cc2433f94e5594964085f973 19.7 -2a60daeff0cdb039b20b2058aaad7dae7bcd2c1c 20.0 -06c9d3ffae80d7f5786c0a454d040d253d47fc03 20.1 -919a40f1843131249f98104c73f3aee3fc835e67 20.1.1 -74c4ffbe1f399345eb4f6a64785cfff54f7e6e7e 20.2 -1aacb05fbdfe06cee904e7a138a4aa6df7b88a63 20.2.1 -48aa5271ef1cd5379cf91a1c958e490692b978e7 20.2.2 -9c55a3a1268a33b4a57b96b2b9fa2cd0701780ee 20.3 -3e87e975a95c780eec497ef9e5a742f7adfb77ec 20.3.1 -06692c64fb9b5843331a918ab7093f151412ec8e 20.4 -f8174392e9e9c6a21ea5df0f22cb4ca885c799ca 20.5 -114f3dbc8a73dacbce2ebe08bb70ca76ab18390e v20.6.0 -a3d4006688fe5e754d0e709a52a00b8191819979 v20.6.1 -2831509712601a78fddf46e51d6f41ae0f92bd0e v20.6.2 -8b46dc41cb234c435b950a879214a6dee54c9dd2 v20.6.3 -7258be20fe93bbf936dc1a81ce71c04c5880663e v20.6.4 -7e0ab283db4e6f780777f7f06af475f044631fa1 v20.6.5 -57d63b38e85515d06e06d3cea62e35e6c54b5093 v20.6.6 -57d63b38e85515d06e06d3cea62e35e6c54b5093 v20.6.6 -b04dbdd161d7f68903a53e1dbd1fa5b5fde73f94 v20.6.6 -0804d30b6ead64e0e324aefd67439b84df2d1c01 v20.6.7 -a00910db03ec15865e4c8506820d4ad1df3e26f3 v20.6.8 -0262ab29fc2417b502a55f49b7fd43528fbd3df4 v20.7.0 -7f56b6f40de39456c78507a14c288709712881cb v20.8.0 -8cf9340669ae26e2b31f68b9c3f885ab7bdd65ce v20.8.1 -8bf8aaa139bb6a36fcd243214d6730a214ae08f5 v20.9.0 -c72faa468919fd2f226c97e94d4e64a6506860e5 v20.10.0 -3b5fdd077c7d83d02c4979ad69cc0bf199b47587 v20.10.1 -ddd3f81eb9e0860bf95c380c50a72c52a215231f v21.0.0 -018e4a727cf691d6404cd24ffb25e8eebea2fad4 v20.6.8 -02643fe9503033edd2fc5a54c8d4361a6c185be4 v21.1.0 -40b8fac6db119aca9c462993d01908492769fc4f v21.2.0 -40b8fac6db119aca9c462993d01908492769fc4f v21.2.0 -9959424676a4aac1c14e430ff6f4210fdb0442d9 v21.2.0 -694111eadb10fe6003078895a2cbb803ce514ef2 v21.2.1 -274f33435e9c3ba5019f2a2bfe478fa2db0da41d v21.2.2 -451fbedb4c226d8ea5b6eab1e21679c9a4ec4a93 v22.0.0 -f5c4923b0400d61f67699c2d54388878f9e0c8bd v22.0.1 -8610a8b9635f15d33f94fccb295fd34aa6fbddee v22.0.2 -efee7d74a8478c0d08c801fb520e41b6e04d0dda v22.0.3 -77b20c09b04775cc936ab5d16cbc46ff05fc7080 v22.0.4 -d5832e5deb77027da474e79e5f047e9a81f7edf8 v22.0.5 diff --git a/.travis.yml b/.travis.yml index e91f7e78..b8bca7cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ env: - LC_ALL=C LC_CTYPE=C script: # avoid VersionConflict when newer version is required - - pip install -U pytest + - pip install -U 'pytest>=3.0.2' # Output the env, because the travis docs just can't be trusted - env @@ -20,14 +20,14 @@ script: # update egg_info based on setup.py in checkout - python bootstrap.py - - python setup.py test --addopts='-rs' + - python setup.py test --addopts='-rsx' before_deploy: - export SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES=1 deploy: provider: pypi # Also update server in setup.cfg - server: https://upload.pypi.io/legacy/ + server: https://upload.pypi.org/legacy/ on: tags: true all_branches: true diff --git a/CHANGES.rst b/CHANGES.rst index b763607c..7c4ff813 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,336 @@ CHANGES ======= +v28.0.0 +------- + +* #733: Do not search excluded directories for packages. + This introduced a backwards incompatible change in ``find_packages()`` + so that ``find_packages(exclude=['foo']) == []``, excluding subpackages of ``foo``. + Previously, ``find_packages(exclude=['foo']) == ['foo.bar']``, + even though the parent ``foo`` package was excluded. + +* #795: Bump certifi. + +* #719: Suppress decoding errors and instead log a warning + when metadata cannot be decoded. + +v27.3.1 +------- + +* #790: In MSVC monkeypatching, explicitly patch each + function by name in the target module instead of inferring + the module from the function's ``__module__``. Improves + compatibility with other packages that might have previously + patched distutils functions (i.e. NumPy). + +v27.3.0 +------- + +* #794: In test command, add installed eggs to PYTHONPATH + when invoking tests so that subprocesses will also have the + dependencies available. Fixes `tox 330 + <https://github.com/tox-dev/tox/issues/330>`_. + +* #795: Update vendored pyparsing 2.1.9. + +v27.2.0 +------- + +* #520 and #513: Suppress ValueErrors in fixup_namespace_packages + when lookup fails. + +* Nicer, more consistent interfaces for msvc monkeypatching. + +v27.1.2 +------- + +* #779 via #781: Fix circular import. + +v27.1.1 +------- + +* #778: Fix MSVC monkeypatching. + +v27.1.0 +------- + +* Introduce the (private) ``monkey`` module to encapsulate + the distutils monkeypatching behavior. + +v27.0.0 +------- + +* Now use Warehouse by default for + ``upload``, patching ``distutils.config.PyPIRCCommand`` to + affect default behavior. + + Any config in .pypirc should be updated to replace + + https://pypi.python.org/pypi/ + + with + + https://upload.pypi.org/legacy/ + + Similarly, any passwords stored in the keyring should be + updated to use this new value for "system". + + The ``upload_docs`` command will continue to use the python.org + site, but the command is now deprecated. Users are urged to use + Read The Docs instead. + +* #776: Use EXT_SUFFIX for py_limited_api renaming. + +* #774 and #775: Use LegacyVersion from packaging when + detecting numpy versions. + +v26.1.1 +------- + +* Re-release of 26.1.0 with pytest pinned to allow for automated + deployement and thus proper packaging environment variables, + fixing issues with missing executable launchers. + +v26.1.0 +------- + +* #763: ``pkg_resources.get_default_cache`` now defers to the + `appdirs project <https://pypi.org/project/appdirs>`_ to + resolve the cache directory. Adds a vendored dependency on + appdirs to pkg_resources. + +v26.0.0 +------- + +* #748: By default, sdists are now produced in gzipped tarfile + format by default on all platforms, adding forward compatibility + for the same behavior in Python 3.6 (See Python #27819). + +* #459 via #736: On Windows with script launchers, + sys.argv[0] now reflects + the name of the entry point, consistent with the behavior in + distlib and pip wrappers. + +* #752 via #753: When indicating ``py_limited_api`` to Extension, + it must be passed as a keyword argument. + +v25.4.0 +------- + +* Add Extension(py_limited_api=True). When set to a truthy value, + that extension gets a filename apropriate for code using Py_LIMITED_API. + When used correctly this allows a single compiled extension to work on + all future versions of CPython 3. + The py_limited_api argument only controls the filename. To be + compatible with multiple versions of Python 3, the C extension + will also need to set -DPy_LIMITED_API=... and be modified to use + only the functions in the limited API. + +v25.3.0 +------- + +* #739 Fix unquoted libpaths by fixing compatibility between `numpy.distutils` and `distutils._msvccompiler` for numpy < 1.11.2 (Fix issue #728, error also fixed in Numpy). + +* #731: Bump certifi. + +* Style updates. See #740, #741, #743, #744, #742, #747. + +* #735: include license file. + +v25.2.0 +------- + +* #612 via #730: Add a LICENSE file which needs to be provided by the terms of + the MIT license. + +v25.1.6 +------- + +* #725: revert `library_dir_option` patch (Error is related to `numpy.distutils` and make errors on non Numpy users). + +v25.1.5 +------- + +* #720 +* #723: Improve patch for `library_dir_option`. + +v25.1.4 +------- + +* #717 +* #713 +* #707: Fix Python 2 compatibility for MSVC by catching errors properly. +* #715: Fix unquoted libpaths by patching `library_dir_option`. + +v25.1.3 +------- + +* #714 and #704: Revert fix as it breaks other components + downstream that can't handle unicode. See #709, #710, + and #712. + +v25.1.2 +------- + +* #704: Fix errors when installing a zip sdist that contained + files named with non-ascii characters on Windows would + crash the install when it attempted to clean up the build. +* #646: MSVC compatibility - catch errors properly in + RegistryInfo.lookup. +* #702: Prevent UnboundLocalError when initial working_set + is empty. + +v25.1.1 +------- + +* #686: Fix issue in sys.path ordering by pkg_resources when + rewrite technique is "raw". +* #699: Fix typo in msvc support. + +v25.1.0 +------- + +* #609: Setuptools will now try to download a distribution from + the next possible download location if the first download fails. + This means you can now specify multiple links as ``dependency_links`` + and all links will be tried until a working download link is encountered. + +v25.0.2 +------- + +* #688: Fix AttributeError in setup.py when invoked not from + the current directory. + +v25.0.1 +------- + +* Cleanup of setup.py script. + +* Fixed documentation builders by allowing setup.py + to be imported without having bootstrapped the + metadata. + +* More style cleanup. See #677, #678, #679, #681, #685. + +v25.0.0 +------- + +* #674: Default ``sys.path`` manipulation by easy-install.pth + is now "raw", meaning that when writing easy-install.pth + during any install operation, the ``sys.path`` will not be + rewritten and will no longer give preference to easy_installed + packages. + + To retain the old behavior when using any easy_install + operation (including ``setup.py install`` when setuptools is + present), set the environment variable: + + SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite + + This project hopes that that few if any environments find it + necessary to retain the old behavior, and intends to drop + support for it altogether in a future release. Please report + any relevant concerns in the ticket for this change. + +v24.3.1 +------- + +* #398: Fix shebang handling on Windows in script + headers where spaces in ``sys.executable`` would + produce an improperly-formatted shebang header, + introduced in 12.0 with the fix for #188. + +* #663, #670: More style updates. + +v24.3.0 +------- + +* #516: Disable ``os.link`` to avoid hard linking + in ``sdist.make_distribution``, avoiding errors on + systems that support hard links but not on the + file system in which the build is occurring. + +v24.2.1 +------- + +* #667: Update Metadata-Version to 1.2 when + ``python_requires`` is supplied. + +v24.2.0 +------- + +* #631: Add support for ``python_requires`` keyword. + +v24.1.1 +------- + +* More style updates. See #660, #661, #641. + +v24.1.0 +------- + +* #659: ``setup.py`` now will fail fast and with a helpful + error message when the necessary metadata is missing. +* More style updates. See #656, #635, #640, + #644, #650, #652, and #655. + +v24.0.3 +------- + +* Updated style in much of the codebase to match + community expectations. See #632, #633, #634, + #637, #639, #638, #642, #648. + +v24.0.2 +------- + +* If MSVC++14 is needed ``setuptools.msvc`` now redirect + user to Visual C++ Build Tools web page. + +v24.0.1 +------- + +* #625 and #626: Fixes on ``setuptools.msvc`` mainly + for Python 2 and Linux. + +v24.0.0 +------- + +* Pull Request #174: Add more aggressive support for + standalone Microsoft Visual C++ compilers in + msvc9compiler patch. + Particularly : Windows SDK 6.1 and 7.0 + (MSVC++ 9.0), Windows SDK 7.1 (MSVC++ 10.0), + Visual C++ Build Tools 2015 (MSVC++14) +* Renamed ``setuptools.msvc9_support`` to + ``setuptools.msvc``. + +v23.2.1 +------- + +Re-release of v23.2.0, which was missing the intended +commits. + +* #623: Remove used of deprecated 'U' flag when reading + manifests. + +v23.1.0 +------- + +* #619: Deprecated ``tag_svn_revision`` distribution + option. + +v23.0.0 +------- + +* #611: Removed ARM executables for CLI and GUI script + launchers on Windows. If this was a feature you cared + about, please comment in the ticket. +* #604: Removed docs building support. The project + now relies on documentation hosted at + https://setuptools.readthedocs.io/. + v22.0.5 ------- @@ -113,7 +443,7 @@ v20.8.0 v20.7.0 ------- -* Refactored extra enviroment marker processing +* Refactored extra environment marker processing in WorkingSet. * Issue #533: Fixed intermittent test failures. * Issue #536: In msvc9_support, trap additional exceptions @@ -200,7 +530,7 @@ v20.6.0 20.2.1 ------ -* Issue #499: Restore compatiblity for legacy versions +* Issue #499: Restore compatibility for legacy versions by bumping to packaging 16.4. 20.2 @@ -432,7 +762,8 @@ v20.6.0 18.3 ---- -* Setuptools now allows disabling of the manipulation of the sys.path +* BB Pull Request #135: Setuptools now allows disabling of + the manipulation of the sys.path during the processing of the easy-install.pth file. To do so, set the environment variable ``SETUPTOOLS_SYS_PATH_TECHNIQUE`` to anything but "rewrite" (consider "raw"). During any install operation @@ -1132,12 +1463,12 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Issue #125: Prevent Subversion support from creating a ~/.subversion directory just for checking the presence of a Subversion repository. -* Issue #12: Namespace packages are now imported lazily. That is, the mere +* Issue #12: Namespace packages are now imported lazily. That is, the mere declaration of a namespace package in an egg on ``sys.path`` no longer - causes it to be imported when ``pkg_resources`` is imported. Note that this + causes it to be imported when ``pkg_resources`` is imported. Note that this change means that all of a namespace package's ``__init__.py`` files must include a ``declare_namespace()`` call in order to ensure that they will be - handled properly at runtime. In 2.x it was possible to get away without + handled properly at runtime. In 2.x it was possible to get away without including the declaration, but only at the cost of forcing namespace packages to be imported early, which 3.0 no longer does. * Issue #148: When building (bdist_egg), setuptools no longer adds @@ -1918,7 +2249,7 @@ how it parses version numbers. * Distribute #67: Fixed doc typo (PEP 381/PEP 382). * Distribute no longer shadows setuptools if we require a 0.7-series - setuptools. And an error is raised when installing a 0.7 setuptools with + setuptools. And an error is raised when installing a 0.7 setuptools with distribute. * When run from within buildout, no attempt is made to modify an existing @@ -2218,7 +2549,7 @@ easy_install * Fix ``bdist_egg`` not including files in subdirectories of ``.egg-info``. * Allow ``.py`` files found by the ``include_package_data`` option to be - automatically included. Remove duplicate data file matches if both + automatically included. Remove duplicate data file matches if both ``include_package_data`` and ``package_data`` are used to refer to the same files. @@ -2259,7 +2590,7 @@ easy_install directory doesn't need to support .pth files. * ``MANIFEST.in`` is now forcibly closed when any errors occur while reading - it. Previously, the file could be left open and the actual error would be + it. Previously, the file could be left open and the actual error would be masked by problems trying to remove the open file on Windows systems. 0.6a10 @@ -2272,7 +2603,7 @@ easy_install * The ``sdist`` command no longer uses the traditional ``MANIFEST`` file to create source distributions. ``MANIFEST.in`` is still read and processed, - as are the standard defaults and pruning. But the manifest is built inside + as are the standard defaults and pruning. But the manifest is built inside the project's ``.egg-info`` directory as ``SOURCES.txt``, and it is rebuilt every time the ``egg_info`` command is run. @@ -2308,11 +2639,11 @@ easy_install command so that you can more easily wrap a "flat" egg in a system package. * Enhanced ``bdist_rpm`` so that it installs single-version eggs that - don't rely on a ``.pth`` file. The ``--no-egg`` option has been removed, + don't rely on a ``.pth`` file. The ``--no-egg`` option has been removed, since all RPMs are now built in a more backwards-compatible format. * Support full roundtrip translation of eggs to and from ``bdist_wininst`` - format. Running ``bdist_wininst`` on a setuptools-based package wraps the + format. Running ``bdist_wininst`` on a setuptools-based package wraps the egg in an .exe that will safely install it as an egg (i.e., with metadata and entry-point wrapper scripts), and ``easy_install`` can turn the .exe back into an ``.egg`` file or directory and install it as such. @@ -2336,19 +2667,19 @@ easy_install * Fixed some problems with fresh checkouts of projects that don't include ``.egg-info/PKG-INFO`` under revision control and put the project's source - code directly in the project directory. If such a package had any + code directly in the project directory. If such a package had any requirements that get processed before the ``egg_info`` command can be run, the setup scripts would fail with a "Missing 'Version:' header and/or PKG-INFO file" error, because the egg runtime interpreted the unbuilt metadata in a directory on ``sys.path`` (i.e. the current directory) as - being a corrupted egg. Setuptools now monkeypatches the distribution + being a corrupted egg. Setuptools now monkeypatches the distribution metadata cache to pretend that the egg has valid version information, until it has a chance to make it actually be so (via the ``egg_info`` command). 0.6a5 ----- - * Fixed missing gui/cli .exe files in distribution. Fixed bugs in tests. + * Fixed missing gui/cli .exe files in distribution. Fixed bugs in tests. 0.6a3 ----- @@ -2361,8 +2692,8 @@ easy_install ----- * Added ``console_scripts`` entry point group to allow installing scripts - without the need to create separate script files. On Windows, console - scripts get an ``.exe`` wrapper so you can just type their name. On other + without the need to create separate script files. On Windows, console + scripts get an ``.exe`` wrapper so you can just type their name. On other platforms, the scripts are written without a file extension. 0.6a1 @@ -2372,7 +2703,7 @@ easy_install the target package, using a ``--no-egg`` option. * The ``build_ext`` command now works better when using the ``--inplace`` - option and multiple Python versions. It now makes sure that all extensions + option and multiple Python versions. It now makes sure that all extensions match the current Python version, even if newer copies were built for a different Python version. @@ -2402,11 +2733,11 @@ easy_install * ``setuptools`` now finds its commands, ``setup()`` argument validators, and metadata writers using entry points, so that they can be extended by - third-party packages. See `Creating distutils Extensions + third-party packages. See `Creating distutils Extensions <http://pythonhosted.org/setuptools/setuptools.html#creating-distutils-extensions>`_ for more details. - * The vestigial ``depends`` command has been removed. It was never finished + * The vestigial ``depends`` command has been removed. It was never finished or documented, and never would have worked without EasyInstall - which it pre-dated and was never compatible with. @@ -2440,9 +2771,9 @@ easy_install * ``setup.py install`` now automatically detects when an "unmanaged" package or module is going to be on ``sys.path`` ahead of a package being installed, - thereby preventing the newer version from being imported. If this occurs, + thereby preventing the newer version from being imported. If this occurs, a warning message is output to ``sys.stderr``, but installation proceeds - anyway. The warning message informs the user what files or directories + anyway. The warning message informs the user what files or directories need deleting, and advises them they can also use EasyInstall (with the ``--delete-conflicting`` option) to do it automatically. @@ -2463,14 +2794,14 @@ easy_install * The "egg_info" command now always sets the distribution metadata to "safe" forms of the distribution name and version, so that distribution files will be generated with parseable names (i.e., ones that don't include '-' in the - name or version). Also, this means that if you use the various ``--tag`` + name or version). Also, this means that if you use the various ``--tag`` options of "egg_info", any distributions generated will use the tags in the version, not just egg distributions. * Added support for defining command aliases in distutils configuration files, - under the "[aliases]" section. To prevent recursion and to allow aliases to + under the "[aliases]" section. To prevent recursion and to allow aliases to call the command of the same name, a given alias can be expanded only once - per command-line invocation. You can define new aliases with the "alias" + per command-line invocation. You can define new aliases with the "alias" command, either for the local, global, or per-user configuration. * Added "rotate" command to delete old distribution files, given a set of @@ -2478,7 +2809,7 @@ easy_install recently-modified distribution files matching each pattern.) * Added "saveopts" command that saves all command-line options for the current - invocation to the local, global, or per-user configuration file. Useful for + invocation to the local, global, or per-user configuration file. Useful for setting defaults without having to hand-edit a configuration file. * Added a "setopt" command that sets a single option in a specified distutils @@ -2497,7 +2828,7 @@ easy_install * Beefed up the "sdist" command so that if you don't have a MANIFEST.in, it will include all files under revision control (CVS or Subversion) in the current directory, and it will regenerate the list every time you create a - source distribution, not just when you tell it to. This should make the + source distribution, not just when you tell it to. This should make the default "do what you mean" more often than the distutils' default behavior did, while still retaining the old behavior in the presence of MANIFEST.in. @@ -2512,14 +2843,14 @@ easy_install 0.5a5 ----- - * Added ``develop`` command to ``setuptools``-based packages. This command + * Added ``develop`` command to ``setuptools``-based packages. This command installs an ``.egg-link`` pointing to the package's source directory, and script wrappers that ``execfile()`` the source versions of the package's - scripts. This lets you put your development checkout(s) on sys.path without + scripts. This lets you put your development checkout(s) on sys.path without having to actually install them. (To uninstall the link, use use ``setup.py develop --uninstall``.) - * Added ``egg_info`` command to ``setuptools``-based packages. This command + * Added ``egg_info`` command to ``setuptools``-based packages. This command just creates or updates the "projectname.egg-info" directory, without building an egg. (It's used by the ``bdist_egg``, ``test``, and ``develop`` commands.) @@ -2527,11 +2858,11 @@ easy_install * Enhanced the ``test`` command so that it doesn't install the package, but instead builds any C extensions in-place, updates the ``.egg-info`` metadata, adds the source directory to ``sys.path``, and runs the tests - directly on the source. This avoids an "unmanaged" installation of the + directly on the source. This avoids an "unmanaged" installation of the package to ``site-packages`` or elsewhere. * Made ``easy_install`` a standard ``setuptools`` command, moving it from - the ``easy_install`` module to ``setuptools.command.easy_install``. Note + the ``easy_install`` module to ``setuptools.command.easy_install``. Note that if you were importing or extending it, you must now change your imports accordingly. ``easy_install.py`` is still installed as a script, but not as a module. @@ -2542,10 +2873,10 @@ easy_install * Setup scripts using setuptools can now list their dependencies directly in the setup.py file, without having to manually create a ``depends.txt`` file. The ``install_requires`` and ``extras_require`` arguments to ``setup()`` - are used to create a dependencies file automatically. If you are manually + are used to create a dependencies file automatically. If you are manually creating ``depends.txt`` right now, please switch to using these setup arguments as soon as practical, because ``depends.txt`` support will be - removed in the 0.6 release cycle. For documentation on the new arguments, + removed in the 0.6 release cycle. For documentation on the new arguments, see the ``setuptools.dist.Distribution`` class. * Setup scripts using setuptools now always install using ``easy_install`` @@ -2554,7 +2885,7 @@ easy_install 0.5a1 ----- - * Added support for "self-installation" bootstrapping. Packages can now + * Added support for "self-installation" bootstrapping. Packages can now include ``ez_setup.py`` in their source distribution, and add the following to their ``setup.py``, in order to automatically bootstrap installation of setuptools as part of their setup process:: @@ -2574,21 +2905,21 @@ easy_install * All downloads are now managed by the ``PackageIndex`` class (which is now subclassable and replaceable), so that embedders can more easily override - download logic, give download progress reports, etc. The class has also + download logic, give download progress reports, etc. The class has also been moved to the new ``setuptools.package_index`` module. * The ``Installer`` class no longer handles downloading, manages a temporary - directory, or tracks the ``zip_ok`` option. Downloading is now handled + directory, or tracks the ``zip_ok`` option. Downloading is now handled by ``PackageIndex``, and ``Installer`` has become an ``easy_install`` command class based on ``setuptools.Command``. * There is a new ``setuptools.sandbox.run_setup()`` API to invoke a setup script in a directory sandbox, and a new ``setuptools.archive_util`` module - with an ``unpack_archive()`` API. These were split out of EasyInstall to + with an ``unpack_archive()`` API. These were split out of EasyInstall to allow reuse by other tools and applications. * ``setuptools.Command`` now supports reinitializing commands using keyword - arguments to set/reset options. Also, ``Command`` subclasses can now set + arguments to set/reset options. Also, ``Command`` subclasses can now set their ``command_consumes_arguments`` attribute to ``True`` in order to receive an ``args`` option containing the rest of the command line. @@ -2597,7 +2928,7 @@ easy_install * Added new options to ``bdist_egg`` to allow tagging the egg's version number with a subversion revision number, the current date, or an explicit tag - value. Run ``setup.py bdist_egg --help`` to get more information. + value. Run ``setup.py bdist_egg --help`` to get more information. * Misc. bug fixes diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..6e0693b4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (C) 2016 Jason R Coombs <jaraco@jaraco.com> + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in index cfd1b001..e25a5ea5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ recursive-include pkg_resources *.py *.txt include *.py include *.rst include MANIFEST.in +include LICENSE include launcher.c include msvc-build-launcher.cmd include pytest.ini @@ -37,6 +37,10 @@ Powershell command. Start up Powershell and paste this command:: > (Invoke-WebRequest https://bootstrap.pypa.io/ez_setup.py).Content | python - +.. image:: https://badges.gitter.im/pypa/setuptools.svg + :alt: Join the chat at https://gitter.im/pypa/setuptools + :target: https://gitter.im/pypa/setuptools?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge + You must start the Powershell with Administrative privileges or you may choose to install a user-local installation:: @@ -53,7 +57,7 @@ The recommended way to install setuptools on Windows is to download distribution file and install it for you. Once installation is complete, you will find an ``easy_install`` program in -your Python ``Scripts`` subdirectory. For simple invocation and best results, +your Python ``Scripts`` subdirectory. For simple invocation and best results, add this directory to your ``PATH`` environment variable, if it is not already present. If you did a user-local install, the ``Scripts`` subdirectory is ``$env:APPDATA\Python\Scripts``. @@ -130,7 +134,7 @@ Downloads ========= All setuptools downloads can be found at `the project's home page in the Python -Package Index`_. Scroll to the very bottom of the page to find the links. +Package Index`_. Scroll to the very bottom of the page to find the links. .. _the project's home page in the Python Package Index: https://pypi.python.org/pypi/setuptools @@ -171,17 +175,17 @@ learning about Setuptools, Python Eggs, and EasyInstall: * `The Internal Structure of Python Eggs`_ Questions, comments, and bug reports should be directed to the `distutils-sig -mailing list`_. If you have written (or know of) any tutorials, documentation, +mailing list`_. If you have written (or know of) any tutorials, documentation, plug-ins, or other resources for setuptools users, please let us know about -them there, so this reference list can be updated. If you have working, +them there, so this reference list can be updated. If you have working, *tested* patches to correct problems or add features, you may submit them to the `setuptools bug tracker`_. .. _setuptools bug tracker: https://github.com/pypa/setuptools/issues -.. _The Internal Structure of Python Eggs: https://pythonhosted.org/setuptools/formats.html -.. _The setuptools Developer's Guide: https://pythonhosted.org/setuptools/setuptools.html -.. _The pkg_resources API reference: https://pythonhosted.org/setuptools/pkg_resources.html -.. _The EasyInstall user's guide and reference manual: https://pythonhosted.org/setuptools/easy_install.html +.. _The Internal Structure of Python Eggs: https://setuptools.readthedocs.io/en/latest/formats.html +.. _The setuptools Developer's Guide: https://setuptools.readthedocs.io/en/latest/developer-guide.html +.. _The pkg_resources API reference: https://setuptools.readthedocs.io/en/latest/pkg_resources.html +.. _The EasyInstall user's guide and reference manual: https://setuptools.readthedocs.io/en/latest/easy_install.html .. _distutils-sig mailing list: http://mail.python.org/pipermail/distutils-sig/ @@ -190,13 +194,13 @@ Credits ------- * The original design for the ``.egg`` format and the ``pkg_resources`` API was - co-created by Phillip Eby and Bob Ippolito. Bob also implemented the first + co-created by Phillip Eby and Bob Ippolito. Bob also implemented the first version of ``pkg_resources``, and supplied the OS X operating system version compatibility algorithm. * Ian Bicking implemented many early "creature comfort" features of easy_install, including support for downloading via Sourceforge and - Subversion repositories. Ian's comments on the Web-SIG about WSGI + Subversion repositories. Ian's comments on the Web-SIG about WSGI application deployment also inspired the concept of "entry points" in eggs, and he has given talks at PyCon and elsewhere to inform and educate the community about eggs and setuptools. @@ -211,7 +215,7 @@ Credits * Significant parts of the implementation of setuptools were funded by the Open Source Applications Foundation, to provide a plug-in infrastructure for the - Chandler PIM application. In addition, many OSAF staffers (such as Mike + Chandler PIM application. In addition, many OSAF staffers (such as Mike "Code Bear" Taylor) contributed their time and stress as guinea pigs for the use of eggs and setuptools, even before eggs were "cool". (Thanks, guys!) @@ -220,7 +224,7 @@ Credits and addressed many defects. * Since the merge with Distribute, Jason R. Coombs is the - maintainer of setuptools. The project is maintained in coordination with + maintainer of setuptools. The project is maintained in coordination with the Python Packaging Authority (PyPA) and the larger Python community. .. _files: diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..ebf446bf --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,17 @@ +environment: + + matrix: + - PYTHON: "C:\\Python35-x64" + - PYTHON: "C:\\Python27-x64" + +install: + # symlink python from a directory with a space + - "mklink /d \"C:\\Program Files\\Python\" %PYTHON%" + - "SET PYTHON=\"C:\\Program Files\\Python\"" + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + +build: off + +test_script: + - "python bootstrap.py" + - "python setup.py test --addopts='-rsx'" diff --git a/bootstrap.py b/bootstrap.py index 70f96258..bf7fb431 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -27,6 +27,7 @@ minimal_egg_info = textwrap.dedent(""" requires.txt = setuptools.command.egg_info:write_requirements """) + def ensure_egg_info(): if os.path.exists('setuptools.egg-info'): return diff --git a/docs/conf.py b/docs/conf.py index 72c1ce43..1ebc6572 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,9 +32,6 @@ templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.txt' -# The encoding of source files. -#source_encoding = 'utf-8' - # The master toctree document. master_doc = 'index' @@ -51,43 +48,13 @@ version = setuptools_scm.get_version(root='..', relative_to=__file__) # The full version, including alpha/beta/rc tags. release = version -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -#unused_docs = [] - # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = [] -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - # -- Options for HTML output --------------------------------------------------- @@ -95,11 +62,6 @@ pygments_style = 'sphinx' # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'nature' -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - # Add any paths that contain custom themes here, relative to this directory. html_theme_path = ['_theme'] @@ -110,24 +72,6 @@ html_title = "Setuptools documentation" # A shorter title for the navigation bar. Default is the same as html_title. html_short_title = "Setuptools" -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. html_use_smartypants = True @@ -135,42 +79,18 @@ html_use_smartypants = True # Custom sidebar templates, maps document names to template names. html_sidebars = {'index': 'indexsidebar.html'} -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - # If false, no module index is generated. html_use_modindex = False # If false, no index is generated. html_use_index = False -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a <link> tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' - # Output file base name for HTML help builder. htmlhelp_basename = 'Setuptoolsdoc' # -- Options for LaTeX output -------------------------------------------------- -# The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' - # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ @@ -178,23 +98,6 @@ latex_documents = [ 'The fellowship of the packaging', 'manual'), ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_use_modindex = True - link_files = { '../CHANGES.rst': dict( using=dict( diff --git a/docs/developer-guide.txt b/docs/developer-guide.txt index 7cd3c6d2..70e7f1cd 100644 --- a/docs/developer-guide.txt +++ b/docs/developer-guide.txt @@ -32,8 +32,8 @@ User support and discussions are done through the issue tracker (for specific) issues, through the distutils-sig mailing list, or on IRC (Freenode) at #pypa. -Discussions about development happen on the pypa-dev mailing list or on IRC -(Freenode) at #pypa-dev. +Discussions about development happen on the pypa-dev mailing list or on +`Gitter <https://gitter.im/pypa/setuptools>`_. ----------------- Authoring Tickets @@ -111,15 +111,10 @@ Setuptools follows ``semver``. Building Documentation ---------------------- -Setuptools relies on the Sphinx system for building documentation and in -particular the ``build_sphinx`` distutils command. To build the -documentation, invoke:: +Setuptools relies on the Sphinx system for building documentation. +To accommodate RTD, docs must be built from the docs/ directory. - python setup.py build_sphinx +To build them, you need to have installed the requirements specified +in docs/requirements.txt. One way to do this is to use rwt: -from the root of the repository. Setuptools will download a compatible -build of Sphinx and any requisite plugins and then build the -documentation in the build/sphinx directory. - -Setuptools does not support invoking the doc builder from the docs/ -directory as some tools expect. + setuptools/docs$ python -m rwt -r requirements.txt -- -m sphinx . html diff --git a/docs/easy_install.txt b/docs/easy_install.txt index a66909b1..bd9f0e86 100644 --- a/docs/easy_install.txt +++ b/docs/easy_install.txt @@ -577,7 +577,7 @@ activated or deactivated. As a result, if you are using EasyInstall to upgrade an existing package, or to install a package with the same name as an existing package, EasyInstall will warn you of the conflict. (This is an improvement over ``setup.py -install``, becuase the ``distutils`` just install new packages on top of old +install``, because the ``distutils`` just install new packages on top of old ones, possibly combining two unrelated packages or leaving behind modules that have been deleted in the newer version of the package.) @@ -606,7 +606,7 @@ can be safely installed as a zipfile, and then acts on its analysis. (Previous versions would not install a package as a zipfile unless you used the ``--zip-ok`` option.) -The current analysis approach is fairly conservative; it currenly looks for: +The current analysis approach is fairly conservative; it currently looks for: * Any use of the ``__file__`` or ``__path__`` variables (which should be replaced with ``pkg_resources`` API calls) @@ -768,7 +768,7 @@ Command-Line Options run by scripts that are already installed. ``--user`` (New in 0.6.11) - Use the the user-site-packages as specified in :pep:`370` + Use the user-site-packages as specified in :pep:`370` instead of the global site-packages. ``--always-copy, -a`` (New in 0.5a4) diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 57818281..0f955663 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -302,6 +302,10 @@ unless you need the associated ``setuptools`` feature. installed to support those features. See the section below on `Declaring Dependencies`_ for details and examples of the format of this argument. +``python_requires`` + A string corresponding to a version specifier (as defined in PEP 440) for + the Python version, used to specify the Requires-Python defined in PEP 345. + ``setup_requires`` A string or list of strings specifying what other distributions need to be present in order for the *setup script* to run. ``setuptools`` will @@ -917,7 +921,7 @@ use its resource management API. See also `Accessing Package Resources`_ for a quick example of converting code that uses ``__file__`` to use ``pkg_resources`` instead. -.. _Resource Management API: http://peak.telecommunity.com/DevCenter/PythonEggs#resource-management +.. _Resource Management API: http://peak.telecommunity.com/DevCenter/PkgResources#resourcemanager-api .. _Accessing Package Resources: http://peak.telecommunity.com/DevCenter/PythonEggs#accessing-package-resources diff --git a/msvc-build-launcher.cmd b/msvc-build-launcher.cmd index e54c4f6c..92da290e 100644 --- a/msvc-build-launcher.cmd +++ b/msvc-build-launcher.cmd @@ -35,21 +35,5 @@ if "%ERRORLEVEL%"=="0" ( echo Windows SDK 6.1 not found to build Windows 64-bit version
)
-REM Windows RT ARM build requires both freeware
-REM "Visual Studio Express 2012 for Windows 8" and
-REM "Visual Studio Express 2012 for Windows Desktop" to be installed from
-REM http://www.microsoft.com/visualstudio/eng/products/visual-studio-express-products
-set PATH=%PATH_OLD%
-set PATH=C:\Program Files\Microsoft Visual Studio 11.0\VC;%PATH%
-set PATH=C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC;%PATH%
-call VCVARSALL x86_arm >nul 2>&1
-if "%ERRORLEVEL%"=="0" (
- echo Building Windows RT Version ...
- cl /D "GUI=0" /D "WIN32_LEAN_AND_MEAN" /D _ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE launcher.c /O2 /link /MACHINE:ARM /SUBSYSTEM:CONSOLE /out:setuptools/cli-arm-32.exe
- cl /D "GUI=1" /D "WIN32_LEAN_AND_MEAN" /D _ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE launcher.c /O2 /link /MACHINE:ARM /SUBSYSTEM:WINDOWS /out:setuptools/gui-arm-32.exe
-) else (
- echo Visual Studio ^(Express^) 2012 not found to build Windows RT Version
-)
-
set PATH=%PATH_OLD%
diff --git a/pavement.py b/pavement.py index 303e9bac..f85617d4 100644 --- a/pavement.py +++ b/pavement.py @@ -3,19 +3,23 @@ import re from paver.easy import task, path as Path import pip + def remove_all(paths): for path in paths: path.rmtree() if path.isdir() else path.remove() + @task def update_vendored(): vendor = Path('pkg_resources/_vendor') + # pip uninstall doesn't support -t, so do it manually remove_all(vendor.glob('packaging*')) remove_all(vendor.glob('six*')) remove_all(vendor.glob('pyparsing*')) + remove_all(vendor.glob('appdirs*')) install_args = [ 'install', - '-r', str(vendor/'vendored.txt'), + '-r', str(vendor / 'vendored.txt'), '-t', str(vendor), ] pip.main(install_args) diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index 2eab8230..37bf1482 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -1,3 +1,5 @@ +# coding: utf-8 + """ Package resource API -------------------- @@ -65,6 +67,7 @@ try: except ImportError: importlib_machinery = None +from pkg_resources.extern import appdirs from pkg_resources.extern import packaging __import__('pkg_resources.extern.packaging.version') __import__('pkg_resources.extern.packaging.specifiers') @@ -155,7 +158,7 @@ class _SetuptoolsVersionMixin(object): # pad for numeric comparison yield part.zfill(8) else: - yield '*'+part + yield '*' + part # ensure that alpha/beta/candidate are before final yield '*final' @@ -209,36 +212,44 @@ def parse_version(v): _state_vars = {} + def _declare_state(vartype, **kw): globals().update(kw) _state_vars.update(dict.fromkeys(kw, vartype)) + def __getstate__(): state = {} g = globals() for k, v in _state_vars.items(): - state[k] = g['_sget_'+v](g[k]) + state[k] = g['_sget_' + v](g[k]) return state + def __setstate__(state): g = globals() for k, v in state.items(): - g['_sset_'+_state_vars[k]](k, g[k], v) + g['_sset_' + _state_vars[k]](k, g[k], v) return state + def _sget_dict(val): return val.copy() + def _sset_dict(key, ob, state): ob.clear() ob.update(state) + def _sget_object(val): return val.__getstate__() + def _sset_object(key, ob, state): ob.__setstate__(state) + _sget_none = _sset_none = lambda *args: None @@ -265,9 +276,10 @@ def get_supported_platform(): pass return plat + __all__ = [ # Basic resource access and distribution/entry point discovery - 'require', 'run_script', 'get_provider', 'get_distribution', + 'require', 'run_script', 'get_provider', 'get_distribution', 'load_entry_point', 'get_entry_map', 'get_entry_info', 'iter_entry_points', 'resource_string', 'resource_stream', 'resource_filename', @@ -311,10 +323,12 @@ __all__ = [ 'run_main', 'AvailableDistributions', ] + class ResolutionError(Exception): """Abstract base for dependency resolution errors""" + def __repr__(self): - return self.__class__.__name__+repr(self.args) + return self.__class__.__name__ + repr(self.args) class VersionConflict(ResolutionError): @@ -391,6 +405,8 @@ class DistributionNotFound(ResolutionError): class UnknownExtra(ResolutionError): """Distribution doesn't have an "extra feature" of the given name""" + + _provider_factories = {} PY_MAJOR = sys.version[:3] @@ -400,6 +416,7 @@ SOURCE_DIST = 1 CHECKOUT_DIST = 0 DEVELOP_DIST = -1 + def register_loader_type(loader_type, provider_factory): """Register `provider_factory` to make providers for `loader_type` @@ -409,6 +426,7 @@ def register_loader_type(loader_type, provider_factory): """ _provider_factories[loader_type] = provider_factory + def get_provider(moduleOrReq): """Return an IResourceProvider for the named module or requirement""" if isinstance(moduleOrReq, Requirement): @@ -421,6 +439,7 @@ def get_provider(moduleOrReq): loader = getattr(module, '__loader__', None) return _find_adapter(_provider_factories, loader)(module) + def _macosx_vers(_cache=[]): if not _cache: version = platform.mac_ver()[0] @@ -436,9 +455,11 @@ def _macosx_vers(_cache=[]): _cache.append(version.split('.')) return _cache[0] + def _macosx_arch(machine): return {'PowerPC': 'ppc', 'Power_Macintosh': 'ppc'}.get(machine, machine) + def get_build_platform(): """Return this platform's string for platform-specific distributions @@ -464,6 +485,7 @@ def get_build_platform(): pass return plat + macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)") darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)") # XXX backward compat @@ -477,7 +499,7 @@ def compatible_platforms(provided, required): XXX Needs compatibility checks for Linux and other unixy OSes. """ - if provided is None or required is None or provided==required: + if provided is None or required is None or provided == required: # easy case return True @@ -524,9 +546,11 @@ def run_script(dist_spec, script_name): ns['__name__'] = name require(dist_spec)[0].run_script(script_name, ns) + # backward compatibility run_main = run_script + def get_distribution(dist): """Return a current distribution object for a Requirement or string""" if isinstance(dist, six.string_types): @@ -537,14 +561,17 @@ def get_distribution(dist): raise TypeError("Expected string, Requirement, or Distribution", dist) return dist + def load_entry_point(dist, group, name): """Return `name` entry point of `group` for `dist` or raise ImportError""" return get_distribution(dist).load_entry_point(group, name) + def get_entry_map(dist, group=None): """Return the entry point map for `group`, or the full entry map""" return get_distribution(dist).get_entry_map(group) + def get_entry_info(dist, group, name): """Return the EntryPoint object for `group`+`name`, or ``None``""" return get_distribution(dist).get_entry_info(group, name) @@ -732,7 +759,7 @@ class WorkingSet(object): for key in self.entry_keys[item]: if key not in seen: - seen[key]=1 + seen[key] = 1 yield self.by_key[key] def add(self, dist, entry=None, insert=True, replace=False): @@ -752,8 +779,8 @@ class WorkingSet(object): if entry is None: entry = dist.location - keys = self.entry_keys.setdefault(entry,[]) - keys2 = self.entry_keys.setdefault(dist.location,[]) + keys = self.entry_keys.setdefault(entry, []) + keys2 = self.entry_keys.setdefault(dist.location, []) if not replace and dist.key in self.by_key: # ignore hidden distros return @@ -947,11 +974,17 @@ class WorkingSet(object): return needed - def subscribe(self, callback): - """Invoke `callback` for all distributions (including existing ones)""" + def subscribe(self, callback, existing=True): + """Invoke `callback` for all distributions + + If `existing=True` (default), + call on all existing ones, as well. + """ if callback in self.callbacks: return self.callbacks.append(callback) + if not existing: + return for dist in self: callback(dist) @@ -1027,7 +1060,7 @@ class Environment(object): is returned. """ return (self.python is None or dist.py_version is None - or dist.py_version==self.python) \ + or dist.py_version == self.python) \ and compatible_platforms(dist.platform, self.platform) def remove(self, dist): @@ -1232,7 +1265,7 @@ class ResourceManager: extract, as it tracks the generated names for possible cleanup later. """ extract_path = self.extraction_path or get_default_cache() - target_path = os.path.join(extract_path, archive_name+'-tmp', *names) + target_path = os.path.join(extract_path, archive_name + '-tmp', *names) try: _bypass_ensure_directory(target_path) except: @@ -1326,49 +1359,18 @@ class ResourceManager: """ # XXX -def get_default_cache(): - """Determine the default cache location - This returns the ``PYTHON_EGG_CACHE`` environment variable, if set. - Otherwise, on Windows, it returns a "Python-Eggs" subdirectory of the - "Application Data" directory. On all other systems, it's "~/.python-eggs". +def get_default_cache(): """ - try: - return os.environ['PYTHON_EGG_CACHE'] - except KeyError: - pass + Return the ``PYTHON_EGG_CACHE`` environment variable + or a platform-relevant user cache dir for an app + named "Python-Eggs". + """ + return ( + os.environ.get('PYTHON_EGG_CACHE') + or appdirs.user_cache_dir(appname='Python-Eggs') + ) - if os.name!='nt': - return os.path.expanduser('~/.python-eggs') - - # XXX this may be locale-specific! - app_data = 'Application Data' - app_homes = [ - # best option, should be locale-safe - (('APPDATA',), None), - (('USERPROFILE',), app_data), - (('HOMEDRIVE','HOMEPATH'), app_data), - (('HOMEPATH',), app_data), - (('HOME',), None), - # 95/98/ME - (('WINDIR',), app_data), - ] - - for keys, subdir in app_homes: - dirname = '' - for key in keys: - if key in os.environ: - dirname = os.path.join(dirname, os.environ[key]) - else: - break - else: - if subdir: - dirname = os.path.join(dirname, subdir) - return os.path.join(dirname, 'Python-Eggs') - else: - raise RuntimeError( - "Please set the PYTHON_EGG_CACHE enviroment variable" - ) def safe_name(name): """Convert an arbitrary string to a standard distribution name @@ -1386,7 +1388,7 @@ def safe_version(version): # normalize the version return str(packaging.version.Version(version)) except packaging.version.InvalidVersion: - version = version.replace(' ','.') + version = version.replace(' ', '.') return re.sub('[^A-Za-z0-9.]+', '-', version) @@ -1404,7 +1406,7 @@ def to_filename(name): Any '-' characters are currently replaced with '_'. """ - return name.replace('-','_') + return name.replace('-', '_') def invalid_marker(text): @@ -1462,16 +1464,11 @@ class NullProvider: def has_metadata(self, name): return self.egg_info and self._has(self._fn(self.egg_info, name)) - if sys.version_info <= (3,): - def get_metadata(self, name): - if not self.egg_info: - return "" - return self._get(self._fn(self.egg_info, name)) - else: - def get_metadata(self, name): - if not self.egg_info: - return "" - return self._get(self._fn(self.egg_info, name)).decode("utf-8") + def get_metadata(self, name): + if not self.egg_info: + return "" + value = self._get(self._fn(self.egg_info, name)) + return value.decode('utf-8') if six.PY3 else value def get_metadata_lines(self, name): return yield_lines(self.get_metadata(name)) @@ -1491,7 +1488,7 @@ class NullProvider: return [] def run_script(self, script_name, namespace): - script = 'scripts/'+script_name + script = 'scripts/' + script_name if not self.has_metadata(script): raise ResolutionError("No script named %r" % script_name) script_text = self.get_metadata(script).replace('\r\n', '\n') @@ -1507,7 +1504,7 @@ class NullProvider: cache[script_filename] = ( len(script_text), 0, script_text.split('\n'), script_filename ) - script_code = compile(script_text, script_filename,'exec') + script_code = compile(script_text, script_filename, 'exec') exec(script_code, namespace, namespace) def _has(self, path): @@ -1537,6 +1534,7 @@ class NullProvider: "Can't perform this operation for loaders without 'get_data()'" ) + register_loader_type(object, NullProvider) @@ -1552,7 +1550,7 @@ class EggProvider(NullProvider): # of multiple eggs; that's why we use module_path instead of .archive path = self.module_path old = None - while path!=old: + while path != old: if _is_unpacked_egg(path): self.egg_name = os.path.basename(path) self.egg_info = os.path.join(path, 'EGG-INFO') @@ -1561,6 +1559,7 @@ class EggProvider(NullProvider): old = path path, base = os.path.split(path) + class DefaultProvider(EggProvider): """Provides access to package resources in the filesystem""" @@ -1586,6 +1585,7 @@ class DefaultProvider(EggProvider): type(None)) register_loader_type(loader_cls, cls) + DefaultProvider._register() @@ -1600,6 +1600,7 @@ class EmptyProvider(NullProvider): def __init__(self): pass + empty_provider = EmptyProvider() @@ -1678,7 +1679,7 @@ class ZipProvider(EggProvider): def __init__(self, module): EggProvider.__init__(self, module) - self.zip_pre = self.loader.archive+os.sep + self.zip_pre = self.loader.archive + os.sep def _zipinfo_name(self, fspath): # Convert a virtual filename (full path to file) into a zipfile subpath @@ -1692,9 +1693,9 @@ class ZipProvider(EggProvider): def _parts(self, zip_path): # Convert a zipfile subpath into an egg-relative path part list. # pseudo-fs path - fspath = self.zip_pre+zip_path - if fspath.startswith(self.egg_root+os.sep): - return fspath[len(self.egg_root)+1:].split(os.sep) + fspath = self.zip_pre + zip_path + if fspath.startswith(self.egg_root + os.sep): + return fspath[len(self.egg_root) + 1:].split(os.sep) raise AssertionError( "%s is not a subpath of %s" % (fspath, self.egg_root) ) @@ -1765,7 +1766,7 @@ class ZipProvider(EggProvider): # so proceed. return real_path # Windows, del old file and retry - elif os.name=='nt': + elif os.name == 'nt': unlink(real_path) rename(tmpnam, real_path) return real_path @@ -1785,7 +1786,7 @@ class ZipProvider(EggProvider): if not os.path.isfile(file_path): return False stat = os.stat(file_path) - if stat.st_size!=size or stat.st_mtime!=timestamp: + if stat.st_size != size or stat.st_mtime != timestamp: return False # check that the contents match zip_contents = self.loader.get_data(zip_path) @@ -1835,6 +1836,7 @@ class ZipProvider(EggProvider): def _resource_to_zip(self, resource_name): return self._zipinfo_name(self._fn(self.module_path, resource_name)) + register_loader_type(zipimport.zipimporter, ZipProvider) @@ -1854,20 +1856,24 @@ class FileMetadata(EmptyProvider): self.path = path def has_metadata(self, name): - return name=='PKG-INFO' and os.path.isfile(self.path) + return name == 'PKG-INFO' and os.path.isfile(self.path) def get_metadata(self, name): - if name=='PKG-INFO': - with io.open(self.path, encoding='utf-8') as f: - try: - metadata = f.read() - except UnicodeDecodeError as exc: - # add path context to error message - tmpl = " in {self.path}" - exc.reason += tmpl.format(self=self) - raise - return metadata - raise KeyError("No metadata except PKG-INFO is available") + if name != 'PKG-INFO': + raise KeyError("No metadata except PKG-INFO is available") + + with io.open(self.path, encoding='utf-8', errors="replace") as f: + metadata = f.read() + self._warn_on_replacement(metadata) + return metadata + + def _warn_on_replacement(self, metadata): + # Python 2.6 and 3.2 compat for: replacement_char = '�' + replacement_char = b'\xef\xbf\xbd'.decode('utf-8') + if replacement_char in metadata: + tmpl = "{self.path} could not be properly decoded in UTF-8" + msg = tmpl.format(**locals()) + warnings.warn(msg) def get_metadata_lines(self, name): return yield_lines(self.get_metadata(name)) @@ -1904,7 +1910,7 @@ class EggMetadata(ZipProvider): def __init__(self, importer): """Create a metadata provider from a zipimporter""" - self.zip_pre = importer.archive+os.sep + self.zip_pre = importer.archive + os.sep self.loader = importer if importer.prefix: self.module_path = os.path.join(importer.archive, importer.prefix) @@ -1912,7 +1918,9 @@ class EggMetadata(ZipProvider): self.module_path = importer.archive self._setup_prefix() -_declare_state('dict', _distribution_finders = {}) + +_declare_state('dict', _distribution_finders={}) + def register_finder(importer_type, distribution_finder): """Register `distribution_finder` to find distributions in sys.path items @@ -1930,6 +1938,7 @@ def find_distributions(path_item, only=False): finder = _find_adapter(_distribution_finders, importer) return finder(importer, path_item, only) + def find_eggs_in_zip(importer, path_item, only=False): """ Find eggs in zip files; possibly multiple nested eggs. @@ -1950,12 +1959,17 @@ def find_eggs_in_zip(importer, path_item, only=False): for dist in find_eggs_in_zip(zipimport.zipimporter(subpath), subpath): yield dist + register_finder(zipimport.zipimporter, find_eggs_in_zip) + def find_nothing(importer, path_item, only=False): return () + + register_finder(object, find_nothing) + def find_on_path(importer, path_item, only=False): """Yield distributions accessible on a sys.path directory""" path_item = _normalize_cached(path_item) @@ -1964,7 +1978,7 @@ def find_on_path(importer, path_item, only=False): if _is_unpacked_egg(path_item): yield Distribution.from_filename( path_item, metadata=PathMetadata( - path_item, os.path.join(path_item,'EGG-INFO') + path_item, os.path.join(path_item, 'EGG-INFO') ) ) else: @@ -1996,6 +2010,8 @@ def find_on_path(importer, path_item, only=False): for item in dists: yield item break + + register_finder(pkgutil.ImpImporter, find_on_path) if hasattr(importlib_machinery, 'FileFinder'): @@ -2022,6 +2038,7 @@ def register_namespace_handler(importer_type, namespace_handler): """ _namespace_handlers[importer_type] = namespace_handler + def _handle_ns(packageName, path_item): """Ensure that named package includes a subpath of path_item (if needed)""" @@ -2036,7 +2053,7 @@ def _handle_ns(packageName, path_item): module = sys.modules[packageName] = types.ModuleType(packageName) module.__path__ = [] _set_parent_ns(packageName) - elif not hasattr(module,'__path__'): + elif not hasattr(module, '__path__'): raise TypeError("Not a package:", packageName) handler = _find_adapter(_namespace_handlers, importer) subpath = handler(importer, path_item, packageName, module) @@ -2054,6 +2071,16 @@ def _rebuild_mod_path(orig_path, package_name, module): corresponding to their sys.path order """ sys_path = [_normalize_cached(p) for p in sys.path] + + def safe_sys_path_index(entry): + """ + Workaround for #520 and #513. + """ + try: + return sys_path.index(entry) + except ValueError: + return float('inf') + def position_in_sys_path(path): """ Return the ordinal of the path based on its position in sys.path @@ -2061,7 +2088,7 @@ def _rebuild_mod_path(orig_path, package_name, module): path_parts = path.split(os.sep) module_parts = package_name.count('.') + 1 parts = path_parts[:-module_parts] - return sys_path.index(_normalize_cached(os.sep.join(parts))) + return safe_sys_path_index(_normalize_cached(os.sep.join(parts))) orig_path.sort(key=position_in_sys_path) module.__path__[:] = [_normalize_cached(p) for p in orig_path] @@ -2088,8 +2115,8 @@ def declare_namespace(packageName): # Track what packages are namespaces, so when new path items are added, # they can be updated - _namespace_packages.setdefault(parent,[]).append(packageName) - _namespace_packages.setdefault(packageName,[]) + _namespace_packages.setdefault(parent, []).append(packageName) + _namespace_packages.setdefault(packageName, []) for path_item in path: # Ensure all the parent's path items are reflected in the child, @@ -2099,29 +2126,32 @@ def declare_namespace(packageName): finally: _imp.release_lock() + def fixup_namespace_packages(path_item, parent=None): """Ensure that previously-declared namespace packages include path_item""" _imp.acquire_lock() try: - for package in _namespace_packages.get(parent,()): + for package in _namespace_packages.get(parent, ()): subpath = _handle_ns(package, path_item) if subpath: fixup_namespace_packages(subpath, package) finally: _imp.release_lock() + def file_ns_handler(importer, path_item, packageName, module): """Compute an ns-package subpath for a filesystem or zipfile importer""" subpath = os.path.join(path_item, packageName.split('.')[-1]) normalized = _normalize_cached(subpath) for item in module.__path__: - if _normalize_cached(item)==normalized: + if _normalize_cached(item) == normalized: break else: # Only return the path if it's not already there return subpath + register_namespace_handler(pkgutil.ImpImporter, file_ns_handler) register_namespace_handler(zipimport.zipimporter, file_ns_handler) @@ -2132,6 +2162,7 @@ if hasattr(importlib_machinery, 'FileFinder'): def null_ns_handler(importer, path_item, packageName, module): return None + register_namespace_handler(object, null_ns_handler) @@ -2139,6 +2170,7 @@ def normalize_path(filename): """Normalize a file/dir name for comparison purposes""" return os.path.normcase(os.path.realpath(filename)) + def _normalize_cached(filename, _cache={}): try: return _cache[filename] @@ -2146,6 +2178,7 @@ def _normalize_cached(filename, _cache={}): _cache[filename] = result = normalize_path(filename) return result + def _is_unpacked_egg(path): """ Determine if given path appears to be an unpacked egg. @@ -2154,6 +2187,7 @@ def _is_unpacked_egg(path): path.lower().endswith('.egg') ) + def _set_parent_ns(packageName): parts = packageName.split('.') name = parts.pop() @@ -2175,6 +2209,7 @@ def yield_lines(strs): for s in yield_lines(ss): yield s + MODULE = re.compile(r"\w+(\.\w+)*$").match EGG_NAME = re.compile( r""" @@ -2293,7 +2328,7 @@ class EntryPoint(object): ep = cls.parse(line, dist) if ep.name in this: raise ValueError("Duplicate entry point", group, ep.name) - this[ep.name]=ep + this[ep.name] = ep return this @classmethod @@ -2355,7 +2390,7 @@ class Distribution(object): @classmethod def from_location(cls, location, basename, metadata=None, **kw): - project_name, version, py_version, platform = [None]*4 + project_name, version, py_version, platform = [None] * 4 basename, ext = os.path.splitext(basename) if ext.lower() in _distributionImpl: cls = _distributionImpl[ext.lower()] @@ -2477,11 +2512,11 @@ class Distribution(object): extra, marker = extra.split(':', 1) if invalid_marker(marker): # XXX warn - reqs=[] + reqs = [] elif not evaluate_marker(marker): - reqs=[] + reqs = [] extra = safe_extra(extra) or None - dm.setdefault(extra,[]).extend(parse_requirements(reqs)) + dm.setdefault(extra, []).extend(parse_requirements(reqs)) return dm def requires(self, extras=()): @@ -2503,11 +2538,11 @@ class Distribution(object): for line in self.get_metadata_lines(name): yield line - def activate(self, path=None): + def activate(self, path=None, replace=False): """Ensure distribution is importable on `path` (default=sys.path)""" if path is None: path = sys.path - self.insert_on(path, replace=True) + self.insert_on(path, replace=replace) if path is sys.path: fixup_namespace_packages(self.location) for pkg in self._get_metadata('namespace_packages.txt'): @@ -2577,7 +2612,7 @@ class Distribution(object): self._get_metadata('entry_points.txt'), self ) if group is not None: - return ep_map.get(group,{}) + return ep_map.get(group, {}) return ep_map def get_entry_info(self, group, name): @@ -2585,7 +2620,24 @@ class Distribution(object): return self.get_entry_map(group).get(name) def insert_on(self, path, loc=None, replace=False): - """Insert self.location in path before its nearest parent directory""" + """Ensure self.location is on path + + If replace=False (default): + - If location is already in path anywhere, do nothing. + - Else: + - If it's an egg and its parent directory is on path, + insert just ahead of the parent. + - Else: add to the end of path. + If replace=True: + - If location is already on path anywhere (not eggs) + or higher priority than its parent (eggs) + do nothing. + - Else: + - If it's an egg and its parent directory is on path, + insert just ahead of the parent, + removing any lower-priority entries. + - Else: add it to the front of path. + """ loc = loc or self.location if not loc: @@ -2593,13 +2645,20 @@ class Distribution(object): nloc = _normalize_cached(loc) bdir = os.path.dirname(nloc) - npath= [(p and _normalize_cached(p) or p) for p in path] + npath = [(p and _normalize_cached(p) or p) for p in path] for p, item in enumerate(npath): if item == nloc: - break + if replace: + break + else: + # don't modify path (even removing duplicates) if found and not replace + return elif item == bdir and self.precedence == EGG_DIST: # if it's an .egg, give it precedence over its directory + # UNLESS it's already been added to sys.path and replace=False + if (not replace) and nloc in npath[p:]: + return if path is sys.path: self.check_version_conflict() path.insert(p, loc) @@ -2617,7 +2676,7 @@ class Distribution(object): # p is the spot where we found or inserted loc; now remove duplicates while True: try: - np = npath.index(nloc, p+1) + np = npath.index(nloc, p + 1) except ValueError: break else: @@ -2657,7 +2716,7 @@ class Distribution(object): return False return True - def clone(self,**kw): + def clone(self, **kw): """Copy this distribution, substituting in any changed keyword args""" names = 'project_name version py_version platform location precedence' for attr in names.split(): @@ -2744,7 +2803,7 @@ _distributionImpl = { } -def issue_warning(*args,**kw): +def issue_warning(*args, **kw): level = 1 g = globals() try: @@ -2758,6 +2817,7 @@ def issue_warning(*args,**kw): class RequirementParseError(ValueError): + def __str__(self): return ' '.join(self.args) @@ -2782,6 +2842,7 @@ def parse_requirements(strs): class Requirement(packaging.requirements.Requirement): + def __init__(self, requirement_string): """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!""" try: @@ -2837,10 +2898,12 @@ class Requirement(packaging.requirements.Requirement): def _get_mro(cls): """Get an mro for a type or classic class""" if not isinstance(cls, type): - class cls(cls, object): pass + class cls(cls, object): + pass return cls.__mro__[1:] return cls.__mro__ + def _find_adapter(registry, ob): """Return an adapter factory for `ob` from `registry`""" for t in _get_mro(getattr(ob, '__class__', type(ob))): @@ -2890,12 +2953,13 @@ def split_sections(s): # wrap up last segment yield section, content -def _mkstemp(*args,**kw): + +def _mkstemp(*args, **kw): old_open = os.open try: # temporarily bypass sandboxing os.open = os_open - return tempfile.mkstemp(*args,**kw) + return tempfile.mkstemp(*args, **kw) finally: # and then put it back os.open = old_open @@ -2946,11 +3010,16 @@ def _initialize_master_working_set(): run_script = working_set.run_script # backward compatibility run_main = run_script - # Activate all distributions already on sys.path, and ensure that - # all distributions added to the working set in the future (e.g. by - # calling ``require()``) will get activated as well. - add_activation_listener(lambda dist: dist.activate()) - working_set.entries=[] + # Activate all distributions already on sys.path with replace=False and + # ensure that all distributions added to the working set in the future + # (e.g. by calling ``require()``) will get activated as well, + # with higher priority (replace=True). + dist = None # ensure dist is defined for del dist below + for dist in working_set: + dist.activate(replace=False) + del dist + add_activation_listener(lambda dist: dist.activate(replace=True), existing=False) + working_set.entries = [] # match order list(map(working_set.add_entry, sys.path)) globals().update(locals()) diff --git a/pkg_resources/_vendor/appdirs.py b/pkg_resources/_vendor/appdirs.py new file mode 100644 index 00000000..f4dba095 --- /dev/null +++ b/pkg_resources/_vendor/appdirs.py @@ -0,0 +1,552 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2005-2010 ActiveState Software Inc. +# Copyright (c) 2013 Eddy Petrișor + +"""Utilities for determining application-specific dirs. + +See <http://github.com/ActiveState/appdirs> for details and usage. +""" +# Dev Notes: +# - MSDN on where to store app data files: +# http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120 +# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html +# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html + +__version_info__ = (1, 4, 0) +__version__ = '.'.join(map(str, __version_info__)) + + +import sys +import os + +PY3 = sys.version_info[0] == 3 + +if PY3: + unicode = str + +if sys.platform.startswith('java'): + import platform + os_name = platform.java_ver()[3][0] + if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc. + system = 'win32' + elif os_name.startswith('Mac'): # "Mac OS X", etc. + system = 'darwin' + else: # "Linux", "SunOS", "FreeBSD", etc. + # Setting this to "linux2" is not ideal, but only Windows or Mac + # are actually checked for and the rest of the module expects + # *sys.platform* style strings. + system = 'linux2' +else: + system = sys.platform + + + +def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> + for a discussion of issues. + + Typical user data directories are: + Mac OS X: ~/Library/Application Support/<AppName> + Unix: ~/.local/share/<AppName> # or in $XDG_DATA_HOME, if defined + Win XP (not roaming): C:\Documents and Settings\<username>\Application Data\<AppAuthor>\<AppName> + Win XP (roaming): C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName> + Win 7 (not roaming): C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName> + Win 7 (roaming): C:\Users\<username>\AppData\Roaming\<AppAuthor>\<AppName> + + For Unix, we follow the XDG spec and support $XDG_DATA_HOME. + That means, by default "~/.local/share/<AppName>". + """ + if system == "win32": + if appauthor is None: + appauthor = appname + const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA" + path = os.path.normpath(_get_win_folder(const)) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) + else: + path = os.path.join(path, appname) + elif system == 'darwin': + path = os.path.expanduser('~/Library/Application Support/') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): + """Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of data dirs should be + returned. By default, the first item from XDG_DATA_DIRS is + returned, or '/usr/local/share/<AppName>', + if XDG_DATA_DIRS is not set + + Typical user data directories are: + Mac OS X: /Library/Application Support/<AppName> + Unix: /usr/local/share/<AppName> or /usr/share/<AppName> + Win XP: C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName> + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + Win 7: C:\ProgramData\<AppAuthor>\<AppName> # Hidden, but writeable on Win 7. + + For Unix, this is using the $XDG_DATA_DIRS[0] default. + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if system == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA")) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) + else: + path = os.path.join(path, appname) + elif system == 'darwin': + path = os.path.expanduser('/Library/Application Support') + if appname: + path = os.path.join(path, appname) + else: + # XDG default for $XDG_DATA_DIRS + # only first, if multipath is False + path = os.getenv('XDG_DATA_DIRS', + os.pathsep.join(['/usr/local/share', '/usr/share'])) + pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [os.sep.join([x, appname]) for x in pathlist] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + + if appname and version: + path = os.path.join(path, version) + return path + + +def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific config dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> + for a discussion of issues. + + Typical user data directories are: + Mac OS X: same as user_data_dir + Unix: ~/.config/<AppName> # or in $XDG_CONFIG_HOME, if defined + Win *: same as user_data_dir + + For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME. + That means, by deafult "~/.config/<AppName>". + """ + if system in ["win32", "darwin"]: + path = user_data_dir(appname, appauthor, None, roaming) + else: + path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): + """Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of config dirs should be + returned. By default, the first item from XDG_CONFIG_DIRS is + returned, or '/etc/xdg/<AppName>', if XDG_CONFIG_DIRS is not set + + Typical user data directories are: + Mac OS X: same as site_data_dir + Unix: /etc/xdg/<AppName> or $XDG_CONFIG_DIRS[i]/<AppName> for each value in + $XDG_CONFIG_DIRS + Win *: same as site_data_dir + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + + For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if system in ["win32", "darwin"]: + path = site_data_dir(appname, appauthor) + if appname and version: + path = os.path.join(path, version) + else: + # XDG default for $XDG_CONFIG_DIRS + # only first, if multipath is False + path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') + pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [os.sep.join([x, appname]) for x in pathlist] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + + +def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific cache dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Cache" to the base app data dir for Windows. See + discussion below. + + Typical user cache directories are: + Mac OS X: ~/Library/Caches/<AppName> + Unix: ~/.cache/<AppName> (XDG default) + Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Cache + Vista: C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Cache + + On Windows the only suggestion in the MSDN docs is that local settings go in + the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming + app data dir (the default returned by `user_data_dir` above). Apps typically + put cache data somewhere *under* the given dir here. Some examples: + ...\Mozilla\Firefox\Profiles\<ProfileName>\Cache + ...\Acme\SuperApp\Cache\1.0 + OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. + This can be disabled with the `opinion=False` option. + """ + if system == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) + else: + path = os.path.join(path, appname) + if opinion: + path = os.path.join(path, "Cache") + elif system == 'darwin': + path = os.path.expanduser('~/Library/Caches') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific log dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Logs" to the base app data dir for Windows, and "log" to the + base cache dir for Unix. See discussion below. + + Typical user cache directories are: + Mac OS X: ~/Library/Logs/<AppName> + Unix: ~/.cache/<AppName>/log # or under $XDG_CACHE_HOME if defined + Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Logs + Vista: C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Logs + + On Windows the only suggestion in the MSDN docs is that local settings + go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in + examples of what some windows apps use for a logs dir.) + + OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA` + value for Windows and appends "log" to the user cache dir for Unix. + This can be disabled with the `opinion=False` option. + """ + if system == "darwin": + path = os.path.join( + os.path.expanduser('~/Library/Logs'), + appname) + elif system == "win32": + path = user_data_dir(appname, appauthor, version) + version = False + if opinion: + path = os.path.join(path, "Logs") + else: + path = user_cache_dir(appname, appauthor, version) + version = False + if opinion: + path = os.path.join(path, "log") + if appname and version: + path = os.path.join(path, version) + return path + + +class AppDirs(object): + """Convenience wrapper for getting application dirs.""" + def __init__(self, appname, appauthor=None, version=None, roaming=False, + multipath=False): + self.appname = appname + self.appauthor = appauthor + self.version = version + self.roaming = roaming + self.multipath = multipath + + @property + def user_data_dir(self): + return user_data_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + + @property + def site_data_dir(self): + return site_data_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + + @property + def user_config_dir(self): + return user_config_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + + @property + def site_config_dir(self): + return site_config_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + + @property + def user_cache_dir(self): + return user_cache_dir(self.appname, self.appauthor, + version=self.version) + + @property + def user_log_dir(self): + return user_log_dir(self.appname, self.appauthor, + version=self.version) + + +#---- internal support stuff + +def _get_win_folder_from_registry(csidl_name): + """This is a fallback technique at best. I'm not sure if using the + registry for this guarantees us the correct answer for all CSIDL_* + names. + """ + import _winreg + + shell_folder_name = { + "CSIDL_APPDATA": "AppData", + "CSIDL_COMMON_APPDATA": "Common AppData", + "CSIDL_LOCAL_APPDATA": "Local AppData", + }[csidl_name] + + key = _winreg.OpenKey( + _winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" + ) + dir, type = _winreg.QueryValueEx(key, shell_folder_name) + return dir + + +def _get_win_folder_with_pywin32(csidl_name): + from win32com.shell import shellcon, shell + dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0) + # Try to make this a unicode path because SHGetFolderPath does + # not return unicode strings when there is unicode data in the + # path. + try: + dir = unicode(dir) + + # Downgrade to short path name if have highbit chars. See + # <http://bugs.activestate.com/show_bug.cgi?id=85099>. + has_high_char = False + for c in dir: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + try: + import win32api + dir = win32api.GetShortPathName(dir) + except ImportError: + pass + except UnicodeError: + pass + return dir + + +def _get_win_folder_with_ctypes(csidl_name): + import ctypes + + csidl_const = { + "CSIDL_APPDATA": 26, + "CSIDL_COMMON_APPDATA": 35, + "CSIDL_LOCAL_APPDATA": 28, + }[csidl_name] + + buf = ctypes.create_unicode_buffer(1024) + ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + + # Downgrade to short path name if have highbit chars. See + # <http://bugs.activestate.com/show_bug.cgi?id=85099>. + has_high_char = False + for c in buf: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + buf2 = ctypes.create_unicode_buffer(1024) + if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + buf = buf2 + + return buf.value + +def _get_win_folder_with_jna(csidl_name): + import array + from com.sun import jna + from com.sun.jna.platform import win32 + + buf_size = win32.WinDef.MAX_PATH * 2 + buf = array.zeros('c', buf_size) + shell = win32.Shell32.INSTANCE + shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf) + dir = jna.Native.toString(buf.tostring()).rstrip("\0") + + # Downgrade to short path name if have highbit chars. See + # <http://bugs.activestate.com/show_bug.cgi?id=85099>. + has_high_char = False + for c in dir: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + buf = array.zeros('c', buf_size) + kernel = win32.Kernel32.INSTANCE + if kernal.GetShortPathName(dir, buf, buf_size): + dir = jna.Native.toString(buf.tostring()).rstrip("\0") + + return dir + +if system == "win32": + try: + import win32com.shell + _get_win_folder = _get_win_folder_with_pywin32 + except ImportError: + try: + from ctypes import windll + _get_win_folder = _get_win_folder_with_ctypes + except ImportError: + try: + import com.sun.jna + _get_win_folder = _get_win_folder_with_jna + except ImportError: + _get_win_folder = _get_win_folder_from_registry + + +#---- self test code + +if __name__ == "__main__": + appname = "MyApp" + appauthor = "MyCompany" + + props = ("user_data_dir", "site_data_dir", + "user_config_dir", "site_config_dir", + "user_cache_dir", "user_log_dir") + + print("-- app dirs (with optional 'version')") + dirs = AppDirs(appname, appauthor, version="1.0") + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'version')") + dirs = AppDirs(appname, appauthor) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'appauthor')") + dirs = AppDirs(appname) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (with disabled 'appauthor')") + dirs = AppDirs(appname, appauthor=False) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) diff --git a/pkg_resources/_vendor/pyparsing.py b/pkg_resources/_vendor/pyparsing.py index 3e02dbee..d602a35a 100644..100755 --- a/pkg_resources/_vendor/pyparsing.py +++ b/pkg_resources/_vendor/pyparsing.py @@ -1,6 +1,6 @@ # module pyparsing.py
#
-# Copyright (c) 2003-2015 Paul T. McGuire
+# Copyright (c) 2003-2016 Paul T. McGuire
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -31,15 +31,18 @@ vs. the traditional lex/yacc approach, or the use of regular expressions. With don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
provides a library of classes that you use to construct the grammar directly in Python.
-Here is a program to parse "Hello, World!" (or any greeting of the form C{"<salutation>, <addressee>!"})::
+Here is a program to parse "Hello, World!" (or any greeting of the form
+C{"<salutation>, <addressee>!"}), built up using L{Word}, L{Literal}, and L{And} elements
+(L{'+'<ParserElement.__add__>} operator gives L{And} expressions, strings are auto-converted to
+L{Literal} expressions)::
from pyparsing import Word, alphas
# define grammar of a greeting
- greet = Word( alphas ) + "," + Word( alphas ) + "!"
+ greet = Word(alphas) + "," + Word(alphas) + "!"
hello = "Hello, World!"
- print (hello, "->", greet.parseString( hello ))
+ print (hello, "->", greet.parseString(hello))
The program outputs the following::
@@ -48,7 +51,7 @@ The program outputs the following:: The Python representation of the grammar is quite readable, owing to the self-explanatory
class names, and the use of '+', '|' and '^' operators.
-The parsed results returned from C{parseString()} can be accessed as a nested list, a dictionary, or an
+The L{ParseResults} object returned from L{ParserElement.parseString<ParserElement.parseString>} can be accessed as a nested list, a dictionary, or an
object with named attributes.
The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
@@ -57,8 +60,8 @@ The pyparsing module handles some of the problems that are typically vexing when - embedded comments
"""
-__version__ = "2.0.6"
-__versionTime__ = "9 Nov 2015 19:03"
+__version__ = "2.1.9"
+__versionTime__ = "10 Sep 2016 15:10 UTC"
__author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"
import string
@@ -70,8 +73,22 @@ import re import sre_constants
import collections
import pprint
-import functools
-import itertools
+import traceback
+import types
+from datetime import datetime
+
+try:
+ from _thread import RLock
+except ImportError:
+ from threading import RLock
+
+try:
+ from collections import OrderedDict as _OrderedDict
+except ImportError:
+ try:
+ from ordereddict import OrderedDict as _OrderedDict
+ except ImportError:
+ _OrderedDict = None
#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
@@ -81,21 +98,23 @@ __all__ = [ 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or',
'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException',
'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException',
-'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', 'Upcase',
+'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter',
'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums',
-'htmlComment', 'javaStyleComment', 'keepOriginalText', 'line', 'lineEnd', 'lineStart', 'lineno',
+'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno',
'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity',
'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass',
+'CloseMatch', 'tokenMap', 'pyparsing_common',
]
-PY_3 = sys.version.startswith('3')
+system_version = tuple(sys.version_info)[:3]
+PY_3 = system_version[0] == 3
if PY_3:
_MAX_INT = sys.maxsize
basestring = str
@@ -123,18 +142,11 @@ else: return str(obj)
except UnicodeEncodeError:
- # The Python docs (http://docs.python.org/ref/customization.html#l2h-182)
- # state that "The return value must be a string object". However, does a
- # unicode object (being a subclass of basestring) count as a "string
- # object"?
- # If so, then return a unicode object:
- return unicode(obj)
- # Else encode it... but how? There are many choices... :)
- # Replace unprintables with escape codes?
- #return unicode(obj).encode(sys.getdefaultencoding(), 'backslashreplace_errors')
- # Replace unprintables with question marks?
- #return unicode(obj).encode(sys.getdefaultencoding(), 'replace')
- # ...
+ # Else encode it
+ ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace')
+ xmlcharref = Regex('&#\d+;')
+ xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:])
+ return xmlcharref.transformString(ret)
# build list of single arg builtins, tolerant of Python version, that can be used as parse actions
singleArgBuiltins = []
@@ -160,7 +172,7 @@ def _xml_escape(data): class _Constants(object):
pass
-alphas = string.ascii_lowercase + string.ascii_uppercase
+alphas = string.ascii_uppercase + string.ascii_lowercase
nums = "0123456789"
hexnums = nums + "ABCDEFabcdef"
alphanums = alphas + nums
@@ -180,6 +192,15 @@ class ParseBaseException(Exception): self.msg = msg
self.pstr = pstr
self.parserElement = elem
+ self.args = (pstr, loc, msg)
+
+ @classmethod
+ def _from_exception(cls, pe):
+ """
+ internal factory method to simplify creating one type of ParseException
+ from another - avoids having __init__ signature conflicts among subclasses
+ """
+ return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement)
def __getattr__( self, aname ):
"""supported attributes by name are:
@@ -212,15 +233,26 @@ class ParseBaseException(Exception): markerString, line_str[line_column:]))
return line_str.strip()
def __dir__(self):
- return "loc msg pstr parserElement lineno col line " \
- "markInputline __str__ __repr__".split()
+ return "lineno col line".split() + dir(type(self))
class ParseException(ParseBaseException):
- """exception thrown when parse expressions don't match class;
- supported attributes by name are:
- - lineno - returns the line number of the exception text
- - col - returns the column number of the exception text
- - line - returns the line containing the exception text
+ """
+ Exception thrown when parse expressions don't match class;
+ supported attributes by name are:
+ - lineno - returns the line number of the exception text
+ - col - returns the column number of the exception text
+ - line - returns the line containing the exception text
+
+ Example::
+ try:
+ Word(nums).setName("integer").parseString("ABC")
+ except ParseException as pe:
+ print(pe)
+ print("column: {}".format(pe.col))
+
+ prints::
+ Expected integer (at char 0), (line:1, col:1)
+ column: 1
"""
pass
@@ -230,12 +262,10 @@ class ParseFatalException(ParseBaseException): pass
class ParseSyntaxException(ParseFatalException):
- """just like C{L{ParseFatalException}}, but thrown internally when an
- C{L{ErrorStop<And._ErrorStop>}} ('-' operator) indicates that parsing is to stop immediately because
- an unbacktrackable syntax error has been found"""
- def __init__(self, pe):
- super(ParseSyntaxException, self).__init__(
- pe.pstr, pe.loc, pe.msg, pe.parserElement)
+ """just like L{ParseFatalException}, but thrown internally when an
+ L{ErrorStop<And._ErrorStop>} ('-' operator) indicates that parsing is to stop
+ immediately because an unbacktrackable syntax error has been found"""
+ pass
#~ class ReparseException(ParseBaseException):
#~ """Experimental class - parse actions can raise this exception to cause
@@ -251,7 +281,7 @@ class ParseSyntaxException(ParseFatalException): #~ self.reparseLoc = restartLoc
class RecursiveGrammarException(Exception):
- """exception thrown by C{validate()} if the grammar could be improperly recursive"""
+ """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive"""
def __init__( self, parseElementList ):
self.parseElementTrace = parseElementList
@@ -264,17 +294,50 @@ class _ParseResultsWithOffset(object): def __getitem__(self,i):
return self.tup[i]
def __repr__(self):
- return repr(self.tup)
+ return repr(self.tup[0])
def setOffset(self,i):
self.tup = (self.tup[0],i)
class ParseResults(object):
- """Structured parse results, to provide multiple means of access to the parsed data:
+ """
+ Structured parse results, to provide multiple means of access to the parsed data:
- as a list (C{len(results)})
- by list index (C{results[0], results[1]}, etc.)
- - by attribute (C{results.<resultsName>})
- """
- def __new__(cls, toklist, name=None, asList=True, modal=True ):
+ - by attribute (C{results.<resultsName>} - see L{ParserElement.setResultsName})
+
+ Example::
+ integer = Word(nums)
+ date_str = (integer.setResultsName("year") + '/'
+ + integer.setResultsName("month") + '/'
+ + integer.setResultsName("day"))
+ # equivalent form:
+ # date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ # parseString returns a ParseResults object
+ result = date_str.parseString("1999/12/31")
+
+ def test(s, fn=repr):
+ print("%s -> %s" % (s, fn(eval(s))))
+ test("list(result)")
+ test("result[0]")
+ test("result['month']")
+ test("result.day")
+ test("'month' in result")
+ test("'minutes' in result")
+ test("result.dump()", str)
+ prints::
+ list(result) -> ['1999', '/', '12', '/', '31']
+ result[0] -> '1999'
+ result['month'] -> '12'
+ result.day -> '31'
+ 'month' in result -> True
+ 'minutes' in result -> False
+ result.dump() -> ['1999', '/', '12', '/', '31']
+ - day: 31
+ - month: 12
+ - year: 1999
+ """
+ def __new__(cls, toklist=None, name=None, asList=True, modal=True ):
if isinstance(toklist, cls):
return toklist
retobj = object.__new__(cls)
@@ -283,12 +346,16 @@ class ParseResults(object): # Performance tuning: we construct a *lot* of these, so keep this
# constructor as small and fast as possible
- def __init__( self, toklist, name=None, asList=True, modal=True, isinstance=isinstance ):
+ def __init__( self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance ):
if self.__doinit:
self.__doinit = False
self.__name = None
self.__parent = None
self.__accumNames = {}
+ self.__asList = asList
+ self.__modal = modal
+ if toklist is None:
+ toklist = []
if isinstance(toklist, list):
self.__toklist = toklist[:]
elif isinstance(toklist, _generatorType):
@@ -331,7 +398,7 @@ class ParseResults(object): if isinstance(v,_ParseResultsWithOffset):
self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
sub = v[0]
- elif isinstance(k,int):
+ elif isinstance(k,(int,slice)):
self.__toklist[k] = v
sub = v
else:
@@ -354,11 +421,6 @@ class ParseResults(object): removed = list(range(*i.indices(mylen)))
removed.reverse()
# fixup indices in token dictionary
- #~ for name in self.__tokdict:
- #~ occurrences = self.__tokdict[name]
- #~ for j in removed:
- #~ for k, (value, position) in enumerate(occurrences):
- #~ occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
for name,occurrences in self.__tokdict.items():
for j in removed:
for k, (value, position) in enumerate(occurrences):
@@ -370,39 +432,52 @@ class ParseResults(object): return k in self.__tokdict
def __len__( self ): return len( self.__toklist )
- def __bool__(self): return len( self.__toklist ) > 0
+ def __bool__(self): return ( not not self.__toklist )
__nonzero__ = __bool__
def __iter__( self ): return iter( self.__toklist )
def __reversed__( self ): return iter( self.__toklist[::-1] )
- def iterkeys( self ):
- """Returns all named result keys."""
+ def _iterkeys( self ):
if hasattr(self.__tokdict, "iterkeys"):
return self.__tokdict.iterkeys()
else:
return iter(self.__tokdict)
- def itervalues( self ):
- """Returns all named result values."""
- return (self[k] for k in self.iterkeys())
+ def _itervalues( self ):
+ return (self[k] for k in self._iterkeys())
- def iteritems( self ):
- return ((k, self[k]) for k in self.iterkeys())
+ def _iteritems( self ):
+ return ((k, self[k]) for k in self._iterkeys())
if PY_3:
- keys = iterkeys
- values = itervalues
- items = iteritems
+ keys = _iterkeys
+ """Returns an iterator of all named result keys (Python 3.x only)."""
+
+ values = _itervalues
+ """Returns an iterator of all named result values (Python 3.x only)."""
+
+ items = _iteritems
+ """Returns an iterator of all named result key-value tuples (Python 3.x only)."""
+
else:
+ iterkeys = _iterkeys
+ """Returns an iterator of all named result keys (Python 2.x only)."""
+
+ itervalues = _itervalues
+ """Returns an iterator of all named result values (Python 2.x only)."""
+
+ iteritems = _iteritems
+ """Returns an iterator of all named result key-value tuples (Python 2.x only)."""
+
def keys( self ):
- """Returns all named result keys."""
+ """Returns all named result keys (as a list in Python 2.x, as an iterator in Python 3.x)."""
return list(self.iterkeys())
def values( self ):
- """Returns all named result values."""
+ """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x)."""
return list(self.itervalues())
def items( self ):
- """Returns all named result keys and values as a list of tuples."""
+ """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x)."""
return list(self.iteritems())
def haskeys( self ):
@@ -411,14 +486,39 @@ class ParseResults(object): return bool(self.__tokdict)
def pop( self, *args, **kwargs):
- """Removes and returns item at specified index (default=last).
- Supports both list and dict semantics for pop(). If passed no
- argument or an integer argument, it will use list semantics
- and pop tokens from the list of parsed tokens. If passed a
- non-integer argument (most likely a string), it will use dict
- semantics and pop the corresponding value from any defined
- results names. A second default return value argument is
- supported, just as in dict.pop()."""
+ """
+ Removes and returns item at specified index (default=C{last}).
+ Supports both C{list} and C{dict} semantics for C{pop()}. If passed no
+ argument or an integer argument, it will use C{list} semantics
+ and pop tokens from the list of parsed tokens. If passed a
+ non-integer argument (most likely a string), it will use C{dict}
+ semantics and pop the corresponding value from any defined
+ results names. A second default return value argument is
+ supported, just as in C{dict.pop()}.
+
+ Example::
+ def remove_first(tokens):
+ tokens.pop(0)
+ print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+ print(OneOrMore(Word(nums)).addParseAction(remove_first).parseString("0 123 321")) # -> ['123', '321']
+
+ label = Word(alphas)
+ patt = label("LABEL") + OneOrMore(Word(nums))
+ print(patt.parseString("AAB 123 321").dump())
+
+ # Use pop() in a parse action to remove named result (note that corresponding value is not
+ # removed from list form of results)
+ def remove_LABEL(tokens):
+ tokens.pop("LABEL")
+ return tokens
+ patt.addParseAction(remove_LABEL)
+ print(patt.parseString("AAB 123 321").dump())
+ prints::
+ ['AAB', '123', '321']
+ - LABEL: AAB
+
+ ['AAB', '123', '321']
+ """
if not args:
args = [-1]
for k,v in kwargs.items():
@@ -438,39 +538,83 @@ class ParseResults(object): return defaultvalue
def get(self, key, defaultValue=None):
- """Returns named result matching the given key, or if there is no
- such name, then returns the given C{defaultValue} or C{None} if no
- C{defaultValue} is specified."""
+ """
+ Returns named result matching the given key, or if there is no
+ such name, then returns the given C{defaultValue} or C{None} if no
+ C{defaultValue} is specified.
+
+ Similar to C{dict.get()}.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ result = date_str.parseString("1999/12/31")
+ print(result.get("year")) # -> '1999'
+ print(result.get("hour", "not specified")) # -> 'not specified'
+ print(result.get("hour")) # -> None
+ """
if key in self:
return self[key]
else:
return defaultValue
def insert( self, index, insStr ):
- """Inserts new element at location index in the list of parsed tokens."""
+ """
+ Inserts new element at location index in the list of parsed tokens.
+
+ Similar to C{list.insert()}.
+
+ Example::
+ print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+
+ # use a parse action to insert the parse location in the front of the parsed results
+ def insert_locn(locn, tokens):
+ tokens.insert(0, locn)
+ print(OneOrMore(Word(nums)).addParseAction(insert_locn).parseString("0 123 321")) # -> [0, '0', '123', '321']
+ """
self.__toklist.insert(index, insStr)
# fixup indices in token dictionary
- #~ for name in self.__tokdict:
- #~ occurrences = self.__tokdict[name]
- #~ for k, (value, position) in enumerate(occurrences):
- #~ occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
for name,occurrences in self.__tokdict.items():
for k, (value, position) in enumerate(occurrences):
occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
def append( self, item ):
- """Add single element to end of ParseResults list of elements."""
+ """
+ Add single element to end of ParseResults list of elements.
+
+ Example::
+ print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+
+ # use a parse action to compute the sum of the parsed integers, and add it to the end
+ def append_sum(tokens):
+ tokens.append(sum(map(int, tokens)))
+ print(OneOrMore(Word(nums)).addParseAction(append_sum).parseString("0 123 321")) # -> ['0', '123', '321', 444]
+ """
self.__toklist.append(item)
def extend( self, itemseq ):
- """Add sequence of elements to end of ParseResults list of elements."""
+ """
+ Add sequence of elements to end of ParseResults list of elements.
+
+ Example::
+ patt = OneOrMore(Word(alphas))
+
+ # use a parse action to append the reverse of the matched strings, to make a palindrome
+ def make_palindrome(tokens):
+ tokens.extend(reversed([t[::-1] for t in tokens]))
+ return ''.join(tokens)
+ print(patt.addParseAction(make_palindrome).parseString("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl'
+ """
if isinstance(itemseq, ParseResults):
self += itemseq
else:
self.__toklist.extend(itemseq)
def clear( self ):
- """Clear all elements and results names."""
+ """
+ Clear all elements and results names.
+ """
del self.__toklist[:]
self.__tokdict.clear()
@@ -511,7 +655,11 @@ class ParseResults(object): def __radd__(self, other):
if isinstance(other,int) and other == 0:
+ # useful for merging many ParseResults using sum() builtin
return self.copy()
+ else:
+ # this may raise a TypeError - so be it
+ return other + self
def __repr__( self ):
return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
@@ -531,18 +679,60 @@ class ParseResults(object): return out
def asList( self ):
- """Returns the parse results as a nested list of matching tokens, all converted to strings."""
+ """
+ Returns the parse results as a nested list of matching tokens, all converted to strings.
+
+ Example::
+ patt = OneOrMore(Word(alphas))
+ result = patt.parseString("sldkj lsdkj sldkj")
+ # even though the result prints in string-like form, it is actually a pyparsing ParseResults
+ print(type(result), result) # -> <class 'pyparsing.ParseResults'> ['sldkj', 'lsdkj', 'sldkj']
+
+ # Use asList() to create an actual list
+ result_list = result.asList()
+ print(type(result_list), result_list) # -> <class 'list'> ['sldkj', 'lsdkj', 'sldkj']
+ """
return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist]
def asDict( self ):
- """Returns the named parse results as dictionary."""
+ """
+ Returns the named parse results as a nested dictionary.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ result = date_str.parseString('12/31/1999')
+ print(type(result), repr(result)) # -> <class 'pyparsing.ParseResults'> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]})
+
+ result_dict = result.asDict()
+ print(type(result_dict), repr(result_dict)) # -> <class 'dict'> {'day': '1999', 'year': '12', 'month': '31'}
+
+ # even though a ParseResults supports dict-like access, sometime you just need to have a dict
+ import json
+ print(json.dumps(result)) # -> Exception: TypeError: ... is not JSON serializable
+ print(json.dumps(result.asDict())) # -> {"month": "31", "day": "1999", "year": "12"}
+ """
if PY_3:
- return dict( self.items() )
+ item_fn = self.items
else:
- return dict( self.iteritems() )
+ item_fn = self.iteritems
+
+ def toItem(obj):
+ if isinstance(obj, ParseResults):
+ if obj.haskeys():
+ return obj.asDict()
+ else:
+ return [toItem(v) for v in obj]
+ else:
+ return obj
+
+ return dict((k,toItem(v)) for k,v in item_fn())
def copy( self ):
- """Returns a new copy of a C{ParseResults} object."""
+ """
+ Returns a new copy of a C{ParseResults} object.
+ """
ret = ParseResults( self.__toklist )
ret.__tokdict = self.__tokdict.copy()
ret.__parent = self.__parent
@@ -551,7 +741,9 @@ class ParseResults(object): return ret
def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
- """Returns the parse results as XML. Tags are created for tokens and lists that have defined results names."""
+ """
+ (Deprecated) Returns the parse results as XML. Tags are created for tokens and lists that have defined results names.
+ """
nl = "\n"
out = []
namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items()
@@ -617,7 +809,27 @@ class ParseResults(object): return None
def getName(self):
- """Returns the results name for this token expression."""
+ """
+ Returns the results name for this token expression. Useful when several
+ different expressions might match at a particular location.
+
+ Example::
+ integer = Word(nums)
+ ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d")
+ house_number_expr = Suppress('#') + Word(nums, alphanums)
+ user_data = (Group(house_number_expr)("house_number")
+ | Group(ssn_expr)("ssn")
+ | Group(integer)("age"))
+ user_info = OneOrMore(user_data)
+
+ result = user_info.parseString("22 111-22-3333 #221B")
+ for item in result:
+ print(item.getName(), ':', item[0])
+ prints::
+ age : 22
+ ssn : 111-22-3333
+ house_number : 221B
+ """
if self.__name:
return self.__name
elif self.__parent:
@@ -628,45 +840,77 @@ class ParseResults(object): return None
elif (len(self) == 1 and
len(self.__tokdict) == 1 and
- self.__tokdict.values()[0][0][1] in (0,-1)):
- return self.__tokdict.keys()[0]
+ next(iter(self.__tokdict.values()))[0][1] in (0,-1)):
+ return next(iter(self.__tokdict.keys()))
else:
return None
- def dump(self,indent='',depth=0):
- """Diagnostic method for listing out the contents of a C{ParseResults}.
- Accepts an optional C{indent} argument so that this string can be embedded
- in a nested display of other data."""
+ def dump(self, indent='', depth=0, full=True):
+ """
+ Diagnostic method for listing out the contents of a C{ParseResults}.
+ Accepts an optional C{indent} argument so that this string can be embedded
+ in a nested display of other data.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ result = date_str.parseString('12/31/1999')
+ print(result.dump())
+ prints::
+ ['12', '/', '31', '/', '1999']
+ - day: 1999
+ - month: 31
+ - year: 12
+ """
out = []
NL = '\n'
out.append( indent+_ustr(self.asList()) )
- if self.haskeys():
- items = sorted(self.items())
- for k,v in items:
- if out:
- out.append(NL)
- out.append( "%s%s- %s: " % (indent,(' '*depth), k) )
- if isinstance(v,ParseResults):
- if v:
- out.append( v.dump(indent,depth+1) )
+ if full:
+ if self.haskeys():
+ items = sorted(self.items())
+ for k,v in items:
+ if out:
+ out.append(NL)
+ out.append( "%s%s- %s: " % (indent,(' '*depth), k) )
+ if isinstance(v,ParseResults):
+ if v:
+ out.append( v.dump(indent,depth+1) )
+ else:
+ out.append(_ustr(v))
else:
out.append(_ustr(v))
- else:
- out.append(_ustr(v))
- elif any(isinstance(vv,ParseResults) for vv in self):
- v = self
- for i,vv in enumerate(v):
- if isinstance(vv,ParseResults):
- out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) ))
- else:
- out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv)))
+ elif any(isinstance(vv,ParseResults) for vv in self):
+ v = self
+ for i,vv in enumerate(v):
+ if isinstance(vv,ParseResults):
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) ))
+ else:
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv)))
return "".join(out)
def pprint(self, *args, **kwargs):
- """Pretty-printer for parsed results as a list, using the C{pprint} module.
- Accepts additional positional or keyword args as defined for the
- C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint})"""
+ """
+ Pretty-printer for parsed results as a list, using the C{pprint} module.
+ Accepts additional positional or keyword args as defined for the
+ C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint})
+
+ Example::
+ ident = Word(alphas, alphanums)
+ num = Word(nums)
+ func = Forward()
+ term = ident | num | Group('(' + func + ')')
+ func <<= ident + Group(Optional(delimitedList(term)))
+ result = func.parseString("fna a,b,(fnb c,d,200),100")
+ result.pprint(width=40)
+ prints::
+ ['fna',
+ ['a',
+ 'b',
+ ['(', 'fnb', ['c', 'd', '200'], ')'],
+ '100']]
+ """
pprint.pprint(self.asList(), *args, **kwargs)
# add support for pickle protocol
@@ -690,8 +934,11 @@ class ParseResults(object): else:
self.__parent = None
+ def __getnewargs__(self):
+ return self.__toklist, self.__name, self.__asList, self.__modal
+
def __dir__(self):
- return dir(super(ParseResults,self)) + list(self.keys())
+ return (dir(type(self)) + list(self.keys()))
collections.MutableMapping.register(ParseResults)
@@ -771,6 +1018,31 @@ def _trim_arity(func, maxargs=2): return lambda s,l,t: func(t)
limit = [0]
foundArity = [False]
+
+ # traceback return data structure changed in Py3.5 - normalize back to plain tuples
+ if system_version[:2] >= (3,5):
+ def extract_stack(limit=0):
+ # special handling for Python 3.5.0 - extra deep call stack by 1
+ offset = -3 if system_version == (3,5,0) else -2
+ frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset]
+ return [(frame_summary.filename, frame_summary.lineno)]
+ def extract_tb(tb, limit=0):
+ frames = traceback.extract_tb(tb, limit=limit)
+ frame_summary = frames[-1]
+ return [(frame_summary.filename, frame_summary.lineno)]
+ else:
+ extract_stack = traceback.extract_stack
+ extract_tb = traceback.extract_tb
+
+ # synthesize what would be returned by traceback.extract_stack at the call to
+ # user's parse action 'func', so that we don't incur call penalty at parse time
+
+ LINE_DIFF = 6
+ # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND
+ # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!
+ this_line = extract_stack(limit=2)[-1]
+ pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF)
+
def wrapper(*args):
while 1:
try:
@@ -778,12 +1050,33 @@ def _trim_arity(func, maxargs=2): foundArity[0] = True
return ret
except TypeError:
- if limit[0] <= maxargs and not foundArity[0]:
+ # re-raise TypeErrors if they did not come from our arity testing
+ if foundArity[0]:
+ raise
+ else:
+ try:
+ tb = sys.exc_info()[-1]
+ if not extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth:
+ raise
+ finally:
+ del tb
+
+ if limit[0] <= maxargs:
limit[0] += 1
continue
raise
+
+ # copy func name to wrapper for sensible debug output
+ func_name = "<parse action>"
+ try:
+ func_name = getattr(func, '__name__',
+ getattr(func, '__class__').__name__)
+ except Exception:
+ func_name = str(func)
+ wrapper.__name__ = func_name
+
return wrapper
-
+
class ParserElement(object):
"""Abstract base level parser element class."""
DEFAULT_WHITE_CHARS = " \n\t\r"
@@ -791,7 +1084,16 @@ class ParserElement(object): @staticmethod
def setDefaultWhitespaceChars( chars ):
- """Overrides the default whitespace chars
+ r"""
+ Overrides the default whitespace chars
+
+ Example::
+ # default whitespace chars are space, <TAB> and newline
+ OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl']
+
+ # change to just treat newline as significant
+ ParserElement.setDefaultWhitespaceChars(" \t")
+ OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def']
"""
ParserElement.DEFAULT_WHITE_CHARS = chars
@@ -799,8 +1101,22 @@ class ParserElement(object): def inlineLiteralsUsing(cls):
"""
Set class to be used for inclusion of string literals into a parser.
+
+ Example::
+ # default literal class used is Literal
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31']
+
+
+ # change to Suppress
+ ParserElement.inlineLiteralsUsing(Suppress)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ date_str.parseString("1999/12/31") # -> ['1999', '12', '31']
"""
- ParserElement.literalStringClass = cls
+ ParserElement._literalStringClass = cls
def __init__( self, savelist=False ):
self.parseAction = list()
@@ -826,8 +1142,21 @@ class ParserElement(object): self.callDuringTry = False
def copy( self ):
- """Make a copy of this C{ParserElement}. Useful for defining different parse actions
- for the same parsing pattern, using copies of the original parse element."""
+ """
+ Make a copy of this C{ParserElement}. Useful for defining different parse actions
+ for the same parsing pattern, using copies of the original parse element.
+
+ Example::
+ integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K")
+ integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+
+ print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M"))
+ prints::
+ [5120, 100, 655360, 268435456]
+ Equivalent form of C{expr.copy()} is just C{expr()}::
+ integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+ """
cpy = copy.copy( self )
cpy.parseAction = self.parseAction[:]
cpy.ignoreExprs = self.ignoreExprs[:]
@@ -836,7 +1165,13 @@ class ParserElement(object): return cpy
def setName( self, name ):
- """Define name for this expression, for use in debugging."""
+ """
+ Define name for this expression, makes debugging and exception messages clearer.
+
+ Example::
+ Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1)
+ Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1)
+ """
self.name = name
self.errmsg = "Expected " + self.name
if hasattr(self,"exception"):
@@ -844,15 +1179,24 @@ class ParserElement(object): return self
def setResultsName( self, name, listAllMatches=False ):
- """Define name for referencing matching tokens as a nested attribute
- of the returned parse results.
- NOTE: this returns a *copy* of the original C{ParserElement} object;
- this is so that the client can define a basic element, such as an
- integer, and reference it in multiple places with different names.
-
- You can also set results names using the abbreviated syntax,
- C{expr("name")} in place of C{expr.setResultsName("name")} -
- see L{I{__call__}<__call__>}.
+ """
+ Define name for referencing matching tokens as a nested attribute
+ of the returned parse results.
+ NOTE: this returns a *copy* of the original C{ParserElement} object;
+ this is so that the client can define a basic element, such as an
+ integer, and reference it in multiple places with different names.
+
+ You can also set results names using the abbreviated syntax,
+ C{expr("name")} in place of C{expr.setResultsName("name")} -
+ see L{I{__call__}<__call__>}.
+
+ Example::
+ date_str = (integer.setResultsName("year") + '/'
+ + integer.setResultsName("month") + '/'
+ + integer.setResultsName("day"))
+
+ # equivalent form:
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
"""
newself = self.copy()
if name.endswith("*"):
@@ -881,42 +1225,76 @@ class ParserElement(object): return self
def setParseAction( self, *fns, **kwargs ):
- """Define action to perform when successfully matching parse element definition.
- Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)},
- C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
- - s = the original string being parsed (see note below)
- - loc = the location of the matching substring
- - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object
- If the functions in fns modify the tokens, they can return them as the return
- value from fn, and the modified list of tokens will replace the original.
- Otherwise, fn does not need to return any value.
-
- Note: the default parsing behavior is to expand tabs in the input string
- before starting the parsing process. See L{I{parseString}<parseString>} for more information
- on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
- consistent view of the parsed string, the parse location, and line and column
- positions within the parsed string.
- """
+ """
+ Define action to perform when successfully matching parse element definition.
+ Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)},
+ C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
+ - s = the original string being parsed (see note below)
+ - loc = the location of the matching substring
+ - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object
+ If the functions in fns modify the tokens, they can return them as the return
+ value from fn, and the modified list of tokens will replace the original.
+ Otherwise, fn does not need to return any value.
+
+ Optional keyword arguments:
+ - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{parseString}<parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer + '/' + integer + '/' + integer
+
+ date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31']
+
+ # use parse action to convert to ints at parse time
+ integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ date_str = integer + '/' + integer + '/' + integer
+
+ # note that integer fields are now ints, not strings
+ date_str.parseString("1999/12/31") # -> [1999, '/', 12, '/', 31]
+ """
self.parseAction = list(map(_trim_arity, list(fns)))
self.callDuringTry = kwargs.get("callDuringTry", False)
return self
def addParseAction( self, *fns, **kwargs ):
- """Add parse action to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}."""
+ """
+ Add parse action to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}.
+
+ See examples in L{I{copy}<copy>}.
+ """
self.parseAction += list(map(_trim_arity, list(fns)))
self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
return self
def addCondition(self, *fns, **kwargs):
"""Add a boolean predicate function to expression's list of parse actions. See
- L{I{setParseAction}<setParseAction>}. Optional keyword argument C{message} can
- be used to define a custom message to be used in the raised exception."""
- msg = kwargs.get("message") or "failed user-defined condition"
+ L{I{setParseAction}<setParseAction>} for function call signatures. Unlike C{setParseAction},
+ functions passed to C{addCondition} need to return boolean success/fail of the condition.
+
+ Optional keyword arguments:
+ - message = define a custom message to be used in the raised exception
+ - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException
+
+ Example::
+ integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ year_int = integer.copy()
+ year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later")
+ date_str = year_int + '/' + integer + '/' + integer
+
+ result = date_str.parseString("1999/12/31") # -> Exception: Only support years 2000 and later (at char 0), (line:1, col:1)
+ """
+ msg = kwargs.get("message", "failed user-defined condition")
+ exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException
for fn in fns:
def pa(s,l,t):
if not bool(_trim_arity(fn)(s,l,t)):
- raise ParseException(s,l,msg)
- return t
+ raise exc_type(s,l,msg)
self.parseAction.append(pa)
self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
return self
@@ -1043,43 +1421,132 @@ class ParserElement(object): return self._parse( instring, loc, doActions=False )[0]
except ParseFatalException:
raise ParseException( instring, loc, self.errmsg, self)
+
+ def canParseNext(self, instring, loc):
+ try:
+ self.tryParse(instring, loc)
+ except (ParseException, IndexError):
+ return False
+ else:
+ return True
+
+ class _UnboundedCache(object):
+ def __init__(self):
+ cache = {}
+ self.not_in_cache = not_in_cache = object()
+
+ def get(self, key):
+ return cache.get(key, not_in_cache)
+
+ def set(self, key, value):
+ cache[key] = value
+
+ def clear(self):
+ cache.clear()
+
+ self.get = types.MethodType(get, self)
+ self.set = types.MethodType(set, self)
+ self.clear = types.MethodType(clear, self)
+
+ if _OrderedDict is not None:
+ class _FifoCache(object):
+ def __init__(self, size):
+ self.not_in_cache = not_in_cache = object()
+
+ cache = _OrderedDict()
+
+ def get(self, key):
+ return cache.get(key, not_in_cache)
+
+ def set(self, key, value):
+ cache[key] = value
+ if len(cache) > size:
+ cache.popitem(False)
+
+ def clear(self):
+ cache.clear()
+
+ self.get = types.MethodType(get, self)
+ self.set = types.MethodType(set, self)
+ self.clear = types.MethodType(clear, self)
+
+ else:
+ class _FifoCache(object):
+ def __init__(self, size):
+ self.not_in_cache = not_in_cache = object()
+
+ cache = {}
+ key_fifo = collections.deque([], size)
+
+ def get(self, key):
+ return cache.get(key, not_in_cache)
+
+ def set(self, key, value):
+ cache[key] = value
+ if len(cache) > size:
+ cache.pop(key_fifo.popleft(), None)
+ key_fifo.append(key)
+
+ def clear(self):
+ cache.clear()
+ key_fifo.clear()
+
+ self.get = types.MethodType(get, self)
+ self.set = types.MethodType(set, self)
+ self.clear = types.MethodType(clear, self)
+
+ # argument cache for optimizing repeated calls when backtracking through recursive expressions
+ packrat_cache = {} # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail
+ packrat_cache_lock = RLock()
+ packrat_cache_stats = [0, 0]
# this method gets repeatedly called during backtracking with the same arguments -
# we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
- lookup = (self,instring,loc,callPreParse,doActions)
- if lookup in ParserElement._exprArgCache:
- value = ParserElement._exprArgCache[ lookup ]
- if isinstance(value, Exception):
- raise value
- return (value[0],value[1].copy())
- else:
- try:
- value = self._parseNoCache( instring, loc, doActions, callPreParse )
- ParserElement._exprArgCache[ lookup ] = (value[0],value[1].copy())
- return value
- except ParseBaseException as pe:
- pe.__traceback__ = None
- ParserElement._exprArgCache[ lookup ] = pe
- raise
+ HIT, MISS = 0, 1
+ lookup = (self, instring, loc, callPreParse, doActions)
+ with ParserElement.packrat_cache_lock:
+ cache = ParserElement.packrat_cache
+ value = cache.get(lookup)
+ if value is cache.not_in_cache:
+ ParserElement.packrat_cache_stats[MISS] += 1
+ try:
+ value = self._parseNoCache(instring, loc, doActions, callPreParse)
+ except ParseBaseException as pe:
+ # cache a copy of the exception, without the traceback
+ cache.set(lookup, pe.__class__(*pe.args))
+ raise
+ else:
+ cache.set(lookup, (value[0], value[1].copy()))
+ return value
+ else:
+ ParserElement.packrat_cache_stats[HIT] += 1
+ if isinstance(value, Exception):
+ raise value
+ return (value[0], value[1].copy())
_parse = _parseNoCache
- # argument cache for optimizing repeated calls when backtracking through recursive expressions
- _exprArgCache = {}
@staticmethod
def resetCache():
- ParserElement._exprArgCache.clear()
+ ParserElement.packrat_cache.clear()
+ ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats)
_packratEnabled = False
@staticmethod
- def enablePackrat():
+ def enablePackrat(cache_size_limit=128):
"""Enables "packrat" parsing, which adds memoizing to the parsing logic.
Repeated parse attempts at the same string location (which happens
often in many complex grammars) can immediately return a cached value,
instead of re-executing parsing/validating code. Memoizing is done of
both valid results and parsing exceptions.
-
+
+ Parameters:
+ - cache_size_limit - (default=C{128}) - if an integer value is provided
+ will limit the size of the packrat cache; if None is passed, then
+ the cache size will be unbounded; if 0 is passed, the cache will
+ be effectively disabled.
+
This speedup may break existing programs that use parse actions that
have side-effects. For this reason, packrat parsing is disabled when
you first import pyparsing. To activate the packrat feature, your
@@ -1088,32 +1555,45 @@ class ParserElement(object): C{enablePackrat} before calling C{psyco.full()}. If you do not do this,
Python will crash. For best results, call C{enablePackrat()} immediately
after importing pyparsing.
+
+ Example::
+ import pyparsing
+ pyparsing.ParserElement.enablePackrat()
"""
if not ParserElement._packratEnabled:
ParserElement._packratEnabled = True
+ if cache_size_limit is None:
+ ParserElement.packrat_cache = ParserElement._UnboundedCache()
+ else:
+ ParserElement.packrat_cache = ParserElement._FifoCache(cache_size_limit)
ParserElement._parse = ParserElement._parseCache
def parseString( self, instring, parseAll=False ):
- """Execute the parse expression with the given string.
- This is the main interface to the client code, once the complete
- expression has been built.
-
- If you want the grammar to require that the entire input string be
- successfully parsed, then set C{parseAll} to True (equivalent to ending
- the grammar with C{L{StringEnd()}}).
-
- Note: C{parseString} implicitly calls C{expandtabs()} on the input string,
- in order to report proper column numbers in parse actions.
- If the input string contains tabs and
- the grammar uses parse actions that use the C{loc} argument to index into the
- string being parsed, you can ensure you have a consistent view of the input
- string by:
- - calling C{parseWithTabs} on your grammar before calling C{parseString}
- (see L{I{parseWithTabs}<parseWithTabs>})
- - define your parse action using the full C{(s,loc,toks)} signature, and
- reference the input string using the parse action's C{s} argument
- - explictly expand the tabs in your input string before calling
- C{parseString}
+ """
+ Execute the parse expression with the given string.
+ This is the main interface to the client code, once the complete
+ expression has been built.
+
+ If you want the grammar to require that the entire input string be
+ successfully parsed, then set C{parseAll} to True (equivalent to ending
+ the grammar with C{L{StringEnd()}}).
+
+ Note: C{parseString} implicitly calls C{expandtabs()} on the input string,
+ in order to report proper column numbers in parse actions.
+ If the input string contains tabs and
+ the grammar uses parse actions that use the C{loc} argument to index into the
+ string being parsed, you can ensure you have a consistent view of the input
+ string by:
+ - calling C{parseWithTabs} on your grammar before calling C{parseString}
+ (see L{I{parseWithTabs}<parseWithTabs>})
+ - define your parse action using the full C{(s,loc,toks)} signature, and
+ reference the input string using the parse action's C{s} argument
+ - explictly expand the tabs in your input string before calling
+ C{parseString}
+
+ Example::
+ Word('a').parseString('aaaaabaaa') # -> ['aaaaa']
+ Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text
"""
ParserElement.resetCache()
if not self.streamlined:
@@ -1139,14 +1619,35 @@ class ParserElement(object): return tokens
def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ):
- """Scan the input string for expression matches. Each match will return the
- matching tokens, start location, and end location. May be called with optional
- C{maxMatches} argument, to clip scanning after 'n' matches are found. If
- C{overlap} is specified, then overlapping matches will be reported.
-
- Note that the start and end locations are reported relative to the string
- being parsed. See L{I{parseString}<parseString>} for more information on parsing
- strings with embedded tabs."""
+ """
+ Scan the input string for expression matches. Each match will return the
+ matching tokens, start location, and end location. May be called with optional
+ C{maxMatches} argument, to clip scanning after 'n' matches are found. If
+ C{overlap} is specified, then overlapping matches will be reported.
+
+ Note that the start and end locations are reported relative to the string
+ being parsed. See L{I{parseString}<parseString>} for more information on parsing
+ strings with embedded tabs.
+
+ Example::
+ source = "sldjf123lsdjjkf345sldkjf879lkjsfd987"
+ print(source)
+ for tokens,start,end in Word(alphas).scanString(source):
+ print(' '*start + '^'*(end-start))
+ print(' '*start + tokens[0])
+
+ prints::
+
+ sldjf123lsdjjkf345sldkjf879lkjsfd987
+ ^^^^^
+ sldjf
+ ^^^^^^^
+ lsdjjkf
+ ^^^^^^
+ sldkjf
+ ^^^^^^
+ lkjsfd
+ """
if not self.streamlined:
self.streamline()
for e in self.ignoreExprs:
@@ -1189,12 +1690,22 @@ class ParserElement(object): raise exc
def transformString( self, instring ):
- """Extension to C{L{scanString}}, to modify matching text with modified tokens that may
- be returned from a parse action. To use C{transformString}, define a grammar and
- attach a parse action to it that modifies the returned token list.
- Invoking C{transformString()} on a target string will then scan for matches,
- and replace the matched text patterns according to the logic in the parse
- action. C{transformString()} returns the resulting transformed string."""
+ """
+ Extension to C{L{scanString}}, to modify matching text with modified tokens that may
+ be returned from a parse action. To use C{transformString}, define a grammar and
+ attach a parse action to it that modifies the returned token list.
+ Invoking C{transformString()} on a target string will then scan for matches,
+ and replace the matched text patterns according to the logic in the parse
+ action. C{transformString()} returns the resulting transformed string.
+
+ Example::
+ wd = Word(alphas)
+ wd.setParseAction(lambda toks: toks[0].title())
+
+ print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york."))
+ Prints::
+ Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York.
+ """
out = []
lastE = 0
# force preservation of <TAB>s, to minimize unwanted transformation of string, and to
@@ -1222,9 +1733,18 @@ class ParserElement(object): raise exc
def searchString( self, instring, maxMatches=_MAX_INT ):
- """Another extension to C{L{scanString}}, simplifying the access to the tokens found
- to match the given parse expression. May be called with optional
- C{maxMatches} argument, to clip searching after 'n' matches are found.
+ """
+ Another extension to C{L{scanString}}, simplifying the access to the tokens found
+ to match the given parse expression. May be called with optional
+ C{maxMatches} argument, to clip searching after 'n' matches are found.
+
+ Example::
+ # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters
+ cap_word = Word(alphas.upper(), alphas.lower())
+
+ print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))
+ prints::
+ ['More', 'Iron', 'Lead', 'Gold', 'I']
"""
try:
return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
@@ -1235,10 +1755,42 @@ class ParserElement(object): # catch and re-raise exception from here, clears out pyparsing internal stack trace
raise exc
+ def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False):
+ """
+ Generator method to split a string using the given expression as a separator.
+ May be called with optional C{maxsplit} argument, to limit the number of splits;
+ and the optional C{includeSeparators} argument (default=C{False}), if the separating
+ matching text should be included in the split results.
+
+ Example::
+ punc = oneOf(list(".,;:/-!?"))
+ print(list(punc.split("This, this?, this sentence, is badly punctuated!")))
+ prints::
+ ['This', ' this', '', ' this sentence', ' is badly punctuated', '']
+ """
+ splits = 0
+ last = 0
+ for t,s,e in self.scanString(instring, maxMatches=maxsplit):
+ yield instring[last:s]
+ if includeSeparators:
+ yield t[0]
+ last = e
+ yield instring[last:]
+
def __add__(self, other ):
- """Implementation of + operator - returns C{L{And}}"""
+ """
+ Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement
+ converts them to L{Literal}s by default.
+
+ Example::
+ greet = Word(alphas) + "," + Word(alphas) + "!"
+ hello = "Hello, World!"
+ print (hello, "->", greet.parseString(hello))
+ Prints::
+ Hello, World! -> ['Hello', ',', 'World', '!']
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1246,9 +1798,11 @@ class ParserElement(object): return And( [ self, other ] )
def __radd__(self, other ):
- """Implementation of + operator when left operand is not a C{L{ParserElement}}"""
+ """
+ Implementation of + operator when left operand is not a C{L{ParserElement}}
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1256,9 +1810,11 @@ class ParserElement(object): return other + self
def __sub__(self, other):
- """Implementation of - operator, returns C{L{And}} with error stop"""
+ """
+ Implementation of - operator, returns C{L{And}} with error stop
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1266,9 +1822,11 @@ class ParserElement(object): return And( [ self, And._ErrorStop(), other ] )
def __rsub__(self, other ):
- """Implementation of - operator when left operand is not a C{L{ParserElement}}"""
+ """
+ Implementation of - operator when left operand is not a C{L{ParserElement}}
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1276,24 +1834,24 @@ class ParserElement(object): return other - self
def __mul__(self,other):
- """Implementation of * operator, allows use of C{expr * 3} in place of
- C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer
- tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples
- may also include C{None} as in:
- - C{expr*(n,None)} or C{expr*(n,)} is equivalent
+ """
+ Implementation of * operator, allows use of C{expr * 3} in place of
+ C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer
+ tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples
+ may also include C{None} as in:
+ - C{expr*(n,None)} or C{expr*(n,)} is equivalent
to C{expr*n + L{ZeroOrMore}(expr)}
(read as "at least n instances of C{expr}")
- - C{expr*(None,n)} is equivalent to C{expr*(0,n)}
+ - C{expr*(None,n)} is equivalent to C{expr*(0,n)}
(read as "0 to n instances of C{expr}")
- - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)}
- - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)}
-
- Note that C{expr*(None,n)} does not raise an exception if
- more than n exprs exist in the input stream; that is,
- C{expr*(None,n)} does not enforce a maximum number of expr
- occurrences. If this behavior is desired, then write
- C{expr*(None,n) + ~expr}
-
+ - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)}
+ - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)}
+
+ Note that C{expr*(None,n)} does not raise an exception if
+ more than n exprs exist in the input stream; that is,
+ C{expr*(None,n)} does not enforce a maximum number of expr
+ occurrences. If this behavior is desired, then write
+ C{expr*(None,n) + ~expr}
"""
if isinstance(other,int):
minElements, optElements = other,0
@@ -1347,9 +1905,11 @@ class ParserElement(object): return self.__mul__(other)
def __or__(self, other ):
- """Implementation of | operator - returns C{L{MatchFirst}}"""
+ """
+ Implementation of | operator - returns C{L{MatchFirst}}
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1357,9 +1917,11 @@ class ParserElement(object): return MatchFirst( [ self, other ] )
def __ror__(self, other ):
- """Implementation of | operator when left operand is not a C{L{ParserElement}}"""
+ """
+ Implementation of | operator when left operand is not a C{L{ParserElement}}
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1367,9 +1929,11 @@ class ParserElement(object): return other | self
def __xor__(self, other ):
- """Implementation of ^ operator - returns C{L{Or}}"""
+ """
+ Implementation of ^ operator - returns C{L{Or}}
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1377,9 +1941,11 @@ class ParserElement(object): return Or( [ self, other ] )
def __rxor__(self, other ):
- """Implementation of ^ operator when left operand is not a C{L{ParserElement}}"""
+ """
+ Implementation of ^ operator when left operand is not a C{L{ParserElement}}
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1387,9 +1953,11 @@ class ParserElement(object): return other ^ self
def __and__(self, other ):
- """Implementation of & operator - returns C{L{Each}}"""
+ """
+ Implementation of & operator - returns C{L{Each}}
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1397,9 +1965,11 @@ class ParserElement(object): return Each( [ self, other ] )
def __rand__(self, other ):
- """Implementation of & operator when left operand is not a C{L{ParserElement}}"""
+ """
+ Implementation of & operator when left operand is not a C{L{ParserElement}}
+ """
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
if not isinstance( other, ParserElement ):
warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
@@ -1407,41 +1977,49 @@ class ParserElement(object): return other & self
def __invert__( self ):
- """Implementation of ~ operator - returns C{L{NotAny}}"""
+ """
+ Implementation of ~ operator - returns C{L{NotAny}}
+ """
return NotAny( self )
def __call__(self, name=None):
- """Shortcut for C{L{setResultsName}}, with C{listAllMatches=default}::
- userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
- could be written as::
- userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
-
- If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be
- passed as C{True}.
+ """
+ Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}.
+
+ If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be
+ passed as C{True}.
- If C{name} is omitted, same as calling C{L{copy}}.
- """
+ If C{name} is omitted, same as calling C{L{copy}}.
+
+ Example::
+ # these are equivalent
+ userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
+ userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
+ """
if name is not None:
return self.setResultsName(name)
else:
return self.copy()
def suppress( self ):
- """Suppresses the output of this C{ParserElement}; useful to keep punctuation from
- cluttering up returned output.
+ """
+ Suppresses the output of this C{ParserElement}; useful to keep punctuation from
+ cluttering up returned output.
"""
return Suppress( self )
def leaveWhitespace( self ):
- """Disables the skipping of whitespace before matching the characters in the
- C{ParserElement}'s defined pattern. This is normally only used internally by
- the pyparsing module, but may be needed in some whitespace-sensitive grammars.
+ """
+ Disables the skipping of whitespace before matching the characters in the
+ C{ParserElement}'s defined pattern. This is normally only used internally by
+ the pyparsing module, but may be needed in some whitespace-sensitive grammars.
"""
self.skipWhitespace = False
return self
def setWhitespaceChars( self, chars ):
- """Overrides the default whitespace chars
+ """
+ Overrides the default whitespace chars
"""
self.skipWhitespace = True
self.whiteChars = chars
@@ -1449,26 +2027,41 @@ class ParserElement(object): return self
def parseWithTabs( self ):
- """Overrides default behavior to expand C{<TAB>}s to spaces before parsing the input string.
- Must be called before C{parseString} when the input grammar contains elements that
- match C{<TAB>} characters."""
+ """
+ Overrides default behavior to expand C{<TAB>}s to spaces before parsing the input string.
+ Must be called before C{parseString} when the input grammar contains elements that
+ match C{<TAB>} characters.
+ """
self.keepTabs = True
return self
def ignore( self, other ):
- """Define expression to be ignored (e.g., comments) while doing pattern
- matching; may be called repeatedly, to define multiple comment or other
- ignorable patterns.
"""
+ Define expression to be ignored (e.g., comments) while doing pattern
+ matching; may be called repeatedly, to define multiple comment or other
+ ignorable patterns.
+
+ Example::
+ patt = OneOrMore(Word(alphas))
+ patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj']
+
+ patt.ignore(cStyleComment)
+ patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd']
+ """
+ if isinstance(other, basestring):
+ other = Suppress(other)
+
if isinstance( other, Suppress ):
if other not in self.ignoreExprs:
- self.ignoreExprs.append( other.copy() )
+ self.ignoreExprs.append(other)
else:
self.ignoreExprs.append( Suppress( other.copy() ) )
return self
def setDebugActions( self, startAction, successAction, exceptionAction ):
- """Enable display of debugging messages while doing pattern matching."""
+ """
+ Enable display of debugging messages while doing pattern matching.
+ """
self.debugActions = (startAction or _defaultStartDebugAction,
successAction or _defaultSuccessDebugAction,
exceptionAction or _defaultExceptionDebugAction)
@@ -1476,8 +2069,40 @@ class ParserElement(object): return self
def setDebug( self, flag=True ):
- """Enable display of debugging messages while doing pattern matching.
- Set C{flag} to True to enable, False to disable."""
+ """
+ Enable display of debugging messages while doing pattern matching.
+ Set C{flag} to True to enable, False to disable.
+
+ Example::
+ wd = Word(alphas).setName("alphaword")
+ integer = Word(nums).setName("numword")
+ term = wd | integer
+
+ # turn on debugging for wd
+ wd.setDebug()
+
+ OneOrMore(term).parseString("abc 123 xyz 890")
+
+ prints::
+ Match alphaword at loc 0(1,1)
+ Matched alphaword -> ['abc']
+ Match alphaword at loc 3(1,4)
+ Exception raised:Expected alphaword (at char 4), (line:1, col:5)
+ Match alphaword at loc 7(1,8)
+ Matched alphaword -> ['xyz']
+ Match alphaword at loc 11(1,12)
+ Exception raised:Expected alphaword (at char 12), (line:1, col:13)
+ Match alphaword at loc 15(1,16)
+ Exception raised:Expected alphaword (at char 15), (line:1, col:16)
+
+ The output shown is that produced by the default debug actions - custom debug actions can be
+ specified using L{setDebugActions}. Prior to attempting
+ to match the C{wd} expression, the debugging message C{"Match <exprname> at loc <n>(<line>,<col>)"}
+ is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"}
+ message is shown. Also note the use of L{setName} to assign a human-readable name to the expression,
+ which makes debugging and exception messages easier to understand - for instance, the default
+ name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}.
+ """
if flag:
self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
else:
@@ -1499,20 +2124,22 @@ class ParserElement(object): pass
def validate( self, validateTrace=[] ):
- """Check defined expressions for valid structure, check for infinite recursive definitions."""
+ """
+ Check defined expressions for valid structure, check for infinite recursive definitions.
+ """
self.checkRecursion( [] )
def parseFile( self, file_or_filename, parseAll=False ):
- """Execute the parse expression on the given file or filename.
- If a filename is specified (instead of a file object),
- the entire file is opened, read, and closed before parsing.
+ """
+ Execute the parse expression on the given file or filename.
+ If a filename is specified (instead of a file object),
+ the entire file is opened, read, and closed before parsing.
"""
try:
file_contents = file_or_filename.read()
except AttributeError:
- f = open(file_or_filename, "r")
- file_contents = f.read()
- f.close()
+ with open(file_or_filename, "r") as f:
+ file_contents = f.read()
try:
return self.parseString(file_contents, parseAll)
except ParseBaseException as exc:
@@ -1524,13 +2151,9 @@ class ParserElement(object): def __eq__(self,other):
if isinstance(other, ParserElement):
- return self is other or self.__dict__ == other.__dict__
+ return self is other or vars(self) == vars(other)
elif isinstance(other, basestring):
- try:
- self.parseString(_ustr(other), parseAll=True)
- return True
- except ParseBaseException:
- return False
+ return self.matches(other)
else:
return super(ParserElement,self)==other
@@ -1546,40 +2169,161 @@ class ParserElement(object): def __rne__(self,other):
return not (self == other)
- def runTests(self, tests, parseAll=False):
- """Execute the parse expression on a series of test strings, showing each
- test, the parsed results or where the parse failed. Quick and easy way to
- run a parse expression against a list of sample strings.
+ def matches(self, testString, parseAll=True):
+ """
+ Method for quick testing of a parser against a test string. Good for simple
+ inline microtests of sub expressions while building up larger parser.0
- Parameters:
- - tests - a list of separate test strings, or a multiline string of test strings
- - parseAll - (default=False) - flag to pass to C{L{parseString}} when running tests
+ Parameters:
+ - testString - to test against this expression for a match
+ - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests
+
+ Example::
+ expr = Word(nums)
+ assert expr.matches("100")
+ """
+ try:
+ self.parseString(_ustr(testString), parseAll=parseAll)
+ return True
+ except ParseBaseException:
+ return False
+
+ def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False):
+ """
+ Execute the parse expression on a series of test strings, showing each
+ test, the parsed results or where the parse failed. Quick and easy way to
+ run a parse expression against a list of sample strings.
+
+ Parameters:
+ - tests - a list of separate test strings, or a multiline string of test strings
+ - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests
+ - comment - (default=C{'#'}) - expression for indicating embedded comments in the test
+ string; pass None to disable comment filtering
+ - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline;
+ if False, only dump nested list
+ - printResults - (default=C{True}) prints test output to stdout
+ - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing
+
+ Returns: a (success, results) tuple, where success indicates that all tests succeeded
+ (or failed if C{failureTests} is True), and the results contain a list of lines of each
+ test's output
+
+ Example::
+ number_expr = pyparsing_common.number.copy()
+
+ result = number_expr.runTests('''
+ # unsigned integer
+ 100
+ # negative integer
+ -100
+ # float with scientific notation
+ 6.02e23
+ # integer with scientific notation
+ 1e-12
+ ''')
+ print("Success" if result[0] else "Failed!")
+
+ result = number_expr.runTests('''
+ # stray character
+ 100Z
+ # missing leading digit before '.'
+ -.100
+ # too many '.'
+ 3.14.159
+ ''', failureTests=True)
+ print("Success" if result[0] else "Failed!")
+ prints::
+ # unsigned integer
+ 100
+ [100]
+
+ # negative integer
+ -100
+ [-100]
+
+ # float with scientific notation
+ 6.02e23
+ [6.02e+23]
+
+ # integer with scientific notation
+ 1e-12
+ [1e-12]
+
+ Success
+
+ # stray character
+ 100Z
+ ^
+ FAIL: Expected end of text (at char 3), (line:1, col:4)
+
+ # missing leading digit before '.'
+ -.100
+ ^
+ FAIL: Expected {real number with scientific notation | real number | signed integer} (at char 0), (line:1, col:1)
+
+ # too many '.'
+ 3.14.159
+ ^
+ FAIL: Expected end of text (at char 4), (line:1, col:5)
+
+ Success
"""
if isinstance(tests, basestring):
- tests = map(str.strip, tests.splitlines())
+ tests = list(map(str.strip, tests.rstrip().splitlines()))
+ if isinstance(comment, basestring):
+ comment = Literal(comment)
+ allResults = []
+ comments = []
+ success = True
for t in tests:
- out = [t]
+ if comment is not None and comment.matches(t, False) or comments and not t:
+ comments.append(t)
+ continue
+ if not t:
+ continue
+ out = ['\n'.join(comments), t]
+ comments = []
try:
- out.append(self.parseString(t, parseAll=parseAll).dump())
- except ParseException as pe:
+ result = self.parseString(t, parseAll=parseAll)
+ out.append(result.dump(full=fullDump))
+ success = success and not failureTests
+ except ParseBaseException as pe:
+ fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else ""
if '\n' in t:
out.append(line(pe.loc, t))
- out.append(' '*(col(pe.loc,t)-1) + '^')
+ out.append(' '*(col(pe.loc,t)-1) + '^' + fatal)
else:
- out.append(' '*pe.loc + '^')
- out.append(str(pe))
- out.append('')
- print('\n'.join(out))
+ out.append(' '*pe.loc + '^' + fatal)
+ out.append("FAIL: " + str(pe))
+ success = success and failureTests
+ result = pe
+ except Exception as exc:
+ out.append("FAIL-EXCEPTION: " + str(exc))
+ success = success and failureTests
+ result = exc
+
+ if printResults:
+ if fullDump:
+ out.append('')
+ print('\n'.join(out))
+
+ allResults.append((t, result))
+
+ return success, allResults
class Token(ParserElement):
- """Abstract C{ParserElement} subclass, for defining atomic matching patterns."""
+ """
+ Abstract C{ParserElement} subclass, for defining atomic matching patterns.
+ """
def __init__( self ):
super(Token,self).__init__( savelist=False )
class Empty(Token):
- """An empty token, will always match."""
+ """
+ An empty token, will always match.
+ """
def __init__( self ):
super(Empty,self).__init__()
self.name = "Empty"
@@ -1588,7 +2332,9 @@ class Empty(Token): class NoMatch(Token):
- """A token that will never match."""
+ """
+ A token that will never match.
+ """
def __init__( self ):
super(NoMatch,self).__init__()
self.name = "NoMatch"
@@ -1601,7 +2347,19 @@ class NoMatch(Token): class Literal(Token):
- """Token to exactly match a specified string."""
+ """
+ Token to exactly match a specified string.
+
+ Example::
+ Literal('blah').parseString('blah') # -> ['blah']
+ Literal('blah').parseString('blahfooblah') # -> ['blah']
+ Literal('blah').parseString('bla') # -> Exception: Expected "blah"
+
+ For case-insensitive matching, use L{CaselessLiteral}.
+
+ For keyword matching (force word break before and after the matched string),
+ use L{Keyword} or L{CaselessKeyword}.
+ """
def __init__( self, matchString ):
super(Literal,self).__init__()
self.match = matchString
@@ -1627,22 +2385,31 @@ class Literal(Token): return loc+self.matchLen, self.match
raise ParseException(instring, loc, self.errmsg, self)
_L = Literal
-ParserElement.literalStringClass = Literal
+ParserElement._literalStringClass = Literal
class Keyword(Token):
- """Token to exactly match a specified string as a keyword, that is, it must be
- immediately followed by a non-keyword character. Compare with C{L{Literal}}::
- Literal("if") will match the leading C{'if'} in C{'ifAndOnlyIf'}.
- Keyword("if") will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'}
- Accepts two optional constructor arguments in addition to the keyword string:
- C{identChars} is a string of characters that would be valid identifier characters,
- defaulting to all alphanumerics + "_" and "$"; C{caseless} allows case-insensitive
- matching, default is C{False}.
+ """
+ Token to exactly match a specified string as a keyword, that is, it must be
+ immediately followed by a non-keyword character. Compare with C{L{Literal}}:
+ - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}.
+ - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'}
+ Accepts two optional constructor arguments in addition to the keyword string:
+ - C{identChars} is a string of characters that would be valid identifier characters,
+ defaulting to all alphanumerics + "_" and "$"
+ - C{caseless} allows case-insensitive matching, default is C{False}.
+
+ Example::
+ Keyword("start").parseString("start") # -> ['start']
+ Keyword("start").parseString("starting") # -> Exception
+
+ For case-insensitive matching, use L{CaselessKeyword}.
"""
DEFAULT_KEYWORD_CHARS = alphanums+"_$"
- def __init__( self, matchString, identChars=DEFAULT_KEYWORD_CHARS, caseless=False ):
+ def __init__( self, matchString, identChars=None, caseless=False ):
super(Keyword,self).__init__()
+ if identChars is None:
+ identChars = Keyword.DEFAULT_KEYWORD_CHARS
self.match = matchString
self.matchLen = len(matchString)
try:
@@ -1686,9 +2453,15 @@ class Keyword(Token): Keyword.DEFAULT_KEYWORD_CHARS = chars
class CaselessLiteral(Literal):
- """Token to match a specified string, ignoring case of letters.
- Note: the matched results will always be in the case of the given
- match string, NOT the case of the input text.
+ """
+ Token to match a specified string, ignoring case of letters.
+ Note: the matched results will always be in the case of the given
+ match string, NOT the case of the input text.
+
+ Example::
+ OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD']
+
+ (Contrast with example for L{CaselessKeyword}.)
"""
def __init__( self, matchString ):
super(CaselessLiteral,self).__init__( matchString.upper() )
@@ -1703,7 +2476,15 @@ class CaselessLiteral(Literal): raise ParseException(instring, loc, self.errmsg, self)
class CaselessKeyword(Keyword):
- def __init__( self, matchString, identChars=Keyword.DEFAULT_KEYWORD_CHARS ):
+ """
+ Caseless version of L{Keyword}.
+
+ Example::
+ OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD']
+
+ (Contrast with example for L{CaselessLiteral}.)
+ """
+ def __init__( self, matchString, identChars=None ):
super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )
def parseImpl( self, instring, loc, doActions=True ):
@@ -1712,17 +2493,113 @@ class CaselessKeyword(Keyword): return loc+self.matchLen, self.match
raise ParseException(instring, loc, self.errmsg, self)
+class CloseMatch(Token):
+ """
+ A variation on L{Literal} which matches "close" matches, that is,
+ strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters:
+ - C{match_string} - string to be matched
+ - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match
+
+ The results from a successful parse will contain the matched text from the input string and the following named results:
+ - C{mismatches} - a list of the positions within the match_string where mismatches were found
+ - C{original} - the original match_string used to compare against the input string
+
+ If C{mismatches} is an empty list, then the match was an exact match.
+
+ Example::
+ patt = CloseMatch("ATCATCGAATGGA")
+ patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']})
+ patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1)
+
+ # exact match
+ patt.parseString("ATCATCGAATGGA") # -> (['ATCATCGAATGGA'], {'mismatches': [[]], 'original': ['ATCATCGAATGGA']})
+
+ # close match allowing up to 2 mismatches
+ patt = CloseMatch("ATCATCGAATGGA", maxMismatches=2)
+ patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']})
+ """
+ def __init__(self, match_string, maxMismatches=1):
+ super(CloseMatch,self).__init__()
+ self.name = match_string
+ self.match_string = match_string
+ self.maxMismatches = maxMismatches
+ self.errmsg = "Expected %r (with up to %d mismatches)" % (self.match_string, self.maxMismatches)
+ self.mayIndexError = False
+ self.mayReturnEmpty = False
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ start = loc
+ instrlen = len(instring)
+ maxloc = start + len(self.match_string)
+
+ if maxloc <= instrlen:
+ match_string = self.match_string
+ match_stringloc = 0
+ mismatches = []
+ maxMismatches = self.maxMismatches
+
+ for match_stringloc,s_m in enumerate(zip(instring[loc:maxloc], self.match_string)):
+ src,mat = s_m
+ if src != mat:
+ mismatches.append(match_stringloc)
+ if len(mismatches) > maxMismatches:
+ break
+ else:
+ loc = match_stringloc + 1
+ results = ParseResults([instring[start:loc]])
+ results['original'] = self.match_string
+ results['mismatches'] = mismatches
+ return loc, results
+
+ raise ParseException(instring, loc, self.errmsg, self)
+
+
class Word(Token):
- """Token for matching words composed of allowed character sets.
- Defined with string containing all allowed initial characters,
- an optional string containing allowed body characters (if omitted,
- defaults to the initial character set), and an optional minimum,
- maximum, and/or exact length. The default value for C{min} is 1 (a
- minimum value < 1 is not valid); the default values for C{max} and C{exact}
- are 0, meaning no maximum or exact length restriction. An optional
- C{exclude} parameter can list characters that might be found in
- the input C{bodyChars} string; useful to define a word of all printables
- except for one or two characters, for instance.
+ """
+ Token for matching words composed of allowed character sets.
+ Defined with string containing all allowed initial characters,
+ an optional string containing allowed body characters (if omitted,
+ defaults to the initial character set), and an optional minimum,
+ maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction. An optional
+ C{excludeChars} parameter can list characters that might be found in
+ the input C{bodyChars} string; useful to define a word of all printables
+ except for one or two characters, for instance.
+
+ L{srange} is useful for defining custom character set strings for defining
+ C{Word} expressions, using range notation from regular expression character sets.
+
+ A common mistake is to use C{Word} to match a specific literal string, as in
+ C{Word("Address")}. Remember that C{Word} uses the string argument to define
+ I{sets} of matchable characters. This expression would match "Add", "AAA",
+ "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'.
+ To match an exact literal string, use L{Literal} or L{Keyword}.
+
+ pyparsing includes helper strings for building Words:
+ - L{alphas}
+ - L{nums}
+ - L{alphanums}
+ - L{hexnums}
+ - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.)
+ - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.)
+ - L{printables} (any non-whitespace character)
+
+ Example::
+ # a word composed of digits
+ integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9"))
+
+ # a word with a leading capital, and zero or more lowercase
+ capital_word = Word(alphas.upper(), alphas.lower())
+
+ # hostnames are alphanumeric, with leading alpha, and '-'
+ hostname = Word(alphas, alphanums+'-')
+
+ # roman numeral (not a strict parser, accepts invalid mix of characters)
+ roman = Word("IVXLCDM")
+
+ # any string of non-whitespace characters, except for ','
+ csv_value = Word(printables, excludeChars=",")
"""
def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ):
super(Word,self).__init__()
@@ -1837,8 +2714,17 @@ class Word(Token): class Regex(Token):
- """Token for matching strings that match a given regular expression.
- Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
+ """
+ Token for matching strings that match a given regular expression.
+ Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
+ If the given regex contains named groups (defined using C{(?P<name>...)}), these will be preserved as
+ named parse results.
+
+ Example::
+ realnum = Regex(r"[+-]?\d+\.\d*")
+ date = Regex(r'(?P<year>\d{4})-(?P<month>\d\d?)-(?P<day>\d\d?)')
+ # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression
+ roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})")
"""
compiledREtype = type(re.compile("[A-Z]"))
def __init__( self, pattern, flags=0):
@@ -1846,7 +2732,7 @@ class Regex(Token): super(Regex,self).__init__()
if isinstance(pattern, basestring):
- if len(pattern) == 0:
+ if not pattern:
warnings.warn("null string passed to Regex; use Empty() instead",
SyntaxWarning, stacklevel=2)
@@ -1901,23 +2787,36 @@ class Regex(Token): class QuotedString(Token):
- """Token for matching strings that are delimited by quoting characters.
+ r"""
+ Token for matching strings that are delimited by quoting characters.
+
+ Defined with the following parameters:
+ - quoteChar - string of one or more characters defining the quote delimiting string
+ - escChar - character to escape quotes, typically backslash (default=C{None})
+ - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None})
+ - multiline - boolean indicating whether quotes can span multiple lines (default=C{False})
+ - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True})
+ - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar)
+ - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True})
+
+ Example::
+ qs = QuotedString('"')
+ print(qs.searchString('lsjdf "This is the quote" sldjf'))
+ complex_qs = QuotedString('{{', endQuoteChar='}}')
+ print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf'))
+ sql_qs = QuotedString('"', escQuote='""')
+ print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf'))
+ prints::
+ [['This is the quote']]
+ [['This is the "quote"']]
+ [['This is the quote with "embedded" quotes']]
"""
- def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None):
- """
- Defined with the following parameters:
- - quoteChar - string of one or more characters defining the quote delimiting string
- - escChar - character to escape quotes, typically backslash (default=None)
- - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=None)
- - multiline - boolean indicating whether quotes can span multiple lines (default=C{False})
- - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True})
- - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar)
- """
+ def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True):
super(QuotedString,self).__init__()
# remove white space from quote chars - wont work anyway
quoteChar = quoteChar.strip()
- if len(quoteChar) == 0:
+ if not quoteChar:
warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
raise SyntaxError()
@@ -1925,7 +2824,7 @@ class QuotedString(Token): endQuoteChar = quoteChar
else:
endQuoteChar = endQuoteChar.strip()
- if len(endQuoteChar) == 0:
+ if not endQuoteChar:
warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
raise SyntaxError()
@@ -1937,6 +2836,7 @@ class QuotedString(Token): self.escChar = escChar
self.escQuote = escQuote
self.unquoteResults = unquoteResults
+ self.convertWhitespaceEscapes = convertWhitespaceEscapes
if multiline:
self.flags = re.MULTILINE | re.DOTALL
@@ -1990,6 +2890,17 @@ class QuotedString(Token): ret = ret[self.quoteCharLen:-self.endQuoteCharLen]
if isinstance(ret,basestring):
+ # replace escaped whitespace
+ if '\\' in ret and self.convertWhitespaceEscapes:
+ ws_map = {
+ r'\t' : '\t',
+ r'\n' : '\n',
+ r'\f' : '\f',
+ r'\r' : '\r',
+ }
+ for wslit,wschar in ws_map.items():
+ ret = ret.replace(wslit, wschar)
+
# replace escaped characters
if self.escChar:
ret = re.sub(self.escCharReplacePattern,"\g<1>",ret)
@@ -2013,11 +2924,20 @@ class QuotedString(Token): class CharsNotIn(Token):
- """Token for matching words composed of characters *not* in a given set.
- Defined with string containing all disallowed characters, and an optional
- minimum, maximum, and/or exact length. The default value for C{min} is 1 (a
- minimum value < 1 is not valid); the default values for C{max} and C{exact}
- are 0, meaning no maximum or exact length restriction.
+ """
+ Token for matching words composed of characters I{not} in a given set (will
+ include whitespace in matched characters if not listed in the provided exclusion set - see example).
+ Defined with string containing all disallowed characters, and an optional
+ minimum, maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction.
+
+ Example::
+ # define a comma-separated-value as anything that is not a ','
+ csv_value = CharsNotIn(',')
+ print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213"))
+ prints::
+ ['dkls', 'lsdkjf', 's12 34', '@!#', '213']
"""
def __init__( self, notChars, min=1, max=0, exact=0 ):
super(CharsNotIn,self).__init__()
@@ -2075,11 +2995,13 @@ class CharsNotIn(Token): return self.strRepr
class White(Token):
- """Special matching class for matching whitespace. Normally, whitespace is ignored
- by pyparsing grammars. This class is included when some whitespace structures
- are significant. Define with a string containing the whitespace characters to be
- matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments,
- as defined for the C{L{Word}} class."""
+ """
+ Special matching class for matching whitespace. Normally, whitespace is ignored
+ by pyparsing grammars. This class is included when some whitespace structures
+ are significant. Define with a string containing the whitespace characters to be
+ matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments,
+ as defined for the C{L{Word}} class.
+ """
whiteStrs = {
" " : "<SPC>",
"\t": "<TAB>",
@@ -2131,7 +3053,9 @@ class _PositionToken(Token): self.mayIndexError = False
class GoToColumn(_PositionToken):
- """Token to advance to a specific column of input text; useful for tabular report scraping."""
+ """
+ Token to advance to a specific column of input text; useful for tabular report scraping.
+ """
def __init__( self, colno ):
super(GoToColumn,self).__init__()
self.col = colno
@@ -2154,7 +3078,9 @@ class GoToColumn(_PositionToken): return newloc, ret
class LineStart(_PositionToken):
- """Matches if current position is at the beginning of a line within the parse string"""
+ """
+ Matches if current position is at the beginning of a line within the parse string
+ """
def __init__( self ):
super(LineStart,self).__init__()
self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
@@ -2174,7 +3100,9 @@ class LineStart(_PositionToken): return loc, []
class LineEnd(_PositionToken):
- """Matches if current position is at the end of a line within the parse string"""
+ """
+ Matches if current position is at the end of a line within the parse string
+ """
def __init__( self ):
super(LineEnd,self).__init__()
self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
@@ -2192,7 +3120,9 @@ class LineEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self)
class StringStart(_PositionToken):
- """Matches if current position is at the beginning of the parse string"""
+ """
+ Matches if current position is at the beginning of the parse string
+ """
def __init__( self ):
super(StringStart,self).__init__()
self.errmsg = "Expected start of text"
@@ -2205,7 +3135,9 @@ class StringStart(_PositionToken): return loc, []
class StringEnd(_PositionToken):
- """Matches if current position is at the end of the parse string"""
+ """
+ Matches if current position is at the end of the parse string
+ """
def __init__( self ):
super(StringEnd,self).__init__()
self.errmsg = "Expected end of text"
@@ -2221,11 +3153,12 @@ class StringEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self)
class WordStart(_PositionToken):
- """Matches if the current position is at the beginning of a Word, and
- is not preceded by any character in a given set of C{wordChars}
- (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
- use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of
- the string being parsed, or at the beginning of a line.
+ """
+ Matches if the current position is at the beginning of a Word, and
+ is not preceded by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of
+ the string being parsed, or at the beginning of a line.
"""
def __init__(self, wordChars = printables):
super(WordStart,self).__init__()
@@ -2240,11 +3173,12 @@ class WordStart(_PositionToken): return loc, []
class WordEnd(_PositionToken):
- """Matches if the current position is at the end of a Word, and
- is not followed by any character in a given set of C{wordChars}
- (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
- use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of
- the string being parsed, or at the end of a line.
+ """
+ Matches if the current position is at the end of a Word, and
+ is not followed by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of
+ the string being parsed, or at the end of a line.
"""
def __init__(self, wordChars = printables):
super(WordEnd,self).__init__()
@@ -2262,18 +3196,21 @@ class WordEnd(_PositionToken): class ParseExpression(ParserElement):
- """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
+ """
+ Abstract subclass of ParserElement, for combining and post-processing parsed tokens.
+ """
def __init__( self, exprs, savelist = False ):
super(ParseExpression,self).__init__(savelist)
if isinstance( exprs, _generatorType ):
exprs = list(exprs)
if isinstance( exprs, basestring ):
- self.exprs = [ Literal( exprs ) ]
- elif isinstance( exprs, collections.Sequence ):
+ self.exprs = [ ParserElement._literalStringClass( exprs ) ]
+ elif isinstance( exprs, collections.Iterable ):
+ exprs = list(exprs)
# if sequence of strings provided, wrap with Literal
if all(isinstance(expr, basestring) for expr in exprs):
- exprs = map(Literal, exprs)
+ exprs = map(ParserElement._literalStringClass, exprs)
self.exprs = list(exprs)
else:
try:
@@ -2351,7 +3288,7 @@ class ParseExpression(ParserElement): self.mayReturnEmpty |= other.mayReturnEmpty
self.mayIndexError |= other.mayIndexError
- self.errmsg = "Expected " + str(self)
+ self.errmsg = "Expected " + _ustr(self)
return self
@@ -2371,9 +3308,19 @@ class ParseExpression(ParserElement): return ret
class And(ParseExpression):
- """Requires all given C{ParseExpression}s to be found in the given order.
- Expressions may be separated by whitespace.
- May be constructed using the C{'+'} operator.
+ """
+ Requires all given C{ParseExpression}s to be found in the given order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'+'} operator.
+ May also be constructed using the C{'-'} operator, which will suppress backtracking.
+
+ Example::
+ integer = Word(nums)
+ name_expr = OneOrMore(Word(alphas))
+
+ expr = And([integer("id"),name_expr("name"),integer("age")])
+ # more easily written as:
+ expr = integer("id") + name_expr("name") + integer("age")
"""
class _ErrorStop(Empty):
@@ -2405,9 +3352,9 @@ class And(ParseExpression): raise
except ParseBaseException as pe:
pe.__traceback__ = None
- raise ParseSyntaxException(pe)
+ raise ParseSyntaxException._from_exception(pe)
except IndexError:
- raise ParseSyntaxException( ParseException(instring, len(instring), self.errmsg, self) )
+ raise ParseSyntaxException(instring, len(instring), self.errmsg, self)
else:
loc, exprtokens = e._parse( instring, loc, doActions )
if exprtokens or exprtokens.haskeys():
@@ -2416,7 +3363,7 @@ class And(ParseExpression): def __iadd__(self, other ):
if isinstance( other, basestring ):
- other = Literal( other )
+ other = ParserElement._literalStringClass( other )
return self.append( other ) #And( [ self, other ] )
def checkRecursion( self, parseElementList ):
@@ -2437,9 +3384,18 @@ class And(ParseExpression): class Or(ParseExpression):
- """Requires that at least one C{ParseExpression} is found.
- If two expressions match, the expression that matches the longest string will be used.
- May be constructed using the C{'^'} operator.
+ """
+ Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the expression that matches the longest string will be used.
+ May be constructed using the C{'^'} operator.
+
+ Example::
+ # construct Or using '^' operator
+
+ number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums))
+ print(number.searchString("123 3.1416 789"))
+ prints::
+ [['123'], ['3.1416'], ['789']]
"""
def __init__( self, exprs, savelist = False ):
super(Or,self).__init__(exprs, savelist)
@@ -2488,7 +3444,7 @@ class Or(ParseExpression): def __ixor__(self, other ):
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
return self.append( other ) #Or( [ self, other ] )
def __str__( self ):
@@ -2507,9 +3463,21 @@ class Or(ParseExpression): class MatchFirst(ParseExpression):
- """Requires that at least one C{ParseExpression} is found.
- If two expressions match, the first one listed is the one that will match.
- May be constructed using the C{'|'} operator.
+ """
+ Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the first one listed is the one that will match.
+ May be constructed using the C{'|'} operator.
+
+ Example::
+ # construct MatchFirst using '|' operator
+
+ # watch the order of expressions to match
+ number = Word(nums) | Combine(Word(nums) + '.' + Word(nums))
+ print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']]
+
+ # put more selective expression first
+ number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums)
+ print(number.searchString("123 3.1416 789")) # Better -> [['123'], ['3.1416'], ['789']]
"""
def __init__( self, exprs, savelist = False ):
super(MatchFirst,self).__init__(exprs, savelist)
@@ -2544,7 +3512,7 @@ class MatchFirst(ParseExpression): def __ior__(self, other ):
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass( other )
+ other = ParserElement._literalStringClass( other )
return self.append( other ) #MatchFirst( [ self, other ] )
def __str__( self ):
@@ -2563,9 +3531,58 @@ class MatchFirst(ParseExpression): class Each(ParseExpression):
- """Requires all given C{ParseExpression}s to be found, but in any order.
- Expressions may be separated by whitespace.
- May be constructed using the C{'&'} operator.
+ """
+ Requires all given C{ParseExpression}s to be found, but in any order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'&'} operator.
+
+ Example::
+ color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN")
+ shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON")
+ integer = Word(nums)
+ shape_attr = "shape:" + shape_type("shape")
+ posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn")
+ color_attr = "color:" + color("color")
+ size_attr = "size:" + integer("size")
+
+ # use Each (using operator '&') to accept attributes in any order
+ # (shape and posn are required, color and size are optional)
+ shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr)
+
+ shape_spec.runTests('''
+ shape: SQUARE color: BLACK posn: 100, 120
+ shape: CIRCLE size: 50 color: BLUE posn: 50,80
+ color:GREEN size:20 shape:TRIANGLE posn:20,40
+ '''
+ )
+ prints::
+ shape: SQUARE color: BLACK posn: 100, 120
+ ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']]
+ - color: BLACK
+ - posn: ['100', ',', '120']
+ - x: 100
+ - y: 120
+ - shape: SQUARE
+
+
+ shape: CIRCLE size: 50 color: BLUE posn: 50,80
+ ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE', 'posn:', ['50', ',', '80']]
+ - color: BLUE
+ - posn: ['50', ',', '80']
+ - x: 50
+ - y: 80
+ - shape: CIRCLE
+ - size: 50
+
+
+ color: GREEN size: 20 shape: TRIANGLE posn: 20,40
+ ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE', 'posn:', ['20', ',', '40']]
+ - color: GREEN
+ - posn: ['20', ',', '40']
+ - x: 20
+ - y: 40
+ - shape: TRIANGLE
+ - size: 20
"""
def __init__( self, exprs, savelist = True ):
super(Each,self).__init__(exprs, savelist)
@@ -2619,17 +3636,7 @@ class Each(ParseExpression): loc,results = e._parse(instring,loc,doActions)
resultlist.append(results)
- finalResults = ParseResults([])
- for r in resultlist:
- dups = {}
- for k in r.keys():
- if k in finalResults:
- tmp = ParseResults(finalResults[k])
- tmp += ParseResults(r[k])
- dups[k] = tmp
- finalResults += ParseResults(r)
- for k,v in dups.items():
- finalResults[k] = v
+ finalResults = sum(resultlist, ParseResults([]))
return loc, finalResults
def __str__( self ):
@@ -2648,11 +3655,16 @@ class Each(ParseExpression): class ParseElementEnhance(ParserElement):
- """Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens."""
+ """
+ Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens.
+ """
def __init__( self, expr, savelist=False ):
super(ParseElementEnhance,self).__init__(savelist)
if isinstance( expr, basestring ):
- expr = Literal(expr)
+ if issubclass(ParserElement._literalStringClass, Token):
+ expr = ParserElement._literalStringClass(expr)
+ else:
+ expr = ParserElement._literalStringClass(Literal(expr))
self.expr = expr
self.strRepr = None
if expr is not None:
@@ -2720,10 +3732,22 @@ class ParseElementEnhance(ParserElement): class FollowedBy(ParseElementEnhance):
- """Lookahead matching of the given parse expression. C{FollowedBy}
- does *not* advance the parsing position within the input string, it only
+ """
+ Lookahead matching of the given parse expression. C{FollowedBy}
+ does I{not} advance the parsing position within the input string, it only
verifies that the specified parse expression matches at the current
- position. C{FollowedBy} always returns a null token list."""
+ position. C{FollowedBy} always returns a null token list.
+
+ Example::
+ # use FollowedBy to match a label only if it is followed by a ':'
+ data_word = Word(alphas)
+ label = data_word + FollowedBy(':')
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+
+ OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint()
+ prints::
+ [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']]
+ """
def __init__( self, expr ):
super(FollowedBy,self).__init__(expr)
self.mayReturnEmpty = True
@@ -2734,11 +3758,16 @@ class FollowedBy(ParseElementEnhance): class NotAny(ParseElementEnhance):
- """Lookahead to disallow matching with the given parse expression. C{NotAny}
- does *not* advance the parsing position within the input string, it only
- verifies that the specified parse expression does *not* match at the current
- position. Also, C{NotAny} does *not* skip over leading whitespace. C{NotAny}
- always returns a null token list. May be constructed using the '~' operator."""
+ """
+ Lookahead to disallow matching with the given parse expression. C{NotAny}
+ does I{not} advance the parsing position within the input string, it only
+ verifies that the specified parse expression does I{not} match at the current
+ position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny}
+ always returns a null token list. May be constructed using the '~' operator.
+
+ Example::
+
+ """
def __init__( self, expr ):
super(NotAny,self).__init__(expr)
#~ self.leaveWhitespace()
@@ -2747,11 +3776,7 @@ class NotAny(ParseElementEnhance): self.errmsg = "Found unwanted token, "+_ustr(self.expr)
def parseImpl( self, instring, loc, doActions=True ):
- try:
- self.expr.tryParse( instring, loc )
- except (ParseException,IndexError):
- pass
- else:
+ if self.expr.canParseNext(instring, loc):
raise ParseException(instring, loc, self.errmsg, self)
return loc, []
@@ -2764,80 +3789,114 @@ class NotAny(ParseElementEnhance): return self.strRepr
-
-class ZeroOrMore(ParseElementEnhance):
- """Optional repetition of zero or more of the given expression."""
- def __init__( self, expr ):
- super(ZeroOrMore,self).__init__(expr)
- self.mayReturnEmpty = True
+class _MultipleMatch(ParseElementEnhance):
+ def __init__( self, expr, stopOn=None):
+ super(_MultipleMatch, self).__init__(expr)
+ ender = stopOn
+ if isinstance(ender, basestring):
+ ender = ParserElement._literalStringClass(ender)
+ self.not_ender = ~ender if ender is not None else None
def parseImpl( self, instring, loc, doActions=True ):
- tokens = []
+ self_expr_parse = self.expr._parse
+ self_skip_ignorables = self._skipIgnorables
+ check_ender = self.not_ender is not None
+ if check_ender:
+ try_not_ender = self.not_ender.tryParse
+
+ # must be at least one (but first see if we are the stopOn sentinel;
+ # if so, fail)
+ if check_ender:
+ try_not_ender(instring, loc)
+ loc, tokens = self_expr_parse( instring, loc, doActions, callPreParse=False )
try:
- loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
- hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
+ hasIgnoreExprs = (not not self.ignoreExprs)
while 1:
+ if check_ender:
+ try_not_ender(instring, loc)
if hasIgnoreExprs:
- preloc = self._skipIgnorables( instring, loc )
+ preloc = self_skip_ignorables( instring, loc )
else:
preloc = loc
- loc, tmptokens = self.expr._parse( instring, preloc, doActions )
+ loc, tmptokens = self_expr_parse( instring, preloc, doActions )
if tmptokens or tmptokens.haskeys():
tokens += tmptokens
except (ParseException,IndexError):
pass
return loc, tokens
+
+class OneOrMore(_MultipleMatch):
+ """
+ Repetition of one or more of the given expression.
+
+ Parameters:
+ - expr - expression that must match one or more times
+ - stopOn - (default=C{None}) - expression for a terminating sentinel
+ (only required if the sentinel would ordinarily match the repetition
+ expression)
+
+ Example::
+ data_word = Word(alphas)
+ label = data_word + FollowedBy(':')
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+ text = "shape: SQUARE posn: upper left color: BLACK"
+ OneOrMore(attr_expr).parseString(text).pprint() # Fail! read 'color' as data instead of next label -> [['shape', 'SQUARE color']]
+
+ # use stopOn attribute for OneOrMore to avoid reading label string as part of the data
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+ OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
+
+ # could also be written as
+ (attr_expr * (1,)).parseString(text).pprint()
+ """
def __str__( self ):
if hasattr(self,"name"):
return self.name
if self.strRepr is None:
- self.strRepr = "[" + _ustr(self.expr) + "]..."
+ self.strRepr = "{" + _ustr(self.expr) + "}..."
return self.strRepr
def setResultsName( self, name, listAllMatches=False ):
- ret = super(ZeroOrMore,self).setResultsName(name,listAllMatches)
+ ret = super(OneOrMore,self).setResultsName(name,listAllMatches)
ret.saveAsList = True
return ret
+class ZeroOrMore(_MultipleMatch):
+ """
+ Optional repetition of zero or more of the given expression.
+
+ Parameters:
+ - expr - expression that must match zero or more times
+ - stopOn - (default=C{None}) - expression for a terminating sentinel
+ (only required if the sentinel would ordinarily match the repetition
+ expression)
-class OneOrMore(ParseElementEnhance):
- """Repetition of one or more of the given expression."""
+ Example: similar to L{OneOrMore}
+ """
+ def __init__( self, expr, stopOn=None):
+ super(ZeroOrMore,self).__init__(expr, stopOn=stopOn)
+ self.mayReturnEmpty = True
+
def parseImpl( self, instring, loc, doActions=True ):
- # must be at least one
- loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
try:
- hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
- while 1:
- if hasIgnoreExprs:
- preloc = self._skipIgnorables( instring, loc )
- else:
- preloc = loc
- loc, tmptokens = self.expr._parse( instring, preloc, doActions )
- if tmptokens or tmptokens.haskeys():
- tokens += tmptokens
+ return super(ZeroOrMore, self).parseImpl(instring, loc, doActions)
except (ParseException,IndexError):
- pass
-
- return loc, tokens
+ return loc, []
def __str__( self ):
if hasattr(self,"name"):
return self.name
if self.strRepr is None:
- self.strRepr = "{" + _ustr(self.expr) + "}..."
+ self.strRepr = "[" + _ustr(self.expr) + "]..."
return self.strRepr
- def setResultsName( self, name, listAllMatches=False ):
- ret = super(OneOrMore,self).setResultsName(name,listAllMatches)
- ret.saveAsList = True
- return ret
-
class _NullToken(object):
def __bool__(self):
return False
@@ -2847,9 +3906,39 @@ class _NullToken(object): _optionalNotMatched = _NullToken()
class Optional(ParseElementEnhance):
- """Optional matching of the given expression.
- A default return string can also be specified, if the optional expression
- is not found.
+ """
+ Optional matching of the given expression.
+
+ Parameters:
+ - expr - expression that must match zero or more times
+ - default (optional) - value to be returned if the optional expression is not found.
+
+ Example::
+ # US postal code can be a 5-digit zip, plus optional 4-digit qualifier
+ zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4)))
+ zip.runTests('''
+ # traditional ZIP code
+ 12345
+
+ # ZIP+4 form
+ 12101-0001
+
+ # invalid ZIP
+ 98765-
+ ''')
+ prints::
+ # traditional ZIP code
+ 12345
+ ['12345']
+
+ # ZIP+4 form
+ 12101-0001
+ ['12101-0001']
+
+ # invalid ZIP
+ 98765-
+ ^
+ FAIL: Expected end of text (at char 5), (line:1, col:6)
"""
def __init__( self, expr, default=_optionalNotMatched ):
super(Optional,self).__init__( expr, savelist=False )
@@ -2879,13 +3968,60 @@ class Optional(ParseElementEnhance): return self.strRepr
-
class SkipTo(ParseElementEnhance):
- """Token for skipping over all undefined text until the matched expression is found.
- If C{include} is set to true, the matched expression is also parsed (the skipped text
- and matched expression are returned as a 2-element list). The C{ignore}
- argument is used to define grammars (typically quoted strings and comments) that
- might contain false matches.
+ """
+ Token for skipping over all undefined text until the matched expression is found.
+
+ Parameters:
+ - expr - target expression marking the end of the data to be skipped
+ - include - (default=C{False}) if True, the target expression is also parsed
+ (the skipped text and target expression are returned as a 2-element list).
+ - ignore - (default=C{None}) used to define grammars (typically quoted strings and
+ comments) that might contain false matches to the target expression
+ - failOn - (default=C{None}) define expressions that are not allowed to be
+ included in the skipped test; if found before the target expression is found,
+ the SkipTo is not a match
+
+ Example::
+ report = '''
+ Outstanding Issues Report - 1 Jan 2000
+
+ # | Severity | Description | Days Open
+ -----+----------+-------------------------------------------+-----------
+ 101 | Critical | Intermittent system crash | 6
+ 94 | Cosmetic | Spelling error on Login ('log|n') | 14
+ 79 | Minor | System slow when running too many reports | 47
+ '''
+ integer = Word(nums)
+ SEP = Suppress('|')
+ # use SkipTo to simply match everything up until the next SEP
+ # - ignore quoted strings, so that a '|' character inside a quoted string does not match
+ # - parse action will call token.strip() for each matched token, i.e., the description body
+ string_data = SkipTo(SEP, ignore=quotedString)
+ string_data.setParseAction(tokenMap(str.strip))
+ ticket_expr = (integer("issue_num") + SEP
+ + string_data("sev") + SEP
+ + string_data("desc") + SEP
+ + integer("days_open"))
+
+ for tkt in ticket_expr.searchString(report):
+ print tkt.dump()
+ prints::
+ ['101', 'Critical', 'Intermittent system crash', '6']
+ - days_open: 6
+ - desc: Intermittent system crash
+ - issue_num: 101
+ - sev: Critical
+ ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14']
+ - days_open: 14
+ - desc: Spelling error on Login ('log|n')
+ - issue_num: 94
+ - sev: Cosmetic
+ ['79', 'Minor', 'System slow when running too many reports', '47']
+ - days_open: 47
+ - desc: System slow when running too many reports
+ - issue_num: 79
+ - sev: Minor
"""
def __init__( self, other, include=False, ignore=None, failOn=None ):
super( SkipTo, self ).__init__( other )
@@ -2894,77 +4030,85 @@ class SkipTo(ParseElementEnhance): self.mayIndexError = False
self.includeMatch = include
self.asList = False
- if failOn is not None and isinstance(failOn, basestring):
- self.failOn = Literal(failOn)
+ if isinstance(failOn, basestring):
+ self.failOn = ParserElement._literalStringClass(failOn)
else:
self.failOn = failOn
self.errmsg = "No match found for "+_ustr(self.expr)
def parseImpl( self, instring, loc, doActions=True ):
- startLoc = loc
+ startloc = loc
instrlen = len(instring)
expr = self.expr
- failParse = False
- while loc <= instrlen:
- try:
- if self.failOn:
+ expr_parse = self.expr._parse
+ self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None
+ self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None
+
+ tmploc = loc
+ while tmploc <= instrlen:
+ if self_failOn_canParseNext is not None:
+ # break if failOn expression matches
+ if self_failOn_canParseNext(instring, tmploc):
+ break
+
+ if self_ignoreExpr_tryParse is not None:
+ # advance past ignore expressions
+ while 1:
try:
- self.failOn.tryParse(instring, loc)
+ tmploc = self_ignoreExpr_tryParse(instring, tmploc)
except ParseBaseException:
- pass
- else:
- failParse = True
- raise ParseException(instring, loc, "Found expression " + str(self.failOn))
- failParse = False
- if self.ignoreExpr is not None:
- while 1:
- try:
- loc = self.ignoreExpr.tryParse(instring,loc)
- # print("found ignoreExpr, advance to", loc)
- except ParseBaseException:
- break
- expr._parse( instring, loc, doActions=False, callPreParse=False )
- skipText = instring[startLoc:loc]
- if self.includeMatch:
- loc,mat = expr._parse(instring,loc,doActions,callPreParse=False)
- if mat:
- skipRes = ParseResults( skipText )
- skipRes += mat
- return loc, [ skipRes ]
- else:
- return loc, [ skipText ]
- else:
- return loc, [ skipText ]
- except (ParseException,IndexError):
- if failParse:
- raise
- else:
- loc += 1
- raise ParseException(instring, loc, self.errmsg, self)
+ break
+
+ try:
+ expr_parse(instring, tmploc, doActions=False, callPreParse=False)
+ except (ParseException, IndexError):
+ # no match, advance loc in string
+ tmploc += 1
+ else:
+ # matched skipto expr, done
+ break
+
+ else:
+ # ran off the end of the input string without matching skipto expr, fail
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ # build up return values
+ loc = tmploc
+ skiptext = instring[startloc:loc]
+ skipresult = ParseResults(skiptext)
+
+ if self.includeMatch:
+ loc, mat = expr_parse(instring,loc,doActions,callPreParse=False)
+ skipresult += mat
+
+ return loc, skipresult
class Forward(ParseElementEnhance):
- """Forward declaration of an expression to be defined later -
- used for recursive grammars, such as algebraic infix notation.
- When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator.
-
- Note: take care when assigning to C{Forward} not to overlook precedence of operators.
- Specifically, '|' has a lower precedence than '<<', so that::
- fwdExpr << a | b | c
- will actually be evaluated as::
- (fwdExpr << a) | b | c
- thereby leaving b and c out as parseable alternatives. It is recommended that you
- explicitly group the values inserted into the C{Forward}::
- fwdExpr << (a | b | c)
- Converting to use the '<<=' operator instead will avoid this problem.
+ """
+ Forward declaration of an expression to be defined later -
+ used for recursive grammars, such as algebraic infix notation.
+ When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator.
+
+ Note: take care when assigning to C{Forward} not to overlook precedence of operators.
+ Specifically, '|' has a lower precedence than '<<', so that::
+ fwdExpr << a | b | c
+ will actually be evaluated as::
+ (fwdExpr << a) | b | c
+ thereby leaving b and c out as parseable alternatives. It is recommended that you
+ explicitly group the values inserted into the C{Forward}::
+ fwdExpr << (a | b | c)
+ Converting to use the '<<=' operator instead will avoid this problem.
+
+ See L{ParseResults.pprint} for an example of a recursive parser created using
+ C{Forward}.
"""
def __init__( self, other=None ):
super(Forward,self).__init__( other, savelist=False )
def __lshift__( self, other ):
if isinstance( other, basestring ):
- other = ParserElement.literalStringClass(other)
+ other = ParserElement._literalStringClass(other)
self.expr = other
- self.mayReturnEmpty = other.mayReturnEmpty
self.strRepr = None
self.mayIndexError = self.expr.mayIndexError
self.mayReturnEmpty = self.expr.mayReturnEmpty
@@ -2998,7 +4142,9 @@ class Forward(ParseElementEnhance): def __str__( self ):
if hasattr(self,"name"):
return self.name
+ return self.__class__.__name__ + ": ..."
+ # stubbed out for now - creates awful memory and perf issues
self._revertClass = self.__class__
self.__class__ = _ForwardNoRecurse
try:
@@ -3023,26 +4169,29 @@ class _ForwardNoRecurse(Forward): return "..."
class TokenConverter(ParseElementEnhance):
- """Abstract subclass of C{ParseExpression}, for converting parsed results."""
+ """
+ Abstract subclass of C{ParseExpression}, for converting parsed results.
+ """
def __init__( self, expr, savelist=False ):
super(TokenConverter,self).__init__( expr )#, savelist )
self.saveAsList = False
-class Upcase(TokenConverter):
- """Converter to upper case all matching tokens."""
- def __init__(self, *args):
- super(Upcase,self).__init__(*args)
- warnings.warn("Upcase class is deprecated, use upcaseTokens parse action instead",
- DeprecationWarning,stacklevel=2)
-
- def postParse( self, instring, loc, tokenlist ):
- return list(map( str.upper, tokenlist ))
-
-
class Combine(TokenConverter):
- """Converter to concatenate all matching tokens to a single string.
- By default, the matching patterns must also be contiguous in the input string;
- this can be disabled by specifying C{'adjacent=False'} in the constructor.
+ """
+ Converter to concatenate all matching tokens to a single string.
+ By default, the matching patterns must also be contiguous in the input string;
+ this can be disabled by specifying C{'adjacent=False'} in the constructor.
+
+ Example::
+ real = Word(nums) + '.' + Word(nums)
+ print(real.parseString('3.1416')) # -> ['3', '.', '1416']
+ # will also erroneously match the following
+ print(real.parseString('3. 1416')) # -> ['3', '.', '1416']
+
+ real = Combine(Word(nums) + '.' + Word(nums))
+ print(real.parseString('3.1416')) # -> ['3.1416']
+ # no match when there are internal spaces
+ print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...)
"""
def __init__( self, expr, joinString="", adjacent=True ):
super(Combine,self).__init__( expr )
@@ -3072,7 +4221,19 @@ class Combine(TokenConverter): return retToks
class Group(TokenConverter):
- """Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions."""
+ """
+ Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions.
+
+ Example::
+ ident = Word(alphas)
+ num = Word(nums)
+ term = ident | num
+ func = ident + Optional(delimitedList(term))
+ print(func.parseString("fn a,b,100")) # -> ['fn', 'a', 'b', '100']
+
+ func = ident + Group(Optional(delimitedList(term)))
+ print(func.parseString("fn a,b,100")) # -> ['fn', ['a', 'b', '100']]
+ """
def __init__( self, expr ):
super(Group,self).__init__( expr )
self.saveAsList = True
@@ -3081,9 +4242,40 @@ class Group(TokenConverter): return [ tokenlist ]
class Dict(TokenConverter):
- """Converter to return a repetitive expression as a list, but also as a dictionary.
- Each element can also be referenced using the first token in the expression as its key.
- Useful for tabular report scraping when the first column can be used as a item key.
+ """
+ Converter to return a repetitive expression as a list, but also as a dictionary.
+ Each element can also be referenced using the first token in the expression as its key.
+ Useful for tabular report scraping when the first column can be used as a item key.
+
+ Example::
+ data_word = Word(alphas)
+ label = data_word + FollowedBy(':')
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+ text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+ attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+
+ # print attributes as plain groups
+ print(OneOrMore(attr_expr).parseString(text).dump())
+
+ # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names
+ result = Dict(OneOrMore(Group(attr_expr))).parseString(text)
+ print(result.dump())
+
+ # access named fields as dict entries, or output as dict
+ print(result['shape'])
+ print(result.asDict())
+ prints::
+ ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap']
+
+ [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+ - color: light blue
+ - posn: upper left
+ - shape: SQUARE
+ - texture: burlap
+ SQUARE
+ {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'}
+ See more examples at L{ParseResults} of accessing fields by results name.
"""
def __init__( self, expr ):
super(Dict,self).__init__( expr )
@@ -3115,7 +4307,24 @@ class Dict(TokenConverter): class Suppress(TokenConverter):
- """Converter for ignoring the results of a parsed expression."""
+ """
+ Converter for ignoring the results of a parsed expression.
+
+ Example::
+ source = "a, b, c,d"
+ wd = Word(alphas)
+ wd_list1 = wd + ZeroOrMore(',' + wd)
+ print(wd_list1.parseString(source))
+
+ # often, delimiters that are useful during parsing are just in the
+ # way afterward - use Suppress to keep them out of the parsed output
+ wd_list2 = wd + ZeroOrMore(Suppress(',') + wd)
+ print(wd_list2.parseString(source))
+ prints::
+ ['a', ',', 'b', ',', 'c', ',', 'd']
+ ['a', 'b', 'c', 'd']
+ (See also L{delimitedList}.)
+ """
def postParse( self, instring, loc, tokenlist ):
return []
@@ -3124,7 +4333,9 @@ class Suppress(TokenConverter): class OnlyOnce(object):
- """Wrapper for parse actions, to ensure they are only called once."""
+ """
+ Wrapper for parse actions, to ensure they are only called once.
+ """
def __init__(self, methodCall):
self.callable = _trim_arity(methodCall)
self.called = False
@@ -3138,20 +4349,39 @@ class OnlyOnce(object): self.called = False
def traceParseAction(f):
- """Decorator for debugging parse actions."""
+ """
+ Decorator for debugging parse actions.
+
+ When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".}
+ When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised.
+
+ Example::
+ wd = Word(alphas)
+
+ @traceParseAction
+ def remove_duplicate_chars(tokens):
+ return ''.join(sorted(set(''.join(tokens)))
+
+ wds = OneOrMore(wd).setParseAction(remove_duplicate_chars)
+ print(wds.parseString("slkdjs sld sldd sdlf sdljf"))
+ prints::
+ >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {}))
+ <<leaving remove_duplicate_chars (ret: 'dfjkls')
+ ['dfjkls']
+ """
f = _trim_arity(f)
def z(*paArgs):
- thisFunc = f.func_name
+ thisFunc = f.__name__
s,l,t = paArgs[-3:]
if len(paArgs)>3:
thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
- sys.stderr.write( ">>entering %s(line: '%s', %d, %s)\n" % (thisFunc,line(l,s),l,t) )
+ sys.stderr.write( ">>entering %s(line: '%s', %d, %r)\n" % (thisFunc,line(l,s),l,t) )
try:
ret = f(*paArgs)
except Exception as exc:
sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) )
raise
- sys.stderr.write( "<<leaving %s (ret: %s)\n" % (thisFunc,ret) )
+ sys.stderr.write( "<<leaving %s (ret: %r)\n" % (thisFunc,ret) )
return ret
try:
z.__name__ = f.__name__
@@ -3163,12 +4393,17 @@ def traceParseAction(f): # global helpers
#
def delimitedList( expr, delim=",", combine=False ):
- """Helper to define a delimited list of expressions - the delimiter defaults to ','.
- By default, the list elements and delimiters can have intervening whitespace, and
- comments, but this can be overridden by passing C{combine=True} in the constructor.
- If C{combine} is set to C{True}, the matching tokens are returned as a single token
- string, with the delimiters included; otherwise, the matching tokens are returned
- as a list of tokens, with the delimiters suppressed.
+ """
+ Helper to define a delimited list of expressions - the delimiter defaults to ','.
+ By default, the list elements and delimiters can have intervening whitespace, and
+ comments, but this can be overridden by passing C{combine=True} in the constructor.
+ If C{combine} is set to C{True}, the matching tokens are returned as a single token
+ string, with the delimiters included; otherwise, the matching tokens are returned
+ as a list of tokens, with the delimiters suppressed.
+
+ Example::
+ delimitedList(Word(alphas)).parseString("aa,bb,cc") # -> ['aa', 'bb', 'cc']
+ delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE']
"""
dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
if combine:
@@ -3177,11 +4412,22 @@ def delimitedList( expr, delim=",", combine=False ): return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)
def countedArray( expr, intExpr=None ):
- """Helper to define a counted list of expressions.
- This helper defines a pattern of the form::
- integer expr expr expr...
- where the leading integer tells how many expr expressions follow.
- The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
+ """
+ Helper to define a counted list of expressions.
+ This helper defines a pattern of the form::
+ integer expr expr expr...
+ where the leading integer tells how many expr expressions follow.
+ The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
+
+ If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value.
+
+ Example::
+ countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd']
+
+ # in this parser, the leading integer value is given in binary,
+ # '10' indicating that 2 values are in the array
+ binaryConstant = Word('01').setParseAction(lambda t: int(t[0], 2))
+ countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef') # -> ['ab', 'cd']
"""
arrayExpr = Forward()
def countFieldParseAction(s,l,t):
@@ -3194,7 +4440,7 @@ def countedArray( expr, intExpr=None ): intExpr = intExpr.copy()
intExpr.setName("arrayLen")
intExpr.addParseAction(countFieldParseAction, callDuringTry=True)
- return ( intExpr + arrayExpr )
+ return ( intExpr + arrayExpr ).setName('(len) ' + _ustr(expr) + '...')
def _flatten(L):
ret = []
@@ -3206,16 +4452,17 @@ def _flatten(L): return ret
def matchPreviousLiteral(expr):
- """Helper to define an expression that is indirectly defined from
- the tokens matched in a previous expression, that is, it looks
- for a 'repeat' of a previous expression. For example::
- first = Word(nums)
- second = matchPreviousLiteral(first)
- matchExpr = first + ":" + second
- will match C{"1:1"}, but not C{"1:2"}. Because this matches a
- previous literal, will also match the leading C{"1:1"} in C{"1:10"}.
- If this is not desired, use C{matchPreviousExpr}.
- Do *not* use with packrat parsing enabled.
+ """
+ Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousLiteral(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches a
+ previous literal, will also match the leading C{"1:1"} in C{"1:10"}.
+ If this is not desired, use C{matchPreviousExpr}.
+ Do I{not} use with packrat parsing enabled.
"""
rep = Forward()
def copyTokenToRepeater(s,l,t):
@@ -3225,24 +4472,26 @@ def matchPreviousLiteral(expr): else:
# flatten t tokens
tflat = _flatten(t.asList())
- rep << And( [ Literal(tt) for tt in tflat ] )
+ rep << And(Literal(tt) for tt in tflat)
else:
rep << Empty()
expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ rep.setName('(prev) ' + _ustr(expr))
return rep
def matchPreviousExpr(expr):
- """Helper to define an expression that is indirectly defined from
- the tokens matched in a previous expression, that is, it looks
- for a 'repeat' of a previous expression. For example::
- first = Word(nums)
- second = matchPreviousExpr(first)
- matchExpr = first + ":" + second
- will match C{"1:1"}, but not C{"1:2"}. Because this matches by
- expressions, will *not* match the leading C{"1:1"} in C{"1:10"};
- the expressions are evaluated first, and then compared, so
- C{"1"} is compared with C{"10"}.
- Do *not* use with packrat parsing enabled.
+ """
+ Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousExpr(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches by
+ expressions, will I{not} match the leading C{"1:1"} in C{"1:10"};
+ the expressions are evaluated first, and then compared, so
+ C{"1"} is compared with C{"10"}.
+ Do I{not} use with packrat parsing enabled.
"""
rep = Forward()
e2 = expr.copy()
@@ -3255,6 +4504,7 @@ def matchPreviousExpr(expr): raise ParseException("",0,"")
rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ rep.setName('(prev) ' + _ustr(expr))
return rep
def _escapeRegexRangeChars(s):
@@ -3266,16 +4516,27 @@ def _escapeRegexRangeChars(s): return _ustr(s)
def oneOf( strs, caseless=False, useRegex=True ):
- """Helper to quickly define a set of alternative Literals, and makes sure to do
- longest-first testing when there is a conflict, regardless of the input order,
- but returns a C{L{MatchFirst}} for best performance.
-
- Parameters:
- - strs - a string of space-delimited literals, or a list of string literals
- - caseless - (default=False) - treat all literals as caseless
- - useRegex - (default=True) - as an optimization, will generate a Regex
+ """
+ Helper to quickly define a set of alternative Literals, and makes sure to do
+ longest-first testing when there is a conflict, regardless of the input order,
+ but returns a C{L{MatchFirst}} for best performance.
+
+ Parameters:
+ - strs - a string of space-delimited literals, or a collection of string literals
+ - caseless - (default=C{False}) - treat all literals as caseless
+ - useRegex - (default=C{True}) - as an optimization, will generate a Regex
object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or
if creating a C{Regex} raises an exception)
+
+ Example::
+ comp_oper = oneOf("< = > <= >= !=")
+ var = Word(alphas)
+ number = Word(nums)
+ term = var | number
+ comparison_expr = term + comp_oper + term
+ print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12"))
+ prints::
+ [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']]
"""
if caseless:
isequal = ( lambda a,b: a.upper() == b.upper() )
@@ -3289,12 +4550,10 @@ def oneOf( strs, caseless=False, useRegex=True ): symbols = []
if isinstance(strs,basestring):
symbols = strs.split()
- elif isinstance(strs, collections.Sequence):
- symbols = list(strs[:])
- elif isinstance(strs, _generatorType):
+ elif isinstance(strs, collections.Iterable):
symbols = list(strs)
else:
- warnings.warn("Invalid argument to oneOf, expected string or list",
+ warnings.warn("Invalid argument to oneOf, expected string or iterable",
SyntaxWarning, stacklevel=2)
if not symbols:
return NoMatch()
@@ -3318,41 +4577,76 @@ def oneOf( strs, caseless=False, useRegex=True ): #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
try:
if len(symbols)==len("".join(symbols)):
- return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) )
+ return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) ).setName(' | '.join(symbols))
else:
- return Regex( "|".join(re.escape(sym) for sym in symbols) )
+ return Regex( "|".join(re.escape(sym) for sym in symbols) ).setName(' | '.join(symbols))
except:
warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
SyntaxWarning, stacklevel=2)
# last resort, just use MatchFirst
- return MatchFirst( [ parseElementClass(sym) for sym in symbols ] )
+ return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols))
def dictOf( key, value ):
- """Helper to easily and clearly define a dictionary by specifying the respective patterns
- for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens
- in the proper order. The key pattern can include delimiting markers or punctuation,
- as long as they are suppressed, thereby leaving the significant key text. The value
- pattern can include named results, so that the C{Dict} results can include named token
- fields.
+ """
+ Helper to easily and clearly define a dictionary by specifying the respective patterns
+ for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens
+ in the proper order. The key pattern can include delimiting markers or punctuation,
+ as long as they are suppressed, thereby leaving the significant key text. The value
+ pattern can include named results, so that the C{Dict} results can include named token
+ fields.
+
+ Example::
+ text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+ attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+ print(OneOrMore(attr_expr).parseString(text).dump())
+
+ attr_label = label
+ attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)
+
+ # similar to Dict, but simpler call format
+ result = dictOf(attr_label, attr_value).parseString(text)
+ print(result.dump())
+ print(result['shape'])
+ print(result.shape) # object attribute access works too
+ print(result.asDict())
+ prints::
+ [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+ - color: light blue
+ - posn: upper left
+ - shape: SQUARE
+ - texture: burlap
+ SQUARE
+ SQUARE
+ {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'}
"""
return Dict( ZeroOrMore( Group ( key + value ) ) )
def originalTextFor(expr, asString=True):
- """Helper to return the original, untokenized text for a given expression. Useful to
- restore the parsed fields of an HTML start tag into the raw tag text itself, or to
- revert separate tokens with intervening whitespace back to the original matching
- input text. Simpler to use than the parse action C{L{keepOriginalText}}, and does not
- require the inspect module to chase up the call stack. By default, returns a
- string containing the original parsed text.
+ """
+ Helper to return the original, untokenized text for a given expression. Useful to
+ restore the parsed fields of an HTML start tag into the raw tag text itself, or to
+ revert separate tokens with intervening whitespace back to the original matching
+ input text. By default, returns astring containing the original parsed text.
- If the optional C{asString} argument is passed as C{False}, then the return value is a
- C{L{ParseResults}} containing any results names that were originally matched, and a
- single token containing the original matched text from the input string. So if
- the expression passed to C{L{originalTextFor}} contains expressions with defined
- results names, you must set C{asString} to C{False} if you want to preserve those
- results name values."""
+ If the optional C{asString} argument is passed as C{False}, then the return value is a
+ C{L{ParseResults}} containing any results names that were originally matched, and a
+ single token containing the original matched text from the input string. So if
+ the expression passed to C{L{originalTextFor}} contains expressions with defined
+ results names, you must set C{asString} to C{False} if you want to preserve those
+ results name values.
+
+ Example::
+ src = "this is test <b> bold <i>text</i> </b> normal text "
+ for tag in ("b","i"):
+ opener,closer = makeHTMLTags(tag)
+ patt = originalTextFor(opener + SkipTo(closer) + closer)
+ print(patt.searchString(src)[0])
+ prints::
+ ['<b> bold <i>text</i> </b>']
+ ['<i>text</i>']
+ """
locMarker = Empty().setParseAction(lambda s,loc,t: loc)
endlocMarker = locMarker.copy()
endlocMarker.callPreparse = False
@@ -3361,27 +4655,37 @@ def originalTextFor(expr, asString=True): extractText = lambda s,l,t: s[t._original_start:t._original_end]
else:
def extractText(s,l,t):
- del t[:]
- t.insert(0, s[t._original_start:t._original_end])
- del t["_original_start"]
- del t["_original_end"]
+ t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]]
matchExpr.setParseAction(extractText)
+ matchExpr.ignoreExprs = expr.ignoreExprs
return matchExpr
def ungroup(expr):
- """Helper to undo pyparsing's default grouping of And expressions, even
- if all but one are non-empty."""
+ """
+ Helper to undo pyparsing's default grouping of And expressions, even
+ if all but one are non-empty.
+ """
return TokenConverter(expr).setParseAction(lambda t:t[0])
def locatedExpr(expr):
- """Helper to decorate a returned token with its starting and ending locations in the input string.
- This helper adds the following results names:
- - locn_start = location where matched expression begins
- - locn_end = location where matched expression ends
- - value = the actual parsed results
-
- Be careful if the input text contains C{<TAB>} characters, you may want to call
- C{L{ParserElement.parseWithTabs}}
+ """
+ Helper to decorate a returned token with its starting and ending locations in the input string.
+ This helper adds the following results names:
+ - locn_start = location where matched expression begins
+ - locn_end = location where matched expression ends
+ - value = the actual parsed results
+
+ Be careful if the input text contains C{<TAB>} characters, you may want to call
+ C{L{ParserElement.parseWithTabs}}
+
+ Example::
+ wd = Word(alphas)
+ for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"):
+ print(match)
+ prints::
+ [[0, 'ljsdf', 5]]
+ [[8, 'lksdjjf', 15]]
+ [[18, 'lkkjj', 23]]
"""
locator = Empty().setParseAction(lambda s,l,t: l)
return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end"))
@@ -3402,21 +4706,22 @@ _charRange = Group(_singleChar + Suppress("-") + _singleChar) _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
def srange(s):
- r"""Helper to easily define string ranges for use in Word construction. Borrows
- syntax from regexp '[]' string range definitions::
- srange("[0-9]") -> "0123456789"
- srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz"
- srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
- The input string must be enclosed in []'s, and the returned string is the expanded
- character set joined into a single string.
- The values enclosed in the []'s may be::
- a single character
- an escaped character with a leading backslash (such as \- or \])
- an escaped hex character with a leading '\x' (\x21, which is a '!' character)
- (\0x## is also supported for backwards compatibility)
- an escaped octal character with a leading '\0' (\041, which is a '!' character)
- a range of any of the above, separated by a dash ('a-z', etc.)
- any combination of the above ('aeiouy', 'a-zA-Z0-9_$', etc.)
+ r"""
+ Helper to easily define string ranges for use in Word construction. Borrows
+ syntax from regexp '[]' string range definitions::
+ srange("[0-9]") -> "0123456789"
+ srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz"
+ srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
+ The input string must be enclosed in []'s, and the returned string is the expanded
+ character set joined into a single string.
+ The values enclosed in the []'s may be:
+ - a single character
+ - an escaped character with a leading backslash (such as C{\-} or C{\]})
+ - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character)
+ (C{\0x##} is also supported for backwards compatibility)
+ - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character)
+ - a range of any of the above, separated by a dash (C{'a-z'}, etc.)
+ - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.)
"""
_expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1))
try:
@@ -3425,8 +4730,9 @@ def srange(s): return ""
def matchOnlyAtCol(n):
- """Helper method for defining parse actions that require matching at a specific
- column in the input text.
+ """
+ Helper method for defining parse actions that require matching at a specific
+ column in the input text.
"""
def verifyCol(strg,locn,toks):
if col(locn,strg) != n:
@@ -3434,57 +4740,83 @@ def matchOnlyAtCol(n): return verifyCol
def replaceWith(replStr):
- """Helper method for common parse actions that simply return a literal value. Especially
- useful when used with C{L{transformString<ParserElement.transformString>}()}.
"""
- #def _replFunc(*args):
- # return [replStr]
- #return _replFunc
- return functools.partial(next, itertools.repeat([replStr]))
+ Helper method for common parse actions that simply return a literal value. Especially
+ useful when used with C{L{transformString<ParserElement.transformString>}()}.
+
+ Example::
+ num = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ na = oneOf("N/A NA").setParseAction(replaceWith(math.nan))
+ term = na | num
+
+ OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234]
+ """
+ return lambda s,l,t: [replStr]
def removeQuotes(s,l,t):
- """Helper parse action for removing quotation marks from parsed quoted strings.
- To use, add this parse action to quoted string using::
- quotedString.setParseAction( removeQuotes )
"""
- return t[0][1:-1]
+ Helper parse action for removing quotation marks from parsed quoted strings.
-def upcaseTokens(s,l,t):
- """Helper parse action to convert tokens to upper case."""
- return [ tt.upper() for tt in map(_ustr,t) ]
+ Example::
+ # by default, quotation marks are included in parsed results
+ quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"]
-def downcaseTokens(s,l,t):
- """Helper parse action to convert tokens to lower case."""
- return [ tt.lower() for tt in map(_ustr,t) ]
+ # use removeQuotes to strip quotation marks from parsed results
+ quotedString.setParseAction(removeQuotes)
+ quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["Now is the Winter of our Discontent"]
+ """
+ return t[0][1:-1]
+
+def tokenMap(func, *args):
+ """
+ Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional
+ args are passed, they are forwarded to the given function as additional arguments after
+ the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the
+ parsed data to an integer using base 16.
+
+ Example (compare the last to example in L{ParserElement.transformString}::
+ hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16))
+ hex_ints.runTests('''
+ 00 11 22 aa FF 0a 0d 1a
+ ''')
+
+ upperword = Word(alphas).setParseAction(tokenMap(str.upper))
+ OneOrMore(upperword).runTests('''
+ my kingdom for a horse
+ ''')
+
+ wd = Word(alphas).setParseAction(tokenMap(str.title))
+ OneOrMore(wd).setParseAction(' '.join).runTests('''
+ now is the winter of our discontent made glorious summer by this sun of york
+ ''')
+ prints::
+ 00 11 22 aa FF 0a 0d 1a
+ [0, 17, 34, 170, 255, 10, 13, 26]
+
+ my kingdom for a horse
+ ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE']
+
+ now is the winter of our discontent made glorious summer by this sun of york
+ ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York']
+ """
+ def pa(s,l,t):
+ return [func(tokn, *args) for tokn in t]
-def keepOriginalText(s,startLoc,t):
- """DEPRECATED - use new helper method C{L{originalTextFor}}.
- Helper parse action to preserve original parsed text,
- overriding any nested parse actions."""
- try:
- endloc = getTokensEndLoc()
- except ParseException:
- raise ParseFatalException("incorrect usage of keepOriginalText - may only be called as a parse action")
- del t[:]
- t += ParseResults(s[startLoc:endloc])
- return t
-
-def getTokensEndLoc():
- """Method to be called from within a parse action to determine the end
- location of the parsed tokens."""
- import inspect
- fstack = inspect.stack()
try:
- # search up the stack (through intervening argument normalizers) for correct calling routine
- for f in fstack[2:]:
- if f[3] == "_parseNoCache":
- endloc = f[0].f_locals["loc"]
- return endloc
- else:
- raise ParseFatalException("incorrect usage of getTokensEndLoc - may only be called from within a parse action")
- finally:
- del fstack
+ func_name = getattr(func, '__name__',
+ getattr(func, '__class__').__name__)
+ except Exception:
+ func_name = str(func)
+ pa.__name__ = func_name
+ return pa
+
+upcaseTokens = tokenMap(lambda t: _ustr(t).upper())
+"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}"""
+
+downcaseTokens = tokenMap(lambda t: _ustr(t).lower())
+"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}"""
+
def _makeTags(tagStr, xml):
"""Internal helper to construct opening and closing tag expressions, given a tag name"""
if isinstance(tagStr,basestring):
@@ -3508,40 +4840,90 @@ def _makeTags(tagStr, xml): Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
closeTag = Combine(_L("</") + tagStr + ">")
- openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % tagStr)
- closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % tagStr)
+ openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % resname)
+ closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % resname)
openTag.tag = resname
closeTag.tag = resname
return openTag, closeTag
def makeHTMLTags(tagStr):
- """Helper to construct opening and closing tag expressions for HTML, given a tag name"""
+ """
+ Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches
+ tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values.
+
+ Example::
+ text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>'
+ # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple
+ a,a_end = makeHTMLTags("A")
+ link_expr = a + SkipTo(a_end)("link_text") + a_end
+
+ for link in link_expr.searchString(text):
+ # attributes in the <A> tag (like "href" shown here) are also accessible as named results
+ print(link.link_text, '->', link.href)
+ prints::
+ pyparsing -> http://pyparsing.wikispaces.com
+ """
return _makeTags( tagStr, False )
def makeXMLTags(tagStr):
- """Helper to construct opening and closing tag expressions for XML, given a tag name"""
+ """
+ Helper to construct opening and closing tag expressions for XML, given a tag name. Matches
+ tags only in the given upper/lower case.
+
+ Example: similar to L{makeHTMLTags}
+ """
return _makeTags( tagStr, True )
def withAttribute(*args,**attrDict):
- """Helper to create a validating parse action to be used with start tags created
- with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag
- with a required attribute value, to avoid false matches on common tags such as
- C{<TD>} or C{<DIV>}.
-
- Call C{withAttribute} with a series of attribute names and values. Specify the list
- of filter attributes names and values as:
- - keyword arguments, as in C{(align="right")}, or
- - as an explicit dict with C{**} operator, when an attribute name is also a Python
+ """
+ Helper to create a validating parse action to be used with start tags created
+ with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag
+ with a required attribute value, to avoid false matches on common tags such as
+ C{<TD>} or C{<DIV>}.
+
+ Call C{withAttribute} with a series of attribute names and values. Specify the list
+ of filter attributes names and values as:
+ - keyword arguments, as in C{(align="right")}, or
+ - as an explicit dict with C{**} operator, when an attribute name is also a Python
reserved word, as in C{**{"class":"Customer", "align":"right"}}
- - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
- For attribute names with a namespace prefix, you must use the second form. Attribute
- names are matched insensitive to upper/lower case.
+ - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
+ For attribute names with a namespace prefix, you must use the second form. Attribute
+ names are matched insensitive to upper/lower case.
- If just testing for C{class} (with or without a namespace), use C{L{withClass}}.
-
- To verify that the attribute exists, but without specifying a value, pass
- C{withAttribute.ANY_VALUE} as the value.
- """
+ If just testing for C{class} (with or without a namespace), use C{L{withClass}}.
+
+ To verify that the attribute exists, but without specifying a value, pass
+ C{withAttribute.ANY_VALUE} as the value.
+
+ Example::
+ html = '''
+ <div>
+ Some text
+ <div type="grid">1 4 0 1 0</div>
+ <div type="graph">1,3 2,3 1,1</div>
+ <div>this has no type</div>
+ </div>
+
+ '''
+ div,div_end = makeHTMLTags("div")
+
+ # only match div tag having a type attribute with value "grid"
+ div_grid = div().setParseAction(withAttribute(type="grid"))
+ grid_expr = div_grid + SkipTo(div | div_end)("body")
+ for grid_header in grid_expr.searchString(html):
+ print(grid_header.body)
+
+ # construct a match with any div tag having a type attribute, regardless of the value
+ div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE))
+ div_expr = div_any_type + SkipTo(div | div_end)("body")
+ for div_header in div_expr.searchString(html):
+ print(div_header.body)
+ prints::
+ 1 4 0 1 0
+
+ 1 4 0 1 0
+ 1,3 2,3 1,1
+ """
if args:
attrs = args[:]
else:
@@ -3558,9 +4940,37 @@ def withAttribute(*args,**attrDict): withAttribute.ANY_VALUE = object()
def withClass(classname, namespace=''):
- """Simplified version of C{L{withAttribute}} when matching on a div class - made
- difficult because C{class} is a reserved word in Python.
- """
+ """
+ Simplified version of C{L{withAttribute}} when matching on a div class - made
+ difficult because C{class} is a reserved word in Python.
+
+ Example::
+ html = '''
+ <div>
+ Some text
+ <div class="grid">1 4 0 1 0</div>
+ <div class="graph">1,3 2,3 1,1</div>
+ <div>this <div> has no class</div>
+ </div>
+
+ '''
+ div,div_end = makeHTMLTags("div")
+ div_grid = div().setParseAction(withClass("grid"))
+
+ grid_expr = div_grid + SkipTo(div | div_end)("body")
+ for grid_header in grid_expr.searchString(html):
+ print(grid_header.body)
+
+ div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE))
+ div_expr = div_any_type + SkipTo(div | div_end)("body")
+ for div_header in div_expr.searchString(html):
+ print(div_header.body)
+ prints::
+ 1 4 0 1 0
+
+ 1 4 0 1 0
+ 1,3 2,3 1,1
+ """
classattr = "%s:class" % namespace if namespace else "class"
return withAttribute(**{classattr : classname})
@@ -3569,40 +4979,69 @@ opAssoc.LEFT = object() opAssoc.RIGHT = object()
def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ):
- """Helper method for constructing grammars of expressions made up of
- operators working in a precedence hierarchy. Operators may be unary or
- binary, left- or right-associative. Parse actions can also be attached
- to operator expressions.
-
- Parameters:
- - baseExpr - expression representing the most basic element for the nested
- - opList - list of tuples, one for each operator precedence level in the
- expression grammar; each tuple is of the form
- (opExpr, numTerms, rightLeftAssoc, parseAction), where:
- - opExpr is the pyparsing expression for the operator;
- may also be a string, which will be converted to a Literal;
- if numTerms is 3, opExpr is a tuple of two expressions, for the
- two operators separating the 3 terms
- - numTerms is the number of terms for this operator (must
- be 1, 2, or 3)
- - rightLeftAssoc is the indicator whether the operator is
- right or left associative, using the pyparsing-defined
- constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
- - parseAction is the parse action to be associated with
- expressions matching this operator expression (the
- parse action tuple member may be omitted)
- - lpar - expression for matching left-parentheses (default=Suppress('('))
- - rpar - expression for matching right-parentheses (default=Suppress(')'))
+ """
+ Helper method for constructing grammars of expressions made up of
+ operators working in a precedence hierarchy. Operators may be unary or
+ binary, left- or right-associative. Parse actions can also be attached
+ to operator expressions.
+
+ Parameters:
+ - baseExpr - expression representing the most basic element for the nested
+ - opList - list of tuples, one for each operator precedence level in the
+ expression grammar; each tuple is of the form
+ (opExpr, numTerms, rightLeftAssoc, parseAction), where:
+ - opExpr is the pyparsing expression for the operator;
+ may also be a string, which will be converted to a Literal;
+ if numTerms is 3, opExpr is a tuple of two expressions, for the
+ two operators separating the 3 terms
+ - numTerms is the number of terms for this operator (must
+ be 1, 2, or 3)
+ - rightLeftAssoc is the indicator whether the operator is
+ right or left associative, using the pyparsing-defined
+ constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
+ - parseAction is the parse action to be associated with
+ expressions matching this operator expression (the
+ parse action tuple member may be omitted)
+ - lpar - expression for matching left-parentheses (default=C{Suppress('(')})
+ - rpar - expression for matching right-parentheses (default=C{Suppress(')')})
+
+ Example::
+ # simple example of four-function arithmetic with ints and variable names
+ integer = pyparsing_common.signed_integer
+ varname = pyparsing_common.identifier
+
+ arith_expr = infixNotation(integer | varname,
+ [
+ ('-', 1, opAssoc.RIGHT),
+ (oneOf('* /'), 2, opAssoc.LEFT),
+ (oneOf('+ -'), 2, opAssoc.LEFT),
+ ])
+
+ arith_expr.runTests('''
+ 5+3*6
+ (5+3)*6
+ -2--11
+ ''', fullDump=False)
+ prints::
+ 5+3*6
+ [[5, '+', [3, '*', 6]]]
+
+ (5+3)*6
+ [[[5, '+', 3], '*', 6]]
+
+ -2--11
+ [[['-', 2], '-', ['-', 11]]]
"""
ret = Forward()
lastExpr = baseExpr | ( lpar + ret + rpar )
for i,operDef in enumerate(opList):
opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
+ termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr
if arity == 3:
if opExpr is None or len(opExpr) != 2:
raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions")
opExpr1, opExpr2 = opExpr
- thisExpr = Forward()#.setName("expr%d" % i)
+ thisExpr = Forward().setName(termName)
if rightLeftAssoc == opAssoc.LEFT:
if arity == 1:
matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) )
@@ -3636,37 +5075,77 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): raise ValueError("operator must indicate right or left associativity")
if pa:
matchExpr.setParseAction( pa )
- thisExpr <<= ( matchExpr | lastExpr )
+ thisExpr <<= ( matchExpr.setName(termName) | lastExpr )
lastExpr = thisExpr
ret <<= lastExpr
return ret
+
operatorPrecedence = infixNotation
+"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release."""
-dblQuotedString = Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*"').setName("string enclosed in double quotes")
-sglQuotedString = Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*'").setName("string enclosed in single quotes")
-quotedString = Regex(r'''(?:"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*")|(?:'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*')''').setName("quotedString using single or double quotes")
-unicodeString = Combine(_L('u') + quotedString.copy())
+dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes")
+sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes")
+quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"'|
+ Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("quotedString using single or double quotes")
+unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal")
def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()):
- """Helper method for defining nested lists enclosed in opening and closing
- delimiters ("(" and ")" are the default).
-
- Parameters:
- - opener - opening character for a nested list (default="("); can also be a pyparsing expression
- - closer - closing character for a nested list (default=")"); can also be a pyparsing expression
- - content - expression for items within the nested lists (default=None)
- - ignoreExpr - expression for ignoring opening and closing delimiters (default=quotedString)
-
- If an expression is not provided for the content argument, the nested
- expression will capture all whitespace-delimited content between delimiters
- as a list of separate values.
-
- Use the C{ignoreExpr} argument to define expressions that may contain
- opening or closing characters that should not be treated as opening
- or closing characters for nesting, such as quotedString or a comment
- expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}.
- The default is L{quotedString}, but if no expressions are to be ignored,
- then pass C{None} for this argument.
+ """
+ Helper method for defining nested lists enclosed in opening and closing
+ delimiters ("(" and ")" are the default).
+
+ Parameters:
+ - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression
+ - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression
+ - content - expression for items within the nested lists (default=C{None})
+ - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString})
+
+ If an expression is not provided for the content argument, the nested
+ expression will capture all whitespace-delimited content between delimiters
+ as a list of separate values.
+
+ Use the C{ignoreExpr} argument to define expressions that may contain
+ opening or closing characters that should not be treated as opening
+ or closing characters for nesting, such as quotedString or a comment
+ expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}.
+ The default is L{quotedString}, but if no expressions are to be ignored,
+ then pass C{None} for this argument.
+
+ Example::
+ data_type = oneOf("void int short long char float double")
+ decl_data_type = Combine(data_type + Optional(Word('*')))
+ ident = Word(alphas+'_', alphanums+'_')
+ number = pyparsing_common.number
+ arg = Group(decl_data_type + ident)
+ LPAR,RPAR = map(Suppress, "()")
+
+ code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment))
+
+ c_function = (decl_data_type("type")
+ + ident("name")
+ + LPAR + Optional(delimitedList(arg), [])("args") + RPAR
+ + code_body("body"))
+ c_function.ignore(cStyleComment)
+
+ source_code = '''
+ int is_odd(int x) {
+ return (x%2);
+ }
+
+ int dec_to_hex(char hchar) {
+ if (hchar >= '0' && hchar <= '9') {
+ return (ord(hchar)-ord('0'));
+ } else {
+ return (10+ord(hchar)-ord('A'));
+ }
+ }
+ '''
+ for func in c_function.searchString(source_code):
+ print("%(name)s (%(type)s) args: %(args)s" % func)
+
+ prints::
+ is_odd (int) args: [['int', 'x']]
+ dec_to_hex (int) args: [['char', 'hchar']]
"""
if opener == closer:
raise ValueError("opening and closing strings cannot be the same")
@@ -3697,23 +5176,86 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
else:
ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) )
+ ret.setName('nested %s%s expression' % (opener,closer))
return ret
def indentedBlock(blockStatementExpr, indentStack, indent=True):
- """Helper method for defining space-delimited indentation blocks, such as
- those used to define block statements in Python source code.
+ """
+ Helper method for defining space-delimited indentation blocks, such as
+ those used to define block statements in Python source code.
- Parameters:
- - blockStatementExpr - expression defining syntax of statement that
+ Parameters:
+ - blockStatementExpr - expression defining syntax of statement that
is repeated within the indented block
- - indentStack - list created by caller to manage indentation stack
+ - indentStack - list created by caller to manage indentation stack
(multiple statementWithIndentedBlock expressions within a single grammar
should share a common indentStack)
- - indent - boolean indicating whether block must be indented beyond the
+ - indent - boolean indicating whether block must be indented beyond the
the current level; set to False for block of left-most statements
- (default=True)
-
- A valid block must contain at least one C{blockStatement}.
+ (default=C{True})
+
+ A valid block must contain at least one C{blockStatement}.
+
+ Example::
+ data = '''
+ def A(z):
+ A1
+ B = 100
+ G = A2
+ A2
+ A3
+ B
+ def BB(a,b,c):
+ BB1
+ def BBA():
+ bba1
+ bba2
+ bba3
+ C
+ D
+ def spam(x,y):
+ def eggs(z):
+ pass
+ '''
+
+
+ indentStack = [1]
+ stmt = Forward()
+
+ identifier = Word(alphas, alphanums)
+ funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":")
+ func_body = indentedBlock(stmt, indentStack)
+ funcDef = Group( funcDecl + func_body )
+
+ rvalue = Forward()
+ funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")")
+ rvalue << (funcCall | identifier | Word(nums))
+ assignment = Group(identifier + "=" + rvalue)
+ stmt << ( funcDef | assignment | identifier )
+
+ module_body = OneOrMore(stmt)
+
+ parseTree = module_body.parseString(data)
+ parseTree.pprint()
+ prints::
+ [['def',
+ 'A',
+ ['(', 'z', ')'],
+ ':',
+ [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]],
+ 'B',
+ ['def',
+ 'BB',
+ ['(', 'a', 'b', 'c', ')'],
+ ':',
+ [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]],
+ 'C',
+ 'D',
+ ['def',
+ 'spam',
+ ['(', 'x', 'y', ')'],
+ ':',
+ [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]]
"""
def checkPeerIndent(s,l,t):
if l >= len(s): return
@@ -3738,9 +5280,9 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): indentStack.pop()
NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress())
- INDENT = Empty() + Empty().setParseAction(checkSubIndent)
- PEER = Empty().setParseAction(checkPeerIndent)
- UNDENT = Empty().setParseAction(checkUnindent)
+ INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT')
+ PEER = Empty().setParseAction(checkPeerIndent).setName('')
+ UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT')
if indent:
smExpr = Group( Optional(NL) +
#~ FollowedBy(blockStatementExpr) +
@@ -3749,57 +5291,387 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): smExpr = Group( Optional(NL) +
(OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )
blockStatementExpr.ignore(_bslash + LineEnd())
- return smExpr
+ return smExpr.setName('indented block')
alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
-anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:"))
-commonHTMLEntity = Combine(_L("&") + oneOf("gt lt amp nbsp quot").setResultsName("entity") +";").streamline()
-_htmlEntityMap = dict(zip("gt lt amp nbsp quot".split(),'><& "'))
-replaceHTMLEntity = lambda t : t.entity in _htmlEntityMap and _htmlEntityMap[t.entity] or None
+anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:").setName('any tag'))
+_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(),'><& "\''))
+commonHTMLEntity = Regex('&(?P<entity>' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity")
+def replaceHTMLEntity(t):
+ """Helper parser action to replace common HTML entities with their special characters"""
+ return _htmlEntityMap.get(t.entity)
# it's easy to get these comment structures wrong - they're very common, so may as well make them available
-cStyleComment = Regex(r"/\*(?:[^*]*\*+)+?/").setName("C style comment")
+cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment")
+"Comment of the form C{/* ... */}"
+
+htmlComment = Regex(r"<!--[\s\S]*?-->").setName("HTML comment")
+"Comment of the form C{<!-- ... -->}"
+
+restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line")
+dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment")
+"Comment of the form C{// ... (to end of line)}"
-htmlComment = Regex(r"<!--[\s\S]*?-->")
-restOfLine = Regex(r".*").leaveWhitespace()
-dblSlashComment = Regex(r"\/\/(\\\n|.)*").setName("// comment")
-cppStyleComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))").setName("C++ style comment")
+cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment")
+"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}"
javaStyleComment = cppStyleComment
+"Same as C{L{cppStyleComment}}"
+
pythonStyleComment = Regex(r"#.*").setName("Python style comment")
+"Comment of the form C{# ... (to end of line)}"
+
_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') +
Optional( Word(" \t") +
~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList")
+"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas.
+ This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}."""
+
+# some other useful expressions - using lower-case class name since we are really using this as a namespace
+class pyparsing_common:
+ """
+ Here are some common low-level expressions that may be useful in jump-starting parser development:
+ - numeric forms (L{integers<integer>}, L{reals<real>}, L{scientific notation<sci_real>})
+ - common L{programming identifiers<identifier>}
+ - network addresses (L{MAC<mac_address>}, L{IPv4<ipv4_address>}, L{IPv6<ipv6_address>})
+ - ISO8601 L{dates<iso8601_date>} and L{datetime<iso8601_datetime>}
+ - L{UUID<uuid>}
+ - L{comma-separated list<comma_separated_list>}
+ Parse actions:
+ - C{L{convertToInteger}}
+ - C{L{convertToFloat}}
+ - C{L{convertToDate}}
+ - C{L{convertToDatetime}}
+ - C{L{stripHTMLTags}}
+ - C{L{upcaseTokens}}
+ - C{L{downcaseTokens}}
+
+ Example::
+ pyparsing_common.number.runTests('''
+ # any int or real number, returned as the appropriate type
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ ''')
+
+ pyparsing_common.fnumber.runTests('''
+ # any int or real number, returned as float
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ ''')
+
+ pyparsing_common.hex_integer.runTests('''
+ # hex numbers
+ 100
+ FF
+ ''')
+
+ pyparsing_common.fraction.runTests('''
+ # fractions
+ 1/2
+ -3/4
+ ''')
+
+ pyparsing_common.mixed_integer.runTests('''
+ # mixed fractions
+ 1
+ 1/2
+ -3/4
+ 1-3/4
+ ''')
+
+ import uuid
+ pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+ pyparsing_common.uuid.runTests('''
+ # uuid
+ 12345678-1234-5678-1234-567812345678
+ ''')
+ prints::
+ # any int or real number, returned as the appropriate type
+ 100
+ [100]
+
+ -100
+ [-100]
+
+ +100
+ [100]
+
+ 3.14159
+ [3.14159]
+
+ 6.02e23
+ [6.02e+23]
+
+ 1e-12
+ [1e-12]
+
+ # any int or real number, returned as float
+ 100
+ [100.0]
+
+ -100
+ [-100.0]
+
+ +100
+ [100.0]
+
+ 3.14159
+ [3.14159]
+
+ 6.02e23
+ [6.02e+23]
+
+ 1e-12
+ [1e-12]
+
+ # hex numbers
+ 100
+ [256]
+
+ FF
+ [255]
+
+ # fractions
+ 1/2
+ [0.5]
+
+ -3/4
+ [-0.75]
+
+ # mixed fractions
+ 1
+ [1]
+
+ 1/2
+ [0.5]
+
+ -3/4
+ [-0.75]
+
+ 1-3/4
+ [1.75]
+
+ # uuid
+ 12345678-1234-5678-1234-567812345678
+ [UUID('12345678-1234-5678-1234-567812345678')]
+ """
+
+ convertToInteger = tokenMap(int)
+ """
+ Parse action for converting parsed integers to Python int
+ """
+
+ convertToFloat = tokenMap(float)
+ """
+ Parse action for converting parsed numbers to Python float
+ """
+
+ integer = Word(nums).setName("integer").setParseAction(convertToInteger)
+ """expression that parses an unsigned integer, returns an int"""
+
+ hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int,16))
+ """expression that parses a hexadecimal integer, returns an int"""
+
+ signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger)
+ """expression that parses an integer with optional leading sign, returns an int"""
+
+ fraction = (signed_integer().setParseAction(convertToFloat) + '/' + signed_integer().setParseAction(convertToFloat)).setName("fraction")
+ """fractional expression of an integer divided by an integer, returns a float"""
+ fraction.addParseAction(lambda t: t[0]/t[-1])
+
+ mixed_integer = (fraction | signed_integer + Optional(Optional('-').suppress() + fraction)).setName("fraction or mixed integer-fraction")
+ """mixed integer of the form 'integer - fraction', with optional leading integer, returns float"""
+ mixed_integer.addParseAction(sum)
+
+ real = Regex(r'[+-]?\d+\.\d*').setName("real number").setParseAction(convertToFloat)
+ """expression that parses a floating point number and returns a float"""
+
+ sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat)
+ """expression that parses a floating point number with optional scientific notation and returns a float"""
+
+ # streamlining this expression makes the docs nicer-looking
+ number = (sci_real | real | signed_integer).streamline()
+ """any numeric expression, returns the corresponding Python type"""
+
+ fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat)
+ """any int or real number, returned as float"""
+
+ identifier = Word(alphas+'_', alphanums+'_').setName("identifier")
+ """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')"""
+
+ ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address")
+ "IPv4 address (C{0.0.0.0 - 255.255.255.255})"
+
+ _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer")
+ _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address")
+ _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part)*(0,6)) + "::" + Optional(_ipv6_part + (':' + _ipv6_part)*(0,6))).setName("short IPv6 address")
+ _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8)
+ _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address")
+ ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address")
+ "IPv6 address (long, short, or mixed form)"
+
+ mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address")
+ "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)"
+
+ @staticmethod
+ def convertToDate(fmt="%Y-%m-%d"):
+ """
+ Helper to create a parse action for converting parsed date string to Python datetime.date
+
+ Params -
+ - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"})
+
+ Example::
+ date_expr = pyparsing_common.iso8601_date.copy()
+ date_expr.setParseAction(pyparsing_common.convertToDate())
+ print(date_expr.parseString("1999-12-31"))
+ prints::
+ [datetime.date(1999, 12, 31)]
+ """
+ def cvt_fn(s,l,t):
+ try:
+ return datetime.strptime(t[0], fmt).date()
+ except ValueError as ve:
+ raise ParseException(s, l, str(ve))
+ return cvt_fn
+
+ @staticmethod
+ def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"):
+ """
+ Helper to create a parse action for converting parsed datetime string to Python datetime.datetime
+
+ Params -
+ - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"})
+
+ Example::
+ dt_expr = pyparsing_common.iso8601_datetime.copy()
+ dt_expr.setParseAction(pyparsing_common.convertToDatetime())
+ print(dt_expr.parseString("1999-12-31T23:59:59.999"))
+ prints::
+ [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)]
+ """
+ def cvt_fn(s,l,t):
+ try:
+ return datetime.strptime(t[0], fmt)
+ except ValueError as ve:
+ raise ParseException(s, l, str(ve))
+ return cvt_fn
+
+ iso8601_date = Regex(r'(?P<year>\d{4})(?:-(?P<month>\d\d)(?:-(?P<day>\d\d))?)?').setName("ISO8601 date")
+ "ISO8601 date (C{yyyy-mm-dd})"
+
+ iso8601_datetime = Regex(r'(?P<year>\d{4})-(?P<month>\d\d)-(?P<day>\d\d)[T ](?P<hour>\d\d):(?P<minute>\d\d)(:(?P<second>\d\d(\.\d*)?)?)?(?P<tz>Z|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime")
+ "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}"
+
+ uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID")
+ "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})"
+
+ _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress()
+ @staticmethod
+ def stripHTMLTags(s, l, tokens):
+ """
+ Parse action to remove HTML tags from web page HTML source
+
+ Example::
+ # strip HTML links from normal text
+ text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>'
+ td,td_end = makeHTMLTags("TD")
+ table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end
+
+ print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page'
+ """
+ return pyparsing_common._html_stripper.transformString(tokens[0])
+
+ _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',')
+ + Optional( White(" \t") ) ) ).streamline().setName("commaItem")
+ comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list")
+ """Predefined expression of 1 or more printable words or quoted strings, separated by commas."""
+
+ upcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).upper()))
+ """Parse action to convert tokens to upper case."""
+
+ downcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).lower()))
+ """Parse action to convert tokens to lower case."""
if __name__ == "__main__":
- selectToken = CaselessLiteral( "select" )
- fromToken = CaselessLiteral( "from" )
-
- ident = Word( alphas, alphanums + "_$" )
- columnName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
- columnNameList = Group( delimitedList( columnName ) ).setName("columns")
- tableName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
- tableNameList = Group( delimitedList( tableName ) ).setName("tables")
- simpleSQL = ( selectToken + \
- ( '*' | columnNameList ).setResultsName( "columns" ) + \
- fromToken + \
- tableNameList.setResultsName( "tables" ) )
-
- simpleSQL.runTests("""\
- SELECT * from XYZZY, ABC
- select * from SYS.XYZZY
- Select A from Sys.dual
- Select AA,BB,CC from Sys.dual
- Select A, B, C from Sys.dual
- Select A, B, C from Sys.dual
- Xelect A, B, C from Sys.dual
- Select A, B, C frox Sys.dual
- Select
- Select ^^^ frox Sys.dual
- Select A, B, C from Sys.dual, Table2""")
+ selectToken = CaselessLiteral("select")
+ fromToken = CaselessLiteral("from")
+
+ ident = Word(alphas, alphanums + "_$")
+
+ columnName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+ columnNameList = Group(delimitedList(columnName)).setName("columns")
+ columnSpec = ('*' | columnNameList)
+
+ tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+ tableNameList = Group(delimitedList(tableName)).setName("tables")
+ simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables")
+
+ # demo runTests method, including embedded comments in test string
+ simpleSQL.runTests("""
+ # '*' as column list and dotted table name
+ select * from SYS.XYZZY
+
+ # caseless match on "SELECT", and casts back to "select"
+ SELECT * from XYZZY, ABC
+
+ # list of column names, and mixed case SELECT keyword
+ Select AA,BB,CC from Sys.dual
+
+ # multiple tables
+ Select A, B, C from Sys.dual, Table2
+
+ # invalid SELECT keyword - should fail
+ Xelect A, B, C from Sys.dual
+
+ # incomplete command - should fail
+ Select
+
+ # invalid column name - should fail
+ Select ^^^ frox Sys.dual
+
+ """)
+
+ pyparsing_common.number.runTests("""
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ """)
+
+ # any int or real number, returned as float
+ pyparsing_common.fnumber.runTests("""
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ """)
+
+ pyparsing_common.hex_integer.runTests("""
+ 100
+ FF
+ """)
+
+ import uuid
+ pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+ pyparsing_common.uuid.runTests("""
+ 12345678-1234-5678-1234-567812345678
+ """)
diff --git a/pkg_resources/_vendor/vendored.txt b/pkg_resources/_vendor/vendored.txt index 46532c0a..22e1309c 100644 --- a/pkg_resources/_vendor/vendored.txt +++ b/pkg_resources/_vendor/vendored.txt @@ -1,3 +1,4 @@ packaging==16.7 -pyparsing==2.0.6 +pyparsing==2.1.9 six==1.10.0 +appdirs==1.4.0 diff --git a/pkg_resources/extern/__init__.py b/pkg_resources/extern/__init__.py index 6758d36f..b4156fec 100644 --- a/pkg_resources/extern/__init__.py +++ b/pkg_resources/extern/__init__.py @@ -6,6 +6,7 @@ class VendorImporter: A PEP 302 meta path importer for finding optionally-vendored or otherwise naturally-installed packages from root_name. """ + def __init__(self, root_name, vendored_names=(), vendor_pkg=None): self.root_name = root_name self.vendored_names = set(vendored_names) @@ -67,5 +68,6 @@ class VendorImporter: if self not in sys.meta_path: sys.meta_path.append(self) -names = 'packaging', 'pyparsing', 'six' + +names = 'packaging', 'pyparsing', 'six', 'appdirs' VendorImporter(__name__, names).install() diff --git a/pkg_resources/tests/test_markers.py b/pkg_resources/tests/test_markers.py index 8d451de3..78810b6e 100644 --- a/pkg_resources/tests/test_markers.py +++ b/pkg_resources/tests/test_markers.py @@ -5,6 +5,7 @@ except ImportError: from pkg_resources import evaluate_marker + @mock.patch('platform.python_version', return_value='2.7.10') def test_ordering(python_version_mock): assert evaluate_marker("python_full_version > '2.7.3'") is True diff --git a/pkg_resources/tests/test_pkg_resources.py b/pkg_resources/tests/test_pkg_resources.py index 8b276ffc..361fe657 100644 --- a/pkg_resources/tests/test_pkg_resources.py +++ b/pkg_resources/tests/test_pkg_resources.py @@ -24,6 +24,7 @@ try: except NameError: unicode = str + def timestamp(dt): """ Return a timestamp for a local, naive datetime instance. @@ -34,13 +35,16 @@ def timestamp(dt): # Python 3.2 and earlier return time.mktime(dt.timetuple()) + class EggRemover(unicode): + def __call__(self): if self in sys.path: sys.path.remove(self) if os.path.exists(self): os.remove(self) + class TestZipProvider(object): finalizers = [] @@ -94,7 +98,9 @@ class TestZipProvider(object): assert f.read() == 'hello, world!' manager.cleanup_resources() + class TestResourceManager(object): + def test_get_cache_path(self): mgr = pkg_resources.ResourceManager() path = mgr.get_cache_path('foo') @@ -107,6 +113,7 @@ class TestIndependence: """ Tests to ensure that pkg_resources runs independently from setuptools. """ + def test_setuptools_not_imported(self): """ In a separate Python environment, import pkg_resources and assert @@ -122,7 +129,6 @@ class TestIndependence: subprocess.check_call(cmd) - class TestDeepVersionLookupDistutils(object): @pytest.fixture diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index 31847dc8..2ed56233 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import os import sys import string +import platform from pkg_resources.extern.six.moves import map @@ -33,6 +34,7 @@ class Metadata(pkg_resources.EmptyProvider): dist_from_fn = pkg_resources.Distribution.from_filename + class TestDistro: def testCollection(self): @@ -50,15 +52,15 @@ class TestDistro: assert list(ad) == ['foopkg'] # Distributions sort by version - assert [dist.version for dist in ad['FooPkg']] == ['1.4','1.3-1','1.2'] + assert [dist.version for dist in ad['FooPkg']] == ['1.4', '1.3-1', '1.2'] # Removing a distribution leaves sequence alone ad.remove(ad['FooPkg'][1]) - assert [dist.version for dist in ad['FooPkg']] == ['1.4','1.2'] + assert [dist.version for dist in ad['FooPkg']] == ['1.4', '1.2'] # And inserting adds them in order ad.add(dist_from_fn("FooPkg-1.9.egg")) - assert [dist.version for dist in ad['FooPkg']] == ['1.9','1.4','1.2'] + assert [dist.version for dist in ad['FooPkg']] == ['1.9', '1.4', '1.2'] ws = WorkingSet([]) foo12 = dist_from_fn("FooPkg-1.2-py2.4.egg") @@ -85,7 +87,7 @@ class TestDistro: ws.add(foo14) assert ad.best_match(req, ws).version == '1.4' - def checkFooPkg(self,d): + def checkFooPkg(self, d): assert d.project_name == "FooPkg" assert d.key == "foopkg" assert d.version == "1.3.post1" @@ -96,13 +98,13 @@ class TestDistro: def testDistroBasics(self): d = Distribution( "/some/path", - project_name="FooPkg",version="1.3-1",py_version="2.4",platform="win32" + project_name="FooPkg", version="1.3-1", py_version="2.4", platform="win32" ) self.checkFooPkg(d) d = Distribution("/some/path") assert d.py_version == sys.version[:3] - assert d.platform == None + assert d.platform is None def testDistroParse(self): d = dist_from_fn("FooPkg-1.3.post1-py2.4-win32.egg") @@ -113,8 +115,8 @@ class TestDistro: def testDistroMetadata(self): d = Distribution( "/some/path", project_name="FooPkg", py_version="2.4", platform="win32", - metadata = Metadata( - ('PKG-INFO',"Metadata-Version: 1.0\nVersion: 1.3-1\n") + metadata=Metadata( + ('PKG-INFO', "Metadata-Version: 1.0\nVersion: 1.3-1\n") ) ) self.checkFooPkg(d) @@ -152,7 +154,7 @@ class TestDistro: list(map(ws.add, targets)) with pytest.raises(VersionConflict): ws.resolve(parse_requirements("Foo==0.9"), ad) - ws = WorkingSet([]) # reset + ws = WorkingSet([]) # reset # Request an extra that causes an unresolved dependency for "Baz" with pytest.raises(pkg_resources.DistributionNotFound): @@ -163,7 +165,7 @@ class TestDistro: ad.add(Baz) # Activation list now includes resolved dependency - assert list(ws.resolve(parse_requirements("Foo[bar]"), ad)) ==[Foo,Baz] + assert list(ws.resolve(parse_requirements("Foo[bar]"), ad)) == [Foo, Baz] # Requests for conflicting versions produce VersionConflict with pytest.raises(VersionConflict) as vc: ws.resolve(parse_requirements("Foo==1.2\nFoo!=1.2"), ad) @@ -217,7 +219,7 @@ class TestDistro: quux = Distribution.from_filename("/foo_dir/quux-1.0.dist-info") ad.add(quux) res = list(ws.resolve(parse_requirements("Foo[baz]"), ad)) - assert res == [Foo,quux] + assert res == [Foo, quux] def test_marker_evaluation_with_multiple_extras(self): ad = pkg_resources.Environment([]) @@ -237,7 +239,7 @@ class TestDistro: fred = Distribution.from_filename("/foo_dir/fred-0.1.dist-info") ad.add(fred) res = list(ws.resolve(parse_requirements("Foo[baz,bar]"), ad)) - assert sorted(res) == [fred,quux,Foo] + assert sorted(res) == [fred, quux, Foo] def test_marker_evaluation_with_extras_loop(self): ad = pkg_resources.Environment([]) @@ -273,19 +275,19 @@ class TestDistro: docutils>=0.3 [fastcgi] fcgiapp>=0.1""") - self.checkRequires(d,"Twisted>=1.5") + self.checkRequires(d, "Twisted>=1.5") self.checkRequires( - d,"Twisted>=1.5 ZConfig>=2.0 docutils>=0.3".split(), ["docgen"] + d, "Twisted>=1.5 ZConfig>=2.0 docutils>=0.3".split(), ["docgen"] ) self.checkRequires( - d,"Twisted>=1.5 fcgiapp>=0.1".split(), ["fastcgi"] + d, "Twisted>=1.5 fcgiapp>=0.1".split(), ["fastcgi"] ) self.checkRequires( - d,"Twisted>=1.5 ZConfig>=2.0 docutils>=0.3 fcgiapp>=0.1".split(), - ["docgen","fastcgi"] + d, "Twisted>=1.5 ZConfig>=2.0 docutils>=0.3 fcgiapp>=0.1".split(), + ["docgen", "fastcgi"] ) self.checkRequires( - d,"Twisted>=1.5 fcgiapp>=0.1 ZConfig>=2.0 docutils>=0.3".split(), + d, "Twisted>=1.5 fcgiapp>=0.1 ZConfig>=2.0 docutils>=0.3".split(), ["fastcgi", "docgen"] ) with pytest.raises(pkg_resources.UnknownExtra): @@ -293,6 +295,7 @@ class TestDistro: class TestWorkingSet: + def test_find_conflicting(self): ws = WorkingSet([]) Foo = Distribution.from_filename("/foo_dir/Foo-1.2.egg") @@ -347,7 +350,7 @@ class TestEntryPoints: def setup_method(self, method): self.dist = Distribution.from_filename( - "FooPkg-1.2-py2.4.egg", metadata=Metadata(('requires.txt','[x]'))) + "FooPkg-1.2-py2.4.egg", metadata=Metadata(('requires.txt', '[x]'))) def testBasics(self): ep = EntryPoint( @@ -379,6 +382,7 @@ class TestEntryPoints: assert ep.name == 'html+mako' reject_specs = "foo", "x=a:b:c", "q=x/na", "fez=pish:tush-z", "x=f[a]>2" + @pytest.mark.parametrize("reject_spec", reject_specs) def test_reject_spec(self, reject_spec): with pytest.raises(ValueError): @@ -404,7 +408,7 @@ class TestEntryPoints: submap_expect = dict( feature1=EntryPoint('feature1', 'somemodule', ['somefunction']), - feature2=EntryPoint('feature2', 'another.module', ['SomeClass'], ['extra1','extra2']), + feature2=EntryPoint('feature2', 'another.module', ['SomeClass'], ['extra1', 'extra2']), feature3=EntryPoint('feature3', 'this.module', extras=['something']) ) submap_str = """ @@ -422,10 +426,10 @@ class TestEntryPoints: EntryPoint.parse_group("x", ["foo=baz", "foo=bar"]) def testParseMap(self): - m = EntryPoint.parse_map({'xyz':self.submap_str}) + m = EntryPoint.parse_map({'xyz': self.submap_str}) self.checkSubMap(m['xyz']) assert list(m.keys()) == ['xyz'] - m = EntryPoint.parse_map("[xyz]\n"+self.submap_str) + m = EntryPoint.parse_map("[xyz]\n" + self.submap_str) self.checkSubMap(m['xyz']) assert list(m.keys()) == ['xyz'] with pytest.raises(ValueError): @@ -433,6 +437,7 @@ class TestEntryPoints: with pytest.raises(ValueError): EntryPoint.parse_map(self.submap_str) + class TestRequirements: def testBasics(self): @@ -479,7 +484,7 @@ class TestRequirements: hash(( "twisted", packaging.specifiers.SpecifierSet(">=1.2"), - frozenset(["foo","bar"]), + frozenset(["foo", "bar"]), None )) ) @@ -520,9 +525,9 @@ class TestParsing: assert list(parse_requirements('')) == [] def testYielding(self): - for inp,out in [ - ([], []), ('x',['x']), ([[]],[]), (' x\n y', ['x','y']), - (['x\n\n','y'], ['x','y']), + for inp, out in [ + ([], []), ('x', ['x']), ([[]], []), (' x\n y', ['x', 'y']), + (['x\n\n', 'y'], ['x', 'y']), ]: assert list(pkg_resources.yield_lines(inp)) == out @@ -625,9 +630,9 @@ class TestParsing: req, = parse_requirements('foo >= 1.0, < 3') def testVersionEquality(self): - def c(s1,s2): - p1, p2 = parse_version(s1),parse_version(s2) - assert p1 == p2, (s1,s2,p1,p2) + def c(s1, s2): + p1, p2 = parse_version(s1), parse_version(s2) + assert p1 == p2, (s1, s2, p1, p2) c('1.2-rc1', '1.2rc1') c('0.4', '0.4.0') @@ -641,13 +646,13 @@ class TestParsing: c('1.2.a', '1.2a') def testVersionOrdering(self): - def c(s1,s2): - p1, p2 = parse_version(s1),parse_version(s2) - assert p1<p2, (s1,s2,p1,p2) + def c(s1, s2): + p1, p2 = parse_version(s1), parse_version(s2) + assert p1 < p2, (s1, s2, p1, p2) - c('2.1','2.1.1') - c('2a1','2b0') - c('2a1','2.1') + c('2.1', '2.1.1') + c('2a1', '2b0') + c('2a1', '2.1') c('2.3a1', '2.3') c('2.1-1', '2.1-2') c('2.1-1', '2.1.1') @@ -659,18 +664,18 @@ class TestParsing: c('0.4', '4.0') c('0.0.4', '0.4.0') c('0post1', '0.4post1') - c('2.1.0-rc1','2.1.0') - c('2.1dev','2.1a0') + c('2.1.0-rc1', '2.1.0') + c('2.1dev', '2.1a0') - torture =""" + torture = """ 0.80.1-3 0.80.1-2 0.80.1-1 0.79.9999+0.80.0pre4-1 0.79.9999+0.80.0pre2-3 0.79.9999+0.80.0pre2-2 0.77.2-1 0.77.1-1 0.77.0-1 """.split() - for p,v1 in enumerate(torture): - for v2 in torture[p+1:]: - c(v2,v1) + for p, v1 in enumerate(torture): + for v2 in torture[p + 1:]: + c(v2, v1) def testVersionBuildout(self): """ @@ -763,6 +768,9 @@ class TestNamespaces: pkg_resources._namespace_packages = saved_ns_pkgs sys.path = saved_sys_path + issue591 = pytest.mark.xfail(platform.system() == 'Windows', reason="#591") + + @issue591 def test_two_levels_deep(self, symlinked_tmpdir): """ Test nested namespace packages @@ -796,6 +804,7 @@ class TestNamespaces: ] assert pkg1.pkg2.__path__ == expected + @issue591 def test_path_order(self, symlinked_tmpdir): """ Test that if multiple versions of the same namespace package subpackage @@ -1,3 +1,6 @@ [pytest] addopts=--doctest-modules --ignore release.py --ignore setuptools/lib2to3_ex.py --ignore tests/manual_test.py --ignore tests/shlib_test --doctest-glob=pkg_resources/api_tests.txt --ignore scripts/upload-old-releases-as-zip.py --ignore pavement.py -norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern +norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern .* +flake8-ignore = + setuptools/site-patch.py F821 + setuptools/py*compat.py F811 @@ -10,7 +10,7 @@ binary = bdist_egg upload --show-response test = pytest [upload] -repository = https://upload.pypi.io/legacy/ +repository = https://upload.pypi.org/legacy/ [sdist] formats = gztar zip @@ -8,23 +8,27 @@ import os import sys import textwrap -# Allow to run setup.py from another directory. -os.chdir(os.path.dirname(os.path.abspath(__file__))) +import setuptools -src_root = None -from distutils.util import convert_path +here = os.path.dirname(__file__) -command_ns = {} -init_path = convert_path('setuptools/command/__init__.py') -with open(init_path) as init_file: - exec(init_file.read(), command_ns) -SETUP_COMMANDS = command_ns['__all__'] +def require_metadata(): + "Prevent improper installs without necessary metadata. See #659" + if not os.path.exists('setuptools.egg-info'): + msg = "Cannot build setuptools without metadata. Run bootstrap.py" + raise RuntimeError(msg) -import setuptools -scripts = [] +def read_commands(): + command_ns = {} + cmd_module_path = 'setuptools/command/__init__.py' + init_path = os.path.join(here, cmd_module_path) + with open(init_path) as init_file: + exec(init_file.read(), command_ns) + return command_ns['__all__'] + def _gen_console_scripts(): yield "easy_install = setuptools.command.easy_install:main" @@ -41,21 +45,27 @@ def _gen_console_scripts(): yield ("easy_install-{shortver} = setuptools.command.easy_install:main" .format(shortver=sys.version[:3])) -console_scripts = list(_gen_console_scripts()) -readme_file = io.open('README.rst', encoding='utf-8') - -with readme_file: +readme_path = os.path.join(here, 'README.rst') +with io.open(readme_path, encoding='utf-8') as readme_file: long_description = readme_file.read() -package_data = { - 'setuptools': ['script (dev).tmpl', 'script.tmpl', 'site-patch.py']} +package_data = dict( + setuptools=['script (dev).tmpl', 'script.tmpl', 'site-patch.py'], +) + force_windows_specific_files = ( os.environ.get("SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES") not in (None, "", "0") ) -if (sys.platform == 'win32' or (os.name == 'java' and os._name == 'nt')) \ - or force_windows_specific_files: + +include_windows_files = ( + sys.platform == 'win32' or + os.name == 'java' and os._name == 'nt' or + force_windows_specific_files +) + +if include_windows_files: package_data.setdefault('setuptools', []).extend(['*.exe']) package_data.setdefault('setuptools.command', []).extend(['*.xml']) @@ -64,17 +74,29 @@ pytest_runner = ['pytest-runner'] if needs_pytest else [] needs_wheel = set(['release', 'bdist_wheel']).intersection(sys.argv) wheel = ['wheel'] if needs_wheel else [] + +def pypi_link(pkg_filename): + """ + Given the filename, including md5 fragment, construct the + dependency link for PyPI. + """ + root = 'https://pypi.python.org/packages/source' + name, sep, rest = pkg_filename.partition('-') + parts = root, name[0], name, pkg_filename + return '/'.join(parts) + + setup_params = dict( name="setuptools", use_scm_version=True, description="Easily download, build, install, upgrade, and uninstall " - "Python packages", + "Python packages", author="Python Packaging Authority", author_email="distutils-sig@python.org", long_description=long_description, keywords="CPAN PyPI distutils eggs package management", url="https://github.com/pypa/setuptools", - src_root=src_root, + src_root=None, packages=setuptools.find_packages(exclude=['*.tests']), package_data=package_data, @@ -85,7 +107,7 @@ setup_params = dict( entry_points={ "distutils.commands": [ "%(cmd)s = setuptools.command.%(cmd)s:%(cmd)s" % locals() - for cmd in SETUP_COMMANDS + for cmd in read_commands() ], "distutils.setup_keywords": [ "eager_resources = setuptools.dist:assert_string_list", @@ -94,6 +116,7 @@ setup_params = dict( "install_requires = setuptools.dist:check_requirements", "tests_require = setuptools.dist:check_requirements", "setup_requires = setuptools.dist:check_requirements", + "python_requires = setuptools.dist:check_specifier", "entry_points = setuptools.dist:check_entry_points", "test_suite = setuptools.dist:check_test_suite", "zip_safe = setuptools.dist:assert_bool", @@ -119,7 +142,7 @@ setup_params = dict( "depends.txt = setuptools.command.egg_info:warn_depends_obsolete", "dependency_links.txt = setuptools.command.egg_info:overwrite_arg", ], - "console_scripts": console_scripts, + "console_scripts": list(_gen_console_scripts()), "setuptools.installation": ['eggsecutable = setuptools.command.easy_install:bootstrap'], @@ -144,16 +167,21 @@ setup_params = dict( """).strip().splitlines(), extras_require={ "ssl:sys_platform=='win32'": "wincertstore==0.2", - "certs": "certifi==2016.2.28", + "certs": "certifi==2016.8.31", }, dependency_links=[ - 'https://pypi.python.org/packages/source/c/certifi/certifi-2016.2.28.tar.gz#md5=5d672aa766e1f773c75cfeccd02d3650', - 'https://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2', + pypi_link( + 'certifi-2016.8.31.tar.gz#md5=2f22d484a36d38d98be74f9eeb2846ec', + ), + pypi_link( + 'wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2', + ), ], scripts=[], tests_require=[ 'setuptools[ssl]', - 'pytest>=2.8', + 'pytest-flake8', + 'pytest>=3.0.2', ] + (['mock'] if sys.version_info[:2] < (3, 3) else []), setup_requires=[ 'setuptools_scm>=1.9', @@ -161,4 +189,7 @@ setup_params = dict( ) if __name__ == '__main__': + # allow setup.py to run from another directory + here and os.chdir(here) + require_metadata() dist = setuptools.setup(**setup_params) diff --git a/setuptools/__init__.py b/setuptools/__init__.py index b8cec4c3..42f6a5d6 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -4,20 +4,20 @@ import os import functools import distutils.core import distutils.filelist -from distutils.core import Command as _Command from distutils.util import convert_path from fnmatch import fnmatchcase -from setuptools.extern.six.moves import filterfalse, map +from setuptools.extern.six.moves import filter, filterfalse, map import pkg_resources from setuptools.extension import Extension -from setuptools.dist import Distribution, Feature, _get_unpatched +from setuptools.dist import Distribution, Feature from setuptools.depends import Require +from . import monkey __all__ = [ 'setup', 'Distribution', 'Feature', 'Command', 'Extension', 'Require', - 'find_packages' + 'find_packages', ] __version__ = pkg_resources.require('setuptools')[0].version @@ -32,12 +32,18 @@ lib2to3_fixer_packages = ['lib2to3.fixes'] class PackageFinder(object): + """ + Generate a list of all Python packages found within a directory + """ + @classmethod def find(cls, where='.', exclude=(), include=('*',)): """Return a list all Python packages found within directory 'where' - 'where' should be supplied as a "cross-platform" (i.e. URL-style) - path; it will be converted to the appropriate local path syntax. + 'where' is the root directory which will be searched for packages. It + should be supplied as a "cross-platform" (i.e. URL-style) path; it will + be converted to the appropriate local path syntax. + 'exclude' is a sequence of package names to exclude; '*' can be used as a wildcard in the names, such that 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself). @@ -46,78 +52,64 @@ class PackageFinder(object): specified, only the named packages will be included. If it's not specified, all found packages will be included. 'include' can contain shell style wildcard patterns just like 'exclude'. - - The list of included packages is built up first and then any - explicitly excluded packages are removed from it. """ - out = cls._find_packages_iter(convert_path(where)) - out = cls.require_parents(out) - includes = cls._build_filter(*include) - excludes = cls._build_filter('ez_setup', '*__pycache__', *exclude) - out = filter(includes, out) - out = filterfalse(excludes, out) - return list(out) - @staticmethod - def require_parents(packages): - """ - Exclude any apparent package that apparently doesn't include its - parent. - - For example, exclude 'foo.bar' if 'foo' is not present. - """ - found = [] - for pkg in packages: - base, sep, child = pkg.rpartition('.') - if base and base not in found: - continue - found.append(pkg) - yield pkg + return list(cls._find_packages_iter( + convert_path(where), + cls._build_filter('ez_setup', '*__pycache__', *exclude), + cls._build_filter(*include))) - @staticmethod - def _candidate_dirs(base_path): + @classmethod + def _find_packages_iter(cls, where, exclude, include): """ - Return all dirs in base_path that might be packages. + All the packages found in 'where' that pass the 'include' filter, but + not the 'exclude' filter. """ - has_dot = lambda name: '.' in name - for root, dirs, files in os.walk(base_path, followlinks=True): - # Exclude directories that contain a period, as they cannot be - # packages. Mutate the list to avoid traversal. - dirs[:] = filterfalse(has_dot, dirs) - for dir in dirs: - yield os.path.relpath(os.path.join(root, dir), base_path) - - @classmethod - def _find_packages_iter(cls, base_path): - candidates = cls._candidate_dirs(base_path) - return ( - path.replace(os.path.sep, '.') - for path in candidates - if cls._looks_like_package(os.path.join(base_path, path)) - ) + for root, dirs, files in os.walk(where, followlinks=True): + # Copy dirs to iterate over it, then empty dirs. + all_dirs = dirs[:] + dirs[:] = [] + + for dir in all_dirs: + full_path = os.path.join(root, dir) + rel_path = os.path.relpath(full_path, where) + package = rel_path.replace(os.path.sep, '.') + + # Check if the directory is a package and passes the filters + if ('.' not in dir + and include(package) + and not exclude(package) + and cls._looks_like_package(full_path)): + yield package + dirs.append(dir) @staticmethod def _looks_like_package(path): + """Does a directory look like a package?""" return os.path.isfile(os.path.join(path, '__init__.py')) @staticmethod def _build_filter(*patterns): """ Given a list of patterns, return a callable that will be true only if - the input matches one of the patterns. + the input matches at least one of the patterns. """ return lambda name: any(fnmatchcase(name, pat=pat) for pat in patterns) + class PEP420PackageFinder(PackageFinder): + @staticmethod def _looks_like_package(path): return True + find_packages = PackageFinder.find setup = distutils.core.setup -_Command = _get_unpatched(_Command) +_Command = monkey.get_unpatched(distutils.core.Command) + class Command(_Command): __doc__ = _Command.__doc__ @@ -137,9 +129,6 @@ class Command(_Command): vars(cmd).update(kw) return cmd -# we can't patch distutils.cmd, alas -distutils.core.Command = Command - def _find_all_simple(path): """ @@ -165,5 +154,4 @@ def findall(dir=os.curdir): return list(files) -# fix findall bug in distutils (http://bugs.python.org/issue12885) -distutils.filelist.findall = findall +monkey.patch_all() diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index b3c9fa56..b6411cc5 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -1,24 +1,27 @@ """Utilities for extracting common archive formats""" - -__all__ = [ - "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", - "UnrecognizedFormat", "extraction_drivers", "unpack_directory", -] - import zipfile import tarfile import os import shutil import posixpath import contextlib -from pkg_resources import ensure_directory, ContextualZipFile from distutils.errors import DistutilsError +from pkg_resources import ensure_directory, ContextualZipFile + + +__all__ = [ + "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", + "UnrecognizedFormat", "extraction_drivers", "unpack_directory", +] + + class UnrecognizedFormat(DistutilsError): """Couldn't recognize the archive type""" -def default_filter(src,dst): + +def default_filter(src, dst): """The default progress/filter callback; returns True for all files""" return dst @@ -167,4 +170,5 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): pass return True + extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile diff --git a/setuptools/cli-arm-32.exe b/setuptools/cli-arm-32.exe Binary files differdeleted file mode 100644 index 2f40402d..00000000 --- a/setuptools/cli-arm-32.exe +++ /dev/null diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 9cebd7fa..cbea7537 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -8,8 +8,8 @@ from distutils import log from types import CodeType import sys import os -import marshal import textwrap +import marshal from setuptools.extern import six @@ -129,7 +129,7 @@ class bdist_egg(Command): self.distribution.data_files.append(item) try: - log.info("installing package data to %s" % self.bdist_dir) + log.info("installing package data to %s", self.bdist_dir) self.call_command('install_data', force=0, root=None) finally: self.distribution.data_files = old @@ -152,7 +152,7 @@ class bdist_egg(Command): self.run_command("egg_info") # We run install_lib before install_data, because some data hacks # pull their data path from the install_lib command. - log.info("installing library code to %s" % self.bdist_dir) + log.info("installing library code to %s", self.bdist_dir) instcmd = self.get_finalized_command('install') old_root = instcmd.root instcmd.root = None @@ -169,7 +169,7 @@ class bdist_egg(Command): pyfile = os.path.join(self.bdist_dir, strip_module(filename) + '.py') self.stubs.append(pyfile) - log.info("creating stub loader for %s" % ext_name) + log.info("creating stub loader for %s", ext_name) if not self.dry_run: write_stub(os.path.basename(ext_name), pyfile) to_compile.append(pyfile) @@ -186,14 +186,14 @@ class bdist_egg(Command): self.mkpath(egg_info) if self.distribution.scripts: script_dir = os.path.join(egg_info, 'scripts') - log.info("installing scripts to %s" % script_dir) + log.info("installing scripts to %s", script_dir) self.call_command('install_scripts', install_dir=script_dir, no_ep=1) self.copy_metadata_to(egg_info) native_libs = os.path.join(egg_info, "native_libs.txt") if all_outputs: - log.info("writing %s" % native_libs) + log.info("writing %s", native_libs) if not self.dry_run: ensure_directory(native_libs) libs_file = open(native_libs, 'wt') @@ -201,7 +201,7 @@ class bdist_egg(Command): libs_file.write('\n') libs_file.close() elif os.path.isfile(native_libs): - log.info("removing %s" % native_libs) + log.info("removing %s", native_libs) if not self.dry_run: os.unlink(native_libs) @@ -432,6 +432,7 @@ def can_scan(): # Attribute names of options for commands that might need to be convinced to # install to the egg build directory + INSTALL_DIRECTORY_ATTRS = [ 'install_lib', 'install_dir', 'install_data', 'install_base' ] @@ -457,7 +458,7 @@ def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True, p = path[len(base_dir) + 1:] if not dry_run: z.write(path, p) - log.debug("adding '%s'" % p) + log.debug("adding '%s'", p) compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED if not dry_run: diff --git a/setuptools/command/bdist_wininst.py b/setuptools/command/bdist_wininst.py index 073de97b..8243c917 100755 --- a/setuptools/command/bdist_wininst.py +++ b/setuptools/command/bdist_wininst.py @@ -2,6 +2,7 @@ import distutils.command.bdist_wininst as orig class bdist_wininst(orig.bdist_wininst): + def reinitialize_command(self, command, reinit_subcommands=0): """ Supplement reinitialize_command to work around diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 1caf8c81..454c91fb 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -1,14 +1,16 @@ +import os +import sys +import itertools +import imp from distutils.command.build_ext import build_ext as _du_build_ext from distutils.file_util import copy_file from distutils.ccompiler import new_compiler -from distutils.sysconfig import customize_compiler +from distutils.sysconfig import customize_compiler, get_config_var from distutils.errors import DistutilsError from distutils import log -import os -import sys -import itertools from setuptools.extension import Library +from setuptools.extern import six try: # Attempt to use Cython for building extensions, if available @@ -16,10 +18,8 @@ try: except ImportError: _build_ext = _du_build_ext -from distutils.sysconfig import get_config_var - -get_config_var("LDSHARED") # make sure _config_vars is initialized -del get_config_var +# make sure _config_vars is initialized +get_config_var("LDSHARED") from distutils.sysconfig import _config_vars as _CONFIG_VARS @@ -59,7 +59,18 @@ elif os.name != 'nt': if_dl = lambda s: s if have_rtld else '' + +def get_abi3_suffix(): + """Return the file extension for an abi3-compliant Extension()""" + for suffix, _, _ in (s for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION): + if '.abi3' in suffix: # Unix + return suffix + elif suffix == '.pyd': # Windows + return suffix + + class build_ext(_build_ext): + def run(self): """Build extensions in build directory, then copy if --inplace""" old_inplace, self.inplace = self.inplace, 0 @@ -94,6 +105,15 @@ class build_ext(_build_ext): filename = _build_ext.get_ext_filename(self, fullname) if fullname in self.ext_map: ext = self.ext_map[fullname] + use_abi3 = ( + six.PY3 + and getattr(ext, 'py_limited_api') + and get_abi3_suffix() + ) + if use_abi3: + so_ext = get_config_var('EXT_SUFFIX') + filename = filename[:-len(so_ext)] + filename = filename + get_abi3_suffix() if isinstance(ext, Library): fn, ext = os.path.splitext(filename) return self.shlib_compiler.library_filename(fn, libtype) diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 0bad8295..b5de9bda 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -15,6 +15,7 @@ try: from setuptools.lib2to3_ex import Mixin2to3 except ImportError: class Mixin2to3: + def run_2to3(self, files, doctests=True): "do nothing" diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py index 11b5df10..3eb86120 100755 --- a/setuptools/command/develop.py +++ b/setuptools/command/develop.py @@ -186,6 +186,7 @@ class VersionlessRequirement(object): >>> str(adapted_dist.as_requirement()) 'foo' """ + def __init__(self, dist): self.__dist = dist diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index ccc66cf7..a3792ce2 100755 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -8,7 +8,7 @@ A tool for doing automatic download/extract/build of distutils-based Python packages. For detailed documentation, see the accompanying EasyInstall.txt file, or visit the `EasyInstall home page`__. -__ https://pythonhosted.org/setuptools/easy_install.html +__ https://setuptools.readthedocs.io/en/latest/easy_install.html """ @@ -32,7 +32,6 @@ import zipfile import re import stat import random -import platform import textwrap import warnings import site @@ -50,8 +49,9 @@ from setuptools.sandbox import run_setup from setuptools.py31compat import get_path, get_config_vars from setuptools.command import setopt from setuptools.archive_util import unpack_archive -from setuptools.package_index import PackageIndex -from setuptools.package_index import URL_SCHEME +from setuptools.package_index import ( + PackageIndex, parse_requirement_arg, URL_SCHEME, +) from setuptools.command import bdist_egg, egg_info from pkg_resources import ( yield_lines, normalize_path, resource_string, ensure_directory, @@ -432,7 +432,7 @@ class easy_install(Command): """ try: pid = os.getpid() - except: + except Exception: pid = random.randint(0, sys.maxsize) return os.path.join(self.install_dir, "test-easy-install-%s" % pid) @@ -513,7 +513,7 @@ class easy_install(Command): For information on other options, you may wish to consult the documentation at: - https://pythonhosted.org/setuptools/easy_install.html + https://setuptools.readthedocs.io/en/latest/easy_install.html Please make the appropriate changes for your system and try again. """).lstrip() @@ -930,7 +930,7 @@ class easy_install(Command): destination, fix_zipimporter_caches=new_dist_is_zipped, ) - except: + except Exception: update_dist_caches(destination, fix_zipimporter_caches=False) raise @@ -1257,7 +1257,8 @@ class easy_install(Command): * You can set up the installation directory to support ".pth" files by using one of the approaches described here: - https://pythonhosted.org/setuptools/easy_install.html#custom-installation-locations + https://setuptools.readthedocs.io/en/latest/easy_install.html#custom-installation-locations + Please make the appropriate changes for your system and try again.""").lstrip() @@ -1522,15 +1523,6 @@ def get_exe_prefixes(exe_filename): return prefixes -def parse_requirement_arg(spec): - try: - return Requirement.parse(spec) - except ValueError: - raise DistutilsError( - "Not a URL, existing file, or requirement spec: %r" % (spec,) - ) - - class PthDistributions(Environment): """A .pth file with Distribution paths in it""" @@ -1662,7 +1654,7 @@ class RewritePthDistributions(PthDistributions): """) -if os.environ.get('SETUPTOOLS_SYS_PATH_TECHNIQUE', 'rewrite') == 'rewrite': +if os.environ.get('SETUPTOOLS_SYS_PATH_TECHNIQUE', 'raw') == 'rewrite': PthDistributions = RewritePthDistributions @@ -1832,6 +1824,7 @@ def _remove_and_clear_zip_directory_cache_data(normalized_path): normalized_path, zipimport._zip_directory_cache, updater=clear_and_remove_cached_zip_archive_directory_data) + # PyPy Python implementation does not allow directly writing to the # zipimport._zip_directory_cache and so prevents us from attempting to correct # its content. The best we can do there is clear the problematic cache content @@ -1987,10 +1980,20 @@ class CommandSpec(list): return self._render(self + list(self.options)) @staticmethod + def _strip_quotes(item): + _QUOTES = '"\'' + for q in _QUOTES: + if item.startswith(q) and item.endswith(q): + return item[1:-1] + return item + + @staticmethod def _render(items): - cmdline = subprocess.list2cmdline(items) + cmdline = subprocess.list2cmdline( + CommandSpec._strip_quotes(item.strip()) for item in items) return '#!' + cmdline + '\n' + # For pbr compat; will be removed in a future version. sys_executable = CommandSpec._sys_executable() @@ -2008,10 +2011,12 @@ class ScriptWriter(object): template = textwrap.dedent(""" # EASY-INSTALL-ENTRY-SCRIPT: %(spec)r,%(group)r,%(name)r __requires__ = %(spec)r + import re import sys from pkg_resources import load_entry_point if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) sys.exit( load_entry_point(%(spec)r, %(group)r, %(name)r)() ) @@ -2159,6 +2164,7 @@ class WindowsScriptWriter(ScriptWriter): class WindowsExecutableLauncherWriter(WindowsScriptWriter): + @classmethod def _get_script_args(cls, type_, name, header, script_text): """ @@ -2203,8 +2209,6 @@ def get_win_launcher(type): Returns the executable as a byte string. """ launcher_fn = '%s.exe' % type - if platform.machine().lower() == 'arm': - launcher_fn = launcher_fn.replace(".", "-arm.") if is_64bit(): launcher_fn = launcher_fn.replace(".", "-64.") else: @@ -2221,39 +2225,7 @@ def load_launcher_manifest(name): def rmtree(path, ignore_errors=False, onerror=auto_chmod): - """Recursively delete a directory tree. - - This code is taken from the Python 2.4 version of 'shutil', because - the 2.3 version doesn't really work right. - """ - if ignore_errors: - def onerror(*args): - pass - elif onerror is None: - def onerror(*args): - raise - names = [] - try: - names = os.listdir(path) - except os.error: - onerror(os.listdir, path, sys.exc_info()) - for name in names: - fullname = os.path.join(path, name) - try: - mode = os.lstat(fullname).st_mode - except os.error: - mode = 0 - if stat.S_ISDIR(mode): - rmtree(fullname, ignore_errors, onerror) - else: - try: - os.remove(fullname) - except os.error: - onerror(os.remove, fullname, sys.exc_info()) - try: - os.rmdir(path) - except os.error: - onerror(os.rmdir, path, sys.exc_info()) + return shutil.rmtree(path, ignore_errors, onerror) def current_umask(): diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 8e1502a5..5183eedc 100755 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -52,8 +52,10 @@ class egg_info(Command): ] boolean_options = ['tag-date', 'tag-svn-revision'] - negative_opt = {'no-svn-revision': 'tag-svn-revision', - 'no-date': 'tag-date'} + negative_opt = { + 'no-svn-revision': 'tag-svn-revision', + 'no-date': 'tag-date', + } def initialize_options(self): self.egg_name = None @@ -197,6 +199,10 @@ class egg_info(Command): if self.tag_build: version += self.tag_build if self.tag_svn_revision: + warnings.warn( + "tag_svn_revision is deprecated and will not be honored " + "in a future release" + ) version += '-r%s' % self.get_svn_revision() if self.tag_date: version += time.strftime("-%Y%m%d") diff --git a/setuptools/command/install_lib.py b/setuptools/command/install_lib.py index 78fe6891..2b31c3e3 100644 --- a/setuptools/command/install_lib.py +++ b/setuptools/command/install_lib.py @@ -3,6 +3,7 @@ import imp from itertools import product, starmap import distutils.command.install_lib as orig + class install_lib(orig.install_lib): """Don't add compiled flags to filenames of non-Python files""" diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py index be66cb22..16234273 100755 --- a/setuptools/command/install_scripts.py +++ b/setuptools/command/install_scripts.py @@ -1,6 +1,7 @@ from distutils import log import distutils.command.install_scripts as orig import os +import sys from pkg_resources import Distribution, PathMetadata, ensure_directory @@ -37,6 +38,10 @@ class install_scripts(orig.install_scripts): if is_wininst: exec_param = "python.exe" writer = ei.WindowsScriptWriter + if exec_param == sys.executable: + # In case the path to the Python executable contains a space, wrap + # it so it's not split up. + exec_param = [exec_param] # resolve the writer to the environment writer = writer.best() cmd = writer.command_spec_class.best().from_param(exec_param) diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 6640d4e3..1d4f5d54 100755 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -4,6 +4,7 @@ import distutils.command.sdist as orig import os import sys import io +import contextlib from setuptools.extern import six @@ -15,6 +16,7 @@ READMES = 'README', 'README.rst', 'README.txt' _default_revctrl = list + def walk_revctrl(dirname=''): """Find all files under revision control""" for ep in pkg_resources.iter_entry_points('setuptools.file_finders'): @@ -64,6 +66,43 @@ class sdist(orig.sdist): if data not in dist_files: dist_files.append(data) + def initialize_options(self): + orig.sdist.initialize_options(self) + + self._default_to_gztar() + + def _default_to_gztar(self): + # only needed on Python prior to 3.6. + if sys.version_info >= (3, 6, 0, 'beta', 1): + return + self.formats = ['gztar'] + + def make_distribution(self): + """ + Workaround for #516 + """ + with self._remove_os_link(): + orig.sdist.make_distribution(self) + + @staticmethod + @contextlib.contextmanager + def _remove_os_link(): + """ + In a context, remove and restore os.link if it exists + """ + class NoValue: + pass + orig_val = getattr(os, 'link', NoValue) + try: + del os.link + except Exception: + pass + try: + yield + finally: + if orig_val is not NoValue: + setattr(os, 'link', orig_val) + def __read_template_hack(self): # This grody hack closes the template file (MANIFEST.in) if an # exception occurs during read_template. @@ -71,7 +110,7 @@ class sdist(orig.sdist): # file. try: orig.sdist.read_template(self) - except: + except Exception: _, _, tb = sys.exc_info() tb.tb_next.tb_frame.f_locals['template'].close() raise @@ -179,7 +218,7 @@ class sdist(orig.sdist): distribution. """ log.info("reading manifest file '%s'", self.manifest) - manifest = open(self.manifest, 'rbU') + manifest = open(self.manifest, 'rb') for line in manifest: # The manifest must contain UTF-8. See #303. if six.PY3: diff --git a/setuptools/command/test.py b/setuptools/command/test.py index 39746a02..38bbcd8b 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -1,10 +1,13 @@ +import os +import operator import sys import contextlib +import itertools from distutils.errors import DistutilsOptionError from unittest import TestLoader from setuptools.extern import six -from setuptools.extern.six.moves import map +from setuptools.extern.six.moves import map, filter from pkg_resources import (resource_listdir, resource_exists, normalize_path, working_set, _namespace_packages, @@ -14,6 +17,7 @@ from setuptools.py31compat import unittest_main class ScanningLoader(TestLoader): + def loadTestsFromModule(self, module, pattern=None): """Return a suite of all tests cases contained in the given module @@ -46,6 +50,7 @@ class ScanningLoader(TestLoader): # adapted from jaraco.classes.properties:NonDataProperty class NonDataProperty(object): + def __init__(self, fget): self.fget = fget @@ -110,7 +115,7 @@ class test(Command): func() @contextlib.contextmanager - def project_on_sys_path(self): + def project_on_sys_path(self, include_dists=[]): with_2to3 = six.PY3 and getattr(self.distribution, 'use_2to3', False) if with_2to3: @@ -142,23 +147,57 @@ class test(Command): old_modules = sys.modules.copy() try: - sys.path.insert(0, normalize_path(ei_cmd.egg_base)) + project_path = normalize_path(ei_cmd.egg_base) + sys.path.insert(0, project_path) working_set.__init__() add_activation_listener(lambda dist: dist.activate()) require('%s==%s' % (ei_cmd.egg_name, ei_cmd.egg_version)) - yield + with self.paths_on_pythonpath([project_path]): + yield finally: sys.path[:] = old_path sys.modules.clear() sys.modules.update(old_modules) working_set.__init__() + @staticmethod + @contextlib.contextmanager + def paths_on_pythonpath(paths): + """ + Add the indicated paths to the head of the PYTHONPATH environment + variable so that subprocesses will also see the packages at + these paths. + + Do this in a context that restores the value on exit. + """ + nothing = object() + orig_pythonpath = os.environ.get('PYTHONPATH', nothing) + current_pythonpath = os.environ.get('PYTHONPATH', '') + try: + prefix = os.pathsep.join(paths) + to_join = filter(None, [prefix, current_pythonpath]) + new_path = os.pathsep.join(to_join) + if new_path: + os.environ['PYTHONPATH'] = new_path + yield + finally: + if orig_pythonpath is nothing: + os.environ.pop('PYTHONPATH', None) + else: + os.environ['PYTHONPATH'] = orig_pythonpath + + @staticmethod + def install_dists(dist): + """ + Install the requirements indicated by self.distribution and + return an iterable of the dists that were built. + """ + ir_d = dist.fetch_build_eggs(dist.install_requires or []) + tr_d = dist.fetch_build_eggs(dist.tests_require or []) + return itertools.chain(ir_d, tr_d) + def run(self): - if self.distribution.install_requires: - self.distribution.fetch_build_eggs( - self.distribution.install_requires) - if self.distribution.tests_require: - self.distribution.fetch_build_eggs(self.distribution.tests_require) + installed_dists = self.install_dists(self.distribution) cmd = ' '.join(self._argv) if self.dry_run: @@ -166,8 +205,11 @@ class test(Command): return self.announce('running "%s"' % cmd) - with self.project_on_sys_path(): - self.run_tests() + + paths = map(operator.attrgetter('location'), installed_dists) + with self.paths_on_pythonpath(paths): + with self.project_on_sys_path(): + self.run_tests() def run_tests(self): # Purge modules under test from sys.modules. The test loader will diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 01b49046..269dc2d5 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -29,6 +29,10 @@ def _encode(s): class upload_docs(upload): + # override the default repository as upload_docs isn't + # supported by Warehouse (and won't be). + DEFAULT_REPOSITORY = 'https://pypi.python.org/pypi/' + description = 'Upload documentation to PyPI' user_options = [ @@ -53,6 +57,7 @@ class upload_docs(upload): self.target_dir = None def finalize_options(self): + log.warn("Upload_docs command is deprecated. Use RTD instead.") upload.finalize_options(self) if self.upload_dir is None: if self.has_sphinx(): @@ -105,7 +110,7 @@ class upload_docs(upload): if not isinstance(values, list): values = [values] for value in values: - if type(value) is tuple: + if isinstance(value, tuple): title += '; filename="%s"' % value[0] value = value[1] else: diff --git a/setuptools/depends.py b/setuptools/depends.py index 9f7c9a35..d5a344ad 100644 --- a/setuptools/depends.py +++ b/setuptools/depends.py @@ -1,8 +1,8 @@ import sys import imp import marshal -from imp import PKG_DIRECTORY, PY_COMPILED, PY_SOURCE, PY_FROZEN from distutils.version import StrictVersion +from imp import PKG_DIRECTORY, PY_COMPILED, PY_SOURCE, PY_FROZEN from setuptools.extern import six @@ -10,6 +10,7 @@ __all__ = [ 'Require', 'find_module', 'get_module_constant', 'extract_constant' ] + class Require: """A prerequisite to building or installing a distribution""" @@ -30,7 +31,7 @@ class Require: def full_name(self): """Return full package/distribution name, w/version""" if self.requested_version is not None: - return '%s-%s' % (self.name,self.requested_version) + return '%s-%s' % (self.name, self.requested_version) return self.name def version_ok(self, version): @@ -39,7 +40,6 @@ class Require: str(version) != "unknown" and version >= self.requested_version def get_version(self, paths=None, default="unknown"): - """Get version number of installed module, 'None', or 'default' Search 'paths' for module. If not found, return 'None'. If found, @@ -52,8 +52,9 @@ class Require: if self.attribute is None: try: - f,p,i = find_module(self.module,paths) - if f: f.close() + f, p, i = find_module(self.module, paths) + if f: + f.close() return default except ImportError: return None @@ -78,28 +79,27 @@ class Require: def _iter_code(code): - """Yield '(op,arg)' pair for each operation in code object 'code'""" from array import array from dis import HAVE_ARGUMENT, EXTENDED_ARG - bytes = array('b',code.co_code) + bytes = array('b', code.co_code) eof = len(code.co_code) ptr = 0 extended_arg = 0 - while ptr<eof: + while ptr < eof: op = bytes[ptr] - if op>=HAVE_ARGUMENT: + if op >= HAVE_ARGUMENT: - arg = bytes[ptr+1] + bytes[ptr+2]*256 + extended_arg + arg = bytes[ptr + 1] + bytes[ptr + 2] * 256 + extended_arg ptr += 3 - if op==EXTENDED_ARG: + if op == EXTENDED_ARG: long_type = six.integer_types[-1] extended_arg = arg * long_type(65536) continue @@ -108,7 +108,7 @@ def _iter_code(code): arg = None ptr += 1 - yield op,arg + yield op, arg def find_module(module, paths=None): @@ -118,20 +118,19 @@ def find_module(module, paths=None): while parts: part = parts.pop(0) - f, path, (suffix,mode,kind) = info = imp.find_module(part, paths) + f, path, (suffix, mode, kind) = info = imp.find_module(part, paths) - if kind==PKG_DIRECTORY: + if kind == PKG_DIRECTORY: parts = parts or ['__init__'] paths = [path] elif parts: - raise ImportError("Can't find %r in %s" % (parts,module)) + raise ImportError("Can't find %r in %s" % (parts, module)) return info def get_module_constant(module, symbol, default=-1, paths=None): - """Find 'module' by searching 'paths', and extract 'symbol' Return 'None' if 'module' does not exist on 'paths', or it does not define @@ -145,12 +144,12 @@ def get_module_constant(module, symbol, default=-1, paths=None): return None try: - if kind==PY_COMPILED: + if kind == PY_COMPILED: f.read(8) # skip magic & date code = marshal.load(f) - elif kind==PY_FROZEN: + elif kind == PY_FROZEN: code = imp.get_frozen_object(module) - elif kind==PY_SOURCE: + elif kind == PY_SOURCE: code = compile(f.read(), path, 'exec') else: # Not something we can parse; we'll have to import it. :( @@ -192,9 +191,9 @@ def extract_constant(code, symbol, default=-1): for op, arg in _iter_code(code): - if op==LOAD_CONST: + if op == LOAD_CONST: const = code.co_consts[arg] - elif arg==name_idx and (op==STORE_NAME or op==STORE_GLOBAL): + elif arg == name_idx and (op == STORE_NAME or op == STORE_GLOBAL): return const else: const = default @@ -214,4 +213,5 @@ def _update_globals(): del globals()[name] __all__.remove(name) + _update_globals() diff --git a/setuptools/dist.py b/setuptools/dist.py index 086e0a58..364f2b4d 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -2,16 +2,15 @@ __all__ = ['Distribution'] import re import os -import sys import warnings import numbers import distutils.log import distutils.core import distutils.cmd import distutils.dist -from distutils.core import Distribution as _Distribution from distutils.errors import (DistutilsOptionError, DistutilsPlatformError, DistutilsSetupError) +from distutils.util import rfc822_escape from setuptools.extern import six from setuptools.extern.six.moves import map @@ -19,70 +18,94 @@ from pkg_resources.extern import packaging from setuptools.depends import Require from setuptools import windows_support +from setuptools.monkey import get_unpatched import pkg_resources def _get_unpatched(cls): - """Protect against re-patching the distutils if reloaded + warnings.warn("Do not call this function", DeprecationWarning) + return get_unpatched(cls) - Also ensures that no other distutils extension monkeypatched the distutils - first. - """ - while cls.__module__.startswith('setuptools'): - cls, = cls.__bases__ - if not cls.__module__.startswith('distutils'): - raise AssertionError( - "distutils has already been patched by %r" % cls - ) - return cls -_Distribution = _get_unpatched(_Distribution) - -def _patch_distribution_metadata_write_pkg_info(): +# Based on Python 3.5 version +def write_pkg_file(self, file): + """Write the PKG-INFO format data to a file object. """ - Workaround issue #197 - Python 3 prior to 3.2.2 uses an environment-local - encoding to save the pkg_info. Monkey-patch its write_pkg_info method to - correct this undesirable behavior. + version = '1.0' + if (self.provides or self.requires or self.obsoletes or + self.classifiers or self.download_url): + version = '1.1' + # Setuptools specific for PEP 345 + if hasattr(self, 'python_requires'): + version = '1.2' + + file.write('Metadata-Version: %s\n' % version) + file.write('Name: %s\n' % self.get_name()) + file.write('Version: %s\n' % self.get_version()) + file.write('Summary: %s\n' % self.get_description()) + file.write('Home-page: %s\n' % self.get_url()) + file.write('Author: %s\n' % self.get_contact()) + file.write('Author-email: %s\n' % self.get_contact_email()) + file.write('License: %s\n' % self.get_license()) + if self.download_url: + file.write('Download-URL: %s\n' % self.download_url) + + long_desc = rfc822_escape(self.get_long_description()) + file.write('Description: %s\n' % long_desc) + + keywords = ','.join(self.get_keywords()) + if keywords: + file.write('Keywords: %s\n' % keywords) + + self._write_list(file, 'Platform', self.get_platforms()) + self._write_list(file, 'Classifier', self.get_classifiers()) + + # PEP 314 + self._write_list(file, 'Requires', self.get_requires()) + self._write_list(file, 'Provides', self.get_provides()) + self._write_list(file, 'Obsoletes', self.get_obsoletes()) + + # Setuptools specific for PEP 345 + if hasattr(self, 'python_requires'): + file.write('Requires-Python: %s\n' % self.python_requires) + + +# from Python 3.4 +def write_pkg_info(self, base_dir): + """Write the PKG-INFO file into the release tree. """ - environment_local = (3,) <= sys.version_info[:3] < (3, 2, 2) - if not environment_local: - return - - # from Python 3.4 - def write_pkg_info(self, base_dir): - """Write the PKG-INFO file into the release tree. - """ - with open(os.path.join(base_dir, 'PKG-INFO'), 'w', - encoding='UTF-8') as pkg_info: - self.write_pkg_file(pkg_info) + with open(os.path.join(base_dir, 'PKG-INFO'), 'w', + encoding='UTF-8') as pkg_info: + self.write_pkg_file(pkg_info) - distutils.dist.DistributionMetadata.write_pkg_info = write_pkg_info -_patch_distribution_metadata_write_pkg_info() sequence = tuple, list + def check_importable(dist, attr, value): try: - ep = pkg_resources.EntryPoint.parse('x='+value) + ep = pkg_resources.EntryPoint.parse('x=' + value) assert not ep.extras - except (TypeError,ValueError,AttributeError,AssertionError): + except (TypeError, ValueError, AttributeError, AssertionError): raise DistutilsSetupError( "%r must be importable 'module:attrs' string (got %r)" - % (attr,value) + % (attr, value) ) def assert_string_list(dist, attr, value): """Verify that value is a string list or None""" try: - assert ''.join(value)!=value - except (TypeError,ValueError,AttributeError,AssertionError): + assert ''.join(value) != value + except (TypeError, ValueError, AttributeError, AssertionError): raise DistutilsSetupError( - "%r must be a list of strings (got %r)" % (attr,value) + "%r must be a list of strings (got %r)" % (attr, value) ) + + def check_nsp(dist, attr, value): """Verify that namespace packages are valid""" - assert_string_list(dist,attr,value) + assert_string_list(dist, attr, value) for nsp in value: if not dist.has_contents_for(nsp): raise DistutilsSetupError( @@ -97,22 +120,24 @@ def check_nsp(dist, attr, value): " is not: please correct this in setup.py", nsp, parent ) + def check_extras(dist, attr, value): """Verify that extras_require mapping is valid""" try: - for k,v in value.items(): + for k, v in value.items(): if ':' in k: - k,m = k.split(':',1) + k, m = k.split(':', 1) if pkg_resources.invalid_marker(m): - raise DistutilsSetupError("Invalid environment marker: "+m) + raise DistutilsSetupError("Invalid environment marker: " + m) list(pkg_resources.parse_requirements(v)) - except (TypeError,ValueError,AttributeError): + except (TypeError, ValueError, AttributeError): raise DistutilsSetupError( "'extras_require' must be a dictionary whose values are " "strings or lists of strings containing valid project/version " "requirement specifiers." ) + def assert_bool(dist, attr, value): """Verify that value is True, False, 0, or 1""" if bool(value) != value: @@ -131,6 +156,19 @@ def check_requirements(dist, attr, value): ) raise DistutilsSetupError(tmpl.format(attr=attr, error=error)) + +def check_specifier(dist, attr, value): + """Verify that value is a valid version specifier""" + try: + packaging.specifiers.SpecifierSet(value) + except packaging.specifiers.InvalidSpecifier as error: + tmpl = ( + "{attr!r} must be a string or list of strings " + "containing valid version specifiers; {error}" + ) + raise DistutilsSetupError(tmpl.format(attr=attr, error=error)) + + def check_entry_points(dist, attr, value): """Verify that entry_points map is parseable""" try: @@ -138,25 +176,30 @@ def check_entry_points(dist, attr, value): except ValueError as e: raise DistutilsSetupError(e) + def check_test_suite(dist, attr, value): if not isinstance(value, six.string_types): raise DistutilsSetupError("test_suite must be a string") + def check_package_data(dist, attr, value): """Verify that value is a dictionary of package names to glob lists""" - if isinstance(value,dict): - for k,v in value.items(): - if not isinstance(k,str): break - try: iter(v) + if isinstance(value, dict): + for k, v in value.items(): + if not isinstance(k, str): + break + try: + iter(v) except TypeError: break else: return raise DistutilsSetupError( - attr+" must be a dictionary mapping package names to lists of " + attr + " must be a dictionary mapping package names to lists of " "wildcard patterns" ) + def check_packages(dist, attr, value): for pkgname in value: if not re.match(r'\w+(\.\w+)*', pkgname): @@ -166,6 +209,9 @@ def check_packages(dist, attr, value): ) +_Distribution = get_unpatched(distutils.core.Distribution) + + class Distribution(_Distribution): """Distribution with support for features, tests, and package data @@ -264,12 +310,12 @@ class Distribution(_Distribution): # Make sure we have any eggs needed to interpret 'attrs' if attrs is not None: self.dependency_links = attrs.pop('dependency_links', []) - assert_string_list(self,'dependency_links',self.dependency_links) + assert_string_list(self, 'dependency_links', self.dependency_links) if attrs and 'setup_requires' in attrs: self.fetch_build_eggs(attrs['setup_requires']) for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'): vars(self).setdefault(ep.name, None) - _Distribution.__init__(self,attrs) + _Distribution.__init__(self, attrs) if isinstance(self.metadata.version, numbers.Number): # Some people apparently take "version number" too literally :) self.metadata.version = str(self.metadata.version) @@ -293,6 +339,8 @@ class Distribution(_Distribution): "setuptools, pip, and PyPI. Please see PEP 440 for more " "details." % self.metadata.version ) + if getattr(self, 'python_requires', None): + self.metadata.python_requires = self.python_requires def parse_command_line(self): """Process features after parsing command line options""" @@ -301,9 +349,9 @@ class Distribution(_Distribution): self._finalize_features() return result - def _feature_attrname(self,name): + def _feature_attrname(self, name): """Convert feature name to corresponding option attribute name""" - return 'with_'+name.replace('-','_') + return 'with_' + name.replace('-', '_') def fetch_build_eggs(self, requires): """Resolve pre-setup requirements""" @@ -314,6 +362,7 @@ class Distribution(_Distribution): ) for dist in resolved_dists: pkg_resources.working_set.add(dist, replace=True) + return resolved_dists def finalize_options(self): _Distribution.finalize_options(self) @@ -321,7 +370,7 @@ class Distribution(_Distribution): self._set_global_opts_from_features() for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'): - value = getattr(self,ep.name,None) + value = getattr(self, ep.name, None) if value is not None: ep.require(installer=self.fetch_build_egg) ep.load()(self, ep.name, value) @@ -354,7 +403,7 @@ class Distribution(_Distribution): cmd.package_index.to_scan = [] except AttributeError: from setuptools.command.easy_install import easy_install - dist = self.__class__({'script_args':['easy_install']}) + dist = self.__class__({'script_args': ['easy_install']}) dist.parse_config_files() opts = dist.get_option_dict('easy_install') keep = ( @@ -385,20 +434,20 @@ class Distribution(_Distribution): go = [] no = self.negative_opt.copy() - for name,feature in self.features.items(): - self._set_feature(name,None) + for name, feature in self.features.items(): + self._set_feature(name, None) feature.validate(self) if feature.optional: descr = feature.description incdef = ' (default)' - excdef='' + excdef = '' if not feature.include_by_default(): excdef, incdef = incdef, excdef - go.append(('with-'+name, None, 'include '+descr+incdef)) - go.append(('without-'+name, None, 'exclude '+descr+excdef)) - no['without-'+name] = 'with-'+name + go.append(('with-' + name, None, 'include ' + descr + incdef)) + go.append(('without-' + name, None, 'exclude ' + descr + excdef)) + no['without-' + name] = 'with-' + name self.global_options = self.feature_options = go + self.global_options self.negative_opt = self.feature_negopt = no @@ -407,25 +456,25 @@ class Distribution(_Distribution): """Add/remove features and resolve dependencies between them""" # First, flag all the enabled items (and thus their dependencies) - for name,feature in self.features.items(): + for name, feature in self.features.items(): enabled = self.feature_is_included(name) if enabled or (enabled is None and feature.include_by_default()): feature.include_in(self) - self._set_feature(name,1) + self._set_feature(name, 1) # Then disable the rest, so that off-by-default features don't # get flagged as errors when they're required by an enabled feature - for name,feature in self.features.items(): + for name, feature in self.features.items(): if not self.feature_is_included(name): feature.exclude_from(self) - self._set_feature(name,0) + self._set_feature(name, 0) def get_command_class(self, command): """Pluggable version of get_command_class()""" if command in self.cmdclass: return self.cmdclass[command] - for ep in pkg_resources.iter_entry_points('distutils.commands',command): + for ep in pkg_resources.iter_entry_points('distutils.commands', command): ep.require(installer=self.fetch_build_egg) self.cmdclass[command] = cmdclass = ep.load() return cmdclass @@ -448,26 +497,26 @@ class Distribution(_Distribution): self.cmdclass[ep.name] = cmdclass return _Distribution.get_command_list(self) - def _set_feature(self,name,status): + def _set_feature(self, name, status): """Set feature's inclusion status""" - setattr(self,self._feature_attrname(name),status) + setattr(self, self._feature_attrname(name), status) - def feature_is_included(self,name): + def feature_is_included(self, name): """Return 1 if feature is included, 0 if excluded, 'None' if unknown""" - return getattr(self,self._feature_attrname(name)) + return getattr(self, self._feature_attrname(name)) - def include_feature(self,name): + def include_feature(self, name): """Request inclusion of feature named 'name'""" - if self.feature_is_included(name)==0: + if self.feature_is_included(name) == 0: descr = self.features[name].description raise DistutilsOptionError( descr + " is required, but was excluded or is not available" ) self.features[name].include_in(self) - self._set_feature(name,1) + self._set_feature(name, 1) - def include(self,**attrs): + def include(self, **attrs): """Add items to distribution that are named in keyword arguments For example, 'dist.exclude(py_modules=["x"])' would add 'x' to @@ -482,86 +531,86 @@ class Distribution(_Distribution): will try to call 'dist._include_foo({"bar":"baz"})', which can then handle whatever special inclusion logic is needed. """ - for k,v in attrs.items(): - include = getattr(self, '_include_'+k, None) + for k, v in attrs.items(): + include = getattr(self, '_include_' + k, None) if include: include(v) else: - self._include_misc(k,v) + self._include_misc(k, v) - def exclude_package(self,package): + def exclude_package(self, package): """Remove packages, modules, and extensions in named package""" - pfx = package+'.' + pfx = package + '.' if self.packages: self.packages = [ p for p in self.packages - if p != package and not p.startswith(pfx) + if p != package and not p.startswith(pfx) ] if self.py_modules: self.py_modules = [ p for p in self.py_modules - if p != package and not p.startswith(pfx) + if p != package and not p.startswith(pfx) ] if self.ext_modules: self.ext_modules = [ p for p in self.ext_modules - if p.name != package and not p.name.startswith(pfx) + if p.name != package and not p.name.startswith(pfx) ] - def has_contents_for(self,package): + def has_contents_for(self, package): """Return true if 'exclude_package(package)' would do something""" - pfx = package+'.' + pfx = package + '.' for p in self.iter_distribution_names(): - if p==package or p.startswith(pfx): + if p == package or p.startswith(pfx): return True - def _exclude_misc(self,name,value): + def _exclude_misc(self, name, value): """Handle 'exclude()' for list/tuple attrs without a special handler""" - if not isinstance(value,sequence): + if not isinstance(value, sequence): raise DistutilsSetupError( "%s: setting must be a list or tuple (%r)" % (name, value) ) try: - old = getattr(self,name) + old = getattr(self, name) except AttributeError: raise DistutilsSetupError( "%s: No such distribution setting" % name ) - if old is not None and not isinstance(old,sequence): + if old is not None and not isinstance(old, sequence): raise DistutilsSetupError( - name+": this setting cannot be changed via include/exclude" + name + ": this setting cannot be changed via include/exclude" ) elif old: - setattr(self,name,[item for item in old if item not in value]) + setattr(self, name, [item for item in old if item not in value]) - def _include_misc(self,name,value): + def _include_misc(self, name, value): """Handle 'include()' for list/tuple attrs without a special handler""" - if not isinstance(value,sequence): + if not isinstance(value, sequence): raise DistutilsSetupError( "%s: setting must be a list (%r)" % (name, value) ) try: - old = getattr(self,name) + old = getattr(self, name) except AttributeError: raise DistutilsSetupError( "%s: No such distribution setting" % name ) if old is None: - setattr(self,name,value) - elif not isinstance(old,sequence): + setattr(self, name, value) + elif not isinstance(old, sequence): raise DistutilsSetupError( - name+": this setting cannot be changed via include/exclude" + name + ": this setting cannot be changed via include/exclude" ) else: - setattr(self,name,old+[item for item in value if item not in old]) + setattr(self, name, old + [item for item in value if item not in old]) - def exclude(self,**attrs): + def exclude(self, **attrs): """Remove items from distribution that are named in keyword arguments For example, 'dist.exclude(py_modules=["x"])' would remove 'x' from @@ -577,15 +626,15 @@ class Distribution(_Distribution): will try to call 'dist._exclude_foo({"bar":"baz"})', which can then handle whatever special exclusion logic is needed. """ - for k,v in attrs.items(): - exclude = getattr(self, '_exclude_'+k, None) + for k, v in attrs.items(): + exclude = getattr(self, '_exclude_' + k, None) if exclude: exclude(v) else: - self._exclude_misc(k,v) + self._exclude_misc(k, v) - def _exclude_packages(self,packages): - if not isinstance(packages,sequence): + def _exclude_packages(self, packages): + if not isinstance(packages, sequence): raise DistutilsSetupError( "packages: setting must be a list or tuple (%r)" % (packages,) ) @@ -600,17 +649,17 @@ class Distribution(_Distribution): command = args[0] aliases = self.get_option_dict('aliases') while command in aliases: - src,alias = aliases[command] + src, alias = aliases[command] del aliases[command] # ensure each alias can expand only once! import shlex - args[:1] = shlex.split(alias,True) + args[:1] = shlex.split(alias, True) command = args[0] nargs = _Distribution._parse_command_opts(self, parser, args) # Handle commands that want to consume all remaining arguments cmd_class = self.get_command_class(command) - if getattr(cmd_class,'command_consumes_arguments',None): + if getattr(cmd_class, 'command_consumes_arguments', None): self.get_option_dict(command)['args'] = ("command line", nargs) if nargs is not None: return [] @@ -629,31 +678,31 @@ class Distribution(_Distribution): d = {} - for cmd,opts in self.command_options.items(): + for cmd, opts in self.command_options.items(): - for opt,(src,val) in opts.items(): + for opt, (src, val) in opts.items(): if src != "command line": continue - opt = opt.replace('_','-') + opt = opt.replace('_', '-') - if val==0: + if val == 0: cmdobj = self.get_command_obj(cmd) neg_opt = self.negative_opt.copy() - neg_opt.update(getattr(cmdobj,'negative_opt',{})) - for neg,pos in neg_opt.items(): - if pos==opt: - opt=neg - val=None + neg_opt.update(getattr(cmdobj, 'negative_opt', {})) + for neg, pos in neg_opt.items(): + if pos == opt: + opt = neg + val = None break else: raise AssertionError("Shouldn't be able to get here") - elif val==1: + elif val == 1: val = None - d.setdefault(cmd,{})[opt] = val + d.setdefault(cmd, {})[opt] = val return d @@ -667,7 +716,7 @@ class Distribution(_Distribution): yield module for ext in self.ext_modules or (): - if isinstance(ext,tuple): + if isinstance(ext, tuple): name, buildinfo = ext else: name = ext.name @@ -711,11 +760,6 @@ class Distribution(_Distribution): sys.stdout.detach(), encoding, errors, newline, line_buffering) -# Install it throughout the distutils -for module in distutils.dist, distutils.core, distutils.cmd: - module.Distribution = Distribution - - class Feature: """ **deprecated** -- The `Feature` facility was never completely implemented @@ -790,16 +834,17 @@ class Feature: self.standard = standard self.available = available self.optional = optional - if isinstance(require_features,(str,Require)): + if isinstance(require_features, (str, Require)): require_features = require_features, self.require_features = [ - r for r in require_features if isinstance(r,str) + r for r in require_features if isinstance(r, str) ] - er = [r for r in require_features if not isinstance(r,str)] - if er: extras['require_features'] = er + er = [r for r in require_features if not isinstance(r, str)] + if er: + extras['require_features'] = er - if isinstance(remove,str): + if isinstance(remove, str): remove = remove, self.remove = remove self.extras = extras @@ -814,8 +859,7 @@ class Feature: """Should this feature be included by default?""" return self.available and self.standard - def include_in(self,dist): - + def include_in(self, dist): """Ensure feature and its requirements are included in distribution You may override this in a subclass to perform additional operations on @@ -826,7 +870,7 @@ class Feature: if not self.available: raise DistutilsPlatformError( - self.description+" is required, " + self.description + " is required, " "but is not available on this platform" ) @@ -835,8 +879,7 @@ class Feature: for f in self.require_features: dist.include_feature(f) - def exclude_from(self,dist): - + def exclude_from(self, dist): """Ensure feature is excluded from distribution You may override this in a subclass to perform additional operations on @@ -851,8 +894,7 @@ class Feature: for item in self.remove: dist.exclude_package(item) - def validate(self,dist): - + def validate(self, dist): """Verify that feature makes sense in context of distribution This method is called by the distribution just before it parses its diff --git a/setuptools/extension.py b/setuptools/extension.py index d10609b6..03068d35 100644 --- a/setuptools/extension.py +++ b/setuptools/extension.py @@ -1,4 +1,3 @@ -import sys import re import functools import distutils.core @@ -7,18 +6,14 @@ import distutils.extension from setuptools.extern.six.moves import map -from .dist import _get_unpatched -from . import msvc9_support +from .monkey import get_unpatched -_Extension = _get_unpatched(distutils.core.Extension) - -msvc9_support.patch_for_specialized_compiler() def _have_cython(): """ Return True if Cython can be imported. """ - cython_impl = 'Cython.Distutils.build_ext', + cython_impl = 'Cython.Distutils.build_ext' try: # from (cython_impl) import build_ext __import__(cython_impl, fromlist=['build_ext']).build_ext @@ -27,13 +22,23 @@ def _have_cython(): pass return False + # for compatibility have_pyrex = _have_cython +_Extension = get_unpatched(distutils.core.Extension) + + class Extension(_Extension): """Extension that uses '.c' files in place of '.pyx' files""" + def __init__(self, name, sources, *args, **kw): + # The *args is needed for compatibility as calls may use positional + # arguments. py_limited_api may be set only via keyword. + self.py_limited_api = kw.pop("py_limited_api", False) + _Extension.__init__(self, name, sources, *args, **kw) + def _convert_pyx_sources_to_lang(self): """ Replace sources with .pyx extensions to sources with the target @@ -48,10 +53,6 @@ class Extension(_Extension): sub = functools.partial(re.sub, '.pyx$', target_ext) self.sources = list(map(sub, self.sources)) + class Library(Extension): """Just like a regular Extension, but built as a library instead""" - -distutils.core.Extension = Extension -distutils.extension.Extension = Extension -if 'distutils.command.build_ext' in sys.modules: - sys.modules['distutils.command.build_ext'].Extension = Extension diff --git a/setuptools/gui-arm-32.exe b/setuptools/gui-arm-32.exe Binary files differdeleted file mode 100644 index 537aff37..00000000 --- a/setuptools/gui-arm-32.exe +++ /dev/null diff --git a/setuptools/launch.py b/setuptools/launch.py index b05cbd2c..308283ea 100644 --- a/setuptools/launch.py +++ b/setuptools/launch.py @@ -18,9 +18,9 @@ def run(): __builtins__ script_name = sys.argv[1] namespace = dict( - __file__ = script_name, - __name__ = '__main__', - __doc__ = None, + __file__=script_name, + __name__='__main__', + __doc__=None, ) sys.argv[:] = sys.argv[1:] diff --git a/setuptools/lib2to3_ex.py b/setuptools/lib2to3_ex.py index feef591a..c8632bc5 100644 --- a/setuptools/lib2to3_ex.py +++ b/setuptools/lib2to3_ex.py @@ -10,9 +10,12 @@ This module raises an ImportError on Python 2. from distutils.util import Mixin2to3 as _Mixin2to3 from distutils import log from lib2to3.refactor import RefactoringTool, get_fixers_from_package + import setuptools + class DistutilsRefactoringTool(RefactoringTool): + def log_error(self, msg, *args, **kw): log.error(msg, *args) @@ -22,15 +25,17 @@ class DistutilsRefactoringTool(RefactoringTool): def log_debug(self, msg, *args): log.debug(msg, *args) + class Mixin2to3(_Mixin2to3): - def run_2to3(self, files, doctests = False): + + def run_2to3(self, files, doctests=False): # See of the distribution option has been set, otherwise check the # setuptools default. if self.distribution.use_2to3 is not True: return if not files: return - log.info("Fixing "+" ".join(files)) + log.info("Fixing " + " ".join(files)) self.__build_fixer_names() self.__exclude_fixers() if doctests: @@ -41,7 +46,8 @@ class Mixin2to3(_Mixin2to3): _Mixin2to3.run_2to3(self, files) def __build_fixer_names(self): - if self.fixer_names: return + if self.fixer_names: + return self.fixer_names = [] for p in setuptools.lib2to3_fixer_packages: self.fixer_names.extend(get_fixers_from_package(p)) diff --git a/setuptools/monkey.py b/setuptools/monkey.py new file mode 100644 index 00000000..43b97b4d --- /dev/null +++ b/setuptools/monkey.py @@ -0,0 +1,187 @@ +""" +Monkey patching of distutils. +""" + +import sys +import distutils.filelist +import platform +import types +import functools + +from .py26compat import import_module +from setuptools.extern import six + +import setuptools + + +__all__ = [] +""" +Everything is private. Contact the project team +if you think you need this functionality. +""" + + +def get_unpatched(item): + lookup = ( + get_unpatched_class if isinstance(item, six.class_types) else + get_unpatched_function if isinstance(item, types.FunctionType) else + lambda item: None + ) + return lookup(item) + + +def get_unpatched_class(cls): + """Protect against re-patching the distutils if reloaded + + Also ensures that no other distutils extension monkeypatched the distutils + first. + """ + while cls.__module__.startswith('setuptools'): + cls, = cls.__bases__ + if not cls.__module__.startswith('distutils'): + msg = "distutils has already been patched by %r" % cls + raise AssertionError(msg) + return cls + + +def patch_all(): + # we can't patch distutils.cmd, alas + distutils.core.Command = setuptools.Command + + has_issue_12885 = ( + sys.version_info < (3, 4, 6) + or + (3, 5) < sys.version_info <= (3, 5, 3) + or + (3, 6) < sys.version_info + ) + + if has_issue_12885: + # fix findall bug in distutils (http://bugs.python.org/issue12885) + distutils.filelist.findall = setuptools.findall + + needs_warehouse = ( + sys.version_info < (2, 7, 13) + or + (3, 0) < sys.version_info < (3, 3, 7) + or + (3, 4) < sys.version_info < (3, 4, 6) + or + (3, 5) < sys.version_info <= (3, 5, 3) + or + (3, 6) < sys.version_info + ) + + if needs_warehouse: + warehouse = 'https://upload.pypi.org/legacy/' + distutils.config.PyPIRCCommand.DEFAULT_REPOSITORY = warehouse + + _patch_distribution_metadata_write_pkg_file() + _patch_distribution_metadata_write_pkg_info() + + # Install Distribution throughout the distutils + for module in distutils.dist, distutils.core, distutils.cmd: + module.Distribution = setuptools.dist.Distribution + + # Install the patched Extension + distutils.core.Extension = setuptools.extension.Extension + distutils.extension.Extension = setuptools.extension.Extension + if 'distutils.command.build_ext' in sys.modules: + sys.modules['distutils.command.build_ext'].Extension = ( + setuptools.extension.Extension + ) + + patch_for_msvc_specialized_compiler() + + +def _patch_distribution_metadata_write_pkg_file(): + """Patch write_pkg_file to also write Requires-Python/Requires-External""" + distutils.dist.DistributionMetadata.write_pkg_file = ( + setuptools.dist.write_pkg_file + ) + + +def _patch_distribution_metadata_write_pkg_info(): + """ + Workaround issue #197 - Python 3 prior to 3.2.2 uses an environment-local + encoding to save the pkg_info. Monkey-patch its write_pkg_info method to + correct this undesirable behavior. + """ + environment_local = (3,) <= sys.version_info[:3] < (3, 2, 2) + if not environment_local: + return + + distutils.dist.DistributionMetadata.write_pkg_info = ( + setuptools.dist.write_pkg_info + ) + + +def patch_func(replacement, target_mod, func_name): + """ + Patch func_name in target_mod with replacement + + Important - original must be resolved by name to avoid + patching an already patched function. + """ + original = getattr(target_mod, func_name) + + # set the 'unpatched' attribute on the replacement to + # point to the original. + vars(replacement).setdefault('unpatched', original) + + # replace the function in the original module + setattr(target_mod, func_name, replacement) + + +def get_unpatched_function(candidate): + return getattr(candidate, 'unpatched') + + +def patch_for_msvc_specialized_compiler(): + """ + Patch functions in distutils to use standalone Microsoft Visual C++ + compilers. + """ + # import late to avoid circular imports on Python < 3.5 + msvc = import_module('setuptools.msvc') + + if platform.system() != 'Windows': + # Compilers only availables on Microsoft Windows + return + + def patch_params(mod_name, func_name): + """ + Prepare the parameters for patch_func to patch indicated function. + """ + repl_prefix = 'msvc9_' if 'msvc9' in mod_name else 'msvc14_' + repl_name = repl_prefix + func_name.lstrip('_') + repl = getattr(msvc, repl_name) + mod = import_module(mod_name) + if not hasattr(mod, func_name): + raise ImportError(func_name) + return repl, mod, func_name + + # Python 2.7 to 3.4 + msvc9 = functools.partial(patch_params, 'distutils.msvc9compiler') + + # Python 3.5+ + msvc14 = functools.partial(patch_params, 'distutils._msvccompiler') + + try: + # Patch distutils.msvc9compiler + patch_func(*msvc9('find_vcvarsall')) + patch_func(*msvc9('query_vcvarsall')) + except ImportError: + pass + + try: + # Patch distutils._msvccompiler._get_vc_env + patch_func(*msvc14('_get_vc_env')) + except ImportError: + pass + + try: + # Patch distutils._msvccompiler.gen_lib_options for Numpy + patch_func(*msvc14('gen_lib_options')) + except ImportError: + pass diff --git a/setuptools/msvc.py b/setuptools/msvc.py new file mode 100644 index 00000000..e9665e10 --- /dev/null +++ b/setuptools/msvc.py @@ -0,0 +1,1189 @@ +""" +Improved support for Microsoft Visual C++ compilers. + +Known supported compilers: +-------------------------- +Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64); + Microsoft Windows SDK 7.0 (x86, x64, ia64); + Microsoft Windows SDK 6.1 (x86, x64, ia64) + +Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + +Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) +""" + +import os +import sys +import platform +import itertools +import distutils.errors +from pkg_resources.extern.packaging.version import LegacyVersion + +from setuptools.extern.six.moves import filterfalse + +from .monkey import get_unpatched + +if platform.system() == 'Windows': + from setuptools.extern.six.moves import winreg + safe_env = os.environ +else: + """ + Mock winreg and environ so the module can be imported + on this platform. + """ + class winreg: + HKEY_USERS = None + HKEY_CURRENT_USER = None + HKEY_LOCAL_MACHINE = None + HKEY_CLASSES_ROOT = None + safe_env = dict() + +try: + from distutils.msvc9compiler import Reg +except ImportError: + pass + + +def msvc9_find_vcvarsall(version): + """ + Patched "distutils.msvc9compiler.find_vcvarsall" to use the standalone + compiler build for Python (VCForPython). Fall back to original behavior + when the standalone compiler is not available. + + Redirect the path of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + + Parameters + ---------- + version: float + Required Microsoft Visual C++ version. + + Return + ------ + vcvarsall.bat path: str + """ + VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' + key = VC_BASE % ('', version) + try: + # Per-user installs register the compiler path here + productdir = Reg.get_value(key, "installdir") + except KeyError: + try: + # All-user installs on a 64-bit system register here + key = VC_BASE % ('Wow6432Node\\', version) + productdir = Reg.get_value(key, "installdir") + except KeyError: + productdir = None + + if productdir: + vcvarsall = os.path.os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + + return get_unpatched(msvc9_find_vcvarsall)(version) + + +def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs): + """ + Patched "distutils.msvc9compiler.query_vcvarsall" for support standalones + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64); + Microsoft Windows SDK 7.0 (x86, x64, ia64); + Microsoft Windows SDK 6.1 (x86, x64, ia64) + + Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + + Parameters + ---------- + ver: float + Required Microsoft Visual C++ version. + arch: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environement from vcvarsall.bat (Classical way) + try: + orig = get_unpatched(msvc9_query_vcvarsall) + return orig(ver, arch, *args, **kwargs) + except distutils.errors.DistutilsPlatformError: + # Pass error if Vcvarsall.bat is missing + pass + except ValueError: + # Pass error if environment not set after executing vcvarsall.bat + pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(arch, ver).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, ver, arch) + raise + + +def msvc14_get_vc_env(plat_spec): + """ + Patched "distutils._msvccompiler._get_vc_env" for support standalones + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) + + Parameters + ---------- + plat_spec: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environment from vcvarsall.bat (Classical way) + try: + return get_unpatched(msvc14_get_vc_env)(plat_spec) + except distutils.errors.DistutilsPlatformError: + # Pass error Vcvarsall.bat is missing + pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(plat_spec, vc_min_ver=14.0).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, 14.0) + raise + + +def msvc14_gen_lib_options(*args, **kwargs): + """ + Patched "distutils._msvccompiler.gen_lib_options" for fix + compatibility between "numpy.distutils" and "distutils._msvccompiler" + (for Numpy < 1.11.2) + """ + if "numpy.distutils" in sys.modules: + import numpy as np + if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'): + return np.distutils.ccompiler.gen_lib_options(*args, **kwargs) + return get_unpatched(msvc14_gen_lib_options)(*args, **kwargs) + + +def _augment_exception(exc, version, arch=''): + """ + Add details to the exception message to help guide the user + as to what action will resolve it. + """ + # Error if MSVC++ directory not found or environment not set + message = exc.args[0] + + if "vcvarsall" in message.lower() or "visual c" in message.lower(): + # Special error message if MSVC++ not installed + tmpl = 'Microsoft Visual C++ {version:0.1f} is required.' + message = tmpl.format(**locals()) + msdownload = 'www.microsoft.com/download/details.aspx?id=%d' + if version == 9.0: + if arch.lower().find('ia64') > -1: + # For VC++ 9.0, if IA64 support is needed, redirect user + # to Windows SDK 7.0 + message += ' Get it with "Microsoft Windows SDK 7.0": ' + message += msdownload % 3138 + else: + # For VC++ 9.0 redirect user to Vc++ for Python 2.7 : + # This redirection link is maintained by Microsoft. + # Contact vspython@microsoft.com if it needs updating. + message += ' Get it from http://aka.ms/vcpython27' + elif version == 10.0: + # For VC++ 10.0 Redirect user to Windows SDK 7.1 + message += ' Get it with "Microsoft Windows SDK 7.1": ' + message += msdownload % 8279 + elif version >= 14.0: + # For VC++ 14.0 Redirect user to Visual C++ Build Tools + message += (' Get it with "Microsoft Visual C++ Build Tools": ' + r'http://landinghub.visualstudio.com/' + 'visual-cpp-build-tools') + + exc.args = (message, ) + + +class PlatformInfo: + """ + Current and Target Architectures informations. + + Parameters + ---------- + arch: str + Target architecture. + """ + current_cpu = safe_env.get('processor_architecture', '').lower() + + def __init__(self, arch): + self.arch = arch.lower().replace('x64', 'amd64') + + @property + def target_cpu(self): + return self.arch[self.arch.find('_') + 1:] + + def target_is_x86(self): + return self.target_cpu == 'x86' + + def current_is_x86(self): + return self.current_cpu == 'x86' + + def current_dir(self, hidex86=False, x64=False): + """ + Current platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\target', or '' (see hidex86 parameter) + """ + return ( + '' if (self.current_cpu == 'x86' and hidex86) else + r'\x64' if (self.current_cpu == 'amd64' and x64) else + r'\%s' % self.current_cpu + ) + + def target_dir(self, hidex86=False, x64=False): + """ + Target platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\current', or '' (see hidex86 parameter) + """ + return ( + '' if (self.target_cpu == 'x86' and hidex86) else + r'\x64' if (self.target_cpu == 'amd64' and x64) else + r'\%s' % self.target_cpu + ) + + def cross_dir(self, forcex86=False): + """ + Cross platform specific subfolder. + + Parameters + ---------- + forcex86: bool + Use 'x86' as current architecture even if current acritecture is + not x86. + + Return + ------ + subfolder: str + '' if target architecture is current architecture, + '\current_target' if not. + """ + current = 'x86' if forcex86 else self.current_cpu + return ( + '' if self.target_cpu == current else + self.target_dir().replace('\\', '\\%s_' % current) + ) + + +class RegistryInfo: + """ + Microsoft Visual Studio related registry informations. + + Parameters + ---------- + platform_info: PlatformInfo + "PlatformInfo" instance. + """ + HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + + def __init__(self, platform_info): + self.pi = platform_info + + @property + def visualstudio(self): + """ + Microsoft Visual Studio root registry key. + """ + return 'VisualStudio' + + @property + def sxs(self): + """ + Microsoft Visual Studio SxS registry key. + """ + return os.path.join(self.visualstudio, 'SxS') + + @property + def vc(self): + """ + Microsoft Visual C++ VC7 registry key. + """ + return os.path.join(self.sxs, 'VC7') + + @property + def vs(self): + """ + Microsoft Visual Studio VS7 registry key. + """ + return os.path.join(self.sxs, 'VS7') + + @property + def vc_for_python(self): + """ + Microsoft Visual C++ for Python registry key. + """ + return r'DevDiv\VCForPython' + + @property + def microsoft_sdk(self): + """ + Microsoft SDK registry key. + """ + return 'Microsoft SDKs' + + @property + def windows_sdk(self): + """ + Microsoft Windows/Platform SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'Windows') + + @property + def netfx_sdk(self): + """ + Microsoft .NET Framework SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'NETFXSDK') + + @property + def windows_kits_roots(self): + """ + Microsoft Windows Kits Roots registry key. + """ + return r'Windows Kits\Installed Roots' + + def microsoft(self, key, x86=False): + """ + Return key in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + x86: str + Force x86 software registry. + + Return + ------ + str: value + """ + node64 = '' if self.pi.current_is_x86() or x86 else r'\Wow6432Node' + return os.path.join('Software', node64, 'Microsoft', key) + + def lookup(self, key, name): + """ + Look for values in registry in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + name: str + Value name to find. + + Return + ------ + str: value + """ + KEY_READ = winreg.KEY_READ + openkey = winreg.OpenKey + ms = self.microsoft + for hkey in self.HKEYS: + try: + bkey = openkey(hkey, ms(key), 0, KEY_READ) + except (OSError, IOError): + if not self.pi.current_is_x86(): + try: + bkey = openkey(hkey, ms(key, True), 0, KEY_READ) + except (OSError, IOError): + continue + else: + continue + try: + return winreg.QueryValueEx(bkey, name)[0] + except (OSError, IOError): + pass + + +class SystemInfo: + """ + Microsoft Windows and Visual Studio related system inormations. + + Parameters + ---------- + registry_info: RegistryInfo + "RegistryInfo" instance. + vc_ver: float + Required Microsoft Visual C++ version. + """ + # Variables and properties in this class use originals CamelCase variables + # names from Microsoft source files for more easy comparaison. + WinDir = safe_env.get('WinDir', '') + ProgramFiles = safe_env.get('ProgramFiles', '') + ProgramFilesx86 = safe_env.get('ProgramFiles(x86)', ProgramFiles) + + def __init__(self, registry_info, vc_ver=None): + self.ri = registry_info + self.pi = self.ri.pi + if vc_ver: + self.vc_ver = vc_ver + else: + try: + self.vc_ver = self.find_available_vc_vers()[-1] + except IndexError: + err = 'No Microsoft Visual C++ version found' + raise distutils.errors.DistutilsPlatformError(err) + + def find_available_vc_vers(self): + """ + Find all available Microsoft Visual C++ versions. + """ + vckeys = (self.ri.vc, self.ri.vc_for_python) + vc_vers = [] + for hkey in self.ri.HKEYS: + for key in vckeys: + try: + bkey = winreg.OpenKey(hkey, key, 0, winreg.KEY_READ) + except (OSError, IOError): + continue + subkeys, values, _ = winreg.QueryInfoKey(bkey) + for i in range(values): + try: + ver = float(winreg.EnumValue(bkey, i)[0]) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + for i in range(subkeys): + try: + ver = float(winreg.EnumKey(bkey, i)) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + return sorted(vc_vers) + + @property + def VSInstallDir(self): + """ + Microsoft Visual Studio directory. + """ + # Default path + name = 'Microsoft Visual Studio %0.1f' % self.vc_ver + default = os.path.join(self.ProgramFilesx86, name) + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vs, '%0.1f' % self.vc_ver) or default + + @property + def VCInstallDir(self): + """ + Microsoft Visual C++ directory. + """ + # Default path + default = r'Microsoft Visual Studio %0.1f\VC' % self.vc_ver + guess_vc = os.path.join(self.ProgramFilesx86, default) + + # Try to get "VC++ for Python" path from registry as default path + reg_path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + python_vc = self.ri.lookup(reg_path, 'installdir') + default_vc = os.path.join(python_vc, 'VC') if python_vc else guess_vc + + # Try to get path from registry, if fail use default path + path = self.ri.lookup(self.ri.vc, '%0.1f' % self.vc_ver) or default_vc + + if not os.path.isdir(path): + msg = 'Microsoft Visual C++ directory not found' + raise distutils.errors.DistutilsPlatformError(msg) + + return path + + @property + def WindowsSdkVersion(self): + """ + Microsoft Windows SDK versions. + """ + # Set Windows SDK versions for specified MSVC++ version + if self.vc_ver <= 9.0: + return ('7.0', '6.1', '6.0a') + elif self.vc_ver == 10.0: + return ('7.1', '7.0a') + elif self.vc_ver == 11.0: + return ('8.0', '8.0a') + elif self.vc_ver == 12.0: + return ('8.1', '8.1a') + elif self.vc_ver >= 14.0: + return ('10.0', '8.1') + + @property + def WindowsSdkDir(self): + """ + Microsoft Windows SDK directory. + """ + sdkdir = '' + for ver in self.WindowsSdkVersion: + # Try to get it from registry + loc = os.path.join(self.ri.windows_sdk, 'v%s' % ver) + sdkdir = self.ri.lookup(loc, 'installationfolder') + if sdkdir: + break + if not sdkdir or not os.path.isdir(sdkdir): + # Try to get "VC++ for Python" version from registry + path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + install_base = self.ri.lookup(path, 'installdir') + if install_base: + sdkdir = os.path.join(install_base, 'WinSDK') + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default new path + for ver in self.WindowsSdkVersion: + intver = ver[:ver.rfind('.')] + path = r'Microsoft SDKs\Windows Kits\%s' % (intver) + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default old path + for ver in self.WindowsSdkVersion: + path = r'Microsoft SDKs\Windows\v%s' % ver + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir: + # If fail, use Platform SDK + sdkdir = os.path.join(self.VCInstallDir, 'PlatformSDK') + return sdkdir + + @property + def WindowsSDKExecutablePath(self): + """ + Microsoft Windows SDK executable directory. + """ + # Find WinSDK NetFx Tools registry dir name + if self.vc_ver <= 11.0: + netfxver = 35 + arch = '' + else: + netfxver = 40 + hidex86 = True if self.vc_ver <= 12.0 else False + arch = self.pi.current_dir(x64=True, hidex86=hidex86) + fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\', '-')) + + # liste all possibles registry paths + regpaths = [] + if self.vc_ver >= 14.0: + for ver in self.NetFxSdkVersion: + regpaths += [os.path.join(self.ri.netfx_sdk, ver, fx)] + + for ver in self.WindowsSdkVersion: + regpaths += [os.path.join(self.ri.windows_sdk, 'v%sA' % ver, fx)] + + # Return installation folder from the more recent path + for path in regpaths: + execpath = self.ri.lookup(path, 'installationfolder') + if execpath: + break + return execpath + + @property + def FSharpInstallDir(self): + """ + Microsoft Visual F# directory. + """ + path = r'%0.1f\Setup\F#' % self.vc_ver + path = os.path.join(self.ri.visualstudio, path) + return self.ri.lookup(path, 'productdir') or '' + + @property + def UniversalCRTSdkDir(self): + """ + Microsoft Universal CRT SDK directory. + """ + # Set Kit Roots versions for specified MSVC++ version + if self.vc_ver >= 14.0: + vers = ('10', '81') + else: + vers = () + + # Find path of the more recent Kit + for ver in vers: + sdkdir = self.ri.lookup(self.ri.windows_kits_roots, + 'kitsroot%s' % ver) + if sdkdir: + break + return sdkdir or '' + + @property + def NetFxSdkVersion(self): + """ + Microsoft .NET Framework SDK versions. + """ + # Set FxSdk versions for specified MSVC++ version + if self.vc_ver >= 14.0: + return ('4.6.1', '4.6') + else: + return () + + @property + def NetFxSdkDir(self): + """ + Microsoft .NET Framework SDK directory. + """ + for ver in self.NetFxSdkVersion: + loc = os.path.join(self.ri.netfx_sdk, ver) + sdkdir = self.ri.lookup(loc, 'kitsinstallationfolder') + if sdkdir: + break + return sdkdir or '' + + @property + def FrameworkDir32(self): + """ + Microsoft .NET Framework 32bit directory. + """ + # Default path + guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\Framework') + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vc, 'frameworkdir32') or guess_fw + + @property + def FrameworkDir64(self): + """ + Microsoft .NET Framework 64bit directory. + """ + # Default path + guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\Framework64') + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vc, 'frameworkdir64') or guess_fw + + @property + def FrameworkVersion32(self): + """ + Microsoft .NET Framework 32bit versions. + """ + return self._find_dot_net_versions(32) + + @property + def FrameworkVersion64(self): + """ + Microsoft .NET Framework 64bit versions. + """ + return self._find_dot_net_versions(64) + + def _find_dot_net_versions(self, bits=32): + """ + Find Microsoft .NET Framework versions. + + Parameters + ---------- + bits: int + Platform number of bits: 32 or 64. + """ + # Find actual .NET version + ver = self.ri.lookup(self.ri.vc, 'frameworkver%d' % bits) or '' + + # Set .NET versions for specified MSVC++ version + if self.vc_ver >= 12.0: + frameworkver = (ver, 'v4.0') + elif self.vc_ver >= 10.0: + frameworkver = ('v4.0.30319' if ver.lower()[:2] != 'v4' else ver, + 'v3.5') + elif self.vc_ver == 9.0: + frameworkver = ('v3.5', 'v2.0.50727') + if self.vc_ver == 8.0: + frameworkver = ('v3.0', 'v2.0.50727') + return frameworkver + + +class EnvironmentInfo: + """ + Return environment variables for specified Microsoft Visual C++ version + and platform : Lib, Include, Path and libpath. + + This function is compatible with Microsoft Visual C++ 9.0 to 14.0. + + Script created by analysing Microsoft environment configuration files like + "vcvars[...].bat", "SetEnv.Cmd", "vcbuildtools.bat", ... + + Parameters + ---------- + arch: str + Target architecture. + vc_ver: float + Required Microsoft Visual C++ version. If not set, autodetect the last + version. + vc_min_ver: float + Minimum Microsoft Visual C++ version. + """ + # Variables and properties in this class use originals CamelCase variables + # names from Microsoft source files for more easy comparaison. + + def __init__(self, arch, vc_ver=None, vc_min_ver=None): + self.pi = PlatformInfo(arch) + self.ri = RegistryInfo(self.pi) + self.si = SystemInfo(self.ri, vc_ver) + + if vc_min_ver: + if self.vc_ver < vc_min_ver: + err = 'No suitable Microsoft Visual C++ version found' + raise distutils.errors.DistutilsPlatformError(err) + + @property + def vc_ver(self): + """ + Microsoft Visual C++ version. + """ + return self.si.vc_ver + + @property + def VSTools(self): + """ + Microsoft Visual Studio Tools + """ + paths = [r'Common7\IDE', r'Common7\Tools'] + + if self.vc_ver >= 14.0: + arch_subdir = self.pi.current_dir(hidex86=True, x64=True) + paths += [r'Common7\IDE\CommonExtensions\Microsoft\TestWindow'] + paths += [r'Team Tools\Performance Tools'] + paths += [r'Team Tools\Performance Tools%s' % arch_subdir] + + return [os.path.join(self.si.VSInstallDir, path) for path in paths] + + @property + def VCIncludes(self): + """ + Microsoft Visual C++ & Microsoft Foundation Class Includes + """ + return [os.path.join(self.si.VCInstallDir, 'Include'), + os.path.join(self.si.VCInstallDir, r'ATLMFC\Include')] + + @property + def VCLibraries(self): + """ + Microsoft Visual C++ & Microsoft Foundation Class Libraries + """ + arch_subdir = self.pi.target_dir(hidex86=True) + paths = ['Lib%s' % arch_subdir, r'ATLMFC\Lib%s' % arch_subdir] + + if self.vc_ver >= 14.0: + paths += [r'Lib\store%s' % arch_subdir] + + return [os.path.join(self.si.VCInstallDir, path) for path in paths] + + @property + def VCStoreRefs(self): + """ + Microsoft Visual C++ store references Libraries + """ + if self.vc_ver < 14.0: + return [] + return [os.path.join(self.si.VCInstallDir, r'Lib\store\references')] + + @property + def VCTools(self): + """ + Microsoft Visual C++ Tools + """ + si = self.si + tools = [os.path.join(si.VCInstallDir, 'VCPackages')] + + forcex86 = True if self.vc_ver <= 10.0 else False + arch_subdir = self.pi.cross_dir(forcex86) + if arch_subdir: + tools += [os.path.join(si.VCInstallDir, 'Bin%s' % arch_subdir)] + + if self.vc_ver >= 14.0: + path = 'Bin%s' % self.pi.current_dir(hidex86=True) + tools += [os.path.join(si.VCInstallDir, path)] + + else: + tools += [os.path.join(si.VCInstallDir, 'Bin')] + + return tools + + @property + def OSLibraries(self): + """ + Microsoft Windows SDK Libraries + """ + if self.vc_ver <= 10.0: + arch_subdir = self.pi.target_dir(hidex86=True, x64=True) + return [os.path.join(self.si.WindowsSdkDir, 'Lib%s' % arch_subdir)] + + else: + arch_subdir = self.pi.target_dir(x64=True) + lib = os.path.join(self.si.WindowsSdkDir, 'lib') + libver = self._get_content_dirname(lib) + return [os.path.join(lib, '%sum%s' % (libver, arch_subdir))] + + @property + def OSIncludes(self): + """ + Microsoft Windows SDK Include + """ + include = os.path.join(self.si.WindowsSdkDir, 'include') + + if self.vc_ver <= 10.0: + return [include, os.path.join(include, 'gl')] + + else: + if self.vc_ver >= 14.0: + sdkver = self._get_content_dirname(include) + else: + sdkver = '' + return [os.path.join(include, '%sshared' % sdkver), + os.path.join(include, '%sum' % sdkver), + os.path.join(include, '%swinrt' % sdkver)] + + @property + def OSLibpath(self): + """ + Microsoft Windows SDK Libraries Paths + """ + ref = os.path.join(self.si.WindowsSdkDir, 'References') + libpath = [] + + if self.vc_ver <= 9.0: + libpath += self.OSLibraries + + if self.vc_ver >= 11.0: + libpath += [os.path.join(ref, r'CommonConfiguration\Neutral')] + + if self.vc_ver >= 14.0: + libpath += [ + ref, + os.path.join(self.si.WindowsSdkDir, 'UnionMetadata'), + os.path.join( + ref, + 'Windows.Foundation.UniversalApiContract', + '1.0.0.0', + ), + os.path.join( + ref, + 'Windows.Foundation.FoundationContract', + '1.0.0.0', + ), + os.path.join( + ref, + 'Windows.Networking.Connectivity.WwanContract', + '1.0.0.0', + ), + os.path.join( + self.si.WindowsSdkDir, + 'ExtensionSDKs', + 'Microsoft.VCLibs', + '%0.1f' % self.vc_ver, + 'References', + 'CommonConfiguration', + 'neutral', + ), + ] + return libpath + + @property + def SdkTools(self): + """ + Microsoft Windows SDK Tools + """ + bin_dir = 'Bin' if self.vc_ver <= 11.0 else r'Bin\x86' + tools = [os.path.join(self.si.WindowsSdkDir, bin_dir)] + + if not self.pi.current_is_x86(): + arch_subdir = self.pi.current_dir(x64=True) + path = 'Bin%s' % arch_subdir + tools += [os.path.join(self.si.WindowsSdkDir, path)] + + if self.vc_ver == 10.0 or self.vc_ver == 11.0: + if self.pi.target_is_x86(): + arch_subdir = '' + else: + arch_subdir = self.pi.current_dir(hidex86=True, x64=True) + path = r'Bin\NETFX 4.0 Tools%s' % arch_subdir + tools += [os.path.join(self.si.WindowsSdkDir, path)] + + if self.si.WindowsSDKExecutablePath: + tools += [self.si.WindowsSDKExecutablePath] + + return tools + + @property + def SdkSetup(self): + """ + Microsoft Windows SDK Setup + """ + if self.vc_ver > 9.0: + return [] + + return [os.path.join(self.si.WindowsSdkDir, 'Setup')] + + @property + def FxTools(self): + """ + Microsoft .NET Framework Tools + """ + pi = self.pi + si = self.si + + if self.vc_ver <= 10.0: + include32 = True + include64 = not pi.target_is_x86() and not pi.current_is_x86() + else: + include32 = pi.target_is_x86() or pi.current_is_x86() + include64 = pi.current_cpu == 'amd64' or pi.target_cpu == 'amd64' + + tools = [] + if include32: + tools += [os.path.join(si.FrameworkDir32, ver) + for ver in si.FrameworkVersion32] + if include64: + tools += [os.path.join(si.FrameworkDir64, ver) + for ver in si.FrameworkVersion64] + return tools + + @property + def NetFxSDKLibraries(self): + """ + Microsoft .Net Framework SDK Libraries + """ + if self.vc_ver < 14.0 or not self.si.NetFxSdkDir: + return [] + + arch_subdir = self.pi.target_dir(x64=True) + return [os.path.join(self.si.NetFxSdkDir, r'lib\um%s' % arch_subdir)] + + @property + def NetFxSDKIncludes(self): + """ + Microsoft .Net Framework SDK Includes + """ + if self.vc_ver < 14.0 or not self.si.NetFxSdkDir: + return [] + + return [os.path.join(self.si.NetFxSdkDir, r'include\um')] + + @property + def VsTDb(self): + """ + Microsoft Visual Studio Team System Database + """ + return [os.path.join(self.si.VSInstallDir, r'VSTSDB\Deploy')] + + @property + def MSBuild(self): + """ + Microsoft Build Engine + """ + if self.vc_ver < 12.0: + return [] + + arch_subdir = self.pi.current_dir(hidex86=True) + path = r'MSBuild\%0.1f\bin%s' % (self.vc_ver, arch_subdir) + return [os.path.join(self.si.ProgramFilesx86, path)] + + @property + def HTMLHelpWorkshop(self): + """ + Microsoft HTML Help Workshop + """ + if self.vc_ver < 11.0: + return [] + + return [os.path.join(self.si.ProgramFilesx86, 'HTML Help Workshop')] + + @property + def UCRTLibraries(self): + """ + Microsoft Universal CRT Libraries + """ + if self.vc_ver < 14.0: + return [] + + arch_subdir = self.pi.target_dir(x64=True) + lib = os.path.join(self.si.UniversalCRTSdkDir, 'lib') + ucrtver = self._get_content_dirname(lib) + return [os.path.join(lib, '%sucrt%s' % (ucrtver, arch_subdir))] + + @property + def UCRTIncludes(self): + """ + Microsoft Universal CRT Include + """ + if self.vc_ver < 14.0: + return [] + + include = os.path.join(self.si.UniversalCRTSdkDir, 'include') + ucrtver = self._get_content_dirname(include) + return [os.path.join(include, '%sucrt' % ucrtver)] + + @property + def FSharp(self): + """ + Microsoft Visual F# + """ + if self.vc_ver < 11.0 and self.vc_ver > 12.0: + return [] + + return self.si.FSharpInstallDir + + @property + def VCRuntimeRedist(self): + """ + Microsoft Visual C++ runtime redistribuable dll + """ + arch_subdir = self.pi.target_dir(x64=True) + vcruntime = 'redist%s\\Microsoft.VC%d0.CRT\\vcruntime%d0.dll' + vcruntime = vcruntime % (arch_subdir, self.vc_ver, self.vc_ver) + return os.path.join(self.si.VCInstallDir, vcruntime) + + def return_env(self, exists=True): + """ + Return environment dict. + + Parameters + ---------- + exists: bool + It True, only return existing paths. + """ + env = dict( + include=self._build_paths('include', + [self.VCIncludes, + self.OSIncludes, + self.UCRTIncludes, + self.NetFxSDKIncludes], + exists), + lib=self._build_paths('lib', + [self.VCLibraries, + self.OSLibraries, + self.FxTools, + self.UCRTLibraries, + self.NetFxSDKLibraries], + exists), + libpath=self._build_paths('libpath', + [self.VCLibraries, + self.FxTools, + self.VCStoreRefs, + self.OSLibpath], + exists), + path=self._build_paths('path', + [self.VCTools, + self.VSTools, + self.VsTDb, + self.SdkTools, + self.SdkSetup, + self.FxTools, + self.MSBuild, + self.HTMLHelpWorkshop, + self.FSharp], + exists), + ) + if self.vc_ver >= 14 and os.path.isfile(self.VCRuntimeRedist): + env['py_vcruntime_redist'] = self.VCRuntimeRedist + return env + + def _build_paths(self, name, spec_path_lists, exists): + """ + Given an environment variable name and specified paths, + return a pathsep-separated string of paths containing + unique, extant, directories from those paths and from + the environment variable. Raise an error if no paths + are resolved. + """ + # flatten spec_path_lists + spec_paths = itertools.chain.from_iterable(spec_path_lists) + env_paths = safe_env.get(name, '').split(os.pathsep) + paths = itertools.chain(spec_paths, env_paths) + extant_paths = list(filter(os.path.isdir, paths)) if exists else paths + if not extant_paths: + msg = "%s environment variable is empty" % name.upper() + raise distutils.errors.DistutilsPlatformError(msg) + unique_paths = self._unique_everseen(extant_paths) + return os.pathsep.join(unique_paths) + + # from Python docs + def _unique_everseen(self, iterable, key=None): + """ + List unique elements, preserving order. + Remember all elements ever seen. + + _unique_everseen('AAAABBBCCDAABBB') --> A B C D + + _unique_everseen('ABBCcAD', str.lower) --> A B C D + """ + seen = set() + seen_add = seen.add + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element + + def _get_content_dirname(self, path): + """ + Return name of the first dir in path or '' if no dir found. + + Parameters + ---------- + path: str + Path where search dir. + + Return + ------ + foldername: str + "name\" or "" + """ + try: + name = os.listdir(path) + if name: + return '%s\\' % name[0] + return '' + except (OSError, IOError): + return '' diff --git a/setuptools/msvc9_support.py b/setuptools/msvc9_support.py deleted file mode 100644 index 9d869580..00000000 --- a/setuptools/msvc9_support.py +++ /dev/null @@ -1,63 +0,0 @@ -try: - import distutils.msvc9compiler -except Exception: - pass - -unpatched = dict() - -def patch_for_specialized_compiler(): - """ - Patch functions in distutils.msvc9compiler to use the standalone compiler - build for Python (Windows only). Fall back to original behavior when the - standalone compiler is not available. - """ - if 'distutils' not in globals(): - # The module isn't available to be patched - return - - if unpatched: - # Already patched - return - - unpatched.update(vars(distutils.msvc9compiler)) - - distutils.msvc9compiler.find_vcvarsall = find_vcvarsall - distutils.msvc9compiler.query_vcvarsall = query_vcvarsall - -def find_vcvarsall(version): - Reg = distutils.msvc9compiler.Reg - VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' - key = VC_BASE % ('', version) - try: - # Per-user installs register the compiler path here - productdir = Reg.get_value(key, "installdir") - except KeyError: - try: - # All-user installs on a 64-bit system register here - key = VC_BASE % ('Wow6432Node\\', version) - productdir = Reg.get_value(key, "installdir") - except KeyError: - productdir = None - - if productdir: - import os - vcvarsall = os.path.join(productdir, "vcvarsall.bat") - if os.path.isfile(vcvarsall): - return vcvarsall - - return unpatched['find_vcvarsall'](version) - -def query_vcvarsall(version, *args, **kwargs): - try: - return unpatched['query_vcvarsall'](version, *args, **kwargs) - except distutils.errors.DistutilsPlatformError as exc: - if exc and "vcvarsall.bat" in exc.args[0]: - message = 'Microsoft Visual C++ %0.1f is required (%s).' % (version, exc.args[0]) - if int(version) == 9: - # This redirection link is maintained by Microsoft. - # Contact vspython@microsoft.com if it needs updating. - raise distutils.errors.DistutilsPlatformError( - message + ' Get it from http://aka.ms/vcpython27' - ) - raise distutils.errors.DistutilsPlatformError(message) - raise diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e87504db..3e8d6818 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -37,7 +37,7 @@ PYPI_MD5 = re.compile( '<a href="([^"#]+)">([^<]+)</a>\n\s+\\(<a (?:title="MD5 hash"\n\s+)' 'href="[^?]+\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\\)' ) -URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match +URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() __all__ = [ @@ -52,6 +52,15 @@ _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}" user_agent = _tmpl.format(py_major=sys.version[:3], **globals()) +def parse_requirement_arg(spec): + try: + return Requirement.parse(spec) + except ValueError: + raise DistutilsError( + "Not a URL, existing file, or requirement spec: %r" % (spec,) + ) + + def parse_bdist_wininst(name): """Return (base,pyversion) or (None,None) for possible .exe name""" @@ -62,41 +71,45 @@ def parse_bdist_wininst(name): if lower.endswith('.win32.exe'): base = name[:-10] plat = 'win32' - elif lower.startswith('.win32-py',-16): + elif lower.startswith('.win32-py', -16): py_ver = name[-7:-4] base = name[:-16] plat = 'win32' elif lower.endswith('.win-amd64.exe'): base = name[:-14] plat = 'win-amd64' - elif lower.startswith('.win-amd64-py',-20): + elif lower.startswith('.win-amd64-py', -20): py_ver = name[-7:-4] base = name[:-20] plat = 'win-amd64' - return base,py_ver,plat + return base, py_ver, plat def egg_info_for_url(url): parts = urllib.parse.urlparse(url) scheme, server, path, parameters, query, fragment = parts base = urllib.parse.unquote(path.split('/')[-1]) - if server=='sourceforge.net' and base=='download': # XXX Yuck + if server == 'sourceforge.net' and base == 'download': # XXX Yuck base = urllib.parse.unquote(path.split('/')[-2]) - if '#' in base: base, fragment = base.split('#',1) - return base,fragment + if '#' in base: + base, fragment = base.split('#', 1) + return base, fragment + def distros_for_url(url, metadata=None): """Yield egg or source distribution objects that might be found at a URL""" base, fragment = egg_info_for_url(url) - for dist in distros_for_location(url, base, metadata): yield dist + for dist in distros_for_location(url, base, metadata): + yield dist if fragment: match = EGG_FRAGMENT.match(fragment) if match: for dist in interpret_distro_name( - url, match.group(1), metadata, precedence = CHECKOUT_DIST + url, match.group(1), metadata, precedence=CHECKOUT_DIST ): yield dist + def distros_for_location(location, basename, metadata=None): """Yield egg or source distribution objects based on basename""" if basename.endswith('.egg.zip'): @@ -118,6 +131,7 @@ def distros_for_location(location, basename, metadata=None): return interpret_distro_name(location, basename, metadata) return [] # no extension matched + def distros_for_filename(filename, metadata=None): """Yield possible egg or source distribution objects based on a filename""" return distros_for_location( @@ -152,13 +166,14 @@ def interpret_distro_name( # it is a bdist_dumb, not an sdist -- bail out return - for p in range(1,len(parts)+1): + for p in range(1, len(parts) + 1): yield Distribution( location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), - py_version=py_version, precedence = precedence, - platform = platform + py_version=py_version, precedence=precedence, + platform=platform ) + # From Python 2.7 docs def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." @@ -177,6 +192,7 @@ def unique_everseen(iterable, key=None): seen_add(k) yield element + def unique_values(func): """ Wrap a function returning an iterable such that the resulting iterable @@ -187,9 +203,11 @@ def unique_values(func): return unique_everseen(func(*args, **kwargs)) return wrapper + REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) # this line is here to fix emacs' cruddy broken syntax highlighting + @unique_values def find_external_links(url, page): """Find rel="homepage" and rel="download" links in `page`, yielding URLs""" @@ -203,8 +221,8 @@ def find_external_links(url, page): for tag in ("<th>Home Page", "<th>Download URL"): pos = page.find(tag) - if pos!=-1: - match = HREF.search(page,pos) + if pos != -1: + match = HREF.search(page, pos) if match: yield urllib.parse.urljoin(url, htmldecode(match.group(1))) @@ -213,6 +231,7 @@ class ContentChecker(object): """ A null content checker that defines the interface for checking content """ + def feed(self, block): """ Feed a block of data to the hash. @@ -232,6 +251,7 @@ class ContentChecker(object): """ return + class HashChecker(ContentChecker): pattern = re.compile( r'(?P<hash_name>sha1|sha224|sha384|sha256|sha512|md5)=' @@ -272,16 +292,22 @@ class PackageIndex(Environment): self, index_url="https://pypi.python.org/simple", hosts=('*',), ca_bundle=None, verify_ssl=True, *args, **kw ): - Environment.__init__(self,*args,**kw) - self.index_url = index_url + "/"[:not index_url.endswith('/')] + Environment.__init__(self, *args, **kw) + self.index_url = index_url + "/" [:not index_url.endswith('/')] self.scanned_urls = {} self.fetched_urls = {} self.package_pages = {} - self.allows = re.compile('|'.join(map(translate,hosts))).match + self.allows = re.compile('|'.join(map(translate, hosts))).match self.to_scan = [] - if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()): + use_ssl = ( + verify_ssl + and ssl_support.is_available + and (ca_bundle or ssl_support.find_ca_bundle()) + ) + if use_ssl: self.opener = ssl_support.opener_for(ca_bundle) - else: self.opener = urllib.request.urlopen + else: + self.opener = urllib.request.urlopen def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -308,8 +334,10 @@ class PackageIndex(Environment): self.info("Reading %s", url) self.fetched_urls[url] = True # prevent multiple fetch attempts - f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url) - if f is None: return + tmpl = "Download error on %s: %%s -- Some packages may not be found!" + f = self.open_url(url, tmpl % url) + if f is None: + return self.fetched_urls[f.url] = True if 'html' not in f.headers.get('content-type', '').lower(): f.close() # not html, we can't process it @@ -317,7 +345,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() - if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. + if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. if isinstance(f, urllib.error.HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' @@ -328,7 +356,7 @@ class PackageIndex(Environment): for match in HREF.finditer(page): link = urllib.parse.urljoin(base, htmldecode(match.group(1))) self.process_url(link) - if url.startswith(self.index_url) and getattr(f,'code',None)!=404: + if url.startswith(self.index_url) and getattr(f, 'code', None) != 404: page = self.process_index(url, page) def process_filename(self, fn, nested=False): @@ -340,7 +368,7 @@ class PackageIndex(Environment): if os.path.isdir(fn) and not nested: path = os.path.realpath(fn) for item in os.listdir(path): - self.process_filename(os.path.join(path,item), True) + self.process_filename(os.path.join(path, item), True) dists = distros_for_filename(fn) if dists: @@ -349,7 +377,8 @@ class PackageIndex(Environment): def url_ok(self, url, fatal=False): s = URL_SCHEME(url) - if (s and s.group(1).lower()=='file') or self.allows(urllib.parse.urlparse(url)[1]): + is_file = s and s.group(1).lower() == 'file' + if is_file or self.allows(urllib.parse.urlparse(url)[1]): return True msg = ("\nNote: Bypassing %s (disallowed host; see " "http://bit.ly/1dg9ijs for details).\n") @@ -384,7 +413,7 @@ class PackageIndex(Environment): dist.precedence = SOURCE_DIST self.add(dist) - def process_index(self,url,page): + def process_index(self, url, page): """Process the contents of a PyPI page""" def scan(link): # Process a URL to see if it's for a package page @@ -392,11 +421,11 @@ class PackageIndex(Environment): parts = list(map( urllib.parse.unquote, link[len(self.index_url):].split('/') )) - if len(parts)==2 and '#' not in parts[1]: + if len(parts) == 2 and '#' not in parts[1]: # it's a package page, sanitize and index it pkg = safe_name(parts[0]) ver = safe_version(parts[1]) - self.package_pages.setdefault(pkg.lower(),{})[link] = True + self.package_pages.setdefault(pkg.lower(), {})[link] = True return to_filename(pkg), to_filename(ver) return None, None @@ -415,13 +444,13 @@ class PackageIndex(Environment): base, frag = egg_info_for_url(new_url) if base.endswith('.py') and not frag: if ver: - new_url+='#egg=%s-%s' % (pkg,ver) + new_url += '#egg=%s-%s' % (pkg, ver) else: self.need_version_info(url) self.scan_url(new_url) return PYPI_MD5.sub( - lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1,3,2), page + lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page ) else: return "" # no sense double-scanning non-package pages @@ -434,24 +463,25 @@ class PackageIndex(Environment): def scan_all(self, msg=None, *args): if self.index_url not in self.fetched_urls: - if msg: self.warn(msg,*args) + if msg: + self.warn(msg, *args) self.info( "Scanning index of all packages (this may take a while)" ) self.scan_url(self.index_url) def find_packages(self, requirement): - self.scan_url(self.index_url + requirement.unsafe_name+'/') + self.scan_url(self.index_url + requirement.unsafe_name + '/') if not self.package_pages.get(requirement.key): # Fall back to safe version of the name - self.scan_url(self.index_url + requirement.project_name+'/') + self.scan_url(self.index_url + requirement.project_name + '/') if not self.package_pages.get(requirement.key): # We couldn't find the target package, so search the index page too self.not_found_in_index(requirement) - for url in list(self.package_pages.get(requirement.key,())): + for url in list(self.package_pages.get(requirement.key, ())): # scan each page that might be related to the desired package self.scan_url(url) @@ -462,7 +492,7 @@ class PackageIndex(Environment): if dist in requirement: return dist self.debug("%s does not match %s", requirement, dist) - return super(PackageIndex, self).obtain(requirement,installer) + return super(PackageIndex, self).obtain(requirement, installer) def check_hash(self, checker, filename, tfp): """ @@ -527,27 +557,21 @@ class PackageIndex(Environment): of `tmpdir`, and the local filename is returned. Various errors may be raised if a problem occurs during downloading. """ - if not isinstance(spec,Requirement): + if not isinstance(spec, Requirement): scheme = URL_SCHEME(spec) if scheme: # It's a url, download it to tmpdir found = self._download_url(scheme.group(1), spec, tmpdir) base, fragment = egg_info_for_url(spec) if base.endswith('.py'): - found = self.gen_setup(found,fragment,tmpdir) + found = self.gen_setup(found, fragment, tmpdir) return found elif os.path.exists(spec): # Existing file or directory, just return it return spec else: - try: - spec = Requirement.parse(spec) - except ValueError: - raise DistutilsError( - "Not a URL, existing file, or requirement spec: %r" % - (spec,) - ) - return getattr(self.fetch_distribution(spec, tmpdir),'location',None) + spec = parse_requirement_arg(spec) + return getattr(self.fetch_distribution(spec, tmpdir), 'location', None) def fetch_distribution( self, requirement, tmpdir, force_scan=False, source=False, @@ -581,22 +605,24 @@ class PackageIndex(Environment): for dist in env[req.key]: - if dist.precedence==DEVELOP_DIST and not develop_ok: + if dist.precedence == DEVELOP_DIST and not develop_ok: if dist not in skipped: - self.warn("Skipping development or system egg: %s",dist) + self.warn("Skipping development or system egg: %s", dist) skipped[dist] = 1 continue - if dist in req and (dist.precedence<=SOURCE_DIST or not source): - return dist + if dist in req and (dist.precedence <= SOURCE_DIST or not source): + dist.download_location = self.download(dist.location, tmpdir) + if os.path.exists(dist.download_location): + return dist if force_scan: self.prescan() self.find_packages(requirement) dist = find(requirement) - if local_index is not None: - dist = dist or find(requirement, local_index) + if not dist and local_index is not None: + dist = find(requirement, local_index) if dist is None: if self.to_scan is not None: @@ -609,13 +635,13 @@ class PackageIndex(Environment): if dist is None: self.warn( - "No local packages or download links found for %s%s", + "No local packages or working download links found for %s%s", (source and "a source distribution of " or ""), requirement, ) else: self.info("Best match: %s", dist) - return dist.clone(location=self.download(dist.location, tmpdir)) + return dist.clone(location=dist.download_location) def fetch(self, requirement, tmpdir, force_scan=False, source=False): """Obtain a file suitable for fulfilling `requirement` @@ -625,7 +651,7 @@ class PackageIndex(Environment): ``location`` of the downloaded distribution instead of a distribution object. """ - dist = self.fetch_distribution(requirement,tmpdir,force_scan,source) + dist = self.fetch_distribution(requirement, tmpdir, force_scan, source) if dist is not None: return dist.location return None @@ -637,7 +663,7 @@ class PackageIndex(Environment): interpret_distro_name(filename, match.group(1), None) if d.version ] or [] - if len(dists)==1: # unambiguous ``#egg`` fragment + if len(dists) == 1: # unambiguous ``#egg`` fragment basename = os.path.basename(filename) # Make sure the file has been downloaded to the temp dir. @@ -646,7 +672,7 @@ class PackageIndex(Environment): from setuptools.command.easy_install import samefile if not samefile(filename, dst): shutil.copy2(filename, dst) - filename=dst + filename = dst with open(os.path.join(tmpdir, 'setup.py'), 'w') as file: file.write( @@ -663,7 +689,7 @@ class PackageIndex(Environment): raise DistutilsError( "Can't unambiguously interpret project/version identifier %r; " "any dashes in the name or version should be escaped using " - "underscores. %r" % (fragment,dists) + "underscores. %r" % (fragment, dists) ) else: raise DistutilsError( @@ -672,6 +698,7 @@ class PackageIndex(Environment): ) dl_blocksize = 8192 + def _download_to(self, url, filename): self.info("Downloading %s", url) # Download the file @@ -681,7 +708,7 @@ class PackageIndex(Environment): fp = self.open_url(strip_fragment(url)) if isinstance(fp, urllib.error.HTTPError): raise DistutilsError( - "Can't download %s: %s %s" % (url, fp.code,fp.msg) + "Can't download %s: %s %s" % (url, fp.code, fp.msg) ) headers = fp.info() blocknum = 0 @@ -692,7 +719,7 @@ class PackageIndex(Environment): sizes = get_all_headers(headers, 'Content-Length') size = max(map(int, sizes)) self.reporthook(url, filename, blocknum, bs, size) - with open(filename,'wb') as tfp: + with open(filename, 'wb') as tfp: while True: block = fp.read(bs) if block: @@ -705,7 +732,8 @@ class PackageIndex(Environment): self.check_hash(checker, filename, tfp) return headers finally: - if fp: fp.close() + if fp: + fp.close() def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op @@ -751,24 +779,24 @@ class PackageIndex(Environment): name, fragment = egg_info_for_url(url) if name: while '..' in name: - name = name.replace('..','.').replace('\\','_') + name = name.replace('..', '.').replace('\\', '_') else: name = "__downloaded__" # default if URL has no path contents if name.endswith('.egg.zip'): name = name[:-4] # strip the extra .zip before download - filename = os.path.join(tmpdir,name) + filename = os.path.join(tmpdir, name) # Download the file # - if scheme=='svn' or scheme.startswith('svn+'): + if scheme == 'svn' or scheme.startswith('svn+'): return self._download_svn(url, filename) - elif scheme=='git' or scheme.startswith('git+'): + elif scheme == 'git' or scheme.startswith('git+'): return self._download_git(url, filename) elif scheme.startswith('hg+'): return self._download_hg(url, filename) - elif scheme=='file': + elif scheme == 'file': return urllib.request.url2pathname(urllib.parse.urlparse(url)[2]) else: self.url_ok(url, True) # raises error if not allowed @@ -779,7 +807,7 @@ class PackageIndex(Environment): def _attempt_download(self, url, filename): headers = self._download_to(url, filename) - if 'html' in headers.get('content-type','').lower(): + if 'html' in headers.get('content-type', '').lower(): return self._download_html(url, headers, filename) else: return filename @@ -797,22 +825,22 @@ class PackageIndex(Environment): break # not an index page file.close() os.unlink(filename) - raise DistutilsError("Unexpected HTML page found at "+url) + raise DistutilsError("Unexpected HTML page found at " + url) def _download_svn(self, url, filename): - url = url.split('#',1)[0] # remove any fragment for svn's sake + url = url.split('#', 1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: - netloc, path = path[2:].split('/',1) + netloc, path = path[2:].split('/', 1) auth, host = splituser(netloc) if auth: if ':' in auth: - user, pw = auth.split(':',1) + user, pw = auth.split(':', 1) creds = " --username=%s --password=%s" % (user, pw) else: - creds = " --username="+auth + creds = " --username=" + auth netloc = host parts = scheme, netloc, url, p, q, f url = urllib.parse.urlunparse(parts) @@ -827,7 +855,7 @@ class PackageIndex(Environment): scheme = scheme.split('+', 1)[-1] # Some fragment identification fails - path = path.split('#',1)[0] + path = path.split('#', 1)[0] rev = None if '@' in path: @@ -839,7 +867,7 @@ class PackageIndex(Environment): return url, rev def _download_git(self, url, filename): - filename = filename.split('#',1)[0] + filename = filename.split('#', 1)[0] url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) self.info("Doing git clone from %s to %s", url, filename) @@ -855,7 +883,7 @@ class PackageIndex(Environment): return filename def _download_hg(self, url, filename): - filename = filename.split('#',1)[0] + filename = filename.split('#', 1)[0] url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) self.info("Doing hg clone from %s to %s", url, filename) @@ -879,16 +907,20 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) + # This pattern matches a character entity reference (a decimal numeric # references, a hexadecimal numeric reference, or a named reference). entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub + def uchr(c): if not isinstance(c, int): return c - if c>255: return six.unichr(c) + if c > 255: + return six.unichr(c) return chr(c) + def decode_entity(match): what = match.group(1) if what.startswith('#x'): @@ -899,10 +931,12 @@ def decode_entity(match): what = six.moves.html_entities.name2codepoint.get(what, match.group(0)) return uchr(what) + def htmldecode(text): """Decode HTML entities in the given text.""" return entity_sub(decode_entity, text) + def socket_timeout(timeout=15): def _socket_timeout(func): def _socket_timeout(*args, **kwargs): @@ -915,6 +949,7 @@ def socket_timeout(timeout=15): return _socket_timeout return _socket_timeout + def _encode_auth(auth): """ A function compatible with Python 2.3-3.3 that will encode @@ -935,12 +970,14 @@ def _encode_auth(auth): # convert back to a string encoded = encoded_bytes.decode() # strip the trailing carriage return - return encoded.replace('\n','') + return encoded.replace('\n', '') + class Credential(object): """ A username/password pair. Use like a namedtuple. """ + def __init__(self, username, password): self.username = username self.password = password @@ -952,6 +989,7 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) + class PyPIConfig(configparser.RawConfigParser): def __init__(self): @@ -1011,7 +1049,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): if cred: auth = str(cred) info = cred.username, url - log.info('Authenticating as %s for %s (from .pypirc)' % info) + log.info('Authenticating as %s for %s (from .pypirc)', *info) if auth: auth = "Basic " + _encode_auth(auth) @@ -1029,12 +1067,13 @@ def open_with_auth(url, opener=urllib.request.urlopen): # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) - if s2==scheme and h2==host: + if s2 == scheme and h2 == host: parts = s2, netloc, path2, param2, query2, frag2 fp.url = urllib.parse.urlunparse(parts) return fp + # adding a timeout to avoid freezing package_index open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) @@ -1042,6 +1081,7 @@ open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) def fix_sf_url(url): return url # backward compatibility + def local_open(url): """Read a local path, with special support for directories""" scheme, server, path, param, query, frag = urllib.parse.urlparse(url) diff --git a/setuptools/py26compat.py b/setuptools/py26compat.py index 40cbb88e..5778cdf1 100644 --- a/setuptools/py26compat.py +++ b/setuptools/py26compat.py @@ -9,6 +9,7 @@ try: except ImportError: from urllib import splittag + def strip_fragment(url): """ In `Python 8280 <http://bugs.python.org/issue8280>`_, Python 2.7 and @@ -18,5 +19,13 @@ def strip_fragment(url): url, fragment = splittag(url) return url -if sys.version_info >= (2,7): + +if sys.version_info >= (2, 7): strip_fragment = lambda x: x + + +try: + from importlib import import_module +except ImportError: + def import_module(module_name): + return __import__(module_name, fromlist=['__name__']) diff --git a/setuptools/py27compat.py b/setuptools/py27compat.py index 702f7d65..57eb150b 100644 --- a/setuptools/py27compat.py +++ b/setuptools/py27compat.py @@ -4,12 +4,14 @@ Compatibility Support for Python 2.7 and earlier import sys + def get_all_headers(message, key): """ Given an HTTPMessage, return all headers matching a given key. """ return message.get_all(key) + if sys.version_info < (3,): def get_all_headers(message, key): return message.getheaders(key) diff --git a/setuptools/py31compat.py b/setuptools/py31compat.py index 8fe6dd9d..1e959e28 100644 --- a/setuptools/py31compat.py +++ b/setuptools/py31compat.py @@ -8,10 +8,11 @@ try: from sysconfig import get_config_vars, get_path except ImportError: from distutils.sysconfig import get_config_vars, get_python_lib + def get_path(name): if name not in ('platlib', 'purelib'): raise ValueError("Name must be purelib or platlib") - return get_python_lib(name=='platlib') + return get_python_lib(name == 'platlib') try: # Python >=3.2 @@ -19,14 +20,16 @@ try: except ImportError: import shutil import tempfile + class TemporaryDirectory(object): """ Very simple temporary directory context manager. Will try to delete afterward, but will also ignore OS and similar errors on deletion. """ + def __init__(self): - self.name = None # Handle mkdtemp raising an exception + self.name = None # Handle mkdtemp raising an exception self.name = tempfile.mkdtemp() def __enter__(self): @@ -35,7 +38,7 @@ except ImportError: def __exit__(self, exctype, excvalue, exctrace): try: shutil.rmtree(self.name, True) - except OSError: #removal errors are not the only possible + except OSError: # removal errors are not the only possible pass self.name = None diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 23e296b1..2babb636 100755 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -29,6 +29,7 @@ __all__ = [ "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup", ] + def _execfile(filename, globals, locals=None): """ Python 3 implementation of execfile. @@ -117,6 +118,7 @@ class ExceptionSaver: A Context Manager that will save an exception, serialized, and restore it later. """ + def __enter__(self): return self @@ -232,11 +234,12 @@ def run_setup(setup_script, args): setup_dir = os.path.abspath(os.path.dirname(setup_script)) with setup_context(setup_dir): try: - sys.argv[:] = [setup_script]+list(args) + sys.argv[:] = [setup_script] + list(args) sys.path.insert(0, setup_dir) # reset to include setup dir, w/clean callback list working_set.__init__() - working_set.callbacks.append(lambda dist:dist.activate()) + working_set.callbacks.append(lambda dist: dist.activate()) + def runner(): ns = dict(__file__=setup_script, __name__='__main__') _execfile(setup_script, ns) @@ -255,12 +258,12 @@ class AbstractSandbox: def __init__(self): self._attrs = [ name for name in dir(_os) - if not name.startswith('_') and hasattr(self,name) + if not name.startswith('_') and hasattr(self, name) ] def _copy(self, source): for name in self._attrs: - setattr(os, name, getattr(source,name)) + setattr(os, name, getattr(source, name)) def run(self, func): """Run 'func' under os sandboxing""" @@ -279,22 +282,25 @@ class AbstractSandbox: self._copy(_os) def _mk_dual_path_wrapper(name): - original = getattr(_os,name) - def wrap(self,src,dst,*args,**kw): + original = getattr(_os, name) + + def wrap(self, src, dst, *args, **kw): if self._active: - src,dst = self._remap_pair(name,src,dst,*args,**kw) - return original(src,dst,*args,**kw) + src, dst = self._remap_pair(name, src, dst, *args, **kw) + return original(src, dst, *args, **kw) return wrap for name in ["rename", "link", "symlink"]: - if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name) + if hasattr(_os, name): + locals()[name] = _mk_dual_path_wrapper(name) def _mk_single_path_wrapper(name, original=None): - original = original or getattr(_os,name) - def wrap(self,path,*args,**kw): + original = original or getattr(_os, name) + + def wrap(self, path, *args, **kw): if self._active: - path = self._remap_input(name,path,*args,**kw) - return original(path,*args,**kw) + path = self._remap_input(name, path, *args, **kw) + return original(path, *args, **kw) return wrap if _file: @@ -305,49 +311,54 @@ class AbstractSandbox: "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat", "startfile", "mkfifo", "mknod", "pathconf", "access" ]: - if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name) + if hasattr(_os, name): + locals()[name] = _mk_single_path_wrapper(name) def _mk_single_with_return(name): - original = getattr(_os,name) - def wrap(self,path,*args,**kw): + original = getattr(_os, name) + + def wrap(self, path, *args, **kw): if self._active: - path = self._remap_input(name,path,*args,**kw) - return self._remap_output(name, original(path,*args,**kw)) - return original(path,*args,**kw) + path = self._remap_input(name, path, *args, **kw) + return self._remap_output(name, original(path, *args, **kw)) + return original(path, *args, **kw) return wrap for name in ['readlink', 'tempnam']: - if hasattr(_os,name): locals()[name] = _mk_single_with_return(name) + if hasattr(_os, name): + locals()[name] = _mk_single_with_return(name) def _mk_query(name): - original = getattr(_os,name) - def wrap(self,*args,**kw): - retval = original(*args,**kw) + original = getattr(_os, name) + + def wrap(self, *args, **kw): + retval = original(*args, **kw) if self._active: return self._remap_output(name, retval) return retval return wrap for name in ['getcwd', 'tmpnam']: - if hasattr(_os,name): locals()[name] = _mk_query(name) + if hasattr(_os, name): + locals()[name] = _mk_query(name) - def _validate_path(self,path): + def _validate_path(self, path): """Called to remap or validate any path, whether input or output""" return path - def _remap_input(self,operation,path,*args,**kw): + def _remap_input(self, operation, path, *args, **kw): """Called for path inputs""" return self._validate_path(path) - def _remap_output(self,operation,path): + def _remap_output(self, operation, path): """Called for path outputs""" return self._validate_path(path) - def _remap_pair(self,operation,src,dst,*args,**kw): + def _remap_pair(self, operation, src, dst, *args, **kw): """Called for path pairs like rename, link, and symlink operations""" return ( - self._remap_input(operation+'-from',src,*args,**kw), - self._remap_input(operation+'-to',dst,*args,**kw) + self._remap_input(operation + '-from', src, *args, **kw), + self._remap_input(operation + '-to', dst, *args, **kw) ) @@ -364,6 +375,7 @@ except ImportError: # it appears pywin32 is not installed, so no need to exclude. pass + class DirectorySandbox(AbstractSandbox): """Restrict operations to a single subdirectory - pseudo-chroot""" @@ -380,7 +392,7 @@ class DirectorySandbox(AbstractSandbox): def __init__(self, sandbox, exceptions=_EXCEPTIONS): self._sandbox = os.path.normcase(os.path.realpath(sandbox)) - self._prefix = os.path.join(self._sandbox,'') + self._prefix = os.path.join(self._sandbox, '') self._exceptions = [ os.path.normcase(os.path.realpath(path)) for path in exceptions @@ -395,12 +407,12 @@ class DirectorySandbox(AbstractSandbox): def _file(self, path, mode='r', *args, **kw): if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path): self._violation("file", path, mode, *args, **kw) - return _file(path,mode,*args,**kw) + return _file(path, mode, *args, **kw) def _open(self, path, mode='r', *args, **kw): if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path): self._violation("open", path, mode, *args, **kw) - return _open(path,mode,*args,**kw) + return _open(path, mode, *args, **kw) def tmpnam(self): self._violation("tmpnam") @@ -440,19 +452,21 @@ class DirectorySandbox(AbstractSandbox): """Called for path pairs like rename, link, and symlink operations""" if not self._ok(src) or not self._ok(dst): self._violation(operation, src, dst, *args, **kw) - return (src,dst) + return (src, dst) def open(self, file, flags, mode=0o777, *args, **kw): """Called for low-level os.open()""" if flags & WRITE_FLAGS and not self._ok(file): self._violation("os.open", file, flags, mode, *args, **kw) - return _os.open(file,flags,mode, *args, **kw) + return _os.open(file, flags, mode, *args, **kw) + WRITE_FLAGS = functools.reduce( operator.or_, [getattr(_os, a, 0) for a in "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()] ) + class SandboxViolation(DistutilsError): """A setup script attempted to modify the filesystem outside the sandbox""" @@ -468,29 +482,4 @@ script by hand. Please inform the package's author and the EasyInstall maintainers to find out if a fix or workaround is available.""" % self.args - - - - - - - - - - - - - - - - - - - - - - - - - # diff --git a/setuptools/site-patch.py b/setuptools/site-patch.py index c2168019..f09ab522 100644 --- a/setuptools/site-patch.py +++ b/setuptools/site-patch.py @@ -2,18 +2,17 @@ def __boot(): import sys import os PYTHONPATH = os.environ.get('PYTHONPATH') - if PYTHONPATH is None or (sys.platform=='win32' and not PYTHONPATH): + if PYTHONPATH is None or (sys.platform == 'win32' and not PYTHONPATH): PYTHONPATH = [] else: PYTHONPATH = PYTHONPATH.split(os.pathsep) - pic = getattr(sys,'path_importer_cache',{}) + pic = getattr(sys, 'path_importer_cache', {}) stdpath = sys.path[len(PYTHONPATH):] mydir = os.path.dirname(__file__) - #print "searching",stdpath,sys.path for item in stdpath: - if item==mydir or not item: + if item == mydir or not item: continue # skip if current dir. on Windows, or my own directory importer = pic.get(item) if importer is not None: @@ -24,26 +23,24 @@ def __boot(): break else: try: - import imp # Avoid import loop in Python >= 3.3 - stream, path, descr = imp.find_module('site',[item]) + import imp # Avoid import loop in Python >= 3.3 + stream, path, descr = imp.find_module('site', [item]) except ImportError: continue if stream is None: continue try: # This should actually reload the current module - imp.load_module('site',stream,path,descr) + imp.load_module('site', stream, path, descr) finally: stream.close() break else: raise ImportError("Couldn't find the real 'site' module") - #print "loaded", __file__ + known_paths = dict([(makepath(item)[1], 1) for item in sys.path]) # 2.2 comp - known_paths = dict([(makepath(item)[1],1) for item in sys.path]) # 2.2 comp - - oldpos = getattr(sys,'__egginsert',0) # save old insertion position + oldpos = getattr(sys, '__egginsert', 0) # save old insertion position sys.__egginsert = 0 # and reset the current one for item in PYTHONPATH: @@ -58,7 +55,7 @@ def __boot(): for item in sys.path: p, np = makepath(item) - if np==nd and insert_at is None: + if np == nd and insert_at is None: # We've hit the first 'system' path entry, so added entries go here insert_at = len(new_path) @@ -71,6 +68,7 @@ def __boot(): sys.path[:] = new_path -if __name__=='site': + +if __name__ == 'site': __boot() del __boot diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py index 657197cf..f4ba8a92 100644 --- a/setuptools/ssl_support.py +++ b/setuptools/ssl_support.py @@ -161,6 +161,7 @@ class VerifyingHTTPSHandler(HTTPSHandler): class VerifyingHTTPSConn(HTTPSConnection): """Simple verifying connection: no auth, subclasses, timeouts, etc.""" + def __init__(self, host, ca_bundle, **kw): HTTPSConnection.__init__(self, host, **kw) self.ca_bundle = ca_bundle @@ -192,6 +193,7 @@ class VerifyingHTTPSConn(HTTPSConnection): self.sock.close() raise + def opener_for(ca_bundle=None): """Get a urlopen() replacement that uses ca_bundle for verification""" return urllib.request.build_opener( @@ -201,6 +203,7 @@ def opener_for(ca_bundle=None): _wincerts = None + def get_win_certfile(): global _wincerts if _wincerts is not None: @@ -212,6 +215,7 @@ def get_win_certfile(): return None class MyCertFile(CertFile): + def __init__(self, stores=(), certs=()): CertFile.__init__(self) for store in stores: @@ -231,7 +235,7 @@ def get_win_certfile(): def find_ca_bundle(): """Return an existing CA bundle path, or None""" - if os.name=='nt': + if os.name == 'nt': return get_win_certfile() else: for cert_path in cert_paths: diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index 32447356..53bd836c 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -27,7 +27,7 @@ def makeSetup(**args): distutils.core._setup_stop_after = "commandline" # Don't let system command line leak into tests! - args.setdefault('script_args',['install']) + args.setdefault('script_args', ['install']) try: return setuptools.setup(**args) @@ -40,6 +40,7 @@ needs_bytecode = pytest.mark.skipif( reason="bytecode support not available", ) + class TestDepends: def testExtractConst(self): @@ -55,35 +56,35 @@ class TestDepends: fc = six.get_function_code(f1) # unrecognized name - assert dep.extract_constant(fc,'q', -1) is None + assert dep.extract_constant(fc, 'q', -1) is None # constant assigned - dep.extract_constant(fc,'x', -1) == "test" + dep.extract_constant(fc, 'x', -1) == "test" # expression assigned - dep.extract_constant(fc,'y', -1) == -1 + dep.extract_constant(fc, 'y', -1) == -1 # recognized name, not assigned - dep.extract_constant(fc,'z', -1) is None + dep.extract_constant(fc, 'z', -1) is None def testFindModule(self): with pytest.raises(ImportError): dep.find_module('no-such.-thing') with pytest.raises(ImportError): dep.find_module('setuptools.non-existent') - f,p,i = dep.find_module('setuptools.tests') + f, p, i = dep.find_module('setuptools.tests') f.close() @needs_bytecode def testModuleExtract(self): from email import __version__ - assert dep.get_module_constant('email','__version__') == __version__ - assert dep.get_module_constant('sys','version') == sys.version - assert dep.get_module_constant('setuptools.tests','__doc__') == __doc__ + assert dep.get_module_constant('email', '__version__') == __version__ + assert dep.get_module_constant('sys', 'version') == sys.version + assert dep.get_module_constant('setuptools.tests', '__doc__') == __doc__ @needs_bytecode def testRequire(self): - req = Require('Email','1.0.3','email') + req = Require('Email', '1.0.3', 'email') assert req.name == 'Email' assert req.module == 'email' @@ -100,12 +101,12 @@ class TestDepends: assert req.is_present() assert req.is_current() - req = Require('Email 3000','03000','email',format=LooseVersion) + req = Require('Email 3000', '03000', 'email', format=LooseVersion) assert req.is_present() assert not req.is_current() assert not req.version_ok('unknown') - req = Require('Do-what-I-mean','1.0','d-w-i-m') + req = Require('Do-what-I-mean', '1.0', 'd-w-i-m') assert not req.is_present() assert not req.is_current() @@ -124,22 +125,22 @@ class TestDepends: class TestDistro: def setup_method(self, method): - self.e1 = Extension('bar.ext',['bar.c']) + self.e1 = Extension('bar.ext', ['bar.c']) self.e2 = Extension('c.y', ['y.c']) self.dist = makeSetup( packages=['a', 'a.b', 'a.b.c', 'b', 'c'], - py_modules=['b.d','x'], - ext_modules = (self.e1, self.e2), - package_dir = {}, + py_modules=['b.d', 'x'], + ext_modules=(self.e1, self.e2), + package_dir={}, ) def testDistroType(self): - assert isinstance(self.dist,setuptools.dist.Distribution) + assert isinstance(self.dist, setuptools.dist.Distribution) def testExcludePackage(self): self.dist.exclude_package('a') - assert self.dist.packages == ['b','c'] + assert self.dist.packages == ['b', 'c'] self.dist.exclude_package('b') assert self.dist.packages == ['c'] @@ -168,7 +169,7 @@ class TestDistro: assert self.dist.ext_modules == [self.e2, self.e1] def testExcludePackages(self): - self.dist.exclude(packages=['c','b','a']) + self.dist.exclude(packages=['c', 'b', 'a']) assert self.dist.packages == [] assert self.dist.py_modules == ['x'] assert self.dist.ext_modules == [self.e1] @@ -198,13 +199,13 @@ class TestDistro: with pytest.raises(DistutilsSetupError): self.dist.exclude(nonexistent_option='x') with pytest.raises(DistutilsSetupError): - self.dist.include(packages={'x':'y'}) + self.dist.include(packages={'x': 'y'}) with pytest.raises(DistutilsSetupError): - self.dist.exclude(packages={'x':'y'}) + self.dist.exclude(packages={'x': 'y'}) with pytest.raises(DistutilsSetupError): - self.dist.include(ext_modules={'x':'y'}) + self.dist.include(ext_modules={'x': 'y'}) with pytest.raises(DistutilsSetupError): - self.dist.exclude(ext_modules={'x':'y'}) + self.dist.exclude(ext_modules={'x': 'y'}) with pytest.raises(DistutilsSetupError): self.dist.include(package_dir=['q']) @@ -215,31 +216,31 @@ class TestDistro: class TestFeatures: def setup_method(self, method): - self.req = Require('Distutils','1.0.3','distutils') + self.req = Require('Distutils', '1.0.3', 'distutils') self.dist = makeSetup( features={ - 'foo': Feature("foo",standard=True,require_features=['baz',self.req]), - 'bar': Feature("bar", standard=True, packages=['pkg.bar'], + 'foo': Feature("foo", standard=True, require_features=['baz', self.req]), + 'bar': Feature("bar", standard=True, packages=['pkg.bar'], py_modules=['bar_et'], remove=['bar.ext'], - ), + ), 'baz': Feature( "baz", optional=False, packages=['pkg.baz'], - scripts = ['scripts/baz_it'], - libraries=[('libfoo','foo/foofoo.c')] + scripts=['scripts/baz_it'], + libraries=[('libfoo', 'foo/foofoo.c')] ), 'dwim': Feature("DWIM", available=False, remove='bazish'), }, script_args=['--without-bar', 'install'], - packages = ['pkg.bar', 'pkg.foo'], - py_modules = ['bar_et', 'bazish'], - ext_modules = [Extension('bar.ext',['bar.c'])] + packages=['pkg.bar', 'pkg.foo'], + py_modules=['bar_et', 'bazish'], + ext_modules=[Extension('bar.ext', ['bar.c'])] ) def testDefaults(self): assert not Feature( - "test",standard=True,remove='x',available=False + "test", standard=True, remove='x', available=False ).include_by_default() - assert Feature("test",standard=True,remove='x').include_by_default() + assert Feature("test", standard=True, remove='x').include_by_default() # Feature must have either kwargs, removes, or require_features with pytest.raises(DistutilsSetupError): Feature("test") @@ -251,32 +252,32 @@ class TestFeatures: def testFeatureOptions(self): dist = self.dist assert ( - ('with-dwim',None,'include DWIM') in dist.feature_options + ('with-dwim', None, 'include DWIM') in dist.feature_options ) assert ( - ('without-dwim',None,'exclude DWIM (default)') in dist.feature_options + ('without-dwim', None, 'exclude DWIM (default)') in dist.feature_options ) assert ( - ('with-bar',None,'include bar (default)') in dist.feature_options + ('with-bar', None, 'include bar (default)') in dist.feature_options ) assert ( - ('without-bar',None,'exclude bar') in dist.feature_options + ('without-bar', None, 'exclude bar') in dist.feature_options ) assert dist.feature_negopt['without-foo'] == 'with-foo' assert dist.feature_negopt['without-bar'] == 'with-bar' assert dist.feature_negopt['without-dwim'] == 'with-dwim' - assert (not 'without-baz' in dist.feature_negopt) + assert ('without-baz' not in dist.feature_negopt) def testUseFeatures(self): dist = self.dist assert dist.with_foo == 1 assert dist.with_bar == 0 assert dist.with_baz == 1 - assert (not 'bar_et' in dist.py_modules) - assert (not 'pkg.bar' in dist.packages) + assert ('bar_et' not in dist.py_modules) + assert ('pkg.bar' not in dist.packages) assert ('pkg.baz' in dist.packages) assert ('scripts/baz_it' in dist.scripts) - assert (('libfoo','foo/foofoo.c') in dist.libraries) + assert (('libfoo', 'foo/foofoo.c') in dist.libraries) assert dist.ext_modules == [] assert dist.require_features == [self.req] @@ -287,7 +288,8 @@ class TestFeatures: def testFeatureWithInvalidRemove(self): with pytest.raises(SystemExit): - makeSetup(features={'x':Feature('x', remove='y')}) + makeSetup(features={'x': Feature('x', remove='y')}) + class TestCommandTests: @@ -296,7 +298,7 @@ class TestCommandTests: assert (isinstance(test_cmd, distutils.cmd.Command)) def testLongOptSuiteWNoDefault(self): - ts1 = makeSetup(script_args=['test','--test-suite=foo.tests.suite']) + ts1 = makeSetup(script_args=['test', '--test-suite=foo.tests.suite']) ts1 = ts1.get_command_obj('test') ts1.ensure_finalized() assert ts1.test_suite == 'foo.tests.suite' @@ -309,7 +311,7 @@ class TestCommandTests: def testDefaultWModuleOnCmdLine(self): ts3 = makeSetup( test_suite='bar.tests', - script_args=['test','-m','foo.tests'] + script_args=['test', '-m', 'foo.tests'] ).get_command_obj('test') ts3.ensure_finalized() assert ts3.test_module == 'foo.tests' @@ -317,7 +319,7 @@ class TestCommandTests: def testConflictingOptions(self): ts4 = makeSetup( - script_args=['test','-m','bar.tests', '-s','foo.tests.suite'] + script_args=['test', '-m', 'bar.tests', '-s', 'foo.tests.suite'] ).get_command_obj('test') with pytest.raises(DistutilsOptionError): ts4.ensure_finalized() @@ -325,4 +327,4 @@ class TestCommandTests: def testNoSuite(self): ts5 = makeSetup().get_command_obj('test') ts5.ensure_finalized() - assert ts5.test_suite == None + assert ts5.test_suite is None diff --git a/setuptools/tests/contexts.py b/setuptools/tests/contexts.py index ae28c7c3..535ae107 100644 --- a/setuptools/tests/contexts.py +++ b/setuptools/tests/contexts.py @@ -10,7 +10,7 @@ import pkg_resources @contextlib.contextmanager -def tempdir(cd=lambda dir:None, **kwargs): +def tempdir(cd=lambda dir: None, **kwargs): temp_dir = tempfile.mkdtemp(**kwargs) orig_dir = os.getcwd() try: diff --git a/setuptools/tests/environment.py b/setuptools/tests/environment.py index a23c0504..b0e3bd36 100644 --- a/setuptools/tests/environment.py +++ b/setuptools/tests/environment.py @@ -25,11 +25,11 @@ def run_setup_py(cmd, pypath=None, path=None, for envname in os.environ: env[envname] = os.environ[envname] - #override the python path if needed + # override the python path if needed if pypath is not None: env["PYTHONPATH"] = pypath - #overide the execution path if needed + # overide the execution path if needed if path is not None: env["PATH"] = path if not env.get("PATH", ""): @@ -50,11 +50,11 @@ def run_setup_py(cmd, pypath=None, path=None, except OSError: return 1, '' - #decode the console string if needed - if hasattr(data, "decode"): + # decode the console string if needed + if hasattr(data, "decode"): # use the default encoding data = data.decode() data = unicodedata.normalize('NFC', data) - #communciate calls wait() + # communciate calls wait() return proc.returncode, data diff --git a/setuptools/tests/fixtures.py b/setuptools/tests/fixtures.py index c70c38cb..5204c8d1 100644 --- a/setuptools/tests/fixtures.py +++ b/setuptools/tests/fixtures.py @@ -1,24 +1,20 @@ -try: - from unittest import mock -except ImportError: - import mock import pytest from . import contexts @pytest.yield_fixture -def user_override(): +def user_override(monkeypatch): """ Override site.USER_BASE and site.USER_SITE with temporary directories in a context. """ with contexts.tempdir() as user_base: - with mock.patch('site.USER_BASE', user_base): - with contexts.tempdir() as user_site: - with mock.patch('site.USER_SITE', user_site): - with contexts.save_user_site_setting(): - yield + monkeypatch.setattr('site.USER_BASE', user_base) + with contexts.tempdir() as user_site: + monkeypatch.setattr('site.USER_SITE', user_site) + with contexts.save_user_site_setting(): + yield @pytest.yield_fixture diff --git a/setuptools/tests/py26compat.py b/setuptools/tests/py26compat.py index 7211f275..18cece05 100644 --- a/setuptools/tests/py26compat.py +++ b/setuptools/tests/py26compat.py @@ -2,12 +2,14 @@ import sys import tarfile import contextlib + def _tarfile_open_ex(*args, **kwargs): """ Extend result as a context manager. """ return contextlib.closing(tarfile.open(*args, **kwargs)) + if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 2): tarfile_open = _tarfile_open_ex else: diff --git a/setuptools/tests/server.py b/setuptools/tests/server.py index 6a687937..9e5fefb7 100644 --- a/setuptools/tests/server.py +++ b/setuptools/tests/server.py @@ -18,6 +18,7 @@ class IndexServer(BaseHTTPServer.HTTPServer): # The index files should be located in setuptools/tests/indexes s.stop() """ + def __init__(self, server_address=('', 0), RequestHandlerClass=SimpleHTTPServer.SimpleHTTPRequestHandler): BaseHTTPServer.HTTPServer.__init__(self, server_address, @@ -42,16 +43,20 @@ class IndexServer(BaseHTTPServer.HTTPServer): port = self.server_port return 'http://127.0.0.1:%s/setuptools/tests/indexes/' % port + class RequestRecorder(BaseHTTPServer.BaseHTTPRequestHandler): + def do_GET(self): requests = vars(self.server).setdefault('requests', []) requests.append(self) self.send_response(200, 'OK') + class MockServer(BaseHTTPServer.HTTPServer, threading.Thread): """ A simple HTTP Server that records the requests made to it. """ + def __init__(self, server_address=('', 0), RequestHandlerClass=RequestRecorder): BaseHTTPServer.HTTPServer.__init__(self, server_address, diff --git a/setuptools/tests/test_archive_util.py b/setuptools/tests/test_archive_util.py new file mode 100644 index 00000000..b789e9ac --- /dev/null +++ b/setuptools/tests/test_archive_util.py @@ -0,0 +1,42 @@ +# coding: utf-8 + +import tarfile +import io + +from setuptools.extern import six + +import pytest + +from setuptools import archive_util + + +@pytest.fixture +def tarfile_with_unicode(tmpdir): + """ + Create a tarfile containing only a file whose name is + a zero byte file called testimäge.png. + """ + tarobj = io.BytesIO() + + with tarfile.open(fileobj=tarobj, mode="w:gz") as tgz: + data = b"" + + filename = "testimäge.png" + if six.PY2: + filename = filename.decode('utf-8') + + t = tarfile.TarInfo(filename) + t.size = len(data) + + tgz.addfile(t, io.BytesIO(data)) + + target = tmpdir / 'unicode-pkg-1.0.tar.gz' + with open(str(target), mode='wb') as tf: + tf.write(tarobj.getvalue()) + return str(target) + + +@pytest.mark.xfail(reason="#710 and #712") +def test_unicode_files(tarfile_with_unicode, tmpdir): + target = tmpdir / 'out' + archive_util.unpack_archive(tarfile_with_unicode, six.text_type(target)) diff --git a/setuptools/tests/test_bdist_egg.py b/setuptools/tests/test_bdist_egg.py index ccfb2ea7..c77aa226 100644 --- a/setuptools/tests/test_bdist_egg.py +++ b/setuptools/tests/test_bdist_egg.py @@ -9,30 +9,33 @@ from setuptools.dist import Distribution from . import contexts + SETUP_PY = """\ from setuptools import setup setup(name='foo', py_modules=['hi']) """ + @pytest.yield_fixture def setup_context(tmpdir): - with (tmpdir/'setup.py').open('w') as f: + with (tmpdir / 'setup.py').open('w') as f: f.write(SETUP_PY) - with (tmpdir/'hi.py').open('w') as f: + with (tmpdir / 'hi.py').open('w') as f: f.write('1\n') with tmpdir.as_cwd(): yield tmpdir class Test: + def test_bdist_egg(self, setup_context, user_override): dist = Distribution(dict( script_name='setup.py', script_args=['bdist_egg'], name='foo', py_modules=['hi'] - )) + )) os.makedirs(os.path.join('build', 'src')) with contexts.quiet(): dist.parse_command_line() diff --git a/setuptools/tests/test_build_ext.py b/setuptools/tests/test_build_ext.py index 0719ba44..ac002f44 100644 --- a/setuptools/tests/test_build_ext.py +++ b/setuptools/tests/test_build_ext.py @@ -1,9 +1,16 @@ +import sys import distutils.command.build_ext as orig +from distutils.sysconfig import get_config_var -from setuptools.command.build_ext import build_ext +from setuptools.extern import six + +from setuptools.command.build_ext import build_ext, get_abi3_suffix from setuptools.dist import Distribution +from setuptools.extension import Extension + class TestBuildExt: + def test_get_ext_filename(self): """ Setuptools needs to give back the same @@ -16,3 +23,24 @@ class TestBuildExt: res = cmd.get_ext_filename('foo') wanted = orig.build_ext.get_ext_filename(cmd, 'foo') assert res == wanted + + def test_abi3_filename(self): + """ + Filename needs to be loadable by several versions + of Python 3 if 'is_abi3' is truthy on Extension() + """ + print(get_abi3_suffix()) + + extension = Extension('spam.eggs', ['eggs.c'], py_limited_api=True) + dist = Distribution(dict(ext_modules=[extension])) + cmd = build_ext(dist) + cmd.finalize_options() + assert 'spam.eggs' in cmd.ext_map + res = cmd.get_ext_filename('spam.eggs') + + if six.PY2 or not get_abi3_suffix(): + assert res.endswith(get_config_var('SO')) + elif sys.platform == 'win32': + assert res.endswith('eggs.pyd') + else: + assert 'abi3' in res diff --git a/setuptools/tests/test_build_py.py b/setuptools/tests/test_build_py.py index ed1703ac..cc701ae6 100644 --- a/setuptools/tests/test_build_py.py +++ b/setuptools/tests/test_build_py.py @@ -26,6 +26,5 @@ def test_directories_in_package_data_glob(tmpdir_as_cwd): package_data={'': ['path/*']}, )) os.makedirs('path/subpath') - #with contexts.quiet(): dist.parse_command_line() dist.run_commands() diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index 1b844499..f1580785 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -26,6 +26,7 @@ setup(name='foo', INIT_PY = """print "foo" """ + @pytest.yield_fixture def temp_user(monkeypatch): with contexts.tempdir() as user_base: @@ -54,6 +55,7 @@ def test_env(tmpdir, temp_user): class TestDevelop: in_virtualenv = hasattr(sys, 'real_prefix') in_venv = hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix + @pytest.mark.skipif(in_virtualenv or in_venv, reason="Cannot run when invoked in a virtualenv or venv") def test_2to3_user_mode(self, test_env): @@ -112,4 +114,4 @@ class TestDevelop: cmd.ensure_finalized() cmd.install_dir = tmpdir cmd.run() - #assert '0.0' not in foocmd_text + # assert '0.0' not in foocmd_text diff --git a/setuptools/tests/test_dist_info.py b/setuptools/tests/test_dist_info.py index 9f226a55..f7e7d2bf 100644 --- a/setuptools/tests/test_dist_info.py +++ b/setuptools/tests/test_dist_info.py @@ -1,8 +1,7 @@ """Test .dist-info style distributions. """ -import os -import shutil -import tempfile + +from __future__ import unicode_literals from setuptools.extern.six.moves import map @@ -14,10 +13,48 @@ from .textwrap import DALS class TestDistInfo: - def test_distinfo(self): + metadata_base = DALS(""" + Metadata-Version: 1.2 + Requires-Dist: splort (==4) + Provides-Extra: baz + Requires-Dist: quux (>=1.1); extra == 'baz' + """) + + @classmethod + def build_metadata(cls, **kwargs): + lines = ( + '{key}: {value}\n'.format(**locals()) + for key, value in kwargs.items() + ) + return cls.metadata_base + ''.join(lines) + + @pytest.fixture + def metadata(self, tmpdir): + dist_info_name = 'VersionedDistribution-2.718.dist-info' + versioned = tmpdir / dist_info_name + versioned.mkdir() + filename = versioned / 'METADATA' + content = self.build_metadata( + Name='VersionedDistribution', + ) + filename.write_text(content, encoding='utf-8') + + dist_info_name = 'UnversionedDistribution.dist-info' + unversioned = tmpdir / dist_info_name + unversioned.mkdir() + filename = unversioned / 'METADATA' + content = self.build_metadata( + Name='UnversionedDistribution', + Version='0.3', + ) + filename.write_text(content, encoding='utf-8') + + return str(tmpdir) + + def test_distinfo(self, metadata): dists = dict( (d.project_name, d) - for d in pkg_resources.find_distributions(self.tmpdir) + for d in pkg_resources.find_distributions(metadata) ) assert len(dists) == 2, dists @@ -25,49 +62,17 @@ class TestDistInfo: unversioned = dists['UnversionedDistribution'] versioned = dists['VersionedDistribution'] - assert versioned.version == '2.718' # from filename - assert unversioned.version == '0.3' # from METADATA + assert versioned.version == '2.718' # from filename + assert unversioned.version == '0.3' # from METADATA - def test_conditional_dependencies(self): + def test_conditional_dependencies(self, metadata): specs = 'splort==4', 'quux>=1.1' requires = list(map(pkg_resources.Requirement.parse, specs)) - for d in pkg_resources.find_distributions(self.tmpdir): + for d in pkg_resources.find_distributions(metadata): assert d.requires() == requires[:1] assert d.requires(extras=('baz',)) == [ requires[0], - pkg_resources.Requirement.parse('quux>=1.1;extra=="baz"')] + pkg_resources.Requirement.parse('quux>=1.1;extra=="baz"'), + ] assert d.extras == ['baz'] - - metadata_template = DALS(""" - Metadata-Version: 1.2 - Name: {name} - {version} - Requires-Dist: splort (==4) - Provides-Extra: baz - Requires-Dist: quux (>=1.1); extra == 'baz' - """) - - def setup_method(self, method): - self.tmpdir = tempfile.mkdtemp() - dist_info_name = 'VersionedDistribution-2.718.dist-info' - versioned = os.path.join(self.tmpdir, dist_info_name) - os.mkdir(versioned) - with open(os.path.join(versioned, 'METADATA'), 'w+') as metadata_file: - metadata = self.metadata_template.format( - name='VersionedDistribution', - version='', - ).replace('\n\n', '\n') - metadata_file.write(metadata) - dist_info_name = 'UnversionedDistribution.dist-info' - unversioned = os.path.join(self.tmpdir, dist_info_name) - os.mkdir(unversioned) - with open(os.path.join(unversioned, 'METADATA'), 'w+') as metadata_file: - metadata = self.metadata_template.format( - name='UnversionedDistribution', - version='Version: 0.3', - ) - metadata_file.write(metadata) - - def teardown_method(self, method): - shutil.rmtree(self.tmpdir) diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index fd06b6ef..82e1d7e8 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -14,9 +14,10 @@ import logging import itertools import distutils.errors import io +import zipfile -from setuptools.extern.six.moves import urllib import time +from setuptools.extern.six.moves import urllib import pytest try: @@ -30,7 +31,7 @@ import setuptools.command.easy_install as ei from setuptools.command.easy_install import PthDistributions from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution -from pkg_resources import working_set +from pkg_resources import normalize_path, working_set from pkg_resources import Distribution as PRDistribution import setuptools.tests.server import pkg_resources @@ -41,6 +42,7 @@ from .textwrap import DALS class FakeDist(object): + def get_entry_map(self, group): if group != 'console_scripts': return {} @@ -49,12 +51,14 @@ class FakeDist(object): def as_requirement(self): return 'spec' + SETUP_PY = DALS(""" from setuptools import setup setup(name='foo') """) + class TestEasyInstallTest: def test_install_site_py(self, tmpdir): @@ -70,10 +74,12 @@ class TestEasyInstallTest: expected = header + DALS(""" # EASY-INSTALL-ENTRY-SCRIPT: 'spec','console_scripts','name' __requires__ = 'spec' + import re import sys from pkg_resources import load_entry_point if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) sys.exit( load_entry_point('spec', 'console_scripts', 'name')() ) @@ -123,16 +129,72 @@ class TestEasyInstallTest: get_site_dirs should always return site dirs reported by site.getsitepackages. """ - mock_gsp = lambda: ['/setuptools/test/site-packages'] + path = normalize_path('/setuptools/test/site-packages') + mock_gsp = lambda: [path] monkeypatch.setattr(site, 'getsitepackages', mock_gsp, raising=False) - assert '/setuptools/test/site-packages' in ei.get_site_dirs() + assert path in ei.get_site_dirs() def test_all_site_dirs_works_without_getsitepackages(self, monkeypatch): monkeypatch.delattr(site, 'getsitepackages', raising=False) assert ei.get_site_dirs() + @pytest.fixture + def sdist_unicode(self, tmpdir): + files = [ + ( + 'setup.py', + DALS(""" + import setuptools + setuptools.setup( + name="setuptools-test-unicode", + version="1.0", + packages=["mypkg"], + include_package_data=True, + ) + """), + ), + ( + 'mypkg/__init__.py', + "", + ), + ( + u'mypkg/\u2603.txt', + "", + ), + ] + sdist_name = 'setuptools-test-unicode-1.0.zip' + sdist = tmpdir / sdist_name + # can't use make_sdist, because the issue only occurs + # with zip sdists. + sdist_zip = zipfile.ZipFile(str(sdist), 'w') + for filename, content in files: + sdist_zip.writestr(filename, content) + sdist_zip.close() + return str(sdist) + + @pytest.mark.xfail(reason="#709 and #710") + # also + #@pytest.mark.xfail(setuptools.tests.is_ascii, + # reason="https://github.com/pypa/setuptools/issues/706") + def test_unicode_filename_in_sdist(self, sdist_unicode, tmpdir, monkeypatch): + """ + The install command should execute correctly even if + the package has unicode filenames. + """ + dist = Distribution({'script_args': ['easy_install']}) + target = (tmpdir / 'target').ensure_dir() + cmd = ei.easy_install( + dist, + install_dir=str(target), + args=['x'], + ) + monkeypatch.setitem(os.environ, 'PYTHONPATH', str(target)) + cmd.ensure_finalized() + cmd.easy_install(sdist_unicode) + class TestPTHFileWriter: + def test_add_from_cwd_site_sets_dirty(self): '''a pth file manager should set dirty if a distribution is in site but also the cwd @@ -154,7 +216,7 @@ class TestPTHFileWriter: @pytest.yield_fixture def setup_context(tmpdir): - with (tmpdir/'setup.py').open('w') as f: + with (tmpdir / 'setup.py').open('w') as f: f.write(SETUP_PY) with tmpdir.as_cwd(): yield tmpdir @@ -266,6 +328,7 @@ def distutils_package(): class TestDistutilsPackage: + def test_bdist_egg_available_on_distutils_pkg(self, distutils_package): run_setup('setup.py', ['bdist_egg']) @@ -527,36 +590,40 @@ def make_trivial_sdist(dist_path, setup_py): dist.addfile(setup_py_file, fileobj=setup_py_bytes) +@pytest.mark.skipif( + sys.platform.startswith('java') and ei.is_sh(sys.executable), + reason="Test cannot run under java when executable is sh" +) class TestScriptHeader: non_ascii_exe = '/Users/José/bin/python' exe_with_spaces = r'C:\Program Files\Python33\python.exe' - @pytest.mark.skipif( - sys.platform.startswith('java') and ei.is_sh(sys.executable), - reason="Test cannot run under java when executable is sh" - ) def test_get_script_header(self): expected = '#!%s\n' % ei.nt_quote_arg(os.path.normpath(sys.executable)) actual = ei.ScriptWriter.get_script_header('#!/usr/local/bin/python') assert actual == expected + def test_get_script_header_args(self): expected = '#!%s -x\n' % ei.nt_quote_arg(os.path.normpath (sys.executable)) actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python -x') assert actual == expected + def test_get_script_header_non_ascii_exe(self): actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', executable=self.non_ascii_exe) expected = '#!%s -x\n' % self.non_ascii_exe assert actual == expected + def test_get_script_header_exe_with_spaces(self): actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', - executable='"'+self.exe_with_spaces+'"') + executable='"' + self.exe_with_spaces + '"') expected = '#!"%s"\n' % self.exe_with_spaces assert actual == expected class TestCommandSpec: + def test_custom_launch_command(self): """ Show how a custom CommandSpec could be used to specify a #! executable @@ -590,17 +657,9 @@ class TestCommandSpec: assert len(cmd) == 2 assert '"' not in cmd.as_header() - def test_sys_executable(self): - """ - CommandSpec.from_string(sys.executable) should contain just that param. - """ - writer = ei.ScriptWriter.best() - cmd = writer.command_spec_class.from_string(sys.executable) - assert len(cmd) == 1 - assert cmd[0] == sys.executable - class TestWindowsScriptWriter: + def test_header(self): hdr = ei.WindowsScriptWriter.get_script_header('') assert hdr.startswith('#!') diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index 3a0db58f..dff2a8c8 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -179,7 +179,7 @@ class TestEggInfo(object): """ % requires_line) build_files({ 'setup.py': setup_script, - }) + }) def test_install_requires_with_markers(self, tmpdir_cwd, env): self._setup_script_with_requires( @@ -210,6 +210,32 @@ class TestEggInfo(object): self._run_install_command(tmpdir_cwd, env) assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + def test_python_requires_egg_info(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """python_requires='>=2.7.12',""") + environ = os.environ.copy().update( + HOME=env.paths['home'], + ) + code, data = environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + env=environ, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + assert 'Requires-Python: >=2.7.12' in pkg_info_lines + assert 'Metadata-Version: 1.2' in pkg_info_lines + + def test_python_requires_install(self, tmpdir_cwd, env): + self._setup_script_with_requires( + """python_requires='>=1.2.3',""") + self._run_install_command(tmpdir_cwd, env) + egg_info_dir = self._find_egg_info_files(env.paths['lib']).base + pkginfo = os.path.join(egg_info_dir, 'PKG-INFO') + assert 'Requires-Python: >=1.2.3' in open(pkginfo).read().split('\n') + def _run_install_command(self, tmpdir_cwd, env, cmd=None, output=None): environ = os.environ.copy().update( HOME=env.paths['home'], @@ -235,6 +261,7 @@ class TestEggInfo(object): def _find_egg_info_files(self, root): class DirList(list): + def __init__(self, files, base): super(DirList, self).__init__(files) self.base = base diff --git a/setuptools/tests/test_find_packages.py b/setuptools/tests/test_find_packages.py index 06a7c02e..9d31ccd7 100644 --- a/setuptools/tests/test_find_packages.py +++ b/setuptools/tests/test_find_packages.py @@ -13,6 +13,8 @@ from setuptools import find_packages find_420_packages = setuptools.PEP420PackageFinder.find # modeled after CPython's test.support.can_symlink + + def can_symlink(): TESTFN = tempfile.mktemp() symlink_path = TESTFN + "can_symlink" @@ -26,13 +28,15 @@ def can_symlink(): globals().update(can_symlink=lambda: can) return can + def has_symlink(): bad_symlink = ( # Windows symlink directory detection is broken on Python 3.2 - platform.system() == 'Windows' and sys.version_info[:2] == (3,2) + platform.system() == 'Windows' and sys.version_info[:2] == (3, 2) ) return can_symlink() and not bad_symlink + class TestFindPackages: def setup_method(self, method): @@ -94,6 +98,15 @@ class TestFindPackages: packages = find_packages(self.dist_dir, exclude=('pkg.*',)) assert packages == ['pkg'] + def test_exclude_recursive(self): + """ + Excluding a parent package should exclude all child packages as well. + """ + self._touch('__init__.py', self.pkg_dir) + self._touch('__init__.py', self.sub_pkg_dir) + packages = find_packages(self.dist_dir, exclude=('pkg',)) + assert packages == [] + def test_include_excludes_other(self): """ If include is specified, other packages should be excluded. diff --git a/setuptools/tests/test_install_scripts.py b/setuptools/tests/test_install_scripts.py new file mode 100644 index 00000000..7393241f --- /dev/null +++ b/setuptools/tests/test_install_scripts.py @@ -0,0 +1,88 @@ +"""install_scripts tests +""" + +import io +import sys + +import pytest + +from setuptools.command.install_scripts import install_scripts +from setuptools.dist import Distribution +from . import contexts + + +class TestInstallScripts: + settings = dict( + name='foo', + entry_points={'console_scripts': ['foo=foo:foo']}, + version='0.0', + ) + unix_exe = '/usr/dummy-test-path/local/bin/python' + unix_spaces_exe = '/usr/bin/env dummy-test-python' + win32_exe = 'C:\\Dummy Test Path\\Program Files\\Python 3.3\\python.exe' + + def _run_install_scripts(self, install_dir, executable=None): + dist = Distribution(self.settings) + dist.script_name = 'setup.py' + cmd = install_scripts(dist) + cmd.install_dir = install_dir + if executable is not None: + bs = cmd.get_finalized_command('build_scripts') + bs.executable = executable + cmd.ensure_finalized() + with contexts.quiet(): + cmd.run() + + @pytest.mark.skipif(sys.platform == 'win32', reason='non-Windows only') + def test_sys_executable_escaping_unix(self, tmpdir, monkeypatch): + """ + Ensure that shebang is not quoted on Unix when getting the Python exe + from sys.executable. + """ + expected = '#!%s\n' % self.unix_exe + monkeypatch.setattr('sys.executable', self.unix_exe) + with tmpdir.as_cwd(): + self._run_install_scripts(str(tmpdir)) + with io.open(str(tmpdir.join('foo')), 'r') as f: + actual = f.readline() + assert actual == expected + + @pytest.mark.skipif(sys.platform != 'win32', reason='Windows only') + def test_sys_executable_escaping_win32(self, tmpdir, monkeypatch): + """ + Ensure that shebang is quoted on Windows when getting the Python exe + from sys.executable and it contains a space. + """ + expected = '#!"%s"\n' % self.win32_exe + monkeypatch.setattr('sys.executable', self.win32_exe) + with tmpdir.as_cwd(): + self._run_install_scripts(str(tmpdir)) + with io.open(str(tmpdir.join('foo-script.py')), 'r') as f: + actual = f.readline() + assert actual == expected + + @pytest.mark.skipif(sys.platform == 'win32', reason='non-Windows only') + def test_executable_with_spaces_escaping_unix(self, tmpdir): + """ + Ensure that shebang on Unix is not quoted, even when a value with spaces + is specified using --executable. + """ + expected = '#!%s\n' % self.unix_spaces_exe + with tmpdir.as_cwd(): + self._run_install_scripts(str(tmpdir), self.unix_spaces_exe) + with io.open(str(tmpdir.join('foo')), 'r') as f: + actual = f.readline() + assert actual == expected + + @pytest.mark.skipif(sys.platform != 'win32', reason='Windows only') + def test_executable_arg_escaping_win32(self, tmpdir): + """ + Ensure that shebang on Windows is quoted when getting a path with spaces + from --executable, that is itself properly quoted. + """ + expected = '#!"%s"\n' % self.win32_exe + with tmpdir.as_cwd(): + self._run_install_scripts(str(tmpdir), '"' + self.win32_exe + '"') + with io.open(str(tmpdir.join('foo-script.py')), 'r') as f: + actual = f.readline() + assert actual == expected diff --git a/setuptools/tests/test_manifest.py b/setuptools/tests/test_manifest.py new file mode 100644 index 00000000..6360270d --- /dev/null +++ b/setuptools/tests/test_manifest.py @@ -0,0 +1,476 @@ +# -*- coding: utf-8 -*- +"""sdist tests""" + +import contextlib +import os +import shutil +import sys +import tempfile +from distutils import log +from distutils.errors import DistutilsTemplateError + +from setuptools.command.egg_info import FileList, egg_info +from setuptools.dist import Distribution +from setuptools.extern import six +from setuptools.tests.textwrap import DALS + +import pytest + +py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only") + + +def make_local_path(s): + """Converts '/' in a string to os.sep""" + return s.replace('/', os.sep) + + +SETUP_ATTRS = { + 'name': 'app', + 'version': '0.0', + 'packages': ['app'], +} + + +SETUP_PY = """\ +from setuptools import setup + +setup(**%r) +""" % SETUP_ATTRS + + +@contextlib.contextmanager +def quiet(): + old_stdout, old_stderr = sys.stdout, sys.stderr + sys.stdout, sys.stderr = six.StringIO(), six.StringIO() + try: + yield + finally: + sys.stdout, sys.stderr = old_stdout, old_stderr + + +def touch(filename): + open(filename, 'w').close() + + +# The set of files always in the manifest, including all files in the +# .egg-info directory +default_files = frozenset(map(make_local_path, [ + 'README.rst', + 'MANIFEST.in', + 'setup.py', + 'app.egg-info/PKG-INFO', + 'app.egg-info/SOURCES.txt', + 'app.egg-info/dependency_links.txt', + 'app.egg-info/top_level.txt', + 'app/__init__.py', +])) + + +class TempDirTestCase(object): + + def setup_method(self, method): + self.temp_dir = tempfile.mkdtemp() + self.old_cwd = os.getcwd() + os.chdir(self.temp_dir) + + def teardown_method(self, method): + os.chdir(self.old_cwd) + shutil.rmtree(self.temp_dir) + + +class TestManifestTest(TempDirTestCase): + + def setup_method(self, method): + super(TestManifestTest, self).setup_method(method) + + f = open(os.path.join(self.temp_dir, 'setup.py'), 'w') + f.write(SETUP_PY) + f.close() + + """ + Create a file tree like: + - LICENSE + - README.rst + - testing.rst + - .hidden.rst + - app/ + - __init__.py + - a.txt + - b.txt + - c.rst + - static/ + - app.js + - app.js.map + - app.css + - app.css.map + """ + + for fname in ['README.rst', '.hidden.rst', 'testing.rst', 'LICENSE']: + touch(os.path.join(self.temp_dir, fname)) + + # Set up the rest of the test package + test_pkg = os.path.join(self.temp_dir, 'app') + os.mkdir(test_pkg) + for fname in ['__init__.py', 'a.txt', 'b.txt', 'c.rst']: + touch(os.path.join(test_pkg, fname)) + + # Some compiled front-end assets to include + static = os.path.join(test_pkg, 'static') + os.mkdir(static) + for fname in ['app.js', 'app.js.map', 'app.css', 'app.css.map']: + touch(os.path.join(static, fname)) + + def make_manifest(self, contents): + """Write a MANIFEST.in.""" + with open(os.path.join(self.temp_dir, 'MANIFEST.in'), 'w') as f: + f.write(DALS(contents)) + + def get_files(self): + """Run egg_info and get all the files to include, as a set""" + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'setup.py' + cmd = egg_info(dist) + cmd.ensure_finalized() + + cmd.run() + + return set(cmd.filelist.files) + + def test_no_manifest(self): + """Check a missing MANIFEST.in includes only the standard files.""" + assert (default_files - set(['MANIFEST.in'])) == self.get_files() + + def test_empty_files(self): + """Check an empty MANIFEST.in includes only the standard files.""" + self.make_manifest("") + assert default_files == self.get_files() + + def test_include(self): + """Include extra rst files in the project root.""" + self.make_manifest("include *.rst") + files = default_files | set([ + 'testing.rst', '.hidden.rst']) + assert files == self.get_files() + + def test_exclude(self): + """Include everything in app/ except the text files""" + l = make_local_path + self.make_manifest( + """ + include app/* + exclude app/*.txt + """) + files = default_files | set([l('app/c.rst')]) + assert files == self.get_files() + + def test_include_multiple(self): + """Include with multiple patterns.""" + l = make_local_path + self.make_manifest("include app/*.txt app/static/*") + files = default_files | set([ + l('app/a.txt'), l('app/b.txt'), + l('app/static/app.js'), l('app/static/app.js.map'), + l('app/static/app.css'), l('app/static/app.css.map')]) + assert files == self.get_files() + + def test_graft(self): + """Include the whole app/static/ directory.""" + l = make_local_path + self.make_manifest("graft app/static") + files = default_files | set([ + l('app/static/app.js'), l('app/static/app.js.map'), + l('app/static/app.css'), l('app/static/app.css.map')]) + assert files == self.get_files() + + def test_graft_global_exclude(self): + """Exclude all *.map files in the project.""" + l = make_local_path + self.make_manifest( + """ + graft app/static + global-exclude *.map + """) + files = default_files | set([ + l('app/static/app.js'), l('app/static/app.css')]) + assert files == self.get_files() + + def test_global_include(self): + """Include all *.rst, *.js, and *.css files in the whole tree.""" + l = make_local_path + self.make_manifest( + """ + global-include *.rst *.js *.css + """) + files = default_files | set([ + '.hidden.rst', 'testing.rst', l('app/c.rst'), + l('app/static/app.js'), l('app/static/app.css')]) + assert files == self.get_files() + + def test_graft_prune(self): + """Include all files in app/, except for the whole app/static/ dir.""" + l = make_local_path + self.make_manifest( + """ + graft app + prune app/static + """) + files = default_files | set([ + l('app/a.txt'), l('app/b.txt'), l('app/c.rst')]) + assert files == self.get_files() + + +class TestFileListTest(TempDirTestCase): + """ + A copy of the relevant bits of distutils/tests/test_filelist.py, + to ensure setuptools' version of FileList keeps parity with distutils. + """ + + def setup_method(self, method): + super(TestFileListTest, self).setup_method(method) + self.threshold = log.set_threshold(log.FATAL) + self._old_log = log.Log._log + log.Log._log = self._log + self.logs = [] + + def teardown_method(self, method): + log.set_threshold(self.threshold) + log.Log._log = self._old_log + super(TestFileListTest, self).teardown_method(method) + + def _log(self, level, msg, args): + if level not in (log.DEBUG, log.INFO, log.WARN, log.ERROR, log.FATAL): + raise ValueError('%s wrong log level' % str(level)) + self.logs.append((level, msg, args)) + + def get_logs(self, *levels): + def _format(msg, args): + if len(args) == 0: + return msg + return msg % args + return [_format(msg, args) for level, msg, args + in self.logs if level in levels] + + def clear_logs(self): + self.logs = [] + + def assertNoWarnings(self): + assert self.get_logs(log.WARN) == [] + self.clear_logs() + + def assertWarnings(self): + assert len(self.get_logs(log.WARN)) > 0 + self.clear_logs() + + def make_files(self, files): + for file in files: + file = os.path.join(self.temp_dir, file) + dirname, basename = os.path.split(file) + if not os.path.exists(dirname): + os.makedirs(dirname) + open(file, 'w').close() + + def test_process_template_line(self): + # testing all MANIFEST.in template patterns + file_list = FileList() + l = make_local_path + + # simulated file list + self.make_files([ + 'foo.tmp', 'ok', 'xo', 'four.txt', + 'buildout.cfg', + # filelist does not filter out VCS directories, + # it's sdist that does + l('.hg/last-message.txt'), + l('global/one.txt'), + l('global/two.txt'), + l('global/files.x'), + l('global/here.tmp'), + l('f/o/f.oo'), + l('dir/graft-one'), + l('dir/dir2/graft2'), + l('dir3/ok'), + l('dir3/sub/ok.txt'), + ]) + + MANIFEST_IN = DALS("""\ + include ok + include xo + exclude xo + include foo.tmp + include buildout.cfg + global-include *.x + global-include *.txt + global-exclude *.tmp + recursive-include f *.oo + recursive-exclude global *.x + graft dir + prune dir3 + """) + + for line in MANIFEST_IN.split('\n'): + if not line: + continue + file_list.process_template_line(line) + + wanted = [ + 'buildout.cfg', + 'four.txt', + 'ok', + l('.hg/last-message.txt'), + l('dir/graft-one'), + l('dir/dir2/graft2'), + l('f/o/f.oo'), + l('global/one.txt'), + l('global/two.txt'), + ] + file_list.sort() + + assert file_list.files == wanted + + def test_exclude_pattern(self): + # return False if no match + file_list = FileList() + assert not file_list.exclude_pattern('*.py') + + # return True if files match + file_list = FileList() + file_list.files = ['a.py', 'b.py'] + assert file_list.exclude_pattern('*.py') + + # test excludes + file_list = FileList() + file_list.files = ['a.py', 'a.txt'] + file_list.exclude_pattern('*.py') + assert file_list.files == ['a.txt'] + + def test_include_pattern(self): + # return False if no match + file_list = FileList() + file_list.set_allfiles([]) + assert not file_list.include_pattern('*.py') + + # return True if files match + file_list = FileList() + file_list.set_allfiles(['a.py', 'b.txt']) + assert file_list.include_pattern('*.py') + + # test * matches all files + file_list = FileList() + assert file_list.allfiles is None + file_list.set_allfiles(['a.py', 'b.txt']) + file_list.include_pattern('*') + assert file_list.allfiles == ['a.py', 'b.txt'] + + def test_process_template(self): + l = make_local_path + # invalid lines + file_list = FileList() + for action in ('include', 'exclude', 'global-include', + 'global-exclude', 'recursive-include', + 'recursive-exclude', 'graft', 'prune', 'blarg'): + try: + file_list.process_template_line(action) + except DistutilsTemplateError: + pass + except Exception: + assert False, "Incorrect error thrown" + else: + assert False, "Should have thrown an error" + + # include + file_list = FileList() + file_list.set_allfiles(['a.py', 'b.txt', l('d/c.py')]) + + file_list.process_template_line('include *.py') + assert file_list.files == ['a.py'] + self.assertNoWarnings() + + file_list.process_template_line('include *.rb') + assert file_list.files == ['a.py'] + self.assertWarnings() + + # exclude + file_list = FileList() + file_list.files = ['a.py', 'b.txt', l('d/c.py')] + + file_list.process_template_line('exclude *.py') + assert file_list.files == ['b.txt', l('d/c.py')] + self.assertNoWarnings() + + file_list.process_template_line('exclude *.rb') + assert file_list.files == ['b.txt', l('d/c.py')] + self.assertWarnings() + + # global-include + file_list = FileList() + file_list.set_allfiles(['a.py', 'b.txt', l('d/c.py')]) + + file_list.process_template_line('global-include *.py') + assert file_list.files == ['a.py', l('d/c.py')] + self.assertNoWarnings() + + file_list.process_template_line('global-include *.rb') + assert file_list.files == ['a.py', l('d/c.py')] + self.assertWarnings() + + # global-exclude + file_list = FileList() + file_list.files = ['a.py', 'b.txt', l('d/c.py')] + + file_list.process_template_line('global-exclude *.py') + assert file_list.files == ['b.txt'] + self.assertNoWarnings() + + file_list.process_template_line('global-exclude *.rb') + assert file_list.files == ['b.txt'] + self.assertWarnings() + + # recursive-include + file_list = FileList() + file_list.set_allfiles(['a.py', l('d/b.py'), l('d/c.txt'), + l('d/d/e.py')]) + + file_list.process_template_line('recursive-include d *.py') + assert file_list.files == [l('d/b.py'), l('d/d/e.py')] + self.assertNoWarnings() + + file_list.process_template_line('recursive-include e *.py') + assert file_list.files == [l('d/b.py'), l('d/d/e.py')] + self.assertWarnings() + + # recursive-exclude + file_list = FileList() + file_list.files = ['a.py', l('d/b.py'), l('d/c.txt'), l('d/d/e.py')] + + file_list.process_template_line('recursive-exclude d *.py') + assert file_list.files == ['a.py', l('d/c.txt')] + self.assertNoWarnings() + + file_list.process_template_line('recursive-exclude e *.py') + assert file_list.files == ['a.py', l('d/c.txt')] + self.assertWarnings() + + # graft + file_list = FileList() + file_list.set_allfiles(['a.py', l('d/b.py'), l('d/d/e.py'), + l('f/f.py')]) + + file_list.process_template_line('graft d') + assert file_list.files == [l('d/b.py'), l('d/d/e.py')] + self.assertNoWarnings() + + file_list.process_template_line('graft e') + assert file_list.files == [l('d/b.py'), l('d/d/e.py')] + self.assertWarnings() + + # prune + file_list = FileList() + file_list.files = ['a.py', l('d/b.py'), l('d/d/e.py'), l('f/f.py')] + + file_list.process_template_line('prune d') + assert file_list.files == ['a.py', l('f/f.py')] + self.assertNoWarnings() + + file_list.process_template_line('prune e') + assert file_list.files == ['a.py', l('f/f.py')] + self.assertWarnings() diff --git a/setuptools/tests/test_msvc9compiler.py b/setuptools/tests/test_msvc.py index 09e0460c..a0c76ea0 100644 --- a/setuptools/tests/test_msvc9compiler.py +++ b/setuptools/tests/test_msvc.py @@ -1,5 +1,5 @@ """ -Tests for msvc9compiler. +Tests for msvc support module. """ import os @@ -69,9 +69,9 @@ class TestModulePatch: def test_patched(self): "Test the module is actually patched" mod_name = distutils.msvc9compiler.find_vcvarsall.__module__ - assert mod_name == "setuptools.msvc9_support", "find_vcvarsall unpatched" + assert mod_name == "setuptools.msvc", "find_vcvarsall unpatched" - def test_no_registry_entryies_means_nothing_found(self): + def test_no_registry_entries_means_nothing_found(self): """ No registry entries or environment variable should lead to an error directing the user to download vcpython27. @@ -83,10 +83,12 @@ class TestModulePatch: with mock_reg(): assert find_vcvarsall(9.0) is None - expected = distutils.errors.DistutilsPlatformError - with pytest.raises(expected) as exc: + try: query_vcvarsall(9.0) - assert 'aka.ms/vcpython27' in str(exc) + except Exception as exc: + expected = distutils.errors.DistutilsPlatformError + assert isinstance(exc, expected) + assert 'aka.ms/vcpython27' in str(exc) @pytest.yield_fixture def user_preferred_setting(self): diff --git a/setuptools/tests/test_packageindex.py b/setuptools/tests/test_packageindex.py index 6a76b5fc..f9bf895b 100644 --- a/setuptools/tests/test_packageindex.py +++ b/setuptools/tests/test_packageindex.py @@ -7,14 +7,24 @@ import distutils.errors from setuptools.extern import six from setuptools.extern.six.moves import urllib, http_client -from .textwrap import DALS import pkg_resources import setuptools.package_index from setuptools.tests.server import IndexServer +from .textwrap import DALS class TestPackageIndex: + def test_regex(self): + hash_url = 'http://other_url?:action=show_md5&' + hash_url += 'digest=0123456789abcdef0123456789abcdef' + doc = """ + <a href="http://some_url">Name</a> + (<a title="MD5 hash" + href="{hash_url}">md5</a>) + """.lstrip().format(**locals()) + assert setuptools.package_index.PYPI_MD5.match(doc) + def test_bad_url_bad_port(self): index = setuptools.package_index.PackageIndex() url = 'http://127.0.0.1:0/nonesuch/test_package_index' @@ -129,7 +139,7 @@ class TestPackageIndex: # the distribution has been found assert 'foobar' in pi # we have only one link, because links are compared without md5 - assert len(pi['foobar'])==1 + assert len(pi['foobar']) == 1 # the link should be from the index assert 'correct_md5' in pi['foobar'][0].location @@ -209,6 +219,7 @@ class TestContentCheckers: class TestPyPIConfig: + def test_percent_in_password(self, tmpdir, monkeypatch): monkeypatch.setitem(os.environ, 'HOME', str(tmpdir)) pypirc = tmpdir / '.pypirc' diff --git a/setuptools/tests/test_sandbox.py b/setuptools/tests/test_sandbox.py index fefd46f7..aa6138e4 100644 --- a/setuptools/tests/test_sandbox.py +++ b/setuptools/tests/test_sandbox.py @@ -57,6 +57,7 @@ class TestSandbox: class TestExceptionSaver: + def test_exception_trapped(self): with setuptools.sandbox.ExceptionSaver(): raise ValueError("details") diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py index d2a1f1bb..16d0eb07 100644 --- a/setuptools/tests/test_sdist.py +++ b/setuptools/tests/test_sdist.py @@ -132,7 +132,6 @@ class TestSdistTest: assert os.path.join('sdist_test', 'b.txt') in manifest assert os.path.join('sdist_test', 'c.rst') not in manifest - def test_defaults_case_sensitivity(self): """ Make sure default files (README.*, etc.) are added in a case-sensitive diff --git a/setuptools/unicode_utils.py b/setuptools/unicode_utils.py index ffab3e24..7c63efd2 100644 --- a/setuptools/unicode_utils.py +++ b/setuptools/unicode_utils.py @@ -3,6 +3,7 @@ import sys from setuptools.extern import six + # HFS Plus uses decomposed UTF-8 def decompose(path): if isinstance(path, six.text_type): diff --git a/setuptools/utils.py b/setuptools/utils.py index 91e4b87f..080b9a8e 100644 --- a/setuptools/utils.py +++ b/setuptools/utils.py @@ -3,9 +3,9 @@ import os.path def cs_path_exists(fspath): - if not os.path.exists(fspath): + if not os.path.exists(fspath): return False # make absolute so we always have a directory abspath = os.path.abspath(fspath) directory, filename = os.path.split(abspath) - return filename in os.listdir(directory)
\ No newline at end of file + return filename in os.listdir(directory) diff --git a/tests/manual_test.py b/tests/manual_test.py index 808fa55a..9987e662 100644 --- a/tests/manual_test.py +++ b/tests/manual_test.py @@ -10,9 +10,11 @@ from string import Template from six.moves import urllib + def _system_call(*args): assert subprocess.call(args) == 0 + def tempdir(func): def _tempdir(*args, **kwargs): test_dir = tempfile.mkdtemp() @@ -25,6 +27,7 @@ def tempdir(func): shutil.rmtree(test_dir) return _tempdir + SIMPLE_BUILDOUT = """\ [buildout] @@ -62,6 +65,7 @@ def test_virtualenv(): res = f.read() assert 'setuptools' in res + @tempdir def test_full(): """virtualenv + pip + buildout""" @@ -87,6 +91,7 @@ def test_full(): assert eggs == ['extensions-0.3-py2.6.egg', 'zc.recipe.egg-1.2.2-py2.6.egg'] + if __name__ == '__main__': test_virtualenv() test_full() @@ -1,5 +1,9 @@ [tox] -envlist = py26,py27,py31,py32,py33,py34 +envlist = py26,py27,py33,py34,py35,pypy,pypy3 [testenv] -commands=python setup.py test +deps= + pytest-flake8 + pytest>=3.0.2 +passenv=APPDATA USERPROFILE HOMEDRIVE HOMEPATH windir +commands=python setup.py test --addopts='-rsx' |