diff options
| author | ?ric Araujo <merwok@netwok.org> | 2010-08-08 02:33:52 +0200 |
|---|---|---|
| committer | ?ric Araujo <merwok@netwok.org> | 2010-08-08 02:33:52 +0200 |
| commit | ed993217835aac361f665cbc6a46414466f83b4b (patch) | |
| tree | 4024a7755759a507e14f8aa478b19dc351645240 /src | |
| parent | 05da924b72e68e8e16ee3d3c5917118a41619c6d (diff) | |
| parent | 3f478eb52c69810dbfd0f87cbc4366d0056eb2e4 (diff) | |
| download | disutils2-ed993217835aac361f665cbc6a46414466f83b4b.tar.gz | |
Merge from Jeremy
Diffstat (limited to 'src')
107 files changed, 4610 insertions, 2387 deletions
diff --git a/src/DEVNOTES.txt b/src/DEVNOTES.txt index 4e506f8..e59656a 100644 --- a/src/DEVNOTES.txt +++ b/src/DEVNOTES.txt @@ -1,10 +1,21 @@ Notes for developers ==================== -- Distutils2 runs from 2.4 to 3.2 (3.x not implemented yet), so - make sure you don't use a syntax that doesn't work under +- Distutils2 runs on Python from 2.4 to 3.2 (3.x not implemented yet), + so make sure you don't use a syntax that doesn't work under one of these Python versions. - Always run tests.sh before you push a change. This implies - that you have all Python versions installed from 2.4 to 2.6. + that you have all Python versions installed from 2.4 to 2.7. +- With Python 2.4, if you want to run tests with runtests.py, or run + just one test directly, be sure to run python2.4 setup.py build_ext + first, else tests won't find _hashlib or _md5. When using tests.sh, + build_ext is automatically done. + +- Renaming to do: + + - DistributionMetadata > Metadata or ReleaseMetadata + - pkgutil > pkgutil.__init__ + new pkgutil.database (or better name) + - RationalizedVersion > Version + - suggest_rationalized_version > suggest diff --git a/src/check.sh b/src/check.sh new file mode 100755 index 0000000..bc77bf9 --- /dev/null +++ b/src/check.sh @@ -0,0 +1,2 @@ +pep8 distutils2 +pyflakes distutils2 diff --git a/src/distutils2/__init__.py b/src/distutils2/__init__.py index 0514da7..4d330b3 100644 --- a/src/distutils2/__init__.py +++ b/src/distutils2/__init__.py @@ -20,7 +20,7 @@ __revision__ = "$Id: __init__.py 78020 2010-02-06 16:37:32Z benjamin.peterson $" __version__ = "1.0a2" -# when set to True, converts doctests by default too +# when set to True, converts doctests by default too run_2to3_on_doctests = True # Standard package names for fixer packages lib2to3_fixer_packages = ['lib2to3.fixes'] diff --git a/src/Modules/_hashopenssl.c b/src/distutils2/_backport/_hashopenssl.c index fdcae5d..fdcae5d 100644 --- a/src/Modules/_hashopenssl.c +++ b/src/distutils2/_backport/_hashopenssl.c diff --git a/src/distutils2/_backport/hashlib.py b/src/distutils2/_backport/hashlib.py index 48f2cfd..dd2b02a 100644 --- a/src/distutils2/_backport/hashlib.py +++ b/src/distutils2/_backport/hashlib.py @@ -65,20 +65,20 @@ __all__ = __always_supported + ('new', 'algorithms') def __get_builtin_constructor(name): if name in ('SHA1', 'sha1'): - import _sha + from distutils2._backport import _sha return _sha.new elif name in ('MD5', 'md5'): - import _md5 + from distutils2._backport import _md5 return _md5.new elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): - import _sha256 + from distutils2._backport import _sha256 bs = name[3:] if bs == '256': return _sha256.sha256 elif bs == '224': return _sha256.sha224 elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): - import _sha512 + from distutils2._backport import _sha512 bs = name[3:] if bs == '512': return _sha512.sha512 @@ -122,7 +122,7 @@ def __hash_new(name, string=''): try: - import _hashlib + from distutils2._backport import _hashlib new = __hash_new __get_hash = __get_openssl_constructor except ImportError: diff --git a/src/Modules/md5.c b/src/distutils2/_backport/md5.c index c35d96c..c35d96c 100644 --- a/src/Modules/md5.c +++ b/src/distutils2/_backport/md5.c diff --git a/src/Modules/md5.h b/src/distutils2/_backport/md5.h index 1718401..1718401 100644 --- a/src/Modules/md5.h +++ b/src/distutils2/_backport/md5.h diff --git a/src/Modules/md5module.c b/src/distutils2/_backport/md5module.c index 5e4f116..5e4f116 100644 --- a/src/Modules/md5module.c +++ b/src/distutils2/_backport/md5module.c diff --git a/src/distutils2/_backport/pkgutil.py b/src/distutils2/_backport/pkgutil.py index b033082..ccf45bf 100644 --- a/src/distutils2/_backport/pkgutil.py +++ b/src/distutils2/_backport/pkgutil.py @@ -20,6 +20,7 @@ except ImportError: import re import warnings + __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', 'walk_packages', 'iter_modules', 'get_data', @@ -27,6 +28,7 @@ __all__ = [ 'Distribution', 'EggInfoDistribution', 'distinfo_dirname', 'get_distributions', 'get_distribution', 'get_file_users', 'provides_distribution', 'obsoletes_distribution', + 'enable_cache', 'disable_cache', 'clear_cache' ] @@ -187,8 +189,8 @@ class ImpImporter(object): searches the current ``sys.path``, plus any modules that are frozen or built-in. - Note that :class:`ImpImporter` does not currently support being used by placement - on ``sys.meta_path``. + Note that :class:`ImpImporter` does not currently support being used by + placement on ``sys.meta_path``. """ def __init__(self, path=None): @@ -577,7 +579,8 @@ def get_data(package, resource): argument should be the name of a package, in standard module format (``foo.bar``). The resource argument should be in the form of a relative filename, using ``'/'`` as the path separator. The parent directory name - ``'..'`` is not allowed, and nor is a rooted name (starting with a ``'/'``). + ``'..'`` is not allowed, and nor is a rooted name (starting with a + ``'/'``). The function returns a binary string, which is the contents of the specified resource. @@ -613,6 +616,97 @@ def get_data(package, resource): DIST_FILES = ('INSTALLER', 'METADATA', 'RECORD', 'REQUESTED',) +# Cache +_cache_name = {} # maps names to Distribution instances +_cache_name_egg = {} # maps names to EggInfoDistribution instances +_cache_path = {} # maps paths to Distribution instances +_cache_path_egg = {} # maps paths to EggInfoDistribution instances +_cache_generated = False # indicates if .dist-info distributions are cached +_cache_generated_egg = False # indicates if .dist-info and .egg are cached +_cache_enabled = True + + +def enable_cache(): + """ + Enables the internal cache. + + Note that this function will not clear the cache in any case, for that + functionality see :func:`clear_cache`. + """ + global _cache_enabled + + _cache_enabled = True + +def disable_cache(): + """ + Disables the internal cache. + + Note that this function will not clear the cache in any case, for that + functionality see :func:`clear_cache`. + """ + global _cache_enabled + + _cache_enabled = False + +def clear_cache(): + """ Clears the internal cache. """ + global _cache_name, _cache_name_egg, cache_path, _cache_path_egg, \ + _cache_generated, _cache_generated_egg + + _cache_name = {} + _cache_name_egg = {} + _cache_path = {} + _cache_path_egg = {} + _cache_generated = False + _cache_generated_egg = False + + +def _yield_distributions(include_dist, include_egg): + """ + Yield .dist-info and .egg(-info) distributions, based on the arguments + + :parameter include_dist: yield .dist-info distributions + :parameter include_egg: yield .egg(-info) distributions + """ + for path in sys.path: + realpath = os.path.realpath(path) + if not os.path.isdir(realpath): + continue + for dir in os.listdir(realpath): + dist_path = os.path.join(realpath, dir) + if include_dist and dir.endswith('.dist-info'): + yield Distribution(dist_path) + elif include_egg and (dir.endswith('.egg-info') or + dir.endswith('.egg')): + yield EggInfoDistribution(dist_path) + + +def _generate_cache(use_egg_info=False): + global _cache_generated, _cache_generated_egg + + if _cache_generated_egg or (_cache_generated and not use_egg_info): + return + else: + gen_dist = not _cache_generated + gen_egg = use_egg_info + + for dist in _yield_distributions(gen_dist, gen_egg): + if isinstance(dist, Distribution): + _cache_path[dist.path] = dist + if not dist.name in _cache_name: + _cache_name[dist.name] = [] + _cache_name[dist.name].append(dist) + else: + _cache_path_egg[dist.path] = dist + if not dist.name in _cache_name_egg: + _cache_name_egg[dist.name] = [] + _cache_name_egg[dist.name].append(dist) + + if gen_dist: + _cache_generated = True + if gen_egg: + _cache_generated_egg = True + class Distribution(object): """Created with the *path* of the ``.dist-info`` directory provided to the @@ -627,15 +721,23 @@ class Distribution(object): """A :class:`distutils2.metadata.DistributionMetadata` instance loaded with the distribution's ``METADATA`` file.""" requested = False - """A boolean that indicates whether the ``REQUESTED`` metadata file is present - (in other words, whether the package was installed by user request).""" + """A boolean that indicates whether the ``REQUESTED`` metadata file is + present (in other words, whether the package was installed by user + request or it was installed as a dependency).""" def __init__(self, path): + if _cache_enabled and path in _cache_path: + self.metadata = _cache_path[path].metadata + else: + metadata_path = os.path.join(path, 'METADATA') + self.metadata = DistributionMetadata(path=metadata_path) + self.path = path - metadata_path = os.path.join(path, 'METADATA') - self.metadata = DistributionMetadata(path=metadata_path) self.name = self.metadata['name'] + if _cache_enabled and not path in _cache_path: + _cache_path[path] = self + def _get_records(self, local=False): RECORD = os.path.join(self.path, 'RECORD') record_reader = csv_reader(open(RECORD, 'rb'), delimiter=',') @@ -756,6 +858,11 @@ class EggInfoDistribution(object): def __init__(self, path): self.path = path + if _cache_enabled and path in _cache_path_egg: + self.metadata = _cache_path_egg[path].metadata + self.name = self.metadata['Name'] + return + # reused from Distribute's pkg_resources def yield_lines(strs): """Yield non-empty/non-comment lines of a ``basestring`` or sequence""" @@ -787,7 +894,7 @@ class EggInfoDistribution(object): requires = zipf.get_data('EGG-INFO/requires.txt') except IOError: requires = None - self.name = self.metadata['name'] + self.name = self.metadata['Name'] elif path.endswith('.egg-info'): if os.path.isdir(path): path = os.path.join(path, 'PKG-INFO') @@ -840,6 +947,9 @@ class EggInfoDistribution(object): else: self.metadata['Requires'] += reqs + if _cache_enabled: + _cache_path_egg[self.path] = self + def get_installed_files(self, local=False): return [] @@ -898,17 +1008,17 @@ def get_distributions(use_egg_info=False): :rtype: iterator of :class:`Distribution` and :class:`EggInfoDistribution` instances""" - for path in sys.path: - realpath = os.path.realpath(path) - if not os.path.isdir(realpath): - continue - for dir in os.listdir(realpath): - if dir.endswith('.dist-info'): - dist = Distribution(os.path.join(realpath, dir)) - yield dist - elif use_egg_info and (dir.endswith('.egg-info') or - dir.endswith('.egg')): - dist = EggInfoDistribution(os.path.join(realpath, dir)) + if not _cache_enabled: + for dist in _yield_distributions(True, use_egg_info): + yield dist + else: + _generate_cache(use_egg_info) + + for dist in _cache_path.itervalues(): + yield dist + + if use_egg_info: + for dist in _cache_path_egg.itervalues(): yield dist @@ -928,17 +1038,19 @@ def get_distribution(name, use_egg_info=False): value is expected. If the directory is not found, ``None`` is returned. :rtype: :class:`Distribution` or :class:`EggInfoDistribution` or None""" - found = None - for dist in get_distributions(): - if dist.name == name: - found = dist - break - if use_egg_info: - for dist in get_distributions(True): + if not _cache_enabled: + for dist in _yield_distributions(True, use_egg_info): if dist.name == name: - found = dist - break - return found + return dist + else: + _generate_cache(use_egg_info) + + if name in _cache_name: + return _cache_name[name][0] + elif use_egg_info and name in _cache_name_egg: + return _cache_name_egg[name][0] + else: + return None def obsoletes_distribution(name, version=None, use_egg_info=False): diff --git a/src/Modules/sha256module.c b/src/distutils2/_backport/sha256module.c index 96cbd10..96cbd10 100644 --- a/src/Modules/sha256module.c +++ b/src/distutils2/_backport/sha256module.c diff --git a/src/Modules/sha512module.c b/src/distutils2/_backport/sha512module.c index dbaec17..dbaec17 100644 --- a/src/Modules/sha512module.c +++ b/src/distutils2/_backport/sha512module.c diff --git a/src/Modules/shamodule.c b/src/distutils2/_backport/shamodule.c index 171dfa1..171dfa1 100644 --- a/src/Modules/shamodule.c +++ b/src/distutils2/_backport/shamodule.c diff --git a/src/distutils2/_backport/tarfile.py b/src/distutils2/_backport/tarfile.py index 0d2c49b..3c95868 100644 --- a/src/distutils2/_backport/tarfile.py +++ b/src/distutils2/_backport/tarfile.py @@ -56,13 +56,6 @@ import operator if not hasattr(os, 'SEEK_SET'): os.SEEK_SET = 0 -if sys.platform == 'mac': - # This module needs work for MacOS9, especially in the area of pathname - # handling. In many places it is assumed a simple substitution of / by the - # local os.path.sep is good enough to convert pathnames, but this does not - # work with the mac rooted:path:name versus :nonrooted:path:name syntax - raise ImportError, "tarfile does not work for platform==mac" - try: import grp, pwd except ImportError: diff --git a/src/distutils2/_backport/tests/__init__.py b/src/distutils2/_backport/tests/__init__.py index e816dd0..78c8142 100644 --- a/src/distutils2/_backport/tests/__init__.py +++ b/src/distutils2/_backport/tests/__init__.py @@ -4,7 +4,7 @@ import sys from distutils2.tests.support import unittest -here = os.path.dirname(__file__) +here = os.path.dirname(__file__) or os.curdir def test_suite(): suite = unittest.TestSuite() @@ -16,4 +16,5 @@ def test_suite(): suite.addTest(module.test_suite()) return suite - +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/src/distutils2/_backport/tests/fake_dists/grammar-1.0a4.dist-info/METADATA b/src/distutils2/_backport/tests/fake_dists/grammar-1.0a4.dist-info/METADATA index 1978be7..0b99f52 100644 --- a/src/distutils2/_backport/tests/fake_dists/grammar-1.0a4.dist-info/METADATA +++ b/src/distutils2/_backport/tests/fake_dists/grammar-1.0a4.dist-info/METADATA @@ -2,3 +2,4 @@ Metadata-Version: 1.2 Name: grammar Version: 1.0a4 Requires-Dist: truffles (>=1.2) +Author: Sherlock Holmes diff --git a/src/distutils2/_backport/tests/fake_dists/truffles-5.0.egg-info b/src/distutils2/_backport/tests/fake_dists/truffles-5.0.egg-info new file mode 100644 index 0000000..45f0cf8 --- /dev/null +++ b/src/distutils2/_backport/tests/fake_dists/truffles-5.0.egg-info @@ -0,0 +1,3 @@ +Metadata-Version: 1.2 +Name: truffles +Version: 5.0 diff --git a/src/distutils2/_backport/tests/test_pkgutil.py b/src/distutils2/_backport/tests/test_pkgutil.py index fafb9a8..c052b4d 100644 --- a/src/distutils2/_backport/tests/test_pkgutil.py +++ b/src/distutils2/_backport/tests/test_pkgutil.py @@ -10,7 +10,7 @@ import zipfile try: from hashlib import md5 except ImportError: - from md5 import md5 + from distutils2._backport.hashlib import md5 from test.test_support import run_unittest, TESTFN from distutils2.tests.support import unittest @@ -25,12 +25,14 @@ except ImportError: except ImportError: from unittest2.compatibility import relpath +# Adapted from Python 2.7's trunk + # TODO Add a test for getting a distribution that is provided by another # distribution. # TODO Add a test for absolute pathed RECORD items (e.g. /etc/myapp/config.ini) -# Adapted from Python 2.7's trunk + class TestPkgUtilData(unittest.TestCase): def setUp(self): @@ -108,10 +110,14 @@ class TestPkgUtilData(unittest.TestCase): del sys.modules[pkg] + # Adapted from Python 2.7's trunk + + class TestPkgUtilPEP302(unittest.TestCase): class MyTestLoader(object): + def load_module(self, fullname): # Create an empty module mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) @@ -120,13 +126,14 @@ class TestPkgUtilPEP302(unittest.TestCase): # Make it a package mod.__path__ = [] # Count how many times the module is reloaded - mod.__dict__['loads'] = mod.__dict__.get('loads',0) + 1 + mod.__dict__['loads'] = mod.__dict__.get('loads', 0) + 1 return mod def get_data(self, path): return "Hello, world!" class MyTestImporter(object): + def find_module(self, fullname, path=None): return TestPkgUtilPEP302.MyTestLoader() @@ -319,7 +326,7 @@ class TestPkgUtilPEP376(unittest.TestCase): current_path = os.path.abspath(os.path.dirname(__file__)) self.sys_path = sys.path[:] self.fake_dists_path = os.path.join(current_path, 'fake_dists') - sys.path[0:0] = [self.fake_dists_path] + sys.path.insert(0, self.fake_dists_path) def tearDown(self): sys.path[:] = self.sys_path @@ -366,8 +373,12 @@ class TestPkgUtilPEP376(unittest.TestCase): if not isinstance(dist, Distribution): self.fail("item received was not a Distribution instance: " "%s" % type(dist)) - if dist.name in dict(fake_dists).keys(): + if dist.name in dict(fake_dists).keys() and \ + dist.path.startswith(self.fake_dists_path): found_dists.append((dist.name, dist.metadata['version'],)) + else: + # check that it doesn't find anything more than this + self.assertFalse(dist.path.startswith(self.fake_dists_path)) # otherwise we don't care what other distributions are found # Finally, test that we found all that we were looking for @@ -375,7 +386,8 @@ class TestPkgUtilPEP376(unittest.TestCase): # Now, test if the egg-info distributions are found correctly as well fake_dists += [('bacon', '0.1'), ('cheese', '2.0.2'), - ('banana', '0.4'), ('strawberry', '0.6')] + ('banana', '0.4'), ('strawberry', '0.6'), + ('truffles', '5.0')] found_dists = [] dists = [dist for dist in get_distributions(use_egg_info=True)] @@ -384,8 +396,11 @@ class TestPkgUtilPEP376(unittest.TestCase): isinstance(dist, EggInfoDistribution)): self.fail("item received was not a Distribution or " "EggInfoDistribution instance: %s" % type(dist)) - if dist.name in dict(fake_dists).keys(): + if dist.name in dict(fake_dists).keys() and \ + dist.path.startswith(self.fake_dists_path): found_dists.append((dist.name, dist.metadata['version'])) + else: + self.assertFalse(dist.path.startswith(self.fake_dists_path)) self.assertListEqual(sorted(fake_dists), sorted(found_dists)) @@ -485,7 +500,7 @@ class TestPkgUtilPEP376(unittest.TestCase): l = [dist.name for dist in provides_distribution('truffles', '>1.5', use_egg_info=True)] - checkLists(l, ['bacon']) + checkLists(l, ['bacon', 'truffles']) l = [dist.name for dist in provides_distribution('truffles', '>=1.0')] checkLists(l, ['choxie', 'towel-stuff']) @@ -549,6 +564,33 @@ class TestPkgUtilPEP376(unittest.TestCase): l = [dist.name for dist in obsoletes_distribution('truffles', '0.2')] checkLists(l, ['towel-stuff']) + def test_yield_distribution(self): + # tests the internal function _yield_distributions + from distutils2._backport.pkgutil import _yield_distributions + checkLists = lambda x, y: self.assertListEqual(sorted(x), sorted(y)) + + eggs = [('bacon', '0.1'), ('banana', '0.4'), ('strawberry', '0.6'), + ('truffles', '5.0'), ('cheese', '2.0.2')] + dists = [('choxie', '2.0.0.9'), ('grammar', '1.0a4'), + ('towel-stuff', '0.1')] + + checkLists([], _yield_distributions(False, False)) + + found = [(dist.name, dist.metadata['Version']) + for dist in _yield_distributions(False, True) + if dist.path.startswith(self.fake_dists_path)] + checkLists(eggs, found) + + found = [(dist.name, dist.metadata['Version']) + for dist in _yield_distributions(True, False) + if dist.path.startswith(self.fake_dists_path)] + checkLists(dists, found) + + found = [(dist.name, dist.metadata['Version']) + for dist in _yield_distributions(True, True) + if dist.path.startswith(self.fake_dists_path)] + checkLists(dists + eggs, found) + def test_suite(): suite = unittest.TestSuite() diff --git a/src/distutils2/command/__init__.py b/src/distutils2/command/__init__.py index 9b3f401..8d77852 100644 --- a/src/distutils2/command/__init__.py +++ b/src/distutils2/command/__init__.py @@ -5,7 +5,8 @@ commands.""" __revision__ = "$Id: __init__.py 71473 2009-04-11 14:55:07Z tarek.ziade $" -__all__ = ['build', +__all__ = ['check', + 'build', 'build_py', 'build_ext', 'build_clib', @@ -16,17 +17,11 @@ __all__ = ['build', 'install_headers', 'install_scripts', 'install_data', + 'install_distinfo', 'sdist', - 'register', 'bdist', 'bdist_dumb', 'bdist_wininst', + 'register', 'upload', - 'check', - # These two are reserved for future use: - #'bdist_sdux', - #'bdist_pkgtool', - # Note: - # bdist_packager is not included because it only provides - # an abstract base class ] diff --git a/src/distutils2/command/bdist_dumb.py b/src/distutils2/command/bdist_dumb.py index 604bcb7..4113a1e 100644 --- a/src/distutils2/command/bdist_dumb.py +++ b/src/distutils2/command/bdist_dumb.py @@ -77,9 +77,7 @@ class bdist_dumb (Command): ("don't know how to create dumb built distributions " + "on platform %s") % os.name - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name')) + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') def run(self): if not self.skip_build: diff --git a/src/distutils2/command/bdist_msi.py b/src/distutils2/command/bdist_msi.py index bbbcbcc..b759a06 100644 --- a/src/distutils2/command/bdist_msi.py +++ b/src/distutils2/command/bdist_msi.py @@ -153,10 +153,7 @@ class bdist_msi (Command): else: self.versions = list(self.all_versions) - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') if self.pre_install_script: raise DistutilsOptionError, "the pre-install-script feature is not yet implemented" diff --git a/src/distutils2/command/bdist_wininst.py b/src/distutils2/command/bdist_wininst.py index 7b55de1..264f4fb 100644 --- a/src/distutils2/command/bdist_wininst.py +++ b/src/distutils2/command/bdist_wininst.py @@ -100,10 +100,7 @@ class bdist_wininst (Command): " option must be specified" % (short_version,) self.target_version = short_version - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') if self.install_script: for script in self.distribution.scripts: @@ -177,8 +174,8 @@ class bdist_wininst (Command): # And make an archive relative to the root of the # pseudo-installation tree. - from tempfile import mktemp - archive_basename = mktemp() + from tempfile import NamedTemporaryFile + archive_basename = NamedTemporaryFile().name fullname = self.distribution.get_fullname() arcname = self.make_archive(archive_basename, "zip", root_dir=self.bdist_dir) diff --git a/src/distutils2/command/build_clib.py b/src/distutils2/command/build_clib.py index 40c4b6e..0a25aa8 100644 --- a/src/distutils2/command/build_clib.py +++ b/src/distutils2/command/build_clib.py @@ -76,9 +76,7 @@ class build_clib(Command): self.set_undefined_options('build', ('build_temp', 'build_clib'), ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force')) + 'compiler', 'debug', 'force') self.libraries = self.distribution.libraries if self.libraries: diff --git a/src/distutils2/command/build_ext.py b/src/distutils2/command/build_ext.py index 7e160ab..29530eb 100644 --- a/src/distutils2/command/build_ext.py +++ b/src/distutils2/command/build_ext.py @@ -177,13 +177,8 @@ class build_ext(Command): def finalize_options(self): self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force'), - ('plat_name', 'plat_name'), - ) + 'build_lib', 'build_temp', 'compiler', + 'debug', 'force', 'plat_name') if self.package is None: self.package = self.distribution.ext_package @@ -292,7 +287,7 @@ class build_ext(Command): "config")) else: # building python standard extensions - self.library_dirs.append('.') + self.library_dirs.append(os.curdir) # for extensions under Linux or Solaris with a shared Python library, # Python's library directory must be appended to library_dirs @@ -305,7 +300,7 @@ class build_ext(Command): self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: # building python standard extensions - self.library_dirs.append('.') + self.library_dirs.append(os.curdir) # The argument parsing will result in self.define being a string, but # it has to be a list of 2-tuples. All the preprocessor symbols diff --git a/src/distutils2/command/build_py.py b/src/distutils2/command/build_py.py index 340c249..505ffdd 100644 --- a/src/distutils2/command/build_py.py +++ b/src/distutils2/command/build_py.py @@ -95,9 +95,7 @@ class build_py(Command, Mixin2to3): self._doctests_2to3 = [] def finalize_options(self): - self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('force', 'force')) + self.set_undefined_options('build', 'build_lib', 'force') # Get the distribution options that are aliases for build_py # options -- list of packages and list of modules. diff --git a/src/distutils2/command/build_scripts.py b/src/distutils2/command/build_scripts.py index ad1c234..b62ade7 100644 --- a/src/distutils2/command/build_scripts.py +++ b/src/distutils2/command/build_scripts.py @@ -40,8 +40,7 @@ class build_scripts (Command): def finalize_options (self): self.set_undefined_options('build', ('build_scripts', 'build_dir'), - ('force', 'force'), - ('executable', 'executable')) + 'force', 'executable') self.scripts = self.distribution.scripts def get_source_files(self): diff --git a/src/distutils2/command/check.py b/src/distutils2/command/check.py index 844672d..b3175e7 100644 --- a/src/distutils2/command/check.py +++ b/src/distutils2/command/check.py @@ -8,13 +8,13 @@ from distutils2.core import Command from distutils2.errors import DistutilsSetupError class check(Command): - """This command checks the meta-data of the package. + """This command checks the metadata of the package. """ description = ("perform some checks on the package") - user_options = [('metadata', 'm', 'Verify meta-data'), + user_options = [('metadata', 'm', 'Verify metadata'), ('restructuredtext', 'r', - ('Checks if long string meta-data syntax ' - 'are reStructuredText-compliant')), + ('Checks if long string metadata syntax ' + 'is reStructuredText-compliant')), ('strict', 's', 'Will exit with an error if a check fails')] @@ -53,7 +53,7 @@ class check(Command): raise DistutilsSetupError(msg) def check_metadata(self): - """Ensures that all required elements of meta-data are supplied. + """Ensures that all required elements of metadata are supplied. name, version, URL, (author and author_email) or (maintainer and maintainer_email)). @@ -62,7 +62,7 @@ class check(Command): """ missing, __ = self.distribution.metadata.check() if missing != []: - self.warn("missing required meta-data: %s" % ', '.join(missing)) + self.warn("missing required metadata: %s" % ', '.join(missing)) def check_restructuredtext(self): """Checks if the long string fields are reST-compliant.""" diff --git a/src/distutils2/command/clean.py b/src/distutils2/command/clean.py index baeaf05..8125dcd 100644 --- a/src/distutils2/command/clean.py +++ b/src/distutils2/command/clean.py @@ -40,13 +40,9 @@ class clean(Command): self.all = None def finalize_options(self): - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib'), - ('build_scripts', 'build_scripts'), - ('build_temp', 'build_temp')) - self.set_undefined_options('bdist', - ('bdist_base', 'bdist_base')) + self.set_undefined_options('build', 'build_base', 'build_lib', + 'build_scripts', 'build_temp') + self.set_undefined_options('bdist', 'bdist_base') def run(self): # remove the build/temp.<plat> directory (unless it's already diff --git a/src/distutils2/command/cmd.py b/src/distutils2/command/cmd.py index ea001b3..73ac9e1 100644 --- a/src/distutils2/command/cmd.py +++ b/src/distutils2/command/cmd.py @@ -51,6 +51,12 @@ class Command(object): # defined. The canonical example is the "install" command. sub_commands = [] + # Pre and post command hooks are run just before or just after the command + # itself. They are simple functions that receive the command instance. They + # should be specified as dotted strings. + pre_hook = None + post_hook = None + # -- Creation/initialization methods ------------------------------- @@ -297,26 +303,30 @@ class Command(object): else: return self.__class__.__name__ - def set_undefined_options(self, src_cmd, *option_pairs): - """Set the values of any "undefined" options from corresponding - option values in some other command object. "Undefined" here means - "is None", which is the convention used to indicate that an option - has not been changed between 'initialize_options()' and - 'finalize_options()'. Usually called from 'finalize_options()' for - options that depend on some other command rather than another - option of the same command. 'src_cmd' is the other command from - which option values will be taken (a command object will be created - for it if necessary); the remaining arguments are - '(src_option,dst_option)' tuples which mean "take the value of - 'src_option' in the 'src_cmd' command object, and copy it to - 'dst_option' in the current command object". + def set_undefined_options(self, src_cmd, *options): + """Set values of undefined options from another command. + + Undefined options are options set to None, which is the convention + used to indicate that an option has not been changed between + 'initialize_options()' and 'finalize_options()'. This method is + usually called from 'finalize_options()' for options that depend on + some other command rather than another option of the same command, + typically subcommands. + + The 'src_cmd' argument is the other command from which option values + will be taken (a command object will be created for it if necessary); + the remaining positional arguments are strings that give the name of + the option to set. If the name is different on the source and target + command, you can pass a tuple with '(name_on_source, name_on_dest)' so + that 'self.name_on_dest' will be set from 'src_cmd.name_on_source'. """ - - # Option_pairs: list of (src_option, dst_option) tuples - src_cmd_obj = self.distribution.get_command_obj(src_cmd) src_cmd_obj.ensure_finalized() - for (src_option, dst_option) in option_pairs: + for obj in options: + if isinstance(obj, tuple): + src_option, dst_option = obj + else: + src_option, dst_option = obj, obj if getattr(self, dst_option) is None: setattr(self, dst_option, getattr(src_cmd_obj, src_option)) @@ -411,7 +421,7 @@ class Command(object): def spawn(self, cmd, search_path=1, level=1): """Spawn an external command respecting dry-run flag.""" - from distutils2.spawn import spawn + from distutils2.util import spawn spawn(cmd, search_path, dry_run= self.dry_run) def make_archive(self, base_name, format, root_dir=None, base_dir=None, diff --git a/src/distutils2/command/install.py b/src/distutils2/command/install.py index 588786f..1a6bd0e 100644 --- a/src/distutils2/command/install.py +++ b/src/distutils2/command/install.py @@ -18,6 +18,7 @@ from distutils2.util import write_file from distutils2.util import convert_path, change_root, get_platform from distutils2.errors import DistutilsOptionError + class install(Command): description = "install everything from build directory" @@ -31,7 +32,7 @@ class install(Command): ('home=', None, "(Unix only) home directory to install under"), - # Or, just set the base director(y|ies) + # Or just set the base director(y|ies) ('install-base=', None, "base installation directory (instead of --prefix or --home)"), ('install-platbase=', None, @@ -40,7 +41,7 @@ class install(Command): ('root=', None, "install everything relative to this alternate root directory"), - # Or, explicitly set the installation scheme + # Or explicitly set the installation scheme ('install-purelib=', None, "installation directory for pure Python module distributions"), ('install-platlib=', None, @@ -62,8 +63,8 @@ class install(Command): ('compile', 'c', "compile .py to .pyc [default]"), ('no-compile', None, "don't compile .py files"), ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), + 'also compile with optimization: -O1 for "python -O", ' + '-O2 for "python -OO", and -O0 to disable [default: -O0]'), # Miscellaneous control options ('force', 'f', @@ -77,26 +78,45 @@ class install(Command): #('install-html=', None, "directory for HTML documentation"), #('install-info=', None, "directory for GNU info files"), + # XXX use a name that makes clear this is the old format ('record=', None, - "filename in which to record list of installed files"), + "filename in which to record a list of installed files " + "(not PEP 376-compliant)"), + + # .dist-info related arguments, read by install_dist_info + ('no-distinfo', None, + "do not create a .dist-info directory"), + ('installer=', None, + "the name of the installer"), + ('requested', None, + "generate a REQUESTED file (i.e."), + ('no-requested', None, + "do not generate a REQUESTED file"), + ('no-record', None, + "do not generate a RECORD file"), ] - boolean_options = ['compile', 'force', 'skip-build'] + boolean_options = ['compile', 'force', 'skip-build', 'no-distinfo', + 'requested', 'no-record'] + + if sys.version >= '2.6': + user_options.append( + ('user', None, + "install in user site-package [%s]" % + get_path('purelib', '%s_user' % os.name))) - user_options.append(('user', None, - "install in user site-package '%s'" % \ - get_path('purelib', '%s_user' % os.name))) - boolean_options.append('user') - negative_opt = {'no-compile' : 'compile'} + boolean_options.append('user') + negative_opt = {'no-compile': 'compile', 'no-requested': 'requested'} def initialize_options(self): - """Initializes options.""" # High-level options: these select both an installation base # and scheme. self.prefix = None self.exec_prefix = None self.home = None + # This attribute is used all over the place, so it's best to + # define it even in < 2.6 self.user = 0 # These select only the installation base; it's up to the user to @@ -160,6 +180,11 @@ class install(Command): self.record = None + # .dist-info related options + self.no_distinfo = None + self.installer = None + self.requested = None + self.no_record = None # -- Option finalizing methods ------------------------------------- # (This is rather more involved than for most commands, @@ -168,7 +193,6 @@ class install(Command): # array of user input is decided. Yes, it's quite complex!) def finalize_options(self): - """Finalizes options.""" # This method (and its pliant slaves, like 'finalize_unix()', # 'finalize_other()', and 'select_scheme()') is where the default # installation directories for modules, extension modules, and @@ -185,18 +209,19 @@ class install(Command): if ((self.prefix or self.exec_prefix or self.home) and (self.install_base or self.install_platbase)): - raise DistutilsOptionError, \ - ("must supply either prefix/exec-prefix/home or " + - "install-base/install-platbase -- not both") + raise DistutilsOptionError( + "must supply either prefix/exec-prefix/home or " + "install-base/install-platbase -- not both") if self.home and (self.prefix or self.exec_prefix): - raise DistutilsOptionError, \ - "must supply either home or prefix/exec-prefix -- not both" + raise DistutilsOptionError( + "must supply either home or prefix/exec-prefix -- not both") if self.user and (self.prefix or self.exec_prefix or self.home or - self.install_base or self.install_platbase): - raise DistutilsOptionError("can't combine user with with prefix/" - "exec_prefix/home or install_(plat)base") + self.install_base or self.install_platbase): + raise DistutilsOptionError( + "can't combine user with prefix/exec_prefix/home or " + "install_base/install_platbase") # Next, stuff that's wrong (or dubious) only on certain platforms. if os.name != "posix": @@ -231,18 +256,19 @@ class install(Command): 'srcdir') metadata = self.distribution.metadata - self.config_vars = {'dist_name': metadata['Name'], - 'dist_version': metadata['Version'], - 'dist_fullname': metadata.get_fullname(), - 'py_version': py_version, - 'py_version_short': py_version[0:3], - 'py_version_nodot': py_version[0] + py_version[2], - 'sys_prefix': prefix, - 'prefix': prefix, - 'sys_exec_prefix': exec_prefix, - 'exec_prefix': exec_prefix, - 'srcdir': srcdir, - } + self.config_vars = { + 'dist_name': metadata['Name'], + 'dist_version': metadata['Version'], + 'dist_fullname': metadata.get_fullname(), + 'py_version': py_version, + 'py_version_short': py_version[:3], + 'py_version_nodot': py_version[:3:2], + 'sys_prefix': prefix, + 'prefix': prefix, + 'sys_exec_prefix': exec_prefix, + 'exec_prefix': exec_prefix, + 'srcdir': srcdir, + } self.config_vars['userbase'] = self.install_userbase self.config_vars['usersite'] = self.install_usersite @@ -270,12 +296,11 @@ class install(Command): # module distribution is pure or not. Of course, if the user # already specified install_lib, use their selection. if self.install_lib is None: - if self.distribution.ext_modules: # has extensions: non-pure + if self.distribution.ext_modules: # has extensions: non-pure self.install_lib = self.install_platlib else: self.install_lib = self.install_purelib - # Convert directories from Unix /-separated syntax to the local # convention. self.convert_paths('lib', 'purelib', 'platlib', @@ -287,7 +312,7 @@ class install(Command): # non-packagized module distributions (hello, Numerical Python!) to # get their own directories. self.handle_extra_path() - self.install_libbase = self.install_lib # needed for .pth file + self.install_libbase = self.install_lib # needed for .pth file self.install_lib = os.path.join(self.install_lib, self.extra_dirs) # If a new root directory was supplied, make all the installation @@ -299,32 +324,16 @@ class install(Command): self.dump_dirs("after prepending root") # Find out the build directories, ie. where to install from. - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib')) + self.set_undefined_options('build', 'build_base', 'build_lib') # Punt on doc directories for now -- after all, we're punting on # documentation completely! - def dump_dirs(self, msg): - """Dumps the list of user options.""" - from distutils2.fancy_getopt import longopt_xlate - log.debug(msg + ":") - for opt in self.user_options: - opt_name = opt[0] - if opt_name[-1] == "=": - opt_name = opt_name[0:-1] - if opt_name in self.negative_opt: - opt_name = self.negative_opt[opt_name] - opt_name = opt_name.translate(longopt_xlate) - val = not getattr(self, opt_name) - else: - opt_name = opt_name.translate(longopt_xlate) - val = getattr(self, opt_name) - log.debug(" %s: %s" % (opt_name, val)) + if self.no_distinfo is None: + self.no_distinfo = False def finalize_unix(self): - """Finalizes options for posix platforms.""" + """Finalize options for posix platforms.""" if self.install_base is not None or self.install_platbase is not None: if ((self.install_lib is None and self.install_purelib is None and @@ -332,15 +341,15 @@ class install(Command): self.install_headers is None or self.install_scripts is None or self.install_data is None): - raise DistutilsOptionError, \ - ("install-base or install-platbase supplied, but " - "installation scheme is incomplete") + raise DistutilsOptionError( + "install-base or install-platbase supplied, but " + "installation scheme is incomplete") return if self.user: if self.install_userbase is None: raise DistutilsPlatformError( - "User base directory is not specified") + "user base directory is not specified") self.install_base = self.install_platbase = self.install_userbase self.select_scheme("posix_user") elif self.home is not None: @@ -349,8 +358,8 @@ class install(Command): else: if self.prefix is None: if self.exec_prefix is not None: - raise DistutilsOptionError, \ - "must not supply exec-prefix without prefix" + raise DistutilsOptionError( + "must not supply exec-prefix without prefix") self.prefix = os.path.normpath(sys.prefix) self.exec_prefix = os.path.normpath(sys.exec_prefix) @@ -364,11 +373,11 @@ class install(Command): self.select_scheme("posix_prefix") def finalize_other(self): - """Finalizes options for non-posix platforms""" + """Finalize options for non-posix platforms""" if self.user: if self.install_userbase is None: raise DistutilsPlatformError( - "User base directory is not specified") + "user base directory is not specified") self.install_base = self.install_platbase = self.install_userbase self.select_scheme(os.name + "_user") elif self.home is not None: @@ -382,11 +391,27 @@ class install(Command): try: self.select_scheme(os.name) except KeyError: - raise DistutilsPlatformError, \ - "I don't know how to install stuff on '%s'" % os.name + raise DistutilsPlatformError( + "no support for installation on '%s'" % os.name) + + def dump_dirs(self, msg): + """Dump the list of user options.""" + log.debug(msg + ":") + for opt in self.user_options: + opt_name = opt[0] + if opt_name[-1] == "=": + opt_name = opt_name[0:-1] + if opt_name in self.negative_opt: + opt_name = self.negative_opt[opt_name] + opt_name = opt_name.replace('-', '_') + val = not getattr(self, opt_name) + else: + opt_name = opt_name.replace('-', '_') + val = getattr(self, opt_name) + log.debug(" %s: %s" % (opt_name, val)) def select_scheme(self, name): - """Sets the install directories by applying the install schemes.""" + """Set the install directories by applying the install schemes.""" # it's the caller's problem if they supply a bad name! scheme = get_paths(name, expand=False) for key, value in scheme.items(): @@ -409,15 +434,14 @@ class install(Command): setattr(self, attr, val) def expand_basedirs(self): - """Calls `os.path.expanduser` on install_base, install_platbase and - root.""" + """Call `os.path.expanduser` on install_{base,platbase} and root.""" self._expand_attrs(['install_base', 'install_platbase', 'root']) def expand_dirs(self): - """Calls `os.path.expanduser` on install dirs.""" + """Call `os.path.expanduser` on install dirs.""" self._expand_attrs(['install_purelib', 'install_platlib', 'install_lib', 'install_headers', - 'install_scripts', 'install_data',]) + 'install_scripts', 'install_data']) def convert_paths(self, *names): """Call `convert_path` over `names`.""" @@ -439,9 +463,9 @@ class install(Command): elif len(self.extra_path) == 2: path_file, extra_dirs = self.extra_path else: - raise DistutilsOptionError, \ - ("'extra_path' option must be a list, tuple, or " - "comma-separated string with 1 or 2 elements") + raise DistutilsOptionError( + "'extra_path' option must be a list, tuple, or " + "comma-separated string with 1 or 2 elements") # convert to local form in case Unix notation used (as it # should be in setup scripts) @@ -527,7 +551,6 @@ class install(Command): else: self.warn("path file '%s' not created" % filename) - # -- Reporting methods --------------------------------------------- def get_outputs(self): @@ -582,9 +605,11 @@ class install(Command): # 'sub_commands': a list of commands this command might have to run to # get its work done. See cmd.py for more info. - sub_commands = [('install_lib', has_lib), + sub_commands = [('install_lib', has_lib), ('install_headers', has_headers), ('install_scripts', has_scripts), - ('install_data', has_data), - ('install_egg_info', lambda self:True), + ('install_data', has_data), + # keep install_distinfo last, as it needs the record + # with files to be completely generated + ('install_distinfo', lambda self: not self.no_distinfo), ] diff --git a/src/distutils2/command/install_data.py b/src/distutils2/command/install_data.py index a9980a1..1256a0d 100644 --- a/src/distutils2/command/install_data.py +++ b/src/distutils2/command/install_data.py @@ -37,9 +37,7 @@ class install_data(Command): def finalize_options(self): self.set_undefined_options('install', ('install_data', 'install_dir'), - ('root', 'root'), - ('force', 'force'), - ) + 'root', 'force') def run(self): self.mkpath(self.install_dir) diff --git a/src/distutils2/command/install_distinfo.py b/src/distutils2/command/install_distinfo.py new file mode 100644 index 0000000..0e30f09 --- /dev/null +++ b/src/distutils2/command/install_distinfo.py @@ -0,0 +1,172 @@ +""" +distutils.command.install_distinfo +================================== + +:Author: Josip Djolonga + +This module implements the ``install_distinfo`` command that creates the +``.dist-info`` directory for the distribution, as specified in :pep:`376`. +Usually, you do not have to call this command directly, it gets called +automatically by the ``install`` command. +""" + +# This file was created from the code for the former command install_egg_info + +import os +import csv +import re +from distutils2.command.cmd import Command +from distutils2 import log +from distutils2._backport.shutil import rmtree +try: + import hashlib +except ImportError: + from distutils2._backport import hashlib + + +class install_distinfo(Command): + """Install a .dist-info directory for the package""" + + description = 'Install a .dist-info directory for the package' + + user_options = [ + ('distinfo-dir=', None, + "directory where the the .dist-info directory will be installed"), + ('installer=', None, + "the name of the installer"), + ('requested', None, + "generate a REQUESTED file"), + ('no-requested', None, + "do not generate a REQUESTED file"), + ('no-record', None, + "do not generate a RECORD file"), + ] + + boolean_options = ['requested', 'no-record'] + + negative_opt = {'no-requested': 'requested'} + + def initialize_options(self): + self.distinfo_dir = None + self.installer = None + self.requested = None + self.no_record = None + + def finalize_options(self): + self.set_undefined_options('install', + 'installer', 'requested', 'no_record') + + self.set_undefined_options('install_lib', + ('install_dir', 'distinfo_dir')) + + if self.installer is None: + # FIXME distutils or distutils2? + # + document default in the option help text above and in install + self.installer = 'distutils' + if self.requested is None: + self.requested = True + if self.no_record is None: + self.no_record = False + + metadata = self.distribution.metadata + + basename = "%s-%s.dist-info" % ( + to_filename(safe_name(metadata['Name'])), + to_filename(safe_version(metadata['Version'])), + ) + + self.distinfo_dir = os.path.join(self.distinfo_dir, basename) + self.outputs = [] + + def run(self): + if not self.dry_run: + target = self.distinfo_dir + + if os.path.isdir(target) and not os.path.islink(target): + rmtree(target) + elif os.path.exists(target): + self.execute(os.unlink, (self.distinfo_dir,), + "Removing " + target) + + self.execute(os.makedirs, (target,), "Creating " + target) + + metadata_path = os.path.join(self.distinfo_dir, 'METADATA') + log.info('Creating %s' % (metadata_path,)) + self.distribution.metadata.write(metadata_path) + self.outputs.append(metadata_path) + + installer_path = os.path.join(self.distinfo_dir, 'INSTALLER') + log.info('Creating %s' % (installer_path,)) + f = open(installer_path, 'w') + try: + f.write(self.installer) + finally: + f.close() + self.outputs.append(installer_path) + + if self.requested: + requested_path = os.path.join(self.distinfo_dir, 'REQUESTED') + log.info('Creating %s' % (requested_path,)) + f = open(requested_path, 'w') + f.close() + self.outputs.append(requested_path) + + if not self.no_record: + record_path = os.path.join(self.distinfo_dir, 'RECORD') + log.info('Creating %s' % (record_path,)) + f = open(record_path, 'wb') + try: + writer = csv.writer(f, delimiter=',', + lineterminator=os.linesep, + quotechar='"') + + install = self.get_finalized_command('install') + + for fpath in install.get_outputs(): + if fpath.endswith('.pyc') or fpath.endswith('.pyo'): + # do not put size and md5 hash, as in PEP-376 + writer.writerow((fpath, '', '')) + else: + size = os.path.getsize(fpath) + fd = open(fpath, 'r') + hash = hashlib.md5() + hash.update(fd.read()) + md5sum = hash.hexdigest() + writer.writerow((fpath, md5sum, size)) + + # add the RECORD file itself + writer.writerow((record_path, '', '')) + self.outputs.append(record_path) + finally: + f.close() + + def get_outputs(self): + return self.outputs + + +# The following functions are taken from setuptools' pkg_resources module. + +def safe_name(name): + """Convert an arbitrary string to a standard distribution name + + Any runs of non-alphanumeric/. characters are replaced with a single '-'. + """ + return re.sub('[^A-Za-z0-9.]+', '-', name) + + +def safe_version(version): + """Convert an arbitrary string to a standard version string + + Spaces become dots, and all other non-alphanumeric characters become + dashes, with runs of multiple dashes condensed to a single dash. + """ + version = version.replace(' ', '.') + return re.sub('[^A-Za-z0-9.]+', '-', version) + + +def to_filename(name): + """Convert a project or version name to its filename-escaped form + + Any '-' characters are currently replaced with '_'. + """ + return name.replace('-', '_') diff --git a/src/distutils2/command/install_egg_info.py b/src/distutils2/command/install_egg_info.py deleted file mode 100644 index c8b568a..0000000 --- a/src/distutils2/command/install_egg_info.py +++ /dev/null @@ -1,83 +0,0 @@ -"""distutils.command.install_egg_info - -Implements the Distutils 'install_egg_info' command, for installing -a package's PKG-INFO metadata.""" - - -from distutils2.command.cmd import Command -from distutils2 import log -from distutils2._backport.shutil import rmtree -import os, sys, re - -class install_egg_info(Command): - """Install an .egg-info file for the package""" - - description = "Install package's PKG-INFO metadata as an .egg-info file" - user_options = [ - ('install-dir=', 'd', "directory to install to"), - ] - - def initialize_options(self): - self.install_dir = None - - def finalize_options(self): - metadata = self.distribution.metadata - self.set_undefined_options('install_lib',('install_dir','install_dir')) - basename = "%s-%s-py%s.egg-info" % ( - to_filename(safe_name(metadata['Name'])), - to_filename(safe_version(metadata['Version'])), - sys.version[:3] - ) - self.target = os.path.join(self.install_dir, basename) - self.outputs = [self.target] - - def run(self): - target = self.target - if os.path.isdir(target) and not os.path.islink(target): - if self.dry_run: - pass # XXX - else: - rmtree(target) - elif os.path.exists(target): - self.execute(os.unlink,(self.target,),"Removing "+target) - elif not os.path.isdir(self.install_dir): - self.execute(os.makedirs, (self.install_dir,), - "Creating "+self.install_dir) - log.info("Writing %s", target) - if not self.dry_run: - f = open(target, 'w') - self.distribution.metadata.write_file(f) - f.close() - - def get_outputs(self): - return self.outputs - - -# The following routines are taken from setuptools' pkg_resources module and -# can be replaced by importing them from pkg_resources once it is included -# in the stdlib. - -def safe_name(name): - """Convert an arbitrary string to a standard distribution name - - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub('[^A-Za-z0-9.]+', '-', name) - - -def safe_version(version): - """Convert an arbitrary string to a standard version string - - Spaces become dots, and all other non-alphanumeric characters become - dashes, with runs of multiple dashes condensed to a single dash. - """ - version = version.replace(' ','.') - return re.sub('[^A-Za-z0-9.]+', '-', version) - - -def to_filename(name): - """Convert a project or version name to its filename-escaped form - - Any '-' characters are currently replaced with '_'. - """ - return name.replace('-','_') diff --git a/src/distutils2/command/install_headers.py b/src/distutils2/command/install_headers.py index 762656d..dd63128 100644 --- a/src/distutils2/command/install_headers.py +++ b/src/distutils2/command/install_headers.py @@ -29,8 +29,7 @@ class install_headers(Command): def finalize_options(self): self.set_undefined_options('install', ('install_headers', 'install_dir'), - ('force', 'force')) - + 'force') def run(self): headers = self.distribution.headers diff --git a/src/distutils2/command/install_lib.py b/src/distutils2/command/install_lib.py index 7a9023b..c5e78f7 100644 --- a/src/distutils2/command/install_lib.py +++ b/src/distutils2/command/install_lib.py @@ -68,11 +68,7 @@ class install_lib(Command): self.set_undefined_options('install', ('build_lib', 'build_dir'), ('install_lib', 'install_dir'), - ('force', 'force'), - ('compile', 'compile'), - ('optimize', 'optimize'), - ('skip_build', 'skip_build'), - ) + 'force', 'compile', 'optimize', 'skip_build') if self.compile is None: self.compile = 1 diff --git a/src/distutils2/command/install_scripts.py b/src/distutils2/command/install_scripts.py index bdf7362..fa7587e 100644 --- a/src/distutils2/command/install_scripts.py +++ b/src/distutils2/command/install_scripts.py @@ -36,9 +36,7 @@ class install_scripts (Command): self.set_undefined_options('build', ('build_scripts', 'build_dir')) self.set_undefined_options('install', ('install_scripts', 'install_dir'), - ('force', 'force'), - ('skip_build', 'skip_build'), - ) + 'force', 'skip_build') def run (self): if not self.skip_build: diff --git a/src/distutils2/command/register.py b/src/distutils2/command/register.py index 29b87e4..3f27272 100644 --- a/src/distutils2/command/register.py +++ b/src/distutils2/command/register.py @@ -13,28 +13,41 @@ import urlparse import StringIO from warnings import warn -from distutils2.core import PyPIRCCommand +from distutils2.command.cmd import Command from distutils2 import log - -class register(PyPIRCCommand): - - description = ("register the distribution with the Python package index") - user_options = PyPIRCCommand.user_options + [ +from distutils2.util import (metadata_to_dict, read_pypirc, generate_pypirc, + DEFAULT_REPOSITORY, DEFAULT_REALM, + get_pypirc_path) + +class register(Command): + + description = "register the distribution with the Python package index" + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), ('list-classifiers', None, - 'list the valid Trove classifiers'), + "list valid Trove classifiers"), ('strict', None , - 'Will stop the registering if the meta-data are not fully compliant') + "stop the registration if the metadata is not fully compliant") ] - boolean_options = PyPIRCCommand.boolean_options + [ - 'verify', 'list-classifiers', 'strict'] + + boolean_options = ['show-response', 'verify', 'list-classifiers', + 'strict'] def initialize_options(self): - PyPIRCCommand.initialize_options(self) + self.repository = None + self.realm = None + self.show_response = 0 self.list_classifiers = 0 self.strict = 0 def finalize_options(self): - PyPIRCCommand.finalize_options(self) + if self.repository is None: + self.repository = DEFAULT_REPOSITORY + if self.realm is None: + self.realm = DEFAULT_REALM # setting options for the `check` subcommand check_options = {'strict': ('register', self.strict), 'restructuredtext': ('register', 1)} @@ -67,7 +80,7 @@ class register(PyPIRCCommand): def _set_config(self): ''' Reads the configuration file and set attributes. ''' - config = self._read_pypirc() + config = read_pypirc(self.repository, self.realm) if config != {}: self.username = config['username'] self.password = config['password'] @@ -75,10 +88,10 @@ class register(PyPIRCCommand): self.realm = config['realm'] self.has_config = True else: - if self.repository not in ('pypi', self.DEFAULT_REPOSITORY): + if self.repository not in ('pypi', DEFAULT_REPOSITORY): raise ValueError('%s not found in .pypirc' % self.repository) if self.repository == 'pypi': - self.repository = self.DEFAULT_REPOSITORY + self.repository = DEFAULT_REPOSITORY self.has_config = False def classifiers(self): @@ -177,14 +190,14 @@ Your selection [default 1]: ''', log.INFO) self.announce(('I can store your PyPI login so future ' 'submissions will be faster.'), log.INFO) self.announce('(the login will be stored in %s)' % \ - self._get_rc_file(), log.INFO) + get_pypirc_path(), log.INFO) choice = 'X' while choice.lower() not in 'yn': choice = raw_input('Save your login (y/N)?') if not choice: choice = 'n' if choice.lower() == 'y': - self._store_pypirc(username, password) + generate_pypirc(username, password) elif choice == '2': data = {':action': 'user'} @@ -221,7 +234,7 @@ Your selection [default 1]: ''', log.INFO) def build_post_data(self, action): # figure the data to send - the metadata plus some additional # information used by the package server - data = self._metadata_to_pypy_dict() + data = metadata_to_dict(self.distribution.metadata) data[':action'] = action return data diff --git a/src/distutils2/command/sdist.py b/src/distutils2/command/sdist.py index 44fca10..464c7fe 100644 --- a/src/distutils2/command/sdist.py +++ b/src/distutils2/command/sdist.py @@ -70,8 +70,8 @@ class sdist(Command): ('dist-dir=', 'd', "directory to put the source distribution archive(s) in " "[default: dist]"), - ('medata-check', None, - "Ensure that all required elements of meta-data " + ('check-metadata', None, + "Ensure that all required elements of metadata " "are supplied. Warn if any missing. [default]"), ('owner=', 'u', "Owner name used when creating a tar file [default: current user]"), @@ -80,7 +80,7 @@ class sdist(Command): ] boolean_options = ['use-defaults', 'prune', - 'manifest-only', 'keep-temp', 'metadata-check'] + 'manifest-only', 'keep-temp', 'check-metadata'] help_options = [ ('help-formats', None, @@ -317,7 +317,7 @@ class sdist(Command): 'self.keep_temp' is true). The list of archive files created is stored so it can be retrieved later by 'get_archive_files()'. """ - # Don't warn about missing meta-data here -- should be (and is!) + # Don't warn about missing metadata here -- should be (and is!) # done elsewhere. base_dir = self.distribution.get_fullname() base_name = os.path.join(self.dist_dir, base_dir) diff --git a/src/distutils2/command/upload.py b/src/distutils2/command/upload.py index 14fe119..18ebb77 100644 --- a/src/distutils2/command/upload.py +++ b/src/distutils2/command/upload.py @@ -11,40 +11,57 @@ import StringIO as StringIO try: from hashlib import md5 except ImportError: - from md5 import md5 + from distutils2._backport.hashlib import md5 from distutils2.errors import DistutilsOptionError -from distutils2.core import PyPIRCCommand -from distutils2.spawn import spawn +from distutils2.util import spawn from distutils2 import log +from distutils2.command.cmd import Command +from distutils2 import log +from distutils2.util import (metadata_to_dict, read_pypirc, + DEFAULT_REPOSITORY, DEFAULT_REALM) + -class upload(PyPIRCCommand): +class upload(Command): - description = "upload binary package to PyPI" + description = "upload distribution to PyPI" - user_options = PyPIRCCommand.user_options + [ + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), ('sign', 's', - 'sign files to upload using gpg'), - ('identity=', 'i', 'GPG identity used to sign files'), + "sign files to upload using gpg"), + ('identity=', 'i', + "GPG identity used to sign files"), + ('upload-docs', None, + "upload documentation too"), ] - boolean_options = PyPIRCCommand.boolean_options + ['sign'] + boolean_options = ['show-response', 'sign'] def initialize_options(self): - PyPIRCCommand.initialize_options(self) + self.repository = None + self.realm = None + self.show_response = 0 self.username = '' self.password = '' self.show_response = 0 self.sign = False self.identity = None + self.upload_docs = False def finalize_options(self): - PyPIRCCommand.finalize_options(self) + if self.repository is None: + self.repository = DEFAULT_REPOSITORY + if self.realm is None: + self.realm = DEFAULT_REALM if self.identity and not self.sign: raise DistutilsOptionError( "Must use --sign for --identity to have meaning" ) - config = self._read_pypirc() + config = read_pypirc(self.repository, self.realm) if config != {}: self.username = config['username'] self.password = config['password'] @@ -61,6 +78,12 @@ class upload(PyPIRCCommand): raise DistutilsOptionError("No dist file created in earlier command") for command, pyversion, filename in self.distribution.dist_files: self.upload_file(command, pyversion, filename) + if self.upload_docs: + upload_docs = self.get_finalized_command("upload_docs") + upload_docs.repository = self.repository + upload_docs.username = self.username + upload_docs.password = self.password + upload_docs.run() # XXX to be refactored with register.post_to_server def upload_file(self, command, pyversion, filename): @@ -81,11 +104,11 @@ class upload(PyPIRCCommand): spawn(gpg_args, dry_run=self.dry_run) - # Fill in the data - send all the meta-data in case we need to + # Fill in the data - send all the metadata in case we need to # register a new release content = open(filename,'rb').read() - data = self._metadata_to_pypy_dict() + data = metadata_to_dict(self.distribution.metadata) # extra upload infos data[':action'] = 'file_upload' diff --git a/src/distutils2/command/upload_docs.py b/src/distutils2/command/upload_docs.py index 256d4c2..7838d85 100644 --- a/src/distutils2/command/upload_docs.py +++ b/src/distutils2/command/upload_docs.py @@ -1,9 +1,11 @@ import base64, httplib, os.path, socket, urlparse, zipfile from cStringIO import StringIO + from distutils2 import log from distutils2.command.upload import upload -from distutils2.core import PyPIRCCommand +from distutils2.command.cmd import Command from distutils2.errors import DistutilsFileError +from distutils2.util import read_pypirc, DEFAULT_REPOSITORY, DEFAULT_REALM def zip_dir(directory): """Compresses recursively contents of directory into a StringIO object""" @@ -47,26 +49,36 @@ def encode_multipart(fields, files, boundary=None): content_type = 'multipart/form-data; boundary=%s' % boundary return content_type, body -class upload_docs(PyPIRCCommand): +class upload_docs(Command): user_options = [ - ('repository=', 'r', "url of repository [default: %s]" % upload.DEFAULT_REPOSITORY), - ('show-response', None, 'display full response text from server'), - ('upload-dir=', None, 'directory to upload'), + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), + ('upload-dir=', None, + "directory to upload"), ] def initialize_options(self): - PyPIRCCommand.initialize_options(self) - self.upload_dir = "build/docs" + self.repository = None + self.realm = None + self.show_response = 0 + self.upload_dir = None + self.username = '' + self.password = '' def finalize_options(self): - PyPIRCCommand.finalize_options(self) - if self.upload_dir == None: + if self.repository is None: + self.repository = DEFAULT_REPOSITORY + if self.realm is None: + self.realm = DEFAULT_REALM + if self.upload_dir is None: build = self.get_finalized_command('build') self.upload_dir = os.path.join(build.build_base, "docs") self.announce('Using upload directory %s' % self.upload_dir) self.verify_upload_dir(self.upload_dir) - config = self._read_pypirc() + config = read_pypirc(self.repository, self.realm) if config != {}: self.username = config['username'] self.password = config['password'] diff --git a/src/distutils2/compiler/ccompiler.py b/src/distutils2/compiler/ccompiler.py index adbf529..c9bb129 100644 --- a/src/distutils2/compiler/ccompiler.py +++ b/src/distutils2/compiler/ccompiler.py @@ -11,8 +11,7 @@ import re from distutils2.errors import (CompileError, LinkError, UnknownFileError, DistutilsPlatformError, DistutilsModuleError) -from distutils2.spawn import spawn -from distutils2.util import split_quoted, execute, newer_group +from distutils2.util import split_quoted, execute, newer_group, spawn from distutils2 import log from shutil import move diff --git a/src/distutils2/config.py b/src/distutils2/config.py deleted file mode 100644 index 30ad479..0000000 --- a/src/distutils2/config.py +++ /dev/null @@ -1,155 +0,0 @@ -"""distutils.pypirc - -Provides the PyPIRCCommand class, the base class for the command classes -that uses .pypirc in the distutils.command package. -""" -import os -from ConfigParser import RawConfigParser - -from distutils2.command.cmd import Command - -DEFAULT_PYPIRC = """\ -[distutils] -index-servers = - pypi - -[pypi] -username:%s -password:%s -""" - -class PyPIRCCommand(Command): - """Base command that knows how to handle the .pypirc file - """ - DEFAULT_REPOSITORY = 'http://pypi.python.org/pypi' - DEFAULT_REALM = 'pypi' - repository = None - realm = None - - user_options = [ - ('repository=', 'r', - "url of repository [default: %s]" % \ - DEFAULT_REPOSITORY), - ('show-response', None, - 'display full response text from server')] - - boolean_options = ['show-response'] - - def _get_rc_file(self): - """Returns rc file path.""" - return os.path.join(os.path.expanduser('~'), '.pypirc') - - def _store_pypirc(self, username, password): - """Creates a default .pypirc file.""" - rc = self._get_rc_file() - f = open(rc, 'w') - try: - f.write(DEFAULT_PYPIRC % (username, password)) - finally: - f.close() - try: - os.chmod(rc, 0600) - except OSError: - # should do something better here - pass - - def _read_pypirc(self): - """Reads the .pypirc file.""" - rc = self._get_rc_file() - if os.path.exists(rc): - self.announce('Using PyPI login from %s' % rc) - repository = self.repository or self.DEFAULT_REPOSITORY - config = RawConfigParser() - config.read(rc) - sections = config.sections() - if 'distutils' in sections: - # let's get the list of servers - index_servers = config.get('distutils', 'index-servers') - _servers = [server.strip() for server in - index_servers.split('\n') - if server.strip() != ''] - if _servers == []: - # nothing set, let's try to get the default pypi - if 'pypi' in sections: - _servers = ['pypi'] - else: - # the file is not properly defined, returning - # an empty dict - return {} - for server in _servers: - current = {'server': server} - current['username'] = config.get(server, 'username') - - # optional params - for key, default in (('repository', - self.DEFAULT_REPOSITORY), - ('realm', self.DEFAULT_REALM), - ('password', None)): - if config.has_option(server, key): - current[key] = config.get(server, key) - else: - current[key] = default - if (current['server'] == repository or - current['repository'] == repository): - return current - elif 'server-login' in sections: - # old format - server = 'server-login' - if config.has_option(server, 'repository'): - repository = config.get(server, 'repository') - else: - repository = self.DEFAULT_REPOSITORY - return {'username': config.get(server, 'username'), - 'password': config.get(server, 'password'), - 'repository': repository, - 'server': server, - 'realm': self.DEFAULT_REALM} - - return {} - - def _metadata_to_pypy_dict(self): - meta = self.distribution.metadata - data = { - 'metadata_version' : meta.version, - 'name': meta['Name'], - 'version': meta['Version'], - 'summary': meta['Summary'], - 'home_page': meta['Home-page'], - 'author': meta['Author'], - 'author_email': meta['Author-email'], - 'license': meta['License'], - 'description': meta['Description'], - 'keywords': meta['Keywords'], - 'platform': meta['Platform'], - 'classifier': meta['Classifier'], - 'download_url': meta['Download-URL'], - } - - if meta.version == '1.2': - data['requires_dist'] = meta['Requires-Dist'] - data['requires_python'] = meta['Requires-Python'] - data['requires_external'] = meta['Requires-External'] - data['provides_dist'] = meta['Provides-Dist'] - data['obsoletes_dist'] = meta['Obsoletes-Dist'] - data['project_url'] = [','.join(url) for url in - meta['Project-URL']] - - elif meta.version == '1.1': - data['provides'] = meta['Provides'] - data['requires'] = meta['Requires'] - data['obsoletes'] = meta['Obsoletes'] - - return data - - def initialize_options(self): - """Initialize options.""" - self.repository = None - self.realm = None - self.show_response = 0 - - def finalize_options(self): - """Finalizes options.""" - if self.repository is None: - self.repository = self.DEFAULT_REPOSITORY - if self.realm is None: - self.realm = self.DEFAULT_REALM diff --git a/src/distutils2/core.py b/src/distutils2/core.py index a81780b..0820adb 100644 --- a/src/distutils2/core.py +++ b/src/distutils2/core.py @@ -19,7 +19,6 @@ from distutils2.util import grok_environment_error # Mainly import these so setup scripts can "from distutils2.core import" them. from distutils2.dist import Distribution from distutils2.command.cmd import Command -from distutils2.config import PyPIRCCommand from distutils2.extension import Extension from distutils2.util import find_packages @@ -34,6 +33,7 @@ usage: %(script)s [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...] or: %(script)s cmd --help """ + def gen_usage(script_name): script = os.path.basename(script_name) return USAGE % {'script': script} @@ -60,6 +60,7 @@ extension_keywords = ('name', 'sources', 'include_dirs', 'extra_objects', 'extra_compile_args', 'extra_link_args', 'swig_opts', 'export_symbols', 'depends', 'language') + def setup(**attrs): """The gateway to the Distutils: do everything your setup script needs to do, in a highly flexible and user-driven way. Briefly: create a @@ -156,7 +157,7 @@ def setup(**attrs): def run_setup(script_name, script_args=None, stop_after="run"): """Run a setup script in a somewhat controlled environment, and return the Distribution instance that drives things. This is useful - if you need to find out the distribution meta-data (passed as + if you need to find out the distribution metadata (passed as keyword args from 'script' to 'setup()', or the contents of the config files or command-line. @@ -206,8 +207,6 @@ def run_setup(script_name, script_args=None, stop_after="run"): # Hmm, should we do something if exiting with a non-zero code # (ie. error)? pass - except: - raise if _setup_distribution is None: raise RuntimeError, \ diff --git a/src/distutils2/depgraph.py b/src/distutils2/depgraph.py index 75bc4d3..2d66a39 100644 --- a/src/distutils2/depgraph.py +++ b/src/distutils2/depgraph.py @@ -1,5 +1,5 @@ -"""Analyse the relationships between the distributions in the system and generate -a dependency graph. +"""Analyse the relationships between the distributions in the system +and generate a dependency graph. """ from distutils2.errors import DistutilsError @@ -135,8 +135,7 @@ def generate_graph(dists): requires = dist.metadata['Requires-Dist'] + dist.metadata['Requires'] for req in requires: predicate = VersionPredicate(req) - comps = req.strip().rsplit(" ", 1) - name = comps[0] + name = predicate.name if not name in provided: graph.add_missing(dist, req) diff --git a/src/distutils2/dist.py b/src/distutils2/dist.py index b393a0e..90b3af1 100644 --- a/src/distutils2/dist.py +++ b/src/distutils2/dist.py @@ -6,19 +6,17 @@ being built/installed/distributed. __revision__ = "$Id: dist.py 77717 2010-01-24 00:33:32Z tarek.ziade $" -import sys, os, re - -try: - import warnings -except ImportError: - warnings = None +import sys +import os +import re +import warnings from ConfigParser import RawConfigParser from distutils2.errors import (DistutilsOptionError, DistutilsArgError, DistutilsModuleError, DistutilsClassError) from distutils2.fancy_getopt import FancyGetopt, translate_longopt -from distutils2.util import check_environ, strtobool +from distutils2.util import check_environ, strtobool, resolve_dotted_name from distutils2 import log from distutils2.metadata import DistributionMetadata @@ -26,7 +24,7 @@ from distutils2.metadata import DistributionMetadata # the same as a Python NAME -- I don't allow leading underscores. The fact # that they're very similar is no coincidence; the default naming scheme is # to look for a Python module named after the command. -command_re = re.compile (r'^[a-zA-Z]([a-zA-Z0-9_]*)$') +command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') class Distribution(object): @@ -43,7 +41,6 @@ class Distribution(object): See the code for 'setup()', in core.py, for details. """ - # 'global_options' describes the command-line options that may be # supplied to the setup script prior to any actual commands. # Eg. "./setup.py -n" or "./setup.py --quiet" both take advantage of @@ -116,7 +113,7 @@ Common commands: (see '--help-commands' for more) ('use-2to3', None, "use 2to3 to make source python 3.x compatible"), ('convert-2to3-doctests', None, - "use 2to3 to convert doctests in seperate text files"), + "use 2to3 to convert doctests in seperate text files"), ] display_option_names = map(lambda x: translate_longopt(x[0]), display_options) @@ -124,10 +121,8 @@ Common commands: (see '--help-commands' for more) # negative options are options that exclude other options negative_opt = {'quiet': 'verbose'} - # -- Creation/initialization methods ------------------------------- - - def __init__ (self, attrs=None): + def __init__(self, attrs=None): """Construct a new Distribution instance: initialize all the attributes of a Distribution, and then use 'attrs' (a dictionary mapping attribute names to values) to assign some of those @@ -145,17 +140,12 @@ Common commands: (see '--help-commands' for more) for attr in self.display_option_names: setattr(self, attr, 0) - # Store the distribution meta-data (name, version, author, and so + # Store the distribution metadata (name, version, author, and so # forth) in a separate object -- we're getting to have enough # information here (and enough command-line options) that it's - # worth it. Also delegate 'get_XXX()' methods to the 'metadata' - # object in a sneaky and underhanded (but efficient!) way. + # worth it. self.metadata = DistributionMetadata() - #for basename in self.metadata._METHOD_BASENAMES: - # method_name = "get_" + basename - # setattr(self, method_name, getattr(self.metadata, method_name)) - # 'cmdclass' maps command names to class objects, so we # can 1) quickly figure out which class to instantiate when # we need to create a new command object, and 2) have a way @@ -257,10 +247,7 @@ Common commands: (see '--help-commands' for more) setattr(self, key, val) else: msg = "Unknown distribution option: %r" % key - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") + warnings.warn(msg) # no-user-cfg is handled before other command line args # because other args override the config files, and this @@ -286,10 +273,10 @@ Common commands: (see '--help-commands' for more) and return the new dictionary; otherwise, return the existing option dictionary. """ - dict = self.command_options.get(command) - if dict is None: - dict = self.command_options[command] = {} - return dict + d = self.command_options.get(command) + if d is None: + d = self.command_options[command] = {} + return d def get_fullname(self): return self.metadata.get_fullname() @@ -385,9 +372,21 @@ Common commands: (see '--help-commands' for more) for opt in options: if opt != '__name__': - val = parser.get(section,opt) + val = parser.get(section, opt) opt = opt.replace('-', '_') - opt_dict[opt] = (filename, val) + + # ... although practicality beats purity :( + if opt.startswith("pre_hook.") or opt.startswith("post_hook."): + hook_type, alias = opt.split(".") + # Hooks are somewhat exceptional, as they are + # gathered from many config files (not overriden as + # other options). + # The option_dict expects {"command": ("filename", # "value")} + # so for hooks, we store only the last config file processed + hook_dict = opt_dict.setdefault(hook_type, (filename, {}))[1] + hook_dict[alias] = val + else: + opt_dict[opt] = (filename, val) # Make the RawConfigParser forget everything (so we retain # the original filenames that options come from) @@ -402,12 +401,12 @@ Common commands: (see '--help-commands' for more) try: if alias: setattr(self, alias, not strtobool(val)) - elif opt in ('verbose', 'dry_run'): # ugh! + elif opt in ('verbose', 'dry_run'): # ugh! setattr(self, opt, strtobool(val)) else: setattr(self, opt, val) except ValueError, msg: - raise DistutilsOptionError, msg + raise DistutilsOptionError(msg) # -- Command-line parsing methods ---------------------------------- @@ -473,7 +472,7 @@ Common commands: (see '--help-commands' for more) # Oops, no commands found -- an end-user error if not self.commands: - raise DistutilsArgError, "no commands supplied" + raise DistutilsArgError("no commands supplied") # All is well: return true return 1 @@ -504,7 +503,7 @@ Common commands: (see '--help-commands' for more) # Pull the current command from the head of the command line command = args[0] if not command_re.match(command): - raise SystemExit, "invalid command name '%s'" % command + raise SystemExit("invalid command name '%s'" % command) self.commands.append(command) # Dig up the command class that implements this command, so we @@ -513,22 +512,21 @@ Common commands: (see '--help-commands' for more) try: cmd_class = self.get_command_class(command) except DistutilsModuleError, msg: - raise DistutilsArgError, msg + raise DistutilsArgError(msg) # Require that the command class be derived from Command -- want # to be sure that the basic "command" interface is implemented. if not issubclass(cmd_class, Command): - raise DistutilsClassError, \ - "command class %s must subclass Command" % cmd_class + raise DistutilsClassError( + "command class %s must subclass Command" % cmd_class) # Also make sure that the command object provides a list of its # known options. if not (hasattr(cmd_class, 'user_options') and isinstance(cmd_class.user_options, list)): - raise DistutilsClassError, \ - ("command class %s must provide " + - "'user_options' attribute (a list of tuples)") % \ - cmd_class + raise DistutilsClassError( + ("command class %s must provide " + "'user_options' attribute (a list of tuples)") % cmd_class) # If the command class has a list of negative alias options, # merge it in with the global negative aliases. @@ -545,7 +543,6 @@ Common commands: (see '--help-commands' for more) else: help_options = [] - # All commands support the global options too, just by adding # in 'global_options'. parser.set_option_table(self.global_options + @@ -559,10 +556,10 @@ Common commands: (see '--help-commands' for more) if (hasattr(cmd_class, 'help_options') and isinstance(cmd_class.help_options, list)): - help_option_found=0 + help_option_found = 0 for (help_option, short, desc, func) in cmd_class.help_options: if hasattr(opts, parser.get_attr_name(help_option)): - help_option_found=1 + help_option_found = 1 if hasattr(func, '__call__'): func() else: @@ -588,7 +585,7 @@ Common commands: (see '--help-commands' for more) objects. """ if getattr(self, 'convert_2to3_doctests', None): - self.convert_2to3_doctests = [os.path.join(p) + self.convert_2to3_doctests = [os.path.join(p) for p in self.convert_2to3_doctests] else: self.convert_2to3_doctests = [] @@ -812,7 +809,7 @@ Common commands: (see '--help-commands' for more) class_name = command try: - __import__ (module_name) + __import__(module_name) module = sys.modules[module_name] except ImportError: continue @@ -820,9 +817,9 @@ Common commands: (see '--help-commands' for more) try: cls = getattr(module, class_name) except AttributeError: - raise DistutilsModuleError, \ - "invalid command '%s' (no class '%s' in module '%s')" \ - % (command, class_name, module_name) + raise DistutilsModuleError( + "invalid command '%s' (no class '%s' in module '%s')" % + (command, class_name, module_name)) self.cmdclass[command] = cls return cls @@ -891,11 +888,11 @@ Common commands: (see '--help-commands' for more) elif hasattr(command_obj, option): setattr(command_obj, option, value) else: - raise DistutilsOptionError, \ - ("error in %s: command '%s' has no such option '%s'" - % (source, command_name, option)) + raise DistutilsOptionError( + "error in %s: command '%s' has no such option '%s'" % + (source, command_name, option)) except ValueError, msg: - raise DistutilsOptionError, msg + raise DistutilsOptionError(msg) def get_reinitialized_command(self, command, reinit_subcommands=0): """Reinitializes a command to the state it was in when first @@ -963,15 +960,23 @@ Common commands: (see '--help-commands' for more) if self.have_run.get(command): return - log.info("running %s", command) cmd_obj = self.get_command_obj(command) cmd_obj.ensure_finalized() + self.run_command_hooks(cmd_obj, 'pre_hook') + log.info("running %s", command) cmd_obj.run() + self.run_command_hooks(cmd_obj, 'post_hook') self.have_run[command] = 1 + def run_command_hooks(self, cmd_obj, hook_kind): + hooks = getattr(cmd_obj, hook_kind) + if hooks is None: + return + for hook in hooks.values(): + hook_func = resolve_dotted_name(hook) + hook_func(cmd_obj) # -- Distribution query methods ------------------------------------ - def has_pure_modules(self): return len(self.packages or self.py_modules or []) > 0 @@ -998,13 +1003,8 @@ Common commands: (see '--help-commands' for more) not self.has_ext_modules() and not self.has_c_libraries()) - # -- Metadata query methods ---------------------------------------- - - # If you're looking for 'get_name()', 'get_version()', and so forth, - # they are defined in a sneaky way: the constructor binds self.get_XXX - # to self.metadata.get_XXX. The actual code is in the - # DistributionMetadata class, below. +# XXX keep for compat or remove? def fix_help_options(options): """Convert a 4-tuple 'help_options' list as found in various command diff --git a/src/distutils2/errors.py b/src/distutils2/errors.py index d5750f4..ef3e300 100644 --- a/src/distutils2/errors.py +++ b/src/distutils2/errors.py @@ -10,31 +10,38 @@ symbols whose names start with "Distutils" and end with "Error".""" __revision__ = "$Id: errors.py 75901 2009-10-28 06:45:18Z tarek.ziade $" + class DistutilsError(Exception): """The root of all Distutils evil.""" + class DistutilsModuleError(DistutilsError): """Unable to load an expected module, or to find an expected class within some module (in particular, command modules and classes).""" + class DistutilsClassError(DistutilsError): """Some command class (or possibly distribution class, if anyone feels a need to subclass Distribution) is found not to be holding up its end of the bargain, ie. implementing some part of the "command "interface.""" + class DistutilsGetoptError(DistutilsError): """The option table provided to 'fancy_getopt()' is bogus.""" + class DistutilsArgError(DistutilsError): """Raised by fancy_getopt in response to getopt.error -- ie. an error in the command line usage.""" + class DistutilsFileError(DistutilsError): """Any problems in the filesystem: expected file not found, etc. Typically this is for problems that we detect before IOError or OSError could be raised.""" + class DistutilsOptionError(DistutilsError): """Syntactic/semantic errors in command options, such as use of mutually conflicting options, or inconsistent options, @@ -43,60 +50,76 @@ class DistutilsOptionError(DistutilsError): files, or what-have-you -- but if we *know* something originated in the setup script, we'll raise DistutilsSetupError instead.""" + class DistutilsSetupError(DistutilsError): """For errors that can be definitely blamed on the setup script, such as invalid keyword arguments to 'setup()'.""" + class DistutilsPlatformError(DistutilsError): """We don't know how to do something on the current platform (but we do know how to do it on some platform) -- eg. trying to compile C files on a platform not supported by a CCompiler subclass.""" + class DistutilsExecError(DistutilsError): """Any problems executing an external program (such as the C compiler, when compiling C files).""" + class DistutilsInternalError(DistutilsError): """Internal inconsistencies or impossibilities (obviously, this should never be seen if the code is working!).""" + class DistutilsTemplateError(DistutilsError): """Syntax error in a file list template.""" + class DistutilsByteCompileError(DistutilsError): """Byte compile error.""" + # Exception classes used by the CCompiler implementation classes class CCompilerError(Exception): """Some compile/link operation failed.""" + class PreprocessError(CCompilerError): """Failure to preprocess one or more C/C++ files.""" + class CompileError(CCompilerError): """Failure to compile one or more C/C++ source files.""" + class LibError(CCompilerError): """Failure to create a static library from one or more C/C++ object files.""" + class LinkError(CCompilerError): """Failure to link one or more C/C++ object files into an executable or shared library file.""" + class UnknownFileError(CCompilerError): """Attempt to process an unknown file type.""" + class MetadataConflictError(DistutilsError): """Attempt to read or write metadata fields that are conflictual.""" + class MetadataUnrecognizedVersionError(DistutilsError): """Unknown metadata version number.""" + class IrrationalVersionError(Exception): """This is an irrational version.""" pass + class HugeMajorVersionNumError(IrrationalVersionError): """An irrational version because the major version number is huge (often because a year or date was used). @@ -105,4 +128,3 @@ class HugeMajorVersionNumError(IrrationalVersionError): This guard can be disabled by setting that option False. """ pass - diff --git a/src/distutils2/extension.py b/src/distutils2/extension.py index 05ccd20..f38bf60 100644 --- a/src/distutils2/extension.py +++ b/src/distutils2/extension.py @@ -17,6 +17,7 @@ import warnings # import that large-ish module (indirectly, through distutils.core) in # order to do anything. + class Extension(object): """Just a collection of attributes that describes an extension module and everything needed to build it (hopefully in a portable @@ -84,7 +85,7 @@ class Extension(object): # When adding arguments to this constructor, be sure to update # setup_keywords in core.py. - def __init__ (self, name, sources, + def __init__(self, name, sources, include_dirs=None, define_macros=None, undef_macros=None, @@ -95,11 +96,11 @@ class Extension(object): extra_compile_args=None, extra_link_args=None, export_symbols=None, - swig_opts = None, + swig_opts=None, depends=None, language=None, optional=None, - **kw # To catch unknown keywords + **kw # To catch unknown keywords ): if not isinstance(name, str): raise AssertionError("'name' must be a string") @@ -134,4 +135,3 @@ class Extension(object): options = ', '.join(sorted(options)) msg = "Unknown Extension options: %s" % options warnings.warn(msg) - diff --git a/src/distutils2/fancy_getopt.py b/src/distutils2/fancy_getopt.py index e0ac736..19cb97b 100644 --- a/src/distutils2/fancy_getopt.py +++ b/src/distutils2/fancy_getopt.py @@ -30,6 +30,7 @@ neg_alias_re = re.compile("^(%s)=!(%s)$" % (longopt_pat, longopt_pat)) # (for use as attributes of some object). longopt_xlate = string.maketrans('-', '_') + class FancyGetopt(object): """Wrapper around the standard 'getopt()' module that provides some handy extra functionality: @@ -42,7 +43,7 @@ class FancyGetopt(object): on the command line sets 'verbose' to false """ - def __init__ (self, option_table=None): + def __init__(self, option_table=None): # The option table is (currently) a list of tuples. The # tuples may have 3 or four values: @@ -180,7 +181,8 @@ class FancyGetopt(object): self.long_opts.append(long) if long[-1] == '=': # option takes an argument? - if short: short = short + ':' + if short: + short = short + ':' long = long[0:-1] self.takes_arg[long] = 1 else: diff --git a/src/distutils2/index/__init__.py b/src/distutils2/index/__init__.py new file mode 100644 index 0000000..312662f --- /dev/null +++ b/src/distutils2/index/__init__.py @@ -0,0 +1,11 @@ +"""Package containing ways to interact with Index APIs. + +""" + +__all__ = ['simple', + 'xmlrpc', + 'dist', + 'errors', + 'mirrors',] + +from dist import ReleaseInfo, ReleasesList, DistInfo diff --git a/src/distutils2/index/base.py b/src/distutils2/index/base.py new file mode 100644 index 0000000..04582a3 --- /dev/null +++ b/src/distutils2/index/base.py @@ -0,0 +1,46 @@ +from distutils2.index.dist import ReleasesList + + +class BaseClient(object): + """Base class containing common methods for the index crawlers/clients""" + + def __init__(self, prefer_final, prefer_source): + self._prefer_final = prefer_final + self._prefer_source = prefer_source + self._index = self + + def _get_prefer_final(self, prefer_final=None): + """Return the prefer_final internal parameter or the specified one if + provided""" + if prefer_final: + return prefer_final + else: + return self._prefer_final + + def _get_prefer_source(self, prefer_source=None): + """Return the prefer_source internal parameter or the specified one if + provided""" + if prefer_source: + return prefer_source + else: + return self._prefer_source + + def _get_project(self, project_name): + """Return an project instance, create it if necessary""" + return self._projects.setdefault(project_name.lower(), + ReleasesList(project_name, index=self._index)) + + def download_distribution(self, requirements, temp_path=None, + prefer_source=None, prefer_final=None): + """Download a distribution from the last release according to the + requirements. + + If temp_path is provided, download to this path, otherwise, create a + temporary location for the download and return it. + """ + prefer_final = self._get_prefer_final(prefer_final) + prefer_source = self._get_prefer_source(prefer_source) + release = self.get_release(requirements, prefer_final) + if release: + dist = release.get_distribution(prefer_source=prefer_source) + return dist.download(temp_path) diff --git a/src/distutils2/index/dist.py b/src/distutils2/index/dist.py new file mode 100644 index 0000000..f5d0fd9 --- /dev/null +++ b/src/distutils2/index/dist.py @@ -0,0 +1,544 @@ +"""distutils2.index.dist + +Provides useful classes to represent the release and distributions retrieved +from indexes. + +A project can have several releases (=versions) and each release can have +several distributions (sdist, bdist). + +The release contains the metadata related informations (see PEP 384), and the +distributions contains download related informations. + +""" +import mimetypes +import re +import tarfile +import tempfile +import urllib +import urlparse +import zipfile + +try: + import hashlib +except ImportError: + from distutils2._backport import hashlib + +from distutils2.errors import IrrationalVersionError +from distutils2.index.errors import (HashDoesNotMatch, UnsupportedHashName, + CantParseArchiveName) +from distutils2.version import (suggest_normalized_version, NormalizedVersion, + get_version_predicate) +from distutils2.metadata import DistributionMetadata +from distutils2.util import untar_file, unzip_file, splitext + +__all__ = ['ReleaseInfo', 'DistInfo', 'ReleasesList', 'get_infos_from_url'] + +EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz .egg".split() +MD5_HASH = re.compile(r'^.*#md5=([a-f0-9]+)$') +DIST_TYPES = ['bdist', 'sdist'] + + +class IndexReference(object): + """Mixin used to store the index reference""" + def set_index(self, index=None): + self._index = index + + +class ReleaseInfo(IndexReference): + """Represent a release of a project (a project with a specific version). + The release contain the _metadata informations related to this specific + version, and is also a container for distribution related informations. + + See the DistInfo class for more information about distributions. + """ + + def __init__(self, name, version, metadata=None, hidden=False, + index=None, **kwargs): + """ + :param name: the name of the distribution + :param version: the version of the distribution + :param metadata: the metadata fields of the release. + :type metadata: dict + :param kwargs: optional arguments for a new distribution. + """ + self.set_index(index) + self.name = name + self._version = None + self.version = version + if metadata: + self.metadata = DistributionMetadata(mapping=metadata) + else: + self.metadata = None + self.dists = {} + self.hidden = hidden + + if 'dist_type' in kwargs: + dist_type = kwargs.pop('dist_type') + self.add_distribution(dist_type, **kwargs) + + def set_version(self, version): + try: + self._version = NormalizedVersion(version) + except IrrationalVersionError: + suggestion = suggest_normalized_version(version) + if suggestion: + self.version = suggestion + else: + raise IrrationalVersionError(version) + + def get_version(self): + return self._version + + version = property(get_version, set_version) + + def fetch_metadata(self): + """If the metadata is not set, use the indexes to get it""" + if not self.metadata: + self._index.get_metadata(self.name, '%s' % self.version) + return self.metadata + + @property + def is_final(self): + """proxy to version.is_final""" + return self.version.is_final + + def fetch_distributions(self): + if self.dists is None: + self._index.get_distributions(self.name, '%s' % self.version) + if self.dists is None: + self.dists = {} + return self.dists + + def add_distribution(self, dist_type='sdist', python_version=None, **params): + """Add distribution informations to this release. + If distribution information is already set for this distribution type, + add the given url paths to the distribution. This can be useful while + some of them fails to download. + + :param dist_type: the distribution type (eg. "sdist", "bdist", etc.) + :param params: the fields to be passed to the distribution object + (see the :class:DistInfo constructor). + """ + if dist_type not in DIST_TYPES: + raise ValueError(dist_type) + if dist_type in self.dists: + self.dists[dist_type].add_url(**params) + else: + self.dists[dist_type] = DistInfo(self, dist_type, + index=self._index, **params) + if python_version: + self.dists[dist_type].python_version = python_version + + def get_distribution(self, dist_type=None, prefer_source=True): + """Return a distribution. + + If dist_type is set, find first for this distribution type, and just + act as an alias of __get_item__. + + If prefer_source is True, search first for source distribution, and if + not return one existing distribution. + """ + if len(self.dists) == 0: + raise LookupError() + if dist_type: + return self[dist_type] + if prefer_source: + if "sdist" in self.dists: + dist = self["sdist"] + else: + dist = self.dists.values()[0] + return dist + + def download(self, temp_path=None, prefer_source=True): + """Download the distribution, using the requirements. + + If more than one distribution match the requirements, use the last + version. + Download the distribution, and put it in the temp_path. If no temp_path + is given, creates and return one. + + Returns the complete absolute path to the downloaded archive. + """ + return self.get_distribution(prefer_source=prefer_source)\ + .download(path=temp_path) + + def set_metadata(self, metadata): + if not self.metadata: + self.metadata = DistributionMetadata() + self.metadata.update(metadata) + + def __getitem__(self, item): + """distributions are available using release["sdist"]""" + return self.dists[item] + + def _check_is_comparable(self, other): + if not isinstance(other, ReleaseInfo): + raise TypeError("cannot compare %s and %s" + % (type(self).__name__, type(other).__name__)) + elif self.name != other.name: + raise TypeError("cannot compare %s and %s" + % (self.name, other.name)) + + def __repr__(self): + return "<%s %s>" % (self.name, self.version) + + def __eq__(self, other): + self._check_is_comparable(other) + return self.version == other.version + + def __lt__(self, other): + self._check_is_comparable(other) + return self.version < other.version + + def __ne__(self, other): + return not self.__eq__(other) + + def __gt__(self, other): + return not (self.__lt__(other) or self.__eq__(other)) + + def __le__(self, other): + return self.__eq__(other) or self.__lt__(other) + + def __ge__(self, other): + return self.__eq__(other) or self.__gt__(other) + + # See http://docs.python.org/reference/datamodel#object.__hash__ + __hash__ = object.__hash__ + + +class DistInfo(IndexReference): + """Represents a distribution retrieved from an index (sdist, bdist, ...) + """ + + def __init__(self, release, dist_type=None, url=None, hashname=None, + hashval=None, is_external=True, python_version=None, + index=None): + """Create a new instance of DistInfo. + + :param release: a DistInfo class is relative to a release. + :param dist_type: the type of the dist (eg. source, bin-*, etc.) + :param url: URL where we found this distribution + :param hashname: the name of the hash we want to use. Refer to the + hashlib.new documentation for more information. + :param hashval: the hash value. + :param is_external: we need to know if the provided url comes from + an index browsing, or from an external resource. + + """ + self.set_index(index) + self.release = release + self.dist_type = dist_type + self.python_version = python_version + self._unpacked_dir = None + # set the downloaded path to None by default. The goal here + # is to not download distributions multiple times + self.downloaded_location = None + # We store urls in dict, because we need to have a bit more infos + # than the simple URL. It will be used later to find the good url to + # use. + # We have two _url* attributes: _url and urls. urls contains a list + # of dict for the different urls, and _url contains the choosen url, in + # order to dont make the selection process multiple times. + self.urls = [] + self._url = None + self.add_url(url, hashname, hashval, is_external) + + def add_url(self, url, hashname=None, hashval=None, is_external=True): + """Add a new url to the list of urls""" + if hashname is not None: + try: + hashlib.new(hashname) + except ValueError: + raise UnsupportedHashName(hashname) + if not url in [u['url'] for u in self.urls]: + self.urls.append({ + 'url': url, + 'hashname': hashname, + 'hashval': hashval, + 'is_external': is_external, + }) + # reset the url selection process + self._url = None + + @property + def url(self): + """Pick up the right url for the list of urls in self.urls""" + # We return internal urls over externals. + # If there is more than one internal or external, return the first + # one. + if self._url is None: + if len(self.urls) > 1: + internals_urls = [u for u in self.urls \ + if u['is_external'] == False] + if len(internals_urls) >= 1: + self._url = internals_urls[0] + if self._url is None: + self._url = self.urls[0] + return self._url + + @property + def is_source(self): + """return if the distribution is a source one or not""" + return self.dist_type == 'sdist' + + def download(self, path=None): + """Download the distribution to a path, and return it. + + If the path is given in path, use this, otherwise, generates a new one + Return the download location. + """ + if path is None: + path = tempfile.mkdtemp() + + # if we do not have downloaded it yet, do it. + if self.downloaded_location is None: + url = self.url['url'] + archive_name = urlparse.urlparse(url)[2].split('/')[-1] + filename, headers = urllib.urlretrieve(url, + path + "/" + archive_name) + self.downloaded_location = filename + self._check_md5(filename) + return self.downloaded_location + + def unpack(self, path=None): + """Unpack the distribution to the given path. + + If not destination is given, creates a temporary location. + + Returns the location of the extracted files (root). + """ + if not self._unpacked_dir: + if path is None: + path = tempfile.mkdtemp() + + filename = self.download() + content_type = mimetypes.guess_type(filename)[0] + + if (content_type == 'application/zip' + or filename.endswith('.zip') + or filename.endswith('.pybundle') + or zipfile.is_zipfile(filename)): + unzip_file(filename, path, flatten=not filename.endswith('.pybundle')) + elif (content_type == 'application/x-gzip' + or tarfile.is_tarfile(filename) + or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')): + untar_file(filename, path) + self._unpacked_dir = path + return self._unpacked_dir + + def _check_md5(self, filename): + """Check that the md5 checksum of the given file matches the one in + url param""" + hashname = self.url['hashname'] + expected_hashval = self.url['hashval'] + if not None in (expected_hashval, hashname): + f = open(filename) + hashval = hashlib.new(hashname) + hashval.update(f.read()) + if hashval.hexdigest() != expected_hashval: + raise HashDoesNotMatch("got %s instead of %s" + % (hashval.hexdigest(), expected_hashval)) + + def __repr__(self): + return "<%s %s %s>" % ( + self.release.name, self.release.version, self.dist_type or "") + + +class ReleasesList(IndexReference): + """A container of Release. + + Provides useful methods and facilities to sort and filter releases. + """ + def __init__(self, name, releases=None, contains_hidden=False, index=None): + self.set_index(index) + self.releases = [] + self.name = name + self.contains_hidden = contains_hidden + if releases: + self.add_releases(releases) + + def fetch_releases(self): + self._index.get_releases(self.name) + return self.releases + + def filter(self, predicate): + """Filter and return a subset of releases matching the given predicate. + """ + return ReleasesList(self.name, [release for release in self.releases + if predicate.match(release.version)], + index=self._index) + + def get_last(self, requirements, prefer_final=None): + """Return the "last" release, that satisfy the given predicates. + + "last" is defined by the version number of the releases, you also could + set prefer_final parameter to True or False to change the order results + """ + predicate = get_version_predicate(requirements) + releases = self.filter(predicate) + releases.sort_releases(prefer_final, reverse=True) + return releases[0] + + def add_releases(self, releases): + """Add releases in the release list. + + :param: releases is a list of ReleaseInfo objects. + """ + for r in releases: + self.add_release(release=r) + + def add_release(self, version=None, dist_type='sdist', release=None, + **dist_args): + """Add a release to the list. + + The release can be passed in the `release` parameter, and in this case, + it will be crawled to extract the useful informations if necessary, or + the release informations can be directly passed in the `version` and + `dist_type` arguments. + + Other keywords arguments can be provided, and will be forwarded to the + distribution creation (eg. the arguments of the DistInfo constructor). + """ + if release: + if release.name.lower() != self.name.lower(): + raise ValueError("%s is not the same project than %s" % + (release.name, self.name)) + version = '%s' % release.version + + if not version in self.get_versions(): + # append only if not already exists + self.releases.append(release) + for dist in release.dists.values(): + for url in dist.urls: + self.add_release(version, dist.dist_type, **url) + else: + matches = [r for r in self.releases if '%s' % r.version == version + and r.name == self.name] + if not matches: + release = ReleaseInfo(self.name, version, index=self._index) + self.releases.append(release) + else: + release = matches[0] + + release.add_distribution(dist_type=dist_type, **dist_args) + + def sort_releases(self, prefer_final=False, reverse=True, *args, **kwargs): + """Sort the results with the given properties. + + The `prefer_final` argument can be used to specify if final + distributions (eg. not dev, bet or alpha) would be prefered or not. + + Results can be inverted by using `reverse`. + + Any other parameter provided will be forwarded to the sorted call. You + cannot redefine the key argument of "sorted" here, as it is used + internally to sort the releases. + """ + + sort_by = [] + if prefer_final: + sort_by.append("is_final") + sort_by.append("version") + + self.releases.sort( + key=lambda i: [getattr(i, arg) for arg in sort_by], + reverse=reverse, *args, **kwargs) + + def get_release(self, version): + """Return a release from it's version. + """ + matches = [r for r in self.releases if "%s" % r.version == version] + if len(matches) != 1: + raise KeyError(version) + return matches[0] + + def get_versions(self): + """Return a list of releases versions contained""" + return ["%s" % r.version for r in self.releases] + + def __getitem__(self, key): + return self.releases[key] + + def __len__(self): + return len(self.releases) + + def __repr__(self): + string = 'Project "%s"' % self.name + if self.get_versions(): + string += ' versions: %s' % ', '.join(self.get_versions()) + return '<%s>' % string + + +def get_infos_from_url(url, probable_dist_name=None, is_external=True): + """Get useful informations from an URL. + + Return a dict of (name, version, url, hashtype, hash, is_external) + + :param url: complete url of the distribution + :param probable_dist_name: A probable name of the project. + :param is_external: Tell if the url commes from an index or from + an external URL. + """ + # if the url contains a md5 hash, get it. + md5_hash = None + match = MD5_HASH.match(url) + if match is not None: + md5_hash = match.group(1) + # remove the hash + url = url.replace("#md5=%s" % md5_hash, "") + + # parse the archive name to find dist name and version + archive_name = urlparse.urlparse(url)[2].split('/')[-1] + extension_matched = False + # remove the extension from the name + for ext in EXTENSIONS: + if archive_name.endswith(ext): + archive_name = archive_name[:-len(ext)] + extension_matched = True + + name, version = split_archive_name(archive_name) + if extension_matched is True: + return {'name': name, + 'version': version, + 'url': url, + 'hashname': "md5", + 'hashval': md5_hash, + 'is_external': is_external, + 'dist_type': 'sdist'} + + +def split_archive_name(archive_name, probable_name=None): + """Split an archive name into two parts: name and version. + + Return the tuple (name, version) + """ + # Try to determine wich part is the name and wich is the version using the + # "-" separator. Take the larger part to be the version number then reduce + # if this not works. + def eager_split(str, maxsplit=2): + # split using the "-" separator + splits = str.rsplit("-", maxsplit) + name = splits[0] + version = "-".join(splits[1:]) + if version.startswith("-"): + version = version[1:] + if suggest_normalized_version(version) is None and maxsplit >= 0: + # we dont get a good version number: recurse ! + return eager_split(str, maxsplit - 1) + else: + return (name, version) + if probable_name is not None: + probable_name = probable_name.lower() + name = None + if probable_name is not None and probable_name in archive_name: + # we get the name from probable_name, if given. + name = probable_name + version = archive_name.lstrip(name) + else: + name, version = eager_split(archive_name) + + version = suggest_normalized_version(version) + if version is not None and name != "": + return (name.lower(), version) + else: + raise CantParseArchiveName(archive_name) diff --git a/src/distutils2/index/errors.py b/src/distutils2/index/errors.py new file mode 100644 index 0000000..730ccce --- /dev/null +++ b/src/distutils2/index/errors.py @@ -0,0 +1,45 @@ +"""distutils2.pypi.errors + +All errors and exceptions raised by PyPiIndex classes. +""" +from distutils2.errors import DistutilsError + + +class IndexesError(DistutilsError): + """The base class for errors of the index python package.""" + + +class ProjectNotFound(IndexesError): + """Project has not been found""" + + +class DistributionNotFound(IndexesError): + """The release has not been found""" + + +class ReleaseNotFound(IndexesError): + """The release has not been found""" + + +class CantParseArchiveName(IndexesError): + """An archive name can't be parsed to find distribution name and version""" + + +class DownloadError(IndexesError): + """An error has occurs while downloading""" + + +class HashDoesNotMatch(DownloadError): + """Compared hashes does not match""" + + +class UnsupportedHashName(IndexesError): + """A unsupported hashname has been used""" + + +class UnableToDownload(IndexesError): + """All mirrors have been tried, without success""" + + +class InvalidSearchField(IndexesError): + """An invalid search field has been used""" diff --git a/src/distutils2/index/mirrors.py b/src/distutils2/index/mirrors.py new file mode 100644 index 0000000..49d5dd1 --- /dev/null +++ b/src/distutils2/index/mirrors.py @@ -0,0 +1,52 @@ +"""Utilities related to the mirror infrastructure defined in PEP 381. +See http://www.python.org/dev/peps/pep-0381/ +""" + +from string import ascii_lowercase +import socket + +DEFAULT_MIRROR_URL = "last.pypi.python.org" + +def get_mirrors(hostname=None): + """Return the list of mirrors from the last record found on the DNS + entry:: + + >>> from distutils2.index.mirrors import get_mirrors + >>> get_mirrors() + ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org', + 'd.pypi.python.org'] + + """ + if hostname is None: + hostname = DEFAULT_MIRROR_URL + + # return the last mirror registered on PyPI. + try: + hostname = socket.gethostbyname_ex(hostname)[0] + except socket.gaierror: + return [] + end_letter = hostname.split(".", 1) + + # determine the list from the last one. + return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])] + +def string_range(last): + """Compute the range of string between "a" and last. + + This works for simple "a to z" lists, but also for "a to zz" lists. + """ + for k in range(len(last)): + for x in product(ascii_lowercase, repeat=k+1): + result = ''.join(x) + yield result + if result == last: + return + +def product(*args, **kwds): + pools = map(tuple, args) * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x+[y] for x in result for y in pool] + for prod in result: + yield tuple(prod) + diff --git a/src/distutils2/index/simple.py b/src/distutils2/index/simple.py new file mode 100644 index 0000000..cdb19c6 --- /dev/null +++ b/src/distutils2/index/simple.py @@ -0,0 +1,434 @@ +"""index.simple + +Contains the class "SimpleIndexCrawler", a simple spider to find and retrieve +distributions on the Python Package Index, using it's "simple" API, +avalaible at http://pypi.python.org/simple/ +""" +from fnmatch import translate +import httplib +import re +import socket +import sys +import urllib2 +import urlparse +import logging +import os + +from distutils2.index.base import BaseClient +from distutils2.index.dist import (ReleasesList, EXTENSIONS, + get_infos_from_url, MD5_HASH) +from distutils2.index.errors import (IndexesError, DownloadError, + UnableToDownload, CantParseArchiveName, + ReleaseNotFound, ProjectNotFound) +from distutils2.index.mirrors import get_mirrors +from distutils2.metadata import DistributionMetadata +from distutils2.version import get_version_predicate +from distutils2 import __version__ as __distutils2_version__ + +__all__ = ['Crawler', 'DEFAULT_SIMPLE_INDEX_URL'] + +# -- Constants ----------------------------------------------- +DEFAULT_SIMPLE_INDEX_URL = "http://a.pypi.python.org/simple/" +DEFAULT_HOSTS = ("*",) +SOCKET_TIMEOUT = 15 +USER_AGENT = "Python-urllib/%s distutils2/%s" % ( + sys.version[:3], __distutils2_version__) + +# -- Regexps ------------------------------------------------- +EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') +HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) +URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match + +# This pattern matches a character entity reference (a decimal numeric +# references, a hexadecimal numeric reference, or a named reference). +ENTITY_SUB = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub +REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) + + +def socket_timeout(timeout=SOCKET_TIMEOUT): + """Decorator to add a socket timeout when requesting pages on PyPI. + """ + def _socket_timeout(func): + def _socket_timeout(self, *args, **kwargs): + old_timeout = socket.getdefaulttimeout() + if hasattr(self, "_timeout"): + timeout = self._timeout + socket.setdefaulttimeout(timeout) + try: + return func(self, *args, **kwargs) + finally: + socket.setdefaulttimeout(old_timeout) + return _socket_timeout + return _socket_timeout + + +def with_mirror_support(): + """Decorator that makes the mirroring support easier""" + def wrapper(func): + def wrapped(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except DownloadError: + # if an error occurs, try with the next index_url + if self._mirrors_tries >= self._mirrors_max_tries: + try: + self._switch_to_next_mirror() + except KeyError: + raise UnableToDownload("Tried all mirrors") + else: + self._mirrors_tries += 1 + self._projects.clear() + return wrapped(self, *args, **kwargs) + return wrapped + return wrapper + + +class Crawler(BaseClient): + """Provides useful tools to request the Python Package Index simple API. + + You can specify both mirrors and mirrors_url, but mirrors_url will only be + used if mirrors is set to None. + + :param index_url: the url of the simple index to search on. + :param prefer_final: if the version is not mentioned, and the last + version is not a "final" one (alpha, beta, etc.), + pick up the last final version. + :param prefer_source: if the distribution type is not mentioned, pick up + the source one if available. + :param follow_externals: tell if following external links is needed or + not. Default is False. + :param hosts: a list of hosts allowed to be processed while using + follow_externals=True. Default behavior is to follow all + hosts. + :param follow_externals: tell if following external links is needed or + not. Default is False. + :param mirrors_url: the url to look on for DNS records giving mirror + adresses. + :param mirrors: a list of mirrors (see PEP 381). + :param timeout: time in seconds to consider a url has timeouted. + :param mirrors_max_tries": number of times to try requesting informations + on mirrors before switching. + """ + + def __init__(self, index_url=DEFAULT_SIMPLE_INDEX_URL, prefer_final=False, + prefer_source=True, hosts=DEFAULT_HOSTS, + follow_externals=False, mirrors_url=None, mirrors=None, + timeout=SOCKET_TIMEOUT, mirrors_max_tries=0): + super(Crawler, self).__init__(prefer_final, prefer_source) + self.follow_externals = follow_externals + + # mirroring attributes. + if not index_url.endswith("/"): + index_url += "/" + # if no mirrors are defined, use the method described in PEP 381. + if mirrors is None: + mirrors = get_mirrors(mirrors_url) + self._mirrors = set(mirrors) + self._mirrors_used = set() + self.index_url = index_url + self._mirrors_max_tries = mirrors_max_tries + self._mirrors_tries = 0 + self._timeout = timeout + + # create a regexp to match all given hosts + self._allowed_hosts = re.compile('|'.join(map(translate, hosts))).match + + # we keep an index of pages we have processed, in order to avoid + # scanning them multple time (eg. if there is multiple pages pointing + # on one) + self._processed_urls = [] + self._projects = {} + + @with_mirror_support() + def search_projects(self, name=None, **kwargs): + """Search the index for projects containing the given name. + + Return a list of names. + """ + index = self._open_url(self.index_url) + projectname = re.compile("""<a[^>]*>(.?[^<]*%s.?[^<]*)</a>""" % name, + flags=re.I) + matching_projects = [] + for match in projectname.finditer(index.read()): + project_name = match.group(1) + matching_projects.append(self._get_project(project_name)) + return matching_projects + + def get_releases(self, requirements, prefer_final=None, + force_update=False): + """Search for releases and return a ReleaseList object containing + the results. + """ + predicate = get_version_predicate(requirements) + if self._projects.has_key(predicate.name.lower()) and not force_update: + return self._projects.get(predicate.name.lower()) + prefer_final = self._get_prefer_final(prefer_final) + self._process_index_page(predicate.name) + + if not self._projects.has_key(predicate.name.lower()): + raise ProjectNotFound() + + releases = self._projects.get(predicate.name.lower()) + releases.sort_releases(prefer_final=prefer_final) + return releases + + def get_release(self, requirements, prefer_final=None): + """Return only one release that fulfill the given requirements""" + predicate = get_version_predicate(requirements) + release = self.get_releases(predicate, prefer_final)\ + .get_last(predicate) + if not release: + raise ReleaseNotFound("No release matches the given criterias") + return release + + def get_distributions(self, project_name, version): + """Return the distributions found on the index for the specific given + release""" + # as the default behavior of get_release is to return a release + # containing the distributions, just alias it. + return self.get_release("%s (%s)" % (project_name, version)) + + def get_metadata(self, project_name, version): + """Return the metadatas from the simple index. + + Currently, download one archive, extract it and use the PKG-INFO file. + """ + release = self.get_distributions(project_name, version) + if not release._metadata: + location = release.get_distribution().unpack() + pkg_info = os.path.join(location, 'PKG-INFO') + release._metadata = DistributionMetadata(pkg_info) + return release + + def _switch_to_next_mirror(self): + """Switch to the next mirror (eg. point self.index_url to the next + mirror url. + + Raise a KeyError if all mirrors have been tried. + """ + self._mirrors_used.add(self.index_url) + index_url = self._mirrors.pop() + if not ("http://" or "https://" or "file://") in index_url: + index_url = "http://%s" % index_url + + if not index_url.endswith("/simple"): + index_url = "%s/simple/" % index_url + + self.index_url = index_url + + def _is_browsable(self, url): + """Tell if the given URL can be browsed or not. + + It uses the follow_externals and the hosts list to tell if the given + url is browsable or not. + """ + # if _index_url is contained in the given URL, we are browsing the + # index, and it's always "browsable". + # local files are always considered browable resources + if self.index_url in url or urlparse.urlparse(url)[0] == "file": + return True + elif self.follow_externals: + if self._allowed_hosts(urlparse.urlparse(url)[1]): # 1 is netloc + return True + else: + return False + return False + + def _is_distribution(self, link): + """Tell if the given URL matches to a distribution name or not. + """ + #XXX find a better way to check that links are distributions + # Using a regexp ? + for ext in EXTENSIONS: + if ext in link: + return True + return False + + def _register_release(self, release=None, release_info={}): + """Register a new release. + + Both a release or a dict of release_info can be provided, the prefered + way (eg. the quicker) is the dict one. + + Return the list of existing releases for the given project. + """ + # Check if the project already has a list of releases (refering to + # the project name). If not, create a new release list. + # Then, add the release to the list. + if release: + name = release.name + else: + name = release_info['name'] + if not name.lower() in self._projects: + self._projects[name.lower()] = ReleasesList(name, + index=self._index) + + if release: + self._projects[name.lower()].add_release(release=release) + else: + name = release_info.pop('name') + version = release_info.pop('version') + dist_type = release_info.pop('dist_type') + self._projects[name.lower()].add_release(version, dist_type, + **release_info) + return self._projects[name.lower()] + + def _process_url(self, url, project_name=None, follow_links=True): + """Process an url and search for distributions packages. + + For each URL found, if it's a download, creates a PyPIdistribution + object. If it's a homepage and we can follow links, process it too. + + :param url: the url to process + :param project_name: the project name we are searching for. + :param follow_links: Do not want to follow links more than from one + level. This parameter tells if we want to follow + the links we find (eg. run recursively this + method on it) + """ + f = self._open_url(url) + base_url = f.url + if url not in self._processed_urls: + self._processed_urls.append(url) + link_matcher = self._get_link_matcher(url) + for link, is_download in link_matcher(f.read(), base_url): + if link not in self._processed_urls: + if self._is_distribution(link) or is_download: + self._processed_urls.append(link) + # it's a distribution, so create a dist object + try: + infos = get_infos_from_url(link, project_name, + is_external=not self.index_url in url) + except CantParseArchiveName, e: + logging.warning("version has not been parsed: %s" + % e) + else: + self._register_release(release_info=infos) + else: + if self._is_browsable(link) and follow_links: + self._process_url(link, project_name, + follow_links=False) + + def _get_link_matcher(self, url): + """Returns the right link matcher function of the given url + """ + if self.index_url in url: + return self._simple_link_matcher + else: + return self._default_link_matcher + + def _get_full_url(self, url, base_url): + return urlparse.urljoin(base_url, self._htmldecode(url)) + + def _simple_link_matcher(self, content, base_url): + """Yield all links with a rel="download" or rel="homepage". + + This matches the simple index requirements for matching links. + If follow_externals is set to False, dont yeld the external + urls. + """ + for match in HREF.finditer(content): + url = self._get_full_url(match.group(1), base_url) + if MD5_HASH.match(url): + yield (url, True) + + for match in REL.finditer(content): + # search for rel links. + tag, rel = match.groups() + rels = map(str.strip, rel.lower().split(',')) + if 'homepage' in rels or 'download' in rels: + for match in HREF.finditer(tag): + url = self._get_full_url(match.group(1), base_url) + if 'download' in rels or self._is_browsable(url): + # yield a list of (url, is_download) + yield (url, 'download' in rels) + + def _default_link_matcher(self, content, base_url): + """Yield all links found on the page. + """ + for match in HREF.finditer(content): + url = self._get_full_url(match.group(1), base_url) + if self._is_browsable(url): + yield (url, False) + + @with_mirror_support() + def _process_index_page(self, name): + """Find and process a PyPI page for the given project name. + + :param name: the name of the project to find the page + """ + # Browse and index the content of the given PyPI page. + url = self.index_url + name + "/" + self._process_url(url, name) + + @socket_timeout() + def _open_url(self, url): + """Open a urllib2 request, handling HTTP authentication, and local + files support. + + """ + scheme, netloc, path, params, query, frag = urlparse.urlparse(url) + + # authentication stuff + if scheme in ('http', 'https'): + auth, host = urllib2.splituser(netloc) + else: + auth = None + + # add index.html automatically for filesystem paths + if scheme == 'file': + if url.endswith('/'): + url += "index.html" + + # add authorization headers if auth is provided + if auth: + auth = "Basic " + \ + urllib2.unquote(auth).encode('base64').strip() + new_url = urlparse.urlunparse(( + scheme, host, path, params, query, frag)) + request = urllib2.Request(new_url) + request.add_header("Authorization", auth) + else: + request = urllib2.Request(url) + request.add_header('User-Agent', USER_AGENT) + try: + fp = urllib2.urlopen(request) + except (ValueError, httplib.InvalidURL), v: + msg = ' '.join([str(arg) for arg in v.args]) + raise IndexesError('%s %s' % (url, msg)) + except urllib2.HTTPError, v: + return v + except urllib2.URLError, v: + raise DownloadError("Download error for %s: %s" % (url, v.reason)) + except httplib.BadStatusLine, v: + raise DownloadError('%s returned a bad status line. ' + 'The server might be down, %s' % (url, v.line)) + except httplib.HTTPException, v: + raise DownloadError("Download error for %s: %s" % (url, v)) + except socket.timeout: + raise DownloadError("The server timeouted") + + if auth: + # Put authentication info back into request URL if same host, + # so that links found on the page will work + s2, h2, path2, param2, query2, frag2 = \ + urlparse.urlparse(fp.url) + if s2 == scheme and h2 == host: + fp.url = urlparse.urlunparse( + (s2, netloc, path2, param2, query2, frag2)) + return fp + + def _decode_entity(self, match): + what = match.group(1) + if what.startswith('#x'): + what = int(what[2:], 16) + elif what.startswith('#'): + what = int(what[1:]) + else: + from htmlentitydefs import name2codepoint + what = name2codepoint.get(what, match.group(0)) + return unichr(what) + + def _htmldecode(self, text): + """Decode HTML entities in the given text.""" + return ENTITY_SUB(self._decode_entity, text) diff --git a/src/distutils2/index/wrapper.py b/src/distutils2/index/wrapper.py new file mode 100644 index 0000000..e3c4551 --- /dev/null +++ b/src/distutils2/index/wrapper.py @@ -0,0 +1,93 @@ +import xmlrpc +import simple + +_WRAPPER_MAPPINGS = {'get_release': 'simple', + 'get_releases': 'simple', + 'search_projects': 'simple', + 'get_metadata': 'xmlrpc', + 'get_distributions': 'simple'} + +_WRAPPER_INDEXES = {'xmlrpc': xmlrpc.Client, + 'simple': simple.Crawler} + +def switch_index_if_fails(func, wrapper): + """Decorator that switch of index (for instance from xmlrpc to simple) + if the first mirror return an empty list or raises an exception. + """ + def decorator(*args, **kwargs): + retry = True + exception = None + methods = [func] + for f in wrapper._indexes.values(): + if f != func.im_self and hasattr(f, func.__name__): + methods.append(getattr(f, func.__name__)) + for method in methods: + try: + response = method(*args, **kwargs) + retry = False + except Exception, e: + exception = e + if not retry: + break + if retry and exception: + raise exception + else: + return response + return decorator + + +class ClientWrapper(object): + """Wrapper around simple and xmlrpc clients, + + Choose the best implementation to use depending the needs, using the given + mappings. + If one of the indexes returns an error, tries to use others indexes. + + :param index: tell wich index to rely on by default. + :param index_classes: a dict of name:class to use as indexes. + :param indexes: a dict of name:index already instantiated + :param mappings: the mappings to use for this wrapper + """ + + def __init__(self, default_index='simple', index_classes=_WRAPPER_INDEXES, + indexes={}, mappings=_WRAPPER_MAPPINGS): + self._projects = {} + self._mappings = mappings + self._indexes = indexes + self._default_index = default_index + + # instantiate the classes and set their _project attribute to the one + # of the wrapper. + for name, cls in index_classes.items(): + obj = self._indexes.setdefault(name, cls()) + obj._projects = self._projects + obj._index = self + + def __getattr__(self, method_name): + """When asking for methods of the wrapper, return the implementation of + the wrapped classes, depending the mapping. + + Decorate the methods to switch of implementation if an error occurs + """ + real_method = None + if method_name in _WRAPPER_MAPPINGS: + obj = self._indexes[_WRAPPER_MAPPINGS[method_name]] + real_method = getattr(obj, method_name) + else: + # the method is not defined in the mappings, so we try first to get + # it via the default index, and rely on others if needed. + try: + real_method = getattr(self._indexes[self._default_index], + method_name) + except AttributeError: + other_indexes = [i for i in self._indexes + if i != self._default_index] + for index in other_indexes: + real_method = getattr(self._indexes[index], method_name, None) + if real_method: + break + if real_method: + return switch_index_if_fails(real_method, self) + else: + raise AttributeError("No index have attribute '%s'" % method_name) + diff --git a/src/distutils2/index/xmlrpc.py b/src/distutils2/index/xmlrpc.py new file mode 100644 index 0000000..b9d66e0 --- /dev/null +++ b/src/distutils2/index/xmlrpc.py @@ -0,0 +1,179 @@ +import logging +import xmlrpclib + +from distutils2.errors import IrrationalVersionError +from distutils2.index.base import BaseClient +from distutils2.index.errors import (ProjectNotFound, InvalidSearchField, + ReleaseNotFound) +from distutils2.index.dist import ReleaseInfo +from distutils2.version import get_version_predicate + +__all__ = ['Client', 'DEFAULT_XMLRPC_INDEX_URL'] + +DEFAULT_XMLRPC_INDEX_URL = 'http://python.org/pypi' + +_SEARCH_FIELDS = ['name', 'version', 'author', 'author_email', 'maintainer', + 'maintainer_email', 'home_page', 'license', 'summary', + 'description', 'keywords', 'platform', 'download_url'] + + +class Client(BaseClient): + """Client to query indexes using XML-RPC method calls. + + If no server_url is specified, use the default PyPI XML-RPC URL, + defined in the DEFAULT_XMLRPC_INDEX_URL constant:: + + >>> client = XMLRPCClient() + >>> client.server_url == DEFAULT_XMLRPC_INDEX_URL + True + + >>> client = XMLRPCClient("http://someurl/") + >>> client.server_url + 'http://someurl/' + """ + + def __init__(self, server_url=DEFAULT_XMLRPC_INDEX_URL, prefer_final=False, + prefer_source=True): + super(Client, self).__init__(prefer_final, prefer_source) + self.server_url = server_url + self._projects = {} + + def get_release(self, requirements, prefer_final=False): + """Return a release with all complete metadata and distribution + related informations. + """ + prefer_final = self._get_prefer_final(prefer_final) + predicate = get_version_predicate(requirements) + releases = self.get_releases(predicate.name) + release = releases.get_last(predicate, prefer_final) + self.get_metadata(release.name, "%s" % release.version) + self.get_distributions(release.name, "%s" % release.version) + return release + + def get_releases(self, requirements, prefer_final=None, show_hidden=True, + force_update=False): + """Return the list of existing releases for a specific project. + + Cache the results from one call to another. + + If show_hidden is True, return the hidden releases too. + If force_update is True, reprocess the index to update the + informations (eg. make a new XML-RPC call). + :: + + >>> client = XMLRPCClient() + >>> client.get_releases('Foo') + ['1.1', '1.2', '1.3'] + + If no such project exists, raise a ProjectNotFound exception:: + + >>> client.get_project_versions('UnexistingProject') + ProjectNotFound: UnexistingProject + + """ + def get_versions(project_name, show_hidden): + return self.proxy.package_releases(project_name, show_hidden) + + predicate = get_version_predicate(requirements) + prefer_final = self._get_prefer_final(prefer_final) + project_name = predicate.name + if not force_update and (project_name.lower() in self._projects): + project = self._projects[project_name.lower()] + if not project.contains_hidden and show_hidden: + # if hidden releases are requested, and have an existing + # list of releases that does not contains hidden ones + all_versions = get_versions(project_name, show_hidden) + existing_versions = project.get_versions() + hidden_versions = list(set(all_versions) - + set(existing_versions)) + for version in hidden_versions: + project.add_release(release=ReleaseInfo(project_name, + version, index=self._index)) + else: + versions = get_versions(project_name, show_hidden) + if not versions: + raise ProjectNotFound(project_name) + project = self._get_project(project_name) + project.add_releases([ReleaseInfo(project_name, version, + index=self._index) + for version in versions]) + project = project.filter(predicate) + if len(project) == 0: + raise ReleaseNotFound("%s" % predicate) + project.sort_releases(prefer_final) + return project + + + def get_distributions(self, project_name, version): + """Grab informations about distributions from XML-RPC. + + Return a ReleaseInfo object, with distribution-related informations + filled in. + """ + url_infos = self.proxy.release_urls(project_name, version) + project = self._get_project(project_name) + if version not in project.get_versions(): + project.add_release(release=ReleaseInfo(project_name, version, + index=self._index)) + release = project.get_release(version) + for info in url_infos: + packagetype = info['packagetype'] + dist_infos = {'url': info['url'], + 'hashval': info['md5_digest'], + 'hashname': 'md5', + 'is_external': False, + 'python_version': info['python_version']} + release.add_distribution(packagetype, **dist_infos) + return release + + def get_metadata(self, project_name, version): + """Retreive project metadatas. + + Return a ReleaseInfo object, with metadata informations filled in. + """ + metadata = self.proxy.release_data(project_name, version) + project = self._get_project(project_name) + if version not in project.get_versions(): + project.add_release(release=ReleaseInfo(project_name, version, + index=self._index)) + release = project.get_release(version) + release.set_metadata(metadata) + return release + + def search_projects(self, name=None, operator="or", **kwargs): + """Find using the keys provided in kwargs. + + You can set operator to "and" or "or". + """ + for key in kwargs: + if key not in _SEARCH_FIELDS: + raise InvalidSearchField(key) + if name: + kwargs["name"] = name + projects = self.proxy.search(kwargs, operator) + for p in projects: + project = self._get_project(p['name']) + try: + project.add_release(release=ReleaseInfo(p['name'], + p['version'], metadata={'summary': p['summary']}, + index=self._index)) + except IrrationalVersionError, e: + logging.warn("Irrational version error found: %s" % e) + + return [self._projects[p['name'].lower()] for p in projects] + + @property + def proxy(self): + """Property used to return the XMLRPC server proxy. + + If no server proxy is defined yet, creates a new one:: + + >>> client = XmlRpcClient() + >>> client.proxy() + <ServerProxy for python.org/pypi> + + """ + if not hasattr(self, '_server_proxy'): + self._server_proxy = xmlrpclib.ServerProxy(self.server_url) + + return self._server_proxy diff --git a/src/distutils2/install_with_deps.py b/src/distutils2/install_with_deps.py new file mode 100644 index 0000000..6951ff9 --- /dev/null +++ b/src/distutils2/install_with_deps.py @@ -0,0 +1,171 @@ +import logging +from distutils2.index import wrapper +from distutils2.index.errors import ProjectNotFound, ReleaseNotFound +from distutils2.depgraph import generate_graph +from distutils2._backport.pkgutil import get_distributions + + +"""Provides installations scripts. + +The goal of this script is to install a release from the indexes (eg. +PyPI), including the dependencies of the releases if needed. + +It uses the work made in pkgutil and by the index crawlers to browse the +installed distributions, and rely on the instalation commands to install. +""" + + +def get_deps(requirements, index): + """Return the dependencies of a project, as a depgraph object. + + Build a :class depgraph.DependencyGraph: for the given requirements + + If the project does not uses Metadata < 1.1, dependencies can't be handled + from here, so it returns an empty list of dependencies. + + :param requirements: is a string containing the version predicate to take + the project name and version specifier from. + :param index: the index to use for making searches. + """ + deps = [] + release = index.get_release(requirements) + requires = release.metadata['Requires-Dist'] + release.metadata['Requires'] + deps.append(release) # include the release we are computing deps. + for req in requires: + deps.extend(get_deps(req, index)) + return deps + + +def install(requirements, index=None, interactive=True, upgrade=True, + prefer_source=True, prefer_final=True): + """Given a list of distributions to install, a list of distributions to + remove, and a list of conflicts, proceed and do what's needed to be done. + + :param requirements: is a *string* containing the requirements for this + project (for instance "FooBar 1.1" or "BarBaz (<1.2) + :param index: If an index is specified, use this one, otherwise, use + :class index.ClientWrapper: to get project metadatas. + :param interactive: if set to True, will prompt the user for interactions + of needed. If false, use the default values. + :param upgrade: If a project exists in a newer version, does the script + need to install the new one, or keep the already installed + version. + :param prefer_source: used to tell if the user prefer source distributions + over built dists. + :param prefer_final: if set to true, pick up the "final" versions (eg. + stable) over the beta, alpha (not final) ones. + """ + # get the default index if none is specified + if not index: + index = wrapper.WrapperClient() + + # check if the project is already installed. + installed_release = get_installed_release(requirements) + + # if a version that satisfy the requirements is already installed + if installed_release and (interactive or upgrade): + new_releases = index.get_releases(requirements) + if (new_releases.get_last(requirements).version > + installed_release.version): + if interactive: + # prompt the user to install the last version of the package. + # set upgrade here. + print "You want to install a package already installed on your" + "system. A new version exists, you could just use the version" + "you have, or upgrade to the latest version" + + upgrade = raw_input("Do you want to install the most recent one ? (Y/n)") or "Y" + if upgrade in ('Y', 'y'): + upgrade = True + else: + upgrade = False + if not upgrade: + return + + # create the depgraph from the dependencies of the release we want to + # install + graph = generate_graph(get_deps(requirements, index)) + from ipdb import set_trace + set_trace() + installed = [] # to uninstall on errors + try: + for release in graph.adjacency_list: + dist = release.get_distribution() + install(dist) + installed.append(dist) + print "%s have been installed on your system" % requirements + except: + print "an error has occured, uninstalling" + for dist in installed: + uninstall_dist(dist) + +class InstallationException(Exception): + pass + +def get_install_info(requirements, index=None, already_installed=None): + """Return the informations on what's going to be installed and upgraded. + + :param requirements: is a *string* containing the requirements for this + project (for instance "FooBar 1.1" or "BarBaz (<1.2)") + :param index: If an index is specified, use this one, otherwise, use + :class index.ClientWrapper: to get project metadatas. + :param already_installed: a list of already installed distributions. + + The results are returned in a dict. For instance:: + + >>> get_install_info("FooBar (<=1.2)") + {'install': [<FooBar 1.1>], 'remove': [], 'conflict': []} + + Conflict contains all the conflicting distributions, if there is a + conflict. + + """ + def update_infos(new_infos, infos): + for key, value in infos.items(): + if key in new_infos: + infos[key].extend(new_infos[key]) + return new_infos + + if not index: + index = wrapper.ClientWrapper() + logging.info("searching releases for %s" % requirements) + + # 1. get all the releases that match the requirements + try: + releases = index.get_releases(requirements) + except (ReleaseNotFound, ProjectNotFound), e: + raise InstallationException('Release not found: "%s"' % requirements) + + # 2. pick up a release, and try to get the dependency tree + release = releases.get_last(requirements) + metadata = release.fetch_metadata() + + # 3. get the distributions already_installed on the system + # 4. and add the one we want to install + if not already_installed: + already_installed = get_distributions() + + logging.info("fetching %s %s dependencies" % ( + release.name, release.version)) + distributions = already_installed + [release] + depgraph = generate_graph(distributions) + + # store all the already_installed packages in a list, in case of rollback. + infos = {'install':[], 'remove': [], 'conflict': []} + + # 5. get what the missing deps are + for dists in depgraph.missing.values(): + if dists: + logging.info("missing dependencies found, installing them") + # we have missing deps + for dist in dists: + update_infos(get_install_info(dist, index, already_installed), + infos) + + # 6. fill in the infos + existing = [d for d in already_installed if d.name == release.name] + if existing: + infos['remove'].append(existing[0]) + infos['conflict'].extend(depgraph.reverse_list[existing[0]]) + infos['install'].append(release) + return infos diff --git a/src/distutils2/manifest.py b/src/distutils2/manifest.py index 87634c9..0d838ae 100644 --- a/src/distutils2/manifest.py +++ b/src/distutils2/manifest.py @@ -22,7 +22,7 @@ __all__ = ['Manifest'] # a \ followed by some spaces + EOL _COLLAPSE_PATTERN = re.compile('\\\w*\n', re.M) -_COMMENTED_LINE = re.compile('#.*?(?=\n)|^\w*\n|\n(?=$)', re.M|re.S) +_COMMENTED_LINE = re.compile('#.*?(?=\n)|^\w*\n|\n(?=$)', re.M | re.S) class Manifest(object): """A list of files built by on exploring the filesystem and filtered by diff --git a/src/distutils2/metadata.py b/src/distutils2/metadata.py index 55f1020..ed80fac 100644 --- a/src/distutils2/metadata.py +++ b/src/distutils2/metadata.py @@ -1,13 +1,12 @@ -""" -Implementation of the Metadata for Python packages +"""Implementation of the Metadata for Python packages PEPs. -Supports all Metadata formats (1.0, 1.1, 1.2). +Supports all metadata formats (1.0, 1.1, 1.2). """ -import re import os import sys import platform +import re from StringIO import StringIO from email import message_from_file from tokenize import tokenize, NAME, OP, STRING, ENDMARKER @@ -53,12 +52,12 @@ PKG_INFO_ENCODING = 'utf-8' PKG_INFO_PREFERRED_VERSION = '1.0' _LINE_PREFIX = re.compile('\n \|') -_241_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', +_241_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', 'Summary', 'Description', 'Keywords', 'Home-page', 'Author', 'Author-email', 'License') -_314_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', +_314_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', 'Supported-Platform', 'Summary', 'Description', 'Keywords', 'Home-page', 'Author', 'Author-email', 'License', 'Classifier', 'Download-URL', 'Obsoletes', @@ -66,7 +65,7 @@ _314_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', _314_MARKERS = ('Obsoletes', 'Provides', 'Requires') -_345_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', +_345_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', 'Supported-Platform', 'Summary', 'Description', 'Keywords', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', @@ -78,12 +77,11 @@ _345_MARKERS = ('Provides-Dist', 'Requires-Dist', 'Requires-Python', 'Obsoletes-Dist', 'Requires-External', 'Maintainer', 'Maintainer-email', 'Project-URL') -_ALL_FIELDS = [] +_ALL_FIELDS = set() +_ALL_FIELDS.update(_241_FIELDS) +_ALL_FIELDS.update(_314_FIELDS) +_ALL_FIELDS.update(_345_FIELDS) -for field in _241_FIELDS + _314_FIELDS + _345_FIELDS: - if field in _ALL_FIELDS: - continue - _ALL_FIELDS.append(field) def _version2fieldlist(version): if version == '1.0': @@ -94,8 +92,9 @@ def _version2fieldlist(version): return _345_FIELDS raise MetadataUnrecognizedVersionError(version) + def _best_version(fields): - """Will detect the best version depending on the fields used.""" + """Detect the best version depending on the fields used.""" def _has_marker(keys, markers): for marker in markers: if marker in keys: @@ -182,19 +181,33 @@ _UNICODEFIELDS = ('Author', 'Maintainer', 'Summary', 'Description') class DistributionMetadata(object): - """Distribution meta-data class (1.0 or 1.2). + """The metadata of a release. + + Supports versions 1.0, 1.1 and 1.2 (auto-detected). You can + instantiate the class with one of these arguments (or none): + - *path*, the path to a METADATA file + - *fileobj* give a file-like object with METADATA as content + - *mapping* is a dict-like object """ + # TODO document that execution_context and platform_dependent are used + # to filter on query, not when setting a key + # also document the mapping API and UNKNOWN default key + def __init__(self, path=None, platform_dependent=False, - execution_context=None, fileobj=None): + execution_context=None, fileobj=None, mapping=None): self._fields = {} self.version = None self.docutils_support = _HAS_DOCUTILS self.platform_dependent = platform_dependent + self.execution_context = execution_context + if [path, fileobj, mapping].count(None) < 2: + raise TypeError('path, fileobj and mapping are exclusive') if path is not None: self.read(path) elif fileobj is not None: self.read_file(fileobj) - self.execution_context = execution_context + elif mapping is not None: + self.update(mapping) def _set_best_version(self): self.version = _best_version(self._fields) @@ -203,10 +216,6 @@ class DistributionMetadata(object): def _write_field(self, file, name, value): file.write('%s: %s\n' % (name, value)) - def _write_list(self, file, name, values): - for value in values: - self._write_field(file, name, value) - def _encode_field(self, value): if isinstance(value, unicode): return value.encode(PKG_INFO_ENCODING) @@ -226,17 +235,15 @@ class DistributionMetadata(object): if name in _ALL_FIELDS: return name name = name.replace('-', '_').lower() - if name in _ATTR2FIELD: - return _ATTR2FIELD[name] - return name + return _ATTR2FIELD.get(name, name) def _default_value(self, name): - if name in _LISTFIELDS + _ELEMENTSFIELD: + if name in _LISTFIELDS or name in _ELEMENTSFIELD: return [] return 'UNKNOWN' def _check_rst_data(self, data): - """Returns warnings when the provided data doesn't compile.""" + """Return warnings when the provided data has syntax errors.""" source_path = StringIO() parser = Parser() settings = frontend.OptionParser().get_default_values() @@ -271,7 +278,7 @@ class DistributionMetadata(object): return _LINE_PREFIX.sub('\n', value) # - # Public APIs + # Public API # def get_fullname(self): return '%s-%s' % (self['Name'], self['Version']) @@ -284,7 +291,7 @@ class DistributionMetadata(object): self.read_file(open(filepath)) def read_file(self, fileob): - """Reads the metadata values from a file object.""" + """Read the metadata values from a file object.""" msg = message_from_file(fileob) self.version = msg['metadata-version'] @@ -302,8 +309,7 @@ class DistributionMetadata(object): self.set(field, value) def write(self, filepath): - """Write the metadata fields into path. - """ + """Write the metadata fields to filepath.""" pkg_info = open(filepath, 'w') try: self.write_file(pkg_info) @@ -311,8 +317,7 @@ class DistributionMetadata(object): pkg_info.close() def write_file(self, fileobject): - """Write the PKG-INFO format data to a file object. - """ + """Write the PKG-INFO format data to a file object.""" self._set_best_version() for field in _version2fieldlist(self.version): values = self.get(field) @@ -330,18 +335,50 @@ class DistributionMetadata(object): for value in values: self._write_field(fileobject, field, value) + def update(self, other=None, **kwargs): + """Set metadata values from the given mapping + + Convert the keys to Metadata fields. Given keys that don't match a + metadata argument will not be used. + + If overwrite is set to False, just add metadata values that are + actually not defined. + + If there is existing values in conflict with the dictionary ones, the + new values prevails. + + Empty values (e.g. None and []) are not setted this way. + """ + def _set(key, value): + if value not in ([], None, '') and key in _ATTR2FIELD: + self.set(self._convert_name(key), value) + + if other is None: + pass + elif hasattr(other, 'iteritems'): # iteritems saves memory and lookups + for k, v in other.iteritems(): + _set(k, v) + elif hasattr(other, 'keys'): + for k in other.keys(): + _set(k, v) + else: + for k, v in other: + _set(k, v) + if kwargs: + self.update(kwargs) + def set(self, name, value): - """Controls then sets a metadata field""" + """Control then set a metadata field.""" name = self._convert_name(name) - if (name in _ELEMENTSFIELD + ('Platform',) and + if ((name in _ELEMENTSFIELD or name == 'Platform') and not isinstance(value, (list, tuple))): if isinstance(value, str): value = value.split(',') else: value = [] elif (name in _LISTFIELDS and - not isinstance(value, (list, tuple))): + not isinstance(value, (list, tuple))): if isinstance(value, str): value = [value] else: @@ -351,13 +388,16 @@ class DistributionMetadata(object): for v in value: # check that the values are valid predicates if not is_valid_predicate(v.split(';')[0]): - warn('"%s" is not a valid predicate' % v) + warn('"%s" is not a valid predicate (field "%s")' % + (v, name)) elif name in _VERSIONS_FIELDS and value is not None: if not is_valid_versions(value): - warn('"%s" is not a valid predicate' % value) + warn('"%s" is not a valid version (field "%s")' % + (value, name)) elif name in _VERSION_FIELDS and value is not None: if not is_valid_version(value): - warn('"%s" is not a valid version' % value) + warn('"%s" is not a valid version (field "%s")' % + (value, name)) if name in _UNICODEFIELDS: value = self._encode_field(value) @@ -368,7 +408,7 @@ class DistributionMetadata(object): self._set_best_version() def get(self, name): - """Gets a metadata field.""" + """Get a metadata field.""" name = self._convert_name(name) if name not in self._fields: return self._default_value(name) @@ -403,24 +443,21 @@ class DistributionMetadata(object): return value def check(self): - """Checks if the metadata are compliant.""" + """Check if the metadata is compliant.""" # XXX should check the versions (if the file was loaded) - missing = [] + missing, warnings = [], [] for attr in ('Name', 'Version', 'Home-page'): value = self[attr] if value == 'UNKNOWN': missing.append(attr) if _HAS_DOCUTILS: - warnings = self._check_rst_data(self['Description']) - else: - warnings = [] + warnings.extend(self._check_rst_data(self['Description'])) # checking metadata 1.2 (XXX needs to check 1.1, 1.0) if self['Metadata-Version'] != '1.2': return missing, warnings - def is_valid_predicates(value): for v in value: if not is_valid_predicate(v.split(';')[0]): @@ -428,16 +465,15 @@ class DistributionMetadata(object): return True for fields, controller in ((_PREDICATE_FIELDS, is_valid_predicates), - (_VERSIONS_FIELDS, is_valid_versions), - (_VERSION_FIELDS, is_valid_version)): + (_VERSIONS_FIELDS, is_valid_versions), + (_VERSION_FIELDS, is_valid_version)): for field in fields: value = self[field] if value == 'UNKNOWN': continue if not controller(value): - warnings.append('Wrong value for "%s": %s' \ - % (field, value)) + warnings.append('Wrong value for %r: %s' % (field, value)) return missing, warnings @@ -454,7 +490,6 @@ class DistributionMetadata(object): # # micro-language for PEP 345 environment markers # -_STR_LIMIT = "'\"" # allowed operators _OPERATORS = {'==': lambda x, y: x == y, @@ -466,17 +501,18 @@ _OPERATORS = {'==': lambda x, y: x == y, 'in': lambda x, y: x in y, 'not in': lambda x, y: x not in y} + def _operate(operation, x, y): return _OPERATORS[operation](x, y) # restricted set of variables _VARS = {'sys.platform': sys.platform, - 'python_version': '%s.%s' % (sys.version_info[0], - sys.version_info[1]), - 'python_full_version': sys.version.split()[0], + 'python_version': sys.version[:3], + 'python_full_version': sys.version.split(' ', 1)[0], 'os.name': os.name, - 'platform.version': platform.version, - 'platform.machine': platform.machine} + 'platform.version': platform.version(), + 'platform.machine': platform.machine()} + class _Operation(object): @@ -499,7 +535,7 @@ class _Operation(object): def _is_string(self, value): if value is None or len(value) < 2: return False - for delimiter in _STR_LIMIT: + for delimiter in '"\'': if value[0] == value[-1] == delimiter: return True return False @@ -510,7 +546,7 @@ class _Operation(object): def _convert(self, value): if value in _VARS: return self._get_var(value) - return value.strip(_STR_LIMIT) + return value.strip('"\'') def _check_name(self, value): if value not in _VARS: @@ -538,6 +574,7 @@ class _Operation(object): right = self._convert(self.right) return _operate(self.op, left, right) + class _OR(object): def __init__(self, left, right=None): self.left = left @@ -547,7 +584,7 @@ class _OR(object): return self.right is not None def __repr__(self): - return 'OR(%s, %s)' % (repr(self.left), repr(self.right)) + return 'OR(%r, %r)' % (self.left, self.right) def __call__(self): return self.left() or self.right() @@ -562,11 +599,12 @@ class _AND(object): return self.right is not None def __repr__(self): - return 'AND(%s, %s)' % (repr(self.left), repr(self.right)) + return 'AND(%r, %r)' % (self.left, self.right) def __call__(self): return self.left() and self.right() + class _CHAIN(object): def __init__(self, execution_context=None): @@ -628,8 +666,9 @@ class _CHAIN(object): return False return True + def _interpret(marker, execution_context=None): - """Interprets a marker and return a result given the environment.""" + """Interpret a marker and return a result depending on environment.""" marker = marker.strip() operations = _CHAIN(execution_context) tokenize(StringIO(marker).readline, operations.eat) diff --git a/src/distutils2/mkpkg.py b/src/distutils2/mkpkg.py index 3fe82e3..0512592 100755 --- a/src/distutils2/mkpkg.py +++ b/src/distutils2/mkpkg.py @@ -30,7 +30,11 @@ # # Detect scripts (not sure how. #! outside of package?) -import sys, os, re, shutil, ConfigParser +import sys +import os +import re +import shutil +import ConfigParser helpText = { @@ -629,19 +633,20 @@ def askYn(question, default = None, helptext = None): print '\nERROR: You must select "Y" or "N".\n' -def ask(question, default = None, helptext = None, required = True, - lengthy = False, multiline = False): - prompt = '%s: ' % ( question, ) +def ask(question, default=None, helptext=None, required=True, + lengthy=False, multiline=False): + prompt = '%s: ' % (question,) if default: - prompt = '%s [%s]: ' % ( question, default ) + prompt = '%s [%s]: ' % (question, default) if default and len(question) + len(default) > 70: - prompt = '%s\n [%s]: ' % ( question, default ) + prompt = '%s\n [%s]: ' % (question, default) if lengthy or multiline: prompt += '\n >' - if not helptext: helptext = 'No additional help available.' - if helptext[0] == '\n': helptext = helptext[1:] - if helptext[-1] == '\n': helptext = helptext[:-1] + if not helptext: + helptext = 'No additional help available.' + + helptext = helptext.strip("\n") while True: sys.stdout.write(prompt) @@ -653,12 +658,14 @@ def ask(question, default = None, helptext = None, required = True, print helptext print '=' * 70 continue - if default and not line: return(default) + if default and not line: + return(default) if not line and required: print '*' * 70 print 'This value cannot be empty.' print '===========================' - if helptext: print helptext + if helptext: + print helptext print '*' * 70 continue return(line) @@ -669,7 +676,8 @@ def buildTroveDict(troveList): for key in troveList: subDict = dict for subkey in key.split(' :: '): - if not subkey in subDict: subDict[subkey] = {} + if not subkey in subDict: + subDict[subkey] = {} subDict = subDict[subkey] return(dict) troveDict = buildTroveDict(troveList) @@ -687,7 +695,8 @@ class SetupClass(object): def lookupOption(self, key): - if not self.config.has_option('DEFAULT', key): return(None) + if not self.config.has_option('DEFAULT', key): + return(None) return(self.config.get('DEFAULT', key)) @@ -706,7 +715,8 @@ class SetupClass(object): self.config.set('DEFAULT', compareKey, self.setupData[compareKey]) - if not valuesDifferent: return + if not valuesDifferent: + return self.config.write(open(os.path.expanduser('~/.pygiver'), 'w')) @@ -718,7 +728,7 @@ class SetupClass(object): def inspectFile(self, path): fp = open(path, 'r') try: - for line in [ fp.readline() for x in range(10) ]: + for line in [fp.readline() for x in range(10)]: m = re.match(r'^#!.*python((?P<major>\d)(\.\d+)?)?$', line) if m: if m.group('major') == '3': @@ -737,14 +747,14 @@ class SetupClass(object): self.setupData['name'] = m.group(1) self.setupData['version'] = m.group(2) - for root, dirs, files in os.walk('.'): + for root, dirs, files in os.walk(os.curdir): for file in files: - if root == '.' and file == 'setup.py': continue + if root == os.curdir and file == 'setup.py': continue fileName = os.path.join(root, file) self.inspectFile(fileName) if file == '__init__.py': - trySrc = os.path.join('.', 'src') + trySrc = os.path.join(os.curdir, 'src') tmpRoot = root if tmpRoot.startswith(trySrc): tmpRoot = tmpRoot[len(trySrc):] @@ -761,16 +771,16 @@ class SetupClass(object): self.setupData.get('version'), helpText['version']) self.setupData['description'] = ask('Package description', self.setupData.get('description'), helpText['description'], - lengthy = True) + lengthy=True) self.setupData['author'] = ask('Author name', self.setupData.get('author'), helpText['author']) self.setupData['author_email'] = ask('Author e-mail address', self.setupData.get('author_email'), helpText['author_email']) self.setupData['url'] = ask('Project URL', - self.setupData.get('url'), helpText['url'], required = False) + self.setupData.get('url'), helpText['url'], required=False) if (askYn('Do you want to set Trove classifiers?', - helptext = helpText['do_classifier']) == 'y'): + helptext=helpText['do_classifier']) == 'y'): self.setTroveClassifier() @@ -781,8 +791,10 @@ class SetupClass(object): def setTroveOther(self, classifierDict): - if askYn('Do you want to set other trove identifiers', 'n', - helpText['trove_generic']) != 'y': return + if askYn('Do you want to set other trove identifiers', + 'n', + helpText['trove_generic']) != 'y': + return self.walkTrove(classifierDict, [troveDict], '') @@ -799,7 +811,7 @@ class SetupClass(object): continue if askYn('Do you want to set items under\n "%s" (%d sub-items)' - % ( key, len(trove[key]) ), 'n', + % (key, len(trove[key])), 'n', helpText['trove_generic']) == 'y': self.walkTrove(classifierDict, trovePath + [trove[key]], desc + ' :: ' + key) @@ -808,15 +820,18 @@ class SetupClass(object): def setTroveLicense(self, classifierDict): while True: license = ask('What license do you use', - helptext = helpText['trove_license'], required = False) - if not license: return + helptext=helpText['trove_license'], + required=False) + if not license: + return licenseWords = license.lower().split(' ') foundList = [] for index in range(len(troveList)): troveItem = troveList[index] - if not troveItem.startswith('License :: '): continue + if not troveItem.startswith('License :: '): + continue troveItem = troveItem[11:].lower() allMatch = True @@ -824,17 +839,20 @@ class SetupClass(object): if not word in troveItem: allMatch = False break - if allMatch: foundList.append(index) + if allMatch: + foundList.append(index) question = 'Matching licenses:\n\n' for i in xrange(1, len(foundList) + 1): - question += ' %s) %s\n' % ( i, troveList[foundList[i - 1]] ) + question += ' %s) %s\n' % (i, troveList[foundList[i - 1]]) question += ('\nType the number of the license you wish to use or ' '? to try again:') - troveLicense = ask(question, required = False) + troveLicense = ask(question, required=False) - if troveLicense == '?': continue - if troveLicense == '': return + if troveLicense == '?': + continue + if troveLicense == '': + return foundIndex = foundList[int(troveLicense) - 1] classifierDict[troveList[foundIndex]] = 1 try: @@ -856,7 +874,7 @@ class SetupClass(object): 6 - Mature 7 - Inactive -Status''', required = False) +Status''', required=False) if devStatus: try: key = { @@ -884,7 +902,8 @@ Status''', required = False) return modified_pkgs def writeSetup(self): - if os.path.exists('setup.py'): shutil.move('setup.py', 'setup.py.old') + if os.path.exists('setup.py'): + shutil.move('setup.py', 'setup.py.old') fp = open('setup.py', 'w') try: diff --git a/src/distutils2/pypi/__init__.py b/src/distutils2/pypi/__init__.py deleted file mode 100644 index 88efa14..0000000 --- a/src/distutils2/pypi/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""distutils2.pypi - -Package containing ways to interact with the PyPI APIs. -""" - -__all__ = ['simple', - 'dist', -] diff --git a/src/distutils2/pypi/dist.py b/src/distutils2/pypi/dist.py deleted file mode 100644 index 418186e..0000000 --- a/src/distutils2/pypi/dist.py +++ /dev/null @@ -1,315 +0,0 @@ -"""distutils2.pypi.dist - -Provides the PyPIDistribution class thats represents a distribution retrieved -on PyPI. -""" -import re -import urlparse -import urllib -import tempfile -from operator import attrgetter - -try: - import hashlib -except ImportError: - from distutils2._backport import hashlib - -from distutils2.version import suggest_normalized_version, NormalizedVersion -from distutils2.pypi.errors import HashDoesNotMatch, UnsupportedHashName - -EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz .egg".split() -MD5_HASH = re.compile(r'^.*#md5=([a-f0-9]+)$') - - -class PyPIDistribution(object): - """Represents a distribution retrieved from PyPI. - - This is a simple container for various attributes as name, version, - downloaded_location, url etc. - - The PyPIDistribution class is used by the pypi.*Index class to return - information about distributions. - """ - - @classmethod - def from_url(cls, url, probable_dist_name=None, is_external=True): - """Build a Distribution from a url archive (egg or zip or tgz). - - :param url: complete url of the distribution - :param probable_dist_name: A probable name of the distribution. - :param is_external: Tell if the url commes from an index or from - an external URL. - """ - # if the url contains a md5 hash, get it. - md5_hash = None - match = MD5_HASH.match(url) - if match is not None: - md5_hash = match.group(1) - # remove the hash - url = url.replace("#md5=%s" % md5_hash, "") - - # parse the archive name to find dist name and version - archive_name = urlparse.urlparse(url)[2].split('/')[-1] - extension_matched = False - # remove the extension from the name - for ext in EXTENSIONS: - if archive_name.endswith(ext): - archive_name = archive_name[:-len(ext)] - extension_matched = True - - name, version = split_archive_name(archive_name) - if extension_matched is True: - return PyPIDistribution(name, version, url=url, url_hashname="md5", - url_hashval=md5_hash, - url_is_external=is_external) - - def __init__(self, name, version, type=None, url=None, url_hashname=None, - url_hashval=None, url_is_external=True): - """Create a new instance of PyPIDistribution. - - :param name: the name of the distribution - :param version: the version of the distribution - :param type: the type of the dist (eg. source, bin-*, etc.) - :param url: URL where we found this distribution - :param url_hashname: the name of the hash we want to use. Refer to the - hashlib.new documentation for more information. - :param url_hashval: the hash value. - :param url_is_external: we need to know if the provided url comes from an - index browsing, or from an external resource. - - """ - self.name = name - self.version = NormalizedVersion(version) - self.type = type - # set the downloaded path to None by default. The goal here - # is to not download distributions multiple times - self.downloaded_location = None - # We store urls in dict, because we need to have a bit more informations - # than the simple URL. It will be used later to find the good url to - # use. - # We have two _url* attributes: _url and _urls. _urls contains a list of - # dict for the different urls, and _url contains the choosen url, in - # order to dont make the selection process multiple times. - self._urls = [] - self._url = None - self.add_url(url, url_hashname, url_hashval, url_is_external) - - def add_url(self, url, hashname=None, hashval=None, is_external=True): - """Add a new url to the list of urls""" - if hashname is not None: - try: - hashlib.new(hashname) - except ValueError: - raise UnsupportedHashName(hashname) - - self._urls.append({ - 'url': url, - 'hashname': hashname, - 'hashval': hashval, - 'is_external': is_external, - }) - # reset the url selection process - self._url = None - - @property - def url(self): - """Pick up the right url for the list of urls in self.urls""" - # We return internal urls over externals. - # If there is more than one internal or external, return the first - # one. - if self._url is None: - if len(self._urls) > 1: - internals_urls = [u for u in self._urls \ - if u['is_external'] == False] - if len(internals_urls) >= 1: - self._url = internals_urls[0] - if self._url is None: - self._url = self._urls[0] - return self._url - - @property - def is_source(self): - """return if the distribution is a source one or not""" - return self.type == 'source' - - @property - def is_final(self): - """proxy to version.is_final""" - return self.version.is_final - - def download(self, path=None): - """Download the distribution to a path, and return it. - - If the path is given in path, use this, otherwise, generates a new one - """ - if path is None: - path = tempfile.mkdtemp() - - # if we do not have downloaded it yet, do it. - if self.downloaded_location is None: - url = self.url['url'] - archive_name = urlparse.urlparse(url)[2].split('/')[-1] - filename, headers = urllib.urlretrieve(url, - path + "/" + archive_name) - self.downloaded_location = filename - self._check_md5(filename) - return self.downloaded_location - - def _check_md5(self, filename): - """Check that the md5 checksum of the given file matches the one in - url param""" - hashname = self.url['hashname'] - expected_hashval = self.url['hashval'] - if not None in (expected_hashval, hashname): - f = open(filename) - hashval = hashlib.new(hashname) - hashval.update(f.read()) - if hashval.hexdigest() != expected_hashval: - raise HashDoesNotMatch("got %s instead of %s" - % (hashval.hexdigest(), expected_hashval)) - - def __repr__(self): - return "%s %s %s %s" \ - % (self.__class__.__name__, self.name, self.version, - self.type or "") - - def _check_is_comparable(self, other): - if not isinstance(other, PyPIDistribution): - raise TypeError("cannot compare %s and %s" - % (type(self).__name__, type(other).__name__)) - elif self.name != other.name: - raise TypeError("cannot compare %s and %s" - % (self.name, other.name)) - - def __eq__(self, other): - self._check_is_comparable(other) - return self.version == other.version - - def __lt__(self, other): - self._check_is_comparable(other) - return self.version < other.version - - def __ne__(self, other): - return not self.__eq__(other) - - def __gt__(self, other): - return not (self.__lt__(other) or self.__eq__(other)) - - def __le__(self, other): - return self.__eq__(other) or self.__lt__(other) - - def __ge__(self, other): - return self.__eq__(other) or self.__gt__(other) - - # See http://docs.python.org/reference/datamodel#object.__hash__ - __hash__ = object.__hash__ - - -class PyPIDistributions(list): - """A container of PyPIDistribution objects. - - Contains methods and facilities to sort and filter distributions. - """ - def __init__(self, list=[]): - # To disable the ability to pass lists on instanciation - super(PyPIDistributions, self).__init__() - for item in list: - self.append(item) - - def filter(self, predicate): - """Filter the distributions and return a subset of distributions that - match the given predicate - """ - return PyPIDistributions( - [dist for dist in self if dist.name == predicate.name and - predicate.match(dist.version)]) - - def get_last(self, predicate, prefer_source=None, prefer_final=None): - """Return the most up to date version, that satisfy the given - predicate - """ - distributions = self.filter(predicate) - distributions.sort_distributions(prefer_source, prefer_final, reverse=True) - return distributions[0] - - def get_same_name_and_version(self): - """Return lists of PyPIDistribution objects that refer to the same - name and version number. This do not consider the type (source, binary, - etc.)""" - processed = [] - duplicates = [] - for dist in self: - if (dist.name, dist.version) not in processed: - processed.append((dist.name, dist.version)) - found_duplicates = [d for d in self if d.name == dist.name and - d.version == dist.version] - if len(found_duplicates) > 1: - duplicates.append(found_duplicates) - return duplicates - - def append(self, o): - """Append a new distribution to the list. - - If a distribution with the same name and version exists, just grab the - URL informations and add a new new url for the existing one. - """ - similar_dists = [d for d in self if d.name == o.name and - d.version == o.version and d.type == o.type] - if len(similar_dists) > 0: - dist = similar_dists[0] - dist.add_url(**o.url) - else: - super(PyPIDistributions, self).append(o) - - def sort_distributions(self, prefer_source=True, prefer_final=False, - reverse=True, *args, **kwargs): - """order the results with the given properties""" - - sort_by = [] - if prefer_final: - sort_by.append("is_final") - sort_by.append("version") - - if prefer_source: - sort_by.append("is_source") - - super(PyPIDistributions, self).sort( - key=lambda i: [getattr(i, arg) for arg in sort_by], - reverse=reverse, *args, **kwargs) - - -def split_archive_name(archive_name, probable_name=None): - """Split an archive name into two parts: name and version. - - Return the tuple (name, version) - """ - # Try to determine wich part is the name and wich is the version using the - # "-" separator. Take the larger part to be the version number then reduce - # if this not works. - def eager_split(str, maxsplit=2): - # split using the "-" separator - splits = str.rsplit("-", maxsplit) - name = splits[0] - version = "-".join(splits[1:]) - if version.startswith("-"): - version = version[1:] - if suggest_normalized_version(version) is None and maxsplit >= 0: - # we dont get a good version number: recurse ! - return eager_split(str, maxsplit - 1) - else: - return (name, version) - if probable_name is not None: - probable_name = probable_name.lower() - name = None - if probable_name is not None and probable_name in archive_name: - # we get the name from probable_name, if given. - name = probable_name - version = archive_name.lstrip(name) - else: - name, version = eager_split(archive_name) - - version = suggest_normalized_version(version) - if version != "" and name != "": - return (name.lower(), version) - else: - raise CantParseArchiveName(archive_name) diff --git a/src/distutils2/pypi/errors.py b/src/distutils2/pypi/errors.py deleted file mode 100644 index c9b232d..0000000 --- a/src/distutils2/pypi/errors.py +++ /dev/null @@ -1,33 +0,0 @@ -"""distutils2.pypi.errors - -All errors and exceptions raised by PyPiIndex classes. -""" -from distutils2.errors import DistutilsError - - -class PyPIError(DistutilsError): - """The base class for errors of the pypi python package.""" - - -class DistributionNotFound(PyPIError): - """No distribution match the given requirements.""" - - -class CantParseArchiveName(PyPIError): - """An archive name can't be parsed to find distribution name and version""" - - -class DownloadError(PyPIError): - """An error has occurs while downloading""" - - -class HashDoesNotMatch(DownloadError): - """Compared hashes does not match""" - - -class UnsupportedHashName(PyPIError): - """A unsupported hashname has been used""" - - -class UnableToDownload(PyPIError): - """All mirrors have been tried, without success""" diff --git a/src/distutils2/pypi/simple.py b/src/distutils2/pypi/simple.py deleted file mode 100644 index 1a6a35d..0000000 --- a/src/distutils2/pypi/simple.py +++ /dev/null @@ -1,393 +0,0 @@ -"""pypi.simple - -Contains the class "SimpleIndex", a simple spider to find and retrieve -distributions on the Python Package Index, using it's "simple" API, -avalaible at http://pypi.python.org/simple/ -""" -from fnmatch import translate -import httplib -import re -import socket -import sys -import urllib2 -import urlparse - -from distutils2.version import VersionPredicate -from distutils2.pypi.dist import (PyPIDistribution, PyPIDistributions, - EXTENSIONS) -from distutils2.pypi.errors import (PyPIError, DistributionNotFound, - DownloadError, UnableToDownload) -from distutils2 import __version__ as __distutils2_version__ - -# -- Constants ----------------------------------------------- -PYPI_DEFAULT_INDEX_URL = "http://pypi.python.org/simple/" -PYPI_DEFAULT_MIRROR_URL = "mirrors.pypi.python.org" -DEFAULT_HOSTS = ("*",) -SOCKET_TIMEOUT = 15 -USER_AGENT = "Python-urllib/%s distutils2/%s" % ( - sys.version[:3], __distutils2_version__) - -# -- Regexps ------------------------------------------------- -EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') -HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) -PYPI_MD5 = re.compile( - '<a href="([^"#]+)">([^<]+)</a>\n\s+\\(<a (?:title="MD5 hash"\n\s+)' - 'href="[^?]+\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\\)') -URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match - -# This pattern matches a character entity reference (a decimal numeric -# references, a hexadecimal numeric reference, or a named reference). -ENTITY_SUB = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub -REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) - - -def socket_timeout(timeout=SOCKET_TIMEOUT): - """Decorator to add a socket timeout when requesting pages on PyPI. - """ - def _socket_timeout(func): - def _socket_timeout(self, *args, **kwargs): - old_timeout = socket.getdefaulttimeout() - if hasattr(self, "_timeout"): - timeout = self._timeout - socket.setdefaulttimeout(timeout) - try: - return func(self, *args, **kwargs) - finally: - socket.setdefaulttimeout(old_timeout) - return _socket_timeout - return _socket_timeout - - -class SimpleIndex(object): - """Provides useful tools to request the Python Package Index simple API - - :param index_url: the url of the simple index to search on. - :param follow_externals: tell if following external links is needed or - not. Default is False. - :param hosts: a list of hosts allowed to be processed while using - follow_externals=True. Default behavior is to follow all - hosts. - :param follow_externals: tell if following external links is needed or - not. Default is False. - :param prefer_source: if there is binary and source distributions, the - source prevails. - :param prefer_final: if the version is not mentioned, and the last - version is not a "final" one (alpha, beta, etc.), - pick up the last final version. - :param mirrors_url: the url to look on for DNS records giving mirror - adresses. - :param mirrors: a list of mirrors to check out if problems - occurs while working with the one given in "url" - :param timeout: time in seconds to consider a url has timeouted. - """ - - def __init__(self, index_url=PYPI_DEFAULT_INDEX_URL, hosts=DEFAULT_HOSTS, - follow_externals=False, prefer_source=True, - prefer_final=False, mirrors_url=PYPI_DEFAULT_MIRROR_URL, - mirrors=None, timeout=SOCKET_TIMEOUT): - self.follow_externals = follow_externals - - if not index_url.endswith("/"): - index_url += "/" - self._index_urls = [index_url] - # if no mirrors are defined, use the method described in PEP 381. - if mirrors is None: - try: - mirrors = socket.gethostbyname_ex(mirrors_url)[-1] - except socket.gaierror: - mirrors = [] - self._index_urls.extend(mirrors) - self._current_index_url = 0 - self._timeout = timeout - self._prefer_source = prefer_source - self._prefer_final = prefer_final - - # create a regexp to match all given hosts - self._allowed_hosts = re.compile('|'.join(map(translate, hosts))).match - - # we keep an index of pages we have processed, in order to avoid - # scanning them multple time (eg. if there is multiple pages pointing - # on one) - self._processed_urls = [] - self._distributions = {} - - def find(self, requirements, prefer_source=None, prefer_final=None): - """Browse the PyPI to find distributions that fullfil the given - requirements. - - :param requirements: A project name and it's distribution, using - version specifiers, as described in PEP345. - :type requirements: You can pass either a version.VersionPredicate - or a string. - :param prefer_source: if there is binary and source distributions, the - source prevails. - :param prefer_final: if the version is not mentioned, and the last - version is not a "final" one (alpha, beta, etc.), - pick up the last final version. - """ - requirements = self._get_version_predicate(requirements) - if prefer_source is None: - prefer_source = self._prefer_source - if prefer_final is None: - prefer_final = self._prefer_final - - # process the index for this project - self._process_pypi_page(requirements.name) - - # filter with requirements and return the results - if requirements.name in self._distributions: - dists = self._distributions[requirements.name].filter(requirements) - dists.sort_distributions(prefer_source=prefer_source, - prefer_final=prefer_final) - else: - dists = [] - - return dists - - def get(self, requirements, *args, **kwargs): - """Browse the PyPI index to find distributions that fullfil the - given requirements, and return the most recent one. - - You can specify prefer_final and prefer_source arguments here. - If not, the default one will be used. - """ - predicate = self._get_version_predicate(requirements) - dists = self.find(predicate, *args, **kwargs) - - if len(dists) == 0: - raise DistributionNotFound(requirements) - - return dists.get_last(predicate) - - def download(self, requirements, temp_path=None, *args, **kwargs): - """Download the distribution, using the requirements. - - If more than one distribution match the requirements, use the last - version. - Download the distribution, and put it in the temp_path. If no temp_path - is given, creates and return one. - - Returns the complete absolute path to the downloaded archive. - - :param requirements: The same as the find attribute of `find`. - - You can specify prefer_final and prefer_source arguments here. - If not, the default one will be used. - """ - return self.get(requirements, *args, **kwargs)\ - .download(path=temp_path) - - def _get_version_predicate(self, requirements): - """Return a VersionPredicate object, from a string or an already - existing object. - """ - if isinstance(requirements, str): - requirements = VersionPredicate(requirements) - return requirements - - @property - def index_url(self): - return self._index_urls[self._current_index_url] - - def _switch_to_next_mirror(self): - """Switch to the next mirror (eg. point self.index_url to the next - url. - """ - # Internally, iter over the _index_url iterable, if we have read all - # of the available indexes, raise an exception. - if self._current_index_url < len(self._index_urls): - self._current_index_url = self._current_index_url + 1 - else: - raise UnableToDownload("All mirrors fails") - - def _is_browsable(self, url): - """Tell if the given URL can be browsed or not. - - It uses the follow_externals and the hosts list to tell if the given - url is browsable or not. - """ - # if _index_url is contained in the given URL, we are browsing the - # index, and it's always "browsable". - # local files are always considered browable resources - if self.index_url in url or urlparse.urlparse(url)[0] == "file": - return True - elif self.follow_externals: - if self._allowed_hosts(urlparse.urlparse(url)[1]): # 1 is netloc - return True - else: - return False - return False - - def _is_distribution(self, link): - """Tell if the given URL matches to a distribution name or not. - """ - #XXX find a better way to check that links are distributions - # Using a regexp ? - for ext in EXTENSIONS: - if ext in link: - return True - return False - - def _register_dist(self, dist): - """Register a distribution as a part of fetched distributions for - SimpleIndex. - - Return the PyPIDistributions object for the specified project name - """ - # Internally, check if a entry exists with the project name, if not, - # create a new one, and if exists, add the dist to the pool. - if not dist.name in self._distributions: - self._distributions[dist.name] = PyPIDistributions() - self._distributions[dist.name].append(dist) - return self._distributions[dist.name] - - def _process_url(self, url, project_name=None, follow_links=True): - """Process an url and search for distributions packages. - - For each URL found, if it's a download, creates a PyPIdistribution - object. If it's a homepage and we can follow links, process it too. - - :param url: the url to process - :param project_name: the project name we are searching for. - :param follow_links: Do not want to follow links more than from one - level. This parameter tells if we want to follow - the links we find (eg. run recursively this - method on it) - """ - f = self._open_url(url) - base_url = f.url - if url not in self._processed_urls: - self._processed_urls.append(url) - link_matcher = self._get_link_matcher(url) - for link, is_download in link_matcher(f.read(), base_url): - if link not in self._processed_urls: - if self._is_distribution(link) or is_download: - self._processed_urls.append(link) - # it's a distribution, so create a dist object - dist = PyPIDistribution.from_url(link, project_name, - is_external=not self.index_url in url) - self._register_dist(dist) - else: - if self._is_browsable(link) and follow_links: - self._process_url(link, project_name, - follow_links=False) - - def _get_link_matcher(self, url): - """Returns the right link matcher function of the given url - """ - if self.index_url in url: - return self._simple_link_matcher - else: - return self._default_link_matcher - - def _simple_link_matcher(self, content, base_url): - """Yield all links with a rel="download" or rel="homepage". - - This matches the simple index requirements for matching links. - If follow_externals is set to False, dont yeld the external - urls. - """ - for match in REL.finditer(content): - tag, rel = match.groups() - rels = map(str.strip, rel.lower().split(',')) - if 'homepage' in rels or 'download' in rels: - for match in HREF.finditer(tag): - url = urlparse.urljoin(base_url, - self._htmldecode(match.group(1))) - if 'download' in rels or self._is_browsable(url): - # yield a list of (url, is_download) - yield (urlparse.urljoin(base_url, url), - 'download' in rels) - - def _default_link_matcher(self, content, base_url): - """Yield all links found on the page. - """ - for match in HREF.finditer(content): - url = urlparse.urljoin(base_url, self._htmldecode(match.group(1))) - if self._is_browsable(url): - yield (url, False) - - def _process_pypi_page(self, name): - """Find and process a PyPI page for the given project name. - - :param name: the name of the project to find the page - """ - try: - # Browse and index the content of the given PyPI page. - url = self.index_url + name + "/" - self._process_url(url, name) - except DownloadError: - # if an error occurs, try with the next index_url - # (provided by the mirrors) - self._switch_to_next_mirror() - self._distributions.clear() - self._process_pypi_page(name) - - @socket_timeout() - def _open_url(self, url): - """Open a urllib2 request, handling HTTP authentication, and local - files support. - - """ - try: - scheme, netloc, path, params, query, frag = urlparse.urlparse(url) - - if scheme in ('http', 'https'): - auth, host = urllib2.splituser(netloc) - else: - auth = None - - # add index.html automatically for filesystem paths - if scheme == 'file': - if url.endswith('/'): - url += "index.html" - - if auth: - auth = "Basic " + \ - urllib2.unquote(auth).encode('base64').strip() - new_url = urlparse.urlunparse(( - scheme, host, path, params, query, frag)) - request = urllib2.Request(new_url) - request.add_header("Authorization", auth) - else: - request = urllib2.Request(url) - request.add_header('User-Agent', USER_AGENT) - fp = urllib2.urlopen(request) - - if auth: - # Put authentication info back into request URL if same host, - # so that links found on the page will work - s2, h2, path2, param2, query2, frag2 = \ - urlparse.urlparse(fp.url) - if s2 == scheme and h2 == host: - fp.url = urlparse.urlunparse( - (s2, netloc, path2, param2, query2, frag2)) - - return fp - except (ValueError, httplib.InvalidURL), v: - msg = ' '.join([str(arg) for arg in v.args]) - raise PyPIError('%s %s' % (url, msg)) - except urllib2.HTTPError, v: - return v - except urllib2.URLError, v: - raise DownloadError("Download error for %s: %s" % (url, v.reason)) - except httplib.BadStatusLine, v: - raise DownloadError('%s returned a bad status line. ' - 'The server might be down, %s' % (url, v.line)) - except httplib.HTTPException, v: - raise DownloadError("Download error for %s: %s" % (url, v)) - - def _decode_entity(self, match): - what = match.group(1) - if what.startswith('#x'): - what = int(what[2:], 16) - elif what.startswith('#'): - what = int(what[1:]) - else: - from htmlentitydefs import name2codepoint - what = name2codepoint.get(what, match.group(0)) - return unichr(what) - - def _htmldecode(self, text): - """Decode HTML entities in the given text.""" - return ENTITY_SUB(self._decode_entity, text) diff --git a/src/distutils2/spawn.py b/src/distutils2/spawn.py deleted file mode 100644 index 08996fc..0000000 --- a/src/distutils2/spawn.py +++ /dev/null @@ -1,192 +0,0 @@ -"""distutils.spawn - -Provides the 'spawn()' function, a front-end to various platform- -specific functions for launching another program in a sub-process. -Also provides the 'find_executable()' to search the path for a given -executable name. -""" - -__revision__ = "$Id: spawn.py 73147 2009-06-02 15:58:43Z tarek.ziade $" - -import sys -import os - -from distutils2.errors import DistutilsPlatformError, DistutilsExecError -from distutils2 import log - -def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None): - """Run another program, specified as a command list 'cmd', in a new process. - - 'cmd' is just the argument list for the new process, ie. - cmd[0] is the program to run and cmd[1:] are the rest of its arguments. - There is no way to run a program with a name different from that of its - executable. - - If 'search_path' is true (the default), the system's executable - search path will be used to find the program; otherwise, cmd[0] - must be the exact path to the executable. If 'dry_run' is true, - the command will not actually be run. - - If 'env' is given, it's a environment dictionary used for the execution - environment. - - Raise DistutilsExecError if running the program fails in any way; just - return on success. - """ - if os.name == 'posix': - _spawn_posix(cmd, search_path, dry_run=dry_run, env=env) - elif os.name == 'nt': - _spawn_nt(cmd, search_path, dry_run=dry_run, env=env) - elif os.name == 'os2': - _spawn_os2(cmd, search_path, dry_run=dry_run, env=env) - else: - raise DistutilsPlatformError, \ - "don't know how to spawn programs on platform '%s'" % os.name - -def _nt_quote_args(args): - """Quote command-line arguments for DOS/Windows conventions. - - Just wraps every argument which contains blanks in double quotes, and - returns a new argument list. - """ - # XXX this doesn't seem very robust to me -- but if the Windows guys - # say it'll work, I guess I'll have to accept it. (What if an arg - # contains quotes? What other magic characters, other than spaces, - # have to be escaped? Is there an escaping mechanism other than - # quoting?) - for i, arg in enumerate(args): - if ' ' in arg: - args[i] = '"%s"' % arg - return args - -def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0, env=None): - executable = cmd[0] - cmd = _nt_quote_args(cmd) - if search_path: - # either we find one or it stays the same - executable = find_executable(executable) or executable - log.info(' '.join([executable] + cmd[1:])) - if not dry_run: - # spawn for NT requires a full path to the .exe - try: - if env is None: - rc = os.spawnv(os.P_WAIT, executable, cmd) - else: - rc = os.spawnve(os.P_WAIT, executable, cmd, env) - - except OSError, exc: - # this seems to happen when the command isn't found - raise DistutilsExecError, \ - "command '%s' failed: %s" % (cmd[0], exc[-1]) - if rc != 0: - # and this reflects the command running but failing - raise DistutilsExecError, \ - "command '%s' failed with exit status %d" % (cmd[0], rc) - -def _spawn_os2(cmd, search_path=1, verbose=0, dry_run=0, env=None): - executable = cmd[0] - if search_path: - # either we find one or it stays the same - executable = find_executable(executable) or executable - log.info(' '.join([executable] + cmd[1:])) - if not dry_run: - # spawnv for OS/2 EMX requires a full path to the .exe - try: - if env is None: - rc = os.spawnv(os.P_WAIT, executable, cmd) - else: - rc = os.spawnve(os.P_WAIT, executable, cmd, env) - - except OSError, exc: - # this seems to happen when the command isn't found - raise DistutilsExecError, \ - "command '%s' failed: %s" % (cmd[0], exc[-1]) - if rc != 0: - # and this reflects the command running but failing - log.debug("command '%s' failed with exit status %d" % (cmd[0], rc)) - raise DistutilsExecError, \ - "command '%s' failed with exit status %d" % (cmd[0], rc) - - -def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0, env=None): - log.info(' '.join(cmd)) - if dry_run: - return - - if env is None: - exec_fn = search_path and os.execvp or os.execv - else: - exec_fn = search_path and os.execvpe or os.execve - - pid = os.fork() - - if pid == 0: # in the child - try: - if env is None: - exec_fn(cmd[0], cmd) - else: - exec_fn(cmd[0], cmd, env) - except OSError, e: - sys.stderr.write("unable to execute %s: %s\n" % - (cmd[0], e.strerror)) - os._exit(1) - - sys.stderr.write("unable to execute %s for unknown reasons" % cmd[0]) - os._exit(1) - else: # in the parent - # Loop until the child either exits or is terminated by a signal - # (ie. keep waiting if it's merely stopped) - while 1: - try: - pid, status = os.waitpid(pid, 0) - except OSError, exc: - import errno - if exc.errno == errno.EINTR: - continue - raise DistutilsExecError, \ - "command '%s' failed: %s" % (cmd[0], exc[-1]) - if os.WIFSIGNALED(status): - raise DistutilsExecError, \ - "command '%s' terminated by signal %d" % \ - (cmd[0], os.WTERMSIG(status)) - - elif os.WIFEXITED(status): - exit_status = os.WEXITSTATUS(status) - if exit_status == 0: - return # hey, it succeeded! - else: - raise DistutilsExecError, \ - "command '%s' failed with exit status %d" % \ - (cmd[0], exit_status) - - elif os.WIFSTOPPED(status): - continue - - else: - raise DistutilsExecError, \ - "unknown error executing '%s': termination status %d" % \ - (cmd[0], status) - -def find_executable(executable, path=None): - """Tries to find 'executable' in the directories listed in 'path'. - - A string listing directories separated by 'os.pathsep'; defaults to - os.environ['PATH']. Returns the complete filename or None if not found. - """ - if path is None: - path = os.environ['PATH'] - paths = path.split(os.pathsep) - base, ext = os.path.splitext(executable) - - if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'): - executable = executable + '.exe' - - if not os.path.isfile(executable): - for p in paths: - f = os.path.join(p, executable) - if os.path.isfile(f): - # the file exists, we have a shot at spawn working - return f - return None - else: - return executable diff --git a/src/distutils2/tests/__init__.py b/src/distutils2/tests/__init__.py index 8a8d187..de40aaa 100644 --- a/src/distutils2/tests/__init__.py +++ b/src/distutils2/tests/__init__.py @@ -23,7 +23,7 @@ from distutils2.tests.support import unittest from test.test_support import TESTFN # use TESTFN from stdlib/test_support. -here = os.path.dirname(__file__) +here = os.path.dirname(__file__) or os.curdir verbose = 1 diff --git a/src/distutils2/tests/conversions/02_after.py b/src/distutils2/tests/conversions/02_after.py index 473d035..bfc2d8f 100644 --- a/src/distutils2/tests/conversions/02_after.py +++ b/src/distutils2/tests/conversions/02_after.py @@ -1,4 +1,4 @@ -# -*- encoding: utf8 -*- +# -*- encoding: utf-8 -*- import sys import os from distutils2.core import setup, Extension diff --git a/src/distutils2/tests/conversions/02_before.py b/src/distutils2/tests/conversions/02_before.py index 6fdb418..f7ccc12 100644 --- a/src/distutils2/tests/conversions/02_before.py +++ b/src/distutils2/tests/conversions/02_before.py @@ -1,4 +1,4 @@ -# -*- encoding: utf8 -*- +# -*- encoding: utf-8 -*- import sys import os from distutils.core import setup, Extension diff --git a/src/distutils2/tests/pypi_server.py b/src/distutils2/tests/pypi_server.py index 3927262..2db4577 100644 --- a/src/distutils2/tests/pypi_server.py +++ b/src/distutils2/tests/pypi_server.py @@ -1,25 +1,59 @@ -"""Mocked PyPI Server implementation, to use in tests. +"""Mock PyPI Server implementation, to use in tests. This module also provides a simple test case to extend if you need to use -the PyPIServer all along your test case. Be sure to read the documentation +the PyPIServer all along your test case. Be sure to read the documentation before any use. + +XXX TODO: + +The mock server can handle simple HTTP request (to simulate a simple index) or +XMLRPC requests, over HTTP. Both does not have the same intergface to deal +with, and I think it's a pain. + +A good idea could be to re-think a bit the way dstributions are handled in the +mock server. As it should return malformed HTML pages, we need to keep the +static behavior. + +I think of something like that: + + >>> server = PyPIMockServer() + >>> server.startHTTP() + >>> server.startXMLRPC() + +Then, the server must have only one port to rely on, eg. + + >>> server.fulladress() + "http://ip:port/" + +It could be simple to have one HTTP server, relaying the requests to the two +implementations (static HTTP and XMLRPC over HTTP). """ import Queue +import SocketServer +import os.path +import select +import socket import threading -import time -import urllib2 + from BaseHTTPServer import HTTPServer from SimpleHTTPServer import SimpleHTTPRequestHandler -import os.path -import select +from SimpleXMLRPCServer import SimpleXMLRPCServer from distutils2.tests.support import unittest PYPI_DEFAULT_STATIC_PATH = os.path.dirname(os.path.abspath(__file__)) + "/pypiserver" +def use_xmlrpc_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = True + return use_pypi_server(*server_args, **server_kwargs) + +def use_http_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = False + return use_pypi_server(*server_args, **server_kwargs) + def use_pypi_server(*server_args, **server_kwargs): - """Decorator to make use of the PyPIServer for test methods, + """Decorator to make use of the PyPIServer for test methods, just when needed, and not for the entire duration of the testcase. """ def wrapper(func): @@ -52,38 +86,59 @@ class PyPIServer(threading.Thread): """ def __init__(self, test_static_path=None, - static_filesystem_paths=["default"], static_uri_paths=["simple"]): + static_filesystem_paths=["default"], + static_uri_paths=["simple"], serve_xmlrpc=False) : """Initialize the server. + Default behavior is to start the HTTP server. You can either start the + xmlrpc server by setting xmlrpc to True. Caution: Only one server will + be started. + static_uri_paths and static_base_path are parameters used to provides respectively the http_paths to serve statically, and where to find the matching files on the filesystem. """ + # we want to launch the server in a new dedicated thread, to not freeze + # tests. threading.Thread.__init__(self) self._run = True - self.httpd = HTTPServer(('', 0), PyPIRequestHandler) - self.httpd.RequestHandlerClass.log_request = lambda *_: None - self.httpd.RequestHandlerClass.pypi_server = self - self.address = (self.httpd.server_name, self.httpd.server_port) - self.request_queue = Queue.Queue() - self._requests = [] - self.default_response_status = 200 - self.default_response_headers = [('Content-type', 'text/plain')] - self.default_response_data = "hello" - - # initialize static paths / filesystems - self.static_uri_paths = static_uri_paths - if test_static_path is not None: - static_filesystem_paths.append(test_static_path) - self.static_filesystem_paths = [PYPI_DEFAULT_STATIC_PATH + "/" + path - for path in static_filesystem_paths] + self._serve_xmlrpc = serve_xmlrpc + + #TODO allow to serve XMLRPC and HTTP static files at the same time. + if not self._serve_xmlrpc: + self.server = HTTPServer(('', 0), PyPIRequestHandler) + self.server.RequestHandlerClass.pypi_server = self + + self.request_queue = Queue.Queue() + self._requests = [] + self.default_response_status = 200 + self.default_response_headers = [('Content-type', 'text/plain')] + self.default_response_data = "hello" + + # initialize static paths / filesystems + self.static_uri_paths = static_uri_paths + if test_static_path is not None: + static_filesystem_paths.append(test_static_path) + self.static_filesystem_paths = [PYPI_DEFAULT_STATIC_PATH + "/" + path + for path in static_filesystem_paths] + else: + # XMLRPC server + self.server = PyPIXMLRPCServer(('', 0)) + self.xmlrpc = XMLRPCMockIndex() + # register the xmlrpc methods + self.server.register_introspection_functions() + self.server.register_instance(self.xmlrpc) + + self.address = (self.server.server_name, self.server.server_port) + # to not have unwanted outputs. + self.server.RequestHandlerClass.log_request = lambda *_: None def run(self): # loop because we can't stop it otherwise, for python < 2.6 while self._run: - r, w, e = select.select([self.httpd], [], [], 0.5) + r, w, e = select.select([self.server], [], [], 0.5) if r: - self.httpd.handle_request() + self.server.handle_request() def stop(self): """self shutdown is not supported for python < 2.6""" @@ -146,7 +201,7 @@ class PyPIRequestHandler(SimpleHTTPRequestHandler): # serve the content from local disc if we request an URL beginning # by a pattern defined in `static_paths` url_parts = self.path.split("/") - if (len(url_parts) > 1 and + if (len(url_parts) > 1 and url_parts[1] in self.pypi_server.static_uri_paths): data = None # always take the last first. @@ -184,7 +239,7 @@ class PyPIRequestHandler(SimpleHTTPRequestHandler): try: status = int(status) except ValueError: - # we probably got something like YYY Codename. + # we probably got something like YYY Codename. # Just get the first 3 digits status = int(status[:3]) @@ -193,3 +248,181 @@ class PyPIRequestHandler(SimpleHTTPRequestHandler): self.send_header(header, value) self.end_headers() self.wfile.write(data) + +class PyPIXMLRPCServer(SimpleXMLRPCServer): + def server_bind(self): + """Override server_bind to store the server name.""" + SocketServer.TCPServer.server_bind(self) + host, port = self.socket.getsockname()[:2] + self.server_name = socket.getfqdn(host) + self.server_port = port + +class MockDist(object): + """Fake distribution, used in the Mock PyPI Server""" + def __init__(self, name, version="1.0", hidden=False, url="http://url/", + type="sdist", filename="", size=10000, + digest="123456", downloads=7, has_sig=False, + python_version="source", comment="comment", + author="John Doe", author_email="john@doe.name", + maintainer="Main Tayner", maintainer_email="maintainer_mail", + project_url="http://project_url/", homepage="http://homepage/", + keywords="", platform="UNKNOWN", classifiers=[], licence="", + description="Description", summary="Summary", stable_version="", + ordering="", documentation_id="", code_kwalitee_id="", + installability_id="", obsoletes=[], obsoletes_dist=[], + provides=[], provides_dist=[], requires=[], requires_dist=[], + requires_external=[], requires_python=""): + + # basic fields + self.name = name + self.version = version + self.hidden = hidden + + # URL infos + self.url = url + self.digest = digest + self.downloads = downloads + self.has_sig = has_sig + self.python_version = python_version + self.comment = comment + self.type = type + + # metadata + self.author = author + self.author_email = author_email + self.maintainer = maintainer + self.maintainer_email = maintainer_email + self.project_url = project_url + self.homepage = homepage + self.keywords = keywords + self.platform = platform + self.classifiers = classifiers + self.licence = licence + self.description = description + self.summary = summary + self.stable_version = stable_version + self.ordering = ordering + self.cheesecake_documentation_id = documentation_id + self.cheesecake_code_kwalitee_id = code_kwalitee_id + self.cheesecake_installability_id = installability_id + + self.obsoletes = obsoletes + self.obsoletes_dist = obsoletes_dist + self.provides = provides + self.provides_dist = provides_dist + self.requires = requires + self.requires_dist = requires_dist + self.requires_external = requires_external + self.requires_python = requires_python + + def url_infos(self): + return { + 'url': self.url, + 'packagetype': self.type, + 'filename': 'filename.tar.gz', + 'size': '6000', + 'md5_digest': self.digest, + 'downloads': self.downloads, + 'has_sig': self.has_sig, + 'python_version': self.python_version, + 'comment_text': self.comment, + } + + def metadata(self): + return { + 'maintainer': self.maintainer, + 'project_url': [self.project_url], + 'maintainer_email': self.maintainer_email, + 'cheesecake_code_kwalitee_id': self.cheesecake_code_kwalitee_id, + 'keywords': self.keywords, + 'obsoletes_dist': self.obsoletes_dist, + 'requires_external': self.requires_external, + 'author': self.author, + 'author_email': self.author_email, + 'download_url': self.url, + 'platform': self.platform, + 'version': self.version, + 'obsoletes': self.obsoletes, + 'provides': self.provides, + 'cheesecake_documentation_id': self.cheesecake_documentation_id, + '_pypi_hidden': self.hidden, + 'description': self.description, + '_pypi_ordering': 19, + 'requires_dist': self.requires_dist, + 'requires_python': self.requires_python, + 'classifiers': [], + 'name': self.name, + 'licence': self.licence, + 'summary': self.summary, + 'home_page': self.homepage, + 'stable_version': self.stable_version, + 'provides_dist': self.provides_dist or "%s (%s)" % (self.name, + self.version), + 'requires': self.requires, + 'cheesecake_installability_id': self.cheesecake_installability_id, + } + + def search_result(self): + return { + '_pypi_ordering': 0, + 'version': self.version, + 'name': self.name, + 'summary': self.summary, + } + +class XMLRPCMockIndex(object): + """Mock XMLRPC server""" + + def __init__(self, dists=[]): + self._dists = dists + + def add_distributions(self, dists): + for dist in dists: + self._dists.append(MockDist(**dist)) + + def set_distributions(self, dists): + self._dists = [] + self.add_distributions(dists) + + def set_search_result(self, result): + """set a predefined search result""" + self._search_result = result + + def _get_search_results(self): + results = [] + for name in self._search_result: + found_dist = [d for d in self._dists if d.name == name] + if found_dist: + results.append(found_dist[0]) + else: + dist = MockDist(name) + results.append(dist) + self._dists.append(dist) + return [r.search_result() for r in results] + + def list_package(self): + return [d.name for d in self._dists] + + def package_releases(self, package_name, show_hidden=False): + if show_hidden: + # return all + return [d.version for d in self._dists if d.name == package_name] + else: + # return only un-hidden + return [d.version for d in self._dists if d.name == package_name + and not d.hidden] + + def release_urls(self, package_name, version): + return [d.url_infos() for d in self._dists + if d.name == package_name and d.version == version] + + def release_data(self, package_name, version): + release = [d for d in self._dists + if d.name == package_name and d.version == version] + if release: + return release[0].metadata() + else: + return {} + + def search(self, spec, operator="and"): + return self._get_search_results() diff --git a/src/distutils2/tests/pypiserver/project_list/simple/index.html b/src/distutils2/tests/pypiserver/project_list/simple/index.html new file mode 100644 index 0000000..b36d728 --- /dev/null +++ b/src/distutils2/tests/pypiserver/project_list/simple/index.html @@ -0,0 +1,5 @@ +<a class="test" href="yeah">FooBar-bar</a> +<a class="test" href="yeah">Foobar-baz</a> +<a class="test" href="yeah">Baz-FooBar</a> +<a class="test" href="yeah">Baz</a> +<a class="test" href="yeah">Foo</a> diff --git a/src/distutils2/tests/support.py b/src/distutils2/tests/support.py index 518cedf..fa83b30 100644 --- a/src/distutils2/tests/support.py +++ b/src/distutils2/tests/support.py @@ -1,8 +1,29 @@ """Support code for distutils2 test cases. Always import unittest from this module, it will be the right version -(standard library unittest for 2.7 and higher, third-party unittest2 +(standard library unittest for 3.2 and higher, third-party unittest2 release for older versions). + +Three helper classes are provided: LoggingSilencer, TempdirManager and +EnvironGuard. They are written to be used as mixins, e.g. :: + + from distutils2.tests.support import unittest + from distutils2.tests.support import LoggingSilencer + + class SomeTestCase(LoggingSilencer, unittest.TestCase): + +If you need to define a setUp method on your test class, you have to +call the mixin class' setUp method or it won't work (same thing for +tearDown): + + def setUp(self): + super(SomeTestCase, self).setUp() + ... # other setup code + +Read each class' docstring to see its purpose and usage. + +Also provided is a DummyCommand class, useful to mock commands in the +tests of another command that needs them (see docstring). """ import os @@ -10,26 +31,36 @@ import sys import shutil import tempfile from copy import deepcopy -import warnings from distutils2 import log from distutils2.log import DEBUG, INFO, WARN, ERROR, FATAL -if sys.version_info >= (2, 7): - # improved unittest package from 2.7's standard library +if sys.version_info >= (3, 2): + # improved unittest package from 3.2's standard library import unittest else: # external release of same package for older versions import unittest2 as unittest +__all__ = ['LoggingSilencer', 'TempdirManager', 'EnvironGuard', + 'DummyCommand', 'unittest'] + + class LoggingSilencer(object): + """TestCase-compatible mixin to catch logging calls. + + Every log message that goes through distutils2.log will get appended to + self.logs instead of being printed. You can check that your code logs + warnings and errors as documented by inspecting that list; helper methods + get_logs and clear_logs are also provided. + """ def setUp(self): super(LoggingSilencer, self).setUp() - self.threshold = log.set_threshold(log.FATAL) + self.threshold = log.set_threshold(FATAL) # catching warnings - # when log will be replaced by logging - # we won't need such monkey-patch anymore + # when log is replaced by logging we won't need + # such monkey-patching anymore self._old_log = log.Log._log log.Log._log = self._log self.logs = [] @@ -45,6 +76,10 @@ class LoggingSilencer(object): self.logs.append((level, msg, args)) def get_logs(self, *levels): + """Return a list of caught messages with level in `levels`. + + Example: self.get_logs(log.WARN, log.DEBUG) -> list + """ def _format(msg, args): if len(args) == 0: return msg @@ -53,48 +88,41 @@ class LoggingSilencer(object): in self.logs if level in levels] def clear_logs(self): - self.logs = [] + """Empty the internal list of caught messages.""" + del self.logs[:] + class TempdirManager(object): - """Mix-in class that handles temporary directories for test cases. + """TestCase-compatible mixin to create temporary directories and files. - This is intended to be used with unittest.TestCase. + Directories and files created in a test_* method will be removed after it + has run. """ def setUp(self): super(TempdirManager, self).setUp() - self.tempdirs = [] - self.tempfiles = [] + self._basetempdir = tempfile.mkdtemp() def tearDown(self): super(TempdirManager, self).tearDown() - while self.tempdirs: - d = self.tempdirs.pop() - shutil.rmtree(d, os.name in ('nt', 'cygwin')) - for file_ in self.tempfiles: - if os.path.exists(file_): - os.remove(file_) + shutil.rmtree(self._basetempdir, os.name in ('nt', 'cygwin')) def mktempfile(self): - """Create a temporary file that will be cleaned up.""" - tempfile_ = tempfile.NamedTemporaryFile() - self.tempfiles.append(tempfile_.name) - return tempfile_ + """Create a read-write temporary file and return it.""" + fd, fn = tempfile.mkstemp(dir=self._basetempdir) + os.close(fd) + return open(fn, 'w+') def mkdtemp(self): - """Create a temporary directory that will be cleaned up. - - Returns the path of the directory. - """ - d = tempfile.mkdtemp() - self.tempdirs.append(d) + """Create a temporary directory and return its path.""" + d = tempfile.mkdtemp(dir=self._basetempdir) return d def write_file(self, path, content='xxx'): - """Writes a file in the given path. - + """Write a file at the given path. - path can be a string or a sequence. + path can be a string, a tuple or a list; if it's a tuple or list, + os.path.join will be used to produce a path. """ if isinstance(path, (list, tuple)): path = os.path.join(*path) @@ -105,41 +133,35 @@ class TempdirManager(object): f.close() def create_dist(self, pkg_name='foo', **kw): - """Will generate a test environment. + """Create a stub distribution object and files. - This function creates: - - a Distribution instance using keywords - - a temporary directory with a package structure + This function creates a Distribution instance (use keyword arguments + to customize it) and a temporary directory with a project structure + (currently an empty directory). - It returns the package directory and the distribution - instance. + It returns the path to the directory and the Distribution instance. + You can use TempdirManager.write_file to write any file in that + directory, e.g. setup scripts or Python modules. """ + # Late import so that third parties can import support without + # loading a ton of distutils2 modules in memory. from distutils2.dist import Distribution tmp_dir = self.mkdtemp() pkg_dir = os.path.join(tmp_dir, pkg_name) os.mkdir(pkg_dir) dist = Distribution(attrs=kw) - return pkg_dir, dist -class DummyCommand: - """Class to store options for retrieval via set_undefined_options().""" - - def __init__(self, **kwargs): - for kw, val in kwargs.items(): - setattr(self, kw, val) - - def ensure_finalized(self): - pass class EnvironGuard(object): + """TestCase-compatible mixin to save and restore the environment.""" def setUp(self): super(EnvironGuard, self).setUp() self.old_environ = deepcopy(os.environ) def tearDown(self): - for key, value in self.old_environ.items(): + for key, value in self.old_environ.iteritems(): if os.environ.get(key) != value: os.environ[key] = value @@ -148,3 +170,18 @@ class EnvironGuard(object): del os.environ[key] super(EnvironGuard, self).tearDown() + + +class DummyCommand(object): + """Class to store options for retrieval via set_undefined_options(). + + Useful for mocking one dependency command in the tests for another + command, see e.g. the dummy build command in test_build_scripts. + """ + + def __init__(self, **kwargs): + for kw, val in kwargs.iteritems(): + setattr(self, kw, val) + + def ensure_finalized(self): + pass diff --git a/src/distutils2/tests/test_Mixin2to3.py b/src/distutils2/tests/test_Mixin2to3.py index a455fc2..4b397c7 100644 --- a/src/distutils2/tests/test_Mixin2to3.py +++ b/src/distutils2/tests/test_Mixin2to3.py @@ -1,6 +1,5 @@ """Tests for distutils.command.build_py.""" import sys -import tempfile import distutils2 from distutils2.tests import support @@ -10,7 +9,7 @@ from distutils2.command.build_py import Mixin2to3 class Mixin2to3TestCase(support.TempdirManager, unittest.TestCase): - @unittest.skipUnless(sys.version > '2.6', 'Need >= 2.6') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_convert_code_only(self): # used to check if code gets converted properly. code_content = "print 'test'\n" @@ -27,7 +26,7 @@ class Mixin2to3TestCase(support.TempdirManager, unittest.TestCase): self.assertEquals(new_code_content, converted_code_content) - @unittest.skipUnless(sys.version > '2.6', 'Need >= 2.6') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_doctests_only(self): # used to check if doctests gets converted properly. doctest_content = '"""\n>>> print test\ntest\n"""\nprint test\n\n' diff --git a/src/distutils2/tests/test_bdist.py b/src/distutils2/tests/test_bdist.py index 345c74c..2be01e1 100644 --- a/src/distutils2/tests/test_bdist.py +++ b/src/distutils2/tests/test_bdist.py @@ -8,8 +8,7 @@ from distutils2.core import Distribution from distutils2.command.bdist import bdist from distutils2.tests import support from distutils2.tests.support import unittest -from distutils2.spawn import find_executable -from distutils2 import spawn +from distutils2.util import find_executable from distutils2.errors import DistutilsExecError class BuildTestCase(support.TempdirManager, diff --git a/src/distutils2/tests/test_bdist_msi.py b/src/distutils2/tests/test_bdist_msi.py index ebde4db..4ad54ef 100644 --- a/src/distutils2/tests/test_bdist_msi.py +++ b/src/distutils2/tests/test_bdist_msi.py @@ -10,8 +10,8 @@ class BDistMSITestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): - @unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") - def test_minial(self): + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") + def test_minimal(self): # minimal test XXX need more tests from distutils2.command.bdist_msi import bdist_msi pkg_pth, dist = self.create_dist() diff --git a/src/distutils2/tests/test_build_clib.py b/src/distutils2/tests/test_build_clib.py index 48af7e1..348f700 100644 --- a/src/distutils2/tests/test_build_clib.py +++ b/src/distutils2/tests/test_build_clib.py @@ -5,7 +5,7 @@ import sys from distutils2.command.build_clib import build_clib from distutils2.errors import DistutilsSetupError from distutils2.tests import support -from distutils2.spawn import find_executable +from distutils2.util import find_executable from distutils2.tests.support import unittest class BuildCLibTestCase(support.TempdirManager, diff --git a/src/distutils2/tests/test_build_ext.py b/src/distutils2/tests/test_build_ext.py index f561923..1ae0bd8 100644 --- a/src/distutils2/tests/test_build_ext.py +++ b/src/distutils2/tests/test_build_ext.py @@ -45,7 +45,7 @@ class BuildExtTestCase(support.TempdirManager, build_ext.USER_BASE = site.USER_BASE # XXX only works with 2.6 > -- dunno why yet - @unittest.skipUnless(sys.version_info >= (2, 6,), 'works for >= 2.6') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_build_ext(self): global ALREADY_TESTED xx_c = os.path.join(self.tmp_dir, 'xxmodule.c') @@ -126,11 +126,8 @@ class BuildExtTestCase(support.TempdirManager, # make sure we get some library dirs under solaris self.assertTrue(len(cmd.library_dirs) > 0) + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_user_site(self): - # site.USER_SITE was introduced in 2.6 - if sys.version < '2.6': - return - import site dist = Distribution({'name': 'xx'}) cmd = build_ext(dist) diff --git a/src/distutils2/tests/test_build_py.py b/src/distutils2/tests/test_build_py.py index 4b46da9..aed24d3 100644 --- a/src/distutils2/tests/test_build_py.py +++ b/src/distutils2/tests/test_build_py.py @@ -95,7 +95,7 @@ class BuildPyTestCase(support.TempdirManager, sys.stdout = old_stdout @unittest.skipUnless(hasattr(sys, 'dont_write_bytecode'), - 'dont_write_bytecode support') + 'sys.dont_write_bytecode not supported') def test_dont_write_bytecode(self): # makes sure byte_compile is not used pkg_dir, dist = self.create_dist() diff --git a/src/distutils2/tests/test_check.py b/src/distutils2/tests/test_check.py index 7a672e1..02d171f 100644 --- a/src/distutils2/tests/test_check.py +++ b/src/distutils2/tests/test_check.py @@ -43,7 +43,7 @@ class CheckTestCase(support.LoggingSilencer, # get an error if there are missing metadata self.assertRaises(DistutilsSetupError, self._run, {}, **{'strict': 1}) - # and of course, no error when all metadata are present + # and of course, no error when all metadata fields are present cmd = self._run(metadata, strict=1) self.assertEqual(len(cmd._warnings), 0) diff --git a/src/distutils2/tests/test_cmd.py b/src/distutils2/tests/test_cmd.py index aa4a709..c768f79 100644 --- a/src/distutils2/tests/test_cmd.py +++ b/src/distutils2/tests/test_cmd.py @@ -98,7 +98,7 @@ class CommandTestCase(unittest.TestCase): def test_ensure_dirname(self): cmd = self.cmd - cmd.option1 = os.path.dirname(__file__) + cmd.option1 = os.path.dirname(__file__) or os.curdir cmd.ensure_dirname('option1') cmd.option2 = 'xxx' self.assertRaises(DistutilsOptionError, cmd.ensure_dirname, 'option2') diff --git a/src/distutils2/tests/test_config.py b/src/distutils2/tests/test_config.py deleted file mode 100644 index 09b6d02..0000000 --- a/src/distutils2/tests/test_config.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Tests for distutils.pypirc.pypirc.""" -import sys -import os -import shutil - -from distutils2.core import PyPIRCCommand -from distutils2.core import Distribution -from distutils2.log import set_threshold -from distutils2.log import WARN - -from distutils2.tests import support -from distutils2.tests.support import unittest - -PYPIRC = """\ -[distutils] - -index-servers = - server1 - server2 - -[server1] -username:me -password:secret - -[server2] -username:meagain -password: secret -realm:acme -repository:http://another.pypi/ -""" - -PYPIRC_OLD = """\ -[server-login] -username:tarek -password:secret -""" - -WANTED = """\ -[distutils] -index-servers = - pypi - -[pypi] -username:tarek -password:xxx -""" - - -class PyPIRCCommandTestCase(support.TempdirManager, - support.LoggingSilencer, - support.EnvironGuard, - unittest.TestCase): - - def setUp(self): - """Patches the environment.""" - super(PyPIRCCommandTestCase, self).setUp() - self.tmp_dir = self.mkdtemp() - os.environ['HOME'] = self.tmp_dir - self.rc = os.path.join(self.tmp_dir, '.pypirc') - self.dist = Distribution() - - class command(PyPIRCCommand): - def __init__(self, dist): - PyPIRCCommand.__init__(self, dist) - def initialize_options(self): - pass - finalize_options = initialize_options - - self._cmd = command - self.old_threshold = set_threshold(WARN) - - def tearDown(self): - """Removes the patch.""" - set_threshold(self.old_threshold) - super(PyPIRCCommandTestCase, self).tearDown() - - def test_server_registration(self): - # This test makes sure PyPIRCCommand knows how to: - # 1. handle several sections in .pypirc - # 2. handle the old format - - # new format - self.write_file(self.rc, PYPIRC) - cmd = self._cmd(self.dist) - config = cmd._read_pypirc() - - config = config.items() - config.sort() - expected = [('password', 'secret'), ('realm', 'pypi'), - ('repository', 'http://pypi.python.org/pypi'), - ('server', 'server1'), ('username', 'me')] - self.assertEqual(config, expected) - - # old format - self.write_file(self.rc, PYPIRC_OLD) - config = cmd._read_pypirc() - config = config.items() - config.sort() - expected = [('password', 'secret'), ('realm', 'pypi'), - ('repository', 'http://pypi.python.org/pypi'), - ('server', 'server-login'), ('username', 'tarek')] - self.assertEqual(config, expected) - - def test_server_empty_registration(self): - cmd = self._cmd(self.dist) - rc = cmd._get_rc_file() - self.assertTrue(not os.path.exists(rc)) - cmd._store_pypirc('tarek', 'xxx') - self.assertTrue(os.path.exists(rc)) - content = open(rc).read() - self.assertEqual(content, WANTED) - -def test_suite(): - return unittest.makeSuite(PyPIRCCommandTestCase) - -if __name__ == "__main__": - unittest.main(defaultTest="test_suite") diff --git a/src/distutils2/tests/test_converter.py b/src/distutils2/tests/test_converter.py index f0717cc..c4cb387 100644 --- a/src/distutils2/tests/test_converter.py +++ b/src/distutils2/tests/test_converter.py @@ -17,7 +17,7 @@ def _read_file(path): class ConverterTestCase(unittest.TestCase): - @unittest.skipUnless(not sys.version < '2.6', 'Needs Python >=2.6') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_conversions(self): # for all XX_before in the conversions/ dir # we run the refactoring tool diff --git a/src/distutils2/tests/test_core.py b/src/distutils2/tests/test_core.py index 9f4ad27..883187e 100644 --- a/src/distutils2/tests/test_core.py +++ b/src/distutils2/tests/test_core.py @@ -64,13 +64,13 @@ class CoreTestCase(support.EnvironGuard, unittest.TestCase): f = self.write_setup(setup_using___file__) for s in ['init', 'config', 'commandline', 'run']: distutils2.core.run_setup(f, stop_after=s) - self.assertRaises(ValueError, distutils2.core.run_setup, + self.assertRaises(ValueError, distutils2.core.run_setup, f, stop_after='bob') def test_run_setup_args(self): f = self.write_setup(setup_using___file__) - d = distutils2.core.run_setup(f, script_args=["--help"], - stop_after="init") + d = distutils2.core.run_setup(f, script_args=["--help"], + stop_after="init") self.assertEqual(['--help'], d.script_args) def test_run_setup_uses_current_dir(self): diff --git a/src/distutils2/tests/test_dist.py b/src/distutils2/tests/test_dist.py index f528684..290ce01 100644 --- a/src/distutils2/tests/test_dist.py +++ b/src/distutils2/tests/test_dist.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- """Tests for distutils2.dist.""" import os @@ -153,6 +153,27 @@ class DistributionTestCase(support.TempdirManager, my_file2 = os.path.join(tmp_dir, 'f2') dist.metadata.write_file(open(my_file, 'w')) + def test_bad_attr(self): + cls = Distribution + + # catching warnings + warns = [] + def _warn(msg): + warns.append(msg) + + old_warn = warnings.warn + warnings.warn = _warn + try: + dist = cls(attrs={'author': 'xxx', + 'name': 'xxx', + 'version': 'xxx', + 'url': 'xxxx', + 'badoptname': 'xxx'}) + finally: + warnings.warn = old_warn + + self.assertTrue(len(warns)==1 and "Unknown distribution" in warns[0]) + def test_empty_options(self): # an empty options dictionary should not stay in the # list of attributes @@ -176,6 +197,21 @@ class DistributionTestCase(support.TempdirManager, self.assertEqual(len(warns), 0) + def test_non_empty_options(self): + # TODO: how to actually use options is not documented except + # for a few cryptic comments in dist.py. If this is to stay + # in the public API, it deserves some better documentation. + + # Here is an example of how it's used out there: + # http://svn.pythonmac.org/py2app/py2app/trunk/doc/index.html#specifying-customizations + cls = Distribution + dist = cls(attrs={'author': 'xxx', + 'name': 'xxx', + 'version': 'xxx', + 'url': 'xxxx', + 'options': dict(sdist=dict(owner="root"))}) + self.assertTrue("owner" in dist.get_option_dict("sdist")) + def test_finalize_options(self): attrs = {'keywords': 'one,two', @@ -240,6 +276,49 @@ class DistributionTestCase(support.TempdirManager, # make sure --no-user-cfg disables the user cfg file self.assertEqual(len(all_files)-1, len(files)) + def test_special_hooks_parsing(self): + temp_home = self.mkdtemp() + config_files = [os.path.join(temp_home, "config1.cfg"), + os.path.join(temp_home, "config2.cfg")] + + # Store two aliased hooks in config files + self.write_file((temp_home, "config1.cfg"), '[test_dist]\npre-hook.a = type') + self.write_file((temp_home, "config2.cfg"), '[test_dist]\npre-hook.b = type') + + sys.argv.extend(["--command-packages", + "distutils2.tests", + "test_dist"]) + cmd = self.create_distribution(config_files).get_command_obj("test_dist") + self.assertEqual(cmd.pre_hook, {"a": 'type', "b": 'type'}) + + + def test_hooks_get_run(self): + temp_home = self.mkdtemp() + config_file = os.path.join(temp_home, "config1.cfg") + + self.write_file((temp_home, "config1.cfg"), textwrap.dedent(''' + [test_dist] + pre-hook.test = distutils2.tests.test_dist.DistributionTestCase.log_pre_call + post-hook.test = distutils2.tests.test_dist.DistributionTestCase.log_post_call''')) + + sys.argv.extend(["--command-packages", + "distutils2.tests", + "test_dist"]) + d = self.create_distribution([config_file]) + cmd = d.get_command_obj("test_dist") + + # prepare the call recorders + record = [] + DistributionTestCase.log_pre_call = staticmethod(lambda _cmd: record.append(('pre', _cmd))) + DistributionTestCase.log_post_call = staticmethod(lambda _cmd: record.append(('post', _cmd))) + test_dist.run = lambda _cmd: record.append(('run', _cmd)) + test_dist.finalize_options = lambda _cmd: record.append(('finalize_options', _cmd)) + + d.run_command('test_dist') + self.assertEqual(record, [('finalize_options', cmd), + ('pre', cmd), + ('run', cmd), + ('post', cmd)]) class MetadataTestCase(support.TempdirManager, support.EnvironGuard, unittest.TestCase): diff --git a/src/distutils2/tests/test_index_dist.py b/src/distutils2/tests/test_index_dist.py new file mode 100644 index 0000000..9020635 --- /dev/null +++ b/src/distutils2/tests/test_index_dist.py @@ -0,0 +1,248 @@ +"""Tests for the distutils2.index.dist module.""" + +import os + +from distutils2.tests.pypi_server import use_pypi_server +from distutils2.tests import run_unittest +from distutils2.tests.support import unittest, TempdirManager +from distutils2.version import VersionPredicate +from distutils2.index.errors import HashDoesNotMatch, UnsupportedHashName +from distutils2.index.dist import (ReleaseInfo, ReleasesList, DistInfo, + split_archive_name, get_infos_from_url) + + +def Dist(*args, **kwargs): + # DistInfo takes a release as a first parameter, avoid this in tests. + return DistInfo(None, *args, **kwargs) + + +class TestReleaseInfo(unittest.TestCase): + + def test_instantiation(self): + # Test the DistInfo class provides us the good attributes when + # given on construction + release = ReleaseInfo("FooBar", "1.1") + self.assertEqual("FooBar", release.name) + self.assertEqual("1.1", "%s" % release.version) + + def test_add_dist(self): + # empty distribution type should assume "sdist" + release = ReleaseInfo("FooBar", "1.1") + release.add_distribution(url="http://example.org/") + # should not fail + release['sdist'] + + def test_get_unknown_distribution(self): + # should raise a KeyError + pass + + def test_get_infos_from_url(self): + # Test that the the URLs are parsed the right way + url_list = { + 'FooBar-1.1.0.tar.gz': { + 'name': 'foobar', # lowercase the name + 'version': '1.1.0', + }, + 'Foo-Bar-1.1.0.zip': { + 'name': 'foo-bar', # keep the dash + 'version': '1.1.0', + }, + 'foobar-1.1b2.tar.gz#md5=123123123123123': { + 'name': 'foobar', + 'version': '1.1b2', + 'url': 'http://example.org/foobar-1.1b2.tar.gz', # no hash + 'hashval': '123123123123123', + 'hashname': 'md5', + }, + 'foobar-1.1-rc2.tar.gz': { # use suggested name + 'name': 'foobar', + 'version': '1.1c2', + 'url': 'http://example.org/foobar-1.1-rc2.tar.gz', + } + } + + for url, attributes in url_list.items(): + # for each url + infos = get_infos_from_url("http://example.org/" + url) + for attribute, expected in attributes.items(): + got = infos.get(attribute) + if attribute == "version": + self.assertEqual("%s" % got, expected) + else: + self.assertEqual(got, expected) + + def test_split_archive_name(self): + # Test we can split the archive names + names = { + 'foo-bar-baz-1.0-rc2': ('foo-bar-baz', '1.0c2'), + 'foo-bar-baz-1.0': ('foo-bar-baz', '1.0'), + 'foobarbaz-1.0': ('foobarbaz', '1.0'), + } + for name, results in names.items(): + self.assertEqual(results, split_archive_name(name)) + + +class TestDistInfo(TempdirManager, unittest.TestCase): + + def test_get_url(self): + # Test that the url property works well + + d = Dist(url="test_url") + self.assertDictEqual(d.url, { + "url": "test_url", + "is_external": True, + "hashname": None, + "hashval": None, + }) + + # add a new url + d.add_url(url="internal_url", is_external=False) + self.assertEqual(d._url, None) + self.assertDictEqual(d.url, { + "url": "internal_url", + "is_external": False, + "hashname": None, + "hashval": None, + }) + self.assertEqual(2, len(d.urls)) + + def test_comparison(self): + # Test that we can compare DistInfoributionInfoList + foo1 = ReleaseInfo("foo", "1.0") + foo2 = ReleaseInfo("foo", "2.0") + bar = ReleaseInfo("bar", "2.0") + # assert we use the version to compare + self.assertTrue(foo1 < foo2) + self.assertFalse(foo1 > foo2) + self.assertFalse(foo1 == foo2) + + # assert we can't compare dists with different names + self.assertRaises(TypeError, foo1.__eq__, bar) + + @use_pypi_server("downloads_with_md5") + def test_download(self, server): + # Download is possible, and the md5 is checked if given + + url = "%s/simple/foobar/foobar-0.1.tar.gz" % server.full_address + # check md5 if given + dist = Dist(url=url, hashname="md5", + hashval="d41d8cd98f00b204e9800998ecf8427e") + dist.download(self.mkdtemp()) + + # a wrong md5 fails + dist2 = Dist(url=url, hashname="md5", hashval="wrongmd5") + + self.assertRaises(HashDoesNotMatch, dist2.download, self.mkdtemp()) + + # we can omit the md5 hash + dist3 = Dist(url=url) + dist3.download(self.mkdtemp()) + + # and specify a temporary location + # for an already downloaded dist + path1 = self.mkdtemp() + dist3.download(path=path1) + # and for a new one + path2_base = self.mkdtemp() + dist4 = Dist(url=url) + path2 = dist4.download(path=path2_base) + self.assertTrue(path2_base in path2) + + def test_hashname(self): + # Invalid hashnames raises an exception on assignation + Dist(hashname="md5", hashval="value") + + self.assertRaises(UnsupportedHashName, Dist, + hashname="invalid_hashname", + hashval="value") + + +class TestReleasesList(unittest.TestCase): + + def test_filter(self): + # Test we filter the distributions the right way, using version + # predicate match method + releases = ReleasesList('FooBar', ( + ReleaseInfo("FooBar", "1.1"), + ReleaseInfo("FooBar", "1.1.1"), + ReleaseInfo("FooBar", "1.2"), + ReleaseInfo("FooBar", "1.2.1"), + )) + filtered = releases.filter(VersionPredicate("FooBar (<1.2)")) + self.assertNotIn(releases[2], filtered) + self.assertNotIn(releases[3], filtered) + self.assertIn(releases[0], filtered) + self.assertIn(releases[1], filtered) + + def test_append(self): + # When adding a new item to the list, the behavior is to test if + # a release with the same name and version number already exists, + # and if so, to add a new distribution for it. If the distribution type + # is already defined too, add url informations to the existing DistInfo + # object. + + releases = ReleasesList("FooBar", [ + ReleaseInfo("FooBar", "1.1", url="external_url", + dist_type="sdist"), + ]) + self.assertEqual(1, len(releases)) + releases.add_release(release=ReleaseInfo("FooBar", "1.1", + url="internal_url", + is_external=False, + dist_type="sdist")) + self.assertEqual(1, len(releases)) + self.assertEqual(2, len(releases[0]['sdist'].urls)) + + releases.add_release(release=ReleaseInfo("FooBar", "1.1.1", + dist_type="sdist")) + self.assertEqual(2, len(releases)) + + # when adding a distribution whith a different type, a new distribution + # has to be added. + releases.add_release(release=ReleaseInfo("FooBar", "1.1.1", + dist_type="bdist")) + self.assertEqual(2, len(releases)) + self.assertEqual(2, len(releases[1].dists)) + + def test_prefer_final(self): + # Can order the distributions using prefer_final + + fb10 = ReleaseInfo("FooBar", "1.0") # final distribution + fb11a = ReleaseInfo("FooBar", "1.1a1") # alpha + fb12a = ReleaseInfo("FooBar", "1.2a1") # alpha + fb12b = ReleaseInfo("FooBar", "1.2b1") # beta + dists = ReleasesList("FooBar", [fb10, fb11a, fb12a, fb12b]) + + dists.sort_releases(prefer_final=True) + self.assertEqual(fb10, dists[0]) + + dists.sort_releases(prefer_final=False) + self.assertEqual(fb12b, dists[0]) + +# def test_prefer_source(self): +# # Ordering support prefer_source +# fb_source = Dist("FooBar", "1.0", type="source") +# fb_binary = Dist("FooBar", "1.0", type="binary") +# fb2_binary = Dist("FooBar", "2.0", type="binary") +# dists = ReleasesList([fb_binary, fb_source]) +# +# dists.sort_distributions(prefer_source=True) +# self.assertEqual(fb_source, dists[0]) +# +# dists.sort_distributions(prefer_source=False) +# self.assertEqual(fb_binary, dists[0]) +# +# dists.append(fb2_binary) +# dists.sort_distributions(prefer_source=True) +# self.assertEqual(fb2_binary, dists[0]) + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestDistInfo)) + suite.addTest(unittest.makeSuite(TestReleaseInfo)) + suite.addTest(unittest.makeSuite(TestReleasesList)) + return suite + +if __name__ == '__main__': + run_unittest(test_suite()) diff --git a/src/distutils2/tests/test_pypi_simple.py b/src/distutils2/tests/test_index_simple.py index 85d8b77..d21bda5 100644 --- a/src/distutils2/tests/test_pypi_simple.py +++ b/src/distutils2/tests/test_index_simple.py @@ -3,36 +3,34 @@ """ import sys import os -import shutil -import tempfile import urllib2 -from distutils2.pypi import simple -from distutils2.tests import support, run_unittest +from distutils2.index.simple import Crawler +from distutils2.tests import support from distutils2.tests.support import unittest from distutils2.tests.pypi_server import (use_pypi_server, PyPIServer, PYPI_DEFAULT_STATIC_PATH) -class PyPISimpleTestCase(support.TempdirManager, - unittest.TestCase): +class SimpleCrawlerTestCase(support.TempdirManager, unittest.TestCase): - def _get_simple_index(self, server, base_url="/simple/", hosts=None, + def _get_simple_crawler(self, server, base_url="/simple/", hosts=None, *args, **kwargs): - """Build and return a SimpleSimpleIndex instance, with the test server + """Build and return a SimpleIndex instance, with the test server urls """ if hosts is None: hosts = (server.full_address.strip("http://"),) kwargs['hosts'] = hosts - return simple.SimpleIndex(server.full_address + base_url, *args, + return Crawler(server.full_address + base_url, *args, **kwargs) - def test_bad_urls(self): - index = simple.SimpleIndex() + @use_pypi_server() + def test_bad_urls(self, server): + crawler = Crawler() url = 'http://127.0.0.1:0/nonesuch/test_simple' try: - v = index._open_url(url) + v = crawler._open_url(url) except Exception, v: self.assertTrue(url in str(v)) else: @@ -41,10 +39,10 @@ class PyPISimpleTestCase(support.TempdirManager, # issue 16 # easy_install inquant.contentmirror.plone breaks because of a typo # in its home URL - index = simple.SimpleIndex(hosts=('www.example.com',)) + crawler = Crawler(hosts=('example.org',)) url = 'url:%20https://svn.plone.org/svn/collective/inquant.contentmirror.plone/trunk' try: - v = index._open_url(url) + v = crawler._open_url(url) except Exception, v: self.assertTrue(url in str(v)) else: @@ -56,10 +54,10 @@ class PyPISimpleTestCase(support.TempdirManager, old_urlopen = urllib2.urlopen urllib2.urlopen = _urlopen - url = 'http://example.com' + url = 'http://example.org' try: try: - v = index._open_url(url) + v = crawler._open_url(url) except Exception, v: self.assertTrue('line' in str(v)) else: @@ -70,91 +68,91 @@ class PyPISimpleTestCase(support.TempdirManager, # issue 20 url = 'http://http://svn.pythonpaste.org/Paste/wphp/trunk' try: - index._open_url(url) + crawler._open_url(url) except Exception, v: self.assertTrue('nonnumeric port' in str(v)) # issue #160 if sys.version_info[0] == 2 and sys.version_info[1] == 7: # this should not fail - url = 'http://example.com' + url = server.full_address page = ('<a href="http://www.famfamfam.com](' 'http://www.famfamfam.com/">') - index._process_url(url, page) + crawler._process_url(url, page) @use_pypi_server("test_found_links") def test_found_links(self, server): - # Browse the index, asking for a specified distribution version + # Browse the index, asking for a specified release version # The PyPI index contains links for version 1.0, 1.1, 2.0 and 2.0.1 - index = self._get_simple_index(server) - last_distribution = index.get("foobar") + crawler = self._get_simple_crawler(server) + last_release = crawler.get_release("foobar") # we have scanned the index page self.assertIn(server.full_address + "/simple/foobar/", - index._processed_urls) + crawler._processed_urls) - # we have found 4 distributions in this page - self.assertEqual(len(index._distributions["foobar"]), 4) + # we have found 4 releases in this page + self.assertEqual(len(crawler._projects["foobar"]), 4) # and returned the most recent one - self.assertEqual("%s" % last_distribution.version, '2.0.1') + self.assertEqual("%s" % last_release.version, '2.0.1') def test_is_browsable(self): - index = simple.SimpleIndex(follow_externals=False) - self.assertTrue(index._is_browsable(index.index_url + "test")) + crawler = Crawler(follow_externals=False) + self.assertTrue(crawler._is_browsable(crawler.index_url + "test")) # Now, when following externals, we can have a list of hosts to trust. # and don't follow other external links than the one described here. - index = simple.SimpleIndex(hosts=["pypi.python.org", "test.org"], + crawler = Crawler(hosts=["pypi.python.org", "example.org"], follow_externals=True) good_urls = ( "http://pypi.python.org/foo/bar", "http://pypi.python.org/simple/foobar", - "http://test.org", - "http://test.org/", - "http://test.org/simple/", + "http://example.org", + "http://example.org/", + "http://example.org/simple/", ) bad_urls = ( "http://python.org", - "http://test.tld", + "http://example.tld", ) for url in good_urls: - self.assertTrue(index._is_browsable(url)) + self.assertTrue(crawler._is_browsable(url)) for url in bad_urls: - self.assertFalse(index._is_browsable(url)) + self.assertFalse(crawler._is_browsable(url)) # allow all hosts - index = simple.SimpleIndex(follow_externals=True, hosts=("*",)) - self.assertTrue(index._is_browsable("http://an-external.link/path")) - self.assertTrue(index._is_browsable("pypi.test.tld/a/path")) + crawler = Crawler(follow_externals=True, hosts=("*",)) + self.assertTrue(crawler._is_browsable("http://an-external.link/path")) + self.assertTrue(crawler._is_browsable("pypi.example.org/a/path")) # specify a list of hosts we want to allow - index = simple.SimpleIndex(follow_externals=True, - hosts=("*.test.tld",)) - self.assertFalse(index._is_browsable("http://an-external.link/path")) - self.assertTrue(index._is_browsable("http://pypi.test.tld/a/path")) + crawler = Crawler(follow_externals=True, + hosts=("*.example.org",)) + self.assertFalse(crawler._is_browsable("http://an-external.link/path")) + self.assertTrue(crawler._is_browsable("http://pypi.example.org/a/path")) @use_pypi_server("with_externals") - def test_restrict_hosts(self, server): + def test_follow_externals(self, server): # Include external pages # Try to request the package index, wich contains links to "externals" # resources. They have to be scanned too. - index = self._get_simple_index(server, follow_externals=True) - index.get("foobar") + crawler = self._get_simple_crawler(server, follow_externals=True) + crawler.get_release("foobar") self.assertIn(server.full_address + "/external/external.html", - index._processed_urls) + crawler._processed_urls) @use_pypi_server("with_real_externals") def test_restrict_hosts(self, server): # Only use a list of allowed hosts is possible # Test that telling the simple pyPI client to not retrieve external # works - index = self._get_simple_index(server, follow_externals=False) - index.get("foobar") + crawler = self._get_simple_crawler(server, follow_externals=False) + crawler.get_release("foobar") self.assertNotIn(server.full_address + "/external/external.html", - index._processed_urls) + crawler._processed_urls) @use_pypi_server(static_filesystem_paths=["with_externals"], static_uri_paths=["simple", "external"]) @@ -168,23 +166,26 @@ class PyPISimpleTestCase(support.TempdirManager, # - someone manually coindexes this link (with the md5 in the url) onto # an external page accessible from the package page. # - someone reuploads the package (with a different md5) - # - while easy_installing, an MD5 error occurs because the external link - # is used + # - while easy_installing, an MD5 error occurs because the external + # link is used # -> The index should use the link from pypi, not the external one. # start an index server index_url = server.full_address + '/simple/' # scan a test index - index = simple.SimpleIndex(index_url, follow_externals=True) - dists = index.find("foobar") + crawler = Crawler(index_url, follow_externals=True) + releases = crawler.get_releases("foobar") server.stop() # we have only one link, because links are compared without md5 - self.assertEqual(len(dists), 1) + self.assertEqual(1, len(releases)) + self.assertEqual(1, len(releases[0].dists)) # the link should be from the index - self.assertEqual('12345678901234567', dists[0].url['hashval']) - self.assertEqual('md5', dists[0].url['hashname']) + self.assertEqual(2, len(releases[0].dists['sdist'].urls)) + self.assertEqual('12345678901234567', + releases[0].dists['sdist'].url['hashval']) + self.assertEqual('md5', releases[0].dists['sdist'].url['hashname']) @use_pypi_server(static_filesystem_paths=["with_norel_links"], static_uri_paths=["simple", "external"]) @@ -194,22 +195,22 @@ class PyPISimpleTestCase(support.TempdirManager, # to not be processed by the package index, while processing "pages". # process the pages - index = self._get_simple_index(server, follow_externals=True) - index.find("foobar") + crawler = self._get_simple_crawler(server, follow_externals=True) + crawler.get_releases("foobar") # now it should have processed only pages with links rel="download" # and rel="homepage" self.assertIn("%s/simple/foobar/" % server.full_address, - index._processed_urls) # it's the simple index page + crawler._processed_urls) # it's the simple index page self.assertIn("%s/external/homepage.html" % server.full_address, - index._processed_urls) # the external homepage is rel="homepage" + crawler._processed_urls) # the external homepage is rel="homepage" self.assertNotIn("%s/external/nonrel.html" % server.full_address, - index._processed_urls) # this link contains no rel=* + crawler._processed_urls) # this link contains no rel=* self.assertNotIn("%s/unrelated-0.2.tar.gz" % server.full_address, - index._processed_urls) # linked from simple index (no rel) + crawler._processed_urls) # linked from simple index (no rel) self.assertIn("%s/foobar-0.1.tar.gz" % server.full_address, - index._processed_urls) # linked from simple index (rel) + crawler._processed_urls) # linked from simple index (rel) self.assertIn("%s/foobar-2.0.tar.gz" % server.full_address, - index._processed_urls) # linked from external homepage (rel) + crawler._processed_urls) # linked from external homepage (rel) def test_uses_mirrors(self): # When the main repository seems down, try using the given mirrors""" @@ -219,18 +220,18 @@ class PyPISimpleTestCase(support.TempdirManager, try: # create the index using both servers - index = simple.SimpleIndex(server.full_address + "/simple/", + crawler = Crawler(server.full_address + "/simple/", hosts=('*',), timeout=1, # set the timeout to 1s for the tests - mirrors=[mirror.full_address + "/simple/",]) + mirrors=[mirror.full_address]) # this should not raise a timeout - self.assertEqual(4, len(index.find("foo"))) + self.assertEqual(4, len(crawler.get_releases("foo"))) finally: mirror.stop() def test_simple_link_matcher(self): # Test that the simple link matcher yields the right links""" - index = simple.SimpleIndex(follow_externals=False) + crawler = Crawler(follow_externals=False) # Here, we define: # 1. one link that must be followed, cause it's a download one @@ -238,27 +239,33 @@ class PyPISimpleTestCase(support.TempdirManager, # returns false for it. # 3. one link that must be followed cause it's a homepage that is # browsable - self.assertTrue(index._is_browsable("%stest" % index.index_url)) - self.assertFalse(index._is_browsable("http://dl-link2")) + # 4. one link that must be followed, because it contain a md5 hash + self.assertTrue(crawler._is_browsable("%stest" % crawler.index_url)) + self.assertFalse(crawler._is_browsable("http://dl-link2")) content = """ <a href="http://dl-link1" rel="download">download_link1</a> <a href="http://dl-link2" rel="homepage">homepage_link1</a> - <a href="%stest" rel="homepage">homepage_link2</a> - """ % index.index_url + <a href="%(index_url)stest" rel="homepage">homepage_link2</a> + <a href="%(index_url)stest/foobar-1.tar.gz#md5=abcdef>download_link2</a> + """ % {'index_url': crawler.index_url } # Test that the simple link matcher yield the good links. - generator = index._simple_link_matcher(content, index.index_url) + generator = crawler._simple_link_matcher(content, crawler.index_url) + self.assertEqual(('%stest/foobar-1.tar.gz#md5=abcdef' % crawler.index_url, + True), generator.next()) self.assertEqual(('http://dl-link1', True), generator.next()) - self.assertEqual(('%stest' % index.index_url, False), + self.assertEqual(('%stest' % crawler.index_url, False), generator.next()) self.assertRaises(StopIteration, generator.next) - # Follow the external links is possible - index.follow_externals = True - generator = index._simple_link_matcher(content, index.index_url) + # Follow the external links is possible (eg. homepages) + crawler.follow_externals = True + generator = crawler._simple_link_matcher(content, crawler.index_url) + self.assertEqual(('%stest/foobar-1.tar.gz#md5=abcdef' % crawler.index_url, + True), generator.next()) self.assertEqual(('http://dl-link1', True), generator.next()) self.assertEqual(('http://dl-link2', False), generator.next()) - self.assertEqual(('%stest' % index.index_url, False), + self.assertEqual(('%stest' % crawler.index_url, False), generator.next()) self.assertRaises(StopIteration, generator.next) @@ -266,12 +273,44 @@ class PyPISimpleTestCase(support.TempdirManager, # Test that we can browse local files""" index_path = os.sep.join(["file://" + PYPI_DEFAULT_STATIC_PATH, "test_found_links", "simple"]) - index = simple.SimpleIndex(index_path) - dists = index.find("foobar") + crawler = Crawler(index_path) + dists = crawler.get_releases("foobar") self.assertEqual(4, len(dists)) + def test_get_link_matcher(self): + crawler = Crawler("http://example.org") + self.assertEqual('_simple_link_matcher', crawler._get_link_matcher( + "http://example.org/some/file").__name__) + self.assertEqual('_default_link_matcher', crawler._get_link_matcher( + "http://other-url").__name__) + + def test_default_link_matcher(self): + crawler = Crawler("http://example.org", mirrors=[]) + crawler.follow_externals = True + crawler._is_browsable = lambda *args:True + base_url = "http://example.org/some/file/" + content = """ +<a href="../homepage" rel="homepage">link</a> +<a href="../download" rel="download">link2</a> +<a href="../simpleurl">link2</a> + """ + found_links = dict(crawler._default_link_matcher(content, + base_url)).keys() + self.assertIn('http://example.org/some/homepage', found_links) + self.assertIn('http://example.org/some/simpleurl', found_links) + self.assertIn('http://example.org/some/download', found_links) + + @use_pypi_server("project_list") + def test_search_projects(self, server): + # we can search the index for some projects, on their names + # the case used no matters here + crawler = self._get_simple_crawler(server) + projects = [p.name for p in crawler.search_projects("Foobar")] + self.assertListEqual(['FooBar-bar', 'Foobar-baz', 'Baz-FooBar'], + projects) + def test_suite(): - return unittest.makeSuite(PyPISimpleTestCase) + return unittest.makeSuite(SimpleCrawlerTestCase) if __name__ == '__main__': unittest.main(defaultTest="test_suite") diff --git a/src/distutils2/tests/test_index_xmlrpc.py b/src/distutils2/tests/test_index_xmlrpc.py new file mode 100644 index 0000000..50ea739 --- /dev/null +++ b/src/distutils2/tests/test_index_xmlrpc.py @@ -0,0 +1,92 @@ +"""Tests for the distutils2.index.xmlrpc module.""" + +from distutils2.tests.pypi_server import use_xmlrpc_server +from distutils2.tests import run_unittest +from distutils2.tests.support import unittest +from distutils2.index.xmlrpc import Client, InvalidSearchField, ProjectNotFound + + +class TestXMLRPCClient(unittest.TestCase): + def _get_client(self, server, *args, **kwargs): + return Client(server.full_address, *args, **kwargs) + + @use_xmlrpc_server() + def test_search_projects(self, server): + client = self._get_client(server) + server.xmlrpc.set_search_result(['FooBar', 'Foo', 'FooFoo']) + results = [r.name for r in client.search_projects(name='Foo')] + self.assertEqual(3, len(results)) + self.assertIn('FooBar', results) + self.assertIn('Foo', results) + self.assertIn('FooFoo', results) + + def test_search_projects_bad_fields(self): + client = Client() + self.assertRaises(InvalidSearchField, client.search_projects, + invalid="test") + + @use_xmlrpc_server() + def test_get_releases(self, server): + client = self._get_client(server) + server.xmlrpc.set_distributions([ + {'name': 'FooBar', 'version': '1.1'}, + {'name': 'FooBar', 'version': '1.2', 'url': 'http://some/url/'}, + {'name': 'FooBar', 'version': '1.3', 'url': 'http://other/url/'}, + ]) + + # use a lambda here to avoid an useless mock call + server.xmlrpc.list_releases = lambda *a, **k: ['1.1', '1.2', '1.3'] + + releases = client.get_releases('FooBar (<=1.2)') + # dont call release_data and release_url; just return name and version. + self.assertEqual(2, len(releases)) + versions = releases.get_versions() + self.assertIn('1.1', versions) + self.assertIn('1.2', versions) + self.assertNotIn('1.3', versions) + + self.assertRaises(ProjectNotFound, client.get_releases,'Foo') + + @use_xmlrpc_server() + def test_get_distributions(self, server): + client = self._get_client(server) + server.xmlrpc.set_distributions([ + {'name':'FooBar', 'version': '1.1', 'url': + 'http://example.org/foobar-1.1-sdist.tar.gz', + 'digest': '1234567', 'type': 'sdist', 'python_version':'source'}, + {'name':'FooBar', 'version': '1.1', 'url': + 'http://example.org/foobar-1.1-bdist.tar.gz', + 'digest': '8912345', 'type': 'bdist'}, + ]) + + releases = client.get_releases('FooBar', '1.1') + client.get_distributions('FooBar', '1.1') + release = releases.get_release('1.1') + self.assertTrue('http://example.org/foobar-1.1-sdist.tar.gz', + release['sdist'].url['url']) + self.assertTrue('http://example.org/foobar-1.1-bdist.tar.gz', + release['bdist'].url['url']) + self.assertEqual(release['sdist'].python_version, 'source') + + @use_xmlrpc_server() + def test_get_metadata(self, server): + client = self._get_client(server) + server.xmlrpc.set_distributions([ + {'name':'FooBar', + 'version': '1.1', + 'keywords': '', + 'obsoletes_dist': ['FooFoo'], + 'requires_external': ['Foo'], + }]) + release = client.get_metadata('FooBar', '1.1') + self.assertEqual(['Foo'], release.metadata['requires_external']) + self.assertEqual(['FooFoo'], release.metadata['obsoletes_dist']) + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestXMLRPCClient)) + return suite + +if __name__ == '__main__': + run_unittest(test_suite()) diff --git a/src/distutils2/tests/test_install.py b/src/distutils2/tests/test_install.py index 5cf6cd9..f3d9c59 100644 --- a/src/distutils2/tests/test_install.py +++ b/src/distutils2/tests/test_install.py @@ -75,11 +75,9 @@ class InstallTestCase(support.TempdirManager, check_path(cmd.install_scripts, os.path.join(destination, "bin")) check_path(cmd.install_data, destination) + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_user_site(self): - # site.USER_SITE was introduced in 2.6 - if sys.version < '2.6': - return - + # test install with --user # preparing the environment for the test self.old_user_base = get_config_var('userbase') self.old_user_site = get_path('purelib', '%s_user' % os.name) @@ -195,14 +193,18 @@ class InstallTestCase(support.TempdirManager, cmd.ensure_finalized() cmd.run() - # let's check the RECORD file was created with one - # line (the egg info file) + # let's check the RECORD file was created with four + # lines, one for each .dist-info entry: METADATA, + # INSTALLER, REQUSTED, RECORD f = open(cmd.record) try: - self.assertEqual(len(f.readlines()), 1) + self.assertEqual(len(f.readlines()), 4) finally: f.close() + # XXX test that fancy_getopt is okay with options named + # record and no-record but unrelated + def _test_debug_mode(self): # this covers the code called when DEBUG is set old_logs_len = len(self.logs) diff --git a/src/distutils2/tests/test_install_distinfo.py b/src/distutils2/tests/test_install_distinfo.py new file mode 100644 index 0000000..fbfa722 --- /dev/null +++ b/src/distutils2/tests/test_install_distinfo.py @@ -0,0 +1,202 @@ +"""Tests for ``distutils2.command.install_distinfo``. """ + +import os +import sys +import csv + +from distutils2.command.install_distinfo import install_distinfo +from distutils2.core import Command +from distutils2.metadata import DistributionMetadata +from distutils2.tests import support +from distutils2.tests.support import unittest + +try: + import hashlib +except ImportError: + from distutils2._backport import hashlib + + +class DummyInstallCmd(Command): + + def __init__(self, dist=None): + self.outputs = [] + self.distribution = dist + + def __getattr__(self, name): + return None + + def ensure_finalized(self): + pass + + def get_outputs(self): + return self.outputs + \ + self.get_finalized_command('install_distinfo').get_outputs() + + +class InstallDistinfoTestCase(support.TempdirManager, + support.LoggingSilencer, + support.EnvironGuard, + unittest.TestCase): + + checkLists = lambda self, x, y: self.assertListEqual(sorted(x), sorted(y)) + + def test_empty_install(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.ensure_finalized() + cmd.run() + + self.checkLists(os.listdir(install_dir), ['foo-1.0.dist-info']) + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'RECORD', 'REQUESTED', 'INSTALLER']) + self.assertEqual(open(os.path.join(dist_info, 'INSTALLER')).read(), + 'distutils') + self.assertEqual(open(os.path.join(dist_info, 'REQUESTED')).read(), + '') + meta_path = os.path.join(dist_info, 'METADATA') + self.assertTrue(DistributionMetadata(path=meta_path).check()) + + def test_installer(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.installer = 'bacon-python' + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.assertEqual(open(os.path.join(dist_info, 'INSTALLER')).read(), + 'bacon-python') + + def test_requested(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.requested = False + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'RECORD', 'INSTALLER']) + + def test_no_record(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.no_record = True + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'REQUESTED', 'INSTALLER']) + + def test_record(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + fake_dists = os.path.join(os.path.dirname(__file__), '..', + '_backport', 'tests', 'fake_dists') + fake_dists = os.path.realpath(fake_dists) + + # for testing, we simply add all files from _backport's fake_dists + dirs = [] + for dir in os.listdir(fake_dists): + full_path = os.path.join(fake_dists, dir) + if (not dir.endswith('.egg') or dir.endswith('.egg-info') or + dir.endswith('.dist-info')) and os.path.isdir(full_path): + dirs.append(full_path) + + for dir in dirs: + for (path, subdirs, files) in os.walk(dir): + install.outputs += [os.path.join(path, f) for f in files] + install.outputs += [os.path.join('path', f + 'c') + for f in files if f.endswith('.py')] + + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + + expected = [] + for f in install.get_outputs(): + if f.endswith('.pyc') or \ + f == os.path.join(install_dir, 'foo-1.0.dist-info', 'RECORD'): + expected.append([f, '', '']) + else: + size = os.path.getsize(f) + md5 = hashlib.md5() + md5.update(open(f).read()) + hash = md5.hexdigest() + expected.append([f, hash, str(size)]) + + parsed = [] + f = open(os.path.join(dist_info, 'RECORD'), 'rb') + try: + reader = csv.reader(f, delimiter=',', + lineterminator=os.linesep, + quotechar='"') + parsed = list(reader) + finally: + f.close() + + self.maxDiff = None + self.checkLists(parsed, expected) + + +def test_suite(): + return unittest.makeSuite(InstallDistinfoTestCase) + + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/src/distutils2/tests/test_install_lib.py b/src/distutils2/tests/test_install_lib.py index 680eaf1..a96c889 100644 --- a/src/distutils2/tests/test_install_lib.py +++ b/src/distutils2/tests/test_install_lib.py @@ -57,9 +57,8 @@ class InstallLibTestCase(support.TempdirManager, # setting up a dist environment cmd.compile = cmd.optimize = 1 cmd.install_dir = pkg_dir - f = os.path.join(pkg_dir, 'foo.py') - self.write_file(f, '# python file') - cmd.distribution.py_modules = [pkg_dir] + f = os.path.join(pkg_dir, '__init__.py') + self.write_file(f, '# python package') cmd.distribution.ext_modules = [Extension('foo', ['xxx'])] cmd.distribution.packages = [pkg_dir] cmd.distribution.script_name = 'setup.py' @@ -74,9 +73,8 @@ class InstallLibTestCase(support.TempdirManager, # setting up a dist environment cmd.compile = cmd.optimize = 1 cmd.install_dir = pkg_dir - f = os.path.join(pkg_dir, 'foo.py') - self.write_file(f, '# python file') - cmd.distribution.py_modules = [pkg_dir] + f = os.path.join(pkg_dir, '__init__.py') + self.write_file(f, '# python package') cmd.distribution.ext_modules = [Extension('foo', ['xxx'])] cmd.distribution.packages = [pkg_dir] cmd.distribution.script_name = 'setup.py' @@ -84,7 +82,8 @@ class InstallLibTestCase(support.TempdirManager, # get_input should return 2 elements self.assertEqual(len(cmd.get_inputs()), 2) - @unittest.skipUnless(bytecode_support, 'sys.dont_write_bytecode not supported') + @unittest.skipUnless(bytecode_support, + 'sys.dont_write_bytecode not supported') def test_dont_write_bytecode(self): # makes sure byte_compile is not used pkg_dir, dist = self.create_dist() diff --git a/src/distutils2/tests/test_install_with_deps.py b/src/distutils2/tests/test_install_with_deps.py new file mode 100644 index 0000000..1b808b2 --- /dev/null +++ b/src/distutils2/tests/test_install_with_deps.py @@ -0,0 +1,152 @@ +"""Tests for the distutils2.index.xmlrpc module.""" + +from distutils2.tests.pypi_server import use_xmlrpc_server +from distutils2.tests import run_unittest +from distutils2.tests.support import unittest +from distutils2.index.xmlrpc import Client +from distutils2.install_with_deps import (get_install_info, + InstallationException) +from distutils2.metadata import DistributionMetadata + +class FakeDist(object): + """A fake distribution object, for tests""" + def __init__(self, name, version, deps): + self.name = name + self.version = version + self.metadata = DistributionMetadata() + self.metadata['Requires-Dist'] = deps + self.metadata['Provides-Dist'] = ['%s (%s)' % (name, version)] + + def __repr__(self): + return '<FakeDist %s>' % self.name + +def get_fake_dists(dists): + objects = [] + for (name, version, deps) in dists: + objects.append(FakeDist(name, version, deps)) + return objects + +class TestInstallWithDeps(unittest.TestCase): + def _get_client(self, server, *args, **kwargs): + return Client(server.full_address, *args, **kwargs) + + @use_xmlrpc_server() + def test_existing_deps(self, server): + # Test that the installer get the dependencies from the metadatas + # and ask the index for this dependencies. + # In this test case, we have choxie that is dependent from towel-stuff + # 0.1, which is in-turn dependent on bacon <= 0.2: + # choxie -> towel-stuff -> bacon. + # Each release metadata is not provided in metadata 1.2. + client = self._get_client(server) + archive_path = '%s/distribution.tar.gz' % server.full_address + server.xmlrpc.set_distributions([ + {'name':'choxie', + 'version': '2.0.0.9', + 'requires_dist': ['towel-stuff (0.1)',], + 'url': archive_path}, + {'name':'towel-stuff', + 'version': '0.1', + 'requires_dist': ['bacon (<= 0.2)',], + 'url': archive_path}, + {'name':'bacon', + 'version': '0.1', + 'requires_dist': [], + 'url': archive_path}, + ]) + installed = get_fake_dists([('bacon', '0.1', []),]) + output = get_install_info("choxie", index=client, + already_installed=installed) + + # we dont have installed bacon as it's already installed on the system. + self.assertEqual(0, len(output['remove'])) + self.assertEqual(2, len(output['install'])) + readable_output = [(o.name, '%s' % o.version) + for o in output['install']] + self.assertIn(('towel-stuff', '0.1'), readable_output) + self.assertIn(('choxie', '2.0.0.9'), readable_output) + + @use_xmlrpc_server() + def test_upgrade_existing_deps(self, server): + # Tests that the existing distributions can be upgraded if needed. + client = self._get_client(server) + archive_path = '%s/distribution.tar.gz' % server.full_address + server.xmlrpc.set_distributions([ + {'name':'choxie', + 'version': '2.0.0.9', + 'requires_dist': ['towel-stuff (0.1)',], + 'url': archive_path}, + {'name':'towel-stuff', + 'version': '0.1', + 'requires_dist': ['bacon (>= 0.2)',], + 'url': archive_path}, + {'name':'bacon', + 'version': '0.2', + 'requires_dist': [], + 'url': archive_path}, + ]) + + output = get_install_info("choxie", index=client, already_installed= + get_fake_dists([('bacon', '0.1', []),])) + installed = [(o.name, '%s' % o.version) for o in output['install']] + + # we need bacon 0.2, but 0.1 is installed. + # So we expect to remove 0.1 and to install 0.2 instead. + remove = [(o.name, '%s' % o.version) for o in output['remove']] + self.assertIn(('choxie', '2.0.0.9'), installed) + self.assertIn(('towel-stuff', '0.1'), installed) + self.assertIn(('bacon', '0.2'), installed) + self.assertIn(('bacon', '0.1'), remove) + self.assertEqual(0, len(output['conflict'])) + + @use_xmlrpc_server() + def test_conflicts(self, server): + # Tests that conflicts are detected + client = self._get_client(server) + archive_path = '%s/distribution.tar.gz' % server.full_address + server.xmlrpc.set_distributions([ + {'name':'choxie', + 'version': '2.0.0.9', + 'requires_dist': ['towel-stuff (0.1)',], + 'url': archive_path}, + {'name':'towel-stuff', + 'version': '0.1', + 'requires_dist': ['bacon (>= 0.2)',], + 'url': archive_path}, + {'name':'bacon', + 'version': '0.2', + 'requires_dist': [], + 'url': archive_path}, + ]) + already_installed = [('bacon', '0.1', []), + ('chicken', '1.1', ['bacon (0.1)'])] + output = get_install_info("choxie", index=client, already_installed= + get_fake_dists(already_installed)) + + # we need bacon 0.2, but 0.1 is installed. + # So we expect to remove 0.1 and to install 0.2 instead. + installed = [(o.name, '%s' % o.version) for o in output['install']] + remove = [(o.name, '%s' % o.version) for o in output['remove']] + conflict = [(o.name, '%s' % o.version) for o in output['conflict']] + self.assertIn(('choxie', '2.0.0.9'), installed) + self.assertIn(('towel-stuff', '0.1'), installed) + self.assertIn(('bacon', '0.2'), installed) + self.assertIn(('bacon', '0.1'), remove) + self.assertIn(('chicken', '1.1'), conflict) + + @use_xmlrpc_server() + def test_installation_unexisting_project(self, server): + # Test that the isntalled raises an exception if the project does not + # exists. + client = self._get_client(server) + self.assertRaises(InstallationException, get_install_info, + 'unexistant project', index=client) + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestInstallWithDeps)) + return suite + +if __name__ == '__main__': + run_unittest(test_suite()) diff --git a/src/distutils2/tests/test_metadata.py b/src/distutils2/tests/test_metadata.py index ee763c0..e7e8d68 100644 --- a/src/distutils2/tests/test_metadata.py +++ b/src/distutils2/tests/test_metadata.py @@ -1,6 +1,7 @@ """Tests for distutils.command.bdist.""" import os import sys +import platform from StringIO import StringIO from distutils2.metadata import (DistributionMetadata, _interpret, @@ -12,25 +13,59 @@ from distutils2.errors import (MetadataConflictError, class DistributionMetadataTestCase(LoggingSilencer, unittest.TestCase): + def test_instantiation(self): + PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') + fp = open(PKG_INFO) + try: + contents = fp.read() + finally: + fp.close() + fp = StringIO(contents) + + m = DistributionMetadata() + self.assertRaises(MetadataUnrecognizedVersionError, m.items) + + m = DistributionMetadata(PKG_INFO) + self.assertEqual(len(m.items()), 22) + + m = DistributionMetadata(fileobj=fp) + self.assertEqual(len(m.items()), 22) + + m = DistributionMetadata(mapping=dict(name='Test', version='1.0')) + self.assertEqual(len(m.items()), 11) + + d = dict(m.items()) + self.assertRaises(TypeError, DistributionMetadata, + PKG_INFO, fileobj=fp) + self.assertRaises(TypeError, DistributionMetadata, + PKG_INFO, mapping=d) + self.assertRaises(TypeError, DistributionMetadata, + fileobj=fp, mapping=d) + self.assertRaises(TypeError, DistributionMetadata, + PKG_INFO, mapping=m, fileobj=fp) def test_interpret(self): - platform = sys.platform + sys_platform = sys.platform version = sys.version.split()[0] os_name = os.name + platform_version = platform.version() + platform_machine = platform.machine() - self.assertTrue(_interpret("sys.platform == '%s'" % platform)) + self.assertTrue(_interpret("sys.platform == '%s'" % sys_platform)) self.assertTrue(_interpret( - "sys.platform == '%s' or python_version == '2.4'" % platform)) + "sys.platform == '%s' or python_version == '2.4'" % sys_platform)) self.assertTrue(_interpret( "sys.platform == '%s' and python_full_version == '%s'" % - (platform, version))) - self.assertTrue(_interpret("'%s' == sys.platform" % platform)) - + (sys_platform, version))) + self.assertTrue(_interpret("'%s' == sys.platform" % sys_platform)) self.assertTrue(_interpret('os.name == "%s"' % os_name)) + self.assertTrue(_interpret( + 'platform.version == "%s" and platform.machine == "%s"' % + (platform_version, platform_machine))) # stuff that need to raise a syntax error ops = ('os.name == os.name', 'os.name == 2', "'2' == '2'", - 'okpjonon', '', 'os.name ==') + 'okpjonon', '', 'os.name ==', 'python_version == 2.4') for op in ops: self.assertRaises(SyntaxError, _interpret, op) @@ -62,17 +97,13 @@ class DistributionMetadataTestCase(LoggingSilencer, unittest.TestCase): PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') metadata = DistributionMetadata(PKG_INFO) - res = StringIO() - metadata.write_file(res) - res.seek(0) - res = res.read() - f = open(PKG_INFO) - try: - # XXX this is not used - wanted = f.read() - finally: - f.close() - self.assertTrue('Keywords: keyring,password,crypt' in res) + out = StringIO() + metadata.write_file(out) + out.seek(0) + res = DistributionMetadata() + res.read_file(out) + for k in metadata.keys(): + self.assertTrue(metadata[k] == res[k]) def test_metadata_markers(self): # see if we can be platform-aware @@ -82,6 +113,10 @@ class DistributionMetadataTestCase(LoggingSilencer, unittest.TestCase): metadata = DistributionMetadata(platform_dependent=True) metadata.read_file(StringIO(content)) self.assertEqual(metadata['Requires-Dist'], ['bar']) + metadata['Name'] = "baz; sys.platform == 'blah'" + # FIXME is None or 'UNKNOWN' correct here? + # where is that documented? + self.assertEquals(metadata['Name'], None) # test with context context = {'sys.platform': 'okook'} @@ -109,15 +144,15 @@ class DistributionMetadataTestCase(LoggingSilencer, unittest.TestCase): metadata.read_file(out) self.assertEqual(wanted, metadata['Description']) - def test_mapper_apis(self): + def test_mapping_api(self): PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') content = open(PKG_INFO).read() content = content % sys.platform - metadata = DistributionMetadata() - metadata.read_file(StringIO(content)) + metadata = DistributionMetadata(fileobj=StringIO(content)) self.assertIn('Version', metadata.keys()) self.assertIn('0.5', metadata.values()) self.assertIn(('Version', '0.5'), metadata.items()) + #TODO test update def test_versions(self): metadata = DistributionMetadata() @@ -211,6 +246,10 @@ class DistributionMetadataTestCase(LoggingSilencer, unittest.TestCase): metadata = DistributionMetadata() metadata['Version'] = 'rr' metadata['Requires-dist'] = ['Foo (a)'] + if metadata.docutils_support: + missing, warnings = metadata.check() + self.assertEqual(len(warnings), 2) + metadata.docutils_support = False missing, warnings = metadata.check() self.assertEqual(missing, ['Name', 'Home-page']) self.assertEqual(len(warnings), 2) diff --git a/src/distutils2/tests/test_msvc9compiler.py b/src/distutils2/tests/test_msvc9compiler.py index 452c3f5..eea83fa 100644 --- a/src/distutils2/tests/test_msvc9compiler.py +++ b/src/distutils2/tests/test_msvc9compiler.py @@ -64,7 +64,7 @@ _CLEANED_MANIFEST = """\ class msvc9compilerTestCase(support.TempdirManager, unittest.TestCase): - @unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") def test_no_compiler(self): # makes sure query_vcvarsall throws # a DistutilsPlatformError if the compiler @@ -86,7 +86,7 @@ class msvc9compilerTestCase(support.TempdirManager, finally: msvc9compiler.find_vcvarsall = old_find_vcvarsall - @unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") def test_reg_class(self): from distutils2.msvccompiler import get_build_version if get_build_version() < 8.0: @@ -110,7 +110,7 @@ class msvc9compilerTestCase(support.TempdirManager, keys = Reg.read_keys(HKCU, r'Control Panel') self.assertTrue('Desktop' in keys) - @unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") def test_remove_visual_c_ref(self): from distutils2.msvc9compiler import MSVCCompiler tempdir = self.mkdtemp() diff --git a/src/distutils2/tests/test_pypi_dist.py b/src/distutils2/tests/test_pypi_dist.py deleted file mode 100644 index 0267760..0000000 --- a/src/distutils2/tests/test_pypi_dist.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Tests for the distutils2.pypi.dist module.""" - -import os -import shutil -import tempfile - -from distutils2.tests.pypi_server import use_pypi_server -from distutils2.tests import run_unittest -from distutils2.tests.support import unittest, TempdirManager -from distutils2.version import VersionPredicate -from distutils2.pypi.errors import HashDoesNotMatch, UnsupportedHashName -from distutils2.pypi.dist import (PyPIDistribution as Dist, - PyPIDistributions as Dists, - split_archive_name) - - -class TestPyPIDistribution(TempdirManager, - unittest.TestCase): - """Tests the pypi.dist.PyPIDistribution class""" - - def test_instanciation(self): - # Test the Distribution class provides us the good attributes when - # given on construction - dist = Dist("FooBar", "1.1") - self.assertEqual("FooBar", dist.name) - self.assertEqual("1.1", "%s" % dist.version) - - def test_create_from_url(self): - # Test that the Distribution object can be built from a single URL - url_list = { - 'FooBar-1.1.0.tar.gz': { - 'name': 'foobar', # lowercase the name - 'version': '1.1', - }, - 'Foo-Bar-1.1.0.zip': { - 'name': 'foo-bar', # keep the dash - 'version': '1.1', - }, - 'foobar-1.1b2.tar.gz#md5=123123123123123': { - 'name': 'foobar', - 'version': '1.1b2', - 'url': { - 'url': 'http://test.tld/foobar-1.1b2.tar.gz', # no hash - 'hashval': '123123123123123', - 'hashname': 'md5', - } - }, - 'foobar-1.1-rc2.tar.gz': { # use suggested name - 'name': 'foobar', - 'version': '1.1c2', - 'url': { - 'url': 'http://test.tld/foobar-1.1-rc2.tar.gz', - } - } - } - - for url, attributes in url_list.items(): - dist = Dist.from_url("http://test.tld/" + url) - for attribute, value in attributes.items(): - if isinstance(value, dict): - mylist = getattr(dist, attribute) - for val in value.keys(): - self.assertEqual(value[val], mylist[val]) - else: - if attribute == "version": - self.assertEqual("%s" % getattr(dist, "version"), value) - else: - self.assertEqual(getattr(dist, attribute), value) - - def test_get_url(self): - # Test that the url property works well - - d = Dist("foobar", "1.1", url="test_url") - self.assertDictEqual(d.url, { - "url": "test_url", - "is_external": True, - "hashname": None, - "hashval": None, - }) - - # add a new url - d.add_url(url="internal_url", is_external=False) - self.assertEqual(d._url, None) - self.assertDictEqual(d.url, { - "url": "internal_url", - "is_external": False, - "hashname": None, - "hashval": None, - }) - self.assertEqual(2, len(d._urls)) - - def test_comparaison(self): - # Test that we can compare PyPIDistributions - foo1 = Dist("foo", "1.0") - foo2 = Dist("foo", "2.0") - bar = Dist("bar", "2.0") - # assert we use the version to compare - self.assertTrue(foo1 < foo2) - self.assertFalse(foo1 > foo2) - self.assertFalse(foo1 == foo2) - - # assert we can't compare dists with different names - self.assertRaises(TypeError, foo1.__eq__, bar) - - def test_split_archive_name(self): - # Test we can split the archive names - names = { - 'foo-bar-baz-1.0-rc2': ('foo-bar-baz', '1.0c2'), - 'foo-bar-baz-1.0': ('foo-bar-baz', '1.0'), - 'foobarbaz-1.0': ('foobarbaz', '1.0'), - } - for name, results in names.items(): - self.assertEqual(results, split_archive_name(name)) - - @use_pypi_server("downloads_with_md5") - def test_download(self, server): - # Download is possible, and the md5 is checked if given - - add_to_tmpdirs = lambda x: self.tempdirs.append(os.path.dirname(x)) - - url = "%s/simple/foobar/foobar-0.1.tar.gz" % server.full_address - # check md5 if given - dist = Dist("FooBar", "0.1", url=url, - url_hashname="md5", url_hashval="d41d8cd98f00b204e9800998ecf8427e") - add_to_tmpdirs(dist.download()) - - # a wrong md5 fails - dist2 = Dist("FooBar", "0.1", url=url, - url_hashname="md5", url_hashval="wrongmd5") - - self.assertRaises(HashDoesNotMatch, dist2.download) - add_to_tmpdirs(dist2.downloaded_location) - - # we can omit the md5 hash - dist3 = Dist("FooBar", "0.1", url=url) - add_to_tmpdirs(dist3.download()) - - # and specify a temporary location - # for an already downloaded dist - path1 = self.mkdtemp() - dist3.download(path=path1) - # and for a new one - path2_base = self.mkdtemp() - dist4 = Dist("FooBar", "0.1", url=url) - path2 = dist4.download(path=path2_base) - self.assertTrue(path2_base in path2) - - def test_hashname(self): - # Invalid hashnames raises an exception on assignation - Dist("FooBar", "0.1", url_hashname="md5", url_hashval="value") - - self.assertRaises(UnsupportedHashName, Dist, "FooBar", "0.1", - url_hashname="invalid_hashname", url_hashval="value") - - -class TestPyPIDistributions(unittest.TestCase): - - def test_filter(self): - # Test we filter the distributions the right way, using version - # predicate match method - dists = Dists(( - Dist("FooBar", "1.1"), - Dist("FooBar", "1.1.1"), - Dist("FooBar", "1.2"), - Dist("FooBar", "1.2.1"), - )) - filtered = dists.filter(VersionPredicate("FooBar (<1.2)")) - self.assertNotIn(dists[2], filtered) - self.assertNotIn(dists[3], filtered) - self.assertIn(dists[0], filtered) - self.assertIn(dists[1], filtered) - - def test_append(self): - # When adding a new item to the list, the behavior is to test if - # a distribution with the same name and version number already exists, - # and if so, to add url informations to the existing PyPIDistribution - # object. - # If no object matches, just add "normally" the object to the list. - - dists = Dists([ - Dist("FooBar", "1.1", url="external_url", type="source"), - ]) - self.assertEqual(1, len(dists)) - dists.append(Dist("FooBar", "1.1", url="internal_url", - url_is_external=False, type="source")) - self.assertEqual(1, len(dists)) - self.assertEqual(2, len(dists[0]._urls)) - - dists.append(Dist("Foobar", "1.1.1", type="source")) - self.assertEqual(2, len(dists)) - - # when adding a distribution whith a different type, a new distribution - # has to be added. - dists.append(Dist("Foobar", "1.1.1", type="binary")) - self.assertEqual(3, len(dists)) - - def test_prefer_final(self): - # Can order the distributions using prefer_final - - fb10 = Dist("FooBar", "1.0") # final distribution - fb11a = Dist("FooBar", "1.1a1") # alpha - fb12a = Dist("FooBar", "1.2a1") # alpha - fb12b = Dist("FooBar", "1.2b1") # beta - dists = Dists([fb10, fb11a, fb12a, fb12b]) - - dists.sort_distributions(prefer_final=True) - self.assertEqual(fb10, dists[0]) - - dists.sort_distributions(prefer_final=False) - self.assertEqual(fb12b, dists[0]) - - def test_prefer_source(self): - # Ordering support prefer_source - fb_source = Dist("FooBar", "1.0", type="source") - fb_binary = Dist("FooBar", "1.0", type="binary") - fb2_binary = Dist("FooBar", "2.0", type="binary") - dists = Dists([fb_binary, fb_source]) - - dists.sort_distributions(prefer_source=True) - self.assertEqual(fb_source, dists[0]) - - dists.sort_distributions(prefer_source=False) - self.assertEqual(fb_binary, dists[0]) - - dists.append(fb2_binary) - dists.sort_distributions(prefer_source=True) - self.assertEqual(fb2_binary, dists[0]) - - def test_get_same_name_and_version(self): - # PyPIDistributions can return a list of "duplicates" - fb_source = Dist("FooBar", "1.0", type="source") - fb_binary = Dist("FooBar", "1.0", type="binary") - fb2_binary = Dist("FooBar", "2.0", type="binary") - dists = Dists([fb_binary, fb_source, fb2_binary]) - duplicates = dists.get_same_name_and_version() - self.assertTrue(1, len(duplicates)) - self.assertIn(fb_source, duplicates[0]) - - -def test_suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(TestPyPIDistribution)) - suite.addTest(unittest.makeSuite(TestPyPIDistributions)) - return suite - -if __name__ == '__main__': - run_unittest(test_suite()) diff --git a/src/distutils2/tests/test_pypi_versions.py b/src/distutils2/tests/test_pypi_versions.py index 7781982..bbb9395 100644 --- a/src/distutils2/tests/test_pypi_versions.py +++ b/src/distutils2/tests/test_pypi_versions.py @@ -1,43 +1,39 @@ -# -## test_pypi_versions.py -## -## A very simple test to see what percentage of the current pypi packages -## have versions that can be converted automatically by distutils' new -## suggest_normalized_version() into PEP-386 compatible versions. -## -## Requires : Python 2.5+ -## -## Written by: ssteinerX@gmail.com -# +"""PEP 386 compatibility test with current distributions on PyPI. + +A very simple test to see what percentage of the current PyPI packages +have versions that can be converted automatically by distutils2's new +suggest_normalized_version into PEP 386-compatible versions. +""" + +# XXX This file does not actually run tests, move it to a script + +# Written by ssteinerX@gmail.com + +import os +import xmlrpclib try: import cPickle as pickle -except: +except ImportError: import pickle -import xmlrpclib -import os.path - from distutils2.version import suggest_normalized_version from distutils2.tests import run_unittest from distutils2.tests.support import unittest def test_pypi(): - # - ## To re-run from scratch, just delete these two .pkl files - # + # FIXME need a better way to do that + # To re-run from scratch, just delete these two .pkl files INDEX_PICKLE_FILE = 'pypi-index.pkl' VERSION_PICKLE_FILE = 'pypi-version.pkl' package_info = version_info = [] - # - ## if there's a saved version of the package list - ## restore it - ## else: - ## pull the list down from pypi - ## save a pickled version of it - # + # if there's a saved version of the package list + # restore it + # else: + # pull the list down from pypi + # save a pickled version of it if os.path.exists(INDEX_PICKLE_FILE): print "Loading saved pypi data..." f = open(INDEX_PICKLE_FILE, 'rb') @@ -57,13 +53,11 @@ def test_pypi(): finally: f.close() - # - ## If there's a saved list of the versions from the packages - ## restore it - ## else - ## extract versions from the package list - ## save a pickled version of it - # + # If there's a saved list of the versions from the packages + # restore it + # else + # extract versions from the package list + # save a pickled version of it versions = [] if os.path.exists(VERSION_PICKLE_FILE): print "Loading saved version info..." diff --git a/src/distutils2/tests/test_register.py b/src/distutils2/tests/test_register.py index be782d1..5081f49 100644 --- a/src/distutils2/tests/test_register.py +++ b/src/distutils2/tests/test_register.py @@ -1,5 +1,5 @@ +# -*- encoding: utf-8 -*- """Tests for distutils.command.register.""" -# -*- encoding: utf8 -*- import sys import os import getpass @@ -19,7 +19,7 @@ from distutils2.errors import DistutilsSetupError from distutils2.tests import support from distutils2.tests.support import unittest -from distutils2.tests.test_config import PYPIRC, PyPIRCCommandTestCase + PYPIRC_NOPASSWORD = """\ [distutils] @@ -68,10 +68,15 @@ class FakeOpener(object): def read(self): return 'xxx' -class RegisterTestCase(PyPIRCCommandTestCase): +class RegisterTestCase(support.TempdirManager, support.EnvironGuard, + unittest.TestCase): def setUp(self): super(RegisterTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir + # patching the password prompt self._old_getpass = getpass.getpass def _getpass(prompt): @@ -148,15 +153,15 @@ class RegisterTestCase(PyPIRCCommandTestCase): self.write_file(self.rc, PYPIRC_NOPASSWORD) cmd = self._get_cmd() - cmd._set_config() cmd.finalize_options() + cmd._set_config() cmd.send_metadata() # dist.password should be set # therefore used afterwards by other commands self.assertEqual(cmd.distribution.password, 'password') - def test_registering(self): + def test_registration(self): # this test runs choice 2 cmd = self._get_cmd() inputs = RawInputs('2', 'tarek', 'tarek@ziade.org') @@ -205,7 +210,7 @@ class RegisterTestCase(PyPIRCCommandTestCase): cmd.strict = 1 self.assertRaises(DistutilsSetupError, cmd.run) - # metadata are OK but long_description is broken + # metadata is OK but long_description is broken metadata = {'home_page': 'xxx', 'author': 'xxx', 'author_email': u'éxéxé', 'name': 'xxx', 'version': 'xxx', diff --git a/src/distutils2/tests/test_sdist.py b/src/distutils2/tests/test_sdist.py index ec4c87a..c974175 100644 --- a/src/distutils2/tests/test_sdist.py +++ b/src/distutils2/tests/test_sdist.py @@ -28,9 +28,8 @@ from distutils2.command.sdist import sdist from distutils2.command.sdist import show_formats from distutils2.core import Distribution from distutils2.tests.support import unittest -from distutils2.tests.test_config import PyPIRCCommandTestCase from distutils2.errors import DistutilsExecError, DistutilsOptionError -from distutils2.spawn import find_executable +from distutils2.util import find_executable from distutils2.tests import support from distutils2.log import WARN try: @@ -58,12 +57,15 @@ somecode%(sep)sdoc.dat somecode%(sep)sdoc.txt """ -class SDistTestCase(PyPIRCCommandTestCase): +class SDistTestCase(support.TempdirManager, support.LoggingSilencer, + support.EnvironGuard, unittest.TestCase): def setUp(self): # PyPIRCCommandTestCase creates a temp dir already # and put it in self.tmp_dir super(SDistTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + os.environ['HOME'] = self.tmp_dir # setting up an environment self.old_path = os.getcwd() os.mkdir(join(self.tmp_dir, 'somecode')) @@ -239,7 +241,7 @@ class SDistTestCase(PyPIRCCommandTestCase): @unittest.skipUnless(zlib, "requires zlib") def test_metadata_check_option(self): - # testing the `medata-check` option + # testing the `check-metadata` option dist, cmd = self.get_cmd(metadata={}) # this should raise some warnings ! @@ -293,7 +295,7 @@ class SDistTestCase(PyPIRCCommandTestCase): self.assertRaises(DistutilsOptionError, cmd.finalize_options) @unittest.skipUnless(zlib, "requires zlib") - @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") + @unittest.skipUnless(UID_GID_SUPPORT, "requires grp and pwd support") def test_make_distribution_owner_group(self): # check if tar and gzip are installed diff --git a/src/distutils2/tests/test_spawn.py b/src/distutils2/tests/test_spawn.py deleted file mode 100644 index 2c7f1ad..0000000 --- a/src/distutils2/tests/test_spawn.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Tests for distutils.spawn.""" -import os -import time -from distutils2.tests import captured_stdout - -from distutils2.spawn import _nt_quote_args -from distutils2.spawn import spawn, find_executable -from distutils2.errors import DistutilsExecError -from distutils2.tests import support -from distutils2.tests.support import unittest - -class SpawnTestCase(support.TempdirManager, - support.LoggingSilencer, - unittest.TestCase): - - def test_nt_quote_args(self): - - for (args, wanted) in ((['with space', 'nospace'], - ['"with space"', 'nospace']), - (['nochange', 'nospace'], - ['nochange', 'nospace'])): - res = _nt_quote_args(args) - self.assertEqual(res, wanted) - - - @unittest.skipUnless(os.name in ('nt', 'posix'), - 'Runs only under posix or nt') - def test_spawn(self): - tmpdir = self.mkdtemp() - - # creating something executable - # through the shell that returns 1 - if os.name == 'posix': - exe = os.path.join(tmpdir, 'foo.sh') - self.write_file(exe, '#!/bin/sh\nexit 1') - os.chmod(exe, 0777) - else: - exe = os.path.join(tmpdir, 'foo.bat') - self.write_file(exe, 'exit 1') - - os.chmod(exe, 0777) - self.assertRaises(DistutilsExecError, spawn, [exe]) - - # now something that works - if os.name == 'posix': - exe = os.path.join(tmpdir, 'foo.sh') - self.write_file(exe, '#!/bin/sh\nexit 0') - os.chmod(exe, 0777) - else: - exe = os.path.join(tmpdir, 'foo.bat') - self.write_file(exe, 'exit 0') - - os.chmod(exe, 0777) - spawn([exe]) # should work without any error - -def test_suite(): - return unittest.makeSuite(SpawnTestCase) - -if __name__ == "__main__": - unittest.main(defaultTest="test_suite") diff --git a/src/distutils2/tests/test_upload.py b/src/distutils2/tests/test_upload.py index 5ca88a1..3959872 100644 --- a/src/distutils2/tests/test_upload.py +++ b/src/distutils2/tests/test_upload.py @@ -1,5 +1,5 @@ +# -*- encoding: utf-8 -*- """Tests for distutils.command.upload.""" -# -*- encoding: utf8 -*- import os import sys @@ -9,7 +9,6 @@ from distutils2.core import Distribution from distutils2.tests import support from distutils2.tests.pypi_server import PyPIServer, PyPIServerTestCase from distutils2.tests.support import unittest -from distutils2.tests.test_config import PYPIRC, PyPIRCCommandTestCase PYPIRC_NOPASSWORD = """\ @@ -22,7 +21,33 @@ index-servers = username:me """ -class UploadTestCase(PyPIServerTestCase, PyPIRCCommandTestCase): +PYPIRC = """\ +[distutils] + +index-servers = + server1 + server2 + +[server1] +username:me +password:secret + +[server2] +username:meagain +password: secret +realm:acme +repository:http://another.pypi/ +""" + + +class UploadTestCase(support.TempdirManager, support.EnvironGuard, + PyPIServerTestCase): + + def setUp(self): + super(UploadTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir def test_finalize_options(self): # new format @@ -76,6 +101,38 @@ class UploadTestCase(PyPIServerTestCase, PyPIRCCommandTestCase): self.assertEqual(handler.command, 'POST') self.assertNotIn('\n', headers['authorization']) + def test_upload_docs(self): + path = os.path.join(self.tmp_dir, 'xxx') + self.write_file(path) + command, pyversion, filename = 'xxx', '2.6', path + dist_files = [(command, pyversion, filename)] + docs_path = os.path.join(self.tmp_dir, "build", "docs") + os.makedirs(docs_path) + self.write_file(os.path.join(docs_path, "index.html"), "yellow") + self.write_file(self.rc, PYPIRC) + + # lets run it + pkg_dir, dist = self.create_dist(dist_files=dist_files, author=u'dédé') + + cmd = upload(dist) + cmd.get_finalized_command("build").run() + cmd.upload_docs = True + cmd.ensure_finalized() + cmd.repository = self.pypi.full_address + try: + prev_dir = os.getcwd() + os.chdir(self.tmp_dir) + cmd.run() + finally: + os.chdir(prev_dir) + + handler, request_data = self.pypi.requests[-1] + action, name, content =\ + request_data.split("----------------GHSKFJDLGDS7543FJKLFHRE75642756743254")[1:4] + + self.assertIn('name=":action"', action) + self.assertIn("doc_upload", action) + def test_suite(): return unittest.makeSuite(UploadTestCase) diff --git a/src/distutils2/tests/test_upload_docs.py b/src/distutils2/tests/test_upload_docs.py index 2147968..c90f6b4 100644 --- a/src/distutils2/tests/test_upload_docs.py +++ b/src/distutils2/tests/test_upload_docs.py @@ -1,18 +1,23 @@ -"""Tests for distutils.command.upload_docs.""" # -*- encoding: utf8 -*- -import httplib, os, os.path, shutil, sys, tempfile, zipfile -from cStringIO import StringIO +"""Tests for distutils.command.upload_docs.""" +import os +import sys +import httplib +import shutil +import zipfile +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO from distutils2.command import upload_docs as upload_docs_mod from distutils2.command.upload_docs import (upload_docs, zip_dir, - encode_multipart) + encode_multipart) from distutils2.core import Distribution - from distutils2.errors import DistutilsFileError, DistutilsOptionError from distutils2.tests import support from distutils2.tests.pypi_server import PyPIServer, PyPIServerTestCase -from distutils2.tests.test_config import PyPIRCCommandTestCase from distutils2.tests.support import unittest @@ -47,16 +52,20 @@ username = real_slim_shady password = long_island """ -class UploadDocsTestCase(PyPIServerTestCase, PyPIRCCommandTestCase): +class UploadDocsTestCase(support.TempdirManager, support.EnvironGuard, + PyPIServerTestCase): def setUp(self): super(UploadDocsTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir self.dist = Distribution() self.dist.metadata['Name'] = "distr-name" self.cmd = upload_docs(self.dist) def test_default_uploaddir(self): - sandbox = tempfile.mkdtemp() + sandbox = self.mkdtemp() previous = os.getcwd() os.chdir(sandbox) try: @@ -69,7 +78,7 @@ class UploadDocsTestCase(PyPIServerTestCase, PyPIRCCommandTestCase): def prepare_sample_dir(self, sample_dir=None): if sample_dir is None: - sample_dir = tempfile.mkdtemp() + sample_dir = self.mkdtemp() os.mkdir(os.path.join(sample_dir, "docs")) self.write_file(os.path.join(sample_dir, "docs", "index.html"), "Ce mortel ennui") self.write_file(os.path.join(sample_dir, "index.html"), "Oh la la") @@ -178,9 +187,13 @@ class UploadDocsTestCase(PyPIServerTestCase, PyPIRCCommandTestCase): def test_show_response(self): orig_stdout = sys.stdout write_args = [] + class MockStdIn(object): def write(self, arg): write_args.append(arg) + def flush(self): + pass + sys.stdout = MockStdIn() try: self.prepare_command() @@ -188,8 +201,10 @@ class UploadDocsTestCase(PyPIServerTestCase, PyPIRCCommandTestCase): self.cmd.run() finally: sys.stdout = orig_stdout + self.assertTrue(write_args[0], "should report the response") - self.assertIn(self.pypi.default_response_data + "\n", write_args[0]) + self.assertIn(self.pypi.default_response_data + "\n", + '\n'.join(write_args)) def test_suite(): return unittest.makeSuite(UploadDocsTestCase) diff --git a/src/distutils2/tests/test_util.py b/src/distutils2/tests/test_util.py index c770c59..26f4ee1 100644 --- a/src/distutils2/tests/test_util.py +++ b/src/distutils2/tests/test_util.py @@ -4,22 +4,60 @@ import sys from copy import copy from StringIO import StringIO import subprocess -import tempfile import time +from distutils2.tests import captured_stdout +from distutils2.tests.support import unittest from distutils2.errors import (DistutilsPlatformError, DistutilsByteCompileError, - DistutilsFileError) - + DistutilsFileError, + DistutilsExecError) from distutils2.util import (convert_path, change_root, - check_environ, split_quoted, strtobool, - rfc822_escape, get_compiler_versions, - _find_exe_version, _MAC_OS_X_LD_VERSION, - byte_compile, find_packages) + check_environ, split_quoted, strtobool, + rfc822_escape, get_compiler_versions, + _find_exe_version, _MAC_OS_X_LD_VERSION, + byte_compile, find_packages, spawn, find_executable, + _nt_quote_args, get_pypirc_path, generate_pypirc, + read_pypirc, resolve_dotted_name) + from distutils2 import util from distutils2.tests import support from distutils2.tests.support import unittest + +PYPIRC = """\ +[distutils] +index-servers = + pypi + server1 + +[pypi] +username:me +password:xxxx + +[server1] +repository:http://example.com +username:tarek +password:secret +""" + +PYPIRC_OLD = """\ +[server-login] +username:tarek +password:secret +""" + +WANTED = """\ +[distutils] +index-servers = + pypi + +[pypi] +username:tarek +password:xxx +""" + + class FakePopen(object): test_class = None def __init__(self, cmd, shell, stdout, stderr): @@ -40,6 +78,9 @@ class UtilTestCase(support.EnvironGuard, def setUp(self): super(UtilTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir # saving the environment self.name = os.name self.platform = sys.platform @@ -246,7 +287,7 @@ class UtilTestCase(support.EnvironGuard, self.assertEqual(res[2], None) @unittest.skipUnless(hasattr(sys, 'dont_write_bytecode'), - 'no dont_write_bytecode support') + 'sys.dont_write_bytecode not supported') def test_dont_write_bytecode(self): # makes sure byte_compile raise a DistutilsError # if sys.dont_write_bytecode is True @@ -259,9 +300,9 @@ class UtilTestCase(support.EnvironGuard, def test_newer(self): self.assertRaises(DistutilsFileError, util.newer, 'xxx', 'xxx') - self.newer_f1 = tempfile.NamedTemporaryFile() + self.newer_f1 = self.mktempfile() time.sleep(1) - self.newer_f2 = tempfile.NamedTemporaryFile() + self.newer_f2 = self.mktempfile() self.assertTrue(util.newer(self.newer_f2.name, self.newer_f1.name)) def test_find_packages(self): @@ -301,7 +342,17 @@ class UtilTestCase(support.EnvironGuard, res = find_packages([root], ['pkg1.pkg2']) self.assertEqual(set(res), set(['pkg1', 'pkg5', 'pkg1.pkg3', 'pkg1.pkg3.pkg6'])) - @unittest.skipUnless(sys.version > '2.6', 'Need Python 2.6 or more') + def test_resolve_dotted_name(self): + self.assertEqual(UtilTestCase, resolve_dotted_name("distutils2.tests.test_util.UtilTestCase")) + self.assertEqual(UtilTestCase.test_resolve_dotted_name, + resolve_dotted_name("distutils2.tests.test_util.UtilTestCase.test_resolve_dotted_name")) + + self.assertRaises(ImportError, resolve_dotted_name, + "distutils2.tests.test_util.UtilTestCaseNot") + self.assertRaises(ImportError, resolve_dotted_name, + "distutils2.tests.test_util.UtilTestCase.nonexistent_attribute") + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_run_2to3_on_code(self): content = "print 'test'" converted_content = "print('test')" @@ -316,7 +367,7 @@ class UtilTestCase(support.EnvironGuard, file_handle.close() self.assertEquals(new_content, converted_content) - @unittest.skipUnless(sys.version > '2.6', 'Need Python 2.6 or more') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_run_2to3_on_doctests(self): # to check if text files containing doctests only get converted. content = ">>> print 'test'\ntest\n" @@ -332,6 +383,80 @@ class UtilTestCase(support.EnvironGuard, file_handle.close() self.assertEquals(new_content, converted_content) + def test_nt_quote_args(self): + + for (args, wanted) in ((['with space', 'nospace'], + ['"with space"', 'nospace']), + (['nochange', 'nospace'], + ['nochange', 'nospace'])): + res = _nt_quote_args(args) + self.assertEqual(res, wanted) + + + @unittest.skipUnless(os.name in ('nt', 'posix'), + 'runs only under posix or nt') + def test_spawn(self): + tmpdir = self.mkdtemp() + + # creating something executable + # through the shell that returns 1 + if os.name == 'posix': + exe = os.path.join(tmpdir, 'foo.sh') + self.write_file(exe, '#!/bin/sh\nexit 1') + os.chmod(exe, 0777) + else: + exe = os.path.join(tmpdir, 'foo.bat') + self.write_file(exe, 'exit 1') + + os.chmod(exe, 0777) + self.assertRaises(DistutilsExecError, spawn, [exe]) + + # now something that works + if os.name == 'posix': + exe = os.path.join(tmpdir, 'foo.sh') + self.write_file(exe, '#!/bin/sh\nexit 0') + os.chmod(exe, 0777) + else: + exe = os.path.join(tmpdir, 'foo.bat') + self.write_file(exe, 'exit 0') + + os.chmod(exe, 0777) + spawn([exe]) # should work without any error + + def test_server_registration(self): + # This test makes sure we know how to: + # 1. handle several sections in .pypirc + # 2. handle the old format + + # new format + self.write_file(self.rc, PYPIRC) + config = read_pypirc() + + config = config.items() + config.sort() + expected = [('password', 'xxxx'), ('realm', 'pypi'), + ('repository', 'http://pypi.python.org/pypi'), + ('server', 'pypi'), ('username', 'me')] + self.assertEqual(config, expected) + + # old format + self.write_file(self.rc, PYPIRC_OLD) + config = read_pypirc() + config = config.items() + config.sort() + expected = [('password', 'secret'), ('realm', 'pypi'), + ('repository', 'http://pypi.python.org/pypi'), + ('server', 'server-login'), ('username', 'tarek')] + self.assertEqual(config, expected) + + def test_server_empty_registration(self): + rc = get_pypirc_path() + self.assertTrue(not os.path.exists(rc)) + generate_pypirc('tarek', 'xxx') + self.assertTrue(os.path.exists(rc)) + content = open(rc).read() + self.assertEqual(content, WANTED) + def test_suite(): return unittest.makeSuite(UtilTestCase) diff --git a/src/distutils2/tests/test_version.py b/src/distutils2/tests/test_version.py index 343b397..1e392f2 100644 --- a/src/distutils2/tests/test_version.py +++ b/src/distutils2/tests/test_version.py @@ -188,6 +188,13 @@ class VersionTestCase(unittest.TestCase): # XXX need to silent the micro version in this case #assert not VersionPredicate('Ho (<3.0,!=2.6)').match('2.6.3') + def test_predicate_name(self): + # Test that names are parsed the right way + + self.assertEqual('Hey', VersionPredicate('Hey (<1.1)').name) + self.assertEqual('Foo-Bar', VersionPredicate('Foo-Bar (1.1)').name) + self.assertEqual('Foo Bar', VersionPredicate('Foo Bar (1.1)').name) + def test_is_final(self): # VersionPredicate knows is a distribution is a final one or not. final_versions = ('1.0', '1.0.post456') diff --git a/src/distutils2/util.py b/src/distutils2/util.py index b9aa3f7..178cb95 100644 --- a/src/distutils2/util.py +++ b/src/distutils2/util.py @@ -5,16 +5,20 @@ Miscellaneous utility functions. __revision__ = "$Id: util.py 77761 2010-01-26 22:46:15Z tarek.ziade $" -import sys import os -import string +import posixpath import re +import string +import sys +import shutil +import tarfile +import zipfile from copy import copy from fnmatch import fnmatchcase +from ConfigParser import RawConfigParser from distutils2.errors import (DistutilsPlatformError, DistutilsFileError, - DistutilsByteCompileError) -from distutils2.spawn import spawn, find_executable + DistutilsByteCompileError, DistutilsExecError) from distutils2 import log from distutils2._backport import sysconfig as _sysconfig @@ -84,8 +88,8 @@ def convert_path(pathname): raise ValueError("path '%s' cannot end with '/'" % pathname) paths = pathname.split('/') - while '.' in paths: - paths.remove('.') + while os.curdir in paths: + paths.remove(os.curdir) if not paths: return os.curdir return os.path.join(*paths) @@ -117,15 +121,6 @@ def change_root(new_root, pathname): path = path[1:] return os.path.join(new_root, path) - elif os.name == 'mac': - if not os.path.isabs(pathname): - return os.path.join(new_root, pathname) - else: - # Chop off volume name from start of path - elements = pathname.split(":", 1) - pathname = ":" + elements[1] - return os.path.join(new_root, pathname) - else: raise DistutilsPlatformError("nothing known about " "platform '%s'" % os.name) @@ -595,7 +590,7 @@ def _package_name(root_path, path): return path[len(root_path) + 1:].replace(os.sep, '.') -def find_packages(paths=('.',), exclude=()): +def find_packages(paths=(os.curdir,), exclude=()): """Return a list all Python packages found recursively within directories 'paths' @@ -642,6 +637,24 @@ def find_packages(paths=('.',), exclude=()): return packages +def resolve_dotted_name(dotted_name): + module_name, rest = dotted_name.split('.')[0], dotted_name.split('.')[1:] + while len(rest) > 0: + try: + ret = __import__(module_name) + break + except ImportError: + if rest == []: + raise + module_name += ('.' + rest[0]) + rest = rest[1:] + while len(rest) > 0: + try: + ret = getattr(ret, rest.pop(0)) + except AttributeError: + raise ImportError + return ret + # utility functions for 2to3 support def run_2to3(files, doctests_only=False, fixer_names=None, options=None, @@ -688,3 +701,419 @@ class Mixin2to3: """ Issues a call to util.run_2to3. """ return run_2to3(files, doctests_only, self.fixer_names, self.options, self.explicit) + + +def splitext(path): + """Like os.path.splitext, but take off .tar too""" + base, ext = posixpath.splitext(path) + if base.lower().endswith('.tar'): + ext = base[-4:] + ext + base = base[:-4] + return base, ext + + +def unzip_file(filename, location, flatten=True): + """Unzip the file (zip file located at filename) to the destination + location""" + if not os.path.exists(location): + os.makedirs(location) + zipfp = open(filename, 'rb') + try: + zip = zipfile.ZipFile(zipfp) + leading = has_leading_dir(zip.namelist()) and flatten + for name in zip.namelist(): + data = zip.read(name) + fn = name + if leading: + fn = split_leading_dir(name)[1] + fn = os.path.join(location, fn) + dir = os.path.dirname(fn) + if not os.path.exists(dir): + os.makedirs(dir) + if fn.endswith('/') or fn.endswith('\\'): + # A directory + if not os.path.exists(fn): + os.makedirs(fn) + else: + fp = open(fn, 'wb') + try: + fp.write(data) + finally: + fp.close() + finally: + zipfp.close() + + +def untar_file(filename, location): + """Untar the file (tar file located at filename) to the destination + location + """ + if not os.path.exists(location): + os.makedirs(location) + if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): + mode = 'r:gz' + elif (filename.lower().endswith('.bz2') + or filename.lower().endswith('.tbz')): + mode = 'r:bz2' + elif filename.lower().endswith('.tar'): + mode = 'r' + else: + mode = 'r:*' + tar = tarfile.open(filename, mode) + try: + leading = has_leading_dir([member.name for member in tar.getmembers()]) + for member in tar.getmembers(): + fn = member.name + if leading: + fn = split_leading_dir(fn)[1] + path = os.path.join(location, fn) + if member.isdir(): + if not os.path.exists(path): + os.makedirs(path) + else: + try: + fp = tar.extractfile(member) + except (KeyError, AttributeError), e: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + continue + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + destfp = open(path, 'wb') + try: + shutil.copyfileobj(fp, destfp) + finally: + destfp.close() + fp.close() + finally: + tar.close() + + +def has_leading_dir(paths): + """Returns true if all the paths have the same leading path name + (i.e., everything is in one subdirectory in an archive)""" + common_prefix = None + for path in paths: + prefix, rest = split_leading_dir(path) + if not prefix: + return False + elif common_prefix is None: + common_prefix = prefix + elif prefix != common_prefix: + return False + return True + + +def split_leading_dir(path): + path = str(path) + path = path.lstrip('/').lstrip('\\') + if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) + or '\\' not in path): + return path.split('/', 1) + elif '\\' in path: + return path.split('\\', 1) + else: + return path, '' + + +def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None): + """Run another program specified as a command list 'cmd' in a new process. + + 'cmd' is just the argument list for the new process, ie. + cmd[0] is the program to run and cmd[1:] are the rest of its arguments. + There is no way to run a program with a name different from that of its + executable. + + If 'search_path' is true (the default), the system's executable + search path will be used to find the program; otherwise, cmd[0] + must be the exact path to the executable. If 'dry_run' is true, + the command will not actually be run. + + If 'env' is given, it's a environment dictionary used for the execution + environment. + + Raise DistutilsExecError if running the program fails in any way; just + return on success. + """ + if os.name == 'posix': + _spawn_posix(cmd, search_path, dry_run=dry_run, env=env) + elif os.name == 'nt': + _spawn_nt(cmd, search_path, dry_run=dry_run, env=env) + elif os.name == 'os2': + _spawn_os2(cmd, search_path, dry_run=dry_run, env=env) + else: + raise DistutilsPlatformError( + "don't know how to spawn programs on platform '%s'" % os.name) + + +def _nt_quote_args(args): + """Quote command-line arguments for DOS/Windows conventions. + + Just wraps every argument which contains blanks in double quotes, and + returns a new argument list. + """ + # XXX this doesn't seem very robust to me -- but if the Windows guys + # say it'll work, I guess I'll have to accept it. (What if an arg + # contains quotes? What other magic characters, other than spaces, + # have to be escaped? Is there an escaping mechanism other than + # quoting?) + for i, arg in enumerate(args): + if ' ' in arg: + args[i] = '"%s"' % arg + return args + + +def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0, env=None): + executable = cmd[0] + cmd = _nt_quote_args(cmd) + if search_path: + # either we find one or it stays the same + executable = find_executable(executable) or executable + log.info(' '.join([executable] + cmd[1:])) + if not dry_run: + # spawn for NT requires a full path to the .exe + try: + if env is None: + rc = os.spawnv(os.P_WAIT, executable, cmd) + else: + rc = os.spawnve(os.P_WAIT, executable, cmd, env) + + except OSError, exc: + # this seems to happen when the command isn't found + raise DistutilsExecError( + "command '%s' failed: %s" % (cmd[0], exc[-1])) + if rc != 0: + # and this reflects the command running but failing + raise DistutilsExecError( + "command '%s' failed with exit status %d" % (cmd[0], rc)) + + +def _spawn_os2(cmd, search_path=1, verbose=0, dry_run=0, env=None): + executable = cmd[0] + if search_path: + # either we find one or it stays the same + executable = find_executable(executable) or executable + log.info(' '.join([executable] + cmd[1:])) + if not dry_run: + # spawnv for OS/2 EMX requires a full path to the .exe + try: + if env is None: + rc = os.spawnv(os.P_WAIT, executable, cmd) + else: + rc = os.spawnve(os.P_WAIT, executable, cmd, env) + + except OSError, exc: + # this seems to happen when the command isn't found + raise DistutilsExecError( + "command '%s' failed: %s" % (cmd[0], exc[-1])) + if rc != 0: + # and this reflects the command running but failing + log.debug("command '%s' failed with exit status %d" % (cmd[0], rc)) + raise DistutilsExecError( + "command '%s' failed with exit status %d" % (cmd[0], rc)) + + +def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0, env=None): + log.info(' '.join(cmd)) + if dry_run: + return + + if env is None: + exec_fn = search_path and os.execvp or os.execv + else: + exec_fn = search_path and os.execvpe or os.execve + + pid = os.fork() + + if pid == 0: # in the child + try: + if env is None: + exec_fn(cmd[0], cmd) + else: + exec_fn(cmd[0], cmd, env) + except OSError, e: + sys.stderr.write("unable to execute %s: %s\n" % + (cmd[0], e.strerror)) + os._exit(1) + + sys.stderr.write("unable to execute %s for unknown reasons" % cmd[0]) + os._exit(1) + else: # in the parent + # Loop until the child either exits or is terminated by a signal + # (ie. keep waiting if it's merely stopped) + while 1: + try: + pid, status = os.waitpid(pid, 0) + except OSError, exc: + import errno + if exc.errno == errno.EINTR: + continue + raise DistutilsExecError( + "command '%s' failed: %s" % (cmd[0], exc[-1])) + if os.WIFSIGNALED(status): + raise DistutilsExecError( + "command '%s' terminated by signal %d" % \ + (cmd[0], os.WTERMSIG(status))) + + elif os.WIFEXITED(status): + exit_status = os.WEXITSTATUS(status) + if exit_status == 0: + return # hey, it succeeded! + else: + raise DistutilsExecError( + "command '%s' failed with exit status %d" % \ + (cmd[0], exit_status)) + + elif os.WIFSTOPPED(status): + continue + + else: + raise DistutilsExecError( + "unknown error executing '%s': termination status %d" % \ + (cmd[0], status)) + + +def find_executable(executable, path=None): + """Tries to find 'executable' in the directories listed in 'path'. + + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + """ + if path is None: + path = os.environ['PATH'] + paths = path.split(os.pathsep) + base, ext = os.path.splitext(executable) + + if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'): + executable = executable + '.exe' + + if not os.path.isfile(executable): + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None + else: + return executable + + +DEFAULT_REPOSITORY = 'http://pypi.python.org/pypi' +DEFAULT_REALM = 'pypi' +DEFAULT_PYPIRC = """\ +[distutils] +index-servers = + pypi + +[pypi] +username:%s +password:%s +""" + +def get_pypirc_path(): + """Returns rc file path.""" + return os.path.join(os.path.expanduser('~'), '.pypirc') + + +def generate_pypirc(username, password): + """Creates a default .pypirc file.""" + rc = get_pypirc_path() + f = open(rc, 'w') + try: + f.write(DEFAULT_PYPIRC % (username, password)) + finally: + f.close() + try: + os.chmod(rc, 0600) + except OSError: + # should do something better here + pass + + +def read_pypirc(repository=DEFAULT_REPOSITORY, realm=DEFAULT_REALM): + """Reads the .pypirc file.""" + rc = get_pypirc_path() + if os.path.exists(rc): + config = RawConfigParser() + config.read(rc) + sections = config.sections() + if 'distutils' in sections: + # let's get the list of servers + index_servers = config.get('distutils', 'index-servers') + _servers = [server.strip() for server in + index_servers.split('\n') + if server.strip() != ''] + if _servers == []: + # nothing set, let's try to get the default pypi + if 'pypi' in sections: + _servers = ['pypi'] + else: + # the file is not properly defined, returning + # an empty dict + return {} + for server in _servers: + current = {'server': server} + current['username'] = config.get(server, 'username') + + # optional params + for key, default in (('repository', + DEFAULT_REPOSITORY), + ('realm', DEFAULT_REALM), + ('password', None)): + if config.has_option(server, key): + current[key] = config.get(server, key) + else: + current[key] = default + if (current['server'] == repository or + current['repository'] == repository): + return current + elif 'server-login' in sections: + # old format + server = 'server-login' + if config.has_option(server, 'repository'): + repository = config.get(server, 'repository') + else: + repository = DEFAULT_REPOSITORY + + return {'username': config.get(server, 'username'), + 'password': config.get(server, 'password'), + 'repository': repository, + 'server': server, + 'realm': DEFAULT_REALM} + + return {} + + +def metadata_to_dict(meta): + """XXX might want to move it to the Metadata class.""" + data = { + 'metadata_version' : meta.version, + 'name': meta['Name'], + 'version': meta['Version'], + 'summary': meta['Summary'], + 'home_page': meta['Home-page'], + 'author': meta['Author'], + 'author_email': meta['Author-email'], + 'license': meta['License'], + 'description': meta['Description'], + 'keywords': meta['Keywords'], + 'platform': meta['Platform'], + 'classifier': meta['Classifier'], + 'download_url': meta['Download-URL'], + } + + if meta.version == '1.2': + data['requires_dist'] = meta['Requires-Dist'] + data['requires_python'] = meta['Requires-Python'] + data['requires_external'] = meta['Requires-External'] + data['provides_dist'] = meta['Provides-Dist'] + data['obsoletes_dist'] = meta['Obsoletes-Dist'] + data['project_url'] = [','.join(url) for url in + meta['Project-URL']] + + elif meta.version == '1.1': + data['provides'] = meta['Provides'] + data['requires'] = meta['Requires'] + data['obsoletes'] = meta['Obsoletes'] + + return data diff --git a/src/distutils2/version.py b/src/distutils2/version.py index e61c60e..64c33f7 100644 --- a/src/distutils2/version.py +++ b/src/distutils2/version.py @@ -321,7 +321,7 @@ def suggest_normalized_version(s): return None -_PREDICATE = re.compile(r"(?i)^\s*([a-z_]\w*(?:\.[a-z_]\w*)*)(.*)") +_PREDICATE = re.compile(r"(?i)^\s*([a-z_][\sa-zA-Z_-]*(?:\.[a-z_]\w*)*)(.*)") _VERSIONS = re.compile(r"^\s*\((.*)\)\s*$") _PLAIN_VERSIONS = re.compile(r"^\s*(.*)\s*$") _SPLIT_CMP = re.compile(r"^\s*(<=|>=|<|>|!=|==)\s*([^\s,]+)\s*$") @@ -349,12 +349,14 @@ class VersionPredicate(object): } def __init__(self, predicate): + self._string = predicate predicate = predicate.strip() match = _PREDICATE.match(predicate) if match is None: raise ValueError('Bad predicate "%s"' % predicate) - self.name, predicates = match.groups() + name, predicates = match.groups() + self.name = name.strip() predicates = predicates.strip() predicates = _VERSIONS.match(predicates) if predicates is not None: @@ -373,6 +375,9 @@ class VersionPredicate(object): return False return True + def __repr__(self): + return self._string + class _Versions(VersionPredicate): def __init__(self, predicate): @@ -417,3 +422,12 @@ def is_valid_version(predicate): return False else: return True + + +def get_version_predicate(requirements): + """Return a VersionPredicate object, from a string or an already + existing object. + """ + if isinstance(requirements, str): + requirements = VersionPredicate(requirements) + return requirements diff --git a/src/runtests-cov.py b/src/runtests-cov.py index 6c2ab41..d6678b4 100755 --- a/src/runtests-cov.py +++ b/src/runtests-cov.py @@ -5,9 +5,20 @@ The tests for distutils2 are defined in the distutils2.tests package. """ import sys -from os.path import dirname, islink, realpath +from os.path import dirname, islink, realpath, join, abspath from optparse import OptionParser +COVERAGE_FILE = join(dirname(abspath(__file__)), '.coverage') + +def get_coverage(): + """ Return a usable coverage object. """ + # deferred import because coverage is optional + import coverage + cov = getattr(coverage, "the_coverage", None) + if not cov: + cov = coverage.coverage(COVERAGE_FILE) + return cov + def ignore_prefixes(module): """ Return a list of prefixes to ignore in the coverage report if we want to completely skip `module`. @@ -16,16 +27,17 @@ def ignore_prefixes(module): # distributions, such a Ubuntu, really like to build link farm in # /usr/lib in order to save a few bytes on the disk. dirnames = [dirname(module.__file__)] - + pymod = module.__file__.rstrip("c") if islink(pymod): dirnames.append(dirname(realpath(pymod))) return dirnames + def parse_opts(): parser = OptionParser(usage="%prog [OPTIONS]", description="run the distutils2 unittests") - + parser.add_option("-q", "--quiet", help="do not print verbose messages", action="store_true", default=False) parser.add_option("-c", "--coverage", action="store_true", default=False, @@ -36,15 +48,23 @@ def parse_opts(): default=False, help=("Show line numbers of statements in each module " "that weren't executed.")) - + opts, args = parser.parse_args() return opts, args + def coverage_report(opts): - import coverage from distutils2.tests.support import unittest - cov = coverage.coverage() - cov.load() + cov = get_coverage() + if hasattr(cov, "load"): + # running coverage 3.x + cov.load() + morfs = None + else: + # running coverage 2.x + cov.cache = COVERAGE_FILE + cov.restore() + morfs = [m for m in cov.cexecuted.keys() if "distutils2" in m] prefixes = ["runtests", "distutils2/tests", "distutils2/_backport"] prefixes += ignore_prefixes(unittest) @@ -63,7 +83,7 @@ def coverage_report(opts): # that module is also completely optional pass - cov.report(omit_prefixes=prefixes, show_missing=opts.show_missing) + cov.report(morfs, omit_prefixes=prefixes, show_missing=opts.show_missing) def test_main(): @@ -71,11 +91,8 @@ def test_main(): verbose = not opts.quiet ret = 0 - if opts.coverage or opts.report: - import coverage - if opts.coverage: - cov = coverage.coverage() + cov = get_coverage() cov.erase() cov.start() if not opts.report: @@ -89,6 +106,7 @@ def test_main(): return ret + def run_tests(verbose): import distutils2.tests from distutils2.tests import run_unittest, reap_children, TestFailed @@ -108,12 +126,11 @@ def run_tests(verbose): finally: reap_children() + if __name__ == "__main__": try: from distutils2.tests.support import unittest except ImportError: sys.stderr.write('Error: You have to install unittest2') sys.exit(1) - sys.exit(test_main()) - diff --git a/src/runtests.py b/src/runtests.py index b9e2477..415d37a 100644 --- a/src/runtests.py +++ b/src/runtests.py @@ -1,9 +1,12 @@ +#!/usr/bin/env python """Tests for distutils2. The tests for distutils2 are defined in the distutils2.tests package. """ + import sys + def test_main(): import distutils2.tests from distutils2.tests import run_unittest, reap_children, TestFailed @@ -23,12 +26,11 @@ def test_main(): finally: reap_children() + if __name__ == "__main__": try: from distutils2.tests.support import unittest except ImportError: sys.stderr.write('Error: You have to install unittest2') sys.exit(1) - sys.exit(test_main()) - diff --git a/src/setup.cfg b/src/setup.cfg new file mode 100644 index 0000000..169aaf6 --- /dev/null +++ b/src/setup.cfg @@ -0,0 +1,3 @@ +[build_ext] +# needed so that tests work without mucking with sys.path +inplace = 1 diff --git a/src/setup.py b/src/setup.py index 5345b37..1233b21 100644 --- a/src/setup.py +++ b/src/setup.py @@ -168,29 +168,25 @@ def prepare_hashlib_extensions(): # The _hashlib module wraps optimized implementations # of hash functions from the OpenSSL library. - exts.append(Extension('_hashlib', ['_hashopenssl.c'], + exts.append(Extension('distutils2._backport._hashlib', + ['distutils2/_backport/_hashopenssl.c'], include_dirs = [ssl_inc_dir], library_dirs = [os.path.dirname(ssl_lib)], libraries = oslibs[os.name])) else: - exts.append(Extension('_sha', ['shamodule.c']) ) - exts.append(Extension('_md5', - sources=['md5module.c', 'md5.c'], - depends=['md5.h']) ) + exts.append(Extension('distutils2._backport._sha', + ['distutils2/_backport/shamodule.c'])) + exts.append(Extension('distutils2._backport._md5', + sources=['distutils2/_backport/md5module.c', + 'distutils2/_backport/md5.c'], + depends=['distutils2/_backport/md5.h']) ) if (not ssl_lib or openssl_ver < 0x00908000): # OpenSSL doesn't do these until 0.9.8 so we'll bring our own - exts.append(Extension('_sha256', ['sha256module.c'])) - exts.append(Extension('_sha512', ['sha512module.c'])) - - def prepend_modules(filename): - return os.path.join('Modules', filename) - - # all the C code is in the Modules subdirectory, prepend the path - for ext in exts: - ext.sources = [prepend_modules(fn) for fn in ext.sources] - if hasattr(ext, 'depends') and ext.depends is not None: - ext.depends = [prepend_modules(fn) for fn in ext.depends] + exts.append(Extension('distutils2._backport._sha256', + ['distutils2/_backport/sha256module.c'])) + exts.append(Extension('distutils2._backport._sha512', + ['distutils2/_backport/sha512module.c'])) return exts diff --git a/src/tests.sh b/src/tests.sh index aa339a3..8e6295d 100755 --- a/src/tests.sh +++ b/src/tests.sh @@ -1,40 +1,40 @@ #!/bin/sh echo -n "Running tests for Python 2.4... " -rm -rf _hashlib.so -python2.4 setup.py build_ext -i -q 2> /dev/null > /dev/null +rm -f distutils2/_backport/_hashlib.so +python2.4 setup.py build_ext -f -q 2> /dev/null > /dev/null python2.4 -Wd runtests.py -q 2> /dev/null if [ $? -ne 0 ];then - echo "Failed" + echo Failed + rm -f distutils2/_backport/_hashlib.so exit 1 else - echo "Success" + echo Success fi echo -n "Running tests for Python 2.5... " -rm -rf _hashlib.so -python2.5 setup.py build_ext -i -q 2> /dev/null > /dev/null python2.5 -Wd runtests.py -q 2> /dev/null if [ $? -ne 0 ];then - echo "Failed" + echo Failed exit 1 else - echo "Success" + echo Success fi echo -n "Running tests for Python 2.6... " -python2.6 -Wd -bb -3 runtests.py -q 2> /dev/null +python2.6 -Wd runtests.py -q 2> /dev/null if [ $? -ne 0 ];then - echo "Failed" + echo Failed exit 1 else - echo "Success" + echo Success fi echo -n "Running tests for Python 2.7... " python2.7 -Wd -bb -3 runtests.py -q 2> /dev/null if [ $? -ne 0 ];then - echo "Failed" + echo Failed exit 1 else - echo "Success" + echo Success fi +echo Good job, commit now! |