diff options
| author | Antoine Reversat <a.reversat@gmail.com> | 2010-08-05 23:01:33 -0400 |
|---|---|---|
| committer | Antoine Reversat <a.reversat@gmail.com> | 2010-08-05 23:01:33 -0400 |
| commit | 363c0ae3f332949a5721b6e01353902b89cde68f (patch) | |
| tree | 56e83ccaca90a0587832c3fbd04f28a3fd3d0dbf /src | |
| parent | b4a2d7104551bbcc443b0816438fd0d59ea8fdac (diff) | |
| parent | 4c007c55658abdc098716fcb3264f8acf1da0f4b (diff) | |
| download | disutils2-363c0ae3f332949a5721b6e01353902b89cde68f.tar.gz | |
Merges changes
Diffstat (limited to 'src')
89 files changed, 3467 insertions, 1612 deletions
diff --git a/src/DEVNOTES.txt b/src/DEVNOTES.txt index 4e506f8..528fcb0 100644 --- a/src/DEVNOTES.txt +++ b/src/DEVNOTES.txt @@ -1,10 +1,22 @@ Notes for developers ==================== -- Distutils2 runs from 2.4 to 3.2 (3.x not implemented yet), so - make sure you don't use a syntax that doesn't work under +- Distutils2 runs on Python from 2.4 to 3.2 (3.x not implemented yet), + so make sure you don't use a syntax that doesn't work under one of these Python versions. - Always run tests.sh before you push a change. This implies - that you have all Python versions installed from 2.4 to 2.6. + that you have all Python versions installed from 2.4 to 2.7. +- With Python 2.4, if you want to run tests with runtests.py, or run + just one test directly, be sure to run python2.4 setup.py build_ext + first, else tests won't find _hashlib or _md5. When using tests.sh, + build_ext is automatically done. + +- Renaming to do: + + - DistributionMetadata > Metadata or ReleaseMetadata + - pkgutil > pkgutil.__init__ + new pkgutil.database (or better name) + - pypi > index + - RationalizedVersion > Version + - suggest_rationalized_version > suggest diff --git a/src/distutils2/__init__.py b/src/distutils2/__init__.py index f37e4ff..4d330b3 100644 --- a/src/distutils2/__init__.py +++ b/src/distutils2/__init__.py @@ -16,8 +16,7 @@ without causing the other modules to be imported: """ __all__ = ['__version__'] -__revision__ = \ - "$Id: __init__.py 78020 2010-02-06 16:37:32Z benjamin.peterson $" +__revision__ = "$Id: __init__.py 78020 2010-02-06 16:37:32Z benjamin.peterson $" __version__ = "1.0a2" diff --git a/src/Modules/_hashopenssl.c b/src/distutils2/_backport/_hashopenssl.c index fdcae5d..fdcae5d 100644 --- a/src/Modules/_hashopenssl.c +++ b/src/distutils2/_backport/_hashopenssl.c diff --git a/src/distutils2/_backport/hashlib.py b/src/distutils2/_backport/hashlib.py index 48f2cfd..dd2b02a 100644 --- a/src/distutils2/_backport/hashlib.py +++ b/src/distutils2/_backport/hashlib.py @@ -65,20 +65,20 @@ __all__ = __always_supported + ('new', 'algorithms') def __get_builtin_constructor(name): if name in ('SHA1', 'sha1'): - import _sha + from distutils2._backport import _sha return _sha.new elif name in ('MD5', 'md5'): - import _md5 + from distutils2._backport import _md5 return _md5.new elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): - import _sha256 + from distutils2._backport import _sha256 bs = name[3:] if bs == '256': return _sha256.sha256 elif bs == '224': return _sha256.sha224 elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): - import _sha512 + from distutils2._backport import _sha512 bs = name[3:] if bs == '512': return _sha512.sha512 @@ -122,7 +122,7 @@ def __hash_new(name, string=''): try: - import _hashlib + from distutils2._backport import _hashlib new = __hash_new __get_hash = __get_openssl_constructor except ImportError: diff --git a/src/Modules/md5.c b/src/distutils2/_backport/md5.c index c35d96c..c35d96c 100644 --- a/src/Modules/md5.c +++ b/src/distutils2/_backport/md5.c diff --git a/src/Modules/md5.h b/src/distutils2/_backport/md5.h index 1718401..1718401 100644 --- a/src/Modules/md5.h +++ b/src/distutils2/_backport/md5.h diff --git a/src/Modules/md5module.c b/src/distutils2/_backport/md5module.c index 5e4f116..5e4f116 100644 --- a/src/Modules/md5module.c +++ b/src/distutils2/_backport/md5module.c diff --git a/src/distutils2/_backport/pkgutil.py b/src/distutils2/_backport/pkgutil.py index b033082..ccf45bf 100644 --- a/src/distutils2/_backport/pkgutil.py +++ b/src/distutils2/_backport/pkgutil.py @@ -20,6 +20,7 @@ except ImportError: import re import warnings + __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', 'walk_packages', 'iter_modules', 'get_data', @@ -27,6 +28,7 @@ __all__ = [ 'Distribution', 'EggInfoDistribution', 'distinfo_dirname', 'get_distributions', 'get_distribution', 'get_file_users', 'provides_distribution', 'obsoletes_distribution', + 'enable_cache', 'disable_cache', 'clear_cache' ] @@ -187,8 +189,8 @@ class ImpImporter(object): searches the current ``sys.path``, plus any modules that are frozen or built-in. - Note that :class:`ImpImporter` does not currently support being used by placement - on ``sys.meta_path``. + Note that :class:`ImpImporter` does not currently support being used by + placement on ``sys.meta_path``. """ def __init__(self, path=None): @@ -577,7 +579,8 @@ def get_data(package, resource): argument should be the name of a package, in standard module format (``foo.bar``). The resource argument should be in the form of a relative filename, using ``'/'`` as the path separator. The parent directory name - ``'..'`` is not allowed, and nor is a rooted name (starting with a ``'/'``). + ``'..'`` is not allowed, and nor is a rooted name (starting with a + ``'/'``). The function returns a binary string, which is the contents of the specified resource. @@ -613,6 +616,97 @@ def get_data(package, resource): DIST_FILES = ('INSTALLER', 'METADATA', 'RECORD', 'REQUESTED',) +# Cache +_cache_name = {} # maps names to Distribution instances +_cache_name_egg = {} # maps names to EggInfoDistribution instances +_cache_path = {} # maps paths to Distribution instances +_cache_path_egg = {} # maps paths to EggInfoDistribution instances +_cache_generated = False # indicates if .dist-info distributions are cached +_cache_generated_egg = False # indicates if .dist-info and .egg are cached +_cache_enabled = True + + +def enable_cache(): + """ + Enables the internal cache. + + Note that this function will not clear the cache in any case, for that + functionality see :func:`clear_cache`. + """ + global _cache_enabled + + _cache_enabled = True + +def disable_cache(): + """ + Disables the internal cache. + + Note that this function will not clear the cache in any case, for that + functionality see :func:`clear_cache`. + """ + global _cache_enabled + + _cache_enabled = False + +def clear_cache(): + """ Clears the internal cache. """ + global _cache_name, _cache_name_egg, cache_path, _cache_path_egg, \ + _cache_generated, _cache_generated_egg + + _cache_name = {} + _cache_name_egg = {} + _cache_path = {} + _cache_path_egg = {} + _cache_generated = False + _cache_generated_egg = False + + +def _yield_distributions(include_dist, include_egg): + """ + Yield .dist-info and .egg(-info) distributions, based on the arguments + + :parameter include_dist: yield .dist-info distributions + :parameter include_egg: yield .egg(-info) distributions + """ + for path in sys.path: + realpath = os.path.realpath(path) + if not os.path.isdir(realpath): + continue + for dir in os.listdir(realpath): + dist_path = os.path.join(realpath, dir) + if include_dist and dir.endswith('.dist-info'): + yield Distribution(dist_path) + elif include_egg and (dir.endswith('.egg-info') or + dir.endswith('.egg')): + yield EggInfoDistribution(dist_path) + + +def _generate_cache(use_egg_info=False): + global _cache_generated, _cache_generated_egg + + if _cache_generated_egg or (_cache_generated and not use_egg_info): + return + else: + gen_dist = not _cache_generated + gen_egg = use_egg_info + + for dist in _yield_distributions(gen_dist, gen_egg): + if isinstance(dist, Distribution): + _cache_path[dist.path] = dist + if not dist.name in _cache_name: + _cache_name[dist.name] = [] + _cache_name[dist.name].append(dist) + else: + _cache_path_egg[dist.path] = dist + if not dist.name in _cache_name_egg: + _cache_name_egg[dist.name] = [] + _cache_name_egg[dist.name].append(dist) + + if gen_dist: + _cache_generated = True + if gen_egg: + _cache_generated_egg = True + class Distribution(object): """Created with the *path* of the ``.dist-info`` directory provided to the @@ -627,15 +721,23 @@ class Distribution(object): """A :class:`distutils2.metadata.DistributionMetadata` instance loaded with the distribution's ``METADATA`` file.""" requested = False - """A boolean that indicates whether the ``REQUESTED`` metadata file is present - (in other words, whether the package was installed by user request).""" + """A boolean that indicates whether the ``REQUESTED`` metadata file is + present (in other words, whether the package was installed by user + request or it was installed as a dependency).""" def __init__(self, path): + if _cache_enabled and path in _cache_path: + self.metadata = _cache_path[path].metadata + else: + metadata_path = os.path.join(path, 'METADATA') + self.metadata = DistributionMetadata(path=metadata_path) + self.path = path - metadata_path = os.path.join(path, 'METADATA') - self.metadata = DistributionMetadata(path=metadata_path) self.name = self.metadata['name'] + if _cache_enabled and not path in _cache_path: + _cache_path[path] = self + def _get_records(self, local=False): RECORD = os.path.join(self.path, 'RECORD') record_reader = csv_reader(open(RECORD, 'rb'), delimiter=',') @@ -756,6 +858,11 @@ class EggInfoDistribution(object): def __init__(self, path): self.path = path + if _cache_enabled and path in _cache_path_egg: + self.metadata = _cache_path_egg[path].metadata + self.name = self.metadata['Name'] + return + # reused from Distribute's pkg_resources def yield_lines(strs): """Yield non-empty/non-comment lines of a ``basestring`` or sequence""" @@ -787,7 +894,7 @@ class EggInfoDistribution(object): requires = zipf.get_data('EGG-INFO/requires.txt') except IOError: requires = None - self.name = self.metadata['name'] + self.name = self.metadata['Name'] elif path.endswith('.egg-info'): if os.path.isdir(path): path = os.path.join(path, 'PKG-INFO') @@ -840,6 +947,9 @@ class EggInfoDistribution(object): else: self.metadata['Requires'] += reqs + if _cache_enabled: + _cache_path_egg[self.path] = self + def get_installed_files(self, local=False): return [] @@ -898,17 +1008,17 @@ def get_distributions(use_egg_info=False): :rtype: iterator of :class:`Distribution` and :class:`EggInfoDistribution` instances""" - for path in sys.path: - realpath = os.path.realpath(path) - if not os.path.isdir(realpath): - continue - for dir in os.listdir(realpath): - if dir.endswith('.dist-info'): - dist = Distribution(os.path.join(realpath, dir)) - yield dist - elif use_egg_info and (dir.endswith('.egg-info') or - dir.endswith('.egg')): - dist = EggInfoDistribution(os.path.join(realpath, dir)) + if not _cache_enabled: + for dist in _yield_distributions(True, use_egg_info): + yield dist + else: + _generate_cache(use_egg_info) + + for dist in _cache_path.itervalues(): + yield dist + + if use_egg_info: + for dist in _cache_path_egg.itervalues(): yield dist @@ -928,17 +1038,19 @@ def get_distribution(name, use_egg_info=False): value is expected. If the directory is not found, ``None`` is returned. :rtype: :class:`Distribution` or :class:`EggInfoDistribution` or None""" - found = None - for dist in get_distributions(): - if dist.name == name: - found = dist - break - if use_egg_info: - for dist in get_distributions(True): + if not _cache_enabled: + for dist in _yield_distributions(True, use_egg_info): if dist.name == name: - found = dist - break - return found + return dist + else: + _generate_cache(use_egg_info) + + if name in _cache_name: + return _cache_name[name][0] + elif use_egg_info and name in _cache_name_egg: + return _cache_name_egg[name][0] + else: + return None def obsoletes_distribution(name, version=None, use_egg_info=False): diff --git a/src/Modules/sha256module.c b/src/distutils2/_backport/sha256module.c index 96cbd10..96cbd10 100644 --- a/src/Modules/sha256module.c +++ b/src/distutils2/_backport/sha256module.c diff --git a/src/Modules/sha512module.c b/src/distutils2/_backport/sha512module.c index dbaec17..dbaec17 100644 --- a/src/Modules/sha512module.c +++ b/src/distutils2/_backport/sha512module.c diff --git a/src/Modules/shamodule.c b/src/distutils2/_backport/shamodule.c index 171dfa1..171dfa1 100644 --- a/src/Modules/shamodule.c +++ b/src/distutils2/_backport/shamodule.c diff --git a/src/distutils2/_backport/tarfile.py b/src/distutils2/_backport/tarfile.py index 0d2c49b..3c95868 100644 --- a/src/distutils2/_backport/tarfile.py +++ b/src/distutils2/_backport/tarfile.py @@ -56,13 +56,6 @@ import operator if not hasattr(os, 'SEEK_SET'): os.SEEK_SET = 0 -if sys.platform == 'mac': - # This module needs work for MacOS9, especially in the area of pathname - # handling. In many places it is assumed a simple substitution of / by the - # local os.path.sep is good enough to convert pathnames, but this does not - # work with the mac rooted:path:name versus :nonrooted:path:name syntax - raise ImportError, "tarfile does not work for platform==mac" - try: import grp, pwd except ImportError: diff --git a/src/distutils2/_backport/tests/__init__.py b/src/distutils2/_backport/tests/__init__.py index e816dd0..78c8142 100644 --- a/src/distutils2/_backport/tests/__init__.py +++ b/src/distutils2/_backport/tests/__init__.py @@ -4,7 +4,7 @@ import sys from distutils2.tests.support import unittest -here = os.path.dirname(__file__) +here = os.path.dirname(__file__) or os.curdir def test_suite(): suite = unittest.TestSuite() @@ -16,4 +16,5 @@ def test_suite(): suite.addTest(module.test_suite()) return suite - +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/src/distutils2/_backport/tests/fake_dists/grammar-1.0a4.dist-info/METADATA b/src/distutils2/_backport/tests/fake_dists/grammar-1.0a4.dist-info/METADATA index 1978be7..0b99f52 100644 --- a/src/distutils2/_backport/tests/fake_dists/grammar-1.0a4.dist-info/METADATA +++ b/src/distutils2/_backport/tests/fake_dists/grammar-1.0a4.dist-info/METADATA @@ -2,3 +2,4 @@ Metadata-Version: 1.2 Name: grammar Version: 1.0a4 Requires-Dist: truffles (>=1.2) +Author: Sherlock Holmes diff --git a/src/distutils2/_backport/tests/fake_dists/truffles-5.0.egg-info b/src/distutils2/_backport/tests/fake_dists/truffles-5.0.egg-info new file mode 100644 index 0000000..45f0cf8 --- /dev/null +++ b/src/distutils2/_backport/tests/fake_dists/truffles-5.0.egg-info @@ -0,0 +1,3 @@ +Metadata-Version: 1.2 +Name: truffles +Version: 5.0 diff --git a/src/distutils2/_backport/tests/test_pkgutil.py b/src/distutils2/_backport/tests/test_pkgutil.py index fafb9a8..c052b4d 100644 --- a/src/distutils2/_backport/tests/test_pkgutil.py +++ b/src/distutils2/_backport/tests/test_pkgutil.py @@ -10,7 +10,7 @@ import zipfile try: from hashlib import md5 except ImportError: - from md5 import md5 + from distutils2._backport.hashlib import md5 from test.test_support import run_unittest, TESTFN from distutils2.tests.support import unittest @@ -25,12 +25,14 @@ except ImportError: except ImportError: from unittest2.compatibility import relpath +# Adapted from Python 2.7's trunk + # TODO Add a test for getting a distribution that is provided by another # distribution. # TODO Add a test for absolute pathed RECORD items (e.g. /etc/myapp/config.ini) -# Adapted from Python 2.7's trunk + class TestPkgUtilData(unittest.TestCase): def setUp(self): @@ -108,10 +110,14 @@ class TestPkgUtilData(unittest.TestCase): del sys.modules[pkg] + # Adapted from Python 2.7's trunk + + class TestPkgUtilPEP302(unittest.TestCase): class MyTestLoader(object): + def load_module(self, fullname): # Create an empty module mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) @@ -120,13 +126,14 @@ class TestPkgUtilPEP302(unittest.TestCase): # Make it a package mod.__path__ = [] # Count how many times the module is reloaded - mod.__dict__['loads'] = mod.__dict__.get('loads',0) + 1 + mod.__dict__['loads'] = mod.__dict__.get('loads', 0) + 1 return mod def get_data(self, path): return "Hello, world!" class MyTestImporter(object): + def find_module(self, fullname, path=None): return TestPkgUtilPEP302.MyTestLoader() @@ -319,7 +326,7 @@ class TestPkgUtilPEP376(unittest.TestCase): current_path = os.path.abspath(os.path.dirname(__file__)) self.sys_path = sys.path[:] self.fake_dists_path = os.path.join(current_path, 'fake_dists') - sys.path[0:0] = [self.fake_dists_path] + sys.path.insert(0, self.fake_dists_path) def tearDown(self): sys.path[:] = self.sys_path @@ -366,8 +373,12 @@ class TestPkgUtilPEP376(unittest.TestCase): if not isinstance(dist, Distribution): self.fail("item received was not a Distribution instance: " "%s" % type(dist)) - if dist.name in dict(fake_dists).keys(): + if dist.name in dict(fake_dists).keys() and \ + dist.path.startswith(self.fake_dists_path): found_dists.append((dist.name, dist.metadata['version'],)) + else: + # check that it doesn't find anything more than this + self.assertFalse(dist.path.startswith(self.fake_dists_path)) # otherwise we don't care what other distributions are found # Finally, test that we found all that we were looking for @@ -375,7 +386,8 @@ class TestPkgUtilPEP376(unittest.TestCase): # Now, test if the egg-info distributions are found correctly as well fake_dists += [('bacon', '0.1'), ('cheese', '2.0.2'), - ('banana', '0.4'), ('strawberry', '0.6')] + ('banana', '0.4'), ('strawberry', '0.6'), + ('truffles', '5.0')] found_dists = [] dists = [dist for dist in get_distributions(use_egg_info=True)] @@ -384,8 +396,11 @@ class TestPkgUtilPEP376(unittest.TestCase): isinstance(dist, EggInfoDistribution)): self.fail("item received was not a Distribution or " "EggInfoDistribution instance: %s" % type(dist)) - if dist.name in dict(fake_dists).keys(): + if dist.name in dict(fake_dists).keys() and \ + dist.path.startswith(self.fake_dists_path): found_dists.append((dist.name, dist.metadata['version'])) + else: + self.assertFalse(dist.path.startswith(self.fake_dists_path)) self.assertListEqual(sorted(fake_dists), sorted(found_dists)) @@ -485,7 +500,7 @@ class TestPkgUtilPEP376(unittest.TestCase): l = [dist.name for dist in provides_distribution('truffles', '>1.5', use_egg_info=True)] - checkLists(l, ['bacon']) + checkLists(l, ['bacon', 'truffles']) l = [dist.name for dist in provides_distribution('truffles', '>=1.0')] checkLists(l, ['choxie', 'towel-stuff']) @@ -549,6 +564,33 @@ class TestPkgUtilPEP376(unittest.TestCase): l = [dist.name for dist in obsoletes_distribution('truffles', '0.2')] checkLists(l, ['towel-stuff']) + def test_yield_distribution(self): + # tests the internal function _yield_distributions + from distutils2._backport.pkgutil import _yield_distributions + checkLists = lambda x, y: self.assertListEqual(sorted(x), sorted(y)) + + eggs = [('bacon', '0.1'), ('banana', '0.4'), ('strawberry', '0.6'), + ('truffles', '5.0'), ('cheese', '2.0.2')] + dists = [('choxie', '2.0.0.9'), ('grammar', '1.0a4'), + ('towel-stuff', '0.1')] + + checkLists([], _yield_distributions(False, False)) + + found = [(dist.name, dist.metadata['Version']) + for dist in _yield_distributions(False, True) + if dist.path.startswith(self.fake_dists_path)] + checkLists(eggs, found) + + found = [(dist.name, dist.metadata['Version']) + for dist in _yield_distributions(True, False) + if dist.path.startswith(self.fake_dists_path)] + checkLists(dists, found) + + found = [(dist.name, dist.metadata['Version']) + for dist in _yield_distributions(True, True) + if dist.path.startswith(self.fake_dists_path)] + checkLists(dists + eggs, found) + def test_suite(): suite = unittest.TestSuite() diff --git a/src/distutils2/command/bdist_dumb.py b/src/distutils2/command/bdist_dumb.py index 604bcb7..4113a1e 100644 --- a/src/distutils2/command/bdist_dumb.py +++ b/src/distutils2/command/bdist_dumb.py @@ -77,9 +77,7 @@ class bdist_dumb (Command): ("don't know how to create dumb built distributions " + "on platform %s") % os.name - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name')) + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') def run(self): if not self.skip_build: diff --git a/src/distutils2/command/bdist_msi.py b/src/distutils2/command/bdist_msi.py index bbbcbcc..b759a06 100644 --- a/src/distutils2/command/bdist_msi.py +++ b/src/distutils2/command/bdist_msi.py @@ -153,10 +153,7 @@ class bdist_msi (Command): else: self.versions = list(self.all_versions) - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') if self.pre_install_script: raise DistutilsOptionError, "the pre-install-script feature is not yet implemented" diff --git a/src/distutils2/command/bdist_wininst.py b/src/distutils2/command/bdist_wininst.py index 7b55de1..264f4fb 100644 --- a/src/distutils2/command/bdist_wininst.py +++ b/src/distutils2/command/bdist_wininst.py @@ -100,10 +100,7 @@ class bdist_wininst (Command): " option must be specified" % (short_version,) self.target_version = short_version - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') if self.install_script: for script in self.distribution.scripts: @@ -177,8 +174,8 @@ class bdist_wininst (Command): # And make an archive relative to the root of the # pseudo-installation tree. - from tempfile import mktemp - archive_basename = mktemp() + from tempfile import NamedTemporaryFile + archive_basename = NamedTemporaryFile().name fullname = self.distribution.get_fullname() arcname = self.make_archive(archive_basename, "zip", root_dir=self.bdist_dir) diff --git a/src/distutils2/command/build_clib.py b/src/distutils2/command/build_clib.py index 40c4b6e..0a25aa8 100644 --- a/src/distutils2/command/build_clib.py +++ b/src/distutils2/command/build_clib.py @@ -76,9 +76,7 @@ class build_clib(Command): self.set_undefined_options('build', ('build_temp', 'build_clib'), ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force')) + 'compiler', 'debug', 'force') self.libraries = self.distribution.libraries if self.libraries: diff --git a/src/distutils2/command/build_ext.py b/src/distutils2/command/build_ext.py index 7e160ab..29530eb 100644 --- a/src/distutils2/command/build_ext.py +++ b/src/distutils2/command/build_ext.py @@ -177,13 +177,8 @@ class build_ext(Command): def finalize_options(self): self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force'), - ('plat_name', 'plat_name'), - ) + 'build_lib', 'build_temp', 'compiler', + 'debug', 'force', 'plat_name') if self.package is None: self.package = self.distribution.ext_package @@ -292,7 +287,7 @@ class build_ext(Command): "config")) else: # building python standard extensions - self.library_dirs.append('.') + self.library_dirs.append(os.curdir) # for extensions under Linux or Solaris with a shared Python library, # Python's library directory must be appended to library_dirs @@ -305,7 +300,7 @@ class build_ext(Command): self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: # building python standard extensions - self.library_dirs.append('.') + self.library_dirs.append(os.curdir) # The argument parsing will result in self.define being a string, but # it has to be a list of 2-tuples. All the preprocessor symbols diff --git a/src/distutils2/command/build_py.py b/src/distutils2/command/build_py.py index c4c6d68..a805a1e 100644 --- a/src/distutils2/command/build_py.py +++ b/src/distutils2/command/build_py.py @@ -95,9 +95,7 @@ class build_py(Command, Mixin2to3): self._doctests_2to3 = [] def finalize_options(self): - self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('force', 'force')) + self.set_undefined_options('build', 'build_lib', 'force') # Get the distribution options that are aliases for build_py # options -- list of packages and list of modules. diff --git a/src/distutils2/command/build_scripts.py b/src/distutils2/command/build_scripts.py index ad1c234..b62ade7 100644 --- a/src/distutils2/command/build_scripts.py +++ b/src/distutils2/command/build_scripts.py @@ -40,8 +40,7 @@ class build_scripts (Command): def finalize_options (self): self.set_undefined_options('build', ('build_scripts', 'build_dir'), - ('force', 'force'), - ('executable', 'executable')) + 'force', 'executable') self.scripts = self.distribution.scripts def get_source_files(self): diff --git a/src/distutils2/command/check.py b/src/distutils2/command/check.py index 844672d..b3175e7 100644 --- a/src/distutils2/command/check.py +++ b/src/distutils2/command/check.py @@ -8,13 +8,13 @@ from distutils2.core import Command from distutils2.errors import DistutilsSetupError class check(Command): - """This command checks the meta-data of the package. + """This command checks the metadata of the package. """ description = ("perform some checks on the package") - user_options = [('metadata', 'm', 'Verify meta-data'), + user_options = [('metadata', 'm', 'Verify metadata'), ('restructuredtext', 'r', - ('Checks if long string meta-data syntax ' - 'are reStructuredText-compliant')), + ('Checks if long string metadata syntax ' + 'is reStructuredText-compliant')), ('strict', 's', 'Will exit with an error if a check fails')] @@ -53,7 +53,7 @@ class check(Command): raise DistutilsSetupError(msg) def check_metadata(self): - """Ensures that all required elements of meta-data are supplied. + """Ensures that all required elements of metadata are supplied. name, version, URL, (author and author_email) or (maintainer and maintainer_email)). @@ -62,7 +62,7 @@ class check(Command): """ missing, __ = self.distribution.metadata.check() if missing != []: - self.warn("missing required meta-data: %s" % ', '.join(missing)) + self.warn("missing required metadata: %s" % ', '.join(missing)) def check_restructuredtext(self): """Checks if the long string fields are reST-compliant.""" diff --git a/src/distutils2/command/clean.py b/src/distutils2/command/clean.py index baeaf05..8125dcd 100644 --- a/src/distutils2/command/clean.py +++ b/src/distutils2/command/clean.py @@ -40,13 +40,9 @@ class clean(Command): self.all = None def finalize_options(self): - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib'), - ('build_scripts', 'build_scripts'), - ('build_temp', 'build_temp')) - self.set_undefined_options('bdist', - ('bdist_base', 'bdist_base')) + self.set_undefined_options('build', 'build_base', 'build_lib', + 'build_scripts', 'build_temp') + self.set_undefined_options('bdist', 'bdist_base') def run(self): # remove the build/temp.<plat> directory (unless it's already diff --git a/src/distutils2/command/cmd.py b/src/distutils2/command/cmd.py index fa6f281..73ac9e1 100644 --- a/src/distutils2/command/cmd.py +++ b/src/distutils2/command/cmd.py @@ -51,6 +51,12 @@ class Command(object): # defined. The canonical example is the "install" command. sub_commands = [] + # Pre and post command hooks are run just before or just after the command + # itself. They are simple functions that receive the command instance. They + # should be specified as dotted strings. + pre_hook = None + post_hook = None + # -- Creation/initialization methods ------------------------------- @@ -297,26 +303,30 @@ class Command(object): else: return self.__class__.__name__ - def set_undefined_options(self, src_cmd, *option_pairs): - """Set the values of any "undefined" options from corresponding - option values in some other command object. "Undefined" here means - "is None", which is the convention used to indicate that an option - has not been changed between 'initialize_options()' and - 'finalize_options()'. Usually called from 'finalize_options()' for - options that depend on some other command rather than another - option of the same command. 'src_cmd' is the other command from - which option values will be taken (a command object will be created - for it if necessary); the remaining arguments are - '(src_option,dst_option)' tuples which mean "take the value of - 'src_option' in the 'src_cmd' command object, and copy it to - 'dst_option' in the current command object". + def set_undefined_options(self, src_cmd, *options): + """Set values of undefined options from another command. + + Undefined options are options set to None, which is the convention + used to indicate that an option has not been changed between + 'initialize_options()' and 'finalize_options()'. This method is + usually called from 'finalize_options()' for options that depend on + some other command rather than another option of the same command, + typically subcommands. + + The 'src_cmd' argument is the other command from which option values + will be taken (a command object will be created for it if necessary); + the remaining positional arguments are strings that give the name of + the option to set. If the name is different on the source and target + command, you can pass a tuple with '(name_on_source, name_on_dest)' so + that 'self.name_on_dest' will be set from 'src_cmd.name_on_source'. """ - - # Option_pairs: list of (src_option, dst_option) tuples - src_cmd_obj = self.distribution.get_command_obj(src_cmd) src_cmd_obj.ensure_finalized() - for (src_option, dst_option) in option_pairs: + for obj in options: + if isinstance(obj, tuple): + src_option, dst_option = obj + else: + src_option, dst_option = obj, obj if getattr(self, dst_option) is None: setattr(self, dst_option, getattr(src_cmd_obj, src_option)) diff --git a/src/distutils2/command/install.py b/src/distutils2/command/install.py index 588786f..da7e26a 100644 --- a/src/distutils2/command/install.py +++ b/src/distutils2/command/install.py @@ -79,15 +79,23 @@ class install(Command): ('record=', None, "filename in which to record list of installed files"), + + # .dist-info related arguments, read by install_dist_info + ('no-distinfo', None, 'do not create a .dist-info directory'), + ('installer=', None, 'the name of the installer'), + ('requested', None, 'generate a REQUESTED file'), + ('no-requested', None, 'do not generate a REQUESTED file'), + ('no-record', None, 'do not generate a RECORD file'), ] - boolean_options = ['compile', 'force', 'skip-build'] + boolean_options = ['compile', 'force', 'skip-build', 'no-dist-info', + 'requested', 'no-dist-record',] user_options.append(('user', None, "install in user site-package '%s'" % \ get_path('purelib', '%s_user' % os.name))) boolean_options.append('user') - negative_opt = {'no-compile' : 'compile'} + negative_opt = {'no-compile' : 'compile', 'no-requested': 'requested'} def initialize_options(self): @@ -160,6 +168,12 @@ class install(Command): self.record = None + # .dist-info related options + self.no_distinfo = None + self.installer = None + self.requested = None + self.no_record = None + # -- Option finalizing methods ------------------------------------- # (This is rather more involved than for most commands, @@ -299,13 +313,14 @@ class install(Command): self.dump_dirs("after prepending root") # Find out the build directories, ie. where to install from. - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib')) + self.set_undefined_options('build', 'build_base', 'build_lib') # Punt on doc directories for now -- after all, we're punting on # documentation completely! + if self.no_distinfo is None: + self.no_distinfo = False + def dump_dirs(self, msg): """Dumps the list of user options.""" from distutils2.fancy_getopt import longopt_xlate @@ -586,5 +601,7 @@ class install(Command): ('install_headers', has_headers), ('install_scripts', has_scripts), ('install_data', has_data), - ('install_egg_info', lambda self:True), + # keep install_distinfo last, as it needs the record + # with files to be completely generated + ('install_distinfo', lambda self: not self.no_distinfo), ] diff --git a/src/distutils2/command/install_data.py b/src/distutils2/command/install_data.py index 06b0a33..5b06b5a 100644 --- a/src/distutils2/command/install_data.py +++ b/src/distutils2/command/install_data.py @@ -37,9 +37,7 @@ class install_data(Command): def finalize_options(self): self.set_undefined_options('install', ('install_data', 'install_dir'), - ('root', 'root'), - ('force', 'force'), - ) + 'root', 'force') def run(self): self.mkpath(self.install_dir) diff --git a/src/distutils2/command/install_distinfo.py b/src/distutils2/command/install_distinfo.py new file mode 100644 index 0000000..4898182 --- /dev/null +++ b/src/distutils2/command/install_distinfo.py @@ -0,0 +1,175 @@ +""" +distutils.command.install_distinfo +================================== + +:Author: Josip Djolonga + +This module implements the ``install_distinfo`` command that creates the +``.dist-info`` directory for the distribution, as specified in :pep:`376`. +Usually, you do not have to call this command directly, it gets called +automatically by the ``install`` command. +""" + +# This file was created from the code for the former command install_egg_info + +import os +import csv +import re +from distutils2.command.cmd import Command +from distutils2 import log +from distutils2._backport.shutil import rmtree +try: + import hashlib +except ImportError: + from distutils2._backport import hashlib + + +class install_distinfo(Command): + """Install a .dist-info directory for the package""" + + description = 'Install a .dist-info directory for the package' + + user_options = [ + ('distinfo-dir=', None, + 'directory where the the .dist-info directory will ' + 'be installed'), + ('installer=', None, 'the name of the installer'), + ('requested', None, 'generate a REQUESTED file'), + ('no-requested', None, 'do not generate a REQUESTED file'), + ('no-record', None, 'do not generate a RECORD file'), + ] + + boolean_options = [ + 'requested', + 'no-record', + ] + + negative_opt = {'no-requested': 'requested'} + + def initialize_options(self): + self.distinfo_dir = None + self.installer = None + self.requested = None + self.no_record = None + + def finalize_options(self): + self.set_undefined_options('install', + ('installer', 'installer'), + ('requested', 'requested'), + ('no_record', 'no_record')) + + self.set_undefined_options('install_lib', + ('install_dir', 'distinfo_dir')) + + if self.installer is None: + self.installer = 'distutils' + if self.requested is None: + self.requested = True + if self.no_record is None: + self.no_record = False + + metadata = self.distribution.metadata + + basename = "%s-%s.dist-info" % ( + to_filename(safe_name(metadata['Name'])), + to_filename(safe_version(metadata['Version'])), + ) + + self.distinfo_dir = os.path.join(self.distinfo_dir, basename) + self.outputs = [] + + def run(self): + if not self.dry_run: + target = self.distinfo_dir + + if os.path.isdir(target) and not os.path.islink(target): + rmtree(target) + elif os.path.exists(target): + self.execute(os.unlink, (self.distinfo_dir,), + "Removing " + target) + + self.execute(os.makedirs, (target,), "Creating " + target) + + metadata_path = os.path.join(self.distinfo_dir, 'METADATA') + log.info('Creating %s' % (metadata_path,)) + self.distribution.metadata.write(metadata_path) + self.outputs.append(metadata_path) + + installer_path = os.path.join(self.distinfo_dir, 'INSTALLER') + log.info('Creating %s' % (installer_path,)) + f = open(installer_path, 'w') + try: + f.write(self.installer) + finally: + f.close() + self.outputs.append(installer_path) + + if self.requested: + requested_path = os.path.join(self.distinfo_dir, 'REQUESTED') + log.info('Creating %s' % (requested_path,)) + f = open(requested_path, 'w') + f.close() + self.outputs.append(requested_path) + + if not self.no_record: + record_path = os.path.join(self.distinfo_dir, 'RECORD') + log.info('Creating %s' % (record_path,)) + f = open(record_path, 'wb') + try: + writer = csv.writer(f, delimiter=',', + lineterminator=os.linesep, + quotechar='"') + + install = self.get_finalized_command('install') + + for fpath in install.get_outputs(): + if fpath.endswith('.pyc') or fpath.endswith('.pyo'): + # do not put size and md5 hash, as in PEP-376 + writer.writerow((fpath, '', '')) + else: + size = os.path.getsize(fpath) + fd = open(fpath, 'r') + hash = hashlib.md5() + hash.update(fd.read()) + md5sum = hash.hexdigest() + writer.writerow((fpath, md5sum, size)) + + # add the RECORD file itself + writer.writerow((record_path, '', '')) + self.outputs.append(record_path) + finally: + f.close() + + def get_outputs(self): + return self.outputs + + +# The following routines are taken from setuptools' pkg_resources module and +# can be replaced by importing them from pkg_resources once it is included +# in the stdlib. + + +def safe_name(name): + """Convert an arbitrary string to a standard distribution name + + Any runs of non-alphanumeric/. characters are replaced with a single '-'. + """ + return re.sub('[^A-Za-z0-9.]+', '-', name) + + +def safe_version(version): + """Convert an arbitrary string to a standard version string + + Spaces become dots, and all other non-alphanumeric characters become + dashes, with runs of multiple dashes condensed to a single dash. + """ + version = version.replace(' ', '.') + return re.sub('[^A-Za-z0-9.]+', '-', version) + + +def to_filename(name): + """Convert a project or version name to its filename-escaped form + + Any '-' characters are currently replaced with '_'. + """ + return name.replace('-', '_') diff --git a/src/distutils2/command/install_egg_info.py b/src/distutils2/command/install_egg_info.py deleted file mode 100644 index c8b568a..0000000 --- a/src/distutils2/command/install_egg_info.py +++ /dev/null @@ -1,83 +0,0 @@ -"""distutils.command.install_egg_info - -Implements the Distutils 'install_egg_info' command, for installing -a package's PKG-INFO metadata.""" - - -from distutils2.command.cmd import Command -from distutils2 import log -from distutils2._backport.shutil import rmtree -import os, sys, re - -class install_egg_info(Command): - """Install an .egg-info file for the package""" - - description = "Install package's PKG-INFO metadata as an .egg-info file" - user_options = [ - ('install-dir=', 'd', "directory to install to"), - ] - - def initialize_options(self): - self.install_dir = None - - def finalize_options(self): - metadata = self.distribution.metadata - self.set_undefined_options('install_lib',('install_dir','install_dir')) - basename = "%s-%s-py%s.egg-info" % ( - to_filename(safe_name(metadata['Name'])), - to_filename(safe_version(metadata['Version'])), - sys.version[:3] - ) - self.target = os.path.join(self.install_dir, basename) - self.outputs = [self.target] - - def run(self): - target = self.target - if os.path.isdir(target) and not os.path.islink(target): - if self.dry_run: - pass # XXX - else: - rmtree(target) - elif os.path.exists(target): - self.execute(os.unlink,(self.target,),"Removing "+target) - elif not os.path.isdir(self.install_dir): - self.execute(os.makedirs, (self.install_dir,), - "Creating "+self.install_dir) - log.info("Writing %s", target) - if not self.dry_run: - f = open(target, 'w') - self.distribution.metadata.write_file(f) - f.close() - - def get_outputs(self): - return self.outputs - - -# The following routines are taken from setuptools' pkg_resources module and -# can be replaced by importing them from pkg_resources once it is included -# in the stdlib. - -def safe_name(name): - """Convert an arbitrary string to a standard distribution name - - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub('[^A-Za-z0-9.]+', '-', name) - - -def safe_version(version): - """Convert an arbitrary string to a standard version string - - Spaces become dots, and all other non-alphanumeric characters become - dashes, with runs of multiple dashes condensed to a single dash. - """ - version = version.replace(' ','.') - return re.sub('[^A-Za-z0-9.]+', '-', version) - - -def to_filename(name): - """Convert a project or version name to its filename-escaped form - - Any '-' characters are currently replaced with '_'. - """ - return name.replace('-','_') diff --git a/src/distutils2/command/install_headers.py b/src/distutils2/command/install_headers.py index 762656d..dd63128 100644 --- a/src/distutils2/command/install_headers.py +++ b/src/distutils2/command/install_headers.py @@ -29,8 +29,7 @@ class install_headers(Command): def finalize_options(self): self.set_undefined_options('install', ('install_headers', 'install_dir'), - ('force', 'force')) - + 'force') def run(self): headers = self.distribution.headers diff --git a/src/distutils2/command/install_lib.py b/src/distutils2/command/install_lib.py index 7a9023b..c5e78f7 100644 --- a/src/distutils2/command/install_lib.py +++ b/src/distutils2/command/install_lib.py @@ -68,11 +68,7 @@ class install_lib(Command): self.set_undefined_options('install', ('build_lib', 'build_dir'), ('install_lib', 'install_dir'), - ('force', 'force'), - ('compile', 'compile'), - ('optimize', 'optimize'), - ('skip_build', 'skip_build'), - ) + 'force', 'compile', 'optimize', 'skip_build') if self.compile is None: self.compile = 1 diff --git a/src/distutils2/command/install_scripts.py b/src/distutils2/command/install_scripts.py index bdf7362..fa7587e 100644 --- a/src/distutils2/command/install_scripts.py +++ b/src/distutils2/command/install_scripts.py @@ -36,9 +36,7 @@ class install_scripts (Command): self.set_undefined_options('build', ('build_scripts', 'build_dir')) self.set_undefined_options('install', ('install_scripts', 'install_dir'), - ('force', 'force'), - ('skip_build', 'skip_build'), - ) + 'force', 'skip_build') def run (self): if not self.skip_build: diff --git a/src/distutils2/command/register.py b/src/distutils2/command/register.py index 40ee4ce..3f27272 100644 --- a/src/distutils2/command/register.py +++ b/src/distutils2/command/register.py @@ -21,15 +21,16 @@ from distutils2.util import (metadata_to_dict, read_pypirc, generate_pypirc, class register(Command): - description = ("register the distribution with the Python package index") - user_options = [('repository=', 'r', - "url of repository [default: %s]" % DEFAULT_REPOSITORY), + description = "register the distribution with the Python package index" + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), ('show-response', None, - 'display full response text from server'), + "display full response text from server"), ('list-classifiers', None, - 'list the valid Trove classifiers'), + "list valid Trove classifiers"), ('strict', None , - 'Will stop the registering if the meta-data are not fully compliant') + "stop the registration if the metadata is not fully compliant") ] boolean_options = ['show-response', 'verify', 'list-classifiers', diff --git a/src/distutils2/command/sdist.py b/src/distutils2/command/sdist.py index 37f74c5..8f7d213 100644 --- a/src/distutils2/command/sdist.py +++ b/src/distutils2/command/sdist.py @@ -70,8 +70,8 @@ class sdist(Command): ('dist-dir=', 'd', "directory to put the source distribution archive(s) in " "[default: dist]"), - ('medata-check', None, - "Ensure that all required elements of meta-data " + ('check-metadata', None, + "Ensure that all required elements of metadata " "are supplied. Warn if any missing. [default]"), ('owner=', 'u', "Owner name used when creating a tar file [default: current user]"), @@ -80,7 +80,7 @@ class sdist(Command): ] boolean_options = ['use-defaults', 'prune', - 'manifest-only', 'keep-temp', 'metadata-check'] + 'manifest-only', 'keep-temp', 'check-metadata'] help_options = [ ('help-formats', None, @@ -362,7 +362,7 @@ class sdist(Command): 'self.keep_temp' is true). The list of archive files created is stored so it can be retrieved later by 'get_archive_files()'. """ - # Don't warn about missing meta-data here -- should be (and is!) + # Don't warn about missing metadata here -- should be (and is!) # done elsewhere. base_dir = self.distribution.get_fullname() base_name = os.path.join(self.dist_dir, base_dir) diff --git a/src/distutils2/command/upload.py b/src/distutils2/command/upload.py index 7938e6e..18ebb77 100644 --- a/src/distutils2/command/upload.py +++ b/src/distutils2/command/upload.py @@ -11,7 +11,7 @@ import StringIO as StringIO try: from hashlib import md5 except ImportError: - from md5 import md5 + from distutils2._backport.hashlib import md5 from distutils2.errors import DistutilsOptionError from distutils2.util import spawn @@ -24,16 +24,19 @@ from distutils2.util import (metadata_to_dict, read_pypirc, class upload(Command): - description = "upload binary package to PyPI" + description = "upload distribution to PyPI" - user_options = [('repository=', 'r', - "url of repository [default: %s]" % \ - DEFAULT_REPOSITORY), + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), ('show-response', None, - 'display full response text from server'), + "display full response text from server"), ('sign', 's', - 'sign files to upload using gpg'), - ('identity=', 'i', 'GPG identity used to sign files'), + "sign files to upload using gpg"), + ('identity=', 'i', + "GPG identity used to sign files"), + ('upload-docs', None, + "upload documentation too"), ] boolean_options = ['show-response', 'sign'] @@ -47,6 +50,7 @@ class upload(Command): self.show_response = 0 self.sign = False self.identity = None + self.upload_docs = False def finalize_options(self): if self.repository is None: @@ -74,6 +78,12 @@ class upload(Command): raise DistutilsOptionError("No dist file created in earlier command") for command, pyversion, filename in self.distribution.dist_files: self.upload_file(command, pyversion, filename) + if self.upload_docs: + upload_docs = self.get_finalized_command("upload_docs") + upload_docs.repository = self.repository + upload_docs.username = self.username + upload_docs.password = self.password + upload_docs.run() # XXX to be refactored with register.post_to_server def upload_file(self, command, pyversion, filename): @@ -94,7 +104,7 @@ class upload(Command): spawn(gpg_args, dry_run=self.dry_run) - # Fill in the data - send all the meta-data in case we need to + # Fill in the data - send all the metadata in case we need to # register a new release content = open(filename,'rb').read() diff --git a/src/distutils2/command/upload_docs.py b/src/distutils2/command/upload_docs.py index 35bc351..7838d85 100644 --- a/src/distutils2/command/upload_docs.py +++ b/src/distutils2/command/upload_docs.py @@ -52,16 +52,19 @@ def encode_multipart(fields, files, boundary=None): class upload_docs(Command): user_options = [ - ('repository=', 'r', "url of repository [default: %s]" % DEFAULT_REPOSITORY), - ('show-response', None, 'display full response text from server'), - ('upload-dir=', None, 'directory to upload'), + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), + ('upload-dir=', None, + "directory to upload"), ] def initialize_options(self): self.repository = None self.realm = None self.show_response = 0 - self.upload_dir = "build/docs" + self.upload_dir = None self.username = '' self.password = '' @@ -70,7 +73,7 @@ class upload_docs(Command): self.repository = DEFAULT_REPOSITORY if self.realm is None: self.realm = DEFAULT_REALM - if self.upload_dir == None: + if self.upload_dir is None: build = self.get_finalized_command('build') self.upload_dir = os.path.join(build.build_base, "docs") self.announce('Using upload directory %s' % self.upload_dir) diff --git a/src/distutils2/core.py b/src/distutils2/core.py index b528634..0820adb 100644 --- a/src/distutils2/core.py +++ b/src/distutils2/core.py @@ -157,7 +157,7 @@ def setup(**attrs): def run_setup(script_name, script_args=None, stop_after="run"): """Run a setup script in a somewhat controlled environment, and return the Distribution instance that drives things. This is useful - if you need to find out the distribution meta-data (passed as + if you need to find out the distribution metadata (passed as keyword args from 'script' to 'setup()', or the contents of the config files or command-line. @@ -207,8 +207,6 @@ def run_setup(script_name, script_args=None, stop_after="run"): # Hmm, should we do something if exiting with a non-zero code # (ie. error)? pass - except: - raise if _setup_distribution is None: raise RuntimeError, \ diff --git a/src/distutils2/dist.py b/src/distutils2/dist.py index cc21d7a..9aeb82c 100644 --- a/src/distutils2/dist.py +++ b/src/distutils2/dist.py @@ -9,18 +9,14 @@ __revision__ = "$Id: dist.py 77717 2010-01-24 00:33:32Z tarek.ziade $" import sys import os import re - -try: - import warnings -except ImportError: - warnings = None +import warnings from ConfigParser import RawConfigParser from distutils2.errors import (DistutilsOptionError, DistutilsArgError, DistutilsModuleError, DistutilsClassError) from distutils2.fancy_getopt import FancyGetopt, translate_longopt -from distutils2.util import check_environ, strtobool +from distutils2.util import check_environ, strtobool, resolve_dotted_name from distutils2 import log from distutils2.metadata import DistributionMetadata @@ -145,11 +141,10 @@ Common commands: (see '--help-commands' for more) for attr in self.display_option_names: setattr(self, attr, 0) - # Store the distribution meta-data (name, version, author, and so + # Store the distribution metadata (name, version, author, and so # forth) in a separate object -- we're getting to have enough # information here (and enough command-line options) that it's - # worth it. Also delegate 'get_XXX()' methods to the 'metadata' - # object in a sneaky and underhanded (but efficient!) way. + # worth it. self.metadata = DistributionMetadata() # 'cmdclass' maps command names to class objects, so we @@ -253,10 +248,7 @@ Common commands: (see '--help-commands' for more) setattr(self, key, val) else: msg = "Unknown distribution option: %r" % key - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") + warnings.warn(msg) # no-user-cfg is handled before other command line args # because other args override the config files, and this @@ -381,9 +373,21 @@ Common commands: (see '--help-commands' for more) for opt in options: if opt != '__name__': - val = parser.get(section, opt) + val = parser.get(section,opt) opt = opt.replace('-', '_') - opt_dict[opt] = (filename, val) + + # ... although practicality beats purity :( + if opt.startswith("pre_hook.") or opt.startswith("post_hook."): + hook_type, alias = opt.split(".") + # Hooks are somewhat exceptional, as they are + # gathered from many config files (not overriden as + # other options). + # The option_dict expects {"command": ("filename", # "value")} + # so for hooks, we store only the last config file processed + hook_dict = opt_dict.setdefault(hook_type, (filename, {}))[1] + hook_dict[alias] = val + else: + opt_dict[opt] = (filename, val) # Make the RawConfigParser forget everything (so we retain # the original filenames that options come from) @@ -948,12 +952,22 @@ Common commands: (see '--help-commands' for more) if self.have_run.get(command): return - log.info("running %s", command) cmd_obj = self.get_command_obj(command) cmd_obj.ensure_finalized() + self.run_command_hooks(cmd_obj, 'pre_hook') + log.info("running %s", command) cmd_obj.run() + self.run_command_hooks(cmd_obj, 'post_hook') self.have_run[command] = 1 + def run_command_hooks(self, cmd_obj, hook_kind): + hooks = getattr(cmd_obj, hook_kind) + if hooks is None: + return + for hook in hooks.values(): + hook_func = resolve_dotted_name(hook) + hook_func(cmd_obj) + # -- Distribution query methods ------------------------------------ def has_pure_modules(self): return len(self.packages or self.py_modules or []) > 0 diff --git a/src/distutils2/index/__init__.py b/src/distutils2/index/__init__.py new file mode 100644 index 0000000..312662f --- /dev/null +++ b/src/distutils2/index/__init__.py @@ -0,0 +1,11 @@ +"""Package containing ways to interact with Index APIs. + +""" + +__all__ = ['simple', + 'xmlrpc', + 'dist', + 'errors', + 'mirrors',] + +from dist import ReleaseInfo, ReleasesList, DistInfo diff --git a/src/distutils2/index/base.py b/src/distutils2/index/base.py new file mode 100644 index 0000000..7c87c90 --- /dev/null +++ b/src/distutils2/index/base.py @@ -0,0 +1,55 @@ +from distutils2.version import VersionPredicate +from distutils2.index.dist import ReleasesList + + +class BaseClient(object): + """Base class containing common methods for the index crawlers/clients""" + + def __init__(self, prefer_final, prefer_source): + self._prefer_final = prefer_final + self._prefer_source = prefer_source + self._index = self + + def _get_version_predicate(self, requirements): + """Return a VersionPredicate object, from a string or an already + existing object. + """ + if isinstance(requirements, str): + requirements = VersionPredicate(requirements) + return requirements + + def _get_prefer_final(self, prefer_final=None): + """Return the prefer_final internal parameter or the specified one if + provided""" + if prefer_final: + return prefer_final + else: + return self._prefer_final + + def _get_prefer_source(self, prefer_source=None): + """Return the prefer_source internal parameter or the specified one if + provided""" + if prefer_source: + return prefer_source + else: + return self._prefer_source + + def _get_project(self, project_name): + """Return an project instance, create it if necessary""" + return self._projects.setdefault(project_name.lower(), + ReleasesList(project_name, index=self._index)) + + def download_distribution(self, requirements, temp_path=None, + prefer_source=None, prefer_final=None): + """Download a distribution from the last release according to the + requirements. + + If temp_path is provided, download to this path, otherwise, create a + temporary location for the download and return it. + """ + prefer_final = self._get_prefer_final(prefer_final) + prefer_source = self._get_prefer_source(prefer_source) + release = self.get_release(requirements, prefer_final) + if release: + dist = release.get_distribution(prefer_source=prefer_source) + return dist.download(temp_path) diff --git a/src/distutils2/index/dist.py b/src/distutils2/index/dist.py new file mode 100644 index 0000000..10251ad --- /dev/null +++ b/src/distutils2/index/dist.py @@ -0,0 +1,549 @@ +"""distutils2.index.dist + +Provides useful classes to represent the release and distributions retrieved +from indexes. + +A project can have several releases (=versions) and each release can have +several distributions (sdist, bdist). + +The release contains the metadata related informations (see PEP 384), and the +distributions contains download related informations. + +""" +import mimetypes +import re +import tarfile +import tempfile +import urllib +import urlparse +import zipfile + +try: + import hashlib +except ImportError: + from distutils2._backport import hashlib + +from distutils2.errors import IrrationalVersionError +from distutils2.index.errors import (HashDoesNotMatch, UnsupportedHashName, + CantParseArchiveName) +from distutils2.version import suggest_normalized_version, NormalizedVersion +from distutils2.metadata import DistributionMetadata +from distutils2.util import untar_file, unzip_file, splitext + +__all__ = ['ReleaseInfo', 'DistInfo', 'ReleasesList', 'get_infos_from_url'] + +EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz .egg".split() +MD5_HASH = re.compile(r'^.*#md5=([a-f0-9]+)$') +DIST_TYPES = ['bdist', 'sdist'] + + +class IndexReference(object): + def set_index(self, index=None): + self._index = index + + +class ReleaseInfo(IndexReference): + """Represent a release of a project (a project with a specific version). + The release contain the _metadata informations related to this specific + version, and is also a container for distribution related informations. + + See the DistInfo class for more information about distributions. + """ + + def __init__(self, name, version, metadata=None, hidden=False, + index=None, **kwargs): + """ + :param name: the name of the distribution + :param version: the version of the distribution + :param metadata: the metadata fields of the release. + :type metadata: dict + :param kwargs: optional arguments for a new distribution. + """ + self.set_index(index) + self.name = name + self._version = None + self.version = version + if metadata: + self._metadata = DistributionMetadata(mapping=metadata) + else: + self._metadata = None + self._dists = {} + self.hidden = hidden + + if 'dist_type' in kwargs: + dist_type = kwargs.pop('dist_type') + self.add_distribution(dist_type, **kwargs) + + def set_version(self, version): + try: + self._version = NormalizedVersion(version) + except IrrationalVersionError: + suggestion = suggest_normalized_version(version) + if suggestion: + self.version = suggestion + else: + raise IrrationalVersionError(version) + + def get_version(self): + return self._version + + version = property(get_version, set_version) + + @property + def metadata(self): + """If the metadata is not set, use the indexes to get it""" + if not self._metadata: + self._index.get_metadata(self.name, '%s' % self.version) + return self._metadata + + @property + def is_final(self): + """proxy to version.is_final""" + return self.version.is_final + + @property + def dists(self): + if self._dists is None: + self._index.get_distributions(self.name, '%s' % self.version) + if self._dists is None: + self._dists = {} + return self._dists + + def add_distribution(self, dist_type='sdist', python_version=None, **params): + """Add distribution informations to this release. + If distribution information is already set for this distribution type, + add the given url paths to the distribution. This can be useful while + some of them fails to download. + + :param dist_type: the distribution type (eg. "sdist", "bdist", etc.) + :param params: the fields to be passed to the distribution object + (see the :class:DistInfo constructor). + """ + if dist_type not in DIST_TYPES: + raise ValueError(dist_type) + if dist_type in self.dists: + self._dists[dist_type].add_url(**params) + else: + self._dists[dist_type] = DistInfo(self, dist_type, + index=self._index, **params) + if python_version: + self._dists[dist_type].python_version = python_version + + def get_distribution(self, dist_type=None, prefer_source=True): + """Return a distribution. + + If dist_type is set, find first for this distribution type, and just + act as an alias of __get_item__. + + If prefer_source is True, search first for source distribution, and if + not return one existing distribution. + """ + if len(self.dists) == 0: + raise LookupError() + if dist_type: + return self[dist_type] + if prefer_source: + if "sdist" in self.dists: + dist = self["sdist"] + else: + dist = self.dists.values()[0] + return dist + + def download(self, temp_path=None, prefer_source=True): + """Download the distribution, using the requirements. + + If more than one distribution match the requirements, use the last + version. + Download the distribution, and put it in the temp_path. If no temp_path + is given, creates and return one. + + Returns the complete absolute path to the downloaded archive. + """ + return self.get_distribution(prefer_source=prefer_source)\ + .download(path=temp_path) + + def set_metadata(self, metadata): + if not self._metadata: + self._metadata = DistributionMetadata() + self._metadata.update(metadata) + + def __getitem__(self, item): + """distributions are available using release["sdist"]""" + return self.dists[item] + + def _check_is_comparable(self, other): + if not isinstance(other, ReleaseInfo): + raise TypeError("cannot compare %s and %s" + % (type(self).__name__, type(other).__name__)) + elif self.name != other.name: + raise TypeError("cannot compare %s and %s" + % (self.name, other.name)) + + def __repr__(self): + return "<%s %s>" % (self.name, self.version) + + def __eq__(self, other): + self._check_is_comparable(other) + return self.version == other.version + + def __lt__(self, other): + self._check_is_comparable(other) + return self.version < other.version + + def __ne__(self, other): + return not self.__eq__(other) + + def __gt__(self, other): + return not (self.__lt__(other) or self.__eq__(other)) + + def __le__(self, other): + return self.__eq__(other) or self.__lt__(other) + + def __ge__(self, other): + return self.__eq__(other) or self.__gt__(other) + + # See http://docs.python.org/reference/datamodel#object.__hash__ + __hash__ = object.__hash__ + + +class DistInfo(IndexReference): + """Represents a distribution retrieved from an index (sdist, bdist, ...) + """ + + def __init__(self, release, dist_type=None, url=None, hashname=None, + hashval=None, is_external=True, python_version=None, + index=None): + """Create a new instance of DistInfo. + + :param release: a DistInfo class is relative to a release. + :param dist_type: the type of the dist (eg. source, bin-*, etc.) + :param url: URL where we found this distribution + :param hashname: the name of the hash we want to use. Refer to the + hashlib.new documentation for more information. + :param hashval: the hash value. + :param is_external: we need to know if the provided url comes from + an index browsing, or from an external resource. + + """ + self.set_index(index) + self.release = release + self.dist_type = dist_type + self.python_version = python_version + self._unpacked_dir = None + # set the downloaded path to None by default. The goal here + # is to not download distributions multiple times + self.downloaded_location = None + # We store urls in dict, because we need to have a bit more infos + # than the simple URL. It will be used later to find the good url to + # use. + # We have two _url* attributes: _url and urls. urls contains a list + # of dict for the different urls, and _url contains the choosen url, in + # order to dont make the selection process multiple times. + self.urls = [] + self._url = None + self.add_url(url, hashname, hashval, is_external) + + def add_url(self, url, hashname=None, hashval=None, is_external=True): + """Add a new url to the list of urls""" + if hashname is not None: + try: + hashlib.new(hashname) + except ValueError: + raise UnsupportedHashName(hashname) + if not url in [u['url'] for u in self.urls]: + self.urls.append({ + 'url': url, + 'hashname': hashname, + 'hashval': hashval, + 'is_external': is_external, + }) + # reset the url selection process + self._url = None + + @property + def url(self): + """Pick up the right url for the list of urls in self.urls""" + # We return internal urls over externals. + # If there is more than one internal or external, return the first + # one. + if self._url is None: + if len(self.urls) > 1: + internals_urls = [u for u in self.urls \ + if u['is_external'] == False] + if len(internals_urls) >= 1: + self._url = internals_urls[0] + if self._url is None: + self._url = self.urls[0] + return self._url + + @property + def is_source(self): + """return if the distribution is a source one or not""" + return self.dist_type == 'sdist' + + def download(self, path=None): + """Download the distribution to a path, and return it. + + If the path is given in path, use this, otherwise, generates a new one + Return the download location. + """ + if path is None: + path = tempfile.mkdtemp() + + # if we do not have downloaded it yet, do it. + if self.downloaded_location is None: + url = self.url['url'] + archive_name = urlparse.urlparse(url)[2].split('/')[-1] + filename, headers = urllib.urlretrieve(url, + path + "/" + archive_name) + self.downloaded_location = filename + self._check_md5(filename) + return self.downloaded_location + + def unpack(self, path=None): + """Unpack the distribution to the given path. + + If not destination is given, creates a temporary location. + + Returns the location of the extracted files (root). + """ + if not self._unpacked_dir: + if path is None: + path = tempfile.mkdtemp() + + filename = self.download() + content_type = mimetypes.guess_type(filename)[0] + + if (content_type == 'application/zip' + or filename.endswith('.zip') + or filename.endswith('.pybundle') + or zipfile.is_zipfile(filename)): + unzip_file(filename, path, flatten=not filename.endswith('.pybundle')) + elif (content_type == 'application/x-gzip' + or tarfile.is_tarfile(filename) + or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')): + untar_file(filename, path) + self._unpacked_dir = path + return self._unpacked_dir + + def _check_md5(self, filename): + """Check that the md5 checksum of the given file matches the one in + url param""" + hashname = self.url['hashname'] + expected_hashval = self.url['hashval'] + if not None in (expected_hashval, hashname): + f = open(filename) + hashval = hashlib.new(hashname) + hashval.update(f.read()) + if hashval.hexdigest() != expected_hashval: + raise HashDoesNotMatch("got %s instead of %s" + % (hashval.hexdigest(), expected_hashval)) + + def __repr__(self): + return "<%s %s %s>" % ( + self.release.name, self.release.version, self.dist_type or "") + + +class ReleasesList(IndexReference): + """A container of Release. + + Provides useful methods and facilities to sort and filter releases. + """ + def __init__(self, name, releases=None, contains_hidden=False, index=None): + self.set_index(index) + self._releases = [] + self.name = name + self.contains_hidden = contains_hidden + if releases: + self.add_releases(releases) + + @property + def releases(self): + if not self._releases: + self.fetch_releases() + return self._releases + + def fetch_releases(self): + self._index.get_releases(self.name) + return self.releases + + def filter(self, predicate): + """Filter and return a subset of releases matching the given predicate. + """ + return ReleasesList(self.name, [release for release in self.releases + if predicate.match(release.version)], + index=self._index) + + def get_last(self, predicate, prefer_final=None): + """Return the "last" release, that satisfy the given predicates. + + "last" is defined by the version number of the releases, you also could + set prefer_final parameter to True or False to change the order results + """ + releases = self.filter(predicate) + releases.sort_releases(prefer_final, reverse=True) + return releases[0] + + def add_releases(self, releases): + """Add releases in the release list. + + :param: releases is a list of ReleaseInfo objects. + """ + for r in releases: + self.add_release(release=r) + + def add_release(self, version=None, dist_type='sdist', release=None, + **dist_args): + """Add a release to the list. + + The release can be passed in the `release` parameter, and in this case, + it will be crawled to extract the useful informations if necessary, or + the release informations can be directly passed in the `version` and + `dist_type` arguments. + + Other keywords arguments can be provided, and will be forwarded to the + distribution creation (eg. the arguments of the DistInfo constructor). + """ + if release: + if release.name.lower() != self.name.lower(): + raise ValueError("%s is not the same project than %s" % + (release.name, self.name)) + version = '%s' % release.version + + if not version in self.get_versions(): + # append only if not already exists + self._releases.append(release) + for dist in release.dists.values(): + for url in dist.urls: + self.add_release(version, dist.dist_type, **url) + else: + matches = [r for r in self._releases if '%s' % r.version == version + and r.name == self.name] + if not matches: + release = ReleaseInfo(self.name, version, index=self._index) + self._releases.append(release) + else: + release = matches[0] + + release.add_distribution(dist_type=dist_type, **dist_args) + + def sort_releases(self, prefer_final=False, reverse=True, *args, **kwargs): + """Sort the results with the given properties. + + The `prefer_final` argument can be used to specify if final + distributions (eg. not dev, bet or alpha) would be prefered or not. + + Results can be inverted by using `reverse`. + + Any other parameter provided will be forwarded to the sorted call. You + cannot redefine the key argument of "sorted" here, as it is used + internally to sort the releases. + """ + + sort_by = [] + if prefer_final: + sort_by.append("is_final") + sort_by.append("version") + + self.releases.sort( + key=lambda i: [getattr(i, arg) for arg in sort_by], + reverse=reverse, *args, **kwargs) + + def get_release(self, version): + """Return a release from it's version. + """ + matches = [r for r in self.releases if "%s" % r.version == version] + if len(matches) != 1: + raise KeyError(version) + return matches[0] + + def get_versions(self): + """Return a list of releases versions contained""" + return ["%s" % r.version for r in self._releases] + + def __getitem__(self, key): + return self.releases[key] + + def __len__(self): + return len(self.releases) + + def __repr__(self): + string = 'Project "%s"' % self.name + if self.get_versions(): + string += ' versions: %s' % ', '.join(self.get_versions()) + return '<%s>' % string + + +def get_infos_from_url(url, probable_dist_name=None, is_external=True): + """Get useful informations from an URL. + + Return a dict of (name, version, url, hashtype, hash, is_external) + + :param url: complete url of the distribution + :param probable_dist_name: A probable name of the project. + :param is_external: Tell if the url commes from an index or from + an external URL. + """ + # if the url contains a md5 hash, get it. + md5_hash = None + match = MD5_HASH.match(url) + if match is not None: + md5_hash = match.group(1) + # remove the hash + url = url.replace("#md5=%s" % md5_hash, "") + + # parse the archive name to find dist name and version + archive_name = urlparse.urlparse(url)[2].split('/')[-1] + extension_matched = False + # remove the extension from the name + for ext in EXTENSIONS: + if archive_name.endswith(ext): + archive_name = archive_name[:-len(ext)] + extension_matched = True + + name, version = split_archive_name(archive_name) + if extension_matched is True: + return {'name': name, + 'version': version, + 'url': url, + 'hashname': "md5", + 'hashval': md5_hash, + 'is_external': is_external, + 'dist_type': 'sdist'} + + +def split_archive_name(archive_name, probable_name=None): + """Split an archive name into two parts: name and version. + + Return the tuple (name, version) + """ + # Try to determine wich part is the name and wich is the version using the + # "-" separator. Take the larger part to be the version number then reduce + # if this not works. + def eager_split(str, maxsplit=2): + # split using the "-" separator + splits = str.rsplit("-", maxsplit) + name = splits[0] + version = "-".join(splits[1:]) + if version.startswith("-"): + version = version[1:] + if suggest_normalized_version(version) is None and maxsplit >= 0: + # we dont get a good version number: recurse ! + return eager_split(str, maxsplit - 1) + else: + return (name, version) + if probable_name is not None: + probable_name = probable_name.lower() + name = None + if probable_name is not None and probable_name in archive_name: + # we get the name from probable_name, if given. + name = probable_name + version = archive_name.lstrip(name) + else: + name, version = eager_split(archive_name) + + version = suggest_normalized_version(version) + if version is not None and name != "": + return (name.lower(), version) + else: + raise CantParseArchiveName(archive_name) diff --git a/src/distutils2/index/errors.py b/src/distutils2/index/errors.py new file mode 100644 index 0000000..730ccce --- /dev/null +++ b/src/distutils2/index/errors.py @@ -0,0 +1,45 @@ +"""distutils2.pypi.errors + +All errors and exceptions raised by PyPiIndex classes. +""" +from distutils2.errors import DistutilsError + + +class IndexesError(DistutilsError): + """The base class for errors of the index python package.""" + + +class ProjectNotFound(IndexesError): + """Project has not been found""" + + +class DistributionNotFound(IndexesError): + """The release has not been found""" + + +class ReleaseNotFound(IndexesError): + """The release has not been found""" + + +class CantParseArchiveName(IndexesError): + """An archive name can't be parsed to find distribution name and version""" + + +class DownloadError(IndexesError): + """An error has occurs while downloading""" + + +class HashDoesNotMatch(DownloadError): + """Compared hashes does not match""" + + +class UnsupportedHashName(IndexesError): + """A unsupported hashname has been used""" + + +class UnableToDownload(IndexesError): + """All mirrors have been tried, without success""" + + +class InvalidSearchField(IndexesError): + """An invalid search field has been used""" diff --git a/src/distutils2/index/mirrors.py b/src/distutils2/index/mirrors.py new file mode 100644 index 0000000..49d5dd1 --- /dev/null +++ b/src/distutils2/index/mirrors.py @@ -0,0 +1,52 @@ +"""Utilities related to the mirror infrastructure defined in PEP 381. +See http://www.python.org/dev/peps/pep-0381/ +""" + +from string import ascii_lowercase +import socket + +DEFAULT_MIRROR_URL = "last.pypi.python.org" + +def get_mirrors(hostname=None): + """Return the list of mirrors from the last record found on the DNS + entry:: + + >>> from distutils2.index.mirrors import get_mirrors + >>> get_mirrors() + ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org', + 'd.pypi.python.org'] + + """ + if hostname is None: + hostname = DEFAULT_MIRROR_URL + + # return the last mirror registered on PyPI. + try: + hostname = socket.gethostbyname_ex(hostname)[0] + except socket.gaierror: + return [] + end_letter = hostname.split(".", 1) + + # determine the list from the last one. + return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])] + +def string_range(last): + """Compute the range of string between "a" and last. + + This works for simple "a to z" lists, but also for "a to zz" lists. + """ + for k in range(len(last)): + for x in product(ascii_lowercase, repeat=k+1): + result = ''.join(x) + yield result + if result == last: + return + +def product(*args, **kwds): + pools = map(tuple, args) * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x+[y] for x in result for y in pool] + for prod in result: + yield tuple(prod) + diff --git a/src/distutils2/index/simple.py b/src/distutils2/index/simple.py new file mode 100644 index 0000000..4a21e04 --- /dev/null +++ b/src/distutils2/index/simple.py @@ -0,0 +1,433 @@ +"""index.simple + +Contains the class "SimpleIndexCrawler", a simple spider to find and retrieve +distributions on the Python Package Index, using it's "simple" API, +avalaible at http://pypi.python.org/simple/ +""" +from fnmatch import translate +import httplib +import re +import socket +import sys +import urllib2 +import urlparse +import logging +import os + +from distutils2.index.base import BaseClient +from distutils2.index.dist import (ReleasesList, EXTENSIONS, + get_infos_from_url, MD5_HASH) +from distutils2.index.errors import (IndexesError, DownloadError, + UnableToDownload, CantParseArchiveName, + ReleaseNotFound, ProjectNotFound) +from distutils2.index.mirrors import get_mirrors +from distutils2.metadata import DistributionMetadata +from distutils2 import __version__ as __distutils2_version__ + +__all__ = ['Crawler', 'DEFAULT_SIMPLE_INDEX_URL'] + +# -- Constants ----------------------------------------------- +DEFAULT_SIMPLE_INDEX_URL = "http://a.pypi.python.org/simple/" +DEFAULT_HOSTS = ("*",) +SOCKET_TIMEOUT = 15 +USER_AGENT = "Python-urllib/%s distutils2/%s" % ( + sys.version[:3], __distutils2_version__) + +# -- Regexps ------------------------------------------------- +EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') +HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) +URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match + +# This pattern matches a character entity reference (a decimal numeric +# references, a hexadecimal numeric reference, or a named reference). +ENTITY_SUB = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub +REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) + + +def socket_timeout(timeout=SOCKET_TIMEOUT): + """Decorator to add a socket timeout when requesting pages on PyPI. + """ + def _socket_timeout(func): + def _socket_timeout(self, *args, **kwargs): + old_timeout = socket.getdefaulttimeout() + if hasattr(self, "_timeout"): + timeout = self._timeout + socket.setdefaulttimeout(timeout) + try: + return func(self, *args, **kwargs) + finally: + socket.setdefaulttimeout(old_timeout) + return _socket_timeout + return _socket_timeout + + +def with_mirror_support(): + """Decorator that makes the mirroring support easier""" + def wrapper(func): + def wrapped(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except DownloadError: + # if an error occurs, try with the next index_url + if self._mirrors_tries >= self._mirrors_max_tries: + try: + self._switch_to_next_mirror() + except KeyError: + raise UnableToDownload("Tried all mirrors") + else: + self._mirrors_tries += 1 + self._projects.clear() + return wrapped(self, *args, **kwargs) + return wrapped + return wrapper + + +class Crawler(BaseClient): + """Provides useful tools to request the Python Package Index simple API. + + You can specify both mirrors and mirrors_url, but mirrors_url will only be + used if mirrors is set to None. + + :param index_url: the url of the simple index to search on. + :param prefer_final: if the version is not mentioned, and the last + version is not a "final" one (alpha, beta, etc.), + pick up the last final version. + :param prefer_source: if the distribution type is not mentioned, pick up + the source one if available. + :param follow_externals: tell if following external links is needed or + not. Default is False. + :param hosts: a list of hosts allowed to be processed while using + follow_externals=True. Default behavior is to follow all + hosts. + :param follow_externals: tell if following external links is needed or + not. Default is False. + :param mirrors_url: the url to look on for DNS records giving mirror + adresses. + :param mirrors: a list of mirrors (see PEP 381). + :param timeout: time in seconds to consider a url has timeouted. + :param mirrors_max_tries": number of times to try requesting informations + on mirrors before switching. + """ + + def __init__(self, index_url=DEFAULT_SIMPLE_INDEX_URL, prefer_final=False, + prefer_source=True, hosts=DEFAULT_HOSTS, + follow_externals=False, mirrors_url=None, mirrors=None, + timeout=SOCKET_TIMEOUT, mirrors_max_tries=0): + super(Crawler, self).__init__(prefer_final, prefer_source) + self.follow_externals = follow_externals + + # mirroring attributes. + if not index_url.endswith("/"): + index_url += "/" + # if no mirrors are defined, use the method described in PEP 381. + if mirrors is None: + mirrors = get_mirrors(mirrors_url) + self._mirrors = set(mirrors) + self._mirrors_used = set() + self.index_url = index_url + self._mirrors_max_tries = mirrors_max_tries + self._mirrors_tries = 0 + self._timeout = timeout + + # create a regexp to match all given hosts + self._allowed_hosts = re.compile('|'.join(map(translate, hosts))).match + + # we keep an index of pages we have processed, in order to avoid + # scanning them multple time (eg. if there is multiple pages pointing + # on one) + self._processed_urls = [] + self._projects = {} + + @with_mirror_support() + def search_projects(self, name=None, **kwargs): + """Search the index for projects containing the given name. + + Return a list of names. + """ + index = self._open_url(self.index_url) + projectname = re.compile("""<a[^>]*>(.?[^<]*%s.?[^<]*)</a>""" % name, + flags=re.I) + matching_projects = [] + for match in projectname.finditer(index.read()): + project_name = match.group(1) + matching_projects.append(self._get_project(project_name)) + return matching_projects + + def get_releases(self, requirements, prefer_final=None, + force_update=False): + """Search for releases and return a ReleaseList object containing + the results. + """ + predicate = self._get_version_predicate(requirements) + if self._projects.has_key(predicate.name.lower()) and not force_update: + return self._projects.get(predicate.name.lower()) + prefer_final = self._get_prefer_final(prefer_final) + self._process_index_page(predicate.name) + + if not self._projects.has_key(predicate.name.lower()): + raise ProjectNotFound() + + releases = self._projects.get(predicate.name.lower()) + releases.sort_releases(prefer_final=prefer_final) + return releases + + def get_release(self, requirements, prefer_final=None): + """Return only one release that fulfill the given requirements""" + predicate = self._get_version_predicate(requirements) + release = self.get_releases(predicate, prefer_final)\ + .get_last(predicate) + if not release: + raise ReleaseNotFound("No release matches the given criterias") + return release + + def get_distributions(self, project_name, version): + """Return the distributions found on the index for the specific given + release""" + # as the default behavior of get_release is to return a release + # containing the distributions, just alias it. + return self.get_release("%s (%s)" % (project_name, version)) + + def get_metadata(self, project_name, version): + """Return the metadatas from the simple index. + + Currently, download one archive, extract it and use the PKG-INFO file. + """ + release = self.get_distributions(project_name, version) + if not release._metadata: + location = release.get_distribution().unpack() + pkg_info = os.path.join(location, 'PKG-INFO') + release._metadata = DistributionMetadata(pkg_info) + return release + + def _switch_to_next_mirror(self): + """Switch to the next mirror (eg. point self.index_url to the next + mirror url. + + Raise a KeyError if all mirrors have been tried. + """ + self._mirrors_used.add(self.index_url) + index_url = self._mirrors.pop() + if not ("http://" or "https://" or "file://") in index_url: + index_url = "http://%s" % index_url + + if not index_url.endswith("/simple"): + index_url = "%s/simple/" % index_url + + self.index_url = index_url + + def _is_browsable(self, url): + """Tell if the given URL can be browsed or not. + + It uses the follow_externals and the hosts list to tell if the given + url is browsable or not. + """ + # if _index_url is contained in the given URL, we are browsing the + # index, and it's always "browsable". + # local files are always considered browable resources + if self.index_url in url or urlparse.urlparse(url)[0] == "file": + return True + elif self.follow_externals: + if self._allowed_hosts(urlparse.urlparse(url)[1]): # 1 is netloc + return True + else: + return False + return False + + def _is_distribution(self, link): + """Tell if the given URL matches to a distribution name or not. + """ + #XXX find a better way to check that links are distributions + # Using a regexp ? + for ext in EXTENSIONS: + if ext in link: + return True + return False + + def _register_release(self, release=None, release_info={}): + """Register a new release. + + Both a release or a dict of release_info can be provided, the prefered + way (eg. the quicker) is the dict one. + + Return the list of existing releases for the given project. + """ + # Check if the project already has a list of releases (refering to + # the project name). If not, create a new release list. + # Then, add the release to the list. + if release: + name = release.name + else: + name = release_info['name'] + if not name.lower() in self._projects: + self._projects[name.lower()] = ReleasesList(name, + index=self._index) + + if release: + self._projects[name.lower()].add_release(release=release) + else: + name = release_info.pop('name') + version = release_info.pop('version') + dist_type = release_info.pop('dist_type') + self._projects[name.lower()].add_release(version, dist_type, + **release_info) + return self._projects[name.lower()] + + def _process_url(self, url, project_name=None, follow_links=True): + """Process an url and search for distributions packages. + + For each URL found, if it's a download, creates a PyPIdistribution + object. If it's a homepage and we can follow links, process it too. + + :param url: the url to process + :param project_name: the project name we are searching for. + :param follow_links: Do not want to follow links more than from one + level. This parameter tells if we want to follow + the links we find (eg. run recursively this + method on it) + """ + f = self._open_url(url) + base_url = f.url + if url not in self._processed_urls: + self._processed_urls.append(url) + link_matcher = self._get_link_matcher(url) + for link, is_download in link_matcher(f.read(), base_url): + if link not in self._processed_urls: + if self._is_distribution(link) or is_download: + self._processed_urls.append(link) + # it's a distribution, so create a dist object + try: + infos = get_infos_from_url(link, project_name, + is_external=not self.index_url in url) + except CantParseArchiveName, e: + logging.warning("version has not been parsed: %s" + % e) + else: + self._register_release(release_info=infos) + else: + if self._is_browsable(link) and follow_links: + self._process_url(link, project_name, + follow_links=False) + + def _get_link_matcher(self, url): + """Returns the right link matcher function of the given url + """ + if self.index_url in url: + return self._simple_link_matcher + else: + return self._default_link_matcher + + def _get_full_url(self, url, base_url): + return urlparse.urljoin(base_url, self._htmldecode(url)) + + def _simple_link_matcher(self, content, base_url): + """Yield all links with a rel="download" or rel="homepage". + + This matches the simple index requirements for matching links. + If follow_externals is set to False, dont yeld the external + urls. + """ + for match in HREF.finditer(content): + url = self._get_full_url(match.group(1), base_url) + if MD5_HASH.match(url): + yield (url, True) + + for match in REL.finditer(content): + # search for rel links. + tag, rel = match.groups() + rels = map(str.strip, rel.lower().split(',')) + if 'homepage' in rels or 'download' in rels: + for match in HREF.finditer(tag): + url = self._get_full_url(match.group(1), base_url) + if 'download' in rels or self._is_browsable(url): + # yield a list of (url, is_download) + yield (url, 'download' in rels) + + def _default_link_matcher(self, content, base_url): + """Yield all links found on the page. + """ + for match in HREF.finditer(content): + url = self._get_full_url(match.group(1), base_url) + if self._is_browsable(url): + yield (url, False) + + @with_mirror_support() + def _process_index_page(self, name): + """Find and process a PyPI page for the given project name. + + :param name: the name of the project to find the page + """ + # Browse and index the content of the given PyPI page. + url = self.index_url + name + "/" + self._process_url(url, name) + + @socket_timeout() + def _open_url(self, url): + """Open a urllib2 request, handling HTTP authentication, and local + files support. + + """ + scheme, netloc, path, params, query, frag = urlparse.urlparse(url) + + # authentication stuff + if scheme in ('http', 'https'): + auth, host = urllib2.splituser(netloc) + else: + auth = None + + # add index.html automatically for filesystem paths + if scheme == 'file': + if url.endswith('/'): + url += "index.html" + + # add authorization headers if auth is provided + if auth: + auth = "Basic " + \ + urllib2.unquote(auth).encode('base64').strip() + new_url = urlparse.urlunparse(( + scheme, host, path, params, query, frag)) + request = urllib2.Request(new_url) + request.add_header("Authorization", auth) + else: + request = urllib2.Request(url) + request.add_header('User-Agent', USER_AGENT) + try: + fp = urllib2.urlopen(request) + except (ValueError, httplib.InvalidURL), v: + msg = ' '.join([str(arg) for arg in v.args]) + raise IndexesError('%s %s' % (url, msg)) + except urllib2.HTTPError, v: + return v + except urllib2.URLError, v: + raise DownloadError("Download error for %s: %s" % (url, v.reason)) + except httplib.BadStatusLine, v: + raise DownloadError('%s returned a bad status line. ' + 'The server might be down, %s' % (url, v.line)) + except httplib.HTTPException, v: + raise DownloadError("Download error for %s: %s" % (url, v)) + except socket.timeout: + raise DownloadError("The server timeouted") + + if auth: + # Put authentication info back into request URL if same host, + # so that links found on the page will work + s2, h2, path2, param2, query2, frag2 = \ + urlparse.urlparse(fp.url) + if s2 == scheme and h2 == host: + fp.url = urlparse.urlunparse( + (s2, netloc, path2, param2, query2, frag2)) + return fp + + def _decode_entity(self, match): + what = match.group(1) + if what.startswith('#x'): + what = int(what[2:], 16) + elif what.startswith('#'): + what = int(what[1:]) + else: + from htmlentitydefs import name2codepoint + what = name2codepoint.get(what, match.group(0)) + return unichr(what) + + def _htmldecode(self, text): + """Decode HTML entities in the given text.""" + return ENTITY_SUB(self._decode_entity, text) diff --git a/src/distutils2/index/wrapper.py b/src/distutils2/index/wrapper.py new file mode 100644 index 0000000..e3c4551 --- /dev/null +++ b/src/distutils2/index/wrapper.py @@ -0,0 +1,93 @@ +import xmlrpc +import simple + +_WRAPPER_MAPPINGS = {'get_release': 'simple', + 'get_releases': 'simple', + 'search_projects': 'simple', + 'get_metadata': 'xmlrpc', + 'get_distributions': 'simple'} + +_WRAPPER_INDEXES = {'xmlrpc': xmlrpc.Client, + 'simple': simple.Crawler} + +def switch_index_if_fails(func, wrapper): + """Decorator that switch of index (for instance from xmlrpc to simple) + if the first mirror return an empty list or raises an exception. + """ + def decorator(*args, **kwargs): + retry = True + exception = None + methods = [func] + for f in wrapper._indexes.values(): + if f != func.im_self and hasattr(f, func.__name__): + methods.append(getattr(f, func.__name__)) + for method in methods: + try: + response = method(*args, **kwargs) + retry = False + except Exception, e: + exception = e + if not retry: + break + if retry and exception: + raise exception + else: + return response + return decorator + + +class ClientWrapper(object): + """Wrapper around simple and xmlrpc clients, + + Choose the best implementation to use depending the needs, using the given + mappings. + If one of the indexes returns an error, tries to use others indexes. + + :param index: tell wich index to rely on by default. + :param index_classes: a dict of name:class to use as indexes. + :param indexes: a dict of name:index already instantiated + :param mappings: the mappings to use for this wrapper + """ + + def __init__(self, default_index='simple', index_classes=_WRAPPER_INDEXES, + indexes={}, mappings=_WRAPPER_MAPPINGS): + self._projects = {} + self._mappings = mappings + self._indexes = indexes + self._default_index = default_index + + # instantiate the classes and set their _project attribute to the one + # of the wrapper. + for name, cls in index_classes.items(): + obj = self._indexes.setdefault(name, cls()) + obj._projects = self._projects + obj._index = self + + def __getattr__(self, method_name): + """When asking for methods of the wrapper, return the implementation of + the wrapped classes, depending the mapping. + + Decorate the methods to switch of implementation if an error occurs + """ + real_method = None + if method_name in _WRAPPER_MAPPINGS: + obj = self._indexes[_WRAPPER_MAPPINGS[method_name]] + real_method = getattr(obj, method_name) + else: + # the method is not defined in the mappings, so we try first to get + # it via the default index, and rely on others if needed. + try: + real_method = getattr(self._indexes[self._default_index], + method_name) + except AttributeError: + other_indexes = [i for i in self._indexes + if i != self._default_index] + for index in other_indexes: + real_method = getattr(self._indexes[index], method_name, None) + if real_method: + break + if real_method: + return switch_index_if_fails(real_method, self) + else: + raise AttributeError("No index have attribute '%s'" % method_name) + diff --git a/src/distutils2/index/xmlrpc.py b/src/distutils2/index/xmlrpc.py new file mode 100644 index 0000000..f1db0aa --- /dev/null +++ b/src/distutils2/index/xmlrpc.py @@ -0,0 +1,175 @@ +import logging +import xmlrpclib + +from distutils2.errors import IrrationalVersionError +from distutils2.index.base import BaseClient +from distutils2.index.errors import ProjectNotFound, InvalidSearchField +from distutils2.index.dist import ReleaseInfo + +__all__ = ['Client', 'DEFAULT_XMLRPC_INDEX_URL'] + +DEFAULT_XMLRPC_INDEX_URL = 'http://python.org/pypi' + +_SEARCH_FIELDS = ['name', 'version', 'author', 'author_email', 'maintainer', + 'maintainer_email', 'home_page', 'license', 'summary', + 'description', 'keywords', 'platform', 'download_url'] + + +class Client(BaseClient): + """Client to query indexes using XML-RPC method calls. + + If no server_url is specified, use the default PyPI XML-RPC URL, + defined in the DEFAULT_XMLRPC_INDEX_URL constant:: + + >>> client = XMLRPCClient() + >>> client.server_url == DEFAULT_XMLRPC_INDEX_URL + True + + >>> client = XMLRPCClient("http://someurl/") + >>> client.server_url + 'http://someurl/' + """ + + def __init__(self, server_url=DEFAULT_XMLRPC_INDEX_URL, prefer_final=False, + prefer_source=True): + super(Client, self).__init__(prefer_final, prefer_source) + self.server_url = server_url + self._projects = {} + + def get_release(self, requirements, prefer_final=False): + """Return a release with all complete metadata and distribution + related informations. + """ + prefer_final = self._get_prefer_final(prefer_final) + predicate = self._get_version_predicate(requirements) + releases = self.get_releases(predicate.name) + release = releases.get_last(predicate, prefer_final) + self.get_metadata(release.name, "%s" % release.version) + self.get_distributions(release.name, "%s" % release.version) + return release + + def get_releases(self, requirements, prefer_final=None, show_hidden=True, + force_update=False): + """Return the list of existing releases for a specific project. + + Cache the results from one call to another. + + If show_hidden is True, return the hidden releases too. + If force_update is True, reprocess the index to update the + informations (eg. make a new XML-RPC call). + :: + + >>> client = XMLRPCClient() + >>> client.get_releases('Foo') + ['1.1', '1.2', '1.3'] + + If no such project exists, raise a ProjectNotFound exception:: + + >>> client.get_project_versions('UnexistingProject') + ProjectNotFound: UnexistingProject + + """ + def get_versions(project_name, show_hidden): + return self.proxy.package_releases(project_name, show_hidden) + + predicate = self._get_version_predicate(requirements) + prefer_final = self._get_prefer_final(prefer_final) + project_name = predicate.name + if not force_update and (project_name.lower() in self._projects): + project = self._projects[project_name.lower()] + if not project.contains_hidden and show_hidden: + # if hidden releases are requested, and have an existing + # list of releases that does not contains hidden ones + all_versions = get_versions(project_name, show_hidden) + existing_versions = project.get_versions() + hidden_versions = list(set(all_versions) - + set(existing_versions)) + for version in hidden_versions: + project.add_release(release=ReleaseInfo(project_name, + version, index=self._index)) + else: + versions = get_versions(project_name, show_hidden) + if not versions: + raise ProjectNotFound(project_name) + project = self._get_project(project_name) + project.add_releases([ReleaseInfo(project_name, version, + index=self._index) + for version in versions]) + project = project.filter(predicate) + project.sort_releases(prefer_final) + return project + + + def get_distributions(self, project_name, version): + """Grab informations about distributions from XML-RPC. + + Return a ReleaseInfo object, with distribution-related informations + filled in. + """ + url_infos = self.proxy.release_urls(project_name, version) + project = self._get_project(project_name) + if version not in project.get_versions(): + project.add_release(release=ReleaseInfo(project_name, version, + index=self._index)) + release = project.get_release(version) + for info in url_infos: + packagetype = info['packagetype'] + dist_infos = {'url': info['url'], + 'hashval': info['md5_digest'], + 'hashname': 'md5', + 'is_external': False, + 'python_version': info['python_version']} + release.add_distribution(packagetype, **dist_infos) + return release + + def get_metadata(self, project_name, version): + """Retreive project metadatas. + + Return a ReleaseInfo object, with metadata informations filled in. + """ + metadata = self.proxy.release_data(project_name, version) + project = self._get_project(project_name) + if version not in project.get_versions(): + project.add_release(release=ReleaseInfo(project_name, version, + index=self._index)) + release = project.get_release(version) + release.set_metadata(metadata) + return release + + def search_projects(self, name=None, operator="or", **kwargs): + """Find using the keys provided in kwargs. + + You can set operator to "and" or "or". + """ + for key in kwargs: + if key not in _SEARCH_FIELDS: + raise InvalidSearchField(key) + if name: + kwargs["name"] = name + projects = self.proxy.search(kwargs, operator) + for p in projects: + project = self._get_project(p['name']) + try: + project.add_release(release=ReleaseInfo(p['name'], + p['version'], metadata={'summary': p['summary']}, + index=self._index)) + except IrrationalVersionError, e: + logging.warn("Irrational version error found: %s" % e) + + return [self._projects[p['name'].lower()] for p in projects] + + @property + def proxy(self): + """Property used to return the XMLRPC server proxy. + + If no server proxy is defined yet, creates a new one:: + + >>> client = XmlRpcClient() + >>> client.proxy() + <ServerProxy for python.org/pypi> + + """ + if not hasattr(self, '_server_proxy'): + self._server_proxy = xmlrpclib.ServerProxy(self.server_url) + + return self._server_proxy diff --git a/src/distutils2/metadata.py b/src/distutils2/metadata.py index a5a73f1..4d1991b 100644 --- a/src/distutils2/metadata.py +++ b/src/distutils2/metadata.py @@ -1,7 +1,6 @@ -""" -Implementation of the Metadata for Python packages +"""Implementation of the Metadata for Python packages PEPs. -Supports all Metadata formats (1.0, 1.1, 1.2). +Supports all metadata formats (1.0, 1.1, 1.2). """ import re @@ -78,12 +77,11 @@ _345_MARKERS = ('Provides-Dist', 'Requires-Dist', 'Requires-Python', 'Obsoletes-Dist', 'Requires-External', 'Maintainer', 'Maintainer-email', 'Project-URL') -_ALL_FIELDS = [] +_ALL_FIELDS = set() +_ALL_FIELDS.update(_241_FIELDS) +_ALL_FIELDS.update(_314_FIELDS) +_ALL_FIELDS.update(_345_FIELDS) -for field in _241_FIELDS + _314_FIELDS + _345_FIELDS: - if field in _ALL_FIELDS: - continue - _ALL_FIELDS.append(field) def _version2fieldlist(version): if version == '1.0': @@ -95,7 +93,7 @@ def _version2fieldlist(version): raise MetadataUnrecognizedVersionError(version) def _best_version(fields): - """Will detect the best version depending on the fields used.""" + """Detect the best version depending on the fields used.""" def _has_marker(keys, markers): for marker in markers: if marker in keys: @@ -182,19 +180,33 @@ _UNICODEFIELDS = ('Author', 'Maintainer', 'Summary', 'Description') class DistributionMetadata(object): - """Distribution meta-data class (1.0 or 1.2). + """The metadata of a release. + + Supports versions 1.0, 1.1 and 1.2 (auto-detected). You can + instantiate the class with one of these arguments (or none): + - *path*, the path to a METADATA file + - *fileobj* give a file-like object with METADATA as content + - *mapping* is a dict-like object """ + # TODO document that execution_context and platform_dependent are used + # to filter on query, not when setting a key + # also document the mapping API and UNKNOWN default key + def __init__(self, path=None, platform_dependent=False, - execution_context=None, fileobj=None): + execution_context=None, fileobj=None, mapping=None): self._fields = {} self.version = None self.docutils_support = _HAS_DOCUTILS self.platform_dependent = platform_dependent + self.execution_context = execution_context + if [path, fileobj, mapping].count(None) < 2: + raise TypeError('path, fileobj and mapping are exclusive') if path is not None: self.read(path) elif fileobj is not None: self.read_file(fileobj) - self.execution_context = execution_context + elif mapping is not None: + self.update(mapping) def _set_best_version(self): self.version = _best_version(self._fields) @@ -222,17 +234,15 @@ class DistributionMetadata(object): if name in _ALL_FIELDS: return name name = name.replace('-', '_').lower() - if name in _ATTR2FIELD: - return _ATTR2FIELD[name] - return name + return _ATTR2FIELD.get(name, name) def _default_value(self, name): - if name in _LISTFIELDS + _ELEMENTSFIELD: + if name in _LISTFIELDS or name in _ELEMENTSFIELD: return [] return 'UNKNOWN' def _check_rst_data(self, data): - """Returns warnings when the provided data doesn't compile.""" + """Return warnings when the provided data has syntax errors.""" source_path = StringIO() parser = Parser() settings = frontend.OptionParser().get_default_values() @@ -267,7 +277,7 @@ class DistributionMetadata(object): return _LINE_PREFIX.sub('\n', value) # - # Public APIs + # Public API # def get_fullname(self): return '%s-%s' % (self['Name'], self['Version']) @@ -280,7 +290,7 @@ class DistributionMetadata(object): self.read_file(open(filepath)) def read_file(self, fileob): - """Reads the metadata values from a file object.""" + """Read the metadata values from a file object.""" msg = message_from_file(fileob) self.version = msg['metadata-version'] @@ -298,8 +308,7 @@ class DistributionMetadata(object): self.set(field, value) def write(self, filepath): - """Write the metadata fields into path. - """ + """Write the metadata fields to filepath.""" pkg_info = open(filepath, 'w') try: self.write_file(pkg_info) @@ -307,8 +316,7 @@ class DistributionMetadata(object): pkg_info.close() def write_file(self, fileobject): - """Write the PKG-INFO format data to a file object. - """ + """Write the PKG-INFO format data to a file object.""" self._set_best_version() for field in _version2fieldlist(self.version): values = self.get(field) @@ -326,18 +334,50 @@ class DistributionMetadata(object): for value in values: self._write_field(fileobject, field, value) + def update(self, other=None, **kwargs): + """Set metadata values from the given mapping + + Convert the keys to Metadata fields. Given keys that don't match a + metadata argument will not be used. + + If overwrite is set to False, just add metadata values that are + actually not defined. + + If there is existing values in conflict with the dictionary ones, the + new values prevails. + + Empty values (e.g. None and []) are not setted this way. + """ + def _set(key, value): + if value not in ([], None) and key in _ATTR2FIELD: + self.set(self._convert_name(key), value) + + if other is None: + pass + elif hasattr(other, 'iteritems'): # iteritems saves memory and lookups + for k, v in other.iteritems(): + _set(k, v) + elif hasattr(other, 'keys'): + for k in other.keys(): + _set(k, v) + else: + for k, v in other: + _set(k, v) + if kwargs: + self.update(kwargs) + def set(self, name, value): - """Controls then sets a metadata field""" + """Control then set a metadata field.""" name = self._convert_name(name) - if (name in _ELEMENTSFIELD + ('Platform',) and + if ((name in _ELEMENTSFIELD or name == 'Platform') and not isinstance(value, (list, tuple))): if isinstance(value, str): value = value.split(',') else: value = [] elif (name in _LISTFIELDS and - not isinstance(value, (list, tuple))): + not isinstance(value, (list, tuple))): if isinstance(value, str): value = [value] else: @@ -364,7 +404,7 @@ class DistributionMetadata(object): self._set_best_version() def get(self, name): - """Gets a metadata field.""" + """Get a metadata field.""" name = self._convert_name(name) if name not in self._fields: return self._default_value(name) @@ -399,9 +439,9 @@ class DistributionMetadata(object): return value def check(self): - """Checks if the metadata are compliant.""" + """Check if the metadata is compliant.""" # XXX should check the versions (if the file was loaded) - missing = [] + missing, warnings = [], [] for attr in ('Name', 'Version', 'Home-page'): value = self[attr] if value == 'UNKNOWN': @@ -409,14 +449,11 @@ class DistributionMetadata(object): if _HAS_DOCUTILS: warnings = self._check_rst_data(self['Description']) - else: - warnings = [] # checking metadata 1.2 (XXX needs to check 1.1, 1.0) if self['Metadata-Version'] != '1.2': return missing, warnings - def is_valid_predicates(value): for v in value: if not is_valid_predicate(v.split(';')[0]): @@ -424,16 +461,15 @@ class DistributionMetadata(object): return True for fields, controller in ((_PREDICATE_FIELDS, is_valid_predicates), - (_VERSIONS_FIELDS, is_valid_versions), - (_VERSION_FIELDS, is_valid_version)): + (_VERSIONS_FIELDS, is_valid_versions), + (_VERSION_FIELDS, is_valid_version)): for field in fields: value = self[field] if value == 'UNKNOWN': continue if not controller(value): - warnings.append('Wrong value for "%s": %s' \ - % (field, value)) + warnings.append('Wrong value for %r: %s' % (field, value)) return missing, warnings @@ -450,7 +486,6 @@ class DistributionMetadata(object): # # micro-language for PEP 345 environment markers # -_STR_LIMIT = "'\"" # allowed operators _OPERATORS = {'==': lambda x, y: x == y, @@ -467,12 +502,11 @@ def _operate(operation, x, y): # restricted set of variables _VARS = {'sys.platform': sys.platform, - 'python_version': '%s.%s' % (sys.version_info[0], - sys.version_info[1]), - 'python_full_version': sys.version.split()[0], + 'python_version': sys.version[:3], + 'python_full_version': sys.version.split(' ', 1)[0], 'os.name': os.name, - 'platform.version': platform.version, - 'platform.machine': platform.machine} + 'platform.version': platform.version(), + 'platform.machine': platform.machine()} class _Operation(object): @@ -495,7 +529,7 @@ class _Operation(object): def _is_string(self, value): if value is None or len(value) < 2: return False - for delimiter in _STR_LIMIT: + for delimiter in '"\'': if value[0] == value[-1] == delimiter: return True return False @@ -506,7 +540,7 @@ class _Operation(object): def _convert(self, value): if value in _VARS: return self._get_var(value) - return value.strip(_STR_LIMIT) + return value.strip('"\'') def _check_name(self, value): if value not in _VARS: @@ -543,7 +577,7 @@ class _OR(object): return self.right is not None def __repr__(self): - return 'OR(%s, %s)' % (repr(self.left), repr(self.right)) + return 'OR(%r, %r)' % (self.left, self.right) def __call__(self): return self.left() or self.right() @@ -558,7 +592,7 @@ class _AND(object): return self.right is not None def __repr__(self): - return 'AND(%s, %s)' % (repr(self.left), repr(self.right)) + return 'AND(%r, %r)' % (self.left, self.right) def __call__(self): return self.left() and self.right() @@ -625,7 +659,7 @@ class _CHAIN(object): return True def _interpret(marker, execution_context=None): - """Interprets a marker and return a result given the environment.""" + """Interpret a marker and return a result depending on environment.""" marker = marker.strip() operations = _CHAIN(execution_context) tokenize(StringIO(marker).readline, operations.eat) diff --git a/src/distutils2/mkpkg.py b/src/distutils2/mkpkg.py index 3827c32..0512592 100755 --- a/src/distutils2/mkpkg.py +++ b/src/distutils2/mkpkg.py @@ -38,7 +38,7 @@ import ConfigParser helpText = { - 'name': ''' + 'name' : ''' The name of the program to be packaged, usually a single word composed of lower-case characters such as "python", "sqlalchemy", or "CherryPy". ''', @@ -624,9 +624,9 @@ troveList = [ 'Topic :: Utilities', ] -def askYn(question, default=None, helptext=None): +def askYn(question, default = None, helptext = None): while True: - answer = ask(question, default, helptext, required=True) + answer = ask(question, default, helptext, required = True) if answer and answer[0].lower() in 'yn': return(answer[0].lower()) @@ -747,15 +747,14 @@ class SetupClass(object): self.setupData['name'] = m.group(1) self.setupData['version'] = m.group(2) - for root, dirs, files in os.walk('.'): + for root, dirs, files in os.walk(os.curdir): for file in files: - if root == '.' and file == 'setup.py': - continue + if root == os.curdir and file == 'setup.py': continue fileName = os.path.join(root, file) self.inspectFile(fileName) if file == '__init__.py': - trySrc = os.path.join('.', 'src') + trySrc = os.path.join(os.curdir, 'src') tmpRoot = root if tmpRoot.startswith(trySrc): tmpRoot = tmpRoot[len(trySrc):] diff --git a/src/distutils2/pypi/__init__.py b/src/distutils2/pypi/__init__.py deleted file mode 100644 index 88efa14..0000000 --- a/src/distutils2/pypi/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""distutils2.pypi - -Package containing ways to interact with the PyPI APIs. -""" - -__all__ = ['simple', - 'dist', -] diff --git a/src/distutils2/pypi/dist.py b/src/distutils2/pypi/dist.py deleted file mode 100644 index 418186e..0000000 --- a/src/distutils2/pypi/dist.py +++ /dev/null @@ -1,315 +0,0 @@ -"""distutils2.pypi.dist - -Provides the PyPIDistribution class thats represents a distribution retrieved -on PyPI. -""" -import re -import urlparse -import urllib -import tempfile -from operator import attrgetter - -try: - import hashlib -except ImportError: - from distutils2._backport import hashlib - -from distutils2.version import suggest_normalized_version, NormalizedVersion -from distutils2.pypi.errors import HashDoesNotMatch, UnsupportedHashName - -EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz .egg".split() -MD5_HASH = re.compile(r'^.*#md5=([a-f0-9]+)$') - - -class PyPIDistribution(object): - """Represents a distribution retrieved from PyPI. - - This is a simple container for various attributes as name, version, - downloaded_location, url etc. - - The PyPIDistribution class is used by the pypi.*Index class to return - information about distributions. - """ - - @classmethod - def from_url(cls, url, probable_dist_name=None, is_external=True): - """Build a Distribution from a url archive (egg or zip or tgz). - - :param url: complete url of the distribution - :param probable_dist_name: A probable name of the distribution. - :param is_external: Tell if the url commes from an index or from - an external URL. - """ - # if the url contains a md5 hash, get it. - md5_hash = None - match = MD5_HASH.match(url) - if match is not None: - md5_hash = match.group(1) - # remove the hash - url = url.replace("#md5=%s" % md5_hash, "") - - # parse the archive name to find dist name and version - archive_name = urlparse.urlparse(url)[2].split('/')[-1] - extension_matched = False - # remove the extension from the name - for ext in EXTENSIONS: - if archive_name.endswith(ext): - archive_name = archive_name[:-len(ext)] - extension_matched = True - - name, version = split_archive_name(archive_name) - if extension_matched is True: - return PyPIDistribution(name, version, url=url, url_hashname="md5", - url_hashval=md5_hash, - url_is_external=is_external) - - def __init__(self, name, version, type=None, url=None, url_hashname=None, - url_hashval=None, url_is_external=True): - """Create a new instance of PyPIDistribution. - - :param name: the name of the distribution - :param version: the version of the distribution - :param type: the type of the dist (eg. source, bin-*, etc.) - :param url: URL where we found this distribution - :param url_hashname: the name of the hash we want to use. Refer to the - hashlib.new documentation for more information. - :param url_hashval: the hash value. - :param url_is_external: we need to know if the provided url comes from an - index browsing, or from an external resource. - - """ - self.name = name - self.version = NormalizedVersion(version) - self.type = type - # set the downloaded path to None by default. The goal here - # is to not download distributions multiple times - self.downloaded_location = None - # We store urls in dict, because we need to have a bit more informations - # than the simple URL. It will be used later to find the good url to - # use. - # We have two _url* attributes: _url and _urls. _urls contains a list of - # dict for the different urls, and _url contains the choosen url, in - # order to dont make the selection process multiple times. - self._urls = [] - self._url = None - self.add_url(url, url_hashname, url_hashval, url_is_external) - - def add_url(self, url, hashname=None, hashval=None, is_external=True): - """Add a new url to the list of urls""" - if hashname is not None: - try: - hashlib.new(hashname) - except ValueError: - raise UnsupportedHashName(hashname) - - self._urls.append({ - 'url': url, - 'hashname': hashname, - 'hashval': hashval, - 'is_external': is_external, - }) - # reset the url selection process - self._url = None - - @property - def url(self): - """Pick up the right url for the list of urls in self.urls""" - # We return internal urls over externals. - # If there is more than one internal or external, return the first - # one. - if self._url is None: - if len(self._urls) > 1: - internals_urls = [u for u in self._urls \ - if u['is_external'] == False] - if len(internals_urls) >= 1: - self._url = internals_urls[0] - if self._url is None: - self._url = self._urls[0] - return self._url - - @property - def is_source(self): - """return if the distribution is a source one or not""" - return self.type == 'source' - - @property - def is_final(self): - """proxy to version.is_final""" - return self.version.is_final - - def download(self, path=None): - """Download the distribution to a path, and return it. - - If the path is given in path, use this, otherwise, generates a new one - """ - if path is None: - path = tempfile.mkdtemp() - - # if we do not have downloaded it yet, do it. - if self.downloaded_location is None: - url = self.url['url'] - archive_name = urlparse.urlparse(url)[2].split('/')[-1] - filename, headers = urllib.urlretrieve(url, - path + "/" + archive_name) - self.downloaded_location = filename - self._check_md5(filename) - return self.downloaded_location - - def _check_md5(self, filename): - """Check that the md5 checksum of the given file matches the one in - url param""" - hashname = self.url['hashname'] - expected_hashval = self.url['hashval'] - if not None in (expected_hashval, hashname): - f = open(filename) - hashval = hashlib.new(hashname) - hashval.update(f.read()) - if hashval.hexdigest() != expected_hashval: - raise HashDoesNotMatch("got %s instead of %s" - % (hashval.hexdigest(), expected_hashval)) - - def __repr__(self): - return "%s %s %s %s" \ - % (self.__class__.__name__, self.name, self.version, - self.type or "") - - def _check_is_comparable(self, other): - if not isinstance(other, PyPIDistribution): - raise TypeError("cannot compare %s and %s" - % (type(self).__name__, type(other).__name__)) - elif self.name != other.name: - raise TypeError("cannot compare %s and %s" - % (self.name, other.name)) - - def __eq__(self, other): - self._check_is_comparable(other) - return self.version == other.version - - def __lt__(self, other): - self._check_is_comparable(other) - return self.version < other.version - - def __ne__(self, other): - return not self.__eq__(other) - - def __gt__(self, other): - return not (self.__lt__(other) or self.__eq__(other)) - - def __le__(self, other): - return self.__eq__(other) or self.__lt__(other) - - def __ge__(self, other): - return self.__eq__(other) or self.__gt__(other) - - # See http://docs.python.org/reference/datamodel#object.__hash__ - __hash__ = object.__hash__ - - -class PyPIDistributions(list): - """A container of PyPIDistribution objects. - - Contains methods and facilities to sort and filter distributions. - """ - def __init__(self, list=[]): - # To disable the ability to pass lists on instanciation - super(PyPIDistributions, self).__init__() - for item in list: - self.append(item) - - def filter(self, predicate): - """Filter the distributions and return a subset of distributions that - match the given predicate - """ - return PyPIDistributions( - [dist for dist in self if dist.name == predicate.name and - predicate.match(dist.version)]) - - def get_last(self, predicate, prefer_source=None, prefer_final=None): - """Return the most up to date version, that satisfy the given - predicate - """ - distributions = self.filter(predicate) - distributions.sort_distributions(prefer_source, prefer_final, reverse=True) - return distributions[0] - - def get_same_name_and_version(self): - """Return lists of PyPIDistribution objects that refer to the same - name and version number. This do not consider the type (source, binary, - etc.)""" - processed = [] - duplicates = [] - for dist in self: - if (dist.name, dist.version) not in processed: - processed.append((dist.name, dist.version)) - found_duplicates = [d for d in self if d.name == dist.name and - d.version == dist.version] - if len(found_duplicates) > 1: - duplicates.append(found_duplicates) - return duplicates - - def append(self, o): - """Append a new distribution to the list. - - If a distribution with the same name and version exists, just grab the - URL informations and add a new new url for the existing one. - """ - similar_dists = [d for d in self if d.name == o.name and - d.version == o.version and d.type == o.type] - if len(similar_dists) > 0: - dist = similar_dists[0] - dist.add_url(**o.url) - else: - super(PyPIDistributions, self).append(o) - - def sort_distributions(self, prefer_source=True, prefer_final=False, - reverse=True, *args, **kwargs): - """order the results with the given properties""" - - sort_by = [] - if prefer_final: - sort_by.append("is_final") - sort_by.append("version") - - if prefer_source: - sort_by.append("is_source") - - super(PyPIDistributions, self).sort( - key=lambda i: [getattr(i, arg) for arg in sort_by], - reverse=reverse, *args, **kwargs) - - -def split_archive_name(archive_name, probable_name=None): - """Split an archive name into two parts: name and version. - - Return the tuple (name, version) - """ - # Try to determine wich part is the name and wich is the version using the - # "-" separator. Take the larger part to be the version number then reduce - # if this not works. - def eager_split(str, maxsplit=2): - # split using the "-" separator - splits = str.rsplit("-", maxsplit) - name = splits[0] - version = "-".join(splits[1:]) - if version.startswith("-"): - version = version[1:] - if suggest_normalized_version(version) is None and maxsplit >= 0: - # we dont get a good version number: recurse ! - return eager_split(str, maxsplit - 1) - else: - return (name, version) - if probable_name is not None: - probable_name = probable_name.lower() - name = None - if probable_name is not None and probable_name in archive_name: - # we get the name from probable_name, if given. - name = probable_name - version = archive_name.lstrip(name) - else: - name, version = eager_split(archive_name) - - version = suggest_normalized_version(version) - if version != "" and name != "": - return (name.lower(), version) - else: - raise CantParseArchiveName(archive_name) diff --git a/src/distutils2/pypi/errors.py b/src/distutils2/pypi/errors.py deleted file mode 100644 index c9b232d..0000000 --- a/src/distutils2/pypi/errors.py +++ /dev/null @@ -1,33 +0,0 @@ -"""distutils2.pypi.errors - -All errors and exceptions raised by PyPiIndex classes. -""" -from distutils2.errors import DistutilsError - - -class PyPIError(DistutilsError): - """The base class for errors of the pypi python package.""" - - -class DistributionNotFound(PyPIError): - """No distribution match the given requirements.""" - - -class CantParseArchiveName(PyPIError): - """An archive name can't be parsed to find distribution name and version""" - - -class DownloadError(PyPIError): - """An error has occurs while downloading""" - - -class HashDoesNotMatch(DownloadError): - """Compared hashes does not match""" - - -class UnsupportedHashName(PyPIError): - """A unsupported hashname has been used""" - - -class UnableToDownload(PyPIError): - """All mirrors have been tried, without success""" diff --git a/src/distutils2/pypi/simple.py b/src/distutils2/pypi/simple.py deleted file mode 100644 index 1a6a35d..0000000 --- a/src/distutils2/pypi/simple.py +++ /dev/null @@ -1,393 +0,0 @@ -"""pypi.simple - -Contains the class "SimpleIndex", a simple spider to find and retrieve -distributions on the Python Package Index, using it's "simple" API, -avalaible at http://pypi.python.org/simple/ -""" -from fnmatch import translate -import httplib -import re -import socket -import sys -import urllib2 -import urlparse - -from distutils2.version import VersionPredicate -from distutils2.pypi.dist import (PyPIDistribution, PyPIDistributions, - EXTENSIONS) -from distutils2.pypi.errors import (PyPIError, DistributionNotFound, - DownloadError, UnableToDownload) -from distutils2 import __version__ as __distutils2_version__ - -# -- Constants ----------------------------------------------- -PYPI_DEFAULT_INDEX_URL = "http://pypi.python.org/simple/" -PYPI_DEFAULT_MIRROR_URL = "mirrors.pypi.python.org" -DEFAULT_HOSTS = ("*",) -SOCKET_TIMEOUT = 15 -USER_AGENT = "Python-urllib/%s distutils2/%s" % ( - sys.version[:3], __distutils2_version__) - -# -- Regexps ------------------------------------------------- -EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') -HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) -PYPI_MD5 = re.compile( - '<a href="([^"#]+)">([^<]+)</a>\n\s+\\(<a (?:title="MD5 hash"\n\s+)' - 'href="[^?]+\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\\)') -URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match - -# This pattern matches a character entity reference (a decimal numeric -# references, a hexadecimal numeric reference, or a named reference). -ENTITY_SUB = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub -REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) - - -def socket_timeout(timeout=SOCKET_TIMEOUT): - """Decorator to add a socket timeout when requesting pages on PyPI. - """ - def _socket_timeout(func): - def _socket_timeout(self, *args, **kwargs): - old_timeout = socket.getdefaulttimeout() - if hasattr(self, "_timeout"): - timeout = self._timeout - socket.setdefaulttimeout(timeout) - try: - return func(self, *args, **kwargs) - finally: - socket.setdefaulttimeout(old_timeout) - return _socket_timeout - return _socket_timeout - - -class SimpleIndex(object): - """Provides useful tools to request the Python Package Index simple API - - :param index_url: the url of the simple index to search on. - :param follow_externals: tell if following external links is needed or - not. Default is False. - :param hosts: a list of hosts allowed to be processed while using - follow_externals=True. Default behavior is to follow all - hosts. - :param follow_externals: tell if following external links is needed or - not. Default is False. - :param prefer_source: if there is binary and source distributions, the - source prevails. - :param prefer_final: if the version is not mentioned, and the last - version is not a "final" one (alpha, beta, etc.), - pick up the last final version. - :param mirrors_url: the url to look on for DNS records giving mirror - adresses. - :param mirrors: a list of mirrors to check out if problems - occurs while working with the one given in "url" - :param timeout: time in seconds to consider a url has timeouted. - """ - - def __init__(self, index_url=PYPI_DEFAULT_INDEX_URL, hosts=DEFAULT_HOSTS, - follow_externals=False, prefer_source=True, - prefer_final=False, mirrors_url=PYPI_DEFAULT_MIRROR_URL, - mirrors=None, timeout=SOCKET_TIMEOUT): - self.follow_externals = follow_externals - - if not index_url.endswith("/"): - index_url += "/" - self._index_urls = [index_url] - # if no mirrors are defined, use the method described in PEP 381. - if mirrors is None: - try: - mirrors = socket.gethostbyname_ex(mirrors_url)[-1] - except socket.gaierror: - mirrors = [] - self._index_urls.extend(mirrors) - self._current_index_url = 0 - self._timeout = timeout - self._prefer_source = prefer_source - self._prefer_final = prefer_final - - # create a regexp to match all given hosts - self._allowed_hosts = re.compile('|'.join(map(translate, hosts))).match - - # we keep an index of pages we have processed, in order to avoid - # scanning them multple time (eg. if there is multiple pages pointing - # on one) - self._processed_urls = [] - self._distributions = {} - - def find(self, requirements, prefer_source=None, prefer_final=None): - """Browse the PyPI to find distributions that fullfil the given - requirements. - - :param requirements: A project name and it's distribution, using - version specifiers, as described in PEP345. - :type requirements: You can pass either a version.VersionPredicate - or a string. - :param prefer_source: if there is binary and source distributions, the - source prevails. - :param prefer_final: if the version is not mentioned, and the last - version is not a "final" one (alpha, beta, etc.), - pick up the last final version. - """ - requirements = self._get_version_predicate(requirements) - if prefer_source is None: - prefer_source = self._prefer_source - if prefer_final is None: - prefer_final = self._prefer_final - - # process the index for this project - self._process_pypi_page(requirements.name) - - # filter with requirements and return the results - if requirements.name in self._distributions: - dists = self._distributions[requirements.name].filter(requirements) - dists.sort_distributions(prefer_source=prefer_source, - prefer_final=prefer_final) - else: - dists = [] - - return dists - - def get(self, requirements, *args, **kwargs): - """Browse the PyPI index to find distributions that fullfil the - given requirements, and return the most recent one. - - You can specify prefer_final and prefer_source arguments here. - If not, the default one will be used. - """ - predicate = self._get_version_predicate(requirements) - dists = self.find(predicate, *args, **kwargs) - - if len(dists) == 0: - raise DistributionNotFound(requirements) - - return dists.get_last(predicate) - - def download(self, requirements, temp_path=None, *args, **kwargs): - """Download the distribution, using the requirements. - - If more than one distribution match the requirements, use the last - version. - Download the distribution, and put it in the temp_path. If no temp_path - is given, creates and return one. - - Returns the complete absolute path to the downloaded archive. - - :param requirements: The same as the find attribute of `find`. - - You can specify prefer_final and prefer_source arguments here. - If not, the default one will be used. - """ - return self.get(requirements, *args, **kwargs)\ - .download(path=temp_path) - - def _get_version_predicate(self, requirements): - """Return a VersionPredicate object, from a string or an already - existing object. - """ - if isinstance(requirements, str): - requirements = VersionPredicate(requirements) - return requirements - - @property - def index_url(self): - return self._index_urls[self._current_index_url] - - def _switch_to_next_mirror(self): - """Switch to the next mirror (eg. point self.index_url to the next - url. - """ - # Internally, iter over the _index_url iterable, if we have read all - # of the available indexes, raise an exception. - if self._current_index_url < len(self._index_urls): - self._current_index_url = self._current_index_url + 1 - else: - raise UnableToDownload("All mirrors fails") - - def _is_browsable(self, url): - """Tell if the given URL can be browsed or not. - - It uses the follow_externals and the hosts list to tell if the given - url is browsable or not. - """ - # if _index_url is contained in the given URL, we are browsing the - # index, and it's always "browsable". - # local files are always considered browable resources - if self.index_url in url or urlparse.urlparse(url)[0] == "file": - return True - elif self.follow_externals: - if self._allowed_hosts(urlparse.urlparse(url)[1]): # 1 is netloc - return True - else: - return False - return False - - def _is_distribution(self, link): - """Tell if the given URL matches to a distribution name or not. - """ - #XXX find a better way to check that links are distributions - # Using a regexp ? - for ext in EXTENSIONS: - if ext in link: - return True - return False - - def _register_dist(self, dist): - """Register a distribution as a part of fetched distributions for - SimpleIndex. - - Return the PyPIDistributions object for the specified project name - """ - # Internally, check if a entry exists with the project name, if not, - # create a new one, and if exists, add the dist to the pool. - if not dist.name in self._distributions: - self._distributions[dist.name] = PyPIDistributions() - self._distributions[dist.name].append(dist) - return self._distributions[dist.name] - - def _process_url(self, url, project_name=None, follow_links=True): - """Process an url and search for distributions packages. - - For each URL found, if it's a download, creates a PyPIdistribution - object. If it's a homepage and we can follow links, process it too. - - :param url: the url to process - :param project_name: the project name we are searching for. - :param follow_links: Do not want to follow links more than from one - level. This parameter tells if we want to follow - the links we find (eg. run recursively this - method on it) - """ - f = self._open_url(url) - base_url = f.url - if url not in self._processed_urls: - self._processed_urls.append(url) - link_matcher = self._get_link_matcher(url) - for link, is_download in link_matcher(f.read(), base_url): - if link not in self._processed_urls: - if self._is_distribution(link) or is_download: - self._processed_urls.append(link) - # it's a distribution, so create a dist object - dist = PyPIDistribution.from_url(link, project_name, - is_external=not self.index_url in url) - self._register_dist(dist) - else: - if self._is_browsable(link) and follow_links: - self._process_url(link, project_name, - follow_links=False) - - def _get_link_matcher(self, url): - """Returns the right link matcher function of the given url - """ - if self.index_url in url: - return self._simple_link_matcher - else: - return self._default_link_matcher - - def _simple_link_matcher(self, content, base_url): - """Yield all links with a rel="download" or rel="homepage". - - This matches the simple index requirements for matching links. - If follow_externals is set to False, dont yeld the external - urls. - """ - for match in REL.finditer(content): - tag, rel = match.groups() - rels = map(str.strip, rel.lower().split(',')) - if 'homepage' in rels or 'download' in rels: - for match in HREF.finditer(tag): - url = urlparse.urljoin(base_url, - self._htmldecode(match.group(1))) - if 'download' in rels or self._is_browsable(url): - # yield a list of (url, is_download) - yield (urlparse.urljoin(base_url, url), - 'download' in rels) - - def _default_link_matcher(self, content, base_url): - """Yield all links found on the page. - """ - for match in HREF.finditer(content): - url = urlparse.urljoin(base_url, self._htmldecode(match.group(1))) - if self._is_browsable(url): - yield (url, False) - - def _process_pypi_page(self, name): - """Find and process a PyPI page for the given project name. - - :param name: the name of the project to find the page - """ - try: - # Browse and index the content of the given PyPI page. - url = self.index_url + name + "/" - self._process_url(url, name) - except DownloadError: - # if an error occurs, try with the next index_url - # (provided by the mirrors) - self._switch_to_next_mirror() - self._distributions.clear() - self._process_pypi_page(name) - - @socket_timeout() - def _open_url(self, url): - """Open a urllib2 request, handling HTTP authentication, and local - files support. - - """ - try: - scheme, netloc, path, params, query, frag = urlparse.urlparse(url) - - if scheme in ('http', 'https'): - auth, host = urllib2.splituser(netloc) - else: - auth = None - - # add index.html automatically for filesystem paths - if scheme == 'file': - if url.endswith('/'): - url += "index.html" - - if auth: - auth = "Basic " + \ - urllib2.unquote(auth).encode('base64').strip() - new_url = urlparse.urlunparse(( - scheme, host, path, params, query, frag)) - request = urllib2.Request(new_url) - request.add_header("Authorization", auth) - else: - request = urllib2.Request(url) - request.add_header('User-Agent', USER_AGENT) - fp = urllib2.urlopen(request) - - if auth: - # Put authentication info back into request URL if same host, - # so that links found on the page will work - s2, h2, path2, param2, query2, frag2 = \ - urlparse.urlparse(fp.url) - if s2 == scheme and h2 == host: - fp.url = urlparse.urlunparse( - (s2, netloc, path2, param2, query2, frag2)) - - return fp - except (ValueError, httplib.InvalidURL), v: - msg = ' '.join([str(arg) for arg in v.args]) - raise PyPIError('%s %s' % (url, msg)) - except urllib2.HTTPError, v: - return v - except urllib2.URLError, v: - raise DownloadError("Download error for %s: %s" % (url, v.reason)) - except httplib.BadStatusLine, v: - raise DownloadError('%s returned a bad status line. ' - 'The server might be down, %s' % (url, v.line)) - except httplib.HTTPException, v: - raise DownloadError("Download error for %s: %s" % (url, v)) - - def _decode_entity(self, match): - what = match.group(1) - if what.startswith('#x'): - what = int(what[2:], 16) - elif what.startswith('#'): - what = int(what[1:]) - else: - from htmlentitydefs import name2codepoint - what = name2codepoint.get(what, match.group(0)) - return unichr(what) - - def _htmldecode(self, text): - """Decode HTML entities in the given text.""" - return ENTITY_SUB(self._decode_entity, text) diff --git a/src/distutils2/tests/__init__.py b/src/distutils2/tests/__init__.py index 8a8d187..de40aaa 100644 --- a/src/distutils2/tests/__init__.py +++ b/src/distutils2/tests/__init__.py @@ -23,7 +23,7 @@ from distutils2.tests.support import unittest from test.test_support import TESTFN # use TESTFN from stdlib/test_support. -here = os.path.dirname(__file__) +here = os.path.dirname(__file__) or os.curdir verbose = 1 diff --git a/src/distutils2/tests/pypi_server.py b/src/distutils2/tests/pypi_server.py index 3927262..dc6ad2f 100644 --- a/src/distutils2/tests/pypi_server.py +++ b/src/distutils2/tests/pypi_server.py @@ -5,19 +5,29 @@ the PyPIServer all along your test case. Be sure to read the documentation before any use. """ +import os import Queue +import SocketServer +import select +import socket import threading -import time -import urllib2 + from BaseHTTPServer import HTTPServer from SimpleHTTPServer import SimpleHTTPRequestHandler -import os.path -import select +from SimpleXMLRPCServer import SimpleXMLRPCServer from distutils2.tests.support import unittest PYPI_DEFAULT_STATIC_PATH = os.path.dirname(os.path.abspath(__file__)) + "/pypiserver" +def use_xmlrpc_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = True + return use_pypi_server(*server_args, **server_kwargs) + +def use_http_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = False + return use_pypi_server(*server_args, **server_kwargs) + def use_pypi_server(*server_args, **server_kwargs): """Decorator to make use of the PyPIServer for test methods, just when needed, and not for the entire duration of the testcase. @@ -41,8 +51,8 @@ class PyPIServerTestCase(unittest.TestCase): self.pypi.start() def tearDown(self): - super(PyPIServerTestCase, self).tearDown() self.pypi.stop() + super(PyPIServerTestCase, self).tearDown() class PyPIServer(threading.Thread): """PyPI Mocked server. @@ -52,38 +62,58 @@ class PyPIServer(threading.Thread): """ def __init__(self, test_static_path=None, - static_filesystem_paths=["default"], static_uri_paths=["simple"]): + static_filesystem_paths=["default"], + static_uri_paths=["simple"], serve_xmlrpc=False) : """Initialize the server. + + Default behavior is to start the HTTP server. You can either start the + xmlrpc server by setting xmlrpc to True. Caution: Only one server will + be started. static_uri_paths and static_base_path are parameters used to provides respectively the http_paths to serve statically, and where to find the matching files on the filesystem. """ + # we want to launch the server in a new dedicated thread, to not freeze + # tests. threading.Thread.__init__(self) self._run = True - self.httpd = HTTPServer(('', 0), PyPIRequestHandler) - self.httpd.RequestHandlerClass.log_request = lambda *_: None - self.httpd.RequestHandlerClass.pypi_server = self - self.address = (self.httpd.server_name, self.httpd.server_port) - self.request_queue = Queue.Queue() - self._requests = [] - self.default_response_status = 200 - self.default_response_headers = [('Content-type', 'text/plain')] - self.default_response_data = "hello" - - # initialize static paths / filesystems - self.static_uri_paths = static_uri_paths - if test_static_path is not None: - static_filesystem_paths.append(test_static_path) - self.static_filesystem_paths = [PYPI_DEFAULT_STATIC_PATH + "/" + path - for path in static_filesystem_paths] + self._serve_xmlrpc = serve_xmlrpc + + if not self._serve_xmlrpc: + self.server = HTTPServer(('', 0), PyPIRequestHandler) + self.server.RequestHandlerClass.pypi_server = self + + self.request_queue = Queue.Queue() + self._requests = [] + self.default_response_status = 200 + self.default_response_headers = [('Content-type', 'text/plain')] + self.default_response_data = "hello" + + # initialize static paths / filesystems + self.static_uri_paths = static_uri_paths + if test_static_path is not None: + static_filesystem_paths.append(test_static_path) + self.static_filesystem_paths = [PYPI_DEFAULT_STATIC_PATH + "/" + path + for path in static_filesystem_paths] + else: + # xmlrpc server + self.server = PyPIXMLRPCServer(('', 0)) + self.xmlrpc = XMLRPCMockIndex() + # register the xmlrpc methods + self.server.register_introspection_functions() + self.server.register_instance(self.xmlrpc) + + self.address = (self.server.server_name, self.server.server_port) + # to not have unwanted outputs. + self.server.RequestHandlerClass.log_request = lambda *_: None def run(self): # loop because we can't stop it otherwise, for python < 2.6 while self._run: - r, w, e = select.select([self.httpd], [], [], 0.5) + r, w, e = select.select([self.server], [], [], 0.5) if r: - self.httpd.handle_request() + self.server.handle_request() def stop(self): """self shutdown is not supported for python < 2.6""" @@ -193,3 +223,180 @@ class PyPIRequestHandler(SimpleHTTPRequestHandler): self.send_header(header, value) self.end_headers() self.wfile.write(data) + +class PyPIXMLRPCServer(SimpleXMLRPCServer): + def server_bind(self): + """Override server_bind to store the server name.""" + SocketServer.TCPServer.server_bind(self) + host, port = self.socket.getsockname()[:2] + self.server_name = socket.getfqdn(host) + self.server_port = port + +class MockDist(object): + """Fake distribution, used in the Mock PyPI Server""" + def __init__(self, name, version="1.0", hidden=False, url="http://url/", + type="sdist", filename="", size=10000, + digest="123456", downloads=7, has_sig=False, + python_version="source", comment="comment", + author="John Doe", author_email="john@doe.name", + maintainer="Main Tayner", maintainer_email="maintainer_mail", + project_url="http://project_url/", homepage="http://homepage/", + keywords="", platform="UNKNOWN", classifiers=[], licence="", + description="Description", summary="Summary", stable_version="", + ordering="", documentation_id="", code_kwalitee_id="", + installability_id="", obsoletes=[], obsoletes_dist=[], + provides=[], provides_dist=[], requires=[], requires_dist=[], + requires_external=[], requires_python=""): + + # basic fields + self.name = name + self.version = version + self.hidden = hidden + + # URL infos + self.url = url + self.digest = digest + self.downloads = downloads + self.has_sig = has_sig + self.python_version = python_version + self.comment = comment + self.type = type + + # metadata + self.author = author + self.author_email = author_email + self.maintainer = maintainer + self.maintainer_email = maintainer_email + self.project_url = project_url + self.homepage = homepage + self.keywords = keywords + self.platform = platform + self.classifiers = classifiers + self.licence = licence + self.description = description + self.summary = summary + self.stable_version = stable_version + self.ordering = ordering + self.cheesecake_documentation_id = documentation_id + self.cheesecake_code_kwalitee_id = code_kwalitee_id + self.cheesecake_installability_id = installability_id + + self.obsoletes = obsoletes + self.obsoletes_dist = obsoletes_dist + self.provides = provides + self.provides_dist = provides_dist + self.requires = requires + self.requires_dist = requires_dist + self.requires_external = requires_external + self.requires_python = requires_python + + def url_infos(self): + return { + 'url': self.url, + 'packagetype': self.type, + 'filename': 'filename.tar.gz', + 'size': '6000', + 'md5_digest': self.digest, + 'downloads': self.downloads, + 'has_sig': self.has_sig, + 'python_version': self.python_version, + 'comment_text': self.comment, + } + + def metadata(self): + return { + 'maintainer': self.maintainer, + 'project_url': [self.project_url], + 'maintainer_email': self.maintainer_email, + 'cheesecake_code_kwalitee_id': self.cheesecake_code_kwalitee_id, + 'keywords': self.keywords, + 'obsoletes_dist': self.obsoletes_dist, + 'requires_external': self.requires_external, + 'author': self.author, + 'author_email': self.author_email, + 'download_url': self.url, + 'platform': self.platform, + 'version': self.version, + 'obsoletes': self.obsoletes, + 'provides': self.provides, + 'cheesecake_documentation_id': self.cheesecake_documentation_id, + '_pypi_hidden': self.hidden, + 'description': self.description, + '_pypi_ordering': 19, + 'requires_dist': self.requires_dist, + 'requires_python': self.requires_python, + 'classifiers': [], + 'name': self.name, + 'licence': self.licence, + 'summary': self.summary, + 'home_page': self.homepage, + 'stable_version': self.stable_version, + 'provides_dist': self.provides_dist, + 'requires': self.requires, + 'cheesecake_installability_id': self.cheesecake_installability_id, + } + + def search_result(self): + return { + '_pypi_ordering': 0, + 'version': self.version, + 'name': self.name, + 'summary': self.summary, + } + +class XMLRPCMockIndex(object): + """Mock XMLRPC server""" + + def __init__(self, dists=[]): + self._dists = dists + + def add_distributions(self, dists): + for dist in dists: + self._dists.append(MockDist(**dist)) + + def set_distributions(self, dists): + self._dists = [] + self.add_distributions(dists) + + def set_search_result(self, result): + """set a predefined search result""" + self._search_result = result + + def _get_search_results(self): + results = [] + for name in self._search_result: + found_dist = [d for d in self._dists if d.name == name] + if found_dist: + results.append(found_dist[0]) + else: + dist = MockDist(name) + results.append(dist) + self._dists.append(dist) + return [r.search_result() for r in results] + + def list_package(self): + return [d.name for d in self._dists] + + def package_releases(self, package_name, show_hidden=False): + if show_hidden: + # return all + return [d.version for d in self._dists if d.name == package_name] + else: + # return only un-hidden + return [d.version for d in self._dists if d.name == package_name + and not d.hidden] + + def release_urls(self, package_name, version): + return [d.url_infos() for d in self._dists + if d.name == package_name and d.version == version] + + def release_data(self, package_name, version): + release = [d for d in self._dists + if d.name == package_name and d.version == version] + if release: + return release[0].metadata() + else: + return {} + + def search(self, spec, operator="and"): + return self._get_search_results() diff --git a/src/distutils2/tests/pypiserver/project_list/simple/index.html b/src/distutils2/tests/pypiserver/project_list/simple/index.html new file mode 100644 index 0000000..b36d728 --- /dev/null +++ b/src/distutils2/tests/pypiserver/project_list/simple/index.html @@ -0,0 +1,5 @@ +<a class="test" href="yeah">FooBar-bar</a> +<a class="test" href="yeah">Foobar-baz</a> +<a class="test" href="yeah">Baz-FooBar</a> +<a class="test" href="yeah">Baz</a> +<a class="test" href="yeah">Foo</a> diff --git a/src/distutils2/tests/support.py b/src/distutils2/tests/support.py index 518cedf..ea9907a 100644 --- a/src/distutils2/tests/support.py +++ b/src/distutils2/tests/support.py @@ -3,6 +3,27 @@ Always import unittest from this module, it will be the right version (standard library unittest for 2.7 and higher, third-party unittest2 release for older versions). + +Three helper classes are provided: LoggingSilencer, TempdirManager and +EnvironGuard. They are written to be used as mixins, e.g. :: + + from distutils2.tests.support import unittest + from distutils2.tests.support import LoggingSilencer + + class SomeTestCase(LoggingSilencer, unittest.TestCase): + +If you need to define a setUp method on your test class, you have to +call the mixin class' setUp method or it won't work (same thing for +tearDown): + + def setUp(self): + super(self.__class__, self).setUp() + ... # other setup code + +Read each class' docstring to see their purpose and usage. + +Also provided is a DummyCommand class, useful to mock commands in the +tests of another command that needs them (see docstring). """ import os @@ -10,26 +31,36 @@ import sys import shutil import tempfile from copy import deepcopy -import warnings from distutils2 import log from distutils2.log import DEBUG, INFO, WARN, ERROR, FATAL -if sys.version_info >= (2, 7): - # improved unittest package from 2.7's standard library +if sys.version_info >= (3, 2): + # improved unittest package from 3.2's standard library import unittest else: # external release of same package for older versions import unittest2 as unittest +__all__ = ['LoggingSilencer', 'TempdirManager', 'EnvironGuard', + 'DummyCommand', 'unittest'] + + class LoggingSilencer(object): + """TestCase-compatible mixin to catch logging calls. + + Every log message that goes through distutils2.log will get appended to + self.logs instead of being printed. You can check that your code logs + warnings and errors as documented by inspecting that list; helper methods + get_logs and clear_logs are also provided. + """ def setUp(self): super(LoggingSilencer, self).setUp() - self.threshold = log.set_threshold(log.FATAL) + self.threshold = log.set_threshold(FATAL) # catching warnings - # when log will be replaced by logging - # we won't need such monkey-patch anymore + # when log is replaced by logging we won't need + # such monkey-patching anymore self._old_log = log.Log._log log.Log._log = self._log self.logs = [] @@ -45,6 +76,10 @@ class LoggingSilencer(object): self.logs.append((level, msg, args)) def get_logs(self, *levels): + """Return a list of caught messages with level in `levels`. + + Example: self.get_logs(log.WARN, log.DEBUG) -> list + """ def _format(msg, args): if len(args) == 0: return msg @@ -53,48 +88,41 @@ class LoggingSilencer(object): in self.logs if level in levels] def clear_logs(self): - self.logs = [] + """Empty the internal list of caught messages.""" + del self.logs[:] + class TempdirManager(object): - """Mix-in class that handles temporary directories for test cases. + """TestCase-compatible mixin to create temporary directories and files. - This is intended to be used with unittest.TestCase. + Directories and files created in a test_* method will be removed after it + has run. """ def setUp(self): super(TempdirManager, self).setUp() - self.tempdirs = [] - self.tempfiles = [] + self._basetempdir = tempfile.mkdtemp() def tearDown(self): super(TempdirManager, self).tearDown() - while self.tempdirs: - d = self.tempdirs.pop() - shutil.rmtree(d, os.name in ('nt', 'cygwin')) - for file_ in self.tempfiles: - if os.path.exists(file_): - os.remove(file_) + shutil.rmtree(self._basetempdir, os.name in ('nt', 'cygwin')) def mktempfile(self): - """Create a temporary file that will be cleaned up.""" - tempfile_ = tempfile.NamedTemporaryFile() - self.tempfiles.append(tempfile_.name) - return tempfile_ + """Create a read-write temporary file and return it.""" + fd, fn = tempfile.mkstemp(dir=self._basetempdir) + os.close(fd) + return open(fn, 'w+') def mkdtemp(self): - """Create a temporary directory that will be cleaned up. - - Returns the path of the directory. - """ - d = tempfile.mkdtemp() - self.tempdirs.append(d) + """Create a temporary directory and return its path.""" + d = tempfile.mkdtemp(dir=self._basetempdir) return d def write_file(self, path, content='xxx'): - """Writes a file in the given path. - + """Write a file at the given path. - path can be a string or a sequence. + path can be a string, a tuple or a list; if it's a tuple or list, + os.path.join will be used to produce a path. """ if isinstance(path, (list, tuple)): path = os.path.join(*path) @@ -105,41 +133,35 @@ class TempdirManager(object): f.close() def create_dist(self, pkg_name='foo', **kw): - """Will generate a test environment. + """Create a stub distribution object and files. - This function creates: - - a Distribution instance using keywords - - a temporary directory with a package structure + This function creates a Distribution instance (use keyword arguments + to customize it) and a temporary directory with a project structure + (currently an empty directory). - It returns the package directory and the distribution - instance. + It returns the path to the directory and the Distribution instance. + You can use TempdirManager.write_file to write any file in that + directory, e.g. setup scripts or Python modules. """ + # Late import so that third parties can import support without + # loading a ton of distutils2 modules in memory. from distutils2.dist import Distribution tmp_dir = self.mkdtemp() pkg_dir = os.path.join(tmp_dir, pkg_name) os.mkdir(pkg_dir) dist = Distribution(attrs=kw) - return pkg_dir, dist -class DummyCommand: - """Class to store options for retrieval via set_undefined_options().""" - - def __init__(self, **kwargs): - for kw, val in kwargs.items(): - setattr(self, kw, val) - - def ensure_finalized(self): - pass class EnvironGuard(object): + """TestCase-compatible mixin to save and restore the environment.""" def setUp(self): super(EnvironGuard, self).setUp() self.old_environ = deepcopy(os.environ) def tearDown(self): - for key, value in self.old_environ.items(): + for key, value in self.old_environ.iteritems(): if os.environ.get(key) != value: os.environ[key] = value @@ -148,3 +170,18 @@ class EnvironGuard(object): del os.environ[key] super(EnvironGuard, self).tearDown() + + +class DummyCommand(object): + """Class to store options for retrieval via set_undefined_options(). + + Useful for mocking one dependency command in the tests for another + command, see e.g. the dummy build command in test_build_scripts. + """ + + def __init__(self, **kwargs): + for kw, val in kwargs.iteritems(): + setattr(self, kw, val) + + def ensure_finalized(self): + pass diff --git a/src/distutils2/tests/test_Mixin2to3.py b/src/distutils2/tests/test_Mixin2to3.py index a455fc2..4b397c7 100644 --- a/src/distutils2/tests/test_Mixin2to3.py +++ b/src/distutils2/tests/test_Mixin2to3.py @@ -1,6 +1,5 @@ """Tests for distutils.command.build_py.""" import sys -import tempfile import distutils2 from distutils2.tests import support @@ -10,7 +9,7 @@ from distutils2.command.build_py import Mixin2to3 class Mixin2to3TestCase(support.TempdirManager, unittest.TestCase): - @unittest.skipUnless(sys.version > '2.6', 'Need >= 2.6') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_convert_code_only(self): # used to check if code gets converted properly. code_content = "print 'test'\n" @@ -27,7 +26,7 @@ class Mixin2to3TestCase(support.TempdirManager, unittest.TestCase): self.assertEquals(new_code_content, converted_code_content) - @unittest.skipUnless(sys.version > '2.6', 'Need >= 2.6') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_doctests_only(self): # used to check if doctests gets converted properly. doctest_content = '"""\n>>> print test\ntest\n"""\nprint test\n\n' diff --git a/src/distutils2/tests/test_bdist_msi.py b/src/distutils2/tests/test_bdist_msi.py index ebde4db..4ad54ef 100644 --- a/src/distutils2/tests/test_bdist_msi.py +++ b/src/distutils2/tests/test_bdist_msi.py @@ -10,8 +10,8 @@ class BDistMSITestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): - @unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") - def test_minial(self): + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") + def test_minimal(self): # minimal test XXX need more tests from distutils2.command.bdist_msi import bdist_msi pkg_pth, dist = self.create_dist() diff --git a/src/distutils2/tests/test_build_ext.py b/src/distutils2/tests/test_build_ext.py index f561923..1ae0bd8 100644 --- a/src/distutils2/tests/test_build_ext.py +++ b/src/distutils2/tests/test_build_ext.py @@ -45,7 +45,7 @@ class BuildExtTestCase(support.TempdirManager, build_ext.USER_BASE = site.USER_BASE # XXX only works with 2.6 > -- dunno why yet - @unittest.skipUnless(sys.version_info >= (2, 6,), 'works for >= 2.6') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_build_ext(self): global ALREADY_TESTED xx_c = os.path.join(self.tmp_dir, 'xxmodule.c') @@ -126,11 +126,8 @@ class BuildExtTestCase(support.TempdirManager, # make sure we get some library dirs under solaris self.assertTrue(len(cmd.library_dirs) > 0) + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_user_site(self): - # site.USER_SITE was introduced in 2.6 - if sys.version < '2.6': - return - import site dist = Distribution({'name': 'xx'}) cmd = build_ext(dist) diff --git a/src/distutils2/tests/test_build_py.py b/src/distutils2/tests/test_build_py.py index 4b46da9..aed24d3 100644 --- a/src/distutils2/tests/test_build_py.py +++ b/src/distutils2/tests/test_build_py.py @@ -95,7 +95,7 @@ class BuildPyTestCase(support.TempdirManager, sys.stdout = old_stdout @unittest.skipUnless(hasattr(sys, 'dont_write_bytecode'), - 'dont_write_bytecode support') + 'sys.dont_write_bytecode not supported') def test_dont_write_bytecode(self): # makes sure byte_compile is not used pkg_dir, dist = self.create_dist() diff --git a/src/distutils2/tests/test_check.py b/src/distutils2/tests/test_check.py index 7a672e1..02d171f 100644 --- a/src/distutils2/tests/test_check.py +++ b/src/distutils2/tests/test_check.py @@ -43,7 +43,7 @@ class CheckTestCase(support.LoggingSilencer, # get an error if there are missing metadata self.assertRaises(DistutilsSetupError, self._run, {}, **{'strict': 1}) - # and of course, no error when all metadata are present + # and of course, no error when all metadata fields are present cmd = self._run(metadata, strict=1) self.assertEqual(len(cmd._warnings), 0) diff --git a/src/distutils2/tests/test_cmd.py b/src/distutils2/tests/test_cmd.py index aa4a709..c768f79 100644 --- a/src/distutils2/tests/test_cmd.py +++ b/src/distutils2/tests/test_cmd.py @@ -98,7 +98,7 @@ class CommandTestCase(unittest.TestCase): def test_ensure_dirname(self): cmd = self.cmd - cmd.option1 = os.path.dirname(__file__) + cmd.option1 = os.path.dirname(__file__) or os.curdir cmd.ensure_dirname('option1') cmd.option2 = 'xxx' self.assertRaises(DistutilsOptionError, cmd.ensure_dirname, 'option2') diff --git a/src/distutils2/tests/test_converter.py b/src/distutils2/tests/test_converter.py index f0717cc..c4cb387 100644 --- a/src/distutils2/tests/test_converter.py +++ b/src/distutils2/tests/test_converter.py @@ -17,7 +17,7 @@ def _read_file(path): class ConverterTestCase(unittest.TestCase): - @unittest.skipUnless(not sys.version < '2.6', 'Needs Python >=2.6') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_conversions(self): # for all XX_before in the conversions/ dir # we run the refactoring tool diff --git a/src/distutils2/tests/test_core.py b/src/distutils2/tests/test_core.py index 9f4ad27..883187e 100644 --- a/src/distutils2/tests/test_core.py +++ b/src/distutils2/tests/test_core.py @@ -64,13 +64,13 @@ class CoreTestCase(support.EnvironGuard, unittest.TestCase): f = self.write_setup(setup_using___file__) for s in ['init', 'config', 'commandline', 'run']: distutils2.core.run_setup(f, stop_after=s) - self.assertRaises(ValueError, distutils2.core.run_setup, + self.assertRaises(ValueError, distutils2.core.run_setup, f, stop_after='bob') def test_run_setup_args(self): f = self.write_setup(setup_using___file__) - d = distutils2.core.run_setup(f, script_args=["--help"], - stop_after="init") + d = distutils2.core.run_setup(f, script_args=["--help"], + stop_after="init") self.assertEqual(['--help'], d.script_args) def test_run_setup_uses_current_dir(self): diff --git a/src/distutils2/tests/test_dist.py b/src/distutils2/tests/test_dist.py index d3d9cae..290ce01 100644 --- a/src/distutils2/tests/test_dist.py +++ b/src/distutils2/tests/test_dist.py @@ -153,6 +153,27 @@ class DistributionTestCase(support.TempdirManager, my_file2 = os.path.join(tmp_dir, 'f2') dist.metadata.write_file(open(my_file, 'w')) + def test_bad_attr(self): + cls = Distribution + + # catching warnings + warns = [] + def _warn(msg): + warns.append(msg) + + old_warn = warnings.warn + warnings.warn = _warn + try: + dist = cls(attrs={'author': 'xxx', + 'name': 'xxx', + 'version': 'xxx', + 'url': 'xxxx', + 'badoptname': 'xxx'}) + finally: + warnings.warn = old_warn + + self.assertTrue(len(warns)==1 and "Unknown distribution" in warns[0]) + def test_empty_options(self): # an empty options dictionary should not stay in the # list of attributes @@ -176,6 +197,21 @@ class DistributionTestCase(support.TempdirManager, self.assertEqual(len(warns), 0) + def test_non_empty_options(self): + # TODO: how to actually use options is not documented except + # for a few cryptic comments in dist.py. If this is to stay + # in the public API, it deserves some better documentation. + + # Here is an example of how it's used out there: + # http://svn.pythonmac.org/py2app/py2app/trunk/doc/index.html#specifying-customizations + cls = Distribution + dist = cls(attrs={'author': 'xxx', + 'name': 'xxx', + 'version': 'xxx', + 'url': 'xxxx', + 'options': dict(sdist=dict(owner="root"))}) + self.assertTrue("owner" in dist.get_option_dict("sdist")) + def test_finalize_options(self): attrs = {'keywords': 'one,two', @@ -240,6 +276,49 @@ class DistributionTestCase(support.TempdirManager, # make sure --no-user-cfg disables the user cfg file self.assertEqual(len(all_files)-1, len(files)) + def test_special_hooks_parsing(self): + temp_home = self.mkdtemp() + config_files = [os.path.join(temp_home, "config1.cfg"), + os.path.join(temp_home, "config2.cfg")] + + # Store two aliased hooks in config files + self.write_file((temp_home, "config1.cfg"), '[test_dist]\npre-hook.a = type') + self.write_file((temp_home, "config2.cfg"), '[test_dist]\npre-hook.b = type') + + sys.argv.extend(["--command-packages", + "distutils2.tests", + "test_dist"]) + cmd = self.create_distribution(config_files).get_command_obj("test_dist") + self.assertEqual(cmd.pre_hook, {"a": 'type', "b": 'type'}) + + + def test_hooks_get_run(self): + temp_home = self.mkdtemp() + config_file = os.path.join(temp_home, "config1.cfg") + + self.write_file((temp_home, "config1.cfg"), textwrap.dedent(''' + [test_dist] + pre-hook.test = distutils2.tests.test_dist.DistributionTestCase.log_pre_call + post-hook.test = distutils2.tests.test_dist.DistributionTestCase.log_post_call''')) + + sys.argv.extend(["--command-packages", + "distutils2.tests", + "test_dist"]) + d = self.create_distribution([config_file]) + cmd = d.get_command_obj("test_dist") + + # prepare the call recorders + record = [] + DistributionTestCase.log_pre_call = staticmethod(lambda _cmd: record.append(('pre', _cmd))) + DistributionTestCase.log_post_call = staticmethod(lambda _cmd: record.append(('post', _cmd))) + test_dist.run = lambda _cmd: record.append(('run', _cmd)) + test_dist.finalize_options = lambda _cmd: record.append(('finalize_options', _cmd)) + + d.run_command('test_dist') + self.assertEqual(record, [('finalize_options', cmd), + ('pre', cmd), + ('run', cmd), + ('post', cmd)]) class MetadataTestCase(support.TempdirManager, support.EnvironGuard, unittest.TestCase): diff --git a/src/distutils2/tests/test_index_dist.py b/src/distutils2/tests/test_index_dist.py new file mode 100644 index 0000000..9020635 --- /dev/null +++ b/src/distutils2/tests/test_index_dist.py @@ -0,0 +1,248 @@ +"""Tests for the distutils2.index.dist module.""" + +import os + +from distutils2.tests.pypi_server import use_pypi_server +from distutils2.tests import run_unittest +from distutils2.tests.support import unittest, TempdirManager +from distutils2.version import VersionPredicate +from distutils2.index.errors import HashDoesNotMatch, UnsupportedHashName +from distutils2.index.dist import (ReleaseInfo, ReleasesList, DistInfo, + split_archive_name, get_infos_from_url) + + +def Dist(*args, **kwargs): + # DistInfo takes a release as a first parameter, avoid this in tests. + return DistInfo(None, *args, **kwargs) + + +class TestReleaseInfo(unittest.TestCase): + + def test_instantiation(self): + # Test the DistInfo class provides us the good attributes when + # given on construction + release = ReleaseInfo("FooBar", "1.1") + self.assertEqual("FooBar", release.name) + self.assertEqual("1.1", "%s" % release.version) + + def test_add_dist(self): + # empty distribution type should assume "sdist" + release = ReleaseInfo("FooBar", "1.1") + release.add_distribution(url="http://example.org/") + # should not fail + release['sdist'] + + def test_get_unknown_distribution(self): + # should raise a KeyError + pass + + def test_get_infos_from_url(self): + # Test that the the URLs are parsed the right way + url_list = { + 'FooBar-1.1.0.tar.gz': { + 'name': 'foobar', # lowercase the name + 'version': '1.1.0', + }, + 'Foo-Bar-1.1.0.zip': { + 'name': 'foo-bar', # keep the dash + 'version': '1.1.0', + }, + 'foobar-1.1b2.tar.gz#md5=123123123123123': { + 'name': 'foobar', + 'version': '1.1b2', + 'url': 'http://example.org/foobar-1.1b2.tar.gz', # no hash + 'hashval': '123123123123123', + 'hashname': 'md5', + }, + 'foobar-1.1-rc2.tar.gz': { # use suggested name + 'name': 'foobar', + 'version': '1.1c2', + 'url': 'http://example.org/foobar-1.1-rc2.tar.gz', + } + } + + for url, attributes in url_list.items(): + # for each url + infos = get_infos_from_url("http://example.org/" + url) + for attribute, expected in attributes.items(): + got = infos.get(attribute) + if attribute == "version": + self.assertEqual("%s" % got, expected) + else: + self.assertEqual(got, expected) + + def test_split_archive_name(self): + # Test we can split the archive names + names = { + 'foo-bar-baz-1.0-rc2': ('foo-bar-baz', '1.0c2'), + 'foo-bar-baz-1.0': ('foo-bar-baz', '1.0'), + 'foobarbaz-1.0': ('foobarbaz', '1.0'), + } + for name, results in names.items(): + self.assertEqual(results, split_archive_name(name)) + + +class TestDistInfo(TempdirManager, unittest.TestCase): + + def test_get_url(self): + # Test that the url property works well + + d = Dist(url="test_url") + self.assertDictEqual(d.url, { + "url": "test_url", + "is_external": True, + "hashname": None, + "hashval": None, + }) + + # add a new url + d.add_url(url="internal_url", is_external=False) + self.assertEqual(d._url, None) + self.assertDictEqual(d.url, { + "url": "internal_url", + "is_external": False, + "hashname": None, + "hashval": None, + }) + self.assertEqual(2, len(d.urls)) + + def test_comparison(self): + # Test that we can compare DistInfoributionInfoList + foo1 = ReleaseInfo("foo", "1.0") + foo2 = ReleaseInfo("foo", "2.0") + bar = ReleaseInfo("bar", "2.0") + # assert we use the version to compare + self.assertTrue(foo1 < foo2) + self.assertFalse(foo1 > foo2) + self.assertFalse(foo1 == foo2) + + # assert we can't compare dists with different names + self.assertRaises(TypeError, foo1.__eq__, bar) + + @use_pypi_server("downloads_with_md5") + def test_download(self, server): + # Download is possible, and the md5 is checked if given + + url = "%s/simple/foobar/foobar-0.1.tar.gz" % server.full_address + # check md5 if given + dist = Dist(url=url, hashname="md5", + hashval="d41d8cd98f00b204e9800998ecf8427e") + dist.download(self.mkdtemp()) + + # a wrong md5 fails + dist2 = Dist(url=url, hashname="md5", hashval="wrongmd5") + + self.assertRaises(HashDoesNotMatch, dist2.download, self.mkdtemp()) + + # we can omit the md5 hash + dist3 = Dist(url=url) + dist3.download(self.mkdtemp()) + + # and specify a temporary location + # for an already downloaded dist + path1 = self.mkdtemp() + dist3.download(path=path1) + # and for a new one + path2_base = self.mkdtemp() + dist4 = Dist(url=url) + path2 = dist4.download(path=path2_base) + self.assertTrue(path2_base in path2) + + def test_hashname(self): + # Invalid hashnames raises an exception on assignation + Dist(hashname="md5", hashval="value") + + self.assertRaises(UnsupportedHashName, Dist, + hashname="invalid_hashname", + hashval="value") + + +class TestReleasesList(unittest.TestCase): + + def test_filter(self): + # Test we filter the distributions the right way, using version + # predicate match method + releases = ReleasesList('FooBar', ( + ReleaseInfo("FooBar", "1.1"), + ReleaseInfo("FooBar", "1.1.1"), + ReleaseInfo("FooBar", "1.2"), + ReleaseInfo("FooBar", "1.2.1"), + )) + filtered = releases.filter(VersionPredicate("FooBar (<1.2)")) + self.assertNotIn(releases[2], filtered) + self.assertNotIn(releases[3], filtered) + self.assertIn(releases[0], filtered) + self.assertIn(releases[1], filtered) + + def test_append(self): + # When adding a new item to the list, the behavior is to test if + # a release with the same name and version number already exists, + # and if so, to add a new distribution for it. If the distribution type + # is already defined too, add url informations to the existing DistInfo + # object. + + releases = ReleasesList("FooBar", [ + ReleaseInfo("FooBar", "1.1", url="external_url", + dist_type="sdist"), + ]) + self.assertEqual(1, len(releases)) + releases.add_release(release=ReleaseInfo("FooBar", "1.1", + url="internal_url", + is_external=False, + dist_type="sdist")) + self.assertEqual(1, len(releases)) + self.assertEqual(2, len(releases[0]['sdist'].urls)) + + releases.add_release(release=ReleaseInfo("FooBar", "1.1.1", + dist_type="sdist")) + self.assertEqual(2, len(releases)) + + # when adding a distribution whith a different type, a new distribution + # has to be added. + releases.add_release(release=ReleaseInfo("FooBar", "1.1.1", + dist_type="bdist")) + self.assertEqual(2, len(releases)) + self.assertEqual(2, len(releases[1].dists)) + + def test_prefer_final(self): + # Can order the distributions using prefer_final + + fb10 = ReleaseInfo("FooBar", "1.0") # final distribution + fb11a = ReleaseInfo("FooBar", "1.1a1") # alpha + fb12a = ReleaseInfo("FooBar", "1.2a1") # alpha + fb12b = ReleaseInfo("FooBar", "1.2b1") # beta + dists = ReleasesList("FooBar", [fb10, fb11a, fb12a, fb12b]) + + dists.sort_releases(prefer_final=True) + self.assertEqual(fb10, dists[0]) + + dists.sort_releases(prefer_final=False) + self.assertEqual(fb12b, dists[0]) + +# def test_prefer_source(self): +# # Ordering support prefer_source +# fb_source = Dist("FooBar", "1.0", type="source") +# fb_binary = Dist("FooBar", "1.0", type="binary") +# fb2_binary = Dist("FooBar", "2.0", type="binary") +# dists = ReleasesList([fb_binary, fb_source]) +# +# dists.sort_distributions(prefer_source=True) +# self.assertEqual(fb_source, dists[0]) +# +# dists.sort_distributions(prefer_source=False) +# self.assertEqual(fb_binary, dists[0]) +# +# dists.append(fb2_binary) +# dists.sort_distributions(prefer_source=True) +# self.assertEqual(fb2_binary, dists[0]) + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestDistInfo)) + suite.addTest(unittest.makeSuite(TestReleaseInfo)) + suite.addTest(unittest.makeSuite(TestReleasesList)) + return suite + +if __name__ == '__main__': + run_unittest(test_suite()) diff --git a/src/distutils2/tests/test_pypi_simple.py b/src/distutils2/tests/test_index_simple.py index 85d8b77..d21bda5 100644 --- a/src/distutils2/tests/test_pypi_simple.py +++ b/src/distutils2/tests/test_index_simple.py @@ -3,36 +3,34 @@ """ import sys import os -import shutil -import tempfile import urllib2 -from distutils2.pypi import simple -from distutils2.tests import support, run_unittest +from distutils2.index.simple import Crawler +from distutils2.tests import support from distutils2.tests.support import unittest from distutils2.tests.pypi_server import (use_pypi_server, PyPIServer, PYPI_DEFAULT_STATIC_PATH) -class PyPISimpleTestCase(support.TempdirManager, - unittest.TestCase): +class SimpleCrawlerTestCase(support.TempdirManager, unittest.TestCase): - def _get_simple_index(self, server, base_url="/simple/", hosts=None, + def _get_simple_crawler(self, server, base_url="/simple/", hosts=None, *args, **kwargs): - """Build and return a SimpleSimpleIndex instance, with the test server + """Build and return a SimpleIndex instance, with the test server urls """ if hosts is None: hosts = (server.full_address.strip("http://"),) kwargs['hosts'] = hosts - return simple.SimpleIndex(server.full_address + base_url, *args, + return Crawler(server.full_address + base_url, *args, **kwargs) - def test_bad_urls(self): - index = simple.SimpleIndex() + @use_pypi_server() + def test_bad_urls(self, server): + crawler = Crawler() url = 'http://127.0.0.1:0/nonesuch/test_simple' try: - v = index._open_url(url) + v = crawler._open_url(url) except Exception, v: self.assertTrue(url in str(v)) else: @@ -41,10 +39,10 @@ class PyPISimpleTestCase(support.TempdirManager, # issue 16 # easy_install inquant.contentmirror.plone breaks because of a typo # in its home URL - index = simple.SimpleIndex(hosts=('www.example.com',)) + crawler = Crawler(hosts=('example.org',)) url = 'url:%20https://svn.plone.org/svn/collective/inquant.contentmirror.plone/trunk' try: - v = index._open_url(url) + v = crawler._open_url(url) except Exception, v: self.assertTrue(url in str(v)) else: @@ -56,10 +54,10 @@ class PyPISimpleTestCase(support.TempdirManager, old_urlopen = urllib2.urlopen urllib2.urlopen = _urlopen - url = 'http://example.com' + url = 'http://example.org' try: try: - v = index._open_url(url) + v = crawler._open_url(url) except Exception, v: self.assertTrue('line' in str(v)) else: @@ -70,91 +68,91 @@ class PyPISimpleTestCase(support.TempdirManager, # issue 20 url = 'http://http://svn.pythonpaste.org/Paste/wphp/trunk' try: - index._open_url(url) + crawler._open_url(url) except Exception, v: self.assertTrue('nonnumeric port' in str(v)) # issue #160 if sys.version_info[0] == 2 and sys.version_info[1] == 7: # this should not fail - url = 'http://example.com' + url = server.full_address page = ('<a href="http://www.famfamfam.com](' 'http://www.famfamfam.com/">') - index._process_url(url, page) + crawler._process_url(url, page) @use_pypi_server("test_found_links") def test_found_links(self, server): - # Browse the index, asking for a specified distribution version + # Browse the index, asking for a specified release version # The PyPI index contains links for version 1.0, 1.1, 2.0 and 2.0.1 - index = self._get_simple_index(server) - last_distribution = index.get("foobar") + crawler = self._get_simple_crawler(server) + last_release = crawler.get_release("foobar") # we have scanned the index page self.assertIn(server.full_address + "/simple/foobar/", - index._processed_urls) + crawler._processed_urls) - # we have found 4 distributions in this page - self.assertEqual(len(index._distributions["foobar"]), 4) + # we have found 4 releases in this page + self.assertEqual(len(crawler._projects["foobar"]), 4) # and returned the most recent one - self.assertEqual("%s" % last_distribution.version, '2.0.1') + self.assertEqual("%s" % last_release.version, '2.0.1') def test_is_browsable(self): - index = simple.SimpleIndex(follow_externals=False) - self.assertTrue(index._is_browsable(index.index_url + "test")) + crawler = Crawler(follow_externals=False) + self.assertTrue(crawler._is_browsable(crawler.index_url + "test")) # Now, when following externals, we can have a list of hosts to trust. # and don't follow other external links than the one described here. - index = simple.SimpleIndex(hosts=["pypi.python.org", "test.org"], + crawler = Crawler(hosts=["pypi.python.org", "example.org"], follow_externals=True) good_urls = ( "http://pypi.python.org/foo/bar", "http://pypi.python.org/simple/foobar", - "http://test.org", - "http://test.org/", - "http://test.org/simple/", + "http://example.org", + "http://example.org/", + "http://example.org/simple/", ) bad_urls = ( "http://python.org", - "http://test.tld", + "http://example.tld", ) for url in good_urls: - self.assertTrue(index._is_browsable(url)) + self.assertTrue(crawler._is_browsable(url)) for url in bad_urls: - self.assertFalse(index._is_browsable(url)) + self.assertFalse(crawler._is_browsable(url)) # allow all hosts - index = simple.SimpleIndex(follow_externals=True, hosts=("*",)) - self.assertTrue(index._is_browsable("http://an-external.link/path")) - self.assertTrue(index._is_browsable("pypi.test.tld/a/path")) + crawler = Crawler(follow_externals=True, hosts=("*",)) + self.assertTrue(crawler._is_browsable("http://an-external.link/path")) + self.assertTrue(crawler._is_browsable("pypi.example.org/a/path")) # specify a list of hosts we want to allow - index = simple.SimpleIndex(follow_externals=True, - hosts=("*.test.tld",)) - self.assertFalse(index._is_browsable("http://an-external.link/path")) - self.assertTrue(index._is_browsable("http://pypi.test.tld/a/path")) + crawler = Crawler(follow_externals=True, + hosts=("*.example.org",)) + self.assertFalse(crawler._is_browsable("http://an-external.link/path")) + self.assertTrue(crawler._is_browsable("http://pypi.example.org/a/path")) @use_pypi_server("with_externals") - def test_restrict_hosts(self, server): + def test_follow_externals(self, server): # Include external pages # Try to request the package index, wich contains links to "externals" # resources. They have to be scanned too. - index = self._get_simple_index(server, follow_externals=True) - index.get("foobar") + crawler = self._get_simple_crawler(server, follow_externals=True) + crawler.get_release("foobar") self.assertIn(server.full_address + "/external/external.html", - index._processed_urls) + crawler._processed_urls) @use_pypi_server("with_real_externals") def test_restrict_hosts(self, server): # Only use a list of allowed hosts is possible # Test that telling the simple pyPI client to not retrieve external # works - index = self._get_simple_index(server, follow_externals=False) - index.get("foobar") + crawler = self._get_simple_crawler(server, follow_externals=False) + crawler.get_release("foobar") self.assertNotIn(server.full_address + "/external/external.html", - index._processed_urls) + crawler._processed_urls) @use_pypi_server(static_filesystem_paths=["with_externals"], static_uri_paths=["simple", "external"]) @@ -168,23 +166,26 @@ class PyPISimpleTestCase(support.TempdirManager, # - someone manually coindexes this link (with the md5 in the url) onto # an external page accessible from the package page. # - someone reuploads the package (with a different md5) - # - while easy_installing, an MD5 error occurs because the external link - # is used + # - while easy_installing, an MD5 error occurs because the external + # link is used # -> The index should use the link from pypi, not the external one. # start an index server index_url = server.full_address + '/simple/' # scan a test index - index = simple.SimpleIndex(index_url, follow_externals=True) - dists = index.find("foobar") + crawler = Crawler(index_url, follow_externals=True) + releases = crawler.get_releases("foobar") server.stop() # we have only one link, because links are compared without md5 - self.assertEqual(len(dists), 1) + self.assertEqual(1, len(releases)) + self.assertEqual(1, len(releases[0].dists)) # the link should be from the index - self.assertEqual('12345678901234567', dists[0].url['hashval']) - self.assertEqual('md5', dists[0].url['hashname']) + self.assertEqual(2, len(releases[0].dists['sdist'].urls)) + self.assertEqual('12345678901234567', + releases[0].dists['sdist'].url['hashval']) + self.assertEqual('md5', releases[0].dists['sdist'].url['hashname']) @use_pypi_server(static_filesystem_paths=["with_norel_links"], static_uri_paths=["simple", "external"]) @@ -194,22 +195,22 @@ class PyPISimpleTestCase(support.TempdirManager, # to not be processed by the package index, while processing "pages". # process the pages - index = self._get_simple_index(server, follow_externals=True) - index.find("foobar") + crawler = self._get_simple_crawler(server, follow_externals=True) + crawler.get_releases("foobar") # now it should have processed only pages with links rel="download" # and rel="homepage" self.assertIn("%s/simple/foobar/" % server.full_address, - index._processed_urls) # it's the simple index page + crawler._processed_urls) # it's the simple index page self.assertIn("%s/external/homepage.html" % server.full_address, - index._processed_urls) # the external homepage is rel="homepage" + crawler._processed_urls) # the external homepage is rel="homepage" self.assertNotIn("%s/external/nonrel.html" % server.full_address, - index._processed_urls) # this link contains no rel=* + crawler._processed_urls) # this link contains no rel=* self.assertNotIn("%s/unrelated-0.2.tar.gz" % server.full_address, - index._processed_urls) # linked from simple index (no rel) + crawler._processed_urls) # linked from simple index (no rel) self.assertIn("%s/foobar-0.1.tar.gz" % server.full_address, - index._processed_urls) # linked from simple index (rel) + crawler._processed_urls) # linked from simple index (rel) self.assertIn("%s/foobar-2.0.tar.gz" % server.full_address, - index._processed_urls) # linked from external homepage (rel) + crawler._processed_urls) # linked from external homepage (rel) def test_uses_mirrors(self): # When the main repository seems down, try using the given mirrors""" @@ -219,18 +220,18 @@ class PyPISimpleTestCase(support.TempdirManager, try: # create the index using both servers - index = simple.SimpleIndex(server.full_address + "/simple/", + crawler = Crawler(server.full_address + "/simple/", hosts=('*',), timeout=1, # set the timeout to 1s for the tests - mirrors=[mirror.full_address + "/simple/",]) + mirrors=[mirror.full_address]) # this should not raise a timeout - self.assertEqual(4, len(index.find("foo"))) + self.assertEqual(4, len(crawler.get_releases("foo"))) finally: mirror.stop() def test_simple_link_matcher(self): # Test that the simple link matcher yields the right links""" - index = simple.SimpleIndex(follow_externals=False) + crawler = Crawler(follow_externals=False) # Here, we define: # 1. one link that must be followed, cause it's a download one @@ -238,27 +239,33 @@ class PyPISimpleTestCase(support.TempdirManager, # returns false for it. # 3. one link that must be followed cause it's a homepage that is # browsable - self.assertTrue(index._is_browsable("%stest" % index.index_url)) - self.assertFalse(index._is_browsable("http://dl-link2")) + # 4. one link that must be followed, because it contain a md5 hash + self.assertTrue(crawler._is_browsable("%stest" % crawler.index_url)) + self.assertFalse(crawler._is_browsable("http://dl-link2")) content = """ <a href="http://dl-link1" rel="download">download_link1</a> <a href="http://dl-link2" rel="homepage">homepage_link1</a> - <a href="%stest" rel="homepage">homepage_link2</a> - """ % index.index_url + <a href="%(index_url)stest" rel="homepage">homepage_link2</a> + <a href="%(index_url)stest/foobar-1.tar.gz#md5=abcdef>download_link2</a> + """ % {'index_url': crawler.index_url } # Test that the simple link matcher yield the good links. - generator = index._simple_link_matcher(content, index.index_url) + generator = crawler._simple_link_matcher(content, crawler.index_url) + self.assertEqual(('%stest/foobar-1.tar.gz#md5=abcdef' % crawler.index_url, + True), generator.next()) self.assertEqual(('http://dl-link1', True), generator.next()) - self.assertEqual(('%stest' % index.index_url, False), + self.assertEqual(('%stest' % crawler.index_url, False), generator.next()) self.assertRaises(StopIteration, generator.next) - # Follow the external links is possible - index.follow_externals = True - generator = index._simple_link_matcher(content, index.index_url) + # Follow the external links is possible (eg. homepages) + crawler.follow_externals = True + generator = crawler._simple_link_matcher(content, crawler.index_url) + self.assertEqual(('%stest/foobar-1.tar.gz#md5=abcdef' % crawler.index_url, + True), generator.next()) self.assertEqual(('http://dl-link1', True), generator.next()) self.assertEqual(('http://dl-link2', False), generator.next()) - self.assertEqual(('%stest' % index.index_url, False), + self.assertEqual(('%stest' % crawler.index_url, False), generator.next()) self.assertRaises(StopIteration, generator.next) @@ -266,12 +273,44 @@ class PyPISimpleTestCase(support.TempdirManager, # Test that we can browse local files""" index_path = os.sep.join(["file://" + PYPI_DEFAULT_STATIC_PATH, "test_found_links", "simple"]) - index = simple.SimpleIndex(index_path) - dists = index.find("foobar") + crawler = Crawler(index_path) + dists = crawler.get_releases("foobar") self.assertEqual(4, len(dists)) + def test_get_link_matcher(self): + crawler = Crawler("http://example.org") + self.assertEqual('_simple_link_matcher', crawler._get_link_matcher( + "http://example.org/some/file").__name__) + self.assertEqual('_default_link_matcher', crawler._get_link_matcher( + "http://other-url").__name__) + + def test_default_link_matcher(self): + crawler = Crawler("http://example.org", mirrors=[]) + crawler.follow_externals = True + crawler._is_browsable = lambda *args:True + base_url = "http://example.org/some/file/" + content = """ +<a href="../homepage" rel="homepage">link</a> +<a href="../download" rel="download">link2</a> +<a href="../simpleurl">link2</a> + """ + found_links = dict(crawler._default_link_matcher(content, + base_url)).keys() + self.assertIn('http://example.org/some/homepage', found_links) + self.assertIn('http://example.org/some/simpleurl', found_links) + self.assertIn('http://example.org/some/download', found_links) + + @use_pypi_server("project_list") + def test_search_projects(self, server): + # we can search the index for some projects, on their names + # the case used no matters here + crawler = self._get_simple_crawler(server) + projects = [p.name for p in crawler.search_projects("Foobar")] + self.assertListEqual(['FooBar-bar', 'Foobar-baz', 'Baz-FooBar'], + projects) + def test_suite(): - return unittest.makeSuite(PyPISimpleTestCase) + return unittest.makeSuite(SimpleCrawlerTestCase) if __name__ == '__main__': unittest.main(defaultTest="test_suite") diff --git a/src/distutils2/tests/test_index_xmlrpc.py b/src/distutils2/tests/test_index_xmlrpc.py new file mode 100644 index 0000000..50ea739 --- /dev/null +++ b/src/distutils2/tests/test_index_xmlrpc.py @@ -0,0 +1,92 @@ +"""Tests for the distutils2.index.xmlrpc module.""" + +from distutils2.tests.pypi_server import use_xmlrpc_server +from distutils2.tests import run_unittest +from distutils2.tests.support import unittest +from distutils2.index.xmlrpc import Client, InvalidSearchField, ProjectNotFound + + +class TestXMLRPCClient(unittest.TestCase): + def _get_client(self, server, *args, **kwargs): + return Client(server.full_address, *args, **kwargs) + + @use_xmlrpc_server() + def test_search_projects(self, server): + client = self._get_client(server) + server.xmlrpc.set_search_result(['FooBar', 'Foo', 'FooFoo']) + results = [r.name for r in client.search_projects(name='Foo')] + self.assertEqual(3, len(results)) + self.assertIn('FooBar', results) + self.assertIn('Foo', results) + self.assertIn('FooFoo', results) + + def test_search_projects_bad_fields(self): + client = Client() + self.assertRaises(InvalidSearchField, client.search_projects, + invalid="test") + + @use_xmlrpc_server() + def test_get_releases(self, server): + client = self._get_client(server) + server.xmlrpc.set_distributions([ + {'name': 'FooBar', 'version': '1.1'}, + {'name': 'FooBar', 'version': '1.2', 'url': 'http://some/url/'}, + {'name': 'FooBar', 'version': '1.3', 'url': 'http://other/url/'}, + ]) + + # use a lambda here to avoid an useless mock call + server.xmlrpc.list_releases = lambda *a, **k: ['1.1', '1.2', '1.3'] + + releases = client.get_releases('FooBar (<=1.2)') + # dont call release_data and release_url; just return name and version. + self.assertEqual(2, len(releases)) + versions = releases.get_versions() + self.assertIn('1.1', versions) + self.assertIn('1.2', versions) + self.assertNotIn('1.3', versions) + + self.assertRaises(ProjectNotFound, client.get_releases,'Foo') + + @use_xmlrpc_server() + def test_get_distributions(self, server): + client = self._get_client(server) + server.xmlrpc.set_distributions([ + {'name':'FooBar', 'version': '1.1', 'url': + 'http://example.org/foobar-1.1-sdist.tar.gz', + 'digest': '1234567', 'type': 'sdist', 'python_version':'source'}, + {'name':'FooBar', 'version': '1.1', 'url': + 'http://example.org/foobar-1.1-bdist.tar.gz', + 'digest': '8912345', 'type': 'bdist'}, + ]) + + releases = client.get_releases('FooBar', '1.1') + client.get_distributions('FooBar', '1.1') + release = releases.get_release('1.1') + self.assertTrue('http://example.org/foobar-1.1-sdist.tar.gz', + release['sdist'].url['url']) + self.assertTrue('http://example.org/foobar-1.1-bdist.tar.gz', + release['bdist'].url['url']) + self.assertEqual(release['sdist'].python_version, 'source') + + @use_xmlrpc_server() + def test_get_metadata(self, server): + client = self._get_client(server) + server.xmlrpc.set_distributions([ + {'name':'FooBar', + 'version': '1.1', + 'keywords': '', + 'obsoletes_dist': ['FooFoo'], + 'requires_external': ['Foo'], + }]) + release = client.get_metadata('FooBar', '1.1') + self.assertEqual(['Foo'], release.metadata['requires_external']) + self.assertEqual(['FooFoo'], release.metadata['obsoletes_dist']) + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestXMLRPCClient)) + return suite + +if __name__ == '__main__': + run_unittest(test_suite()) diff --git a/src/distutils2/tests/test_install.py b/src/distutils2/tests/test_install.py index 5cf6cd9..c7c2616 100644 --- a/src/distutils2/tests/test_install.py +++ b/src/distutils2/tests/test_install.py @@ -75,11 +75,9 @@ class InstallTestCase(support.TempdirManager, check_path(cmd.install_scripts, os.path.join(destination, "bin")) check_path(cmd.install_data, destination) + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_user_site(self): - # site.USER_SITE was introduced in 2.6 - if sys.version < '2.6': - return - + # test install with --user # preparing the environment for the test self.old_user_base = get_config_var('userbase') self.old_user_site = get_path('purelib', '%s_user' % os.name) @@ -195,11 +193,12 @@ class InstallTestCase(support.TempdirManager, cmd.ensure_finalized() cmd.run() - # let's check the RECORD file was created with one - # line (the egg info file) + # let's check the RECORD file was created with four + # lines, one for each .dist-info entry: METADATA, + # INSTALLER, REQUSTED, RECORD f = open(cmd.record) try: - self.assertEqual(len(f.readlines()), 1) + self.assertEqual(len(f.readlines()), 4) finally: f.close() diff --git a/src/distutils2/tests/test_install_distinfo.py b/src/distutils2/tests/test_install_distinfo.py new file mode 100644 index 0000000..fbfa722 --- /dev/null +++ b/src/distutils2/tests/test_install_distinfo.py @@ -0,0 +1,202 @@ +"""Tests for ``distutils2.command.install_distinfo``. """ + +import os +import sys +import csv + +from distutils2.command.install_distinfo import install_distinfo +from distutils2.core import Command +from distutils2.metadata import DistributionMetadata +from distutils2.tests import support +from distutils2.tests.support import unittest + +try: + import hashlib +except ImportError: + from distutils2._backport import hashlib + + +class DummyInstallCmd(Command): + + def __init__(self, dist=None): + self.outputs = [] + self.distribution = dist + + def __getattr__(self, name): + return None + + def ensure_finalized(self): + pass + + def get_outputs(self): + return self.outputs + \ + self.get_finalized_command('install_distinfo').get_outputs() + + +class InstallDistinfoTestCase(support.TempdirManager, + support.LoggingSilencer, + support.EnvironGuard, + unittest.TestCase): + + checkLists = lambda self, x, y: self.assertListEqual(sorted(x), sorted(y)) + + def test_empty_install(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.ensure_finalized() + cmd.run() + + self.checkLists(os.listdir(install_dir), ['foo-1.0.dist-info']) + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'RECORD', 'REQUESTED', 'INSTALLER']) + self.assertEqual(open(os.path.join(dist_info, 'INSTALLER')).read(), + 'distutils') + self.assertEqual(open(os.path.join(dist_info, 'REQUESTED')).read(), + '') + meta_path = os.path.join(dist_info, 'METADATA') + self.assertTrue(DistributionMetadata(path=meta_path).check()) + + def test_installer(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.installer = 'bacon-python' + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.assertEqual(open(os.path.join(dist_info, 'INSTALLER')).read(), + 'bacon-python') + + def test_requested(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.requested = False + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'RECORD', 'INSTALLER']) + + def test_no_record(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.no_record = True + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'REQUESTED', 'INSTALLER']) + + def test_record(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install'] = install + + fake_dists = os.path.join(os.path.dirname(__file__), '..', + '_backport', 'tests', 'fake_dists') + fake_dists = os.path.realpath(fake_dists) + + # for testing, we simply add all files from _backport's fake_dists + dirs = [] + for dir in os.listdir(fake_dists): + full_path = os.path.join(fake_dists, dir) + if (not dir.endswith('.egg') or dir.endswith('.egg-info') or + dir.endswith('.dist-info')) and os.path.isdir(full_path): + dirs.append(full_path) + + for dir in dirs: + for (path, subdirs, files) in os.walk(dir): + install.outputs += [os.path.join(path, f) for f in files] + install.outputs += [os.path.join('path', f + 'c') + for f in files if f.endswith('.py')] + + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.initialize_options() + cmd.distinfo_dir = install_dir + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + + expected = [] + for f in install.get_outputs(): + if f.endswith('.pyc') or \ + f == os.path.join(install_dir, 'foo-1.0.dist-info', 'RECORD'): + expected.append([f, '', '']) + else: + size = os.path.getsize(f) + md5 = hashlib.md5() + md5.update(open(f).read()) + hash = md5.hexdigest() + expected.append([f, hash, str(size)]) + + parsed = [] + f = open(os.path.join(dist_info, 'RECORD'), 'rb') + try: + reader = csv.reader(f, delimiter=',', + lineterminator=os.linesep, + quotechar='"') + parsed = list(reader) + finally: + f.close() + + self.maxDiff = None + self.checkLists(parsed, expected) + + +def test_suite(): + return unittest.makeSuite(InstallDistinfoTestCase) + + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/src/distutils2/tests/test_install_lib.py b/src/distutils2/tests/test_install_lib.py index 680eaf1..a96c889 100644 --- a/src/distutils2/tests/test_install_lib.py +++ b/src/distutils2/tests/test_install_lib.py @@ -57,9 +57,8 @@ class InstallLibTestCase(support.TempdirManager, # setting up a dist environment cmd.compile = cmd.optimize = 1 cmd.install_dir = pkg_dir - f = os.path.join(pkg_dir, 'foo.py') - self.write_file(f, '# python file') - cmd.distribution.py_modules = [pkg_dir] + f = os.path.join(pkg_dir, '__init__.py') + self.write_file(f, '# python package') cmd.distribution.ext_modules = [Extension('foo', ['xxx'])] cmd.distribution.packages = [pkg_dir] cmd.distribution.script_name = 'setup.py' @@ -74,9 +73,8 @@ class InstallLibTestCase(support.TempdirManager, # setting up a dist environment cmd.compile = cmd.optimize = 1 cmd.install_dir = pkg_dir - f = os.path.join(pkg_dir, 'foo.py') - self.write_file(f, '# python file') - cmd.distribution.py_modules = [pkg_dir] + f = os.path.join(pkg_dir, '__init__.py') + self.write_file(f, '# python package') cmd.distribution.ext_modules = [Extension('foo', ['xxx'])] cmd.distribution.packages = [pkg_dir] cmd.distribution.script_name = 'setup.py' @@ -84,7 +82,8 @@ class InstallLibTestCase(support.TempdirManager, # get_input should return 2 elements self.assertEqual(len(cmd.get_inputs()), 2) - @unittest.skipUnless(bytecode_support, 'sys.dont_write_bytecode not supported') + @unittest.skipUnless(bytecode_support, + 'sys.dont_write_bytecode not supported') def test_dont_write_bytecode(self): # makes sure byte_compile is not used pkg_dir, dist = self.create_dist() diff --git a/src/distutils2/tests/test_metadata.py b/src/distutils2/tests/test_metadata.py index e9b70d9..e7e8d68 100644 --- a/src/distutils2/tests/test_metadata.py +++ b/src/distutils2/tests/test_metadata.py @@ -1,6 +1,7 @@ """Tests for distutils.command.bdist.""" import os import sys +import platform from StringIO import StringIO from distutils2.metadata import (DistributionMetadata, _interpret, @@ -12,25 +13,59 @@ from distutils2.errors import (MetadataConflictError, class DistributionMetadataTestCase(LoggingSilencer, unittest.TestCase): + def test_instantiation(self): + PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') + fp = open(PKG_INFO) + try: + contents = fp.read() + finally: + fp.close() + fp = StringIO(contents) + + m = DistributionMetadata() + self.assertRaises(MetadataUnrecognizedVersionError, m.items) + + m = DistributionMetadata(PKG_INFO) + self.assertEqual(len(m.items()), 22) + + m = DistributionMetadata(fileobj=fp) + self.assertEqual(len(m.items()), 22) + + m = DistributionMetadata(mapping=dict(name='Test', version='1.0')) + self.assertEqual(len(m.items()), 11) + + d = dict(m.items()) + self.assertRaises(TypeError, DistributionMetadata, + PKG_INFO, fileobj=fp) + self.assertRaises(TypeError, DistributionMetadata, + PKG_INFO, mapping=d) + self.assertRaises(TypeError, DistributionMetadata, + fileobj=fp, mapping=d) + self.assertRaises(TypeError, DistributionMetadata, + PKG_INFO, mapping=m, fileobj=fp) def test_interpret(self): - platform = sys.platform + sys_platform = sys.platform version = sys.version.split()[0] os_name = os.name + platform_version = platform.version() + platform_machine = platform.machine() - self.assertTrue(_interpret("sys.platform == '%s'" % platform)) + self.assertTrue(_interpret("sys.platform == '%s'" % sys_platform)) self.assertTrue(_interpret( - "sys.platform == '%s' or python_version == '2.4'" % platform)) + "sys.platform == '%s' or python_version == '2.4'" % sys_platform)) self.assertTrue(_interpret( "sys.platform == '%s' and python_full_version == '%s'" % - (platform, version))) - self.assertTrue(_interpret("'%s' == sys.platform" % platform)) - + (sys_platform, version))) + self.assertTrue(_interpret("'%s' == sys.platform" % sys_platform)) self.assertTrue(_interpret('os.name == "%s"' % os_name)) + self.assertTrue(_interpret( + 'platform.version == "%s" and platform.machine == "%s"' % + (platform_version, platform_machine))) # stuff that need to raise a syntax error ops = ('os.name == os.name', 'os.name == 2', "'2' == '2'", - 'okpjonon', '', 'os.name ==') + 'okpjonon', '', 'os.name ==', 'python_version == 2.4') for op in ops: self.assertRaises(SyntaxError, _interpret, op) @@ -79,6 +114,8 @@ class DistributionMetadataTestCase(LoggingSilencer, unittest.TestCase): metadata.read_file(StringIO(content)) self.assertEqual(metadata['Requires-Dist'], ['bar']) metadata['Name'] = "baz; sys.platform == 'blah'" + # FIXME is None or 'UNKNOWN' correct here? + # where is that documented? self.assertEquals(metadata['Name'], None) # test with context @@ -107,15 +144,15 @@ class DistributionMetadataTestCase(LoggingSilencer, unittest.TestCase): metadata.read_file(out) self.assertEqual(wanted, metadata['Description']) - def test_mapper_apis(self): + def test_mapping_api(self): PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') content = open(PKG_INFO).read() content = content % sys.platform - metadata = DistributionMetadata() - metadata.read_file(StringIO(content)) + metadata = DistributionMetadata(fileobj=StringIO(content)) self.assertIn('Version', metadata.keys()) self.assertIn('0.5', metadata.values()) self.assertIn(('Version', '0.5'), metadata.items()) + #TODO test update def test_versions(self): metadata = DistributionMetadata() diff --git a/src/distutils2/tests/test_msvc9compiler.py b/src/distutils2/tests/test_msvc9compiler.py index 452c3f5..eea83fa 100644 --- a/src/distutils2/tests/test_msvc9compiler.py +++ b/src/distutils2/tests/test_msvc9compiler.py @@ -64,7 +64,7 @@ _CLEANED_MANIFEST = """\ class msvc9compilerTestCase(support.TempdirManager, unittest.TestCase): - @unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") def test_no_compiler(self): # makes sure query_vcvarsall throws # a DistutilsPlatformError if the compiler @@ -86,7 +86,7 @@ class msvc9compilerTestCase(support.TempdirManager, finally: msvc9compiler.find_vcvarsall = old_find_vcvarsall - @unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") def test_reg_class(self): from distutils2.msvccompiler import get_build_version if get_build_version() < 8.0: @@ -110,7 +110,7 @@ class msvc9compilerTestCase(support.TempdirManager, keys = Reg.read_keys(HKCU, r'Control Panel') self.assertTrue('Desktop' in keys) - @unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") def test_remove_visual_c_ref(self): from distutils2.msvc9compiler import MSVCCompiler tempdir = self.mkdtemp() diff --git a/src/distutils2/tests/test_pypi_dist.py b/src/distutils2/tests/test_pypi_dist.py deleted file mode 100644 index 0267760..0000000 --- a/src/distutils2/tests/test_pypi_dist.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Tests for the distutils2.pypi.dist module.""" - -import os -import shutil -import tempfile - -from distutils2.tests.pypi_server import use_pypi_server -from distutils2.tests import run_unittest -from distutils2.tests.support import unittest, TempdirManager -from distutils2.version import VersionPredicate -from distutils2.pypi.errors import HashDoesNotMatch, UnsupportedHashName -from distutils2.pypi.dist import (PyPIDistribution as Dist, - PyPIDistributions as Dists, - split_archive_name) - - -class TestPyPIDistribution(TempdirManager, - unittest.TestCase): - """Tests the pypi.dist.PyPIDistribution class""" - - def test_instanciation(self): - # Test the Distribution class provides us the good attributes when - # given on construction - dist = Dist("FooBar", "1.1") - self.assertEqual("FooBar", dist.name) - self.assertEqual("1.1", "%s" % dist.version) - - def test_create_from_url(self): - # Test that the Distribution object can be built from a single URL - url_list = { - 'FooBar-1.1.0.tar.gz': { - 'name': 'foobar', # lowercase the name - 'version': '1.1', - }, - 'Foo-Bar-1.1.0.zip': { - 'name': 'foo-bar', # keep the dash - 'version': '1.1', - }, - 'foobar-1.1b2.tar.gz#md5=123123123123123': { - 'name': 'foobar', - 'version': '1.1b2', - 'url': { - 'url': 'http://test.tld/foobar-1.1b2.tar.gz', # no hash - 'hashval': '123123123123123', - 'hashname': 'md5', - } - }, - 'foobar-1.1-rc2.tar.gz': { # use suggested name - 'name': 'foobar', - 'version': '1.1c2', - 'url': { - 'url': 'http://test.tld/foobar-1.1-rc2.tar.gz', - } - } - } - - for url, attributes in url_list.items(): - dist = Dist.from_url("http://test.tld/" + url) - for attribute, value in attributes.items(): - if isinstance(value, dict): - mylist = getattr(dist, attribute) - for val in value.keys(): - self.assertEqual(value[val], mylist[val]) - else: - if attribute == "version": - self.assertEqual("%s" % getattr(dist, "version"), value) - else: - self.assertEqual(getattr(dist, attribute), value) - - def test_get_url(self): - # Test that the url property works well - - d = Dist("foobar", "1.1", url="test_url") - self.assertDictEqual(d.url, { - "url": "test_url", - "is_external": True, - "hashname": None, - "hashval": None, - }) - - # add a new url - d.add_url(url="internal_url", is_external=False) - self.assertEqual(d._url, None) - self.assertDictEqual(d.url, { - "url": "internal_url", - "is_external": False, - "hashname": None, - "hashval": None, - }) - self.assertEqual(2, len(d._urls)) - - def test_comparaison(self): - # Test that we can compare PyPIDistributions - foo1 = Dist("foo", "1.0") - foo2 = Dist("foo", "2.0") - bar = Dist("bar", "2.0") - # assert we use the version to compare - self.assertTrue(foo1 < foo2) - self.assertFalse(foo1 > foo2) - self.assertFalse(foo1 == foo2) - - # assert we can't compare dists with different names - self.assertRaises(TypeError, foo1.__eq__, bar) - - def test_split_archive_name(self): - # Test we can split the archive names - names = { - 'foo-bar-baz-1.0-rc2': ('foo-bar-baz', '1.0c2'), - 'foo-bar-baz-1.0': ('foo-bar-baz', '1.0'), - 'foobarbaz-1.0': ('foobarbaz', '1.0'), - } - for name, results in names.items(): - self.assertEqual(results, split_archive_name(name)) - - @use_pypi_server("downloads_with_md5") - def test_download(self, server): - # Download is possible, and the md5 is checked if given - - add_to_tmpdirs = lambda x: self.tempdirs.append(os.path.dirname(x)) - - url = "%s/simple/foobar/foobar-0.1.tar.gz" % server.full_address - # check md5 if given - dist = Dist("FooBar", "0.1", url=url, - url_hashname="md5", url_hashval="d41d8cd98f00b204e9800998ecf8427e") - add_to_tmpdirs(dist.download()) - - # a wrong md5 fails - dist2 = Dist("FooBar", "0.1", url=url, - url_hashname="md5", url_hashval="wrongmd5") - - self.assertRaises(HashDoesNotMatch, dist2.download) - add_to_tmpdirs(dist2.downloaded_location) - - # we can omit the md5 hash - dist3 = Dist("FooBar", "0.1", url=url) - add_to_tmpdirs(dist3.download()) - - # and specify a temporary location - # for an already downloaded dist - path1 = self.mkdtemp() - dist3.download(path=path1) - # and for a new one - path2_base = self.mkdtemp() - dist4 = Dist("FooBar", "0.1", url=url) - path2 = dist4.download(path=path2_base) - self.assertTrue(path2_base in path2) - - def test_hashname(self): - # Invalid hashnames raises an exception on assignation - Dist("FooBar", "0.1", url_hashname="md5", url_hashval="value") - - self.assertRaises(UnsupportedHashName, Dist, "FooBar", "0.1", - url_hashname="invalid_hashname", url_hashval="value") - - -class TestPyPIDistributions(unittest.TestCase): - - def test_filter(self): - # Test we filter the distributions the right way, using version - # predicate match method - dists = Dists(( - Dist("FooBar", "1.1"), - Dist("FooBar", "1.1.1"), - Dist("FooBar", "1.2"), - Dist("FooBar", "1.2.1"), - )) - filtered = dists.filter(VersionPredicate("FooBar (<1.2)")) - self.assertNotIn(dists[2], filtered) - self.assertNotIn(dists[3], filtered) - self.assertIn(dists[0], filtered) - self.assertIn(dists[1], filtered) - - def test_append(self): - # When adding a new item to the list, the behavior is to test if - # a distribution with the same name and version number already exists, - # and if so, to add url informations to the existing PyPIDistribution - # object. - # If no object matches, just add "normally" the object to the list. - - dists = Dists([ - Dist("FooBar", "1.1", url="external_url", type="source"), - ]) - self.assertEqual(1, len(dists)) - dists.append(Dist("FooBar", "1.1", url="internal_url", - url_is_external=False, type="source")) - self.assertEqual(1, len(dists)) - self.assertEqual(2, len(dists[0]._urls)) - - dists.append(Dist("Foobar", "1.1.1", type="source")) - self.assertEqual(2, len(dists)) - - # when adding a distribution whith a different type, a new distribution - # has to be added. - dists.append(Dist("Foobar", "1.1.1", type="binary")) - self.assertEqual(3, len(dists)) - - def test_prefer_final(self): - # Can order the distributions using prefer_final - - fb10 = Dist("FooBar", "1.0") # final distribution - fb11a = Dist("FooBar", "1.1a1") # alpha - fb12a = Dist("FooBar", "1.2a1") # alpha - fb12b = Dist("FooBar", "1.2b1") # beta - dists = Dists([fb10, fb11a, fb12a, fb12b]) - - dists.sort_distributions(prefer_final=True) - self.assertEqual(fb10, dists[0]) - - dists.sort_distributions(prefer_final=False) - self.assertEqual(fb12b, dists[0]) - - def test_prefer_source(self): - # Ordering support prefer_source - fb_source = Dist("FooBar", "1.0", type="source") - fb_binary = Dist("FooBar", "1.0", type="binary") - fb2_binary = Dist("FooBar", "2.0", type="binary") - dists = Dists([fb_binary, fb_source]) - - dists.sort_distributions(prefer_source=True) - self.assertEqual(fb_source, dists[0]) - - dists.sort_distributions(prefer_source=False) - self.assertEqual(fb_binary, dists[0]) - - dists.append(fb2_binary) - dists.sort_distributions(prefer_source=True) - self.assertEqual(fb2_binary, dists[0]) - - def test_get_same_name_and_version(self): - # PyPIDistributions can return a list of "duplicates" - fb_source = Dist("FooBar", "1.0", type="source") - fb_binary = Dist("FooBar", "1.0", type="binary") - fb2_binary = Dist("FooBar", "2.0", type="binary") - dists = Dists([fb_binary, fb_source, fb2_binary]) - duplicates = dists.get_same_name_and_version() - self.assertTrue(1, len(duplicates)) - self.assertIn(fb_source, duplicates[0]) - - -def test_suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(TestPyPIDistribution)) - suite.addTest(unittest.makeSuite(TestPyPIDistributions)) - return suite - -if __name__ == '__main__': - run_unittest(test_suite()) diff --git a/src/distutils2/tests/test_pypi_versions.py b/src/distutils2/tests/test_pypi_versions.py index 7781982..bbb9395 100644 --- a/src/distutils2/tests/test_pypi_versions.py +++ b/src/distutils2/tests/test_pypi_versions.py @@ -1,43 +1,39 @@ -# -## test_pypi_versions.py -## -## A very simple test to see what percentage of the current pypi packages -## have versions that can be converted automatically by distutils' new -## suggest_normalized_version() into PEP-386 compatible versions. -## -## Requires : Python 2.5+ -## -## Written by: ssteinerX@gmail.com -# +"""PEP 386 compatibility test with current distributions on PyPI. + +A very simple test to see what percentage of the current PyPI packages +have versions that can be converted automatically by distutils2's new +suggest_normalized_version into PEP 386-compatible versions. +""" + +# XXX This file does not actually run tests, move it to a script + +# Written by ssteinerX@gmail.com + +import os +import xmlrpclib try: import cPickle as pickle -except: +except ImportError: import pickle -import xmlrpclib -import os.path - from distutils2.version import suggest_normalized_version from distutils2.tests import run_unittest from distutils2.tests.support import unittest def test_pypi(): - # - ## To re-run from scratch, just delete these two .pkl files - # + # FIXME need a better way to do that + # To re-run from scratch, just delete these two .pkl files INDEX_PICKLE_FILE = 'pypi-index.pkl' VERSION_PICKLE_FILE = 'pypi-version.pkl' package_info = version_info = [] - # - ## if there's a saved version of the package list - ## restore it - ## else: - ## pull the list down from pypi - ## save a pickled version of it - # + # if there's a saved version of the package list + # restore it + # else: + # pull the list down from pypi + # save a pickled version of it if os.path.exists(INDEX_PICKLE_FILE): print "Loading saved pypi data..." f = open(INDEX_PICKLE_FILE, 'rb') @@ -57,13 +53,11 @@ def test_pypi(): finally: f.close() - # - ## If there's a saved list of the versions from the packages - ## restore it - ## else - ## extract versions from the package list - ## save a pickled version of it - # + # If there's a saved list of the versions from the packages + # restore it + # else + # extract versions from the package list + # save a pickled version of it versions = [] if os.path.exists(VERSION_PICKLE_FILE): print "Loading saved version info..." diff --git a/src/distutils2/tests/test_register.py b/src/distutils2/tests/test_register.py index 3468fa4..5081f49 100644 --- a/src/distutils2/tests/test_register.py +++ b/src/distutils2/tests/test_register.py @@ -161,7 +161,7 @@ class RegisterTestCase(support.TempdirManager, support.EnvironGuard, # therefore used afterwards by other commands self.assertEqual(cmd.distribution.password, 'password') - def test_registering(self): + def test_registration(self): # this test runs choice 2 cmd = self._get_cmd() inputs = RawInputs('2', 'tarek', 'tarek@ziade.org') @@ -210,7 +210,7 @@ class RegisterTestCase(support.TempdirManager, support.EnvironGuard, cmd.strict = 1 self.assertRaises(DistutilsSetupError, cmd.run) - # metadata are OK but long_description is broken + # metadata is OK but long_description is broken metadata = {'home_page': 'xxx', 'author': 'xxx', 'author_email': u'éxéxé', 'name': 'xxx', 'version': 'xxx', diff --git a/src/distutils2/tests/test_sdist.py b/src/distutils2/tests/test_sdist.py index cf60490..c974175 100644 --- a/src/distutils2/tests/test_sdist.py +++ b/src/distutils2/tests/test_sdist.py @@ -241,7 +241,7 @@ class SDistTestCase(support.TempdirManager, support.LoggingSilencer, @unittest.skipUnless(zlib, "requires zlib") def test_metadata_check_option(self): - # testing the `medata-check` option + # testing the `check-metadata` option dist, cmd = self.get_cmd(metadata={}) # this should raise some warnings ! @@ -295,7 +295,7 @@ class SDistTestCase(support.TempdirManager, support.LoggingSilencer, self.assertRaises(DistutilsOptionError, cmd.finalize_options) @unittest.skipUnless(zlib, "requires zlib") - @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") + @unittest.skipUnless(UID_GID_SUPPORT, "requires grp and pwd support") def test_make_distribution_owner_group(self): # check if tar and gzip are installed diff --git a/src/distutils2/tests/test_upload.py b/src/distutils2/tests/test_upload.py index f40e6d9..3959872 100644 --- a/src/distutils2/tests/test_upload.py +++ b/src/distutils2/tests/test_upload.py @@ -101,6 +101,38 @@ class UploadTestCase(support.TempdirManager, support.EnvironGuard, self.assertEqual(handler.command, 'POST') self.assertNotIn('\n', headers['authorization']) + def test_upload_docs(self): + path = os.path.join(self.tmp_dir, 'xxx') + self.write_file(path) + command, pyversion, filename = 'xxx', '2.6', path + dist_files = [(command, pyversion, filename)] + docs_path = os.path.join(self.tmp_dir, "build", "docs") + os.makedirs(docs_path) + self.write_file(os.path.join(docs_path, "index.html"), "yellow") + self.write_file(self.rc, PYPIRC) + + # lets run it + pkg_dir, dist = self.create_dist(dist_files=dist_files, author=u'dédé') + + cmd = upload(dist) + cmd.get_finalized_command("build").run() + cmd.upload_docs = True + cmd.ensure_finalized() + cmd.repository = self.pypi.full_address + try: + prev_dir = os.getcwd() + os.chdir(self.tmp_dir) + cmd.run() + finally: + os.chdir(prev_dir) + + handler, request_data = self.pypi.requests[-1] + action, name, content =\ + request_data.split("----------------GHSKFJDLGDS7543FJKLFHRE75642756743254")[1:4] + + self.assertIn('name=":action"', action) + self.assertIn("doc_upload", action) + def test_suite(): return unittest.makeSuite(UploadTestCase) diff --git a/src/distutils2/tests/test_upload_docs.py b/src/distutils2/tests/test_upload_docs.py index 862b894..c90f6b4 100644 --- a/src/distutils2/tests/test_upload_docs.py +++ b/src/distutils2/tests/test_upload_docs.py @@ -1,13 +1,19 @@ -"""Tests for distutils.command.upload_docs.""" # -*- encoding: utf8 -*- -import httplib, os, os.path, shutil, sys, tempfile, zipfile -from cStringIO import StringIO +"""Tests for distutils.command.upload_docs.""" +import os +import sys +import httplib +import shutil +import zipfile +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO from distutils2.command import upload_docs as upload_docs_mod from distutils2.command.upload_docs import (upload_docs, zip_dir, - encode_multipart) + encode_multipart) from distutils2.core import Distribution - from distutils2.errors import DistutilsFileError, DistutilsOptionError from distutils2.tests import support @@ -59,7 +65,7 @@ class UploadDocsTestCase(support.TempdirManager, support.EnvironGuard, self.cmd = upload_docs(self.dist) def test_default_uploaddir(self): - sandbox = tempfile.mkdtemp() + sandbox = self.mkdtemp() previous = os.getcwd() os.chdir(sandbox) try: @@ -72,7 +78,7 @@ class UploadDocsTestCase(support.TempdirManager, support.EnvironGuard, def prepare_sample_dir(self, sample_dir=None): if sample_dir is None: - sample_dir = tempfile.mkdtemp() + sample_dir = self.mkdtemp() os.mkdir(os.path.join(sample_dir, "docs")) self.write_file(os.path.join(sample_dir, "docs", "index.html"), "Ce mortel ennui") self.write_file(os.path.join(sample_dir, "index.html"), "Oh la la") diff --git a/src/distutils2/tests/test_util.py b/src/distutils2/tests/test_util.py index 8377432..26f4ee1 100644 --- a/src/distutils2/tests/test_util.py +++ b/src/distutils2/tests/test_util.py @@ -4,7 +4,6 @@ import sys from copy import copy from StringIO import StringIO import subprocess -import tempfile import time from distutils2.tests import captured_stdout @@ -19,7 +18,7 @@ from distutils2.util import (convert_path, change_root, _find_exe_version, _MAC_OS_X_LD_VERSION, byte_compile, find_packages, spawn, find_executable, _nt_quote_args, get_pypirc_path, generate_pypirc, - read_pypirc) + read_pypirc, resolve_dotted_name) from distutils2 import util from distutils2.tests import support @@ -288,7 +287,7 @@ class UtilTestCase(support.EnvironGuard, self.assertEqual(res[2], None) @unittest.skipUnless(hasattr(sys, 'dont_write_bytecode'), - 'no dont_write_bytecode support') + 'sys.dont_write_bytecode not supported') def test_dont_write_bytecode(self): # makes sure byte_compile raise a DistutilsError # if sys.dont_write_bytecode is True @@ -301,9 +300,9 @@ class UtilTestCase(support.EnvironGuard, def test_newer(self): self.assertRaises(DistutilsFileError, util.newer, 'xxx', 'xxx') - self.newer_f1 = tempfile.NamedTemporaryFile() + self.newer_f1 = self.mktempfile() time.sleep(1) - self.newer_f2 = tempfile.NamedTemporaryFile() + self.newer_f2 = self.mktempfile() self.assertTrue(util.newer(self.newer_f2.name, self.newer_f1.name)) def test_find_packages(self): @@ -343,7 +342,17 @@ class UtilTestCase(support.EnvironGuard, res = find_packages([root], ['pkg1.pkg2']) self.assertEqual(set(res), set(['pkg1', 'pkg5', 'pkg1.pkg3', 'pkg1.pkg3.pkg6'])) - @unittest.skipUnless(sys.version > '2.6', 'Need Python 2.6 or more') + def test_resolve_dotted_name(self): + self.assertEqual(UtilTestCase, resolve_dotted_name("distutils2.tests.test_util.UtilTestCase")) + self.assertEqual(UtilTestCase.test_resolve_dotted_name, + resolve_dotted_name("distutils2.tests.test_util.UtilTestCase.test_resolve_dotted_name")) + + self.assertRaises(ImportError, resolve_dotted_name, + "distutils2.tests.test_util.UtilTestCaseNot") + self.assertRaises(ImportError, resolve_dotted_name, + "distutils2.tests.test_util.UtilTestCase.nonexistent_attribute") + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_run_2to3_on_code(self): content = "print 'test'" converted_content = "print('test')" @@ -358,7 +367,7 @@ class UtilTestCase(support.EnvironGuard, file_handle.close() self.assertEquals(new_content, converted_content) - @unittest.skipUnless(sys.version > '2.6', 'Need Python 2.6 or more') + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_run_2to3_on_doctests(self): # to check if text files containing doctests only get converted. content = ">>> print 'test'\ntest\n" @@ -385,7 +394,7 @@ class UtilTestCase(support.EnvironGuard, @unittest.skipUnless(os.name in ('nt', 'posix'), - 'Runs only under posix or nt') + 'runs only under posix or nt') def test_spawn(self): tmpdir = self.mkdtemp() diff --git a/src/distutils2/tests/test_version.py b/src/distutils2/tests/test_version.py index 343b397..1e392f2 100644 --- a/src/distutils2/tests/test_version.py +++ b/src/distutils2/tests/test_version.py @@ -188,6 +188,13 @@ class VersionTestCase(unittest.TestCase): # XXX need to silent the micro version in this case #assert not VersionPredicate('Ho (<3.0,!=2.6)').match('2.6.3') + def test_predicate_name(self): + # Test that names are parsed the right way + + self.assertEqual('Hey', VersionPredicate('Hey (<1.1)').name) + self.assertEqual('Foo-Bar', VersionPredicate('Foo-Bar (1.1)').name) + self.assertEqual('Foo Bar', VersionPredicate('Foo Bar (1.1)').name) + def test_is_final(self): # VersionPredicate knows is a distribution is a final one or not. final_versions = ('1.0', '1.0.post456') diff --git a/src/distutils2/util.py b/src/distutils2/util.py index 6f4905b..178cb95 100644 --- a/src/distutils2/util.py +++ b/src/distutils2/util.py @@ -5,10 +5,14 @@ Miscellaneous utility functions. __revision__ = "$Id: util.py 77761 2010-01-26 22:46:15Z tarek.ziade $" -import sys import os -import string +import posixpath import re +import string +import sys +import shutil +import tarfile +import zipfile from copy import copy from fnmatch import fnmatchcase from ConfigParser import RawConfigParser @@ -84,8 +88,8 @@ def convert_path(pathname): raise ValueError("path '%s' cannot end with '/'" % pathname) paths = pathname.split('/') - while '.' in paths: - paths.remove('.') + while os.curdir in paths: + paths.remove(os.curdir) if not paths: return os.curdir return os.path.join(*paths) @@ -117,15 +121,6 @@ def change_root(new_root, pathname): path = path[1:] return os.path.join(new_root, path) - elif os.name == 'mac': - if not os.path.isabs(pathname): - return os.path.join(new_root, pathname) - else: - # Chop off volume name from start of path - elements = pathname.split(":", 1) - pathname = ":" + elements[1] - return os.path.join(new_root, pathname) - else: raise DistutilsPlatformError("nothing known about " "platform '%s'" % os.name) @@ -595,7 +590,7 @@ def _package_name(root_path, path): return path[len(root_path) + 1:].replace(os.sep, '.') -def find_packages(paths=('.',), exclude=()): +def find_packages(paths=(os.curdir,), exclude=()): """Return a list all Python packages found recursively within directories 'paths' @@ -642,6 +637,24 @@ def find_packages(paths=('.',), exclude=()): return packages +def resolve_dotted_name(dotted_name): + module_name, rest = dotted_name.split('.')[0], dotted_name.split('.')[1:] + while len(rest) > 0: + try: + ret = __import__(module_name) + break + except ImportError: + if rest == []: + raise + module_name += ('.' + rest[0]) + rest = rest[1:] + while len(rest) > 0: + try: + ret = getattr(ret, rest.pop(0)) + except AttributeError: + raise ImportError + return ret + # utility functions for 2to3 support def run_2to3(files, doctests_only=False, fixer_names=None, options=None, @@ -690,6 +703,119 @@ class Mixin2to3: self.options, self.explicit) +def splitext(path): + """Like os.path.splitext, but take off .tar too""" + base, ext = posixpath.splitext(path) + if base.lower().endswith('.tar'): + ext = base[-4:] + ext + base = base[:-4] + return base, ext + + +def unzip_file(filename, location, flatten=True): + """Unzip the file (zip file located at filename) to the destination + location""" + if not os.path.exists(location): + os.makedirs(location) + zipfp = open(filename, 'rb') + try: + zip = zipfile.ZipFile(zipfp) + leading = has_leading_dir(zip.namelist()) and flatten + for name in zip.namelist(): + data = zip.read(name) + fn = name + if leading: + fn = split_leading_dir(name)[1] + fn = os.path.join(location, fn) + dir = os.path.dirname(fn) + if not os.path.exists(dir): + os.makedirs(dir) + if fn.endswith('/') or fn.endswith('\\'): + # A directory + if not os.path.exists(fn): + os.makedirs(fn) + else: + fp = open(fn, 'wb') + try: + fp.write(data) + finally: + fp.close() + finally: + zipfp.close() + + +def untar_file(filename, location): + """Untar the file (tar file located at filename) to the destination + location + """ + if not os.path.exists(location): + os.makedirs(location) + if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): + mode = 'r:gz' + elif (filename.lower().endswith('.bz2') + or filename.lower().endswith('.tbz')): + mode = 'r:bz2' + elif filename.lower().endswith('.tar'): + mode = 'r' + else: + mode = 'r:*' + tar = tarfile.open(filename, mode) + try: + leading = has_leading_dir([member.name for member in tar.getmembers()]) + for member in tar.getmembers(): + fn = member.name + if leading: + fn = split_leading_dir(fn)[1] + path = os.path.join(location, fn) + if member.isdir(): + if not os.path.exists(path): + os.makedirs(path) + else: + try: + fp = tar.extractfile(member) + except (KeyError, AttributeError), e: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + continue + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + destfp = open(path, 'wb') + try: + shutil.copyfileobj(fp, destfp) + finally: + destfp.close() + fp.close() + finally: + tar.close() + + +def has_leading_dir(paths): + """Returns true if all the paths have the same leading path name + (i.e., everything is in one subdirectory in an archive)""" + common_prefix = None + for path in paths: + prefix, rest = split_leading_dir(path) + if not prefix: + return False + elif common_prefix is None: + common_prefix = prefix + elif prefix != common_prefix: + return False + return True + + +def split_leading_dir(path): + path = str(path) + path = path.lstrip('/').lstrip('\\') + if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) + or '\\' not in path): + return path.split('/', 1) + elif '\\' in path: + return path.split('\\', 1) + else: + return path, '' + + def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None): """Run another program specified as a command list 'cmd' in a new process. diff --git a/src/distutils2/version.py b/src/distutils2/version.py index e61c60e..4034e24 100644 --- a/src/distutils2/version.py +++ b/src/distutils2/version.py @@ -321,7 +321,7 @@ def suggest_normalized_version(s): return None -_PREDICATE = re.compile(r"(?i)^\s*([a-z_]\w*(?:\.[a-z_]\w*)*)(.*)") +_PREDICATE = re.compile(r"(?i)^\s*([a-z_][\sa-zA-Z_-]*(?:\.[a-z_]\w*)*)(.*)") _VERSIONS = re.compile(r"^\s*\((.*)\)\s*$") _PLAIN_VERSIONS = re.compile(r"^\s*(.*)\s*$") _SPLIT_CMP = re.compile(r"^\s*(<=|>=|<|>|!=|==)\s*([^\s,]+)\s*$") @@ -354,7 +354,8 @@ class VersionPredicate(object): if match is None: raise ValueError('Bad predicate "%s"' % predicate) - self.name, predicates = match.groups() + name, predicates = match.groups() + self.name = name.strip() predicates = predicates.strip() predicates = _VERSIONS.match(predicates) if predicates is not None: diff --git a/src/runtests-cov.py b/src/runtests-cov.py index 6c2ab41..641bbb1 100755 --- a/src/runtests-cov.py +++ b/src/runtests-cov.py @@ -8,6 +8,15 @@ import sys from os.path import dirname, islink, realpath from optparse import OptionParser +def get_coverage(): + """ Return a usable coverage object. """ + # deferred import because coverage is optional + import coverage + cov = getattr(coverage, "the_coverage", None) + if not cov: + cov = coverage.coverage() + return cov + def ignore_prefixes(module): """ Return a list of prefixes to ignore in the coverage report if we want to completely skip `module`. @@ -16,16 +25,17 @@ def ignore_prefixes(module): # distributions, such a Ubuntu, really like to build link farm in # /usr/lib in order to save a few bytes on the disk. dirnames = [dirname(module.__file__)] - + pymod = module.__file__.rstrip("c") if islink(pymod): dirnames.append(dirname(realpath(pymod))) return dirnames + def parse_opts(): parser = OptionParser(usage="%prog [OPTIONS]", description="run the distutils2 unittests") - + parser.add_option("-q", "--quiet", help="do not print verbose messages", action="store_true", default=False) parser.add_option("-c", "--coverage", action="store_true", default=False, @@ -36,15 +46,22 @@ def parse_opts(): default=False, help=("Show line numbers of statements in each module " "that weren't executed.")) - + opts, args = parser.parse_args() return opts, args + def coverage_report(opts): - import coverage from distutils2.tests.support import unittest - cov = coverage.coverage() - cov.load() + cov = get_coverage() + if hasattr(cov, "load"): + # running coverage 3.x + cov.load() + morfs = None + else: + morfs = "distutils2/*" + # running coverage 2.x + cov.restore() prefixes = ["runtests", "distutils2/tests", "distutils2/_backport"] prefixes += ignore_prefixes(unittest) @@ -63,7 +80,7 @@ def coverage_report(opts): # that module is also completely optional pass - cov.report(omit_prefixes=prefixes, show_missing=opts.show_missing) + cov.report(morfs, omit_prefixes=prefixes, show_missing=opts.show_missing) def test_main(): @@ -71,11 +88,8 @@ def test_main(): verbose = not opts.quiet ret = 0 - if opts.coverage or opts.report: - import coverage - if opts.coverage: - cov = coverage.coverage() + cov = get_coverage() cov.erase() cov.start() if not opts.report: @@ -89,6 +103,7 @@ def test_main(): return ret + def run_tests(verbose): import distutils2.tests from distutils2.tests import run_unittest, reap_children, TestFailed @@ -108,12 +123,11 @@ def run_tests(verbose): finally: reap_children() + if __name__ == "__main__": try: from distutils2.tests.support import unittest except ImportError: sys.stderr.write('Error: You have to install unittest2') sys.exit(1) - sys.exit(test_main()) - diff --git a/src/runtests.py b/src/runtests.py index b9e2477..415d37a 100644 --- a/src/runtests.py +++ b/src/runtests.py @@ -1,9 +1,12 @@ +#!/usr/bin/env python """Tests for distutils2. The tests for distutils2 are defined in the distutils2.tests package. """ + import sys + def test_main(): import distutils2.tests from distutils2.tests import run_unittest, reap_children, TestFailed @@ -23,12 +26,11 @@ def test_main(): finally: reap_children() + if __name__ == "__main__": try: from distutils2.tests.support import unittest except ImportError: sys.stderr.write('Error: You have to install unittest2') sys.exit(1) - sys.exit(test_main()) - diff --git a/src/setup.cfg b/src/setup.cfg new file mode 100644 index 0000000..169aaf6 --- /dev/null +++ b/src/setup.cfg @@ -0,0 +1,3 @@ +[build_ext] +# needed so that tests work without mucking with sys.path +inplace = 1 diff --git a/src/setup.py b/src/setup.py index 5345b37..1233b21 100644 --- a/src/setup.py +++ b/src/setup.py @@ -168,29 +168,25 @@ def prepare_hashlib_extensions(): # The _hashlib module wraps optimized implementations # of hash functions from the OpenSSL library. - exts.append(Extension('_hashlib', ['_hashopenssl.c'], + exts.append(Extension('distutils2._backport._hashlib', + ['distutils2/_backport/_hashopenssl.c'], include_dirs = [ssl_inc_dir], library_dirs = [os.path.dirname(ssl_lib)], libraries = oslibs[os.name])) else: - exts.append(Extension('_sha', ['shamodule.c']) ) - exts.append(Extension('_md5', - sources=['md5module.c', 'md5.c'], - depends=['md5.h']) ) + exts.append(Extension('distutils2._backport._sha', + ['distutils2/_backport/shamodule.c'])) + exts.append(Extension('distutils2._backport._md5', + sources=['distutils2/_backport/md5module.c', + 'distutils2/_backport/md5.c'], + depends=['distutils2/_backport/md5.h']) ) if (not ssl_lib or openssl_ver < 0x00908000): # OpenSSL doesn't do these until 0.9.8 so we'll bring our own - exts.append(Extension('_sha256', ['sha256module.c'])) - exts.append(Extension('_sha512', ['sha512module.c'])) - - def prepend_modules(filename): - return os.path.join('Modules', filename) - - # all the C code is in the Modules subdirectory, prepend the path - for ext in exts: - ext.sources = [prepend_modules(fn) for fn in ext.sources] - if hasattr(ext, 'depends') and ext.depends is not None: - ext.depends = [prepend_modules(fn) for fn in ext.depends] + exts.append(Extension('distutils2._backport._sha256', + ['distutils2/_backport/sha256module.c'])) + exts.append(Extension('distutils2._backport._sha512', + ['distutils2/_backport/sha512module.c'])) return exts diff --git a/src/tests.sh b/src/tests.sh index fae8fb0..0930d0e 100755 --- a/src/tests.sh +++ b/src/tests.sh @@ -1,20 +1,18 @@ #!/bin/sh echo -n "Running tests for Python 2.4... " -rm -rf *.so -python2.4 setup.py build_ext -i -q 2> /dev/null > /dev/null +rm -f distutils2/_backport/_hashlib.so +python2.4 setup.py build_ext -f -q 2> /dev/null > /dev/null python2.4 -Wd runtests.py -q 2> /dev/null -rm -rf *.so if [ $? -ne 0 ];then echo "Failed" + rm -f distutils2/_backport/_hashlib.so exit 1 else echo "Success" fi echo -n "Running tests for Python 2.5... " -python2.5 setup.py build_ext -i -q 2> /dev/null > /dev/null python2.5 -Wd runtests.py -q 2> /dev/null -rm -rf *.so if [ $? -ne 0 ];then echo "Failed" exit 1 @@ -23,7 +21,7 @@ else fi echo -n "Running tests for Python 2.6... " -python2.6 -Wd -bb -3 runtests.py -q 2> /dev/null +python2.6 -Wd runtests.py -q 2> /dev/null if [ $? -ne 0 ];then echo "Failed" exit 1 |