From 118edbb2b715c96620b51018c1d28e81f2318053 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Thu, 2 Nov 2017 21:19:39 +0100 Subject: easy_install: add support for installing from wheels Note: wheels are installed as eggs, so each install is self-contained and multiple versions of the same package can be installed at the same time. Limitations: - headers are not supported - resulting egg metadata requirements have their markers stripped --- setuptools/wheel.py | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 setuptools/wheel.py (limited to 'setuptools/wheel.py') diff --git a/setuptools/wheel.py b/setuptools/wheel.py new file mode 100644 index 00000000..6e3df77c --- /dev/null +++ b/setuptools/wheel.py @@ -0,0 +1,125 @@ +'''Wheels support.''' + +from distutils.util import get_platform +import email +import itertools +import os +import re +import zipfile + +from pkg_resources import Distribution, PathMetadata, parse_version +from setuptools import Distribution as SetuptoolsDistribution +from setuptools import pep425tags +from setuptools.command.egg_info import write_requirements + + +WHEEL_NAME = re.compile( + r"""^(?P.+?)-(?P\d.*?) + ((-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?) + )\.whl$""", +re.VERBOSE).match + + +class Wheel(object): + + def __init__(self, filename): + match = WHEEL_NAME(os.path.basename(filename)) + if match is None: + raise ValueError('invalid wheel name: %r' % filename) + self.filename = filename + for k, v in match.groupdict().items(): + setattr(self, k, v) + + def tags(self): + '''List tags (py_version, abi, platform) supported by this wheel.''' + return itertools.product(self.py_version.split('.'), + self.abi.split('.'), + self.platform.split('.')) + + def is_compatible(self): + '''Is the wheel is compatible with the current platform?''' + supported_tags = pep425tags.get_supported() + return next((True for t in self.tags() if t in supported_tags), False) + + def egg_name(self): + return Distribution( + project_name=self.project_name, version=self.version, + platform=(None if self.platform == 'any' else get_platform()), + ).egg_name() + '.egg' + + def install_as_egg(self, destination_eggdir): + '''Install wheel as an egg directory.''' + with zipfile.ZipFile(self.filename) as zf: + dist_basename = '%s-%s' % (self.project_name, self.version) + dist_info = '%s.dist-info' % dist_basename + dist_data = '%s.data' % dist_basename + def get_metadata(name): + with zf.open('%s/%s' % (dist_info, name)) as fp: + value = fp.read().decode('utf-8') + return email.parser.Parser().parsestr(value) + wheel_metadata = get_metadata('WHEEL') + dist_metadata = get_metadata('METADATA') + # Check wheel format version is supported. + wheel_version = parse_version(wheel_metadata.get('Wheel-Version')) + if not parse_version('1.0') <= wheel_version < parse_version('2.0dev0'): + raise ValueError('unsupported wheel format version: %s' % wheel_version) + # Extract to target directory. + os.mkdir(destination_eggdir) + zf.extractall(destination_eggdir) + # Convert metadata. + dist_info = os.path.join(destination_eggdir, dist_info) + dist = Distribution.from_location( + destination_eggdir, dist_info, + metadata=PathMetadata(destination_eggdir, dist_info) + ) + # Note: we need to evaluate and strip markers now, + # as we can't easily convert back from the syntax: + # foobar; "linux" in sys_platform and extra == 'test' + def raw_req(req): + req.marker = None + return str(req) + install_requires = list(sorted(map(raw_req, dist.requires()))) + extras_require = { + extra: list(sorted( + req + for req in map(raw_req, dist.requires((extra,))) + if req not in install_requires + )) + for extra in dist.extras + } + egg_info = os.path.join(destination_eggdir, 'EGG-INFO') + os.rename(dist_info, egg_info) + os.rename(os.path.join(egg_info, 'METADATA'), + os.path.join(egg_info, 'PKG-INFO')) + setup_dist = SetuptoolsDistribution(attrs=dict( + install_requires=install_requires, + extras_require=extras_require, + )) + write_requirements(setup_dist.get_command_obj('egg_info'), + None, os.path.join(egg_info, 'requires.txt')) + # Move data entries to their correct location. + dist_data = os.path.join(destination_eggdir, dist_data) + dist_data_scripts = os.path.join(dist_data, 'scripts') + if os.path.exists(dist_data_scripts): + egg_info_scripts = os.path.join(destination_eggdir, + 'EGG-INFO', 'scripts') + os.mkdir(egg_info_scripts) + for entry in os.listdir(dist_data_scripts): + # Remove bytecode, as it's not properly handled + # during easy_install scripts install phase. + if entry.endswith('.pyc'): + os.unlink(os.path.join(dist_data_scripts, entry)) + else: + os.rename(os.path.join(dist_data_scripts, entry), + os.path.join(egg_info_scripts, entry)) + os.rmdir(dist_data_scripts) + for subdir in filter(os.path.exists, ( + os.path.join(dist_data, d) + for d in ('data', 'headers', 'purelib', 'platlib') + )): + for entry in os.listdir(subdir): + os.rename(os.path.join(subdir, entry), + os.path.join(destination_eggdir, entry)) + os.rmdir(subdir) + if os.path.exists(dist_data): + os.rmdir(dist_data) -- cgit v1.2.1 From e72afd6243713cd0d3f8a5bc5b50fb59934d7ff8 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 26 Nov 2017 23:11:14 +0100 Subject: fix encoding handling of wheels metadata --- setuptools/wheel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/wheel.py') diff --git a/setuptools/wheel.py b/setuptools/wheel.py index 6e3df77c..f711f38b 100644 --- a/setuptools/wheel.py +++ b/setuptools/wheel.py @@ -8,6 +8,7 @@ import re import zipfile from pkg_resources import Distribution, PathMetadata, parse_version +from pkg_resources.extern.six import PY3 from setuptools import Distribution as SetuptoolsDistribution from setuptools import pep425tags from setuptools.command.egg_info import write_requirements @@ -55,7 +56,7 @@ class Wheel(object): dist_data = '%s.data' % dist_basename def get_metadata(name): with zf.open('%s/%s' % (dist_info, name)) as fp: - value = fp.read().decode('utf-8') + value = fp.read().decode('utf-8') if PY3 else fp.read() return email.parser.Parser().parsestr(value) wheel_metadata = get_metadata('WHEEL') dist_metadata = get_metadata('METADATA') -- cgit v1.2.1 From da1c78f354fac3ce177e2869828a34b3e6df1820 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Mon, 27 Nov 2017 13:25:04 +0100 Subject: fix namespace packages handling of wheels --- setuptools/wheel.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'setuptools/wheel.py') diff --git a/setuptools/wheel.py b/setuptools/wheel.py index f711f38b..c2327213 100644 --- a/setuptools/wheel.py +++ b/setuptools/wheel.py @@ -20,6 +20,13 @@ WHEEL_NAME = re.compile( )\.whl$""", re.VERBOSE).match +NAMESPACE_PACKAGE_INIT = '''\ +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + __path__ = __import__('pkgutil').extend_path(__path__, __name__) +''' + class Wheel(object): @@ -124,3 +131,14 @@ class Wheel(object): os.rmdir(subdir) if os.path.exists(dist_data): os.rmdir(dist_data) + # Fix namespace packages. + namespace_packages = os.path.join(egg_info, 'namespace_packages.txt') + if os.path.exists(namespace_packages): + with open(namespace_packages) as fp: + namespace_packages = fp.read().split() + for mod in namespace_packages: + mod_dir = os.path.join(destination_eggdir, *mod.split('.')) + mod_init = os.path.join(mod_dir, '__init__.py') + if os.path.exists(mod_dir) and not os.path.exists(mod_init): + with open(mod_init, 'w') as fp: + fp.write(NAMESPACE_PACKAGE_INIT) -- cgit v1.2.1 From 35c0e9c2d67cbda4033ba0e0b1c20e4df8c24ce7 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Thu, 30 Nov 2017 19:46:16 +0100 Subject: fix `data_files` handling when installing from wheel --- setuptools/wheel.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) (limited to 'setuptools/wheel.py') diff --git a/setuptools/wheel.py b/setuptools/wheel.py index c2327213..9ffe434a 100644 --- a/setuptools/wheel.py +++ b/setuptools/wheel.py @@ -28,6 +28,28 @@ except ImportError: ''' +def unpack(src_dir, dst_dir): + '''Move everything under `src_dir` to `dst_dir`, and delete the former.''' + for dirpath, dirnames, filenames in os.walk(src_dir): + subdir = os.path.relpath(dirpath, src_dir) + for f in filenames: + src = os.path.join(dirpath, f) + dst = os.path.join(dst_dir, subdir, f) + os.renames(src, dst) + for n, d in reversed(list(enumerate(dirnames))): + src = os.path.join(dirpath, d) + dst = os.path.join(dst_dir, subdir, d) + if not os.path.exists(dst): + # Directory does not exist in destination, + # rename it and prune it from os.walk list. + os.renames(src, dst) + del dirnames[n] + # Cleanup. + for dirpath, dirnames, filenames in os.walk(src_dir, topdown=True): + assert not filenames + os.rmdir(dirpath) + + class Wheel(object): def __init__(self, filename): @@ -125,10 +147,7 @@ class Wheel(object): os.path.join(dist_data, d) for d in ('data', 'headers', 'purelib', 'platlib') )): - for entry in os.listdir(subdir): - os.rename(os.path.join(subdir, entry), - os.path.join(destination_eggdir, entry)) - os.rmdir(subdir) + unpack(subdir, destination_eggdir) if os.path.exists(dist_data): os.rmdir(dist_data) # Fix namespace packages. -- cgit v1.2.1