diff options
Diffstat (limited to 'setuptools/command/easy_install.py')
| -rwxr-xr-x | setuptools/command/easy_install.py | 753 |
1 files changed, 478 insertions, 275 deletions
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index ad7f4725..68548272 100755 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -12,6 +12,14 @@ __ https://pythonhosted.org/setuptools/easy_install.html """ +from glob import glob +from distutils.util import get_platform +from distutils.util import convert_path, subst_vars +from distutils.errors import DistutilsArgError, DistutilsOptionError, \ + DistutilsError, DistutilsPlatformError +from distutils.command.install import INSTALL_SCHEMES, SCHEME_KEYS +from distutils import log, dir_util +from distutils.command.build_scripts import first_line_re import sys import os import zipimport @@ -26,21 +34,10 @@ import textwrap import warnings import site import struct -from glob import glob -from distutils import log, dir_util -from distutils.command.build_scripts import first_line_re - -import pkg_resources from setuptools import Command, _dont_write_bytecode from setuptools.sandbox import run_setup from setuptools.py31compat import get_path, get_config_vars - -from distutils.util import get_platform -from distutils.util import convert_path, subst_vars -from distutils.errors import DistutilsArgError, DistutilsOptionError, \ - DistutilsError, DistutilsPlatformError -from distutils.command.install import INSTALL_SCHEMES, SCHEME_KEYS from setuptools.command import setopt from setuptools.archive_util import unpack_archive from setuptools.package_index import PackageIndex @@ -54,18 +51,22 @@ from pkg_resources import ( Distribution, PathMetadata, EggMetadata, WorkingSet, DistributionNotFound, VersionConflict, DEVELOP_DIST, ) +import pkg_resources + sys_executable = os.environ.get('__PYVENV_LAUNCHER__', - os.path.normpath(sys.executable)) + os.path.normpath(sys.executable)) __all__ = [ 'samefile', 'easy_install', 'PthDistributions', 'extract_wininst_cfg', 'main', 'get_exe_prefixes', ] + def is_64bit(): return struct.calcsize("P") == 8 + def samefile(p1, p2): both_exist = os.path.exists(p1) and os.path.exists(p2) use_samefile = hasattr(os.path, 'samefile') and both_exist @@ -75,9 +76,11 @@ def samefile(p1, p2): norm_p2 = os.path.normpath(os.path.normcase(p2)) return norm_p1 == norm_p2 + if PY2: def _to_ascii(s): return s + def isascii(s): try: unicode(s, 'ascii') @@ -87,6 +90,7 @@ if PY2: else: def _to_ascii(s): return s.encode('ascii') + def isascii(s): try: s.encode('ascii') @@ -94,6 +98,7 @@ else: except UnicodeError: return False + class easy_install(Command): """Manage a download/build/install process""" description = "Find/get/install Python packages" @@ -111,22 +116,22 @@ class easy_install(Command): ("index-url=", "i", "base URL of Python Package Index"), ("find-links=", "f", "additional URL(s) to search for packages"), ("build-directory=", "b", - "download/extract/build in DIR; keep the results"), + "download/extract/build in DIR; keep the results"), ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), + "also compile with optimization: -O1 for \"python -O\", " + "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), ('record=', None, - "filename in which to record list of installed files"), + "filename in which to record list of installed files"), ('always-unzip', 'Z', "don't install as a zipfile, no matter what"), - ('site-dirs=','S',"list of directories where .pth files work"), + ('site-dirs=', 'S', "list of directories where .pth files work"), ('editable', 'e', "Install specified packages in editable form"), ('no-deps', 'N', "don't install dependencies"), ('allow-hosts=', 'H', "pattern(s) that hostnames must match"), ('local-snapshots-ok', 'l', - "allow building eggs from local checkouts"), + "allow building eggs from local checkouts"), ('version', None, "print version information and exit"), ('no-find-links', None, - "Don't load find-links defined in packages being installed") + "Don't load find-links defined in packages being installed") ] boolean_options = [ 'zip-ok', 'multi-version', 'exclude-scripts', 'upgrade', 'always-copy', @@ -160,10 +165,10 @@ class easy_install(Command): self.editable = self.no_deps = self.allow_hosts = None self.root = self.prefix = self.no_report = None self.version = None - self.install_purelib = None # for pure module distributions - self.install_platlib = None # non-pure (dists w/ extensions) - self.install_headers = None # for C/C++ headers - self.install_lib = None # set to either purelib or platlib + self.install_purelib = None # for pure module distributions + self.install_platlib = None # non-pure (dists w/ extensions) + self.install_headers = None # for C/C++ headers + self.install_lib = None # set to either purelib or platlib self.install_scripts = None self.install_data = None self.install_base = None @@ -198,7 +203,8 @@ class easy_install(Command): if os.path.exists(filename) or os.path.islink(filename): log.info("Deleting %s", filename) if not self.dry_run: - if os.path.isdir(filename) and not os.path.islink(filename): + if (os.path.isdir(filename) and + not os.path.islink(filename)): rmtree(filename) else: os.unlink(filename) @@ -231,7 +237,7 @@ class easy_install(Command): self.config_vars['usersite'] = self.install_usersite # fix the install_dir if "--user" was used - #XXX: duplicate of the code in the setup command + # XXX: duplicate of the code in the setup command if self.user and site.ENABLE_USER_SITE: self.create_home_path() if self.install_userbase is None: @@ -246,7 +252,8 @@ class easy_install(Command): self.expand_basedirs() self.expand_dirs() - self._expand('install_dir','script_dir','build_directory','site_dirs') + self._expand('install_dir', 'script_dir', 'build_directory', + 'site_dirs') # If a non-default installation directory was specified, default the # script directory to match it. if self.script_dir is None: @@ -258,12 +265,12 @@ class easy_install(Command): # Let install_dir get set by install_lib command, which in turn # gets its info from the install command, and takes into account # --prefix and --home and all that other crud. - self.set_undefined_options('install_lib', - ('install_dir','install_dir') + self.set_undefined_options( + 'install_lib', ('install_dir', 'install_dir') ) # Likewise, set default script_dir from 'install_scripts.install_dir' - self.set_undefined_options('install_scripts', - ('install_dir', 'script_dir') + self.set_undefined_options( + 'install_scripts', ('install_dir', 'script_dir') ) if self.user and self.install_purelib: @@ -277,18 +284,20 @@ class easy_install(Command): self.all_site_dirs = get_site_dirs() if self.site_dirs is not None: site_dirs = [ - os.path.expanduser(s.strip()) for s in self.site_dirs.split(',') + os.path.expanduser(s.strip()) for s in + self.site_dirs.split(',') ] for d in site_dirs: if not os.path.isdir(d): log.warn("%s (in --site-dirs) does not exist", d) elif normalize_path(d) not in normpath: raise DistutilsOptionError( - d+" (in --site-dirs) is not on sys.path" + d + " (in --site-dirs) is not on sys.path" ) else: self.all_site_dirs.append(normalize_path(d)) - if not self.editable: self.check_site_dir() + if not self.editable: + self.check_site_dir() self.index_url = self.index_url or "https://pypi.python.org/simple" self.shadow_path = self.all_site_dirs[:] for path_item in self.install_dir, normalize_path(self.script_dir): @@ -301,9 +310,9 @@ class easy_install(Command): hosts = ['*'] if self.package_index is None: self.package_index = self.create_index( - self.index_url, search_path = self.shadow_path, hosts=hosts, + self.index_url, search_path=self.shadow_path, hosts=hosts, ) - self.local_index = Environment(self.shadow_path+sys.path) + self.local_index = Environment(self.shadow_path + sys.path) if self.find_links is not None: if isinstance(self.find_links, basestring): @@ -311,14 +320,15 @@ class easy_install(Command): else: self.find_links = [] if self.local_snapshots_ok: - self.package_index.scan_egg_links(self.shadow_path+sys.path) + self.package_index.scan_egg_links(self.shadow_path + sys.path) if not self.no_find_links: self.package_index.add_find_links(self.find_links) - self.set_undefined_options('install_lib', ('optimize','optimize')) - if not isinstance(self.optimize,int): + self.set_undefined_options('install_lib', ('optimize', 'optimize')) + if not isinstance(self.optimize, int): try: self.optimize = int(self.optimize) - if not (0 <= self.optimize <= 2): raise ValueError + if not (0 <= self.optimize <= 2): + raise ValueError except ValueError: raise DistutilsOptionError("--optimize must be 0, 1, or 2") @@ -350,7 +360,7 @@ class easy_install(Command): """Calls `os.path.expanduser` on install dirs.""" self._expand_attrs(['install_purelib', 'install_platlib', 'install_lib', 'install_headers', - 'install_scripts', 'install_data',]) + 'install_scripts', 'install_data', ]) def run(self): if self.verbose != self.distribution.verbose: @@ -360,11 +370,12 @@ class easy_install(Command): self.easy_install(spec, not self.no_deps) if self.record: outputs = self.outputs - if self.root: # strip any package prefix + if self.root: # strip any package prefix root_len = len(self.root) for counter in range(len(outputs)): outputs[counter] = outputs[counter][root_len:] from distutils import file_util + self.execute( file_util.write_file, (self.record, outputs), "writing list of installed files to '%s'" % @@ -392,7 +403,7 @@ class easy_install(Command): """Verify that self.install_dir is .pth-capable dir, if needed""" instdir = normalize_path(self.install_dir) - pth_file = os.path.join(instdir,'easy-install.pth') + pth_file = os.path.join(instdir, 'easy-install.pth') # Is it a configured, PYTHONPATH, implicit, or explicit site dir? is_site_dir = instdir in self.all_site_dirs @@ -402,13 +413,14 @@ class easy_install(Command): is_site_dir = self.check_pth_processing() else: # make sure we can write to target dir - testfile = self.pseudo_tempname()+'.write-test' + testfile = self.pseudo_tempname() + '.write-test' test_exists = os.path.exists(testfile) try: - if test_exists: os.unlink(testfile) - open(testfile,'w').close() + if test_exists: + os.unlink(testfile) + open(testfile, 'w').close() os.unlink(testfile) - except (OSError,IOError): + except (OSError, IOError): self.cant_write_to_target() if not is_site_dir and not self.multi_version: @@ -421,13 +433,13 @@ class easy_install(Command): else: self.pth_file = None - PYTHONPATH = os.environ.get('PYTHONPATH','').split(os.pathsep) + PYTHONPATH = os.environ.get('PYTHONPATH', '').split(os.pathsep) if instdir not in map(normalize_path, [_f for _f in PYTHONPATH if _f]): # only PYTHONPATH dirs need a site.py, so pretend it's there self.sitepy_installed = True elif self.multi_version and not os.path.exists(pth_file): - self.sitepy_installed = True # don't need site.py in this case - self.pth_file = None # and don't create a .pth file + self.sitepy_installed = True # don't need site.py in this case + self.pth_file = None # and don't create a .pth file self.install_dir = instdir def cant_write_to_target(self): @@ -473,32 +485,36 @@ Please make the appropriate changes for your system and try again. """Empirically verify whether .pth files are supported in inst. dir""" instdir = self.install_dir log.info("Checking .pth file support in %s", instdir) - pth_file = self.pseudo_tempname()+".pth" - ok_file = pth_file+'.ok' + pth_file = self.pseudo_tempname() + ".pth" + ok_file = pth_file + '.ok' ok_exists = os.path.exists(ok_file) try: - if ok_exists: os.unlink(ok_file) + if ok_exists: + os.unlink(ok_file) dirname = os.path.dirname(ok_file) if not os.path.exists(dirname): os.makedirs(dirname) - f = open(pth_file,'w') - except (OSError,IOError): + f = open(pth_file, 'w') + except (OSError, IOError): self.cant_write_to_target() else: try: - f.write("import os; f = open(%r, 'w'); f.write('OK'); f.close()\n" % (ok_file,)) + f.write("import os; f = open(%r, 'w'); f.write('OK'); " + "f.close()\n" % (ok_file,)) f.close() - f=None + f = None executable = sys.executable - if os.name=='nt': - dirname,basename = os.path.split(executable) - alt = os.path.join(dirname,'pythonw.exe') - if basename.lower()=='python.exe' and os.path.exists(alt): + if os.name == 'nt': + dirname, basename = os.path.split(executable) + alt = os.path.join(dirname, 'pythonw.exe') + if (basename.lower() == 'python.exe' and + os.path.exists(alt)): # use pythonw.exe to avoid opening a console window executable = alt from distutils.spawn import spawn - spawn([executable,'-E','-c','pass'],0) + + spawn([executable, '-E', '-c', 'pass'], 0) if os.path.exists(ok_file): log.info( @@ -527,7 +543,7 @@ Please make the appropriate changes for your system and try again. continue self.install_script( dist, script_name, - dist.get_metadata('scripts/'+script_name) + dist.get_metadata('scripts/' + script_name) ) self.install_wrapper_scripts(dist) @@ -535,7 +551,7 @@ Please make the appropriate changes for your system and try again. if os.path.isdir(path): for base, dirs, files in os.walk(path): for filename in files: - self.outputs.append(os.path.join(base,filename)) + self.outputs.append(os.path.join(base, filename)) else: self.outputs.append(path) @@ -547,7 +563,7 @@ Please make the appropriate changes for your system and try again. % (spec,) ) - def check_editable(self,spec): + def check_editable(self, spec): if not self.editable: return @@ -560,15 +576,17 @@ Please make the appropriate changes for your system and try again. def easy_install(self, spec, deps=False): tmpdir = tempfile.mkdtemp(prefix="easy_install-") download = None - if not self.editable: self.install_site_py() + if not self.editable: + self.install_site_py() try: - if not isinstance(spec,Requirement): + if not isinstance(spec, Requirement): if URL_SCHEME(spec): # It's a url, download it to tmpdir and process self.not_editable(spec) download = self.package_index.download(spec, tmpdir) - return self.install_item(None, download, tmpdir, deps, True) + return self.install_item(None, download, tmpdir, deps, + True) elif os.path.exists(spec): # Existing file or directory, just process it directly @@ -579,15 +597,15 @@ Please make the appropriate changes for your system and try again. self.check_editable(spec) dist = self.package_index.fetch_distribution( - spec, tmpdir, self.upgrade, self.editable, not self.always_copy, - self.local_index + spec, tmpdir, self.upgrade, self.editable, + not self.always_copy, self.local_index ) if dist is None: msg = "Could not find suitable distribution for %r" % spec if self.always_copy: - msg+=" (--always-copy skips system and development eggs)" + msg += " (--always-copy skips system and development eggs)" raise DistutilsError(msg) - elif dist.precedence==DEVELOP_DIST: + elif dist.precedence == DEVELOP_DIST: # .egg-info dists don't need installing, just process deps self.process_distribution(spec, dist, deps, "Using") return dist @@ -614,10 +632,10 @@ Please make the appropriate changes for your system and try again. # at this point, we know it's a local .egg, we just don't know if # it's already installed. for dist in self.local_index[spec.project_name]: - if dist.location==download: + if dist.location == download: break else: - install_needed = True # it's not in the local index + install_needed = True # it's not in the local index log.info("Processing %s", os.path.basename(download)) @@ -646,13 +664,6 @@ Please make the appropriate changes for your system and try again. def process_distribution(self, requirement, dist, deps=True, *info): self.update_pth(dist) self.package_index.add(dist) - # First remove the dist from self.local_index, to avoid problems using - # old cached data in case its underlying file has been replaced. - # - # This is a quick-fix for a zipimporter caching issue in case the dist - # has been implemented as and already loaded from a zip file that got - # replaced later on. For more detailed information see setuptools issue - # #168 at 'http://bitbucket.org/pypa/setuptools/issue/168'. if dist in self.local_index[dist.key]: self.local_index.remove(dist) self.local_index.add(dist) @@ -711,17 +722,18 @@ Please make the appropriate changes for your system and try again. def maybe_move(self, spec, dist_filename, setup_base): dst = os.path.join(self.build_directory, spec.key) if os.path.exists(dst): - msg = "%r already exists in %s; build directory %s will not be kept" + msg = ("%r already exists in %s; build directory %s will not be " + "kept") log.warn(msg, spec.key, self.build_directory, setup_base) return setup_base if os.path.isdir(dist_filename): setup_base = dist_filename else: - if os.path.dirname(dist_filename)==setup_base: - os.unlink(dist_filename) # get it out of the tmp dir + if os.path.dirname(dist_filename) == setup_base: + os.unlink(dist_filename) # get it out of the tmp dir contents = os.listdir(setup_base) - if len(contents)==1: - dist_filename = os.path.join(setup_base,contents[0]) + if len(contents) == 1: + dist_filename = os.path.join(setup_base, contents[0]) if os.path.isdir(dist_filename): # if the only thing there is a directory, move it instead setup_base = dist_filename @@ -739,34 +751,31 @@ Please make the appropriate changes for your system and try again. spec = str(dist.as_requirement()) is_script = is_python_script(script_text, script_name) - def get_template(filename): - """ - There are a couple of template scripts in the package. This - function loads one of them and prepares it for use. - - These templates use triple-quotes to escape variable - substitutions so the scripts get the 2to3 treatment when build - on Python 3. The templates cannot use triple-quotes naturally. - """ - raw_bytes = resource_string('setuptools', template_name) - template_str = raw_bytes.decode('utf-8') - clean_template = template_str.replace('"""', '') - return clean_template - if is_script: - # See https://bitbucket.org/pypa/setuptools/issue/134 for info - # on script file naming and downstream issues with SVR4 - template_name = 'script template.py' - if dev_path: - template_name = template_name.replace('.py', ' (dev).py') script_text = (get_script_header(script_text) + - get_template(template_name) % locals()) + self._load_template(dev_path) % locals()) self.write_script(script_name, _to_ascii(script_text), 'b') + @staticmethod + def _load_template(dev_path): + """ + There are a couple of template scripts in the package. This + function loads one of them and prepares it for use. + """ + # See https://bitbucket.org/pypa/setuptools/issue/134 for info + # on script file naming and downstream issues with SVR4 + name = 'script.tmpl' + if dev_path: + name = name.replace('.tmpl', ' (dev).tmpl') + + raw_bytes = resource_string('setuptools', name) + return raw_bytes.decode('utf-8') + def write_script(self, script_name, contents, mode="t", blockers=()): """Write an executable file to the scripts directory""" - self.delete_blockers( # clean up old .py/.pyw w/o a script - [os.path.join(self.script_dir,x) for x in blockers]) + self.delete_blockers( # clean up old .py/.pyw w/o a script + [os.path.join(self.script_dir, x) for x in blockers] + ) log.info("Installing %s script to %s", script_name, self.script_dir) target = os.path.join(self.script_dir, script_name) self.add_output(target) @@ -776,10 +785,10 @@ Please make the appropriate changes for your system and try again. ensure_directory(target) if os.path.exists(target): os.unlink(target) - f = open(target,"w"+mode) + f = open(target, "w" + mode) f.write(contents) f.close() - chmod(target, 0o777-mask) + chmod(target, 0o777 - mask) def install_eggs(self, spec, dist_filename, tmpdir): # .egg dirs or files are already built, so just return them @@ -795,7 +804,7 @@ Please make the appropriate changes for your system and try again. elif os.path.isdir(dist_filename): setup_base = os.path.abspath(dist_filename) - if (setup_base.startswith(tmpdir) # something we downloaded + if (setup_base.startswith(tmpdir) # something we downloaded and self.build_directory and spec is not None): setup_base = self.maybe_move(spec, dist_filename, setup_base) @@ -806,11 +815,13 @@ Please make the appropriate changes for your system and try again. setups = glob(os.path.join(setup_base, '*', 'setup.py')) if not setups: raise DistutilsError( - "Couldn't find a setup script in %s" % os.path.abspath(dist_filename) + "Couldn't find a setup script in %s" % + os.path.abspath(dist_filename) ) - if len(setups)>1: + if len(setups) > 1: raise DistutilsError( - "Multiple setup scripts in %s" % os.path.abspath(dist_filename) + "Multiple setup scripts in %s" % + os.path.abspath(dist_filename) ) setup_script = setups[0] @@ -823,13 +834,15 @@ Please make the appropriate changes for your system and try again. def egg_distribution(self, egg_path): if os.path.isdir(egg_path): - metadata = PathMetadata(egg_path,os.path.join(egg_path,'EGG-INFO')) + metadata = PathMetadata(egg_path, os.path.join(egg_path, + 'EGG-INFO')) else: metadata = EggMetadata(zipimport.zipimporter(egg_path)) - return Distribution.from_filename(egg_path,metadata=metadata) + return Distribution.from_filename(egg_path, metadata=metadata) def install_egg(self, egg_path, tmpdir): - destination = os.path.join(self.install_dir,os.path.basename(egg_path)) + destination = os.path.join(self.install_dir, + os.path.basename(egg_path)) destination = os.path.abspath(destination) if not self.dry_run: ensure_directory(destination) @@ -839,24 +852,33 @@ Please make the appropriate changes for your system and try again. if os.path.isdir(destination) and not os.path.islink(destination): dir_util.remove_tree(destination, dry_run=self.dry_run) elif os.path.exists(destination): - self.execute(os.unlink,(destination,),"Removing "+destination) - uncache_zipdir(destination) - if os.path.isdir(egg_path): - if egg_path.startswith(tmpdir): - f,m = shutil.move, "Moving" + self.execute(os.unlink, (destination,), "Removing " + + destination) + try: + new_dist_is_zipped = False + if os.path.isdir(egg_path): + if egg_path.startswith(tmpdir): + f, m = shutil.move, "Moving" + else: + f, m = shutil.copytree, "Copying" + elif self.should_unzip(dist): + self.mkpath(destination) + f, m = self.unpack_and_compile, "Extracting" else: - f,m = shutil.copytree, "Copying" - elif self.should_unzip(dist): - self.mkpath(destination) - f,m = self.unpack_and_compile, "Extracting" - elif egg_path.startswith(tmpdir): - f,m = shutil.move, "Moving" - else: - f,m = shutil.copy2, "Copying" - - self.execute(f, (egg_path, destination), - (m+" %s to %s") % - (os.path.basename(egg_path),os.path.dirname(destination))) + new_dist_is_zipped = True + if egg_path.startswith(tmpdir): + f, m = shutil.move, "Moving" + else: + f, m = shutil.copy2, "Copying" + self.execute(f, (egg_path, destination), + (m + " %s to %s") % + (os.path.basename(egg_path), + os.path.dirname(destination))) + update_dist_caches(destination, + fix_zipimporter_caches=new_dist_is_zipped) + except: + update_dist_caches(destination, fix_zipimporter_caches=False) + raise self.add_output(destination) return self.egg_distribution(destination) @@ -871,30 +893,32 @@ Please make the appropriate changes for your system and try again. # Create a dummy distribution object until we build the real distro dist = Distribution( None, - project_name=cfg.get('metadata','name'), - version=cfg.get('metadata','version'), platform=get_platform(), + project_name=cfg.get('metadata', 'name'), + version=cfg.get('metadata', 'version'), platform=get_platform(), ) # Convert the .exe to an unpacked egg - egg_path = dist.location = os.path.join(tmpdir, dist.egg_name()+'.egg') + egg_path = dist.location = os.path.join(tmpdir, dist.egg_name() + + '.egg') egg_tmp = egg_path + '.tmp' _egg_info = os.path.join(egg_tmp, 'EGG-INFO') pkg_inf = os.path.join(_egg_info, 'PKG-INFO') - ensure_directory(pkg_inf) # make sure EGG-INFO dir exists - dist._provider = PathMetadata(egg_tmp, _egg_info) # XXX + ensure_directory(pkg_inf) # make sure EGG-INFO dir exists + dist._provider = PathMetadata(egg_tmp, _egg_info) # XXX self.exe_to_egg(dist_filename, egg_tmp) # Write EGG-INFO/PKG-INFO if not os.path.exists(pkg_inf): - f = open(pkg_inf,'w') + f = open(pkg_inf, 'w') f.write('Metadata-Version: 1.0\n') - for k,v in cfg.items('metadata'): + for k, v in cfg.items('metadata'): if k != 'target_version': - f.write('%s: %s\n' % (k.replace('_','-').title(), v)) + f.write('%s: %s\n' % (k.replace('_', '-').title(), v)) f.close() - script_dir = os.path.join(_egg_info,'scripts') - self.delete_blockers( # delete entry-point scripts to avoid duping - [os.path.join(script_dir,args[0]) for args in get_script_args(dist)] + script_dir = os.path.join(_egg_info, 'scripts') + self.delete_blockers( # delete entry-point scripts to avoid duping + [os.path.join(script_dir, args[0]) for args in + get_script_args(dist)] ) # Build .egg file from tmpdir bdist_egg.make_zipfile( @@ -910,11 +934,12 @@ Please make the appropriate changes for your system and try again. to_compile = [] native_libs = [] top_level = {} - def process(src,dst): + + def process(src, dst): s = src.lower() - for old,new in prefixes: + for old, new in prefixes: if s.startswith(old): - src = new+src[len(old):] + src = new + src[len(old):] parts = src.split('/') dst = os.path.join(egg_tmp, *parts) dl = dst.lower() @@ -922,35 +947,37 @@ Please make the appropriate changes for your system and try again. parts[-1] = bdist_egg.strip_module(parts[-1]) top_level[os.path.splitext(parts[0])[0]] = 1 native_libs.append(src) - elif dl.endswith('.py') and old!='SCRIPTS/': + elif dl.endswith('.py') and old != 'SCRIPTS/': top_level[os.path.splitext(parts[0])[0]] = 1 to_compile.append(dst) return dst if not src.endswith('.pth'): log.warn("WARNING: can't process %s", src) return None + # extract, tracking .pyd/.dll->native_libs and .py -> to_compile unpack_archive(dist_filename, egg_tmp, process) stubs = [] for res in native_libs: - if res.lower().endswith('.pyd'): # create stubs for .pyd's + if res.lower().endswith('.pyd'): # create stubs for .pyd's parts = res.split('/') resource = parts[-1] - parts[-1] = bdist_egg.strip_module(parts[-1])+'.py' + parts[-1] = bdist_egg.strip_module(parts[-1]) + '.py' pyfile = os.path.join(egg_tmp, *parts) to_compile.append(pyfile) stubs.append(pyfile) bdist_egg.write_stub(resource, pyfile) - self.byte_compile(to_compile) # compile .py's - bdist_egg.write_safety_flag(os.path.join(egg_tmp,'EGG-INFO'), + self.byte_compile(to_compile) # compile .py's + bdist_egg.write_safety_flag( + os.path.join(egg_tmp, 'EGG-INFO'), bdist_egg.analyze_egg(egg_tmp, stubs)) # write zip-safety flag - for name in 'top_level','native_libs': + for name in 'top_level', 'native_libs': if locals()[name]: - txt = os.path.join(egg_tmp, 'EGG-INFO', name+'.txt') + txt = os.path.join(egg_tmp, 'EGG-INFO', name + '.txt') if not os.path.exists(txt): - f = open(txt,'w') - f.write('\n'.join(locals()[name])+'\n') + f = open(txt, 'w') + f.write('\n'.join(locals()[name]) + '\n') f.close() def installation_report(self, req, dist, what="Installed"): @@ -968,7 +995,7 @@ these examples, in order to select the desired version: pkg_resources.require("%(name)s==%(version)s") # this exact version pkg_resources.require("%(name)s>=%(version)s") # this version or higher """ - if self.install_dir not in map(normalize_path,sys.path): + if self.install_dir not in map(normalize_path, sys.path): msg += """ Note also that the installation directory must be on sys.path at runtime for @@ -978,7 +1005,7 @@ PYTHONPATH, or by being added to sys.path by your code.) eggloc = dist.location name = dist.project_name version = dist.version - extras = '' # TODO: self.report_extras(req, dist) + extras = '' # TODO: self.report_extras(req, dist) return msg % locals() def report_editable(self, spec, setup_script): @@ -999,15 +1026,15 @@ See the setuptools documentation for the "develop" command for more info. sys.modules.setdefault('distutils.command.egg_info', egg_info) args = list(args) - if self.verbose>2: + if self.verbose > 2: v = 'v' * (self.verbose - 1) - args.insert(0,'-'+v) - elif self.verbose<2: - args.insert(0,'-q') + args.insert(0, '-' + v) + elif self.verbose < 2: + args.insert(0, '-q') if self.dry_run: - args.insert(0,'-n') + args.insert(0, '-n') log.info( - "Running %s %s", setup_script[len(setup_base)+1:], ' '.join(args) + "Running %s %s", setup_script[len(setup_base) + 1:], ' '.join(args) ) try: run_setup(setup_script, args) @@ -1033,11 +1060,11 @@ See the setuptools documentation for the "develop" command for more info. eggs.append(self.install_egg(dist.location, setup_base)) if not eggs and not self.dry_run: log.warn("No eggs found in %s (setup script problem?)", - dist_dir) + dist_dir) return eggs finally: rmtree(dist_dir) - log.set_verbosity(self.verbose) # restore our log verbosity + log.set_verbosity(self.verbose) # restore our log verbosity def _set_fetcher_options(self, base): """ @@ -1047,7 +1074,7 @@ See the setuptools documentation for the "develop" command for more info. are available to that command as well. """ # find the fetch options from easy_install and write them out - # to the setup.cfg file. + # to the setup.cfg file. ei_opts = self.distribution.get_option_dict('easy_install').copy() fetch_directives = ( 'find_links', 'site_dirs', 'index_url', 'optimize', @@ -1055,7 +1082,8 @@ See the setuptools documentation for the "develop" command for more info. ) fetch_options = {} for key, val in ei_opts.items(): - if key not in fetch_directives: continue + if key not in fetch_directives: + continue fetch_options[key.replace('_', '-')] = val[1] # create a settings dictionary suitable for `edit_config` settings = dict(easy_install=fetch_options) @@ -1066,7 +1094,7 @@ See the setuptools documentation for the "develop" command for more info. if self.pth_file is None: return - for d in self.pth_file[dist.key]: # drop old entries + for d in self.pth_file[dist.key]: # drop old entries if self.multi_version or d.location != dist.location: log.info("Removing %s from easy-install.pth file", d) self.pth_file.remove(d) @@ -1081,7 +1109,7 @@ See the setuptools documentation for the "develop" command for more info. ) else: log.info("Adding %s to easy-install.pth file", dist) - self.pth_file.add(dist) # add new entry + self.pth_file.add(dist) # add new entry if dist.location not in self.shadow_path: self.shadow_path.append(dist.location) @@ -1089,19 +1117,20 @@ See the setuptools documentation for the "develop" command for more info. self.pth_file.save() - if dist.key=='setuptools': + if dist.key == 'setuptools': # Ensure that setuptools itself never becomes unavailable! # XXX should this check for latest version? - filename = os.path.join(self.install_dir,'setuptools.pth') - if os.path.islink(filename): os.unlink(filename) + filename = os.path.join(self.install_dir, 'setuptools.pth') + if os.path.islink(filename): + os.unlink(filename) f = open(filename, 'wt') - f.write(self.pth_file.make_relative(dist.location)+'\n') + f.write(self.pth_file.make_relative(dist.location) + '\n') f.close() def unpack_progress(self, src, dst): # Progress filter for unpacking log.debug("Unpacking %s to %s", src, dst) - return dst # only unpack-and-compile skips files for dry run + return dst # only unpack-and-compile skips files for dry run def unpack_and_compile(self, egg_path, destination): to_compile = [] @@ -1112,7 +1141,7 @@ See the setuptools documentation for the "develop" command for more info. to_compile.append(dst) elif dst.endswith('.dll') or dst.endswith('.so'): to_chmod.append(dst) - self.unpack_progress(src,dst) + self.unpack_progress(src, dst) return not self.dry_run and dst or None unpack_archive(egg_path, destination, pf) @@ -1128,6 +1157,7 @@ See the setuptools documentation for the "develop" command for more info. return from distutils.util import byte_compile + try: # try to make the byte compile messages quieter log.set_verbosity(self.verbose - 1) @@ -1139,7 +1169,7 @@ See the setuptools documentation for the "develop" command for more info. dry_run=self.dry_run ) finally: - log.set_verbosity(self.verbose) # restore original verbosity + log.set_verbosity(self.verbose) # restore original verbosity def no_default_version_msg(self): template = """bad install directory or PYTHONPATH @@ -1170,7 +1200,7 @@ Here are some of your options for correcting the problem: https://pythonhosted.org/setuptools/easy_install.html#custom-installation-locations Please make the appropriate changes for your system and try again.""" - return template % (self.install_dir, os.environ.get('PYTHONPATH','')) + return template % (self.install_dir, os.environ.get('PYTHONPATH', '')) def install_site_py(self): """Make sure there's a site.py in the target dir, if needed""" @@ -1184,7 +1214,7 @@ Please make the appropriate changes for your system and try again.""" if os.path.exists(sitepy): log.debug("Checking existing site.py in %s", self.install_dir) - f = open(sitepy,'rb') + f = open(sitepy, 'rb') current = f.read() # we want str, not bytes if PY3: @@ -1201,7 +1231,7 @@ Please make the appropriate changes for your system and try again.""" log.info("Creating %s", sitepy) if not self.dry_run: ensure_directory(sitepy) - f = open(sitepy,'wb') + f = open(sitepy, 'wb') f.write(source) f.close() self.byte_compile([sitepy]) @@ -1219,15 +1249,15 @@ Please make the appropriate changes for your system and try again.""" os.makedirs(path, 0o700) INSTALL_SCHEMES = dict( - posix = dict( - install_dir = '$base/lib/python$py_version_short/site-packages', - script_dir = '$base/bin', + posix=dict( + install_dir='$base/lib/python$py_version_short/site-packages', + script_dir='$base/bin', ), ) DEFAULT_SCHEME = dict( - install_dir = '$base/Lib/site-packages', - script_dir = '$base/Scripts', + install_dir='$base/Lib/site-packages', + script_dir='$base/Scripts', ) def _expand(self, *attrs): @@ -1237,12 +1267,13 @@ Please make the appropriate changes for your system and try again.""" # Set default install_dir/scripts from --prefix config_vars = config_vars.copy() config_vars['base'] = self.prefix - scheme = self.INSTALL_SCHEMES.get(os.name,self.DEFAULT_SCHEME) - for attr,val in scheme.items(): - if getattr(self,attr,None) is None: - setattr(self,attr,val) + scheme = self.INSTALL_SCHEMES.get(os.name, self.DEFAULT_SCHEME) + for attr, val in scheme.items(): + if getattr(self, attr, None) is None: + setattr(self, attr, val) from distutils.util import subst_vars + for attr in attrs: val = getattr(self, attr) if val is not None: @@ -1251,6 +1282,7 @@ Please make the appropriate changes for your system and try again.""" val = os.path.expanduser(val) setattr(self, attr, val) + def get_site_dirs(): # return a list of 'site' dirs sitedirs = [_f for _f in os.environ.get('PYTHONPATH', @@ -1264,10 +1296,10 @@ def get_site_dirs(): sitedirs.append(os.path.join(prefix, "Lib", "site-packages")) elif os.sep == '/': sitedirs.extend([os.path.join(prefix, - "lib", - "python" + sys.version[:3], - "site-packages"), - os.path.join(prefix, "lib", "site-python")]) + "lib", + "python" + sys.version[:3], + "site-packages"), + os.path.join(prefix, "lib", "site-python")]) else: sitedirs.extend( [prefix, os.path.join(prefix, "lib", "site-packages")] @@ -1287,7 +1319,8 @@ def get_site_dirs(): 'site-packages')) lib_paths = get_path('purelib'), get_path('platlib') for site_lib in lib_paths: - if site_lib not in sitedirs: sitedirs.append(site_lib) + if site_lib not in sitedirs: + sitedirs.append(site_lib) if site.ENABLE_USER_SITE: sitedirs.append(site.USER_SITE) @@ -1318,12 +1351,12 @@ def expand_paths(inputs): if not name.endswith('.pth'): # We only care about the .pth files continue - if name in ('easy-install.pth','setuptools.pth'): + if name in ('easy-install.pth', 'setuptools.pth'): # Ignore .pth files that we control continue # Read the .pth file - f = open(os.path.join(dirname,name)) + f = open(os.path.join(dirname, name)) lines = list(yield_lines(f)) f.close() @@ -1343,7 +1376,7 @@ def extract_wininst_cfg(dist_filename): Returns a ConfigParser.RawConfigParser, or None """ - f = open(dist_filename,'rb') + f = open(dist_filename, 'rb') try: endrec = zipfile._EndRecData(f) if endrec is None: @@ -1352,21 +1385,23 @@ def extract_wininst_cfg(dist_filename): prepended = (endrec[9] - endrec[5]) - endrec[6] if prepended < 12: # no wininst data here return None - f.seek(prepended-12) + f.seek(prepended - 12) from setuptools.compat import StringIO, ConfigParser import struct - tag, cfglen, bmlen = struct.unpack("<iii",f.read(12)) + + tag, cfglen, bmlen = struct.unpack("<iii", f.read(12)) if tag not in (0x1234567A, 0x1234567B): - return None # not a valid tag + return None # not a valid tag - f.seek(prepended-(12+cfglen)) - cfg = ConfigParser.RawConfigParser({'version':'','target_version':''}) + f.seek(prepended - (12 + cfglen)) + cfg = ConfigParser.RawConfigParser( + {'version': '', 'target_version': ''}) try: part = f.read(cfglen) # part is in bytes, but we need to read up to the first null - # byte. - if sys.version_info >= (2,6): + # byte. + if sys.version_info >= (2, 6): null_byte = bytes([0]) else: null_byte = chr(0) @@ -1399,25 +1434,25 @@ def get_exe_prefixes(exe_filename): for info in z.infolist(): name = info.filename parts = name.split('/') - if len(parts)==3 and parts[2]=='PKG-INFO': + if len(parts) == 3 and parts[2] == 'PKG-INFO': if parts[1].endswith('.egg-info'): - prefixes.insert(0,('/'.join(parts[:2]), 'EGG-INFO/')) + prefixes.insert(0, ('/'.join(parts[:2]), 'EGG-INFO/')) break if len(parts) != 2 or not name.endswith('.pth'): continue if name.endswith('-nspkg.pth'): continue - if parts[0].upper() in ('PURELIB','PLATLIB'): + if parts[0].upper() in ('PURELIB', 'PLATLIB'): contents = z.read(name) if PY3: contents = contents.decode() for pth in yield_lines(contents): - pth = pth.strip().replace('\\','/') + pth = pth.strip().replace('\\', '/') if not pth.startswith('import'): - prefixes.append((('%s/%s/' % (parts[0],pth)), '')) + prefixes.append((('%s/%s/' % (parts[0], pth)), '')) finally: z.close() - prefixes = [(x.lower(),y) for x, y in prefixes] + prefixes = [(x.lower(), y) for x, y in prefixes] prefixes.sort() prefixes.reverse() return prefixes @@ -1431,6 +1466,7 @@ def parse_requirement_arg(spec): "Not a URL, existing file, or requirement spec: %r" % (spec,) ) + class PthDistributions(Environment): """A .pth file with Distribution paths in it""" @@ -1450,7 +1486,7 @@ class PthDistributions(Environment): saw_import = False seen = dict.fromkeys(self.sitedirs) if os.path.isfile(self.filename): - f = open(self.filename,'rt') + f = open(self.filename, 'rt') for line in f: if line.startswith('import'): saw_import = True @@ -1462,17 +1498,17 @@ class PthDistributions(Environment): # skip non-existent paths, in case somebody deleted a package # manually, and duplicate paths as well path = self.paths[-1] = normalize_path( - os.path.join(self.basedir,path) + os.path.join(self.basedir, path) ) if not os.path.exists(path) or path in seen: - self.paths.pop() # skip it - self.dirty = True # we cleaned up, so we're dirty now :) + self.paths.pop() # skip it + self.dirty = True # we cleaned up, so we're dirty now :) continue seen[path] = 1 f.close() if self.paths and not saw_import: - self.dirty = True # ensure anything we touch has import wrappers + self.dirty = True # ensure anything we touch has import wrappers while self.paths and not self.paths[-1].strip(): self.paths.pop() @@ -1481,7 +1517,7 @@ class PthDistributions(Environment): if not self.dirty: return - data = '\n'.join(map(self.make_relative,self.paths)) + data = '\n'.join(map(self.make_relative, self.paths)) if data: log.debug("Saving %s", self.filename) data = ( @@ -1495,7 +1531,7 @@ class PthDistributions(Environment): if os.path.islink(self.filename): os.unlink(self.filename) - f = open(self.filename,'wt') + f = open(self.filename, 'wt') f.write(data) f.close() @@ -1508,9 +1544,9 @@ class PthDistributions(Environment): def add(self, dist): """Add `dist` to the distribution map""" if (dist.location not in self.paths and ( - dist.location not in self.sitedirs or - dist.location == os.getcwd() # account for '.' being in PYTHONPATH - )): + dist.location not in self.sitedirs or + dist.location == os.getcwd() # account for '.' being in PYTHONPATH + )): self.paths.append(dist.location) self.dirty = True Environment.add(self, dist) @@ -1522,13 +1558,13 @@ class PthDistributions(Environment): self.dirty = True Environment.remove(self, dist) - def make_relative(self,path): + def make_relative(self, path): npath, last = os.path.split(normalize_path(path)) baselen = len(self.basedir) parts = [last] - sep = os.altsep=='/' and '/' or os.sep - while len(npath)>=baselen: - if npath==self.basedir: + sep = os.altsep == '/' and '/' or os.sep + while len(npath) >= baselen: + if npath == self.basedir: parts.append(os.curdir) parts.reverse() return sep.join(parts) @@ -1552,12 +1588,13 @@ def _first_line_re(): def get_script_header(script_text, executable=sys_executable, wininst=False): """Create a #! line, getting options (if any) from script_text""" - first = (script_text+'\n').splitlines()[0] + first = (script_text + '\n').splitlines()[0] match = _first_line_re().match(first) options = '' if match: options = match.group(1) or '' - if options: options = ' '+options + if options: + options = ' ' + options if wininst: executable = "python.exe" else: @@ -1567,50 +1604,199 @@ def get_script_header(script_text, executable=sys_executable, wininst=False): # Non-ascii path to sys.executable, use -x to prevent warnings if options: if options.strip().startswith('-'): - options = ' -x'+options.strip()[1:] - # else: punt, we can't do it, let the warning happen anyway + options = ' -x' + options.strip()[1:] + # else: punt, we can't do it, let the warning happen anyway else: options = ' -x' executable = fix_jython_executable(executable, options) hdr = "#!%(executable)s%(options)s\n" % locals() return hdr + def auto_chmod(func, arg, exc): - if func is os.remove and os.name=='nt': + if func is os.remove and os.name == 'nt': chmod(arg, stat.S_IWRITE) return func(arg) et, ev, _ = sys.exc_info() - reraise(et, (ev[0], ev[1] + (" %s %s" % (func,arg)))) - -def uncache_zipdir(path): - """ - Remove any globally cached zip file related data for `path` + reraise(et, (ev[0], ev[1] + (" %s %s" % (func, arg)))) - Stale zipimport.zipimporter objects need to be removed when a zip file is - replaced as they contain cached zip file directory information. If they are - asked to get data from their zip file, they will use that cached - information to calculate the data location in the zip file. This calculated - location may be incorrect for the replaced zip file, which may in turn - cause the read operation to either fail or return incorrect data. - Note we have no way to clear any local caches from here. That is left up to - whomever is in charge of maintaining that cache. +def update_dist_caches(dist_path, fix_zipimporter_caches): + """ + Fix any globally cached `dist_path` related data + + `dist_path` should be a path of a newly installed egg distribution (zipped + or unzipped). + + sys.path_importer_cache contains finder objects that have been cached when + importing data from the original distribution. Any such finders need to be + cleared since the replacement distribution might be packaged differently, + e.g. a zipped egg distribution might get replaced with an unzipped egg + folder or vice versa. Having the old finders cached may then cause Python + to attempt loading modules from the replacement distribution using an + incorrect loader. + + zipimport.zipimporter objects are Python loaders charged with importing + data packaged inside zip archives. If stale loaders referencing the + original distribution, are left behind, they can fail to load modules from + the replacement distribution. E.g. if an old zipimport.zipimporter instance + is used to load data from a new zipped egg archive, it may cause the + operation to attempt to locate the requested data in the wrong location - + one indicated by the original distribution's zip archive directory + information. Such an operation may then fail outright, e.g. report having + read a 'bad local file header', or even worse, it may fail silently & + return invalid data. + + zipimport._zip_directory_cache contains cached zip archive directory + information for all existing zipimport.zipimporter instances and all such + instances connected to the same archive share the same cached directory + information. + + If asked, and the underlying Python implementation allows it, we can fix + all existing zipimport.zipimporter instances instead of having to track + them down and remove them one by one, by updating their shared cached zip + archive directory information. This, of course, assumes that the + replacement distribution is packaged as a zipped egg. + + If not asked to fix existing zipimport.zipimporter instances, we still do + our best to clear any remaining zipimport.zipimporter related cached data + that might somehow later get used when attempting to load data from the new + distribution and thus cause such load operations to fail. Note that when + tracking down such remaining stale data, we can not catch every conceivable + usage from here, and we clear only those that we know of and have found to + cause problems if left alive. Any remaining caches should be updated by + whomever is in charge of maintaining them, i.e. they should be ready to + handle us replacing their zip archives with new distributions at runtime. """ - normalized_path = normalize_path(path) - _uncache(normalized_path, zipimport._zip_directory_cache) + # There are several other known sources of stale zipimport.zipimporter + # instances that we do not clear here, but might if ever given a reason to + # do so: + # * Global setuptools pkg_resources.working_set (a.k.a. 'master working + # set') may contain distributions which may in turn contain their + # zipimport.zipimporter loaders. + # * Several zipimport.zipimporter loaders held by local variables further + # up the function call stack when running the setuptools installation. + # * Already loaded modules may have their __loader__ attribute set to the + # exact loader instance used when importing them. Python 3.4 docs state + # that this information is intended mostly for introspection and so is + # not expected to cause us problems. + normalized_path = normalize_path(dist_path) _uncache(normalized_path, sys.path_importer_cache) + if fix_zipimporter_caches: + _replace_zip_directory_cache_data(normalized_path) + else: + # Here, even though we do not want to fix existing and now stale + # zipimporter cache information, we still want to remove it. Related to + # Python's zip archive directory information cache, we clear each of + # its stale entries in two phases: + # 1. Clear the entry so attempting to access zip archive information + # via any existing stale zipimport.zipimporter instances fails. + # 2. Remove the entry from the cache so any newly constructed + # zipimport.zipimporter instances do not end up using old stale + # zip archive directory information. + # This whole stale data removal step does not seem strictly necessary, + # but has been left in because it was done before we started replacing + # the zip archive directory information cache content if possible, and + # there are no relevant unit tests that we can depend on to tell us if + # this is really needed. + _remove_and_clear_zip_directory_cache_data(normalized_path) + + +def _collect_zipimporter_cache_entries(normalized_path, cache): + """ + Return zipimporter cache entry keys related to a given normalized path. -def _uncache(normalized_path, cache): - to_remove = [] + Alternative path spellings (e.g. those using different character case or + those using alternative path separators) related to the same path are + included. Any sub-path entries are included as well, i.e. those + corresponding to zip archives embedded in other zip archives. + + """ + result = [] prefix_len = len(normalized_path) for p in cache: np = normalize_path(p) if (np.startswith(normalized_path) and np[prefix_len:prefix_len + 1] in (os.sep, '')): - to_remove.append(p) - for p in to_remove: + result.append(p) + return result + + +def _update_zipimporter_cache(normalized_path, cache, updater=None): + """ + Update zipimporter cache data for a given normalized path. + + Any sub-path entries are processed as well, i.e. those corresponding to zip + archives embedded in other zip archives. + + Given updater is a callable taking a cache entry key and the original entry + (after already removing the entry from the cache), and expected to update + the entry and possibly return a new one to be inserted in its place. + Returning None indicates that the entry should not be replaced with a new + one. If no updater is given, the cache entries are simply removed without + any additional processing, the same as if the updater simply returned None. + + """ + for p in _collect_zipimporter_cache_entries(normalized_path, cache): + # N.B. pypy's custom zipimport._zip_directory_cache implementation does + # not support the complete dict interface: + # * Does not support item assignment, thus not allowing this function + # to be used only for removing existing cache entries. + # * Does not support the dict.pop() method, forcing us to use the + # get/del patterns instead. For more detailed information see the + # following links: + # https://bitbucket.org/pypa/setuptools/issue/202/more-robust-zipimporter-cache-invalidation#comment-10495960 + # https://bitbucket.org/pypy/pypy/src/dd07756a34a41f674c0cacfbc8ae1d4cc9ea2ae4/pypy/module/zipimport/interp_zipimport.py#cl-99 + old_entry = cache[p] del cache[p] + new_entry = updater and updater(p, old_entry) + if new_entry is not None: + cache[p] = new_entry + + +def _uncache(normalized_path, cache): + _update_zipimporter_cache(normalized_path, cache) + + +def _remove_and_clear_zip_directory_cache_data(normalized_path): + def clear_and_remove_cached_zip_archive_directory_data(path, old_entry): + old_entry.clear() + + _update_zipimporter_cache( + normalized_path, zipimport._zip_directory_cache, + updater=clear_and_remove_cached_zip_archive_directory_data) + +# PyPy Python implementation does not allow directly writing to the +# zipimport._zip_directory_cache and so prevents us from attempting to correct +# its content. The best we can do there is clear the problematic cache content +# and have PyPy repopulate it as needed. The downside is that if there are any +# stale zipimport.zipimporter instances laying around, attempting to use them +# will fail due to not having its zip archive directory information available +# instead of being automatically corrected to use the new correct zip archive +# directory information. +if '__pypy__' in sys.builtin_module_names: + _replace_zip_directory_cache_data = \ + _remove_and_clear_zip_directory_cache_data +else: + def _replace_zip_directory_cache_data(normalized_path): + def replace_cached_zip_archive_directory_data(path, old_entry): + # N.B. In theory, we could load the zip directory information just + # once for all updated path spellings, and then copy it locally and + # update its contained path strings to contain the correct + # spelling, but that seems like a way too invasive move (this cache + # structure is not officially documented anywhere and could in + # theory change with new Python releases) for no significant + # benefit. + old_entry.clear() + zipimport.zipimporter(path) + old_entry.update(zipimport._zip_directory_cache[path]) + return old_entry + + _update_zipimporter_cache( + normalized_path, zipimport._zip_directory_cache, + updater=replace_cached_zip_archive_directory_data) + def is_python(text, filename='<string>'): "Is this string a valid Python script?" @@ -1621,15 +1807,18 @@ def is_python(text, filename='<string>'): else: return True + def is_sh(executable): """Determine if the specified executable is a .sh (contains a #! line)""" try: fp = open(executable) magic = fp.read(2) fp.close() - except (OSError,IOError): return executable + except (OSError, IOError): + return executable return magic == '#!' + def nt_quote_arg(arg): """Quote a command line argument according to Windows parsing rules""" @@ -1646,7 +1835,7 @@ def nt_quote_arg(arg): nb += 1 elif c == '"': # double preceding backslashes, then add a \" - result.append('\\' * (nb*2) + '\\"') + result.append('\\' * (nb * 2) + '\\"') nb = 0 else: if nb: @@ -1658,29 +1847,33 @@ def nt_quote_arg(arg): result.append('\\' * nb) if needquote: - result.append('\\' * nb) # double the trailing backslashes + result.append('\\' * nb) # double the trailing backslashes result.append('"') return ''.join(result) + def is_python_script(script_text, filename): """Is this text, as a whole, a Python script? (as opposed to shell/bat/etc. """ if filename.endswith('.py') or filename.endswith('.pyw'): - return True # extension says it's Python + return True # extension says it's Python if is_python(script_text, filename): - return True # it's syntactically valid Python + return True # it's syntactically valid Python if script_text.startswith('#!'): # It begins with a '#!' line, so check if 'python' is in it somewhere return 'python' in script_text.splitlines()[0].lower() - return False # Not any Python I can recognize + return False # Not any Python I can recognize + try: from os import chmod as _chmod except ImportError: # Jython compatibility - def _chmod(*args): pass + def _chmod(*args): + pass + def chmod(path, mode): log.debug("changing mode of %s to %o", path, mode) @@ -1690,10 +1883,12 @@ def chmod(path, mode): e = sys.exc_info()[1] log.debug("chmod failed: %s", e) + def fix_jython_executable(executable, options): if sys.platform.startswith('java') and is_sh(executable): # Workaround for Jython is not needed on Linux systems. import java + if java.lang.System.getProperty("os.name") == "Linux": return executable @@ -1742,19 +1937,19 @@ class ScriptWriter(object): for name, ep in dist.get_entry_map(group).items(): script_text = gen_class.template % locals() for res in gen_class._get_script_args(type_, name, header, - script_text): + script_text): yield res @classmethod def get_writer(cls, force_windows): - if force_windows or sys.platform=='win32': + if force_windows or sys.platform == 'win32': return WindowsScriptWriter.get_writer() return cls @classmethod def _get_script_args(cls, type_, name, header, script_text): # Simply write the stub with no extension. - yield (name, header+script_text) + yield (name, header + script_text) class WindowsScriptWriter(ScriptWriter): @@ -1777,12 +1972,12 @@ class WindowsScriptWriter(ScriptWriter): ext = dict(console='.pya', gui='.pyw')[type_] if ext not in os.environ['PATHEXT'].lower().split(';'): warnings.warn("%s not listed in PATHEXT; scripts will not be " - "recognized as executables." % ext, UserWarning) + "recognized as executables." % ext, UserWarning) old = ['.pya', '.py', '-script.py', '.pyc', '.pyo', '.pyw', '.exe'] old.remove(ext) header = cls._adjust_header(type_, header) - blockers = [name+x for x in old] - yield name+ext, header+script_text, 't', blockers + blockers = [name + x for x in old] + yield name + ext, header + script_text, 't', blockers @staticmethod def _adjust_header(type_, orig_header): @@ -1809,33 +2004,35 @@ class WindowsExecutableLauncherWriter(WindowsScriptWriter): """ For Windows, add a .py extension and an .exe launcher """ - if type_=='gui': + if type_ == 'gui': launcher_type = 'gui' ext = '-script.pyw' old = ['.pyw'] else: launcher_type = 'cli' ext = '-script.py' - old = ['.py','.pyc','.pyo'] + old = ['.py', '.pyc', '.pyo'] hdr = cls._adjust_header(type_, header) - blockers = [name+x for x in old] - yield (name+ext, hdr+script_text, 't', blockers) + blockers = [name + x for x in old] + yield (name + ext, hdr + script_text, 't', blockers) yield ( - name+'.exe', get_win_launcher(launcher_type), - 'b' # write in binary mode + name + '.exe', get_win_launcher(launcher_type), + 'b' # write in binary mode ) if not is_64bit(): # install a manifest for the launcher to prevent Windows - # from detecting it as an installer (which it will for + # from detecting it as an installer (which it will for # launchers like easy_install.exe). Consider only # adding a manifest for launchers detected as installers. # See Distribute #143 for details. m_name = name + '.exe.manifest' yield (m_name, load_launcher_manifest(name), 't') + # for backward-compatibility get_script_args = ScriptWriter.get_script_args + def get_win_launcher(type): """ Load the Windows launcher (executable) suitable for launching a script. @@ -1845,7 +2042,7 @@ def get_win_launcher(type): Returns the executable as a byte string. """ launcher_fn = '%s.exe' % type - if platform.machine().lower()=='arm': + if platform.machine().lower() == 'arm': launcher_fn = launcher_fn.replace(".", "-arm.") if is_64bit(): launcher_fn = launcher_fn.replace(".", "-64.") @@ -1853,6 +2050,7 @@ def get_win_launcher(type): launcher_fn = launcher_fn.replace(".", "-32.") return resource_string('setuptools', launcher_fn) + def load_launcher_manifest(name): manifest = pkg_resources.resource_string(__name__, 'launcher manifest.xml') if PY2: @@ -1860,6 +2058,7 @@ def load_launcher_manifest(name): else: return manifest.decode('utf-8') % vars() + def rmtree(path, ignore_errors=False, onerror=auto_chmod): """Recursively delete a directory tree. @@ -1895,19 +2094,23 @@ def rmtree(path, ignore_errors=False, onerror=auto_chmod): except os.error: onerror(os.rmdir, path, sys.exc_info()) + def current_umask(): tmp = os.umask(0o022) os.umask(tmp) return tmp + def bootstrap(): # This function is called when setuptools*.egg is run using /bin/sh import setuptools + argv0 = os.path.dirname(setuptools.__path__[0]) sys.argv[0] = argv0 sys.argv.append(argv0) main() + def main(argv=None, **kw): from setuptools import setup from setuptools.dist import Distribution @@ -1934,16 +2137,16 @@ usage: %(script)s [options] requirement_or_url ... class DistributionWithoutHelpCommands(Distribution): common_usage = "" - def _show_help(self,*args,**kw): - with_ei_usage(lambda: Distribution._show_help(self,*args,**kw)) + def _show_help(self, *args, **kw): + with_ei_usage(lambda: Distribution._show_help(self, *args, **kw)) if argv is None: argv = sys.argv[1:] - with_ei_usage(lambda: - setup( - script_args = ['-q','easy_install', '-v']+argv, - script_name = sys.argv[0] or 'easy_install', + with_ei_usage( + lambda: setup( + script_args=['-q', 'easy_install', '-v'] + argv, + script_name=sys.argv[0] or 'easy_install', distclass=DistributionWithoutHelpCommands, **kw ) ) |