diff options
Diffstat (limited to 'setuptools')
41 files changed, 1042 insertions, 795 deletions
diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 0ce190b8..0f702848 100644 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -125,6 +125,56 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): os.chmod(target, unix_attributes) +def _resolve_tar_file_or_dir(tar_obj, tar_member_obj): + """Resolve any links and extract link targets as normal files.""" + while tar_member_obj is not None and ( + tar_member_obj.islnk() or tar_member_obj.issym()): + linkpath = tar_member_obj.linkname + if tar_member_obj.issym(): + base = posixpath.dirname(tar_member_obj.name) + linkpath = posixpath.join(base, linkpath) + linkpath = posixpath.normpath(linkpath) + tar_member_obj = tar_obj._getmember(linkpath) + + is_file_or_dir = ( + tar_member_obj is not None and + (tar_member_obj.isfile() or tar_member_obj.isdir()) + ) + if is_file_or_dir: + return tar_member_obj + + raise LookupError('Got unknown file type') + + +def _iter_open_tar(tar_obj, extract_dir, progress_filter): + """Emit member-destination pairs from a tar archive.""" + # don't do any chowning! + tar_obj.chown = lambda *args: None + + with contextlib.closing(tar_obj): + for member in tar_obj: + name = member.name + # don't extract absolute paths or ones with .. in them + if name.startswith('/') or '..' in name.split('/'): + continue + + prelim_dst = os.path.join(extract_dir, *name.split('/')) + + try: + member = _resolve_tar_file_or_dir(tar_obj, member) + except LookupError: + continue + + final_dst = progress_filter(name, prelim_dst) + if not final_dst: + continue + + if final_dst.endswith(os.sep): + final_dst = final_dst[:-1] + + yield member, final_dst + + def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir` @@ -138,38 +188,18 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): raise UnrecognizedFormat( "%s is not a compressed or uncompressed tar file" % (filename,) ) from e - with contextlib.closing(tarobj): - # don't do any chowning! - tarobj.chown = lambda *args: None - for member in tarobj: - name = member.name - # don't extract absolute paths or ones with .. in them - if not name.startswith('/') and '..' not in name.split('/'): - prelim_dst = os.path.join(extract_dir, *name.split('/')) - - # resolve any links and to extract the link targets as normal - # files - while member is not None and ( - member.islnk() or member.issym()): - linkpath = member.linkname - if member.issym(): - base = posixpath.dirname(member.name) - linkpath = posixpath.join(base, linkpath) - linkpath = posixpath.normpath(linkpath) - member = tarobj._getmember(linkpath) - - if member is not None and (member.isfile() or member.isdir()): - final_dst = progress_filter(name, prelim_dst) - if final_dst: - if final_dst.endswith(os.sep): - final_dst = final_dst[:-1] - try: - # XXX Ugh - tarobj._extract_member(member, final_dst) - except tarfile.ExtractError: - # chown/chmod/mkfifo/mknode/makedev failed - pass - return True + + for member, final_dst in _iter_open_tar( + tarobj, extract_dir, progress_filter, + ): + try: + # XXX Ugh + tarobj._extract_member(member, final_dst) + except tarfile.ExtractError: + # chown/chmod/mkfifo/mknode/makedev failed + pass + + return True extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index b9e8a2b3..9dfb2f24 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -101,7 +101,12 @@ def _file_with_extension(directory, extension): f for f in os.listdir(directory) if f.endswith(extension) ) - file, = matching + try: + file, = matching + except ValueError: + raise ValueError( + 'No distribution was found. Ensure that `setup.py` ' + 'is not empty and that it calls `setup()`.') return file diff --git a/setuptools/command/__init__.py b/setuptools/command/__init__.py index 743f5588..570e6957 100644 --- a/setuptools/command/__init__.py +++ b/setuptools/command/__init__.py @@ -2,7 +2,7 @@ __all__ = [ 'alias', 'bdist_egg', 'bdist_rpm', 'build_ext', 'build_py', 'develop', 'easy_install', 'egg_info', 'install', 'install_lib', 'rotate', 'saveopts', 'sdist', 'setopt', 'test', 'install_egg_info', 'install_scripts', - 'bdist_wininst', 'upload_docs', 'build_clib', 'dist_info', + 'upload_docs', 'build_clib', 'dist_info', ] from distutils.command.bdist import bdist diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index a88efb45..e6b1609f 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -2,7 +2,6 @@ Build .egg distributions""" -from distutils.errors import DistutilsSetupError from distutils.dir_util import remove_tree, mkpath from distutils import log from types import CodeType @@ -11,12 +10,10 @@ import os import re import textwrap import marshal -import warnings from pkg_resources import get_build_platform, Distribution, ensure_directory -from pkg_resources import EntryPoint from setuptools.extension import Library -from setuptools import Command, SetuptoolsDeprecationWarning +from setuptools import Command from sysconfig import get_path, get_python_version @@ -153,7 +150,7 @@ class bdist_egg(Command): self.run_command(cmdname) return cmd - def run(self): + def run(self): # noqa: C901 # is too complex (14) # FIXME # Generate metadata first self.run_command("egg_info") # We run install_lib before install_data, because some data hacks @@ -268,49 +265,7 @@ class bdist_egg(Command): return analyze_egg(self.bdist_dir, self.stubs) def gen_header(self): - epm = EntryPoint.parse_map(self.distribution.entry_points or '') - ep = epm.get('setuptools.installation', {}).get('eggsecutable') - if ep is None: - return 'w' # not an eggsecutable, do it the usual way. - - warnings.warn( - "Eggsecutables are deprecated and will be removed in a future " - "version.", - SetuptoolsDeprecationWarning - ) - - if not ep.attrs or ep.extras: - raise DistutilsSetupError( - "eggsecutable entry point (%r) cannot have 'extras' " - "or refer to a module" % (ep,) - ) - - pyver = '{}.{}'.format(*sys.version_info) - pkg = ep.module_name - full = '.'.join(ep.attrs) - base = ep.attrs[0] - basename = os.path.basename(self.egg_output) - - header = ( - "#!/bin/sh\n" - 'if [ `basename $0` = "%(basename)s" ]\n' - 'then exec python%(pyver)s -c "' - "import sys, os; sys.path.insert(0, os.path.abspath('$0')); " - "from %(pkg)s import %(base)s; sys.exit(%(full)s())" - '" "$@"\n' - 'else\n' - ' echo $0 is not the correct name for this egg file.\n' - ' echo Please rename it back to %(basename)s and try again.\n' - ' exec false\n' - 'fi\n' - ) % locals() - - if not self.dry_run: - mkpath(os.path.dirname(self.egg_output), dry_run=self.dry_run) - f = open(self.egg_output, 'w') - f.write(header) - f.close() - return 'a' + return 'w' def copy_metadata_to(self, target_dir): "Copy metadata (egg info) to the target_dir" diff --git a/setuptools/command/bdist_wininst.py b/setuptools/command/bdist_wininst.py deleted file mode 100644 index ff4b6345..00000000 --- a/setuptools/command/bdist_wininst.py +++ /dev/null @@ -1,30 +0,0 @@ -import distutils.command.bdist_wininst as orig -import warnings - -from setuptools import SetuptoolsDeprecationWarning - - -class bdist_wininst(orig.bdist_wininst): - def reinitialize_command(self, command, reinit_subcommands=0): - """ - Supplement reinitialize_command to work around - http://bugs.python.org/issue20819 - """ - cmd = self.distribution.reinitialize_command( - command, reinit_subcommands) - if command in ('install', 'install_lib'): - cmd.install_lib = None - return cmd - - def run(self): - warnings.warn( - "bdist_wininst is deprecated and will be removed in a future " - "version. Use bdist_wheel (wheel packages) instead.", - SetuptoolsDeprecationWarning - ) - - self._is_running = True - try: - orig.bdist_wininst.run(self) - finally: - self._is_running = False diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 9ec83b7d..45adb6a1 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -67,7 +67,7 @@ warnings.filterwarnings("default", category=pkg_resources.PEP440Warning) __all__ = [ 'samefile', 'easy_install', 'PthDistributions', 'extract_wininst_cfg', - 'main', 'get_exe_prefixes', + 'get_exe_prefixes', ] @@ -226,7 +226,7 @@ class easy_install(Command): print(tmpl.format(**locals())) raise SystemExit() - def finalize_options(self): + def finalize_options(self): # noqa: C901 # is too complex (25) # FIXME self.version and self._render_version() py_version = sys.version.split()[0] @@ -437,7 +437,7 @@ class easy_install(Command): def warn_deprecated_options(self): pass - def check_site_dir(self): + def check_site_dir(self): # noqa: C901 # is too complex (12) # FIXME """Verify that self.install_dir is .pth-capable dir, if needed""" instdir = normalize_path(self.install_dir) @@ -713,7 +713,10 @@ class easy_install(Command): if getattr(self, attrname) is None: setattr(self, attrname, scheme[key]) - def process_distribution(self, requirement, dist, deps=True, *info): + # FIXME: 'easy_install.process_distribution' is too complex (12) + def process_distribution( # noqa: C901 + self, requirement, dist, deps=True, *info, + ): self.update_pth(dist) self.package_index.add(dist) if dist in self.local_index[dist.key]: @@ -837,12 +840,19 @@ class easy_install(Command): def install_eggs(self, spec, dist_filename, tmpdir): # .egg dirs or files are already built, so just return them - if dist_filename.lower().endswith('.egg'): - return [self.install_egg(dist_filename, tmpdir)] - elif dist_filename.lower().endswith('.exe'): - return [self.install_exe(dist_filename, tmpdir)] - elif dist_filename.lower().endswith('.whl'): - return [self.install_wheel(dist_filename, tmpdir)] + installer_map = { + '.egg': self.install_egg, + '.exe': self.install_exe, + '.whl': self.install_wheel, + } + try: + install_dist = installer_map[ + dist_filename.lower()[-4:] + ] + except KeyError: + pass + else: + return [install_dist(dist_filename, tmpdir)] # Anything else, try to extract and build setup_base = tmpdir @@ -887,7 +897,8 @@ class easy_install(Command): metadata = EggMetadata(zipimport.zipimporter(egg_path)) return Distribution.from_filename(egg_path, metadata=metadata) - def install_egg(self, egg_path, tmpdir): + # FIXME: 'easy_install.install_egg' is too complex (11) + def install_egg(self, egg_path, tmpdir): # noqa: C901 destination = os.path.join( self.install_dir, os.path.basename(egg_path), @@ -986,7 +997,8 @@ class easy_install(Command): # install the .egg return self.install_egg(egg_path, tmpdir) - def exe_to_egg(self, dist_filename, egg_tmp): + # FIXME: 'easy_install.exe_to_egg' is too complex (12) + def exe_to_egg(self, dist_filename, egg_tmp): # noqa: C901 """Extract a bdist_wininst to the directories an egg would use""" # Check for .pth file and set up prefix translations prefixes = get_exe_prefixes(dist_filename) @@ -1178,22 +1190,24 @@ class easy_install(Command): for key, val in ei_opts.items(): if key not in fetch_directives: continue - fetch_options[key.replace('_', '-')] = val[1] + fetch_options[key] = val[1] # create a settings dictionary suitable for `edit_config` settings = dict(easy_install=fetch_options) cfg_filename = os.path.join(base, 'setup.cfg') setopt.edit_config(cfg_filename, settings) - def update_pth(self, dist): + def update_pth(self, dist): # noqa: C901 # is too complex (11) # FIXME if self.pth_file is None: return for d in self.pth_file[dist.key]: # drop old entries - if self.multi_version or d.location != dist.location: - log.info("Removing %s from easy-install.pth file", d) - self.pth_file.remove(d) - if d.location in self.shadow_path: - self.shadow_path.remove(d.location) + if not self.multi_version and d.location == dist.location: + continue + + log.info("Removing %s from easy-install.pth file", d) + self.pth_file.remove(d) + if d.location in self.shadow_path: + self.shadow_path.remove(d.location) if not self.multi_version: if dist.location in self.pth_file.paths: @@ -1207,19 +1221,21 @@ class easy_install(Command): if dist.location not in self.shadow_path: self.shadow_path.append(dist.location) - if not self.dry_run: + if self.dry_run: + return - self.pth_file.save() + self.pth_file.save() - if dist.key == 'setuptools': - # Ensure that setuptools itself never becomes unavailable! - # XXX should this check for latest version? - filename = os.path.join(self.install_dir, 'setuptools.pth') - if os.path.islink(filename): - os.unlink(filename) - f = open(filename, 'wt') - f.write(self.pth_file.make_relative(dist.location) + '\n') - f.close() + if dist.key != 'setuptools': + return + + # Ensure that setuptools itself never becomes unavailable! + # XXX should this check for latest version? + filename = os.path.join(self.install_dir, 'setuptools.pth') + if os.path.islink(filename): + os.unlink(filename) + with open(filename, 'wt') as f: + f.write(self.pth_file.make_relative(dist.location) + '\n') def unpack_progress(self, src, dst): # Progress filter for unpacking @@ -1360,58 +1376,63 @@ def get_site_dirs(): if sys.exec_prefix != sys.prefix: prefixes.append(sys.exec_prefix) for prefix in prefixes: - if prefix: - if sys.platform in ('os2emx', 'riscos'): - sitedirs.append(os.path.join(prefix, "Lib", "site-packages")) - elif os.sep == '/': - sitedirs.extend([ - os.path.join( - prefix, - "lib", - "python{}.{}".format(*sys.version_info), - "site-packages", - ), - os.path.join(prefix, "lib", "site-python"), - ]) - else: - sitedirs.extend([ + if not prefix: + continue + + if sys.platform in ('os2emx', 'riscos'): + sitedirs.append(os.path.join(prefix, "Lib", "site-packages")) + elif os.sep == '/': + sitedirs.extend([ + os.path.join( prefix, - os.path.join(prefix, "lib", "site-packages"), - ]) - if sys.platform == 'darwin': - # for framework builds *only* we add the standard Apple - # locations. Currently only per-user, but /Library and - # /Network/Library could be added too - if 'Python.framework' in prefix: - home = os.environ.get('HOME') - if home: - home_sp = os.path.join( - home, - 'Library', - 'Python', - '{}.{}'.format(*sys.version_info), - 'site-packages', - ) - sitedirs.append(home_sp) + "lib", + "python{}.{}".format(*sys.version_info), + "site-packages", + ), + os.path.join(prefix, "lib", "site-python"), + ]) + else: + sitedirs.extend([ + prefix, + os.path.join(prefix, "lib", "site-packages"), + ]) + if sys.platform != 'darwin': + continue + + # for framework builds *only* we add the standard Apple + # locations. Currently only per-user, but /Library and + # /Network/Library could be added too + if 'Python.framework' not in prefix: + continue + + home = os.environ.get('HOME') + if not home: + continue + + home_sp = os.path.join( + home, + 'Library', + 'Python', + '{}.{}'.format(*sys.version_info), + 'site-packages', + ) + sitedirs.append(home_sp) lib_paths = get_path('purelib'), get_path('platlib') - for site_lib in lib_paths: - if site_lib not in sitedirs: - sitedirs.append(site_lib) + + sitedirs.extend(s for s in lib_paths if s not in sitedirs) if site.ENABLE_USER_SITE: sitedirs.append(site.USER_SITE) - try: + with contextlib.suppress(AttributeError): sitedirs.extend(site.getsitepackages()) - except AttributeError: - pass sitedirs = list(map(normalize_path, sitedirs)) return sitedirs -def expand_paths(inputs): +def expand_paths(inputs): # noqa: C901 # is too complex (11) # FIXME """Yield sys.path directories that might contain "old-style" packages""" seen = {} @@ -1443,13 +1464,18 @@ def expand_paths(inputs): # Yield existing non-dupe, non-import directory lines from it for line in lines: - if not line.startswith("import"): - line = normalize_path(line.rstrip()) - if line not in seen: - seen[line] = 1 - if not os.path.isdir(line): - continue - yield line, os.listdir(line) + if line.startswith("import"): + continue + + line = normalize_path(line.rstrip()) + if line in seen: + continue + + seen[line] = 1 + if not os.path.isdir(line): + continue + + yield line, os.listdir(line) def extract_wininst_cfg(dist_filename): @@ -2167,7 +2193,7 @@ class WindowsScriptWriter(ScriptWriter): @classmethod def _adjust_header(cls, type_, orig_header): """ - Make sure 'pythonw' is used for gui and and 'python' is used for + Make sure 'pythonw' is used for gui and 'python' is used for console (regardless of what sys.executable is). """ pattern = 'pythonw.exe' @@ -2258,60 +2284,6 @@ def current_umask(): return tmp -def bootstrap(): - # This function is called when setuptools*.egg is run using /bin/sh - import setuptools - - argv0 = os.path.dirname(setuptools.__path__[0]) - sys.argv[0] = argv0 - sys.argv.append(argv0) - main() - - -def main(argv=None, **kw): - from setuptools import setup - from setuptools.dist import Distribution - - class DistributionWithoutHelpCommands(Distribution): - common_usage = "" - - def _show_help(self, *args, **kw): - with _patch_usage(): - Distribution._show_help(self, *args, **kw) - - if argv is None: - argv = sys.argv[1:] - - with _patch_usage(): - setup( - script_args=['-q', 'easy_install', '-v'] + argv, - script_name=sys.argv[0] or 'easy_install', - distclass=DistributionWithoutHelpCommands, - **kw - ) - - -@contextlib.contextmanager -def _patch_usage(): - import distutils.core - USAGE = textwrap.dedent(""" - usage: %(script)s [options] requirement_or_url ... - or: %(script)s --help - """).lstrip() - - def gen_usage(script_name): - return USAGE % dict( - script=os.path.basename(script_name), - ) - - saved = distutils.core.gen_usage - distutils.core.gen_usage = gen_usage - try: - yield - finally: - distutils.core.gen_usage = saved - - class EasyInstallDeprecationWarning(SetuptoolsDeprecationWarning): """ Warning for EasyInstall deprecations, bypassing suppression. diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 0b7ad677..1f120b67 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -8,6 +8,7 @@ from distutils.util import convert_path from distutils import log import distutils.errors import distutils.filelist +import functools import os import re import sys @@ -31,7 +32,7 @@ from setuptools.extern import packaging from setuptools import SetuptoolsDeprecationWarning -def translate_pattern(glob): +def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME """ Translate a file path glob like '*.txt' in to a regular expression. This differs from fnmatch.translate which allows wildcards to match @@ -332,70 +333,74 @@ class FileList(_FileList): # patterns, (dir and patterns), or (dir_pattern). (action, patterns, dir, dir_pattern) = self._parse_template_line(line) + action_map = { + 'include': self.include, + 'exclude': self.exclude, + 'global-include': self.global_include, + 'global-exclude': self.global_exclude, + 'recursive-include': functools.partial( + self.recursive_include, dir, + ), + 'recursive-exclude': functools.partial( + self.recursive_exclude, dir, + ), + 'graft': self.graft, + 'prune': self.prune, + } + log_map = { + 'include': "warning: no files found matching '%s'", + 'exclude': ( + "warning: no previously-included files found " + "matching '%s'" + ), + 'global-include': ( + "warning: no files found matching '%s' " + "anywhere in distribution" + ), + 'global-exclude': ( + "warning: no previously-included files matching " + "'%s' found anywhere in distribution" + ), + 'recursive-include': ( + "warning: no files found matching '%s' " + "under directory '%s'" + ), + 'recursive-exclude': ( + "warning: no previously-included files matching " + "'%s' found under directory '%s'" + ), + 'graft': "warning: no directories found matching '%s'", + 'prune': "no previously-included directories found matching '%s'", + } + + try: + process_action = action_map[action] + except KeyError: + raise DistutilsInternalError( + "this cannot happen: invalid action '{action!s}'". + format(action=action), + ) + # OK, now we know that the action is valid and we have the # right number of words on the line for that action -- so we # can proceed with minimal error-checking. - if action == 'include': - self.debug_print("include " + ' '.join(patterns)) - for pattern in patterns: - if not self.include(pattern): - log.warn("warning: no files found matching '%s'", pattern) - - elif action == 'exclude': - self.debug_print("exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.exclude(pattern): - log.warn(("warning: no previously-included files " - "found matching '%s'"), pattern) - - elif action == 'global-include': - self.debug_print("global-include " + ' '.join(patterns)) - for pattern in patterns: - if not self.global_include(pattern): - log.warn(("warning: no files found matching '%s' " - "anywhere in distribution"), pattern) - - elif action == 'global-exclude': - self.debug_print("global-exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.global_exclude(pattern): - log.warn(("warning: no previously-included files matching " - "'%s' found anywhere in distribution"), - pattern) - - elif action == 'recursive-include': - self.debug_print("recursive-include %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.recursive_include(dir, pattern): - log.warn(("warning: no files found matching '%s' " - "under directory '%s'"), - pattern, dir) - - elif action == 'recursive-exclude': - self.debug_print("recursive-exclude %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.recursive_exclude(dir, pattern): - log.warn(("warning: no previously-included files matching " - "'%s' found under directory '%s'"), - pattern, dir) - - elif action == 'graft': - self.debug_print("graft " + dir_pattern) - if not self.graft(dir_pattern): - log.warn("warning: no directories found matching '%s'", - dir_pattern) - - elif action == 'prune': - self.debug_print("prune " + dir_pattern) - if not self.prune(dir_pattern): - log.warn(("no previously-included directories found " - "matching '%s'"), dir_pattern) - - else: - raise DistutilsInternalError( - "this cannot happen: invalid action '%s'" % action) + + action_is_recursive = action.startswith('recursive-') + if action in {'graft', 'prune'}: + patterns = [dir_pattern] + extra_log_args = (dir, ) if action_is_recursive else () + log_tmpl = log_map[action] + + self.debug_print( + ' '.join( + [action] + + ([dir] if action_is_recursive else []) + + patterns, + ) + ) + for pattern in patterns: + if not process_action(pattern): + log.warn(log_tmpl, pattern, *extra_log_args) def _remove_files(self, predicate): """ diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py index 8c9a15e2..9cd8eb06 100644 --- a/setuptools/command/install_scripts.py +++ b/setuptools/command/install_scripts.py @@ -1,5 +1,6 @@ from distutils import log import distutils.command.install_scripts as orig +from distutils.errors import DistutilsModuleError import os import sys @@ -35,7 +36,7 @@ class install_scripts(orig.install_scripts): try: bw_cmd = self.get_finalized_command("bdist_wininst") is_wininst = getattr(bw_cmd, '_is_running', False) - except ImportError: + except (ImportError, DistutilsModuleError): is_wininst = False writer = ei.ScriptWriter if is_wininst: diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 887b7efa..a6ea814a 100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -4,6 +4,7 @@ import os import sys import io import contextlib +from glob import iglob from setuptools.extern import ordered_set @@ -194,29 +195,41 @@ class sdist(sdist_add_defaults, orig.sdist): """Checks if license_file' or 'license_files' is configured and adds any valid paths to 'self.filelist'. """ - - files = ordered_set.OrderedSet() - opts = self.distribution.get_option_dict('metadata') - # ignore the source of the value - _, license_file = opts.get('license_file', (None, None)) - - if license_file is None: - log.debug("'license_file' option was not specified") - else: - files.add(license_file) - + files = ordered_set.OrderedSet() try: - files.update(self.distribution.metadata.license_files) + license_files = self.distribution.metadata.license_files except TypeError: log.warn("warning: 'license_files' option is malformed") - - for f in files: - if not os.path.exists(f): - log.warn( - "warning: Failed to find the configured license file '%s'", - f) - files.remove(f) - - self.filelist.extend(files) + license_files = ordered_set.OrderedSet() + patterns = license_files if isinstance(license_files, ordered_set.OrderedSet) \ + else ordered_set.OrderedSet(license_files) + + if 'license_file' in opts: + log.warn( + "warning: the 'license_file' option is deprecated, " + "use 'license_files' instead") + patterns.append(opts['license_file'][1]) + + if 'license_file' not in opts and 'license_files' not in opts: + # Default patterns match the ones wheel uses + # See https://wheel.readthedocs.io/en/stable/user_guide.html + # -> 'Including license files in the generated wheel file' + patterns = ('LICEN[CS]E*', 'COPYING*', 'NOTICE*', 'AUTHORS*') + + for pattern in patterns: + for path in iglob(pattern): + if path.endswith('~'): + log.debug( + "ignoring license file '%s' as it looks like a backup", + path) + continue + + if path not in files and os.path.isfile(path): + log.info( + "adding license file '%s' (matched pattern '%s')", + path, pattern) + files.add(path) + + self.filelist.extend(sorted(files)) diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 2559458a..845bff44 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -2,7 +2,7 @@ """upload_docs Implements a Distutils 'upload_docs' subcommand (upload documentation to -PyPI's pythonhosted.org). +sites other than PyPi such as devpi). """ from base64 import standard_b64encode @@ -31,7 +31,7 @@ class upload_docs(upload): # supported by Warehouse (and won't be). DEFAULT_REPOSITORY = 'https://pypi.python.org/pypi/' - description = 'Upload documentation to PyPI' + description = 'Upload documentation to sites other than PyPi such as devpi' user_options = [ ('repository=', 'r', @@ -59,7 +59,7 @@ class upload_docs(upload): if self.upload_dir is None: if self.has_sphinx(): build_sphinx = self.get_finalized_command('build_sphinx') - self.target_dir = build_sphinx.builder_target_dir + self.target_dir = dict(build_sphinx.builder_target_dirs)['html'] else: build = self.get_finalized_command('build') self.target_dir = os.path.join(build.build_base, 'docs') @@ -67,7 +67,7 @@ class upload_docs(upload): self.ensure_dirname('upload_dir') self.target_dir = self.upload_dir if 'pypi.python.org' in self.repository: - log.warn("Upload_docs command is deprecated. Use RTD instead.") + log.warn("Upload_docs command is deprecated for PyPi. Use RTD instead.") self.announce('Using upload directory %s' % self.target_dir) def create_zipfile(self, filename): diff --git a/setuptools/config.py b/setuptools/config.py index af3a3bcb..4a6cd469 100644 --- a/setuptools/config.py +++ b/setuptools/config.py @@ -574,6 +574,7 @@ class ConfigOptionsHandler(ConfigHandler): parse_list_semicolon = partial(self._parse_list, separator=';') parse_bool = self._parse_bool parse_dict = self._parse_dict + parse_cmdclass = self._parse_cmdclass return { 'zip_safe': parse_bool, @@ -594,6 +595,22 @@ class ConfigOptionsHandler(ConfigHandler): 'entry_points': self._parse_file, 'py_modules': parse_list, 'python_requires': SpecifierSet, + 'cmdclass': parse_cmdclass, + } + + def _parse_cmdclass(self, value): + def resolve_class(qualified_class_name): + idx = qualified_class_name.rfind('.') + class_name = qualified_class_name[idx+1:] + pkg_name = qualified_class_name[:idx] + + module = __import__(pkg_name) + + return getattr(module, class_name) + + return { + k: resolve_class(v) + for k, v in self._parse_dict(value).items() } def _parse_packages(self, value): diff --git a/setuptools/dist.py b/setuptools/dist.py index 2c088ef8..c7af35dc 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -11,10 +11,13 @@ import distutils.log import distutils.core import distutils.cmd import distutils.dist +import distutils.command from distutils.util import strtobool from distutils.debug import DEBUG from distutils.fancy_getopt import translate_longopt import itertools +import textwrap +from typing import List, Optional, TYPE_CHECKING from collections import defaultdict from email import message_from_file @@ -29,11 +32,15 @@ from setuptools.extern import ordered_set from . import SetuptoolsDeprecationWarning import setuptools +import setuptools.command from setuptools import windows_support from setuptools.monkey import get_unpatched from setuptools.config import parse_configuration import pkg_resources +if TYPE_CHECKING: + from email.message import Message + __import__('setuptools.extern.packaging.specifiers') __import__('setuptools.extern.packaging.version') @@ -65,61 +72,92 @@ def get_metadata_version(self): return mv -def read_pkg_file(self, file): - """Reads the metadata values from a file object.""" - msg = message_from_file(file) +def rfc822_unescape(content: str) -> str: + """Reverse RFC-822 escaping by removing leading whitespaces from content.""" + lines = content.splitlines() + if len(lines) == 1: + return lines[0].lstrip() + return '\n'.join( + (lines[0].lstrip(), + textwrap.dedent('\n'.join(lines[1:])))) + + +def _read_field_from_msg(msg: "Message", field: str) -> Optional[str]: + """Read Message header field.""" + value = msg[field] + if value == 'UNKNOWN': + return None + return value + - def _read_field(name): - value = msg[name] - if value == 'UNKNOWN': - return None +def _read_field_unescaped_from_msg(msg: "Message", field: str) -> Optional[str]: + """Read Message header field and apply rfc822_unescape.""" + value = _read_field_from_msg(msg, field) + if value is None: return value + return rfc822_unescape(value) - def _read_list(name): - values = msg.get_all(name, None) - if values == []: - return None - return values + +def _read_list_from_msg(msg: "Message", field: str) -> Optional[List[str]]: + """Read Message header field and return all results as list.""" + values = msg.get_all(field, None) + if values == []: + return None + return values + + +def read_pkg_file(self, file): + """Reads the metadata values from a file object.""" + msg = message_from_file(file) self.metadata_version = StrictVersion(msg['metadata-version']) - self.name = _read_field('name') - self.version = _read_field('version') - self.description = _read_field('summary') + self.name = _read_field_from_msg(msg, 'name') + self.version = _read_field_from_msg(msg, 'version') + self.description = _read_field_from_msg(msg, 'summary') # we are filling author only. - self.author = _read_field('author') + self.author = _read_field_from_msg(msg, 'author') self.maintainer = None - self.author_email = _read_field('author-email') + self.author_email = _read_field_from_msg(msg, 'author-email') self.maintainer_email = None - self.url = _read_field('home-page') - self.license = _read_field('license') + self.url = _read_field_from_msg(msg, 'home-page') + self.license = _read_field_from_msg(msg, 'license') if 'download-url' in msg: - self.download_url = _read_field('download-url') + self.download_url = _read_field_from_msg(msg, 'download-url') else: self.download_url = None - self.long_description = _read_field('description') - self.description = _read_field('summary') + self.long_description = _read_field_unescaped_from_msg(msg, 'description') + self.description = _read_field_from_msg(msg, 'summary') if 'keywords' in msg: - self.keywords = _read_field('keywords').split(',') + self.keywords = _read_field_from_msg(msg, 'keywords').split(',') - self.platforms = _read_list('platform') - self.classifiers = _read_list('classifier') + self.platforms = _read_list_from_msg(msg, 'platform') + self.classifiers = _read_list_from_msg(msg, 'classifier') # PEP 314 - these fields only exist in 1.1 if self.metadata_version == StrictVersion('1.1'): - self.requires = _read_list('requires') - self.provides = _read_list('provides') - self.obsoletes = _read_list('obsoletes') + self.requires = _read_list_from_msg(msg, 'requires') + self.provides = _read_list_from_msg(msg, 'provides') + self.obsoletes = _read_list_from_msg(msg, 'obsoletes') else: self.requires = None self.provides = None self.obsoletes = None +def single_line(val): + # quick and dirty validation for description pypa/setuptools#1390 + if '\n' in val: + # TODO after 2021-07-31: Replace with `raise ValueError("newlines not allowed")` + warnings.warn("newlines not allowed and will break in the future") + val = val.replace('\n', ' ') + return val + + # Based on Python 3.5 version -def write_pkg_file(self, file): +def write_pkg_file(self, file): # noqa: C901 # is too complex (14) # FIXME """Write the PKG-INFO format data to a file object. """ version = self.get_metadata_version() @@ -130,7 +168,7 @@ def write_pkg_file(self, file): write_field('Metadata-Version', str(version)) write_field('Name', self.get_name()) write_field('Version', self.get_version()) - write_field('Summary', self.get_description()) + write_field('Summary', single_line(self.get_description())) write_field('Home-page', self.get_url()) if version < StrictVersion('1.2'): @@ -283,7 +321,7 @@ def check_specifier(dist, attr, value): """Verify that value is a valid version specifier""" try: packaging.specifiers.SpecifierSet(value) - except packaging.specifiers.InvalidSpecifier as error: + except (packaging.specifiers.InvalidSpecifier, AttributeError) as error: tmpl = ( "{attr!r} must be a string " "containing valid version specifiers; {error}" @@ -548,7 +586,8 @@ class Distribution(_Distribution): req.marker = None return req - def _parse_config_files(self, filenames=None): + # FIXME: 'Distribution._parse_config_files' is too complex (14) + def _parse_config_files(self, filenames=None): # noqa: C901 """ Adapted from distutils.dist.Distribution.parse_config_files, this method provides the same functionality in subtly-improved @@ -557,14 +596,12 @@ class Distribution(_Distribution): from configparser import ConfigParser # Ignore install directory options if we have a venv - if sys.prefix != sys.base_prefix: - ignore_options = [ - 'install-base', 'install-platbase', 'install-lib', - 'install-platlib', 'install-purelib', 'install-headers', - 'install-scripts', 'install-data', 'prefix', 'exec-prefix', - 'home', 'user', 'root'] - else: - ignore_options = [] + ignore_options = [] if sys.prefix == sys.base_prefix else [ + 'install-base', 'install-platbase', 'install-lib', + 'install-platlib', 'install-purelib', 'install-headers', + 'install-scripts', 'install-data', 'prefix', 'exec-prefix', + 'home', 'user', 'root', + ] ignore_options = frozenset(ignore_options) @@ -575,6 +612,7 @@ class Distribution(_Distribution): self.announce("Distribution.parse_config_files():") parser = ConfigParser() + parser.optionxform = str for filename in filenames: with io.open(filename, encoding='utf-8') as reader: if DEBUG: @@ -585,32 +623,69 @@ class Distribution(_Distribution): opt_dict = self.get_option_dict(section) for opt in options: - if opt != '__name__' and opt not in ignore_options: - val = parser.get(section, opt) - opt = opt.replace('-', '_') - opt_dict[opt] = (filename, val) + if opt == '__name__' or opt in ignore_options: + continue + + val = parser.get(section, opt) + opt = self.warn_dash_deprecation(opt, section) + opt = self.make_option_lowercase(opt, section) + opt_dict[opt] = (filename, val) # Make the ConfigParser forget everything (so we retain # the original filenames that options come from) parser.__init__() + if 'global' not in self.command_options: + return + # If there was a "global" section in the config file, use it # to set Distribution options. - if 'global' in self.command_options: - for (opt, (src, val)) in self.command_options['global'].items(): - alias = self.negative_opt.get(opt) - try: - if alias: - setattr(self, alias, not strtobool(val)) - elif opt in ('verbose', 'dry_run'): # ugh! - setattr(self, opt, strtobool(val)) - else: - setattr(self, opt, val) - except ValueError as e: - raise DistutilsOptionError(e) from e + for (opt, (src, val)) in self.command_options['global'].items(): + alias = self.negative_opt.get(opt) + if alias: + val = not strtobool(val) + elif opt in ('verbose', 'dry_run'): # ugh! + val = strtobool(val) + + try: + setattr(self, alias or opt, val) + except ValueError as e: + raise DistutilsOptionError(e) from e + + def warn_dash_deprecation(self, opt, section): + if section in ( + 'options.extras_require', 'options.data_files', + ): + return opt + + underscore_opt = opt.replace('-', '_') + commands = distutils.command.__all__ + setuptools.command.__all__ + if (not section.startswith('options') and section != 'metadata' + and section not in commands): + return underscore_opt + + if '-' in opt: + warnings.warn( + "Usage of dash-separated '%s' will not be supported in future " + "versions. Please use the underscore name '%s' instead" + % (opt, underscore_opt)) + return underscore_opt + + def make_option_lowercase(self, opt, section): + if section != 'metadata' or opt.islower(): + return opt + + lowercase_opt = opt.lower() + warnings.warn( + "Usage of uppercase key '%s' in '%s' will be deprecated in future " + "versions. Please use lowercase '%s' instead" + % (opt, section, lowercase_opt) + ) + return lowercase_opt - def _set_command_options(self, command_obj, option_dict=None): + # FIXME: 'Distribution._set_command_options' is too complex (14) + def _set_command_options(self, command_obj, option_dict=None): # noqa: C901 """ Set the options for 'command_obj' from 'option_dict'. Basically this means copying elements of a dictionary ('option_dict') to diff --git a/setuptools/extern/__init__.py b/setuptools/extern/__init__.py index b7f30dc2..7df32fde 100644 --- a/setuptools/extern/__init__.py +++ b/setuptools/extern/__init__.py @@ -1,3 +1,4 @@ +import importlib.util import sys @@ -20,17 +21,10 @@ class VendorImporter: yield self.vendor_pkg + '.' yield '' - def find_module(self, fullname, path=None): - """ - Return self when fullname starts with root_name and the - target module is one vendored through this importer. - """ + def _module_matches_namespace(self, fullname): + """Figure out if the target module is vendored.""" root, base, target = fullname.partition(self.root_name + '.') - if root: - return - if not any(map(target.startswith, self.vendored_names)): - return - return self + return not root and any(map(target.startswith, self.vendored_names)) def load_module(self, fullname): """ @@ -54,6 +48,19 @@ class VendorImporter: "distribution.".format(**locals()) ) + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + + def find_spec(self, fullname, path=None, target=None): + """Return a module spec for vendored names.""" + return ( + importlib.util.spec_from_loader(fullname, self) + if self._module_matches_namespace(fullname) else None + ) + def install(self): """ Install this importer into sys.meta_path if not already present. diff --git a/setuptools/glob.py b/setuptools/glob.py index 9d7cbc5d..87062b81 100644 --- a/setuptools/glob.py +++ b/setuptools/glob.py @@ -47,6 +47,8 @@ def iglob(pathname, recursive=False): def _iglob(pathname, recursive): dirname, basename = os.path.split(pathname) + glob_in_dir = glob2 if recursive and _isrecursive(basename) else glob1 + if not has_magic(pathname): if basename: if os.path.lexists(pathname): @@ -56,13 +58,9 @@ def _iglob(pathname, recursive): if os.path.isdir(dirname): yield pathname return + if not dirname: - if recursive and _isrecursive(basename): - for x in glob2(dirname, basename): - yield x - else: - for x in glob1(dirname, basename): - yield x + yield from glob_in_dir(dirname, basename) return # `os.path.split()` returns the argument itself as a dirname if it is a # drive or UNC path. Prevent an infinite recursion if a drive or UNC path @@ -71,12 +69,7 @@ def _iglob(pathname, recursive): dirs = _iglob(dirname, recursive) else: dirs = [dirname] - if has_magic(basename): - if recursive and _isrecursive(basename): - glob_in_dir = glob2 - else: - glob_in_dir = glob1 - else: + if not has_magic(basename): glob_in_dir = glob0 for dirname in dirs: for name in glob_in_dir(dirname, basename): diff --git a/setuptools/installer.py b/setuptools/installer.py index e630b874..57e2b587 100644 --- a/setuptools/installer.py +++ b/setuptools/installer.py @@ -7,7 +7,6 @@ from distutils import log from distutils.errors import DistutilsError import pkg_resources -from setuptools.command.easy_install import easy_install from setuptools.wheel import Wheel @@ -19,54 +18,11 @@ def _fixup_find_links(find_links): return find_links -def _legacy_fetch_build_egg(dist, req): - """Fetch an egg needed for building. - - Legacy path using EasyInstall. - """ - tmp_dist = dist.__class__({'script_args': ['easy_install']}) - opts = tmp_dist.get_option_dict('easy_install') - opts.clear() - opts.update( - (k, v) - for k, v in dist.get_option_dict('easy_install').items() - if k in ( - # don't use any other settings - 'find_links', 'site_dirs', 'index_url', - 'optimize', 'site_dirs', 'allow_hosts', - )) - if dist.dependency_links: - links = dist.dependency_links[:] - if 'find_links' in opts: - links = _fixup_find_links(opts['find_links'][1]) + links - opts['find_links'] = ('setup', links) - install_dir = dist.get_egg_cache_dir() - cmd = easy_install( - tmp_dist, args=["x"], install_dir=install_dir, - exclude_scripts=True, - always_copy=False, build_directory=None, editable=False, - upgrade=False, multi_version=True, no_report=True, user=False - ) - cmd.ensure_finalized() - return cmd.easy_install(req) - - -def fetch_build_egg(dist, req): +def fetch_build_egg(dist, req): # noqa: C901 # is too complex (16) # FIXME """Fetch an egg needed for building. Use pip/wheel to fetch/build a wheel.""" - # Check pip is available. - try: - pkg_resources.get_distribution('pip') - except pkg_resources.DistributionNotFound: - dist.announce( - 'WARNING: The pip package is not available, falling back ' - 'to EasyInstall for handling setup_requires/test_requires; ' - 'this is deprecated and will be removed in a future version.', - log.WARN - ) - return _legacy_fetch_build_egg(dist, req) - # Warn if wheel is not. + # Warn if wheel is not available try: pkg_resources.get_distribution('wheel') except pkg_resources.DistributionNotFound: @@ -80,20 +36,17 @@ def fetch_build_egg(dist, req): if 'allow_hosts' in opts: raise DistutilsError('the `allow-hosts` option is not supported ' 'when using pip to install requirements.') - if 'PIP_QUIET' in os.environ or 'PIP_VERBOSE' in os.environ: - quiet = False - else: - quiet = True + quiet = 'PIP_QUIET' not in os.environ and 'PIP_VERBOSE' not in os.environ if 'PIP_INDEX_URL' in os.environ: index_url = None elif 'index_url' in opts: index_url = opts['index_url'][1] else: index_url = None - if 'find_links' in opts: - find_links = _fixup_find_links(opts['find_links'][1])[:] - else: - find_links = [] + find_links = ( + _fixup_find_links(opts['find_links'][1])[:] if 'find_links' in opts + else [] + ) if dist.dependency_links: find_links.extend(dist.dependency_links) eggs_dir = os.path.realpath(dist.get_egg_cache_dir()) @@ -112,16 +65,12 @@ def fetch_build_egg(dist, req): cmd.append('--quiet') if index_url is not None: cmd.extend(('--index-url', index_url)) - if find_links is not None: - for link in find_links: - cmd.extend(('--find-links', link)) + for link in find_links or []: + cmd.extend(('--find-links', link)) # If requirement is a PEP 508 direct URL, directly pass # the URL to pip, as `req @ url` does not work on the # command line. - if req.url: - cmd.append(req.url) - else: - cmd.append(str(req)) + cmd.append(req.url or str(req)) try: subprocess.check_call(cmd) except subprocess.CalledProcessError as e: diff --git a/setuptools/msvc.py b/setuptools/msvc.py index 1ead72b4..d5e0a952 100644 --- a/setuptools/msvc.py +++ b/setuptools/msvc.py @@ -24,6 +24,7 @@ from io import open from os import listdir, pathsep from os.path import join, isfile, isdir, dirname import sys +import contextlib import platform import itertools import subprocess @@ -724,28 +725,23 @@ class SystemInfo: ms = self.ri.microsoft vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs) vs_vers = [] - for hkey in self.ri.HKEYS: - for key in vckeys: - try: - bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ) - except (OSError, IOError): - continue - with bkey: - subkeys, values, _ = winreg.QueryInfoKey(bkey) - for i in range(values): - try: - ver = float(winreg.EnumValue(bkey, i)[0]) - if ver not in vs_vers: - vs_vers.append(ver) - except ValueError: - pass - for i in range(subkeys): - try: - ver = float(winreg.EnumKey(bkey, i)) - if ver not in vs_vers: - vs_vers.append(ver) - except ValueError: - pass + for hkey, key in itertools.product(self.ri.HKEYS, vckeys): + try: + bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ) + except (OSError, IOError): + continue + with bkey: + subkeys, values, _ = winreg.QueryInfoKey(bkey) + for i in range(values): + with contextlib.suppress(ValueError): + ver = float(winreg.EnumValue(bkey, i)[0]) + if ver not in vs_vers: + vs_vers.append(ver) + for i in range(subkeys): + with contextlib.suppress(ValueError): + ver = float(winreg.EnumKey(bkey, i)) + if ver not in vs_vers: + vs_vers.append(ver) return sorted(vs_vers) def find_programdata_vs_vers(self): @@ -925,8 +921,8 @@ class SystemInfo: """ return self._use_last_dir_name(join(self.WindowsSdkDir, 'lib')) - @property - def WindowsSdkDir(self): + @property # noqa: C901 + def WindowsSdkDir(self): # noqa: C901 # is too complex (12) # FIXME """ Microsoft Windows SDK directory. diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3979b131..123e9582 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -320,7 +320,8 @@ class PackageIndex(Environment): else: self.opener = urllib.request.urlopen - def process_url(self, url, retrieve=False): + # FIXME: 'PackageIndex.process_url' is too complex (14) + def process_url(self, url, retrieve=False): # noqa: C901 """Evaluate a URL as a possible download, and maybe retrieve it""" if url in self.scanned_urls and not retrieve: return @@ -428,49 +429,53 @@ class PackageIndex(Environment): dist.precedence = SOURCE_DIST self.add(dist) + def _scan(self, link): + # Process a URL to see if it's for a package page + NO_MATCH_SENTINEL = None, None + if not link.startswith(self.index_url): + return NO_MATCH_SENTINEL + + parts = list(map( + urllib.parse.unquote, link[len(self.index_url):].split('/') + )) + if len(parts) != 2 or '#' in parts[1]: + return NO_MATCH_SENTINEL + + # it's a package page, sanitize and index it + pkg = safe_name(parts[0]) + ver = safe_version(parts[1]) + self.package_pages.setdefault(pkg.lower(), {})[link] = True + return to_filename(pkg), to_filename(ver) + def process_index(self, url, page): """Process the contents of a PyPI page""" - def scan(link): - # Process a URL to see if it's for a package page - if link.startswith(self.index_url): - parts = list(map( - urllib.parse.unquote, link[len(self.index_url):].split('/') - )) - if len(parts) == 2 and '#' not in parts[1]: - # it's a package page, sanitize and index it - pkg = safe_name(parts[0]) - ver = safe_version(parts[1]) - self.package_pages.setdefault(pkg.lower(), {})[link] = True - return to_filename(pkg), to_filename(ver) - return None, None - # process an index page into the package-page index for match in HREF.finditer(page): try: - scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) + self._scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) except ValueError: pass - pkg, ver = scan(url) # ensure this page is in the page index - if pkg: - # process individual package page - for new_url in find_external_links(url, page): - # Process the found URL - base, frag = egg_info_for_url(new_url) - if base.endswith('.py') and not frag: - if ver: - new_url += '#egg=%s-%s' % (pkg, ver) - else: - self.need_version_info(url) - self.scan_url(new_url) - - return PYPI_MD5.sub( - lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page - ) - else: + pkg, ver = self._scan(url) # ensure this page is in the page index + if not pkg: return "" # no sense double-scanning non-package pages + # process individual package page + for new_url in find_external_links(url, page): + # Process the found URL + base, frag = egg_info_for_url(new_url) + if base.endswith('.py') and not frag: + if ver: + new_url += '#egg=%s-%s' % (pkg, ver) + else: + self.need_version_info(url) + self.scan_url(new_url) + + return PYPI_MD5.sub( + lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page + ) + def need_version_info(self, url): self.scan_all( "Page at %s links to .py file(s) without version info; an index " @@ -591,7 +596,7 @@ class PackageIndex(Environment): spec = parse_requirement_arg(spec) return getattr(self.fetch_distribution(spec, tmpdir), 'location', None) - def fetch_distribution( + def fetch_distribution( # noqa: C901 # is too complex (14) # FIXME self, requirement, tmpdir, force_scan=False, source=False, develop_ok=False, local_index=None): """Obtain a distribution suitable for fulfilling `requirement` @@ -762,7 +767,8 @@ class PackageIndex(Environment): def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op - def open_url(self, url, warning=None): + # FIXME: + def open_url(self, url, warning=None): # noqa: C901 # is too complex (12) if url.startswith('file:'): return local_open(url) try: diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py index eac5e656..b58cca37 100644 --- a/setuptools/ssl_support.py +++ b/setuptools/ssl_support.py @@ -56,7 +56,7 @@ if not CertificateError: pass -if not match_hostname: +if not match_hostname: # noqa: C901 # 'If 59' is too complex (21) # FIXME def _dnsname_match(dn, hostname, max_wildcards=1): """Matching according to RFC 6125, section 6.4.3 diff --git a/setuptools/tests/files.py b/setuptools/tests/files.py deleted file mode 100644 index 71194b9d..00000000 --- a/setuptools/tests/files.py +++ /dev/null @@ -1,38 +0,0 @@ -import os - - -def build_files(file_defs, prefix=""): - """ - Build a set of files/directories, as described by the - file_defs dictionary. - - Each key/value pair in the dictionary is interpreted as - a filename/contents - pair. If the contents value is a dictionary, a directory - is created, and the - dictionary interpreted as the files within it, recursively. - - For example: - - {"README.txt": "A README file", - "foo": { - "__init__.py": "", - "bar": { - "__init__.py": "", - }, - "baz.py": "# Some code", - } - } - """ - for name, contents in file_defs.items(): - full_name = os.path.join(prefix, name) - if isinstance(contents, dict): - os.makedirs(full_name, exist_ok=True) - build_files(contents, prefix=full_name) - else: - if isinstance(contents, bytes): - with open(full_name, 'wb') as f: - f.write(contents) - else: - with open(full_name, 'w') as f: - f.write(contents) diff --git a/setuptools/tests/fixtures.py b/setuptools/tests/fixtures.py index 5204c8d1..a5a172e0 100644 --- a/setuptools/tests/fixtures.py +++ b/setuptools/tests/fixtures.py @@ -1,9 +1,14 @@ +import contextlib +import sys +import shutil +import subprocess + import pytest from . import contexts -@pytest.yield_fixture +@pytest.fixture def user_override(monkeypatch): """ Override site.USER_BASE and site.USER_SITE with temporary directories in @@ -17,7 +22,53 @@ def user_override(monkeypatch): yield -@pytest.yield_fixture +@pytest.fixture def tmpdir_cwd(tmpdir): with tmpdir.as_cwd() as orig: yield orig + + +@pytest.fixture +def tmp_src(request, tmp_path): + """Make a copy of the source dir under `$tmp/src`. + + This fixture is useful whenever it's necessary to run `setup.py` + or `pip install` against the source directory when there's no + control over the number of simultaneous invocations. Such + concurrent runs create and delete directories with the same names + under the target directory and so they influence each other's runs + when they are not being executed sequentially. + """ + tmp_src_path = tmp_path / 'src' + shutil.copytree(request.config.rootdir, tmp_src_path) + return tmp_src_path + + +@pytest.fixture(autouse=True, scope="session") +def workaround_xdist_376(request): + """ + Workaround pytest-dev/pytest-xdist#376 + + ``pytest-xdist`` tends to inject '' into ``sys.path``, + which may break certain isolation expectations. + Remove the entry so the import + machinery behaves the same irrespective of xdist. + """ + if not request.config.pluginmanager.has_plugin('xdist'): + return + + with contextlib.suppress(ValueError): + sys.path.remove('') + + +@pytest.fixture +def sample_project(tmp_path): + """ + Clone the 'sampleproject' and return a path to it. + """ + cmd = ['git', 'clone', 'https://github.com/pypa/sampleproject'] + try: + subprocess.check_call(cmd, cwd=str(tmp_path)) + except Exception: + pytest.skip("Unable to clone sampleproject") + return tmp_path / 'sampleproject' diff --git a/setuptools/tests/requirements.txt b/setuptools/tests/requirements.txt index d0d07f70..b2d84a94 100644 --- a/setuptools/tests/requirements.txt +++ b/setuptools/tests/requirements.txt @@ -11,3 +11,4 @@ paver; python_version>="3.6" futures; python_version=="2.7" pip>=19.1 # For proper file:// URLs support. jaraco.envs +sphinx diff --git a/setuptools/tests/server.py b/setuptools/tests/server.py index 7e213230..6717c053 100644 --- a/setuptools/tests/server.py +++ b/setuptools/tests/server.py @@ -65,7 +65,7 @@ class MockServer(http.server.HTTPServer, threading.Thread): http.server.HTTPServer.__init__( self, server_address, RequestHandlerClass) threading.Thread.__init__(self) - self.setDaemon(True) + self.daemon = True self.requests = [] def run(self): diff --git a/setuptools/tests/test_bdist_deprecations.py b/setuptools/tests/test_bdist_deprecations.py deleted file mode 100644 index 704164aa..00000000 --- a/setuptools/tests/test_bdist_deprecations.py +++ /dev/null @@ -1,23 +0,0 @@ -"""develop tests -""" -import mock - -import pytest - -from setuptools.dist import Distribution -from setuptools import SetuptoolsDeprecationWarning - - -@mock.patch("distutils.command.bdist_wininst.bdist_wininst") -def test_bdist_wininst_warning(distutils_cmd): - dist = Distribution(dict( - script_name='setup.py', - script_args=['bdist_wininst'], - name='foo', - py_modules=['hi'], - )) - dist.parse_command_line() - with pytest.warns(SetuptoolsDeprecationWarning): - dist.run_commands() - - distutils_cmd.run.assert_called_once() diff --git a/setuptools/tests/test_bdist_egg.py b/setuptools/tests/test_bdist_egg.py index 8760ea30..fb5b90b1 100644 --- a/setuptools/tests/test_bdist_egg.py +++ b/setuptools/tests/test_bdist_egg.py @@ -7,7 +7,6 @@ import zipfile import pytest from setuptools.dist import Distribution -from setuptools import SetuptoolsDeprecationWarning from . import contexts @@ -65,17 +64,3 @@ class Test: names = list(zi.filename for zi in zip.filelist) assert 'hi.pyc' in names assert 'hi.py' not in names - - def test_eggsecutable_warning(self, setup_context, user_override): - dist = Distribution(dict( - script_name='setup.py', - script_args=['bdist_egg'], - name='foo', - py_modules=['hi'], - entry_points={ - 'setuptools.installation': - ['eggsecutable = my_package.some_module:main_func']}, - )) - dist.parse_command_line() - with pytest.warns(SetuptoolsDeprecationWarning): - dist.run_commands() diff --git a/setuptools/tests/test_build_ext.py b/setuptools/tests/test_build_ext.py index 838fdb42..b6deebe4 100644 --- a/setuptools/tests/test_build_ext.py +++ b/setuptools/tests/test_build_ext.py @@ -2,12 +2,13 @@ import sys import distutils.command.build_ext as orig from distutils.sysconfig import get_config_var +from jaraco import path + from setuptools.command.build_ext import build_ext, get_abi3_suffix from setuptools.dist import Distribution from setuptools.extension import Extension from . import environment -from .files import build_files from .textwrap import DALS @@ -103,10 +104,10 @@ def test_build_ext_config_handling(tmpdir_cwd): 'setup.cfg': DALS( """ [build] - build-base = foo_build + build_base = foo_build """), } - build_files(files) + path.build(files) code, output = environment.run_setup_py( cmd=['build'], data_stream=(0, 2), ) diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 5462b26a..ab75a189 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -3,15 +3,16 @@ import shutil import tarfile import importlib from concurrent import futures +import re import pytest +from jaraco import path -from .files import build_files from .textwrap import DALS class BuildBackendBase: - def __init__(self, cwd=None, env={}, backend_name='setuptools.build_meta'): + def __init__(self, cwd='.', env={}, backend_name='setuptools.build_meta'): self.cwd = cwd self.env = env self.backend_name = backend_name @@ -108,7 +109,7 @@ defns = [ 'setup.cfg': DALS(""" [metadata] name = foo - version='0.0.0' + version = 0.0.0 [options] py_modules=hello @@ -126,11 +127,11 @@ class TestBuildMetaBackend: backend_name = 'setuptools.build_meta' def get_build_backend(self): - return BuildBackend(cwd='.', backend_name=self.backend_name) + return BuildBackend(backend_name=self.backend_name) @pytest.fixture(params=defns) def build_backend(self, tmpdir, request): - build_files(request.param, prefix=str(tmpdir)) + path.build(request.param, prefix=str(tmpdir)) with tmpdir.as_cwd(): yield self.get_build_backend() @@ -170,7 +171,7 @@ class TestBuildMetaBackend: """), } - build_files(files) + path.build(files) dist_dir = os.path.abspath('preexisting-' + build_type) @@ -262,7 +263,7 @@ class TestBuildMetaBackend: build-backend = "setuptools.build_meta """), } - build_files(files) + path.build(files) build_backend = self.get_build_backend() targz_path = build_backend.build_sdist("temp") with tarfile.open(os.path.join("temp", targz_path)) as tar: @@ -271,7 +272,7 @@ class TestBuildMetaBackend: def test_build_sdist_setup_py_exists(self, tmpdir_cwd): # If build_sdist is called from a script other than setup.py, # ensure setup.py is included - build_files(defns[0]) + path.build(defns[0]) build_backend = self.get_build_backend() targz_path = build_backend.build_sdist("temp") @@ -293,7 +294,7 @@ class TestBuildMetaBackend: """) } - build_files(files) + path.build(files) build_backend = self.get_build_backend() targz_path = build_backend.build_sdist("temp") @@ -315,7 +316,7 @@ class TestBuildMetaBackend: """) } - build_files(files) + path.build(files) build_backend = self.get_build_backend() build_backend.build_sdist("temp") @@ -335,9 +336,9 @@ class TestBuildMetaBackend: } def test_build_sdist_relative_path_import(self, tmpdir_cwd): - build_files(self._relative_path_import_files) + path.build(self._relative_path_import_files) build_backend = self.get_build_backend() - with pytest.raises(ImportError): + with pytest.raises(ImportError, match="^No module named 'hello'$"): build_backend.build_sdist("temp") @pytest.mark.parametrize('setup_literal, requirements', [ @@ -374,7 +375,7 @@ class TestBuildMetaBackend: """), } - build_files(files) + path.build(files) build_backend = self.get_build_backend() @@ -409,7 +410,7 @@ class TestBuildMetaBackend: """), } - build_files(files) + path.build(files) build_backend = self.get_build_backend() @@ -437,11 +438,21 @@ class TestBuildMetaBackend: } def test_sys_argv_passthrough(self, tmpdir_cwd): - build_files(self._sys_argv_0_passthrough) + path.build(self._sys_argv_0_passthrough) build_backend = self.get_build_backend() with pytest.raises(AssertionError): build_backend.build_sdist("temp") + @pytest.mark.parametrize('build_hook', ('build_sdist', 'build_wheel')) + def test_build_with_empty_setuppy(self, build_backend, build_hook): + files = {'setup.py': ''} + path.build(files) + + with pytest.raises( + ValueError, + match=re.escape('No distribution was found.')): + getattr(build_backend, build_hook)("temp") + class TestBuildMetaLegacyBackend(TestBuildMetaBackend): backend_name = 'setuptools.build_meta:__legacy__' @@ -449,13 +460,13 @@ class TestBuildMetaLegacyBackend(TestBuildMetaBackend): # build_meta_legacy-specific tests def test_build_sdist_relative_path_import(self, tmpdir_cwd): # This must fail in build_meta, but must pass in build_meta_legacy - build_files(self._relative_path_import_files) + path.build(self._relative_path_import_files) build_backend = self.get_build_backend() build_backend.build_sdist("temp") def test_sys_argv_passthrough(self, tmpdir_cwd): - build_files(self._sys_argv_0_passthrough) + path.build(self._sys_argv_0_passthrough) build_backend = self.get_build_backend() build_backend.build_sdist("temp") diff --git a/setuptools/tests/test_config.py b/setuptools/tests/test_config.py index 1dee1271..21f1becd 100644 --- a/setuptools/tests/test_config.py +++ b/setuptools/tests/test_config.py @@ -1,3 +1,6 @@ +import types +import sys + import contextlib import configparser @@ -7,6 +10,7 @@ from distutils.errors import DistutilsOptionError, DistutilsFileError from mock import patch from setuptools.dist import Distribution, _Distribution from setuptools.config import ConfigHandler, read_configuration +from distutils.core import Command from .textwrap import DALS @@ -206,8 +210,8 @@ class TestMetadata: fake_env( tmpdir, '[metadata]\n' - 'author-email = test@test.com\n' - 'home-page = http://test.test.com/test/\n' + 'author_email = test@test.com\n' + 'home_page = http://test.test.com/test/\n' 'summary = Short summary\n' 'platform = a, b\n' 'classifier =\n' @@ -503,6 +507,44 @@ class TestMetadata: with get_dist(tmpdir): pass + def test_warn_dash_deprecation(self, tmpdir): + # warn_dash_deprecation() is a method in setuptools.dist + # remove this test and the method when no longer needed + fake_env( + tmpdir, + '[metadata]\n' + 'author-email = test@test.com\n' + 'maintainer_email = foo@foo.com\n' + ) + msg = ("Usage of dash-separated 'author-email' will not be supported " + "in future versions. " + "Please use the underscore name 'author_email' instead") + with pytest.warns(UserWarning, match=msg): + with get_dist(tmpdir) as dist: + metadata = dist.metadata + + assert metadata.author_email == 'test@test.com' + assert metadata.maintainer_email == 'foo@foo.com' + + def test_make_option_lowercase(self, tmpdir): + # remove this test and the method make_option_lowercase() in setuptools.dist + # when no longer needed + fake_env( + tmpdir, + '[metadata]\n' + 'Name = foo\n' + 'description = Some description\n' + ) + msg = ("Usage of uppercase key 'Name' in 'metadata' will be deprecated in " + "future versions. " + "Please use lowercase 'name' instead") + with pytest.warns(UserWarning, match=msg): + with get_dist(tmpdir) as dist: + metadata = dist.metadata + + assert metadata.name == 'foo' + assert metadata.description == 'Some description' + class TestOptions: @@ -768,6 +810,20 @@ class TestOptions: } assert dist.metadata.provides_extras == set(['pdf', 'rest']) + def test_dash_preserved_extras_require(self, tmpdir): + fake_env( + tmpdir, + '[options.extras_require]\n' + 'foo-a = foo\n' + 'foo_b = test\n' + ) + + with get_dist(tmpdir) as dist: + assert dist.extras_require == { + 'foo-a': ['foo'], + 'foo_b': ['test'] + } + def test_entry_points(self, tmpdir): _, config = fake_env( tmpdir, @@ -802,6 +858,24 @@ class TestOptions: with get_dist(tmpdir) as dist: assert dist.entry_points == expected + def test_case_sensitive_entry_points(self, tmpdir): + _, config = fake_env( + tmpdir, + '[options.entry_points]\n' + 'GROUP1 = point1 = pack.module:func, ' + '.point2 = pack.module2:func_rest [rest]\n' + 'group2 = point3 = pack.module:func2\n' + ) + + with get_dist(tmpdir) as dist: + assert dist.entry_points == { + 'GROUP1': [ + 'point1 = pack.module:func', + '.point2 = pack.module2:func_rest [rest]', + ], + 'group2': ['point3 = pack.module:func2'] + } + def test_data_files(self, tmpdir): fake_env( tmpdir, @@ -853,6 +927,26 @@ class TestOptions: with get_dist(tmpdir) as dist: dist.parse_config_files() + def test_cmdclass(self, tmpdir): + class CustomCmd(Command): + pass + + m = types.ModuleType('custom_build', 'test package') + + m.__dict__['CustomCmd'] = CustomCmd + + sys.modules['custom_build'] = m + + fake_env( + tmpdir, + '[options]\n' + 'cmdclass =\n' + ' customcmd = custom_build.CustomCmd\n' + ) + + with get_dist(tmpdir) as dist: + assert dist.cmdclass == {'customcmd': CustomCmd} + saved_dist_init = _Distribution.__init__ diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index 9854420e..df8db4e2 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -7,6 +7,8 @@ import sys import io import subprocess import platform +import pathlib +import textwrap from setuptools.command import test @@ -31,7 +33,7 @@ INIT_PY = """print "foo" """ -@pytest.yield_fixture +@pytest.fixture def temp_user(monkeypatch): with contexts.tempdir() as user_base: with contexts.tempdir() as user_site: @@ -40,7 +42,7 @@ def temp_user(monkeypatch): yield -@pytest.yield_fixture +@pytest.fixture def test_env(tmpdir, temp_user): target = tmpdir foo = target.mkdir('foo') @@ -199,3 +201,55 @@ class TestNamespaces: ] with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(pkg_resources_imp) + + @staticmethod + def install_workaround(site_packages): + site_packages.mkdir(parents=True) + sc = site_packages / 'sitecustomize.py' + sc.write_text(textwrap.dedent(""" + import site + import pathlib + here = pathlib.Path(__file__).parent + site.addsitedir(str(here)) + """).lstrip()) + + @pytest.mark.xfail( + platform.python_implementation() == 'PyPy', + reason="Workaround fails on PyPy (why?)", + ) + def test_editable_prefix(self, tmp_path, sample_project): + """ + Editable install to a prefix should be discoverable. + """ + prefix = tmp_path / 'prefix' + prefix.mkdir() + + # figure out where pip will likely install the package + site_packages = prefix / next( + pathlib.Path(path).relative_to(sys.prefix) + for path in sys.path + if 'site-packages' in path + and path.startswith(sys.prefix) + ) + + # install the workaround + self.install_workaround(site_packages) + + env = dict(os.environ, PYTHONPATH=str(site_packages)) + cmd = [ + sys.executable, + '-m', 'pip', + 'install', + '--editable', + str(sample_project), + '--prefix', str(prefix), + '--no-build-isolation', + ] + subprocess.check_call(cmd, env=env) + + # now run 'sample' with the prefix on the PYTHONPATH + bin = 'Scripts' if platform.system() == 'Windows' else 'bin' + exe = prefix / bin / 'sample' + if sys.version_info < (3, 7) and platform.system() == 'Windows': + exe = str(exe) + subprocess.check_call([exe], env=env) diff --git a/setuptools/tests/test_dist.py b/setuptools/tests/test_dist.py index cb47fb58..dcec1734 100644 --- a/setuptools/tests/test_dist.py +++ b/setuptools/tests/test_dist.py @@ -9,6 +9,9 @@ from setuptools.dist import ( _get_unpatched, check_package_data, DistDeprecationWarning, + check_specifier, + rfc822_escape, + rfc822_unescape, ) from setuptools import sic from setuptools import Distribution @@ -84,6 +87,9 @@ def __read_test_cases(): ('Metadata version 1.1: Provides', params( provides=['package'], )), + ('Metadata Version 1.0: Short long description', params( + long_description='Short long description', + )), ('Metadata version 1.1: Obsoletes', params( obsoletes=['foo'], )), @@ -161,6 +167,7 @@ def test_read_metadata(name, attrs): ('metadata_version', dist_class.get_metadata_version), ('provides', dist_class.get_provides), ('description', dist_class.get_description), + ('long_description', dist_class.get_long_description), ('download_url', dist_class.get_download_url), ('keywords', dist_class.get_keywords), ('platforms', dist_class.get_platforms), @@ -323,3 +330,49 @@ def test_check_package_data(package_data, expected_message): with pytest.raises( DistutilsSetupError, match=re.escape(expected_message)): check_package_data(None, str('package_data'), package_data) + + +def test_check_specifier(): + # valid specifier value + attrs = {'name': 'foo', 'python_requires': '>=3.0, !=3.1'} + dist = Distribution(attrs) + check_specifier(dist, attrs, attrs['python_requires']) + + # invalid specifier value + attrs = {'name': 'foo', 'python_requires': ['>=3.0', '!=3.1']} + with pytest.raises(DistutilsSetupError): + dist = Distribution(attrs) + + +@pytest.mark.parametrize( + 'content, result', + ( + pytest.param( + "Just a single line", + None, + id="single_line", + ), + pytest.param( + "Multiline\nText\nwithout\nextra indents\n", + None, + id="multiline", + ), + pytest.param( + "Multiline\n With\n\nadditional\n indentation", + None, + id="multiline_with_indentation", + ), + pytest.param( + " Leading whitespace", + "Leading whitespace", + id="remove_leading_whitespace", + ), + pytest.param( + " Leading whitespace\nIn\n Multiline comment", + "Leading whitespace\nIn\n Multiline comment", + id="remove_leading_whitespace_multiline", + ), + ) +) +def test_rfc822_unescape(content, result): + assert (result or content) == rfc822_unescape(rfc822_escape(content)) diff --git a/setuptools/tests/test_distutils_adoption.py b/setuptools/tests/test_distutils_adoption.py index a53773df..0e89921c 100644 --- a/setuptools/tests/test_distutils_adoption.py +++ b/setuptools/tests/test_distutils_adoption.py @@ -21,10 +21,10 @@ class VirtualEnv(jaraco.envs.VirtualEnv): @pytest.fixture -def venv(tmpdir): +def venv(tmp_path, tmp_src): env = VirtualEnv() - env.root = path.Path(tmpdir) - env.req = os.getcwd() + env.root = path.Path(tmp_path / 'venv') + env.req = str(tmp_src) return env.create() diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index 26a5e9a6..a3b2d6e6 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -15,8 +15,10 @@ import zipfile import mock import time import re +import subprocess import pytest +from jaraco import path from setuptools import sandbox from setuptools.sandbox import run_setup @@ -25,7 +27,6 @@ from setuptools.command.easy_install import ( EasyInstallDeprecationWarning, ScriptWriter, PthDistributions, WindowsScriptWriter, ) -from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution from pkg_resources import normalize_path, working_set from pkg_resources import Distribution as PRDistribution @@ -34,10 +35,19 @@ from setuptools.tests import fail_on_ascii import pkg_resources from . import contexts -from .files import build_files from .textwrap import DALS +@pytest.fixture(autouse=True) +def pip_disable_index(monkeypatch): + """ + Important: Disable the default index for pip to avoid + querying packages in the index and potentially resolving + and installing packages there. + """ + monkeypatch.setenv('PIP_NO_INDEX', 'true') + + class FakeDist: def get_entry_map(self, group): if group != 'console_scripts': @@ -305,7 +315,7 @@ class TestPTHFileWriter: assert not pth.dirty -@pytest.yield_fixture +@pytest.fixture def setup_context(tmpdir): with (tmpdir / 'setup.py').open('w') as f: f.write(SETUP_PY) @@ -361,7 +371,7 @@ class TestUserInstallTest: f.write('Name: foo\n') return str(tmpdir) - @pytest.yield_fixture() + @pytest.fixture() def install_target(self, tmpdir): target = str(tmpdir) with mock.patch('sys.path', sys.path + [target]): @@ -406,7 +416,7 @@ class TestUserInstallTest: ) -@pytest.yield_fixture +@pytest.fixture def distutils_package(): distutils_setup_py = SETUP_PY.replace( 'from setuptools import setup', @@ -445,22 +455,22 @@ class TestSetupRequires: """ monkeypatch.setenv(str('PIP_RETRIES'), str('0')) monkeypatch.setenv(str('PIP_TIMEOUT'), str('0')) + monkeypatch.setenv('PIP_NO_INDEX', 'false') with contexts.quiet(): # create an sdist that has a build-time dependency. with TestSetupRequires.create_sdist() as dist_file: with contexts.tempdir() as temp_install_dir: with contexts.environment(PYTHONPATH=temp_install_dir): - ei_params = [ + cmd = [ + sys.executable, + '-m', 'setup', + 'easy_install', '--index-url', mock_index.url, '--exclude-scripts', '--install-dir', temp_install_dir, dist_file, ] - with sandbox.save_argv(['easy_install']): - # attempt to install the dist. It should - # fail because it doesn't exist. - with pytest.raises(SystemExit): - easy_install_pkg.main(ei_params) + subprocess.Popen(cmd).wait() # there should have been one requests to the server assert [r.path for r in mock_index.requests] == ['/does-not-exist/'] @@ -618,6 +628,7 @@ class TestSetupRequires: def test_setup_requires_honors_pip_env(self, mock_index, monkeypatch): monkeypatch.setenv(str('PIP_RETRIES'), str('0')) monkeypatch.setenv(str('PIP_TIMEOUT'), str('0')) + monkeypatch.setenv('PIP_NO_INDEX', 'false') monkeypatch.setenv(str('PIP_INDEX_URL'), mock_index.url) with contexts.save_pkg_resources_state(): with contexts.tempdir() as temp_dir: @@ -730,10 +741,10 @@ class TestSetupRequires: assert eggs == ['dep 1.0'] @pytest.mark.parametrize( - 'use_legacy_installer,with_dependency_links_in_setup_py', - itertools.product((False, True), (False, True))) + 'with_dependency_links_in_setup_py', + (False, True)) def test_setup_requires_with_find_links_in_setup_cfg( - self, monkeypatch, use_legacy_installer, + self, monkeypatch, with_dependency_links_in_setup_py): monkeypatch.setenv(str('PIP_RETRIES'), str('0')) monkeypatch.setenv(str('PIP_TIMEOUT'), str('0')) @@ -755,11 +766,9 @@ class TestSetupRequires: fp.write(DALS( ''' from setuptools import installer, setup - if {use_legacy_installer}: - installer.fetch_build_egg = installer._legacy_fetch_build_egg setup(setup_requires='python-xlib==42', dependency_links={dependency_links!r}) - ''').format(use_legacy_installer=use_legacy_installer, # noqa + ''').format( dependency_links=dependency_links)) with open(test_setup_cfg, 'w') as fp: fp.write(DALS( @@ -785,7 +794,7 @@ class TestSetupRequires: # Create source tree for `dep`. dep_pkg = os.path.join(temp_dir, 'dep') os.mkdir(dep_pkg) - build_files({ + path.build({ 'setup.py': DALS(""" import setuptools diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index dc472af4..80d35774 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -6,15 +6,15 @@ import re import stat import time +import pytest +from jaraco import path + from setuptools.command.egg_info import ( egg_info, manifest_maker, EggInfoDeprecationWarning, get_pkg_info_revision, ) from setuptools.dist import Distribution -import pytest - from . import environment -from .files import build_files from .textwrap import DALS from . import contexts @@ -37,7 +37,7 @@ class TestEggInfo: """) def _create_project(self): - build_files({ + path.build({ 'setup.py': self.setup_script, 'hello.py': DALS(""" def run(): @@ -45,7 +45,7 @@ class TestEggInfo: """) }) - @pytest.yield_fixture + @pytest.fixture def env(self): with contexts.tempdir(prefix='setuptools-test.') as env_dir: env = Environment(env_dir) @@ -56,7 +56,7 @@ class TestEggInfo: for dirname in subs ) list(map(os.mkdir, env.paths.values())) - build_files({ + path.build({ env.paths['home']: { '.pydistutils.cfg': DALS(""" [egg_info] @@ -106,7 +106,7 @@ class TestEggInfo: the file should remain unchanged. """ setup_cfg = os.path.join(env.paths['home'], 'setup.cfg') - build_files({ + path.build({ setup_cfg: DALS(""" [egg_info] tag_build = @@ -159,8 +159,10 @@ class TestEggInfo: setup() """) - build_files({'setup.py': setup_script, - 'setup.cfg': setup_config}) + path.build({ + 'setup.py': setup_script, + 'setup.cfg': setup_config, + }) # This command should fail with a ValueError, but because it's # currently configured to use a subprocess, the actual traceback @@ -193,7 +195,7 @@ class TestEggInfo: def test_manifest_template_is_read(self, tmpdir_cwd, env): self._create_project() - build_files({ + path.build({ 'MANIFEST.in': DALS(""" recursive-include docs *.rst """), @@ -216,8 +218,10 @@ class TestEggInfo: ''' ) % ('' if use_setup_cfg else requires) setup_config = requires if use_setup_cfg else '' - build_files({'setup.py': setup_script, - 'setup.cfg': setup_config}) + path.build({ + 'setup.py': setup_script, + 'setup.cfg': setup_config, + }) mismatch_marker = "python_version<'{this_ver}'".format( this_ver=sys.version_info[0], @@ -533,7 +537,7 @@ class TestEggInfo: 'setup.cfg': DALS(""" """), 'LICENSE': "Test license" - }, False), # no license_file attribute + }, True), # no license_file attribute, LICENSE auto-included ({ 'setup.cfg': DALS(""" [metadata] @@ -541,12 +545,20 @@ class TestEggInfo: """), 'MANIFEST.in': "exclude LICENSE", 'LICENSE': "Test license" - }, False) # license file is manually excluded + }, False), # license file is manually excluded + pytest.param({ + 'setup.cfg': DALS(""" + [metadata] + license_file = LICEN[CS]E* + """), + 'LICENSE': "Test license", + }, True, + id="glob_pattern"), ]) def test_setup_cfg_license_file( self, tmpdir_cwd, env, files, license_in_sources): self._create_project() - build_files(files) + path.build(files) environment.run_setup_py( cmd=['egg_info'], @@ -621,7 +633,7 @@ class TestEggInfo: 'setup.cfg': DALS(""" """), 'LICENSE': "Test license" - }, [], ['LICENSE']), # no license_files attribute + }, ['LICENSE'], []), # no license_files attribute, LICENSE auto-included ({ 'setup.cfg': DALS(""" [metadata] @@ -640,12 +652,41 @@ class TestEggInfo: 'MANIFEST.in': "exclude LICENSE-XYZ", 'LICENSE-ABC': "ABC license", 'LICENSE-XYZ': "XYZ license" - }, ['LICENSE-ABC'], ['LICENSE-XYZ']) # subset is manually excluded + }, ['LICENSE-ABC'], ['LICENSE-XYZ']), # subset is manually excluded + pytest.param({ + 'setup.cfg': "", + 'LICENSE-ABC': "ABC license", + 'COPYING-ABC': "ABC copying", + 'NOTICE-ABC': "ABC notice", + 'AUTHORS-ABC': "ABC authors", + 'LICENCE-XYZ': "XYZ license", + 'LICENSE': "License", + 'INVALID-LICENSE': "Invalid license", + }, [ + 'LICENSE-ABC', + 'COPYING-ABC', + 'NOTICE-ABC', + 'AUTHORS-ABC', + 'LICENCE-XYZ', + 'LICENSE', + ], ['INVALID-LICENSE'], + # ('LICEN[CS]E*', 'COPYING*', 'NOTICE*', 'AUTHORS*') + id="default_glob_patterns"), + pytest.param({ + 'setup.cfg': DALS(""" + [metadata] + license_files = + LICENSE* + """), + 'LICENSE-ABC': "ABC license", + 'NOTICE-XYZ': "XYZ notice", + }, ['LICENSE-ABC'], ['NOTICE-XYZ'], + id="no_default_glob_patterns"), ]) def test_setup_cfg_license_files( self, tmpdir_cwd, env, files, incl_licenses, excl_licenses): self._create_project() - build_files(files) + path.build(files) environment.run_setup_py( cmd=['egg_info'], @@ -745,12 +786,33 @@ class TestEggInfo: 'LICENSE-PQR': "PQR license", 'LICENSE-XYZ': "XYZ license" # manually excluded - }, ['LICENSE-XYZ'], ['LICENSE-ABC', 'LICENSE-PQR']) + }, ['LICENSE-XYZ'], ['LICENSE-ABC', 'LICENSE-PQR']), + pytest.param({ + 'setup.cfg': DALS(""" + [metadata] + license_file = LICENSE* + """), + 'LICENSE-ABC': "ABC license", + 'NOTICE-XYZ': "XYZ notice", + }, ['LICENSE-ABC'], ['NOTICE-XYZ'], + id="no_default_glob_patterns"), + pytest.param({ + 'setup.cfg': DALS(""" + [metadata] + license_file = LICENSE* + license_files = + NOTICE* + """), + 'LICENSE-ABC': "ABC license", + 'NOTICE-ABC': "ABC notice", + 'AUTHORS-ABC': "ABC authors", + }, ['LICENSE-ABC', 'NOTICE-ABC'], ['AUTHORS-ABC'], + id="combined_glob_patterrns"), ]) def test_setup_cfg_license_file_license_files( self, tmpdir_cwd, env, files, incl_licenses, excl_licenses): self._create_project() - build_files(files) + path.build(files) environment.run_setup_py( cmd=['egg_info'], @@ -886,7 +948,7 @@ class TestEggInfo: def test_egg_info_tag_only_once(self, tmpdir_cwd, env): self._create_project() - build_files({ + path.build({ 'setup.cfg': DALS(""" [egg_info] tag_build = dev diff --git a/setuptools/tests/test_glob.py b/setuptools/tests/test_glob.py index a0728c5d..e99587f5 100644 --- a/setuptools/tests/test_glob.py +++ b/setuptools/tests/test_glob.py @@ -1,9 +1,8 @@ import pytest +from jaraco import path from setuptools.glob import glob -from .files import build_files - @pytest.mark.parametrize('tree, pattern, matches', ( ('', b'', []), @@ -31,5 +30,5 @@ from .files import build_files )) def test_glob(monkeypatch, tmpdir, tree, pattern, matches): monkeypatch.chdir(tmpdir) - build_files({name: '' for name in tree.split()}) + path.build({name: '' for name in tree.split()}) assert list(sorted(glob(pattern))) == list(sorted(matches)) diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py index 24cef480..b5578312 100644 --- a/setuptools/tests/test_integration.py +++ b/setuptools/tests/test_integration.py @@ -6,11 +6,6 @@ Try to install a few packages. import glob import os import sys -import re -import subprocess -import functools -import tarfile -import zipfile import urllib.request import pytest @@ -20,6 +15,13 @@ from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution +pytestmark = pytest.mark.skipif( + 'platform.python_implementation() == "PyPy" and ' + 'platform.system() == "Windows"', + reason="pypa/setuptools#2496", +) + + def setup_module(module): packages = 'stevedore', 'virtualenvwrapper', 'pbr', 'novaclient' for pkg in packages: @@ -117,56 +119,3 @@ def test_pyuri(install_context): # The package data should be installed. assert os.path.exists(os.path.join(pyuri.location, 'pyuri', 'uri.regex')) - - -build_deps = ['appdirs', 'packaging', 'pyparsing', 'six'] - - -@pytest.mark.parametrize("build_dep", build_deps) -@pytest.mark.skipif( - sys.version_info < (3, 6), reason='run only on late versions') -def test_build_deps_on_distutils(request, tmpdir_factory, build_dep): - """ - All setuptools build dependencies must build without - setuptools. - """ - if 'pyparsing' in build_dep: - pytest.xfail(reason="Project imports setuptools unconditionally") - build_target = tmpdir_factory.mktemp('source') - build_dir = download_and_extract(request, build_dep, build_target) - install_target = tmpdir_factory.mktemp('target') - output = install(build_dir, install_target) - for line in output.splitlines(): - match = re.search('Unknown distribution option: (.*)', line) - allowed_unknowns = [ - 'test_suite', - 'tests_require', - 'python_requires', - 'install_requires', - 'long_description_content_type', - ] - assert not match or match.group(1).strip('"\'') in allowed_unknowns - - -def install(pkg_dir, install_dir): - with open(os.path.join(pkg_dir, 'setuptools.py'), 'w') as breaker: - breaker.write('raise ImportError()') - cmd = [sys.executable, 'setup.py', 'install', '--prefix', str(install_dir)] - env = dict(os.environ, PYTHONPATH=str(pkg_dir)) - output = subprocess.check_output( - cmd, cwd=pkg_dir, env=env, stderr=subprocess.STDOUT) - return output.decode('utf-8') - - -def download_and_extract(request, req, target): - cmd = [ - sys.executable, '-m', 'pip', 'download', '--no-deps', - '--no-binary', ':all:', req, - ] - output = subprocess.check_output(cmd, encoding='utf-8') - filename = re.search('Saved (.*)', output).group(1) - request.addfinalizer(functools.partial(os.remove, filename)) - opener = zipfile.ZipFile if filename.endswith('.zip') else tarfile.open - with opener(filename) as archive: - archive.extractall(target) - return os.path.join(target, os.listdir(target)[0]) diff --git a/setuptools/tests/test_manifest.py b/setuptools/tests/test_manifest.py index 82bdb9c6..589cefb2 100644 --- a/setuptools/tests/test_manifest.py +++ b/setuptools/tests/test_manifest.py @@ -55,6 +55,7 @@ def touch(filename): default_files = frozenset(map(make_local_path, [ 'README.rst', 'MANIFEST.in', + 'LICENSE', 'setup.py', 'app.egg-info/PKG-INFO', 'app.egg-info/SOURCES.txt', diff --git a/setuptools/tests/test_msvc.py b/setuptools/tests/test_msvc.py index 24e38ea8..d1527bfa 100644 --- a/setuptools/tests/test_msvc.py +++ b/setuptools/tests/test_msvc.py @@ -88,7 +88,7 @@ class TestModulePatch: assert isinstance(exc, expected) assert 'aka.ms/vcpython27' in str(exc) - @pytest.yield_fixture + @pytest.fixture def user_preferred_setting(self): """ Set up environment with different install dirs for user vs. system @@ -116,7 +116,7 @@ class TestModulePatch: expected = os.path.join(user_preferred_setting, 'vcvarsall.bat') assert expected == result - @pytest.yield_fixture + @pytest.fixture def local_machine_setting(self): """ Set up environment with only the system environment configured. @@ -138,7 +138,7 @@ class TestModulePatch: expected = os.path.join(local_machine_setting, 'vcvarsall.bat') assert expected == result - @pytest.yield_fixture + @pytest.fixture def x64_preferred_setting(self): """ Set up environment with 64-bit and 32-bit system settings configured diff --git a/setuptools/tests/test_namespaces.py b/setuptools/tests/test_namespaces.py index 6c8c522d..270f90c9 100644 --- a/setuptools/tests/test_namespaces.py +++ b/setuptools/tests/test_namespaces.py @@ -62,8 +62,9 @@ class TestNamespaces: target.mkdir() install_cmd = [ sys.executable, - '-m', 'easy_install', - '-d', str(target), + '-m', 'pip', + 'install', + '-t', str(target), str(pkg), ] with test.test.paths_on_pythonpath([str(target)]): diff --git a/setuptools/tests/test_sphinx_upload_docs.py b/setuptools/tests/test_sphinx_upload_docs.py new file mode 100644 index 00000000..cc5b8293 --- /dev/null +++ b/setuptools/tests/test_sphinx_upload_docs.py @@ -0,0 +1,38 @@ +import pytest + +from jaraco import path + +from setuptools.command.upload_docs import upload_docs +from setuptools.dist import Distribution + + +@pytest.fixture +def sphinx_doc_sample_project(tmpdir_cwd): + path.build({ + 'setup.py': 'from setuptools import setup; setup()', + 'build': { + 'docs': { + 'conf.py': 'project="test"', + 'index.rst': ".. toctree::\ + :maxdepth: 2\ + :caption: Contents:", + }, + }, + }) + + +@pytest.mark.usefixtures('sphinx_doc_sample_project') +class TestSphinxUploadDocs: + def test_sphinx_doc(self): + params = dict( + name='foo', + packages=['test'], + ) + dist = Distribution(params) + + cmd = upload_docs(dist) + + cmd.initialize_options() + assert cmd.upload_dir is None + assert cmd.has_sphinx() is True + cmd.finalize_options() diff --git a/setuptools/tests/test_upload_docs.py b/setuptools/tests/test_upload_docs.py index a26e32a6..55978aad 100644 --- a/setuptools/tests/test_upload_docs.py +++ b/setuptools/tests/test_upload_docs.py @@ -3,6 +3,7 @@ import zipfile import contextlib import pytest +from jaraco import path from setuptools.command.upload_docs import upload_docs from setuptools.dist import Distribution @@ -10,28 +11,20 @@ from setuptools.dist import Distribution from .textwrap import DALS from . import contexts -SETUP_PY = DALS( - """ - from setuptools import setup - - setup(name='foo') - """) - @pytest.fixture def sample_project(tmpdir_cwd): - # setup.py - with open('setup.py', 'wt') as f: - f.write(SETUP_PY) - - os.mkdir('build') - - # A test document. - with open('build/index.html', 'w') as f: - f.write("Hello world.") - - # An empty folder. - os.mkdir('build/empty') + path.build({ + 'setup.py': DALS(""" + from setuptools import setup + + setup(name='foo') + """), + 'build': { + 'index.html': 'Hello world.', + 'empty': {}, + } + }) @pytest.mark.usefixtures('sample_project') diff --git a/setuptools/tests/test_virtualenv.py b/setuptools/tests/test_virtualenv.py index c8ed9e57..399dbaf0 100644 --- a/setuptools/tests/test_virtualenv.py +++ b/setuptools/tests/test_virtualenv.py @@ -1,9 +1,11 @@ import glob import os import sys +import itertools + +import pathlib import pytest -from pytest import yield_fixture from pytest_fixture_config import yield_requires_config import pytest_virtualenv @@ -27,7 +29,7 @@ def pytest_virtualenv_works(virtualenv): @yield_requires_config(pytest_virtualenv.CONFIG, ['virtualenv_executable']) -@yield_fixture(scope='function') +@pytest.fixture(scope='function') def bare_virtualenv(): """ Bare virtualenv (no pip/setuptools/wheel). """ @@ -39,14 +41,11 @@ def bare_virtualenv(): yield venv -SOURCE_DIR = os.path.join(os.path.dirname(__file__), '../..') - - -def test_clean_env_install(bare_virtualenv): +def test_clean_env_install(bare_virtualenv, tmp_src): """ Check setuptools can be installed in a clean environment. """ - bare_virtualenv.run(['python', 'setup.py', 'install'], cd=SOURCE_DIR) + bare_virtualenv.run(['python', 'setup.py', 'install'], cd=tmp_src) def _get_pip_versions(): @@ -67,24 +66,38 @@ def _get_pip_versions(): # No network, disable most of these tests network = False + def mark(param, *marks): + if not isinstance(param, type(pytest.param(''))): + param = pytest.param(param) + return param._replace(marks=param.marks + marks) + + def skip_network(param): + return param if network else mark(param, pytest.mark.skip(reason="no network")) + + issue2599 = pytest.mark.skipif( + sys.version_info > (3, 10), + reason="pypa/setuptools#2599", + ) + network_versions = [ - 'pip==9.0.3', - 'pip==10.0.1', - 'pip==18.1', - 'pip==19.0.1', - 'https://github.com/pypa/pip/archive/master.zip', + mark('pip==9.0.3', issue2599), + mark('pip==10.0.1', issue2599), + mark('pip==18.1', issue2599), + mark('pip==19.3.1', pytest.mark.xfail(reason='pypa/pip#6599')), + 'pip==20.0.2', + 'https://github.com/pypa/pip/archive/main.zip', ] - versions = [None] + [ - pytest.param(v, **({} if network else {'marks': pytest.mark.skip})) - for v in network_versions - ] + versions = itertools.chain( + [None], + map(skip_network, network_versions) + ) - return versions + return list(versions) @pytest.mark.parametrize('pip_version', _get_pip_versions()) -def test_pip_upgrade_from_source(pip_version, virtualenv): +def test_pip_upgrade_from_source(pip_version, tmp_src, virtualenv): """ Check pip can upgrade setuptools from source. """ @@ -103,7 +116,7 @@ def test_pip_upgrade_from_source(pip_version, virtualenv): virtualenv.run(' && '.join(( 'python setup.py -q sdist -d {dist}', 'python setup.py -q bdist_wheel -d {dist}', - )).format(dist=dist_dir), cd=SOURCE_DIR) + )).format(dist=dist_dir), cd=tmp_src) sdist = glob.glob(os.path.join(dist_dir, '*.zip'))[0] wheel = glob.glob(os.path.join(dist_dir, '*.whl'))[0] # Then update from wheel. @@ -112,19 +125,19 @@ def test_pip_upgrade_from_source(pip_version, virtualenv): virtualenv.run('pip install --no-cache-dir --upgrade ' + sdist) -def _check_test_command_install_requirements(virtualenv, tmpdir): +def _check_test_command_install_requirements(virtualenv, tmpdir, cwd): """ Check the test command will install all required dependencies. """ # Install setuptools. - virtualenv.run('python setup.py develop', cd=SOURCE_DIR) + virtualenv.run('python setup.py develop', cd=cwd) def sdist(distname, version): dist_path = tmpdir.join('%s-%s.tar.gz' % (distname, version)) make_nspkg_sdist(str(dist_path), distname, version) return dist_path dependency_links = [ - str(dist_path) + pathlib.Path(str(dist_path)).as_uri() for dist_path in ( sdist('foobar', '2.4'), sdist('bits', '4.2'), @@ -174,22 +187,21 @@ def _check_test_command_install_requirements(virtualenv, tmpdir): assert tmpdir.join('success').check() -def test_test_command_install_requirements(virtualenv, tmpdir): +def test_test_command_install_requirements(virtualenv, tmpdir, request): # Ensure pip/wheel packages are installed. virtualenv.run( "python -c \"__import__('pkg_resources').require(['pip', 'wheel'])\"") - _check_test_command_install_requirements(virtualenv, tmpdir) - - -def test_test_command_install_requirements_when_using_easy_install( - bare_virtualenv, tmpdir): - _check_test_command_install_requirements(bare_virtualenv, tmpdir) + # uninstall setuptools so that 'setup.py develop' works + virtualenv.run("python -m pip uninstall -y setuptools") + # disable index URL so bits and bobs aren't requested from PyPI + virtualenv.env['PIP_NO_INDEX'] = '1' + _check_test_command_install_requirements(virtualenv, tmpdir, request.config.rootdir) -def test_no_missing_dependencies(bare_virtualenv): +def test_no_missing_dependencies(bare_virtualenv, request): """ Quick and dirty test to ensure all external dependencies are vendored. """ for command in ('upload',): # sorted(distutils.command.__all__): - bare_virtualenv.run( - ['python', 'setup.py', command, '-h'], cd=SOURCE_DIR) + cmd = ['python', 'setup.py', command, '-h'] + bare_virtualenv.run(cmd, cd=request.config.rootdir) diff --git a/setuptools/tests/test_wheel.py b/setuptools/tests/test_wheel.py index e56eac14..7345b135 100644 --- a/setuptools/tests/test_wheel.py +++ b/setuptools/tests/test_wheel.py @@ -15,6 +15,7 @@ import sys import zipfile import pytest +from jaraco import path from pkg_resources import Distribution, PathMetadata, PY_MAJOR from setuptools.extern.packaging.utils import canonicalize_name @@ -22,7 +23,6 @@ from setuptools.extern.packaging.tags import parse_tag from setuptools.wheel import Wheel from .contexts import tempdir -from .files import build_files from .textwrap import DALS @@ -91,7 +91,7 @@ def build_wheel(extra_file_defs=None, **kwargs): if extra_file_defs: file_defs.update(extra_file_defs) with tempdir() as source_dir: - build_files(file_defs, source_dir) + path.build(file_defs, source_dir) subprocess.check_call((sys.executable, 'setup.py', '-q', 'bdist_wheel'), cwd=source_dir) yield glob.glob(os.path.join(source_dir, 'dist', '*.whl'))[0] |