Merge with mastersetuptools-scm

12 files changed, 180 insertions, 86 deletions

diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py
index 9cebd7fa..cbea7537 100644
--- a/setuptools/command/bdist_egg.py
+++ b/setuptools/command/bdist_egg.py
@@ -8,8 +8,8 @@ from distutils import log
 from types import CodeType
 import sys
 import os
-import marshal
 import textwrap
+import marshal
 
 from setuptools.extern import six
 
@@ -129,7 +129,7 @@ class bdist_egg(Command):
             self.distribution.data_files.append(item)
 
         try:
-            log.info("installing package data to %s" % self.bdist_dir)
+            log.info("installing package data to %s", self.bdist_dir)
             self.call_command('install_data', force=0, root=None)
         finally:
             self.distribution.data_files = old
@@ -152,7 +152,7 @@ class bdist_egg(Command):
         self.run_command("egg_info")
         # We run install_lib before install_data, because some data hacks
         # pull their data path from the install_lib command.
-        log.info("installing library code to %s" % self.bdist_dir)
+        log.info("installing library code to %s", self.bdist_dir)
         instcmd = self.get_finalized_command('install')
         old_root = instcmd.root
         instcmd.root = None
@@ -169,7 +169,7 @@ class bdist_egg(Command):
             pyfile = os.path.join(self.bdist_dir, strip_module(filename) +
                                   '.py')
             self.stubs.append(pyfile)
-            log.info("creating stub loader for %s" % ext_name)
+            log.info("creating stub loader for %s", ext_name)
             if not self.dry_run:
                 write_stub(os.path.basename(ext_name), pyfile)
             to_compile.append(pyfile)
@@ -186,14 +186,14 @@ class bdist_egg(Command):
         self.mkpath(egg_info)
         if self.distribution.scripts:
             script_dir = os.path.join(egg_info, 'scripts')
-            log.info("installing scripts to %s" % script_dir)
+            log.info("installing scripts to %s", script_dir)
             self.call_command('install_scripts', install_dir=script_dir,
                               no_ep=1)
 
         self.copy_metadata_to(egg_info)
         native_libs = os.path.join(egg_info, "native_libs.txt")
         if all_outputs:
-            log.info("writing %s" % native_libs)
+            log.info("writing %s", native_libs)
             if not self.dry_run:
                 ensure_directory(native_libs)
                 libs_file = open(native_libs, 'wt')
@@ -201,7 +201,7 @@ class bdist_egg(Command):
                 libs_file.write('\n')
                 libs_file.close()
         elif os.path.isfile(native_libs):
-            log.info("removing %s" % native_libs)
+            log.info("removing %s", native_libs)
             if not self.dry_run:
                 os.unlink(native_libs)
 
@@ -432,6 +432,7 @@ def can_scan():
 # Attribute names of options for commands that might need to be convinced to
 # install to the egg build directory
 
+
 INSTALL_DIRECTORY_ATTRS = [
     'install_lib', 'install_dir', 'install_data', 'install_base'
 ]
@@ -457,7 +458,7 @@ def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True,
                 p = path[len(base_dir) + 1:]
                 if not dry_run:
                     z.write(path, p)
-                log.debug("adding '%s'" % p)
+                log.debug("adding '%s'", p)
 
     compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
     if not dry_run:
diff --git a/setuptools/command/bdist_wininst.py b/setuptools/command/bdist_wininst.py
index 073de97b..8243c917 100755
--- a/setuptools/command/bdist_wininst.py
+++ b/setuptools/command/bdist_wininst.py
@@ -2,6 +2,7 @@ import distutils.command.bdist_wininst as orig
 
 
 class bdist_wininst(orig.bdist_wininst):
+
     def reinitialize_command(self, command, reinit_subcommands=0):
         """
         Supplement reinitialize_command to work around
diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py
index 1caf8c81..454c91fb 100644
--- a/setuptools/command/build_ext.py
+++ b/setuptools/command/build_ext.py
@@ -1,14 +1,16 @@
+import os
+import sys
+import itertools
+import imp
 from distutils.command.build_ext import build_ext as _du_build_ext
 from distutils.file_util import copy_file
 from distutils.ccompiler import new_compiler
-from distutils.sysconfig import customize_compiler
+from distutils.sysconfig import customize_compiler, get_config_var
 from distutils.errors import DistutilsError
 from distutils import log
-import os
-import sys
-import itertools
 
 from setuptools.extension import Library
+from setuptools.extern import six
 
 try:
     # Attempt to use Cython for building extensions, if available
@@ -16,10 +18,8 @@ try:
 except ImportError:
     _build_ext = _du_build_ext
 
-from distutils.sysconfig import get_config_var
-
-get_config_var("LDSHARED")  # make sure _config_vars is initialized
-del get_config_var
+# make sure _config_vars is initialized
+get_config_var("LDSHARED")
 from distutils.sysconfig import _config_vars as _CONFIG_VARS
 
 
@@ -59,7 +59,18 @@ elif os.name != 'nt':
 
 if_dl = lambda s: s if have_rtld else ''
 
+
+def get_abi3_suffix():
+    """Return the file extension for an abi3-compliant Extension()"""
+    for suffix, _, _ in (s for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION):
+        if '.abi3' in suffix:   # Unix
+            return suffix
+        elif suffix == '.pyd':  # Windows
+            return suffix
+
+
 class build_ext(_build_ext):
+
     def run(self):
         """Build extensions in build directory, then copy if --inplace"""
         old_inplace, self.inplace = self.inplace, 0
@@ -94,6 +105,15 @@ class build_ext(_build_ext):
         filename = _build_ext.get_ext_filename(self, fullname)
         if fullname in self.ext_map:
             ext = self.ext_map[fullname]
+            use_abi3 = (
+                six.PY3
+                and getattr(ext, 'py_limited_api')
+                and get_abi3_suffix()
+            )
+            if use_abi3:
+                so_ext = get_config_var('EXT_SUFFIX')
+                filename = filename[:-len(so_ext)]
+                filename = filename + get_abi3_suffix()
             if isinstance(ext, Library):
                 fn, ext = os.path.splitext(filename)
                 return self.shlib_compiler.library_filename(fn, libtype)
diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py
index 0bad8295..b5de9bda 100644
--- a/setuptools/command/build_py.py
+++ b/setuptools/command/build_py.py
@@ -15,6 +15,7 @@ try:
     from setuptools.lib2to3_ex import Mixin2to3
 except ImportError:
     class Mixin2to3:
+
         def run_2to3(self, files, doctests=True):
             "do nothing"
 
diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py
index 11b5df10..3eb86120 100755
--- a/setuptools/command/develop.py
+++ b/setuptools/command/develop.py
@@ -186,6 +186,7 @@ class VersionlessRequirement(object):
     >>> str(adapted_dist.as_requirement())
     'foo'
     """
+
     def __init__(self, dist):
         self.__dist = dist
 
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index ccc66cf7..a3792ce2 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -8,7 +8,7 @@ A tool for doing automatic download/extract/build of distutils-based Python
 packages.  For detailed documentation, see the accompanying EasyInstall.txt
 file, or visit the `EasyInstall home page`__.
 
-__ https://pythonhosted.org/setuptools/easy_install.html
+__ https://setuptools.readthedocs.io/en/latest/easy_install.html
 
 """
 
@@ -32,7 +32,6 @@ import zipfile
 import re
 import stat
 import random
-import platform
 import textwrap
 import warnings
 import site
@@ -50,8 +49,9 @@ from setuptools.sandbox import run_setup
 from setuptools.py31compat import get_path, get_config_vars
 from setuptools.command import setopt
 from setuptools.archive_util import unpack_archive
-from setuptools.package_index import PackageIndex
-from setuptools.package_index import URL_SCHEME
+from setuptools.package_index import (
+    PackageIndex, parse_requirement_arg, URL_SCHEME,
+)
 from setuptools.command import bdist_egg, egg_info
 from pkg_resources import (
     yield_lines, normalize_path, resource_string, ensure_directory,
@@ -432,7 +432,7 @@ class easy_install(Command):
         """
         try:
             pid = os.getpid()
-        except:
+        except Exception:
             pid = random.randint(0, sys.maxsize)
         return os.path.join(self.install_dir, "test-easy-install-%s" % pid)
 
@@ -513,7 +513,7 @@ class easy_install(Command):
         For information on other options, you may wish to consult the
         documentation at:
 
-          https://pythonhosted.org/setuptools/easy_install.html
+          https://setuptools.readthedocs.io/en/latest/easy_install.html
 
         Please make the appropriate changes for your system and try again.
         """).lstrip()
@@ -930,7 +930,7 @@ class easy_install(Command):
                     destination,
                     fix_zipimporter_caches=new_dist_is_zipped,
                 )
-            except:
+            except Exception:
                 update_dist_caches(destination, fix_zipimporter_caches=False)
                 raise
 
@@ -1257,7 +1257,8 @@ class easy_install(Command):
         * You can set up the installation directory to support ".pth" files by
           using one of the approaches described here:
 
-          https://pythonhosted.org/setuptools/easy_install.html#custom-installation-locations
+          https://setuptools.readthedocs.io/en/latest/easy_install.html#custom-installation-locations
+
 
         Please make the appropriate changes for your system and try again.""").lstrip()
 
@@ -1522,15 +1523,6 @@ def get_exe_prefixes(exe_filename):
     return prefixes
 
 
-def parse_requirement_arg(spec):
-    try:
-        return Requirement.parse(spec)
-    except ValueError:
-        raise DistutilsError(
-            "Not a URL, existing file, or requirement spec: %r" % (spec,)
-        )
-
-
 class PthDistributions(Environment):
     """A .pth file with Distribution paths in it"""
 
@@ -1662,7 +1654,7 @@ class RewritePthDistributions(PthDistributions):
         """)
 
 
-if os.environ.get('SETUPTOOLS_SYS_PATH_TECHNIQUE', 'rewrite') == 'rewrite':
+if os.environ.get('SETUPTOOLS_SYS_PATH_TECHNIQUE', 'raw') == 'rewrite':
     PthDistributions = RewritePthDistributions
 
 
@@ -1832,6 +1824,7 @@ def _remove_and_clear_zip_directory_cache_data(normalized_path):
         normalized_path, zipimport._zip_directory_cache,
         updater=clear_and_remove_cached_zip_archive_directory_data)
 
+
 # PyPy Python implementation does not allow directly writing to the
 # zipimport._zip_directory_cache and so prevents us from attempting to correct
 # its content. The best we can do there is clear the problematic cache content
@@ -1987,10 +1980,20 @@ class CommandSpec(list):
         return self._render(self + list(self.options))
 
     @staticmethod
+    def _strip_quotes(item):
+        _QUOTES = '"\''
+        for q in _QUOTES:
+            if item.startswith(q) and item.endswith(q):
+                return item[1:-1]
+        return item
+
+    @staticmethod
     def _render(items):
-        cmdline = subprocess.list2cmdline(items)
+        cmdline = subprocess.list2cmdline(
+            CommandSpec._strip_quotes(item.strip()) for item in items)
         return '#!' + cmdline + '\n'
 
+
 # For pbr compat; will be removed in a future version.
 sys_executable = CommandSpec._sys_executable()
 
@@ -2008,10 +2011,12 @@ class ScriptWriter(object):
     template = textwrap.dedent("""
         # EASY-INSTALL-ENTRY-SCRIPT: %(spec)r,%(group)r,%(name)r
         __requires__ = %(spec)r
+        import re
         import sys
         from pkg_resources import load_entry_point
 
         if __name__ == '__main__':
+            sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
             sys.exit(
                 load_entry_point(%(spec)r, %(group)r, %(name)r)()
             )
@@ -2159,6 +2164,7 @@ class WindowsScriptWriter(ScriptWriter):
 
 
 class WindowsExecutableLauncherWriter(WindowsScriptWriter):
+
     @classmethod
     def _get_script_args(cls, type_, name, header, script_text):
         """
@@ -2203,8 +2209,6 @@ def get_win_launcher(type):
     Returns the executable as a byte string.
     """
     launcher_fn = '%s.exe' % type
-    if platform.machine().lower() == 'arm':
-        launcher_fn = launcher_fn.replace(".", "-arm.")
     if is_64bit():
         launcher_fn = launcher_fn.replace(".", "-64.")
     else:
@@ -2221,39 +2225,7 @@ def load_launcher_manifest(name):
 
 
 def rmtree(path, ignore_errors=False, onerror=auto_chmod):
-    """Recursively delete a directory tree.
-
-    This code is taken from the Python 2.4 version of 'shutil', because
-    the 2.3 version doesn't really work right.
-    """
-    if ignore_errors:
-        def onerror(*args):
-            pass
-    elif onerror is None:
-        def onerror(*args):
-            raise
-    names = []
-    try:
-        names = os.listdir(path)
-    except os.error:
-        onerror(os.listdir, path, sys.exc_info())
-    for name in names:
-        fullname = os.path.join(path, name)
-        try:
-            mode = os.lstat(fullname).st_mode
-        except os.error:
-            mode = 0
-        if stat.S_ISDIR(mode):
-            rmtree(fullname, ignore_errors, onerror)
-        else:
-            try:
-                os.remove(fullname)
-            except os.error:
-                onerror(os.remove, fullname, sys.exc_info())
-    try:
-        os.rmdir(path)
-    except os.error:
-        onerror(os.rmdir, path, sys.exc_info())
+    return shutil.rmtree(path, ignore_errors, onerror)
 
 
 def current_umask():
diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py
index 8e1502a5..5183eedc 100755
--- a/setuptools/command/egg_info.py
+++ b/setuptools/command/egg_info.py
@@ -52,8 +52,10 @@ class egg_info(Command):
     ]
 
     boolean_options = ['tag-date', 'tag-svn-revision']
-    negative_opt = {'no-svn-revision': 'tag-svn-revision',
-                    'no-date': 'tag-date'}
+    negative_opt = {
+        'no-svn-revision': 'tag-svn-revision',
+        'no-date': 'tag-date',
+    }
 
     def initialize_options(self):
         self.egg_name = None
@@ -197,6 +199,10 @@ class egg_info(Command):
         if self.tag_build:
             version += self.tag_build
         if self.tag_svn_revision:
+            warnings.warn(
+                "tag_svn_revision is deprecated and will not be honored "
+                "in a future release"
+            )
             version += '-r%s' % self.get_svn_revision()
         if self.tag_date:
             version += time.strftime("-%Y%m%d")
diff --git a/setuptools/command/install_lib.py b/setuptools/command/install_lib.py
index 78fe6891..2b31c3e3 100644
--- a/setuptools/command/install_lib.py
+++ b/setuptools/command/install_lib.py
@@ -3,6 +3,7 @@ import imp
 from itertools import product, starmap
 import distutils.command.install_lib as orig
 
+
 class install_lib(orig.install_lib):
     """Don't add compiled flags to filenames of non-Python files"""
 
diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py
index be66cb22..16234273 100755
--- a/setuptools/command/install_scripts.py
+++ b/setuptools/command/install_scripts.py
@@ -1,6 +1,7 @@
 from distutils import log
 import distutils.command.install_scripts as orig
 import os
+import sys
 
 from pkg_resources import Distribution, PathMetadata, ensure_directory
 
@@ -37,6 +38,10 @@ class install_scripts(orig.install_scripts):
         if is_wininst:
             exec_param = "python.exe"
             writer = ei.WindowsScriptWriter
+        if exec_param == sys.executable:
+            # In case the path to the Python executable contains a space, wrap
+            # it so it's not split up.
+            exec_param = [exec_param]
         # resolve the writer to the environment
         writer = writer.best()
         cmd = writer.command_spec_class.best().from_param(exec_param)
diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py
index 6640d4e3..1d4f5d54 100755
--- a/setuptools/command/sdist.py
+++ b/setuptools/command/sdist.py
@@ -4,6 +4,7 @@ import distutils.command.sdist as orig
 import os
 import sys
 import io
+import contextlib
 
 from setuptools.extern import six
 
@@ -15,6 +16,7 @@ READMES = 'README', 'README.rst', 'README.txt'
 
 _default_revctrl = list
 
+
 def walk_revctrl(dirname=''):
     """Find all files under revision control"""
     for ep in pkg_resources.iter_entry_points('setuptools.file_finders'):
@@ -64,6 +66,43 @@ class sdist(orig.sdist):
             if data not in dist_files:
                 dist_files.append(data)
 
+    def initialize_options(self):
+        orig.sdist.initialize_options(self)
+
+        self._default_to_gztar()
+
+    def _default_to_gztar(self):
+        # only needed on Python prior to 3.6.
+        if sys.version_info >= (3, 6, 0, 'beta', 1):
+            return
+        self.formats = ['gztar']
+
+    def make_distribution(self):
+        """
+        Workaround for #516
+        """
+        with self._remove_os_link():
+            orig.sdist.make_distribution(self)
+
+    @staticmethod
+    @contextlib.contextmanager
+    def _remove_os_link():
+        """
+        In a context, remove and restore os.link if it exists
+        """
+        class NoValue:
+            pass
+        orig_val = getattr(os, 'link', NoValue)
+        try:
+            del os.link
+        except Exception:
+            pass
+        try:
+            yield
+        finally:
+            if orig_val is not NoValue:
+                setattr(os, 'link', orig_val)
+
     def __read_template_hack(self):
         # This grody hack closes the template file (MANIFEST.in) if an
         #  exception occurs during read_template.
@@ -71,7 +110,7 @@ class sdist(orig.sdist):
         #  file.
         try:
             orig.sdist.read_template(self)
-        except:
+        except Exception:
             _, _, tb = sys.exc_info()
             tb.tb_next.tb_frame.f_locals['template'].close()
             raise
@@ -179,7 +218,7 @@ class sdist(orig.sdist):
         distribution.
         """
         log.info("reading manifest file '%s'", self.manifest)
-        manifest = open(self.manifest, 'rbU')
+        manifest = open(self.manifest, 'rb')
         for line in manifest:
             # The manifest must contain UTF-8. See #303.
             if six.PY3:
diff --git a/setuptools/command/test.py b/setuptools/command/test.py
index 39746a02..38bbcd8b 100644
--- a/setuptools/command/test.py
+++ b/setuptools/command/test.py
@@ -1,10 +1,13 @@
+import os
+import operator
 import sys
 import contextlib
+import itertools
 from distutils.errors import DistutilsOptionError
 from unittest import TestLoader
 
 from setuptools.extern import six
-from setuptools.extern.six.moves import map
+from setuptools.extern.six.moves import map, filter
 
 from pkg_resources import (resource_listdir, resource_exists, normalize_path,
                            working_set, _namespace_packages,
@@ -14,6 +17,7 @@ from setuptools.py31compat import unittest_main
 
 
 class ScanningLoader(TestLoader):
+
     def loadTestsFromModule(self, module, pattern=None):
         """Return a suite of all tests cases contained in the given module
 
@@ -46,6 +50,7 @@ class ScanningLoader(TestLoader):
 
 # adapted from jaraco.classes.properties:NonDataProperty
 class NonDataProperty(object):
+
     def __init__(self, fget):
         self.fget = fget
 
@@ -110,7 +115,7 @@ class test(Command):
             func()
 
     @contextlib.contextmanager
-    def project_on_sys_path(self):
+    def project_on_sys_path(self, include_dists=[]):
         with_2to3 = six.PY3 and getattr(self.distribution, 'use_2to3', False)
 
         if with_2to3:
@@ -142,23 +147,57 @@ class test(Command):
         old_modules = sys.modules.copy()
 
         try:
-            sys.path.insert(0, normalize_path(ei_cmd.egg_base))
+            project_path = normalize_path(ei_cmd.egg_base)
+            sys.path.insert(0, project_path)
             working_set.__init__()
             add_activation_listener(lambda dist: dist.activate())
             require('%s==%s' % (ei_cmd.egg_name, ei_cmd.egg_version))
-            yield
+            with self.paths_on_pythonpath([project_path]):
+                yield
         finally:
             sys.path[:] = old_path
             sys.modules.clear()
             sys.modules.update(old_modules)
             working_set.__init__()
 
+    @staticmethod
+    @contextlib.contextmanager
+    def paths_on_pythonpath(paths):
+        """
+        Add the indicated paths to the head of the PYTHONPATH environment
+        variable so that subprocesses will also see the packages at
+        these paths.
+
+        Do this in a context that restores the value on exit.
+        """
+        nothing = object()
+        orig_pythonpath = os.environ.get('PYTHONPATH', nothing)
+        current_pythonpath = os.environ.get('PYTHONPATH', '')
+        try:
+            prefix = os.pathsep.join(paths)
+            to_join = filter(None, [prefix, current_pythonpath])
+            new_path = os.pathsep.join(to_join)
+            if new_path:
+                os.environ['PYTHONPATH'] = new_path
+            yield
+        finally:
+            if orig_pythonpath is nothing:
+                os.environ.pop('PYTHONPATH', None)
+            else:
+                os.environ['PYTHONPATH'] = orig_pythonpath
+
+    @staticmethod
+    def install_dists(dist):
+        """
+        Install the requirements indicated by self.distribution and
+        return an iterable of the dists that were built.
+        """
+        ir_d = dist.fetch_build_eggs(dist.install_requires or [])
+        tr_d = dist.fetch_build_eggs(dist.tests_require or [])
+        return itertools.chain(ir_d, tr_d)
+
     def run(self):
-        if self.distribution.install_requires:
-            self.distribution.fetch_build_eggs(
-                self.distribution.install_requires)
-        if self.distribution.tests_require:
-            self.distribution.fetch_build_eggs(self.distribution.tests_require)
+        installed_dists = self.install_dists(self.distribution)
 
         cmd = ' '.join(self._argv)
         if self.dry_run:
@@ -166,8 +205,11 @@ class test(Command):
             return
 
         self.announce('running "%s"' % cmd)
-        with self.project_on_sys_path():
-            self.run_tests()
+
+        paths = map(operator.attrgetter('location'), installed_dists)
+        with self.paths_on_pythonpath(paths):
+            with self.project_on_sys_path():
+                self.run_tests()
 
     def run_tests(self):
         # Purge modules under test from sys.modules. The test loader will
diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py
index 01b49046..269dc2d5 100644
--- a/setuptools/command/upload_docs.py
+++ b/setuptools/command/upload_docs.py
@@ -29,6 +29,10 @@ def _encode(s):
 
 
 class upload_docs(upload):
+    # override the default repository as upload_docs isn't
+    # supported by Warehouse (and won't be).
+    DEFAULT_REPOSITORY = 'https://pypi.python.org/pypi/'
+
     description = 'Upload documentation to PyPI'
 
     user_options = [
@@ -53,6 +57,7 @@ class upload_docs(upload):
         self.target_dir = None
 
     def finalize_options(self):
+        log.warn("Upload_docs command is deprecated. Use RTD instead.")
         upload.finalize_options(self)
         if self.upload_dir is None:
             if self.has_sphinx():
@@ -105,7 +110,7 @@ class upload_docs(upload):
         if not isinstance(values, list):
             values = [values]
         for value in values:
-            if type(value) is tuple:
+            if isinstance(value, tuple):
                 title += '; filename="%s"' % value[0]
                 value = value[1]
             else: