Merge with 23.2.1

11 files changed, 193 insertions, 76 deletions

diff --git a/setuptools/cli-arm-32.exe b/setuptools/cli-arm-32.exe
deleted file mode 100644
index 2f40402d..00000000
--- a/setuptools/cli-arm-32.exe
+++ /dev/null
diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py
index 8623c777..0bad8295 100644
--- a/setuptools/command/build_py.py
+++ b/setuptools/command/build_py.py
@@ -6,10 +6,10 @@ import fnmatch
 import textwrap
 import io
 import distutils.errors
-import collections
 import itertools
 
-from setuptools.extern.six.moves import map
+from setuptools.extern import six
+from setuptools.extern.six.moves import map, filter, filterfalse
 
 try:
     from setuptools.lib2to3_ex import Mixin2to3
@@ -67,6 +67,9 @@ class build_py(orig.build_py, Mixin2to3):
         return orig.build_py.__getattr__(self, attr)
 
     def build_module(self, module, module_file, package):
+        if six.PY2 and isinstance(package, six.string_types):
+            # avoid errors on Python 2 when unicode is passed (#190)
+            package = package.split('.')
         outfile, copied = orig.build_py.build_module(self, module, module_file,
                                                      package)
         if copied:
@@ -94,12 +97,19 @@ class build_py(orig.build_py, Mixin2to3):
 
     def find_data_files(self, package, src_dir):
         """Return filenames for package's data files in 'src_dir'"""
-        globs = (self.package_data.get('', [])
-                 + self.package_data.get(package, []))
-        files = self.manifest_files.get(package, [])[:]
-        for pattern in globs:
-            # Each pattern has to be converted to a platform-specific path
-            files.extend(glob(os.path.join(src_dir, convert_path(pattern))))
+        patterns = self._get_platform_patterns(
+            self.package_data,
+            package,
+            src_dir,
+        )
+        globs_expanded = map(glob, patterns)
+        # flatten the expanded globs into an iterable of matches
+        globs_matches = itertools.chain.from_iterable(globs_expanded)
+        glob_files = filter(os.path.isfile, globs_matches)
+        files = itertools.chain(
+            self.manifest_files.get(package, []),
+            glob_files,
+        )
         return self.exclude_data_files(package, src_dir, files)
 
     def build_package_data(self):
@@ -184,26 +194,63 @@ class build_py(orig.build_py, Mixin2to3):
 
     def exclude_data_files(self, package, src_dir, files):
         """Filter filenames for package's data files in 'src_dir'"""
-        globs = (
-            self.exclude_package_data.get('', [])
-            + self.exclude_package_data.get(package, [])
+        files = list(files)
+        patterns = self._get_platform_patterns(
+            self.exclude_package_data,
+            package,
+            src_dir,
         )
-        bad = set(
-            item
-            for pattern in globs
-            for item in fnmatch.filter(
-                files,
-                os.path.join(src_dir, convert_path(pattern)),
-            )
+        match_groups = (
+            fnmatch.filter(files, pattern)
+            for pattern in patterns
         )
-        seen = collections.defaultdict(itertools.count)
-        return [
+        # flatten the groups of matches into an iterable of matches
+        matches = itertools.chain.from_iterable(match_groups)
+        bad = set(matches)
+        keepers = (
             fn
             for fn in files
             if fn not in bad
-            # ditch dupes
-            and not next(seen[fn])
-        ]
+        )
+        # ditch dupes
+        return list(_unique_everseen(keepers))
+
+    @staticmethod
+    def _get_platform_patterns(spec, package, src_dir):
+        """
+        yield platfrom-specific path patterns (suitable for glob
+        or fn_match) from a glob-based spec (such as
+        self.package_data or self.exclude_package_data)
+        matching package in src_dir.
+        """
+        raw_patterns = itertools.chain(
+            spec.get('', []),
+            spec.get(package, []),
+        )
+        return (
+            # Each pattern has to be converted to a platform-specific path
+            os.path.join(src_dir, convert_path(pattern))
+            for pattern in raw_patterns
+        )
+
+
+# from Python docs
+def _unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        for element in filterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element
 
 
 def assert_relative(path):
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index ccc66cf7..9ca1554e 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -32,7 +32,6 @@ import zipfile
 import re
 import stat
 import random
-import platform
 import textwrap
 import warnings
 import site
@@ -2203,8 +2202,6 @@ def get_win_launcher(type):
     Returns the executable as a byte string.
     """
     launcher_fn = '%s.exe' % type
-    if platform.machine().lower() == 'arm':
-        launcher_fn = launcher_fn.replace(".", "-arm.")
     if is_64bit():
         launcher_fn = launcher_fn.replace(".", "-64.")
     else:
diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py
index 8e1502a5..5183eedc 100755
--- a/setuptools/command/egg_info.py
+++ b/setuptools/command/egg_info.py
@@ -52,8 +52,10 @@ class egg_info(Command):
     ]
 
     boolean_options = ['tag-date', 'tag-svn-revision']
-    negative_opt = {'no-svn-revision': 'tag-svn-revision',
-                    'no-date': 'tag-date'}
+    negative_opt = {
+        'no-svn-revision': 'tag-svn-revision',
+        'no-date': 'tag-date',
+    }
 
     def initialize_options(self):
         self.egg_name = None
@@ -197,6 +199,10 @@ class egg_info(Command):
         if self.tag_build:
             version += self.tag_build
         if self.tag_svn_revision:
+            warnings.warn(
+                "tag_svn_revision is deprecated and will not be honored "
+                "in a future release"
+            )
             version += '-r%s' % self.get_svn_revision()
         if self.tag_date:
             version += time.strftime("-%Y%m%d")
diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py
index 6640d4e3..f200b946 100755
--- a/setuptools/command/sdist.py
+++ b/setuptools/command/sdist.py
@@ -179,7 +179,7 @@ class sdist(orig.sdist):
         distribution.
         """
         log.info("reading manifest file '%s'", self.manifest)
-        manifest = open(self.manifest, 'rbU')
+        manifest = open(self.manifest, 'rb')
         for line in manifest:
             # The manifest must contain UTF-8. See #303.
             if six.PY3:
diff --git a/setuptools/command/test.py b/setuptools/command/test.py
index 371e913b..39746a02 100644
--- a/setuptools/command/test.py
+++ b/setuptools/command/test.py
@@ -1,6 +1,7 @@
+import sys
+import contextlib
 from distutils.errors import DistutilsOptionError
 from unittest import TestLoader
-import sys
 
 from setuptools.extern import six
 from setuptools.extern.six.moves import map
@@ -102,6 +103,14 @@ class test(Command):
             yield self.test_suite
 
     def with_project_on_sys_path(self, func):
+        """
+        Backward compatibility for project_on_sys_path context.
+        """
+        with self.project_on_sys_path():
+            func()
+
+    @contextlib.contextmanager
+    def project_on_sys_path(self):
         with_2to3 = six.PY3 and getattr(self.distribution, 'use_2to3', False)
 
         if with_2to3:
@@ -137,7 +146,7 @@ class test(Command):
             working_set.__init__()
             add_activation_listener(lambda dist: dist.activate())
             require('%s==%s' % (ei_cmd.egg_name, ei_cmd.egg_version))
-            func()
+            yield
         finally:
             sys.path[:] = old_path
             sys.modules.clear()
@@ -154,9 +163,11 @@ class test(Command):
         cmd = ' '.join(self._argv)
         if self.dry_run:
             self.announce('skipping "%s" (dry run)' % cmd)
-        else:
-            self.announce('running "%s"' % cmd)
-            self.with_project_on_sys_path(self.run_tests)
+            return
+
+        self.announce('running "%s"' % cmd)
+        with self.project_on_sys_path():
+            self.run_tests()
 
     def run_tests(self):
         # Purge modules under test from sys.modules. The test loader will
diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py
index f887b47e..01b49046 100644
--- a/setuptools/command/upload_docs.py
+++ b/setuptools/command/upload_docs.py
@@ -13,6 +13,8 @@ import socket
 import zipfile
 import tempfile
 import shutil
+import itertools
+import functools
 
 from setuptools.extern import six
 from setuptools.extern.six.moves import http_client, urllib
@@ -21,15 +23,9 @@ from pkg_resources import iter_entry_points
 from .upload import upload
 
 
-errors = 'surrogateescape' if six.PY3 else 'strict'
-
-
-# This is not just a replacement for byte literals
-# but works as a general purpose encoder
-def b(s, encoding='utf-8'):
-    if isinstance(s, six.text_type):
-        return s.encode(encoding, errors)
-    return s
+def _encode(s):
+    errors = 'surrogateescape' if six.PY3 else 'strict'
+    return s.encode('utf-8', errors)
 
 
 class upload_docs(upload):
@@ -101,10 +97,48 @@ class upload_docs(upload):
         finally:
             shutil.rmtree(tmp_dir)
 
+    @staticmethod
+    def _build_part(item, sep_boundary):
+        key, values = item
+        title = '\nContent-Disposition: form-data; name="%s"' % key
+        # handle multiple entries for the same name
+        if not isinstance(values, list):
+            values = [values]
+        for value in values:
+            if type(value) is tuple:
+                title += '; filename="%s"' % value[0]
+                value = value[1]
+            else:
+                value = _encode(value)
+            yield sep_boundary
+            yield _encode(title)
+            yield b"\n\n"
+            yield value
+            if value and value[-1:] == b'\r':
+                yield b'\n'  # write an extra newline (lurve Macs)
+
+    @classmethod
+    def _build_multipart(cls, data):
+        """
+        Build up the MIME payload for the POST data
+        """
+        boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
+        sep_boundary = b'\n--' + boundary
+        end_boundary = sep_boundary + b'--'
+        end_items = end_boundary, b"\n",
+        builder = functools.partial(
+            cls._build_part,
+            sep_boundary=sep_boundary,
+        )
+        part_groups = map(builder, data.items())
+        parts = itertools.chain.from_iterable(part_groups)
+        body_items = itertools.chain(parts, end_items)
+        content_type = 'multipart/form-data; boundary=%s' % boundary
+        return b''.join(body_items), content_type
+
     def upload_file(self, filename):
-        f = open(filename, 'rb')
-        content = f.read()
-        f.close()
+        with open(filename, 'rb') as f:
+            content = f.read()
         meta = self.distribution.metadata
         data = {
             ':action': 'doc_upload',
@@ -112,37 +146,13 @@ class upload_docs(upload):
             'content': (os.path.basename(filename), content),
         }
         # set up the authentication
-        credentials = b(self.username + ':' + self.password)
+        credentials = _encode(self.username + ':' + self.password)
         credentials = standard_b64encode(credentials)
         if six.PY3:
             credentials = credentials.decode('ascii')
         auth = "Basic " + credentials
 
-        # Build up the MIME payload for the POST data
-        boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
-        sep_boundary = b('\n--') + b(boundary)
-        end_boundary = sep_boundary + b('--')
-        body = []
-        for key, values in six.iteritems(data):
-            title = '\nContent-Disposition: form-data; name="%s"' % key
-            # handle multiple entries for the same name
-            if not isinstance(values, list):
-                values = [values]
-            for value in values:
-                if type(value) is tuple:
-                    title += '; filename="%s"' % value[0]
-                    value = value[1]
-                else:
-                    value = b(value)
-                body.append(sep_boundary)
-                body.append(b(title))
-                body.append(b("\n\n"))
-                body.append(value)
-                if value and value[-1:] == b('\r'):
-                    body.append(b('\n'))  # write an extra newline (lurve Macs)
-        body.append(end_boundary)
-        body.append(b("\n"))
-        body = b('').join(body)
+        body, ct = self._build_multipart(data)
 
         self.announce("Submitting documentation to %s" % (self.repository),
                       log.INFO)
@@ -164,7 +174,7 @@ class upload_docs(upload):
         try:
             conn.connect()
             conn.putrequest("POST", url)
-            content_type = 'multipart/form-data; boundary=%s' % boundary
+            content_type = ct
             conn.putheader('Content-type', content_type)
             conn.putheader('Content-length', str(len(body)))
             conn.putheader('Authorization', auth)
diff --git a/setuptools/gui-arm-32.exe b/setuptools/gui-arm-32.exe
deleted file mode 100644
index 537aff37..00000000
--- a/setuptools/gui-arm-32.exe
+++ /dev/null
diff --git a/setuptools/package_index.py b/setuptools/package_index.py
index c53343e4..e87504db 100755
--- a/setuptools/package_index.py
+++ b/setuptools/package_index.py
@@ -17,6 +17,7 @@ except ImportError:
 from setuptools.extern import six
 from setuptools.extern.six.moves import urllib, http_client, configparser, map
 
+import setuptools
 from pkg_resources import (
     CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST,
     require, Environment, find_distributions, safe_name, safe_version,
@@ -46,6 +47,11 @@ __all__ = [
 
 _SOCKET_TIMEOUT = 15
 
+
+_tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}"
+user_agent = _tmpl.format(py_major=sys.version[:3], **globals())
+
+
 def parse_bdist_wininst(name):
     """Return (base,pyversion) or (None,None) for possible .exe name"""
 
@@ -202,9 +208,6 @@ def find_external_links(url, page):
             if match:
                 yield urllib.parse.urljoin(url, htmldecode(match.group(1)))
 
-user_agent = "Python-urllib/%s setuptools/%s" % (
-    sys.version[:3], require('setuptools')[0].version
-)
 
 class ContentChecker(object):
     """
diff --git a/setuptools/tests/test_build_py.py b/setuptools/tests/test_build_py.py
new file mode 100644
index 00000000..ed1703ac
--- /dev/null
+++ b/setuptools/tests/test_build_py.py
@@ -0,0 +1,31 @@
+import os
+
+import pytest
+
+from setuptools.dist import Distribution
+
+
+@pytest.yield_fixture
+def tmpdir_as_cwd(tmpdir):
+    with tmpdir.as_cwd():
+        yield tmpdir
+
+
+def test_directories_in_package_data_glob(tmpdir_as_cwd):
+    """
+    Directories matching the glob in package_data should
+    not be included in the package data.
+
+    Regression test for #261.
+    """
+    dist = Distribution(dict(
+        script_name='setup.py',
+        script_args=['build_py'],
+        packages=[''],
+        name='foo',
+        package_data={'': ['path/*']},
+    ))
+    os.makedirs('path/subpath')
+    #with contexts.quiet():
+    dist.parse_command_line()
+    dist.run_commands()
diff --git a/setuptools/tests/test_upload_docs.py b/setuptools/tests/test_upload_docs.py
index cc71cadb..d3dee616 100644
--- a/setuptools/tests/test_upload_docs.py
+++ b/setuptools/tests/test_upload_docs.py
@@ -57,3 +57,15 @@ class TestUploadDocsTest:
 
             with contextlib.closing(zipfile.ZipFile(tmp_file)) as zip_file:
                 assert zip_file.namelist() == ['index.html']
+
+    def test_build_multipart(self):
+        data = dict(
+            a="foo",
+            b="bar",
+            file=('file.txt', b'content'),
+        )
+        body, content_type = upload_docs._build_multipart(data)
+        assert 'form-data' in content_type
+        assert isinstance(body, bytes)
+        assert b'foo' in body
+        assert b'content' in body