3 files changed, 521 insertions, 33 deletions

diff --git a/setuptools/tests/test_archive_util.py b/setuptools/tests/test_archive_util.py
index 3e679c11..b789e9ac 100644
--- a/setuptools/tests/test_archive_util.py
+++ b/setuptools/tests/test_archive_util.py
@@ -1,5 +1,8 @@
 # coding: utf-8
 
+import tarfile
+import io
+
 from setuptools.extern import six
 
 import pytest
@@ -12,41 +15,24 @@ def tarfile_with_unicode(tmpdir):
     """
     Create a tarfile containing only a file whose name is
     a zero byte file called testimäge.png.
-
-    TODO: Is it possible to generate this file programmatically?
     """
-    data = (
-        b'\x1f\x8b\x08\x00R\xfe\x9fW\x00\x03\xed\xd6AO\x13A\x14\x00'
-        b'\xe0\x11b\x8c\x8d\' z\xf02\x07=x\xd9\xce\xcc\xce\xec\xd4\xc3'
-        b'&\xa21\xb4X\\lI\xa3r0\xc3v\xc1\x8a\xec6\xcbbz3\x1cH\x80x\x93'
-        b'x\xeaQ\x12\x13\x0f$z\xd5\x13\x17\xa9U\x8e&\xfc\x06\x13\xff'
-        b'\x82\xb3\xa5\n\xb6J!\x16\x8c\xf0\xbed2\x9d\xd7\xe9v\xba\xb3'
-        b'\xafo\x8c\xe4\xa8\xaa\xa4=U\xf4\xc2\xa4\xf1 \xf2f\xa3\xd2\xcc'
-        b'\xfa\xcb)\xcf(\xfbS\xa8K\x08!\x16\xe78\xee\xa5%\x1a=a\xdb'
-        b'\xe3\x06FLL\x99\xe4R#\x9cbB%\xa5\x1c\xe1J\xb7\x16\xb0\x97'
-        b'\xb9\xd9H\x85z)\x8fT\xa8\xdc\xe0\xcf\xf3\xf4\xb4\xc9\xc9=\xae'
-        b'\xb3\xfdS\xf0\xcf\xfe?\xc1$.\xab\xe8\xa1m\xb4\xed~\x82\x11'
-        b'\xec\xea\xb1gS.\t%&e,\x8e\xa9\xdd1"S\tf\xe2\xfc\x8dt&{\xcf(zO'
-        b'lj\xe9]d\x8c\xec\n\x97\xfc\xc0\xe6\xdc\x12\x84\x9a2AS?\xc2\xfe'
-        b'\xe3\x92?m\xd3\xc4\xbf\xbe\x05\'Z\xfb\xbew\xff;:\xe5\xbf)dK\xfe'
-        b'\x0b*\x04\xc2\xa4\xfbKiw\xc2\xf3\x1f\x9d>\x7f\x06\xf5 4\xa2\\'
-        b'\xec\xe4\xf1]\xdc\x14\xc7\xd0Y\xdd\x98n\xefu\x8b\xc7\xdf\xf6w'
-        b'\xc9\xc1\xb1\xb1\\\xf3e\xfc\x89\xaan\xf9\x96)\xa7v\xe2\x17\xdc'
-        b'`\xc6(\x86Ay"\xa8\x18*\x8a\xc2\xd2\xc4\x9c~"\xf5\x9b\x95\xea'
-        b'\xeb\xc2\xf0\x95Z2][[t>\xd63\x9f\xb2K\x9b\x1b\xce\xed\xfa\x9d7'
-        b'\xb9W\x85\xdaH\xf6\xf3\x87z\xef\xd2\xe5\x17+\x03\xf3\x0b\xb9'
-        b'\xbe[}\xf3\xe7V\xab+h\xa8w\xbd\xecl\x86\xfd\xce\xf3\x9e/\xd7'
-        b'\x9e\xbe}\xb6|ih||uk\xeb>z\xb7v\xf1\xeb\xdf\xdf\xafcf\xa7\xfa'
-        b'\x1f\xde\xbf@\xc7\xfa\xcfEK\xfe[B\x10\xa8\xffGAW\xe9F\xfd\xefP'
-        b'\xfb\x894\x7f[\xfb\xcd\x14\xcef\xae\x0f\xe6tE/\xdc4\xdc\xd0'
-        b'\xd33\x02\xbf9\xcbJq}f\xe0t\xdf\'\x04~\x95\xeb0\x9c\x10\x8e'
-        b'\xd0a\xd7\xfeX\xa7\xfc\x8f\xf3\xe5\xd7\xfc\xe7\xc2\xe2P\xff'
-        b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
-        b'\x008\xa8\xef\xb1g\xe1Z\x00(\x00\x00'
-    )
+    tarobj = io.BytesIO()
+
+    with tarfile.open(fileobj=tarobj, mode="w:gz") as tgz:
+        data = b""
+
+        filename = "testimäge.png"
+        if six.PY2:
+            filename = filename.decode('utf-8')
+
+        t = tarfile.TarInfo(filename)
+        t.size = len(data)
+
+        tgz.addfile(t, io.BytesIO(data))
+
     target = tmpdir / 'unicode-pkg-1.0.tar.gz'
     with open(str(target), mode='wb') as tf:
-        tf.write(data)
+        tf.write(tarobj.getvalue())
     return str(target)
 
 
diff --git a/setuptools/tests/test_build_ext.py b/setuptools/tests/test_build_ext.py
index 5168ebf0..ac002f44 100644
--- a/setuptools/tests/test_build_ext.py
+++ b/setuptools/tests/test_build_ext.py
@@ -1,7 +1,12 @@
+import sys
 import distutils.command.build_ext as orig
+from distutils.sysconfig import get_config_var
 
-from setuptools.command.build_ext import build_ext
+from setuptools.extern import six
+
+from setuptools.command.build_ext import build_ext, get_abi3_suffix
 from setuptools.dist import Distribution
+from setuptools.extension import Extension
 
 
 class TestBuildExt:
@@ -18,3 +23,24 @@ class TestBuildExt:
         res = cmd.get_ext_filename('foo')
         wanted = orig.build_ext.get_ext_filename(cmd, 'foo')
         assert res == wanted
+
+    def test_abi3_filename(self):
+        """
+        Filename needs to be loadable by several versions
+        of Python 3 if 'is_abi3' is truthy on Extension()
+        """
+        print(get_abi3_suffix())
+
+        extension = Extension('spam.eggs', ['eggs.c'], py_limited_api=True)
+        dist = Distribution(dict(ext_modules=[extension]))
+        cmd = build_ext(dist)
+        cmd.finalize_options()
+        assert 'spam.eggs' in cmd.ext_map
+        res = cmd.get_ext_filename('spam.eggs')
+
+        if six.PY2 or not get_abi3_suffix():
+            assert res.endswith(get_config_var('SO'))
+        elif sys.platform == 'win32':
+            assert res.endswith('eggs.pyd')
+        else:
+            assert 'abi3' in res
diff --git a/setuptools/tests/test_manifest.py b/setuptools/tests/test_manifest.py
new file mode 100644
index 00000000..6360270d
--- /dev/null
+++ b/setuptools/tests/test_manifest.py
@@ -0,0 +1,476 @@
+# -*- coding: utf-8 -*-
+"""sdist tests"""
+
+import contextlib
+import os
+import shutil
+import sys
+import tempfile
+from distutils import log
+from distutils.errors import DistutilsTemplateError
+
+from setuptools.command.egg_info import FileList, egg_info
+from setuptools.dist import Distribution
+from setuptools.extern import six
+from setuptools.tests.textwrap import DALS
+
+import pytest
+
+py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only")
+
+
+def make_local_path(s):
+    """Converts '/' in a string to os.sep"""
+    return s.replace('/', os.sep)
+
+
+SETUP_ATTRS = {
+    'name': 'app',
+    'version': '0.0',
+    'packages': ['app'],
+}
+
+
+SETUP_PY = """\
+from setuptools import setup
+
+setup(**%r)
+""" % SETUP_ATTRS
+
+
+@contextlib.contextmanager
+def quiet():
+    old_stdout, old_stderr = sys.stdout, sys.stderr
+    sys.stdout, sys.stderr = six.StringIO(), six.StringIO()
+    try:
+        yield
+    finally:
+        sys.stdout, sys.stderr = old_stdout, old_stderr
+
+
+def touch(filename):
+    open(filename, 'w').close()
+
+
+# The set of files always in the manifest, including all files in the
+# .egg-info directory
+default_files = frozenset(map(make_local_path, [
+    'README.rst',
+    'MANIFEST.in',
+    'setup.py',
+    'app.egg-info/PKG-INFO',
+    'app.egg-info/SOURCES.txt',
+    'app.egg-info/dependency_links.txt',
+    'app.egg-info/top_level.txt',
+    'app/__init__.py',
+]))
+
+
+class TempDirTestCase(object):
+
+    def setup_method(self, method):
+        self.temp_dir = tempfile.mkdtemp()
+        self.old_cwd = os.getcwd()
+        os.chdir(self.temp_dir)
+
+    def teardown_method(self, method):
+        os.chdir(self.old_cwd)
+        shutil.rmtree(self.temp_dir)
+
+
+class TestManifestTest(TempDirTestCase):
+
+    def setup_method(self, method):
+        super(TestManifestTest, self).setup_method(method)
+
+        f = open(os.path.join(self.temp_dir, 'setup.py'), 'w')
+        f.write(SETUP_PY)
+        f.close()
+
+        """
+        Create a file tree like:
+        - LICENSE
+        - README.rst
+        - testing.rst
+        - .hidden.rst
+        - app/
+            - __init__.py
+            - a.txt
+            - b.txt
+            - c.rst
+            - static/
+                - app.js
+                - app.js.map
+                - app.css
+                - app.css.map
+        """
+
+        for fname in ['README.rst', '.hidden.rst', 'testing.rst', 'LICENSE']:
+            touch(os.path.join(self.temp_dir, fname))
+
+        # Set up the rest of the test package
+        test_pkg = os.path.join(self.temp_dir, 'app')
+        os.mkdir(test_pkg)
+        for fname in ['__init__.py', 'a.txt', 'b.txt', 'c.rst']:
+            touch(os.path.join(test_pkg, fname))
+
+        # Some compiled front-end assets to include
+        static = os.path.join(test_pkg, 'static')
+        os.mkdir(static)
+        for fname in ['app.js', 'app.js.map', 'app.css', 'app.css.map']:
+            touch(os.path.join(static, fname))
+
+    def make_manifest(self, contents):
+        """Write a MANIFEST.in."""
+        with open(os.path.join(self.temp_dir, 'MANIFEST.in'), 'w') as f:
+            f.write(DALS(contents))
+
+    def get_files(self):
+        """Run egg_info and get all the files to include, as a set"""
+        dist = Distribution(SETUP_ATTRS)
+        dist.script_name = 'setup.py'
+        cmd = egg_info(dist)
+        cmd.ensure_finalized()
+
+        cmd.run()
+
+        return set(cmd.filelist.files)
+
+    def test_no_manifest(self):
+        """Check a missing MANIFEST.in includes only the standard files."""
+        assert (default_files - set(['MANIFEST.in'])) == self.get_files()
+
+    def test_empty_files(self):
+        """Check an empty MANIFEST.in includes only the standard files."""
+        self.make_manifest("")
+        assert default_files == self.get_files()
+
+    def test_include(self):
+        """Include extra rst files in the project root."""
+        self.make_manifest("include *.rst")
+        files = default_files | set([
+            'testing.rst', '.hidden.rst'])
+        assert files == self.get_files()
+
+    def test_exclude(self):
+        """Include everything in app/ except the text files"""
+        l = make_local_path
+        self.make_manifest(
+            """
+            include app/*
+            exclude app/*.txt
+            """)
+        files = default_files | set([l('app/c.rst')])
+        assert files == self.get_files()
+
+    def test_include_multiple(self):
+        """Include with multiple patterns."""
+        l = make_local_path
+        self.make_manifest("include app/*.txt app/static/*")
+        files = default_files | set([
+            l('app/a.txt'), l('app/b.txt'),
+            l('app/static/app.js'), l('app/static/app.js.map'),
+            l('app/static/app.css'), l('app/static/app.css.map')])
+        assert files == self.get_files()
+
+    def test_graft(self):
+        """Include the whole app/static/ directory."""
+        l = make_local_path
+        self.make_manifest("graft app/static")
+        files = default_files | set([
+            l('app/static/app.js'), l('app/static/app.js.map'),
+            l('app/static/app.css'), l('app/static/app.css.map')])
+        assert files == self.get_files()
+
+    def test_graft_global_exclude(self):
+        """Exclude all *.map files in the project."""
+        l = make_local_path
+        self.make_manifest(
+            """
+            graft app/static
+            global-exclude *.map
+            """)
+        files = default_files | set([
+            l('app/static/app.js'), l('app/static/app.css')])
+        assert files == self.get_files()
+
+    def test_global_include(self):
+        """Include all *.rst, *.js, and *.css files in the whole tree."""
+        l = make_local_path
+        self.make_manifest(
+            """
+            global-include *.rst *.js *.css
+            """)
+        files = default_files | set([
+            '.hidden.rst', 'testing.rst', l('app/c.rst'),
+            l('app/static/app.js'), l('app/static/app.css')])
+        assert files == self.get_files()
+
+    def test_graft_prune(self):
+        """Include all files in app/, except for the whole app/static/ dir."""
+        l = make_local_path
+        self.make_manifest(
+            """
+            graft app
+            prune app/static
+            """)
+        files = default_files | set([
+            l('app/a.txt'), l('app/b.txt'), l('app/c.rst')])
+        assert files == self.get_files()
+
+
+class TestFileListTest(TempDirTestCase):
+    """
+    A copy of the relevant bits of distutils/tests/test_filelist.py,
+    to ensure setuptools' version of FileList keeps parity with distutils.
+    """
+
+    def setup_method(self, method):
+        super(TestFileListTest, self).setup_method(method)
+        self.threshold = log.set_threshold(log.FATAL)
+        self._old_log = log.Log._log
+        log.Log._log = self._log
+        self.logs = []
+
+    def teardown_method(self, method):
+        log.set_threshold(self.threshold)
+        log.Log._log = self._old_log
+        super(TestFileListTest, self).teardown_method(method)
+
+    def _log(self, level, msg, args):
+        if level not in (log.DEBUG, log.INFO, log.WARN, log.ERROR, log.FATAL):
+            raise ValueError('%s wrong log level' % str(level))
+        self.logs.append((level, msg, args))
+
+    def get_logs(self, *levels):
+        def _format(msg, args):
+            if len(args) == 0:
+                return msg
+            return msg % args
+        return [_format(msg, args) for level, msg, args
+                in self.logs if level in levels]
+
+    def clear_logs(self):
+        self.logs = []
+
+    def assertNoWarnings(self):
+        assert self.get_logs(log.WARN) == []
+        self.clear_logs()
+
+    def assertWarnings(self):
+        assert len(self.get_logs(log.WARN)) > 0
+        self.clear_logs()
+
+    def make_files(self, files):
+        for file in files:
+            file = os.path.join(self.temp_dir, file)
+            dirname, basename = os.path.split(file)
+            if not os.path.exists(dirname):
+                os.makedirs(dirname)
+            open(file, 'w').close()
+
+    def test_process_template_line(self):
+        # testing  all MANIFEST.in template patterns
+        file_list = FileList()
+        l = make_local_path
+
+        # simulated file list
+        self.make_files([
+            'foo.tmp', 'ok', 'xo', 'four.txt',
+            'buildout.cfg',
+            # filelist does not filter out VCS directories,
+            # it's sdist that does
+            l('.hg/last-message.txt'),
+            l('global/one.txt'),
+            l('global/two.txt'),
+            l('global/files.x'),
+            l('global/here.tmp'),
+            l('f/o/f.oo'),
+            l('dir/graft-one'),
+            l('dir/dir2/graft2'),
+            l('dir3/ok'),
+            l('dir3/sub/ok.txt'),
+        ])
+
+        MANIFEST_IN = DALS("""\
+        include ok
+        include xo
+        exclude xo
+        include foo.tmp
+        include buildout.cfg
+        global-include *.x
+        global-include *.txt
+        global-exclude *.tmp
+        recursive-include f *.oo
+        recursive-exclude global *.x
+        graft dir
+        prune dir3
+        """)
+
+        for line in MANIFEST_IN.split('\n'):
+            if not line:
+                continue
+            file_list.process_template_line(line)
+
+        wanted = [
+            'buildout.cfg',
+            'four.txt',
+            'ok',
+            l('.hg/last-message.txt'),
+            l('dir/graft-one'),
+            l('dir/dir2/graft2'),
+            l('f/o/f.oo'),
+            l('global/one.txt'),
+            l('global/two.txt'),
+        ]
+        file_list.sort()
+
+        assert file_list.files == wanted
+
+    def test_exclude_pattern(self):
+        # return False if no match
+        file_list = FileList()
+        assert not file_list.exclude_pattern('*.py')
+
+        # return True if files match
+        file_list = FileList()
+        file_list.files = ['a.py', 'b.py']
+        assert file_list.exclude_pattern('*.py')
+
+        # test excludes
+        file_list = FileList()
+        file_list.files = ['a.py', 'a.txt']
+        file_list.exclude_pattern('*.py')
+        assert file_list.files == ['a.txt']
+
+    def test_include_pattern(self):
+        # return False if no match
+        file_list = FileList()
+        file_list.set_allfiles([])
+        assert not file_list.include_pattern('*.py')
+
+        # return True if files match
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', 'b.txt'])
+        assert file_list.include_pattern('*.py')
+
+        # test * matches all files
+        file_list = FileList()
+        assert file_list.allfiles is None
+        file_list.set_allfiles(['a.py', 'b.txt'])
+        file_list.include_pattern('*')
+        assert file_list.allfiles == ['a.py', 'b.txt']
+
+    def test_process_template(self):
+        l = make_local_path
+        # invalid lines
+        file_list = FileList()
+        for action in ('include', 'exclude', 'global-include',
+                       'global-exclude', 'recursive-include',
+                       'recursive-exclude', 'graft', 'prune', 'blarg'):
+            try:
+                file_list.process_template_line(action)
+            except DistutilsTemplateError:
+                pass
+            except Exception:
+                assert False, "Incorrect error thrown"
+            else:
+                assert False, "Should have thrown an error"
+
+        # include
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', 'b.txt', l('d/c.py')])
+
+        file_list.process_template_line('include *.py')
+        assert file_list.files == ['a.py']
+        self.assertNoWarnings()
+
+        file_list.process_template_line('include *.rb')
+        assert file_list.files == ['a.py']
+        self.assertWarnings()
+
+        # exclude
+        file_list = FileList()
+        file_list.files = ['a.py', 'b.txt', l('d/c.py')]
+
+        file_list.process_template_line('exclude *.py')
+        assert file_list.files == ['b.txt', l('d/c.py')]
+        self.assertNoWarnings()
+
+        file_list.process_template_line('exclude *.rb')
+        assert file_list.files == ['b.txt', l('d/c.py')]
+        self.assertWarnings()
+
+        # global-include
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', 'b.txt', l('d/c.py')])
+
+        file_list.process_template_line('global-include *.py')
+        assert file_list.files == ['a.py', l('d/c.py')]
+        self.assertNoWarnings()
+
+        file_list.process_template_line('global-include *.rb')
+        assert file_list.files == ['a.py', l('d/c.py')]
+        self.assertWarnings()
+
+        # global-exclude
+        file_list = FileList()
+        file_list.files = ['a.py', 'b.txt', l('d/c.py')]
+
+        file_list.process_template_line('global-exclude *.py')
+        assert file_list.files == ['b.txt']
+        self.assertNoWarnings()
+
+        file_list.process_template_line('global-exclude *.rb')
+        assert file_list.files == ['b.txt']
+        self.assertWarnings()
+
+        # recursive-include
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', l('d/b.py'), l('d/c.txt'),
+                                l('d/d/e.py')])
+
+        file_list.process_template_line('recursive-include d *.py')
+        assert file_list.files == [l('d/b.py'), l('d/d/e.py')]
+        self.assertNoWarnings()
+
+        file_list.process_template_line('recursive-include e *.py')
+        assert file_list.files == [l('d/b.py'), l('d/d/e.py')]
+        self.assertWarnings()
+
+        # recursive-exclude
+        file_list = FileList()
+        file_list.files = ['a.py', l('d/b.py'), l('d/c.txt'), l('d/d/e.py')]
+
+        file_list.process_template_line('recursive-exclude d *.py')
+        assert file_list.files == ['a.py', l('d/c.txt')]
+        self.assertNoWarnings()
+
+        file_list.process_template_line('recursive-exclude e *.py')
+        assert file_list.files == ['a.py', l('d/c.txt')]
+        self.assertWarnings()
+
+        # graft
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', l('d/b.py'), l('d/d/e.py'),
+                                l('f/f.py')])
+
+        file_list.process_template_line('graft d')
+        assert file_list.files == [l('d/b.py'), l('d/d/e.py')]
+        self.assertNoWarnings()
+
+        file_list.process_template_line('graft e')
+        assert file_list.files == [l('d/b.py'), l('d/d/e.py')]
+        self.assertWarnings()
+
+        # prune
+        file_list = FileList()
+        file_list.files = ['a.py', l('d/b.py'), l('d/d/e.py'), l('f/f.py')]
+
+        file_list.process_template_line('prune d')
+        assert file_list.files == ['a.py', l('f/f.py')]
+        self.assertNoWarnings()
+
+        file_list.process_template_line('prune e')
+        assert file_list.files == ['a.py', l('f/f.py')]
+        self.assertWarnings()