Properly handle filenames with non-ASCII characters. #432

4 files changed, 111 insertions, 2 deletions

diff --git a/tests/helpers.py b/tests/helpers.py
index aa094bc1..2723ea59 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -4,6 +4,7 @@
 """Helpers for coverage.py tests."""
 
 import subprocess
+import sys
 
 
 def run_command(cmd):
@@ -12,8 +13,12 @@ def run_command(cmd):
     Returns the exit status code and the combined stdout and stderr.
 
     """
+    # In some strange cases (PyPy3 in a virtualenv!?) the stdout encoding of
+    # the subprocess is set incorrectly to ascii.  Use an environment variable
+    # to force the encoding to be the same as ours.
     proc = subprocess.Popen(
-        cmd, shell=True,
+        "PYTHONIOENCODING=%s %s" % (sys.__stdout__.encoding, cmd),
+        shell=True,
         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT
         )
diff --git a/tests/test_files.py b/tests/test_files.py
index e3d33285..e7353235 100644
--- a/tests/test_files.py
+++ b/tests/test_files.py
@@ -27,7 +27,7 @@ class FilesTest(CoverageTest):
     def test_simple(self):
         self.make_file("hello.py")
         files.set_relative_directory()
-        self.assertEqual(files.relative_filename("hello.py"), "hello.py")
+        self.assertEqual(files.relative_filename(u"hello.py"), u"hello.py")
         a = self.abs_path("hello.py")
         self.assertNotEqual(a, "hello.py")
         self.assertEqual(files.relative_filename(a), "hello.py")
diff --git a/tests/test_process.py b/tests/test_process.py
index 4902f7c0..8f69877f 100644
--- a/tests/test_process.py
+++ b/tests/test_process.py
@@ -884,6 +884,82 @@ class FailUnderEmptyFilesTest(CoverageTest):
         self.assertEqual(st, 2)
 
 
+class UnicodeFilePathsTest(CoverageTest):
+    """Tests of using non-ascii characters in the names of files."""
+
+    def test_snowman_dot_py(self):
+        # Make a file with a non-ascii character in the filename.
+        self.make_file(u"snowman☃.py", "print('snowman')")
+        out = self.run_command(u"coverage run snowman☃.py")
+        self.assertEqual(out, "snowman\n")
+
+        # The HTML report uses ascii-encoded HTML entities.
+        out = self.run_command("coverage html")
+        self.assertEqual(out, "")
+        self.assert_exists("htmlcov/snowman☃_py.html")
+        with open("htmlcov/index.html") as indexf:
+            index = indexf.read()
+        self.assertIn('<a href="snowman&#9731;_py.html">snowman&#9731;.py</a>', index)
+
+        # The XML report is always UTF8-encoded.
+        out = self.run_command("coverage xml")
+        self.assertEqual(out, "")
+        with open("coverage.xml", "rb") as xmlf:
+            xml = xmlf.read()
+        self.assertIn(u' filename="snowman☃.py"'.encode('utf8'), xml)
+        self.assertIn(u' name="snowman☃.py"'.encode('utf8'), xml)
+
+        report_expected = (
+            u"Name          Stmts   Miss  Cover\n"
+            u"---------------------------------\n"
+            u"snowman☃.py       1      0   100%\n"
+        )
+
+        if env.PY2:
+            report_expected = report_expected.encode("utf8")
+
+        out = self.run_command("coverage report")
+        self.assertEqual(out, report_expected)
+
+    def test_snowman_directory(self):
+        # Make a file with a non-ascii character in the directory name.
+        self.make_file(u"☃/snowman.py", "print('snowman')")
+        out = self.run_command(u"coverage run ☃/snowman.py")
+        self.assertEqual(out, "snowman\n")
+
+        # The HTML report uses ascii-encoded HTML entities.
+        out = self.run_command("coverage html")
+        self.assertEqual(out, "")
+        self.assert_exists("htmlcov/☃_snowman_py.html")
+        with open("htmlcov/index.html") as indexf:
+            index = indexf.read()
+        self.assertIn('<a href="&#9731;_snowman_py.html">&#9731;/snowman.py</a>', index)
+
+        # The XML report is always UTF8-encoded.
+        out = self.run_command("coverage xml")
+        self.assertEqual(out, "")
+        with open("coverage.xml", "rb") as xmlf:
+            xml = xmlf.read()
+        self.assertIn(u' filename="☃/snowman.py"'.encode('utf8'), xml)
+        self.assertIn(u' name="snowman.py"'.encode('utf8'), xml)
+        self.assertIn(
+            u'<package branch-rate="0" complexity="0" line-rate="1" name="☃">'.encode('utf8'),
+            xml
+        )
+
+        report_expected = (
+            u"Name           Stmts   Miss  Cover\n"
+            u"----------------------------------\n"
+            u"☃/snowman.py       1      0   100%\n"
+        )
+
+        if env.PY2:
+            report_expected = report_expected.encode("utf8")
+
+        out = self.run_command("coverage report")
+        self.assertEqual(out, report_expected)
+
+
 def possible_pth_dirs():
     """Produce a sequence of directories for trying to write .pth files."""
     # First look through sys.path, and we find a .pth file, then it's a good
diff --git a/tests/test_summary.py b/tests/test_summary.py
index cf55130a..f1331fea 100644
--- a/tests/test_summary.py
+++ b/tests/test_summary.py
@@ -1,3 +1,4 @@
+# coding: utf8
 # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
 # For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt
 
@@ -404,6 +405,33 @@ class SummaryTest(CoverageTest):
             "mycode.py NotPython: Couldn't parse 'mycode.py' as Python source: 'error' at line 1"
         )
 
+    def test_snowmandotpy_not_python(self):
+        # We run a .py file with a non-ascii name, and when reporting, we can't
+        # parse it as Python.  We should get an error message in the report.
+
+        self.make_file(u"snowman☃.py", "print('snowman')")
+        self.run_command(u"coverage run snowman☃.py")
+        self.make_file(u"snowman☃.py", "This isn't python at all!")
+        report = self.report_from_command(u"coverage report snowman☃.py")
+
+        # Name     Stmts   Miss  Cover
+        # ----------------------------
+        # xxxx   NotPython: Couldn't parse '...' as Python source: 'invalid syntax' at line 1
+        # No data to report.
+
+        last = self.squeezed_lines(report)[-2]
+        # The actual file name varies run to run.
+        last = re.sub(r"parse '.*(snowman.*?\.py)", r"parse '\1", last)
+        # The actual error message varies version to version
+        last = re.sub(r": '.*' at", ": 'error' at", last)
+        expected = (
+            u"snowman☃.py NotPython: "
+            u"Couldn't parse 'snowman☃.py' as Python source: 'error' at line 1"
+        )
+        if env.PY2:
+            expected = expected.encode("utf8")
+        self.assertEqual(last, expected)
+
     def test_dotpy_not_python_ignored(self):
         # We run a .py file, and when reporting, we can't parse it as Python,
         # but we've said to ignore errors, so there's no error reported.