Files with incorrect encoding declarations are no longer ignored. #351

5 files changed, 18 insertions, 26 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 9f1823e2..1abf564f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -22,6 +22,9 @@ Latest
   (`issue 360`_).  It's always fun when the problem is due to a `bug in the
   Python standard library <http://bugs.python.org/issue19035>`_.
 
+- Files with incorrect encoding declaration comments are no longer ignored by
+  the reporting commands, fixing `issue 351`_.
+
 - HTML reports now include a timestamp in the footer, closing `issue 299`_.
   Thanks, Conrad Ho.
 
@@ -36,6 +39,7 @@ Latest
 .. _issue 299: https://bitbucket.org/ned/coveragepy/issue/299/inserted-created-on-yyyy-mm-dd-hh-mm-in
 .. _issue 308: https://bitbucket.org/ned/coveragepy/issue/308/yield-lambda-branch-coverage
 .. _issue 324: https://bitbucket.org/ned/coveragepy/issue/324/yield-in-loop-confuses-branch-coverage
+.. _issue 351: https://bitbucket.org/ned/coveragepy/issue/351/files-with-incorrect-encoding-are-ignored
 .. _issue 354: https://bitbucket.org/ned/coveragepy/issue/354/coverage-combine-should-take-a-list-of
 .. _issue 359: https://bitbucket.org/ned/coveragepy/issue/359/xml-report-chunk-error
 .. _issue 360: https://bitbucket.org/ned/coveragepy/issue/360/html-reports-get-confused-by-l-in-the-code
diff --git a/coverage/phystokens.py b/coverage/phystokens.py
index d21d401c..587ec1c6 100644
--- a/coverage/phystokens.py
+++ b/coverage/phystokens.py
@@ -283,7 +283,7 @@ def compile_unicode(source, filename, mode):
     try:
         code = compile(source, filename, mode)
     except SyntaxError as synerr:
-        if synerr.args[0] != "encoding declaration in Unicode string":
+        if "coding declaration in unicode string" not in synerr.args[0].lower():
             raise
         source = neuter_encoding_declaration(source)
         code = compile(source, filename, mode)
diff --git a/coverage/python.py b/coverage/python.py
index f335f165..3e8e9f39 100644
--- a/coverage/python.py
+++ b/coverage/python.py
@@ -1,8 +1,6 @@
 """Python source expertise for coverage.py"""
 
 import os.path
-import sys
-import tokenize
 import zipimport
 
 from coverage import env
@@ -14,21 +12,15 @@ from coverage.phystokens import source_token_lines, source_encoding
 from coverage.plugin import FileReporter
 
 
-@contract(returns='str')
+@contract(returns='bytes')
 def read_python_source(filename):
     """Read the Python source text from `filename`.
 
-    Returns a str: unicode on Python 3, bytes on Python 2.
+    Returns bytes.
 
     """
-    # Python 3.2 provides `tokenize.open`, the best way to open source files.
-    if sys.version_info >= (3, 2):
-        f = tokenize.open(filename)
-    else:
-        f = open(filename, "rU")
-
-    with f:
-        return f.read()
+    with open(filename, "rb") as f:
+        return f.read().replace(b"\r\n", b"\n").replace(b"\r", b"\n")
 
 
 @contract(returns='unicode')
@@ -50,15 +42,12 @@ def get_python_source(filename):
         # Maybe it's in a zip file?
         source = get_zip_bytes(try_filename)
         if source is not None:
-            if env.PY3:
-                source = source.decode(source_encoding(source), "replace")
             break
     else:
         # Couldn't find source.
         raise NoSource("No source for code: '%s'." % filename)
 
-    if env.PY2:
-        source = source.decode(source_encoding(source), "replace")
+    source = source.decode(source_encoding(source), "replace")
 
     # Python code should always end with a line with a newline.
     if source and source[-1] != '\n':
diff --git a/tests/test_html.py b/tests/test_html.py
index 6f6f42e4..a5878448 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -6,7 +6,6 @@ import os.path
 import re
 
 import coverage
-from coverage import env
 import coverage.html
 from coverage.misc import CoverageException, NotPython, NoSource
 
@@ -297,9 +296,8 @@ class HtmlWithUnparsableFilesTest(HtmlTestHelpers, CoverageTest):
         cov.html_report()
         self.assert_exists("htmlcov/index.html")
 
-    # TODO: enable this test, and then fix this:
-    # https://bitbucket.org/ned/coveragepy/issue/351/files-with-incorrect-encoding-are-ignored
-    def SKIP_THIS_decode_error(self):
+    def test_decode_error(self):
+        # https://bitbucket.org/ned/coveragepy/issue/351/files-with-incorrect-encoding-are-ignored
         # imp.load_module won't load a file with an undecodable character
         # in a comment, though Python will run them.  So we'll change the
         # file after running.
@@ -324,10 +322,7 @@ class HtmlWithUnparsableFilesTest(HtmlTestHelpers, CoverageTest):
         cov.html_report()
 
         html_report = self.get_html_report_content("sub/not_ascii.py")
-        if env.PY2:
-            expected = "# Isn&#39;t this great?&#65533;!"
-        else:
-            expected = "# Isn&#39;t this great?&#203;!"
+        expected = "# Isn&#39;t this great?&#65533;!"
         self.assertIn(expected, html_report)
 
     def test_formfeeds(self):
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 9359c408..2cf39baa 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -142,7 +142,11 @@ class ParserFileTest(CoverageTest):
             fname = fname + ".py"
             self.make_file(fname, text, newline=newline)
             parser = self.parse_file(fname)
-            self.assertEqual(parser.exit_counts(), counts)
+            self.assertEqual(
+                parser.exit_counts(),
+                counts,
+                "Wrong for %r" % fname
+            )
 
     def test_encoding(self):
         self.make_file("encoded.py", """\