Deal properly with dos-ended source files while tokenizing. Fixes issue #53.

3 files changed, 13 insertions, 1 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 4776eaa1..6a8cf844 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -2,6 +2,16 @@
 Change history for Coverage.py
 ------------------------------
 
+
+Next version
+------------
+
+- Source files with DOS line endings are now properly tokenized for syntax
+  coloring on non-DOS machines.  Fixes `issue  53`_.
+
+.. _issue 53: http://bitbucket.org/ned/coveragepy/issue/53
+
+
 Version 3.3.1, 6 March 2010
 ---------------------------
 
diff --git a/coverage/phystokens.py b/coverage/phystokens.py
index 5824b9b9..60b87932 100644
--- a/coverage/phystokens.py
+++ b/coverage/phystokens.py
@@ -77,7 +77,8 @@ def source_token_lines(source):
     ws_tokens = [token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL]
     line = []
     col = 0
-    tokgen = tokenize.generate_tokens(StringIO(source.expandtabs(8)).readline)
+    source = source.expandtabs(8).replace('\r\n', '\n')
+    tokgen = tokenize.generate_tokens(StringIO(source).readline)
     for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):
         mark_start = True
         for part in re.split('(\n)', ttext):
diff --git a/test/test_phystokens.py b/test/test_phystokens.py
index 6b533cf6..0e778510 100644
--- a/test/test_phystokens.py
+++ b/test/test_phystokens.py
@@ -36,6 +36,7 @@ class PhysTokensTest(CoverageTest):
             tokenized += text + "\n"
         # source_token_lines doesn't preserve trailing spaces, so trim all that
         # before comparing.
+        source = source.replace('\r\n', '\n')
         source = re.sub("(?m)[ \t]+$", "", source)
         tokenized = re.sub("(?m)[ \t]+$", "", tokenized)
         self.assertMultiLineEqual(source, tokenized)