diff options
| -rw-r--r-- | CHANGES.txt | 10 | ||||
| -rw-r--r-- | coverage/phystokens.py | 3 | ||||
| -rw-r--r-- | test/test_phystokens.py | 1 |
3 files changed, 13 insertions, 1 deletions
diff --git a/CHANGES.txt b/CHANGES.txt index 4776eaa1..6a8cf844 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,16 @@ Change history for Coverage.py ------------------------------ + +Next version +------------ + +- Source files with DOS line endings are now properly tokenized for syntax + coloring on non-DOS machines. Fixes `issue 53`_. + +.. _issue 53: http://bitbucket.org/ned/coveragepy/issue/53 + + Version 3.3.1, 6 March 2010 --------------------------- diff --git a/coverage/phystokens.py b/coverage/phystokens.py index 5824b9b9..60b87932 100644 --- a/coverage/phystokens.py +++ b/coverage/phystokens.py @@ -77,7 +77,8 @@ def source_token_lines(source): ws_tokens = [token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL] line = [] col = 0 - tokgen = tokenize.generate_tokens(StringIO(source.expandtabs(8)).readline) + source = source.expandtabs(8).replace('\r\n', '\n') + tokgen = tokenize.generate_tokens(StringIO(source).readline) for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen): mark_start = True for part in re.split('(\n)', ttext): diff --git a/test/test_phystokens.py b/test/test_phystokens.py index 6b533cf6..0e778510 100644 --- a/test/test_phystokens.py +++ b/test/test_phystokens.py @@ -36,6 +36,7 @@ class PhysTokensTest(CoverageTest): tokenized += text + "\n" # source_token_lines doesn't preserve trailing spaces, so trim all that # before comparing. + source = source.replace('\r\n', '\n') source = re.sub("(?m)[ \t]+$", "", source) tokenized = re.sub("(?m)[ \t]+$", "", tokenized) self.assertMultiLineEqual(source, tokenized) |