diff options
| author | Ned Batchelder <ned@nedbatchelder.com> | 2010-03-13 08:00:40 -0500 |
|---|---|---|
| committer | Ned Batchelder <ned@nedbatchelder.com> | 2010-03-13 08:00:40 -0500 |
| commit | 9874b43d578920e37ec323c5cc2cd233f0e2e4b7 (patch) | |
| tree | 4f8bc676c322b68d5a2948dcd2fbd318b0f84c13 /coverage/phystokens.py | |
| parent | 5f4ddfceb7243e142ba0211672c7a1f34a50fa0c (diff) | |
| download | python-coveragepy-9874b43d578920e37ec323c5cc2cd233f0e2e4b7.tar.gz | |
Deal properly with dos-ended source files while tokenizing. Fixes issue #53.
Diffstat (limited to 'coverage/phystokens.py')
| -rw-r--r-- | coverage/phystokens.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/coverage/phystokens.py b/coverage/phystokens.py index 5824b9b..60b8793 100644 --- a/coverage/phystokens.py +++ b/coverage/phystokens.py @@ -77,7 +77,8 @@ def source_token_lines(source): ws_tokens = [token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL] line = [] col = 0 - tokgen = tokenize.generate_tokens(StringIO(source.expandtabs(8)).readline) + source = source.expandtabs(8).replace('\r\n', '\n') + tokgen = tokenize.generate_tokens(StringIO(source).readline) for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen): mark_start = True for part in re.split('(\n)', ttext): |