diff options
author | Ned Batchelder <ned@nedbatchelder.com> | 2014-12-12 08:51:19 -0500 |
---|---|---|
committer | Ned Batchelder <ned@nedbatchelder.com> | 2014-12-12 08:51:19 -0500 |
commit | 59889252937e31ab4c3b734922c63dd529e26e03 (patch) | |
tree | 94b7823a9ab7f04f52c8e37cdc29453376b3edc4 /coverage/phystokens.py | |
parent | c76154ce75adbebb461d4cb0a524e6c08b4ed5f7 (diff) | |
download | python-coveragepy-59889252937e31ab4c3b734922c63dd529e26e03.tar.gz |
Source is always Unicode in HTML code. More refactoring to come.
Diffstat (limited to 'coverage/phystokens.py')
-rw-r--r-- | coverage/phystokens.py | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/coverage/phystokens.py b/coverage/phystokens.py index fe77c7d..3fd1165 100644 --- a/coverage/phystokens.py +++ b/coverage/phystokens.py @@ -1,6 +1,7 @@ """Better tokenizing for coverage.py.""" import codecs, keyword, re, sys, token, tokenize + from coverage.parser import generate_tokens @@ -75,6 +76,7 @@ def source_token_lines(source): is indistinguishable from a final line with a newline. """ + ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL]) line = [] col = 0 @@ -108,6 +110,7 @@ def source_token_lines(source): if line: yield line + def source_encoding(source): """Determine the encoding for `source` (a string), according to PEP 263. |