diff options
author | Ned Batchelder <ned@nedbatchelder.com> | 2014-12-12 08:51:19 -0500 |
---|---|---|
committer | Ned Batchelder <ned@nedbatchelder.com> | 2014-12-12 08:51:19 -0500 |
commit | 6055667e8090fc09e55b96ae49d5ac6a37706ede (patch) | |
tree | 12eba2ac0b9cb22476de8ec3d3eb9a6d183ad0b2 /coverage/phystokens.py | |
parent | 24e2488c36e4d19fd9eb06a5b03cd92821b4f2bd (diff) | |
download | python-coveragepy-git-6055667e8090fc09e55b96ae49d5ac6a37706ede.tar.gz |
Source is always Unicode in HTML code. More refactoring to come.
Diffstat (limited to 'coverage/phystokens.py')
-rw-r--r-- | coverage/phystokens.py | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/coverage/phystokens.py b/coverage/phystokens.py index fe77c7de..3fd1165c 100644 --- a/coverage/phystokens.py +++ b/coverage/phystokens.py @@ -1,6 +1,7 @@ """Better tokenizing for coverage.py.""" import codecs, keyword, re, sys, token, tokenize + from coverage.parser import generate_tokens @@ -75,6 +76,7 @@ def source_token_lines(source): is indistinguishable from a final line with a newline. """ + ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL]) line = [] col = 0 @@ -108,6 +110,7 @@ def source_token_lines(source): if line: yield line + def source_encoding(source): """Determine the encoding for `source` (a string), according to PEP 263. |