summaryrefslogtreecommitdiff
path: root/coverage/phystokens.py
diff options
context:
space:
mode:
authorNed Batchelder <ned@nedbatchelder.com>2014-12-12 08:51:19 -0500
committerNed Batchelder <ned@nedbatchelder.com>2014-12-12 08:51:19 -0500
commit59889252937e31ab4c3b734922c63dd529e26e03 (patch)
tree94b7823a9ab7f04f52c8e37cdc29453376b3edc4 /coverage/phystokens.py
parentc76154ce75adbebb461d4cb0a524e6c08b4ed5f7 (diff)
downloadpython-coveragepy-59889252937e31ab4c3b734922c63dd529e26e03.tar.gz
Source is always Unicode in HTML code. More refactoring to come.
Diffstat (limited to 'coverage/phystokens.py')
-rw-r--r--coverage/phystokens.py3
1 files changed, 3 insertions, 0 deletions
diff --git a/coverage/phystokens.py b/coverage/phystokens.py
index fe77c7d..3fd1165 100644
--- a/coverage/phystokens.py
+++ b/coverage/phystokens.py
@@ -1,6 +1,7 @@
"""Better tokenizing for coverage.py."""
import codecs, keyword, re, sys, token, tokenize
+
from coverage.parser import generate_tokens
@@ -75,6 +76,7 @@ def source_token_lines(source):
is indistinguishable from a final line with a newline.
"""
+
ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])
line = []
col = 0
@@ -108,6 +110,7 @@ def source_token_lines(source):
if line:
yield line
+
def source_encoding(source):
"""Determine the encoding for `source` (a string), according to PEP 263.