summaryrefslogtreecommitdiff
path: root/coverage/phystokens.py
diff options
context:
space:
mode:
authorNed Batchelder <ned@nedbatchelder.com>2009-12-03 08:54:27 -0500
committerNed Batchelder <ned@nedbatchelder.com>2009-12-03 08:54:27 -0500
commit3b6e8394c3e50cb733caafb275d2ae85c0397565 (patch)
treedd960443f9ef3772020d74c102d785968bb44dce /coverage/phystokens.py
parent3416380e8e46d839c3111d9f82a5f7d93a218821 (diff)
parent9e4908f37c370250ebc7836e744a59170720a9e3 (diff)
downloadpython-coveragepy-git-3b6e8394c3e50cb733caafb275d2ae85c0397565.tar.gz
Merged default onto config.
--HG-- branch : config
Diffstat (limited to 'coverage/phystokens.py')
-rw-r--r--coverage/phystokens.py12
1 files changed, 6 insertions, 6 deletions
diff --git a/coverage/phystokens.py b/coverage/phystokens.py
index 131b362a..5824b9b9 100644
--- a/coverage/phystokens.py
+++ b/coverage/phystokens.py
@@ -5,13 +5,13 @@ from coverage.backward import StringIO # pylint: disable-msg=W0622
def phys_tokens(toks):
"""Return all physical tokens, even line continuations.
-
+
tokenize.generate_tokens() doesn't return a token for the backslash that
continues lines. This wrapper provides those tokens so that we can
re-create a faithful representation of the original source.
-
+
Returns the same values as generate_tokens()
-
+
"""
last_line = None
last_lineno = -1
@@ -61,13 +61,13 @@ def phys_tokens(toks):
def source_token_lines(source):
"""Generate a series of lines, one for each line in `source`.
-
+
Each line is a list of pairs, each pair is a token::
-
+
[('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ]
Each pair has a token class, and the token text.
-
+
If you concatenate all the token texts, and then join them with newlines,
you should have your original `source` back, with two differences:
trailing whitespace is not preserved, and a final line with no newline