Merged 4.0 to default

1 files changed, 5 insertions, 4 deletions

diff --git a/coverage/phystokens.py b/coverage/phystokens.py
index 5852241..e79ce01 100644
--- a/coverage/phystokens.py
+++ b/coverage/phystokens.py
@@ -1,7 +1,8 @@
 """Better tokenizing for coverage.py."""
 
 import codecs, keyword, re, sys, token, tokenize
-from coverage.backward import StringIO              # pylint: disable=W0622
+from coverage.parser import generate_tokens
+
 
 def phys_tokens(toks):
     """Return all physical tokens, even line continuations.
@@ -18,7 +19,7 @@ def phys_tokens(toks):
     last_ttype = None
     for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks:
         if last_lineno != elineno:
-            if last_line and last_line[-2:] == "\\\n":
+            if last_line and last_line.endswith("\\\n"):
                 # We are at the beginning of a new line, and the last line
                 # ended with a backslash.  We probably have to inject a
                 # backslash token into the stream. Unfortunately, there's more
@@ -74,11 +75,11 @@ def source_token_lines(source):
     is indistinguishable from a final line with a newline.
 
     """
-    ws_tokens = [token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL]
+    ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])
     line = []
     col = 0
     source = source.expandtabs(8).replace('\r\n', '\n')
-    tokgen = tokenize.generate_tokens(StringIO(source).readline)
+    tokgen = generate_tokens(source)
     for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):
         mark_start = True
         for part in re.split('(\n)', ttext):