diff options
| author | Ezio Melotti <ezio.melotti@gmail.com> | 2012-11-03 17:30:51 +0200 |
|---|---|---|
| committer | Ezio Melotti <ezio.melotti@gmail.com> | 2012-11-03 17:30:51 +0200 |
| commit | 7d24b1698a5590df066f829fb4eeef93186d7150 (patch) | |
| tree | c216580e6d3aa39581bfd7cf20b5b9d029bf4825 /Lib/tokenize.py | |
| parent | d7bae5e85ac9ae3a6fbd5c9de5b21be0df7a848d (diff) | |
| download | cpython-git-7d24b1698a5590df066f829fb4eeef93186d7150.tar.gz | |
#16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder.
Diffstat (limited to 'Lib/tokenize.py')
| -rw-r--r-- | Lib/tokenize.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 1cba6e5d9e..ca7b07493c 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -95,7 +95,7 @@ ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) tokenprog, pseudoprog, single3prog, double3prog = map( @@ -362,6 +362,8 @@ def generate_tokens(readline): if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue token, initial = line[start:end], line[start] if initial in numchars or \ |