diff options
| author | Alberto Berti <alberto@metapensiero.it> | 2015-11-09 17:41:39 +0100 |
|---|---|---|
| committer | Alberto Berti <alberto@metapensiero.it> | 2015-11-09 22:15:21 +0100 |
| commit | 1172d1dba26f29c599eac8506d8c0e3fb170b4ee (patch) | |
| tree | ce7f6afd8d0e75e8d262af5a6f2300d285511659 | |
| parent | 8005456562e922403d3118dbcc012f4301730cd7 (diff) | |
| download | sphinx-git-1172d1dba26f29c599eac8506d8c0e3fb170b4ee.tar.gz | |
Treat async/await as anonymous token and deal with them in tokenize.py
.. as cpython's parser does
| -rw-r--r-- | sphinx/pycode/pgen2/grammar.py | 2 | ||||
| -rw-r--r-- | sphinx/pycode/pgen2/tokenize.py | 14 |
2 files changed, 14 insertions, 2 deletions
diff --git a/sphinx/pycode/pgen2/grammar.py b/sphinx/pycode/pgen2/grammar.py index 33643622b..42e6d72ee 100644 --- a/sphinx/pycode/pgen2/grammar.py +++ b/sphinx/pycode/pgen2/grammar.py @@ -165,8 +165,6 @@ opmap_raw = """ //= DOUBLESLASHEQUAL -> RARROW ... ELLIPSIS -async ASYNC -await AWAIT """ opmap = {} diff --git a/sphinx/pycode/pgen2/tokenize.py b/sphinx/pycode/pgen2/tokenize.py index d62535050..c7013bf91 100644 --- a/sphinx/pycode/pgen2/tokenize.py +++ b/sphinx/pycode/pgen2/tokenize.py @@ -360,6 +360,16 @@ def generate_tokens(readline): spos, epos, pos = (lnum, start), (lnum, end), end token, initial = line[start:end], line[start] + if end < max: + next_pseudomatch = pseudoprog.match(line, end) + if next_pseudomatch: + n_start, n_end = next_pseudomatch.span(1) + n_token = line[n_start:n_end] + else: + n_token = None + else: + n_token = None + if initial in numchars or ( initial == '.' and token not in ('.', '...') ): # ordinary number @@ -396,6 +406,10 @@ def generate_tokens(readline): break else: # ordinary string yield (STRING, token, spos, epos, line) + elif token == 'await' and n_token: + yield (AWAIT, token, spos, epos, line) + elif token == 'async' and n_token in ('def', 'for', 'with'): + yield (ASYNC, token, spos, epos, line) elif initial in namechars: # ordinary name yield (NAME, token, spos, epos, line) elif token in ('...',): # ordinary name |