Treat async/await as anonymous token and deal with them in tokenize.py

.. as cpython's parser does

2 files changed, 14 insertions, 2 deletions

diff --git a/sphinx/pycode/pgen2/grammar.py b/sphinx/pycode/pgen2/grammar.py
index 33643622b..42e6d72ee 100644
--- a/sphinx/pycode/pgen2/grammar.py
+++ b/sphinx/pycode/pgen2/grammar.py
@@ -165,8 +165,6 @@ opmap_raw = """
 //= DOUBLESLASHEQUAL
 -> RARROW
 ... ELLIPSIS
-async ASYNC
-await AWAIT
 """
 
 opmap = {}
diff --git a/sphinx/pycode/pgen2/tokenize.py b/sphinx/pycode/pgen2/tokenize.py
index d62535050..c7013bf91 100644
--- a/sphinx/pycode/pgen2/tokenize.py
+++ b/sphinx/pycode/pgen2/tokenize.py
@@ -360,6 +360,16 @@ def generate_tokens(readline):
                 spos, epos, pos = (lnum, start), (lnum, end), end
                 token, initial = line[start:end], line[start]
 
+                if end < max:
+                    next_pseudomatch = pseudoprog.match(line, end)
+                    if next_pseudomatch:
+                        n_start, n_end = next_pseudomatch.span(1)
+                        n_token  = line[n_start:n_end]
+                    else:
+                        n_token = None
+                else:
+                    n_token = None
+
                 if initial in numchars or (
                    initial == '.' and token not in ('.', '...')
                    ):                                      # ordinary number
@@ -396,6 +406,10 @@ def generate_tokens(readline):
                         break
                     else:                                  # ordinary string
                         yield (STRING, token, spos, epos, line)
+                elif token == 'await' and n_token:
+                    yield (AWAIT, token, spos, epos, line)
+                elif token == 'async' and n_token in ('def', 'for', 'with'):
+                    yield (ASYNC, token, spos, epos, line)
                 elif initial in namechars:                 # ordinary name
                     yield (NAME, token, spos, epos, line)
                 elif token in ('...',):                    # ordinary name