1 files changed, 73 insertions, 3 deletions

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 1540d26d8b..184ffe7a7d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -98,10 +98,13 @@ const char *_PyParser_TokenNames[] = {
     "DOUBLESLASH",
     "DOUBLESLASHEQUAL",
     "AT",
+    "ATEQUAL",
     "RARROW",
     "ELLIPSIS",
     /* This table must match the #defines in token.h! */
     "OP",
+    "AWAIT",
+    "ASYNC",
     "<ERRORTOKEN>",
     "<N_TOKENS>"
 };
@@ -123,6 +126,7 @@ tok_new(void)
     tok->tabsize = TABSIZE;
     tok->indent = 0;
     tok->indstack[0] = 0;
+
     tok->atbol = 1;
     tok->pendin = 0;
     tok->prompt = tok->nextprompt = NULL;
@@ -143,6 +147,11 @@ tok_new(void)
     tok->decoding_readline = NULL;
     tok->decoding_buffer = NULL;
 #endif
+
+    tok->async_def = 0;
+    tok->async_def_indent = 0;
+    tok->async_def_nl = 0;
+
     return tok;
 }
 
@@ -266,6 +275,7 @@ get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *t
                         return 0;
                 }
                 *spec = r;
+                break;
             }
         }
     }
@@ -515,9 +525,8 @@ fp_setreadl(struct tok_state *tok, const char* enc)
     if (stream == NULL)
         goto cleanup;
 
-    Py_XDECREF(tok->decoding_readline);
     readline = _PyObject_GetAttrId(stream, &PyId_readline);
-    tok->decoding_readline = readline;
+    Py_XSETREF(tok->decoding_readline, readline);
     if (pos > 0) {
         if (PyObject_CallObject(readline, NULL) == NULL) {
             readline = NULL;
@@ -1129,7 +1138,7 @@ PyToken_OneChar(int c)
     case '}':           return RBRACE;
     case '^':           return CIRCUMFLEX;
     case '~':           return TILDE;
-    case '@':       return AT;
+    case '@':           return AT;
     default:            return OP;
     }
 }
@@ -1205,6 +1214,11 @@ PyToken_TwoChars(int c1, int c2)
         case '=':               return CIRCUMFLEXEQUAL;
         }
         break;
+    case '@':
+        switch (c2) {
+        case '=':               return ATEQUAL;
+        }
+        break;
     }
     return OP;
 }
@@ -1422,6 +1436,21 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
         }
     }
 
+    if (tok->async_def
+        && !blankline
+        && tok->level == 0
+        /* There was a NEWLINE after ASYNC DEF,
+           so we're past the signature. */
+        && tok->async_def_nl
+        /* Current indentation level is less than where
+           the async function was defined */
+        && tok->async_def_indent >= tok->indent)
+    {
+        tok->async_def = 0;
+        tok->async_def_indent = 0;
+        tok->async_def_nl = 0;
+    }
+
  again:
     tok->start = NULL;
     /* Skip spaces */
@@ -1473,6 +1502,42 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             return ERRORTOKEN;
         *p_start = tok->start;
         *p_end = tok->cur;
+
+        /* async/await parsing block. */
+        if (tok->cur - tok->start == 5) {
+            /* Current token length is 5. */
+            if (tok->async_def) {
+                /* We're inside an 'async def' function. */
+                if (memcmp(tok->start, "async", 5) == 0)
+                    return ASYNC;
+                if (memcmp(tok->start, "await", 5) == 0)
+                    return AWAIT;
+            }
+            else if (memcmp(tok->start, "async", 5) == 0) {
+                /* The current token is 'async'.
+                   Look ahead one token.*/
+
+                struct tok_state ahead_tok;
+                char *ahead_tok_start = NULL, *ahead_tok_end = NULL;
+                int ahead_tok_kind;
+
+                memcpy(&ahead_tok, tok, sizeof(ahead_tok));
+                ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
+                                         &ahead_tok_end);
+
+                if (ahead_tok_kind == NAME
+                    && ahead_tok.cur - ahead_tok.start == 3
+                    && memcmp(ahead_tok.start, "def", 3) == 0)
+                {
+                    /* The next token is going to be 'def', so instead of
+                       returning 'async' NAME token, we return ASYNC. */
+                    tok->async_def_indent = tok->indent;
+                    tok->async_def = 1;
+                    return ASYNC;
+                }
+            }
+        }
+
         return NAME;
     }
 
@@ -1484,6 +1549,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
         *p_start = tok->start;
         *p_end = tok->cur - 1; /* Leave '\n' out of the string */
         tok->cont_line = 0;
+        if (tok->async_def) {
+            /* We're somewhere inside an 'async def' function, and
+               we've encountered a NEWLINE after its signature. */
+            tok->async_def_nl = 1;
+        }
         return NEWLINE;
     }