diff options
Diffstat (limited to 'Parser/tokenizer.c')
| -rw-r--r-- | Parser/tokenizer.c | 85 |
1 files changed, 84 insertions, 1 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5e041ea5b3..d4476aea76 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -98,10 +98,13 @@ const char *_PyParser_TokenNames[] = { "DOUBLESLASH", "DOUBLESLASHEQUAL", "AT", + "ATEQUAL", "RARROW", "ELLIPSIS", /* This table must match the #defines in token.h! */ "OP", + "AWAIT", + "ASYNC", "<ERRORTOKEN>", "<N_TOKENS>" }; @@ -123,6 +126,11 @@ tok_new(void) tok->tabsize = TABSIZE; tok->indent = 0; tok->indstack[0] = 0; + + tok->def = 0; + tok->defstack[0] = 0; + tok->deftypestack[0] = 0; + tok->atbol = 1; tok->pendin = 0; tok->prompt = tok->nextprompt = NULL; @@ -1131,7 +1139,7 @@ PyToken_OneChar(int c) case '}': return RBRACE; case '^': return CIRCUMFLEX; case '~': return TILDE; - case '@': return AT; + case '@': return AT; default: return OP; } } @@ -1207,6 +1215,11 @@ PyToken_TwoChars(int c1, int c2) case '=': return CIRCUMFLEXEQUAL; } break; + case '@': + switch (c2) { + case '=': return ATEQUAL; + } + break; } return OP; } @@ -1329,6 +1342,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) int c; int blankline, nonascii; + int tok_len; + struct tok_state ahead_tok; + char *ahead_tok_start = NULL, *ahead_top_end = NULL; + int ahead_tok_kind; + *p_start = *p_end = NULL; nextline: tok->start = NULL; @@ -1416,6 +1434,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) if (tok->pendin != 0) { if (tok->pendin < 0) { tok->pendin++; + + while (tok->def && tok->defstack[tok->def] >= tok->indent) { + tok->def--; + } + return DEDENT; } else { @@ -1475,6 +1498,66 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) return ERRORTOKEN; *p_start = tok->start; *p_end = tok->cur; + + tok_len = tok->cur - tok->start; + if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) { + if (tok->def && tok->deftypestack[tok->def] == 3) { + tok->deftypestack[tok->def] = 2; + } + else if (tok->defstack[tok->def] < tok->indent) { + /* We advance defs stack only when we see "def" *and* + the indentation level was increased relative to the + previous "def". */ + + if (tok->def + 1 >= MAXINDENT) { + tok->done = E_TOODEEP; + tok->cur = tok->inp; + return ERRORTOKEN; + } + + tok->def++; + tok->defstack[tok->def] = tok->indent; + tok->deftypestack[tok->def] = 1; + } + } + else if (tok_len == 5) { + if (memcmp(tok->start, "async", 5) == 0) { + memcpy(&ahead_tok, tok, sizeof(ahead_tok)); + + ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start, + &ahead_top_end); + + if (ahead_tok_kind == NAME && + ahead_tok.cur - ahead_tok.start == 3 && + memcmp(ahead_tok.start, "def", 3) == 0) { + + if (tok->def + 1 >= MAXINDENT) { + tok->done = E_TOODEEP; + tok->cur = tok->inp; + return ERRORTOKEN; + } + + tok->def++; + tok->defstack[tok->def] = tok->indent; + tok->deftypestack[tok->def] = 3; + + return ASYNC; + } + else if (tok->def && tok->deftypestack[tok->def] == 2 + && tok->defstack[tok->def] < tok->indent) { + + return ASYNC; + } + + } + else if (memcmp(tok->start, "await", 5) == 0 + && tok->def && tok->deftypestack[tok->def] == 2 + && tok->defstack[tok->def] < tok->indent) { + + return AWAIT; + } + } + return NAME; } |