diff options
| author | Guido van Rossum <guido@python.org> | 2019-02-27 15:14:06 -0800 |
|---|---|---|
| committer | Guido van Rossum <guido@python.org> | 2019-03-05 11:36:11 -0800 |
| commit | 713d05089de6554928306378ac95a506472a4a51 (patch) | |
| tree | 74f619fec87218063a92e52dcd657ce6e6e74405 | |
| parent | 8caf275763a2c232d81ba0ac1900955215c99bd3 (diff) | |
| download | cpython-git-713d05089de6554928306378ac95a506472a4a51.tar.gz | |
Complete feature_version implementation (except loose ends)
This adds:
- Add ASYNC/AWAIT tokens back to Grammar and regenerate
- Recognize async/await keywords conditionally if feature_version < 7
- Reject await expressions if feature_version < 5
- Docs for ASYNC/AWAIT tokens and for ast.parse(..., feature_version=N)
| -rw-r--r-- | Doc/library/ast.rst | 9 | ||||
| -rw-r--r-- | Doc/library/token-list.inc | 4 | ||||
| -rw-r--r-- | Doc/library/token.rst | 3 | ||||
| -rw-r--r-- | Grammar/Grammar | 8 | ||||
| -rw-r--r-- | Grammar/Tokens | 2 | ||||
| -rw-r--r-- | Include/parsetok.h | 1 | ||||
| -rw-r--r-- | Include/token.h | 10 | ||||
| -rw-r--r-- | Lib/token.py | 16 | ||||
| -rw-r--r-- | Parser/parsetok.c | 2 | ||||
| -rw-r--r-- | Parser/token.c | 2 | ||||
| -rw-r--r-- | Parser/tokenizer.c | 79 | ||||
| -rw-r--r-- | Parser/tokenizer.h | 7 | ||||
| -rw-r--r-- | Python/ast.c | 25 | ||||
| -rw-r--r-- | Python/graminit.c | 210 | ||||
| -rw-r--r-- | Python/pythonrun.c | 2 |
15 files changed, 246 insertions, 134 deletions
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 3df7f9ebc7..7422386116 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -126,7 +126,7 @@ The abstract grammar is currently defined as follows: Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='<unknown>', mode='exec', *, type_comments=False) +.. function:: parse(source, filename='<unknown>', mode='exec', *, type_comments=False, feature_version=-1) Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. @@ -145,13 +145,18 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. + Also, setting ``feature_version`` to the minor version of an + earlier Python 3 version will attempt to parse using that version's + grammar. For example, setting ``feature_version=4`` will allow + the use of ``async`` and ``await`` as variable names. + .. warning:: It is possible to crash the Python interpreter with a sufficiently large/complex string due to stack depth limitations in Python's AST compiler. .. versionchanged:: 3.8 - Added ``type_comments=True`` and ``mode='func_type'``. + Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. .. function:: literal_eval(node_or_string) diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc index cb9fcd79ef..877d39a432 100644 --- a/Doc/library/token-list.inc +++ b/Doc/library/token-list.inc @@ -203,6 +203,10 @@ .. data:: OP +.. data:: AWAIT + +.. data:: ASYNC + .. data:: TYPE_IGNORE .. data:: TYPE_COMMENT diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 4936e9aa08..5c641ef46d 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -88,3 +88,6 @@ the :mod:`tokenize` module. .. versionchanged:: 3.8 Added :data:`TYPE_COMMENT`. + Added :data:`AWAIT` and :data:`ASYNC` tokens back (they're needed + to support parsing older Python versions for :func:`ast.parse` with + ``feature_version`` set to 6 or lower). diff --git a/Grammar/Grammar b/Grammar/Grammar index a425978059..eaebdc4340 100644 --- a/Grammar/Grammar +++ b/Grammar/Grammar @@ -18,7 +18,7 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: 'async' funcdef +async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] func_body_suite parameters: '(' [typedargslist] ')' @@ -70,7 +70,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: 'async' (funcdef | with_stmt | for_stmt) +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite] @@ -106,7 +106,7 @@ arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom_expr ['**' factor] -atom_expr: ['await'] atom trailer* +atom_expr: [AWAIT] atom trailer* atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | @@ -144,7 +144,7 @@ argument: ( test [comp_for] | comp_iter: comp_for | comp_if sync_comp_for: 'for' exprlist 'in' or_test [comp_iter] -comp_for: ['async'] sync_comp_for +comp_for: [ASYNC] sync_comp_for comp_if: 'if' test_nocond [comp_iter] # not used in grammar, but may appear in "node" passed from Parser to Compiler diff --git a/Grammar/Tokens b/Grammar/Tokens index 1d45e05ea2..9de2da5d15 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -55,6 +55,8 @@ ELLIPSIS '...' COLONEQUAL ':=' OP +AWAIT +ASYNC TYPE_IGNORE TYPE_COMMENT ERRORTOKEN diff --git a/Include/parsetok.h b/Include/parsetok.h index be49f0005a..935d733e90 100644 --- a/Include/parsetok.h +++ b/Include/parsetok.h @@ -35,6 +35,7 @@ typedef struct { #define PyPARSE_IGNORE_COOKIE 0x0010 #define PyPARSE_BARRY_AS_BDFL 0x0020 #define PyPARSE_TYPE_COMMENTS 0x0040 +#define PyPARSE_ASYNC_HACKS 0x0080 PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int, perrdetail *); diff --git a/Include/token.h b/Include/token.h index ef6bf969b7..e08708baf1 100644 --- a/Include/token.h +++ b/Include/token.h @@ -65,10 +65,12 @@ extern "C" { #define ELLIPSIS 52 #define COLONEQUAL 53 #define OP 54 -#define TYPE_IGNORE 55 -#define TYPE_COMMENT 56 -#define ERRORTOKEN 57 -#define N_TOKENS 61 +#define AWAIT 55 +#define ASYNC 56 +#define TYPE_IGNORE 57 +#define TYPE_COMMENT 58 +#define ERRORTOKEN 59 +#define N_TOKENS 63 #define NT_OFFSET 256 /* Special definitions for cooperation with parser */ diff --git a/Lib/token.py b/Lib/token.py index 9bf80a5950..493bf04265 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -58,14 +58,16 @@ RARROW = 51 ELLIPSIS = 52 COLONEQUAL = 53 OP = 54 -TYPE_IGNORE = 55 -TYPE_COMMENT = 56 +AWAIT = 55 +ASYNC = 56 +TYPE_IGNORE = 57 +TYPE_COMMENT = 58 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 57 -COMMENT = 58 -NL = 59 -ENCODING = 60 -N_TOKENS = 61 +ERRORTOKEN = 59 +COMMENT = 60 +NL = 61 +ENCODING = 62 +N_TOKENS = 63 # Special definitions for cooperation with parser NT_OFFSET = 256 diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 7a6c886519..ba33a9a058 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -101,6 +101,8 @@ PyParser_ParseStringObject(const char *s, PyObject *filename, Py_INCREF(err_ret->filename); tok->filename = err_ret->filename; + if (*flags & PyPARSE_ASYNC_HACKS) + tok->async_hacks = 1; return parsetok(tok, g, start, err_ret, flags); } diff --git a/Parser/token.c b/Parser/token.c index 228ecffc84..a489668900 100644 --- a/Parser/token.c +++ b/Parser/token.c @@ -61,6 +61,8 @@ const char * const _PyParser_TokenNames[] = { "ELLIPSIS", "COLONEQUAL", "OP", + "AWAIT", + "ASYNC", "TYPE_IGNORE", "TYPE_COMMENT", "<ERRORTOKEN>", diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 44ec41512c..8f0a9c8100 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -84,6 +84,11 @@ tok_new(void) tok->decoding_buffer = NULL; tok->type_comments = 0; + tok->async_hacks = 0; + tok->async_def = 0; + tok->async_def_indent = 0; + tok->async_def_nl = 0; + return tok; } @@ -1196,6 +1201,31 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) } } + /* Peek ahead at the next character */ + c = tok_nextc(tok); + tok_backup(tok, c); + /* Check if we are closing an async function */ + if (tok->async_def + && !blankline + /* Due to some implementation artifacts of type comments, + * a TYPE_COMMENT at the start of a function won't set an + * indentation level and it will produce a NEWLINE after it. + * To avoid spuriously ending an async function due to this, + * wait until we have some non-newline char in front of us. */ + && c != '\n' + && tok->level == 0 + /* There was a NEWLINE after ASYNC DEF, + so we're past the signature. */ + && tok->async_def_nl + /* Current indentation level is less than where + the async function was defined */ + && tok->async_def_indent >= tok->indent) + { + tok->async_def = 0; + tok->async_def_indent = 0; + tok->async_def_nl = 0; + } + again: tok->start = NULL; /* Skip spaces */ @@ -1310,6 +1340,50 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) *p_start = tok->start; *p_end = tok->cur; + /* async/await parsing block. */ + if (tok->cur - tok->start == 5 && tok->start[0] == 'a') { + /* May be an 'async' or 'await' token. For Python 3.7 or + later we recognize them unconditionally. For Python + 3.5 or 3.6 we recognize 'async' in front of 'def', and + either one inside of 'async def'. (Technically we + shouldn't recognize these at all for 3.4 or earlier, + but there's no *valid* Python 3.4 code that would be + rejected, and async functions will be rejected in a + later phase.) */ + if (!tok->async_hacks || tok->async_def) { + /* Always recognize the keywords. */ + if (memcmp(tok->start, "async", 5) == 0) { + return ASYNC; + } + if (memcmp(tok->start, "await", 5) == 0) { + return AWAIT; + } + } + else if (memcmp(tok->start, "async", 5) == 0) { + /* The current token is 'async'. + Look ahead one token to see if that is 'def'. */ + + struct tok_state ahead_tok; + char *ahead_tok_start = NULL, *ahead_tok_end = NULL; + int ahead_tok_kind; + + memcpy(&ahead_tok, tok, sizeof(ahead_tok)); + ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start, + &ahead_tok_end); + + if (ahead_tok_kind == NAME + && ahead_tok.cur - ahead_tok.start == 3 + && memcmp(ahead_tok.start, "def", 3) == 0) + { + /* The next token is going to be 'def', so instead of + returning a plain NAME token, return ASYNC. */ + tok->async_def_indent = tok->indent; + tok->async_def = 1; + return ASYNC; + } + } + } + return NAME; } @@ -1322,6 +1396,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) *p_start = tok->start; *p_end = tok->cur - 1; /* Leave '\n' out of the string */ tok->cont_line = 0; + if (tok->async_def) { + /* We're somewhere inside an 'async def' function, and + we've encountered a NEWLINE after its signature. */ + tok->async_def_nl = 1; + } return NEWLINE; } diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 22e91f7dca..06c7a14b70 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -64,6 +64,13 @@ struct tok_state { const char* input; /* Tokenizer's newline translated copy of the string. */ int type_comments; /* Whether to look for type comments */ + + /* async/await related fields (still needed depending on feature_version) */ + int async_hacks; /* =1 if async/await aren't always keywords */ + int async_def; /* =1 if tokens are inside an 'async def' body. */ + int async_def_indent; /* Indentation level of the outermost 'async def'. */ + int async_def_nl; /* =1 if the outermost 'async def' had at least one + NEWLINE token after it. */ }; extern struct tok_state *PyTokenizer_FromString(const char *, int); diff --git a/Python/ast.c b/Python/ast.c index f1b3dccfbe..2741926d67 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1790,10 +1790,9 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, static stmt_ty ast_for_async_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) { - /* async_funcdef: 'async' funcdef */ + /* async_funcdef: ASYNC funcdef */ REQ(n, async_funcdef); - REQ(CHILD(n, 0), NAME); - assert(strcmp(STR(CHILD(n, 0)), "async") == 0); + REQ(CHILD(n, 0), ASYNC); REQ(CHILD(n, 1), funcdef); return ast_for_funcdef_impl(c, n, decorator_seq, @@ -1812,10 +1811,9 @@ ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) static stmt_ty ast_for_async_stmt(struct compiling *c, const node *n) { - /* async_stmt: 'async' (funcdef | with_stmt | for_stmt) */ + /* async_stmt: ASYNC (funcdef | with_stmt | for_stmt) */ REQ(n, async_stmt); - REQ(CHILD(n, 0), NAME); - assert(strcmp(STR(CHILD(n, 0)), "async") == 0); + REQ(CHILD(n, 0), ASYNC); switch (TYPE(CHILD(n, 1))) { case funcdef: @@ -1966,8 +1964,7 @@ count_comp_fors(struct compiling *c, const node *n) n_fors++; REQ(n, comp_for); if (NCH(n) == 2) { - REQ(CHILD(n, 0), NAME); - assert(strcmp(STR(CHILD(n, 0)), "async") == 0); + REQ(CHILD(n, 0), ASYNC); n = CHILD(n, 1); } else if (NCH(n) == 1) { @@ -2052,8 +2049,7 @@ ast_for_comprehension(struct compiling *c, const node *n) if (NCH(n) == 2) { is_async = 1; - REQ(CHILD(n, 0), NAME); - assert(strcmp(STR(CHILD(n, 0)), "async") == 0); + REQ(CHILD(n, 0), ASYNC); sync_n = CHILD(n, 1); } else { @@ -2711,7 +2707,12 @@ ast_for_atom_expr(struct compiling *c, const node *n) REQ(n, atom_expr); nch = NCH(n); - if (TYPE(CHILD(n, 0)) == NAME && strcmp(STR(CHILD(n, 0)), "await") == 0) { + if (TYPE(CHILD(n, 0)) == AWAIT) { + if (c->c_feature_version < 5) { + ast_error(c, n, + "Await expressions are only supported in Python 3.5 and greater"); + return NULL; + } start = 1; assert(nch > 1); } @@ -2808,7 +2809,7 @@ ast_for_expr(struct compiling *c, const node *n) term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom_expr ['**' factor] - atom_expr: ['await'] atom trailer* + atom_expr: [AWAIT] atom trailer* yield_expr: 'yield' [yield_arg] */ diff --git a/Python/graminit.c b/Python/graminit.c index 6cb7bbc9f7..24f6f6c72a 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -106,7 +106,7 @@ static state states_5[3] = { {1, arcs_5_2}, }; static arc arcs_6_0[1] = { - {16, 1}, + {38, 1}, }; static arc arcs_6_1[1] = { {56, 2}, @@ -120,7 +120,7 @@ static state states_6[3] = { {1, arcs_6_2}, }; static arc arcs_7_0[1] = { - {21, 1}, + {19, 1}, }; static arc arcs_7_1[1] = { {40, 2}, @@ -583,7 +583,7 @@ static state states_19[2] = { {1, arcs_19_1}, }; static arc arcs_20_0[1] = { - {22, 1}, + {20, 1}, }; static arc arcs_20_1[1] = { {98, 2}, @@ -597,7 +597,7 @@ static state states_20[3] = { {1, arcs_20_2}, }; static arc arcs_21_0[1] = { - {31, 1}, + {29, 1}, }; static arc arcs_21_1[1] = { {0, 1}, @@ -621,7 +621,7 @@ static state states_22[2] = { {1, arcs_22_1}, }; static arc arcs_23_0[1] = { - {18, 1}, + {16, 1}, }; static arc arcs_23_1[1] = { {0, 1}, @@ -631,7 +631,7 @@ static state states_23[2] = { {1, arcs_23_1}, }; static arc arcs_24_0[1] = { - {20, 1}, + {18, 1}, }; static arc arcs_24_1[1] = { {0, 1}, @@ -641,7 +641,7 @@ static state states_24[2] = { {1, arcs_24_1}, }; static arc arcs_25_0[1] = { - {33, 1}, + {31, 1}, }; static arc arcs_25_1[2] = { {80, 2}, @@ -666,14 +666,14 @@ static state states_26[2] = { {1, arcs_26_1}, }; static arc arcs_27_0[1] = { - {32, 1}, + {30, 1}, }; static arc arcs_27_1[2] = { {60, 2}, {0, 1}, }; static arc arcs_27_2[2] = { - {24, 3}, + {22, 3}, {0, 2}, }; static arc arcs_27_3[1] = { @@ -701,7 +701,7 @@ static state states_28[2] = { {1, arcs_28_1}, }; static arc arcs_29_0[1] = { - {27, 1}, + {25, 1}, }; static arc arcs_29_1[1] = { {106, 2}, @@ -715,7 +715,7 @@ static state states_29[3] = { {1, arcs_29_2}, }; static arc arcs_30_0[1] = { - {24, 1}, + {22, 1}, }; static arc arcs_30_1[3] = { {107, 2}, @@ -725,11 +725,11 @@ static arc arcs_30_1[3] = { static arc arcs_30_2[4] = { {107, 2}, {9, 2}, - {27, 4}, + {25, 4}, {49, 3}, }; static arc arcs_30_3[1] = { - {27, 4}, + {25, 4}, }; static arc arcs_30_4[3] = { {5, 5}, @@ -832,7 +832,7 @@ static state states_35[2] = { {2, arcs_35_1}, }; static arc arcs_36_0[1] = { - {25, 1}, + {23, 1}, }; static arc arcs_36_1[1] = { {40, 2}, @@ -847,7 +847,7 @@ static state states_36[3] = { {2, arcs_36_2}, }; static arc arcs_37_0[1] = { - {29, 1}, + {27, 1}, }; static arc arcs_37_1[1] = { {40, 2}, @@ -903,7 +903,7 @@ static state states_39[2] = { {1, arcs_39_1}, }; static arc arcs_40_0[1] = { - {16, 1}, + {38, 1}, }; static arc arcs_40_1[3] = { {113, 2}, @@ -919,7 +919,7 @@ static state states_40[3] = { {1, arcs_40_2}, }; static arc arcs_41_0[1] = { - {26, 1}, + {24, 1}, }; static arc arcs_41_1[1] = { {118, 2}, @@ -955,7 +955,7 @@ static state states_41[8] = { {1, arcs_41_7}, }; static arc arcs_42_0[1] = { - {35, 1}, + {33, 1}, }; static arc arcs_42_1[1] = { {118, 2}, @@ -990,7 +990,7 @@ static state states_42[8] = { {1, arcs_42_7}, }; static arc arcs_43_0[1] = { - {23, 1}, + {21, 1}, }; static arc arcs_43_1[1] = { {98, 2}, @@ -1038,7 +1038,7 @@ static state states_43[11] = { {1, arcs_43_10}, }; static arc arcs_44_0[1] = { - {34, 1}, + {32, 1}, }; static arc arcs_44_1[1] = { {59, 2}, @@ -1097,7 +1097,7 @@ static state states_44[13] = { {2, arcs_44_12}, }; static arc arcs_45_0[1] = { - {36, 1}, + {34, 1}, }; static arc arcs_45_1[1] = { {125, 2}, @@ -1218,7 +1218,7 @@ static arc arcs_50_1[1] = { {0, 1}, }; static arc arcs_50_2[2] = { - {26, 3}, + {24, 3}, {0, 2}, }; static arc arcs_50_3[1] = { @@ -1250,7 +1250,7 @@ static state states_51[2] = { {1, arcs_51_1}, }; static arc arcs_52_0[1] = { - {28, 1}, + {26, 1}, }; static arc arcs_52_1[2] = { {59, 2}, @@ -1273,7 +1273,7 @@ static state states_52[5] = { {1, arcs_52_4}, }; static arc arcs_53_0[1] = { - {28, 1}, + {26, 1}, }; static arc arcs_53_1[2] = { {59, 2}, @@ -1318,7 +1318,7 @@ static state states_55[2] = { {2, arcs_55_1}, }; static arc arcs_56_0[2] = { - {30, 1}, + {28, 1}, {139, 2}, }; static arc arcs_56_1[1] = { @@ -1353,13 +1353,13 @@ static arc arcs_58_0[10] = { {146, 1}, {122, 1}, {147, 2}, - {30, 3}, + {28, 3}, }; static arc arcs_58_1[1] = { {0, 1}, }; static arc arcs_58_2[2] = { - {30, 1}, + {28, 1}, {0, 2}, }; static arc arcs_58_3[1] = { @@ -1460,7 +1460,7 @@ static state states_65[2] = { static arc arcs_66_0[4] = { {7, 1}, {8, 1}, - {39, 1}, + {37, 1}, {162, 2}, }; static arc arcs_66_1[1] = { @@ -1494,7 +1494,7 @@ static state states_67[4] = { {1, arcs_67_3}, }; static arc arcs_68_0[2] = { - {17, 1}, + {39, 1}, {164, 2}, }; static arc arcs_68_1[1] = { @@ -1516,7 +1516,7 @@ static arc arcs_69_0[10] = { {12, 2}, {13, 2}, {14, 3}, - {38, 4}, + {36, 4}, {40, 2}, {41, 2}, {42, 5}, @@ -1788,7 +1788,7 @@ static state states_77[14] = { {1, arcs_77_13}, }; static arc arcs_78_0[1] = { - {19, 1}, + {17, 1}, }; static arc arcs_78_1[1] = { {40, 2}, @@ -1874,7 +1874,7 @@ static state states_81[2] = { {1, arcs_81_1}, }; static arc arcs_82_0[1] = { - {23, 1}, + {21, 1}, }; static arc arcs_82_1[1] = { {98, 2}, @@ -1901,7 +1901,7 @@ static state states_82[6] = { {1, arcs_82_5}, }; static arc arcs_83_0[2] = { - {16, 1}, + {38, 1}, {177, 2}, }; static arc arcs_83_1[1] = { @@ -1916,7 +1916,7 @@ static state states_83[3] = { {1, arcs_83_2}, }; static arc arcs_84_0[1] = { - {26, 1}, + {24, 1}, }; static arc arcs_84_1[1] = { {133, 2}, @@ -1945,7 +1945,7 @@ static state states_85[2] = { {1, arcs_85_1}, }; static arc arcs_86_0[1] = { - {37, 1}, + {35, 1}, }; static arc arcs_86_1[2] = { {179, 2}, @@ -1960,7 +1960,7 @@ static state states_86[3] = { {1, arcs_86_2}, }; static arc arcs_87_0[2] = { - {24, 1}, + {22, 1}, {80, 2}, }; static arc arcs_87_1[1] = { @@ -2115,7 +2115,7 @@ static dfa dfas[92] = { {257, "file_input", 0, 2, states_1, "\344\377\377\377\377\027\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {258, "eval_input", 0, 3, states_2, - "\240\173\002\120\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {259, "decorator", 0, 7, states_3, "\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {260, "decorators", 0, 2, states_4, @@ -2123,9 +2123,9 @@ static dfa dfas[92] = { {261, "decorated", 0, 3, states_5, "\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {262, "async_funcdef", 0, 3, states_6, - "\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {263, "funcdef", 0, 9, states_7, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {264, "parameters", 0, 4, states_8, "\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {265, "typedargslist", 0, 22, states_9, @@ -2139,39 +2139,39 @@ static dfa dfas[92] = { {269, "stmt", 0, 2, states_13, "\340\377\377\377\377\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {270, "simple_stmt", 0, 4, states_14, - "\340\373\126\373\343\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\373\325\376\270\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {271, "small_stmt", 0, 2, states_15, - "\340\373\126\373\343\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\373\325\376\270\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {272, "expr_stmt", 0, 6, states_16, - "\340\173\002\120\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\173\000\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {273, "annassign", 0, 5, states_17, "\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {274, "testlist_star_expr", 0, 3, states_18, - "\340\173\002\120\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\173\000\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {275, "augassign", 0, 2, states_19, "\000\000\000\000\000\000\000\000\000\000\340\377\003\000\000\000\000\000\000\000\000\000\000"}, {276, "del_stmt", 0, 3, states_20, - "\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {277, "pass_stmt", 0, 2, states_21, - "\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {278, "flow_stmt", 0, 2, states_22, - "\000\000\024\000\043\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\005\300\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {279, "break_stmt", 0, 2, states_23, - "\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {280, "continue_stmt", 0, 2, states_24, - "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {281, "return_stmt", 0, 3, states_25, - "\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {282, "yield_stmt", 0, 2, states_26, - "\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {283, "raise_stmt", 0, 5, states_27, - "\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {284, "import_stmt", 0, 2, states_28, - "\000\000\000\011\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\100\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {285, "import_name", 0, 3, states_29, - "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {286, "import_from", 0, 8, states_30, - "\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {287, "import_as_name", 0, 4, states_31, "\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {288, "dotted_as_name", 0, 4, states_32, @@ -2183,117 +2183,117 @@ static dfa dfas[92] = { {291, "dotted_name", 0, 2, states_35, "\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {292, "global_stmt", 0, 3, states_36, - "\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {293, "nonlocal_stmt", 0, 3, states_37, - "\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {294, "assert_stmt", 0, 5, states_38, "\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {295, "compound_stmt", 0, 2, states_39, - "\000\004\251\004\034\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\004\052\001\107\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {296, "async_stmt", 0, 3, states_40, - "\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {297, "if_stmt", 0, 8, states_41, - "\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {298, "while_stmt", 0, 8, states_42, - "\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {299, "for_stmt", 0, 11, states_43, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {300, "try_stmt", 0, 13, states_44, - "\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {301, "with_stmt", 0, 6, states_45, - "\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {302, "with_item", 0, 4, states_46, - "\240\173\002\120\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {303, "except_clause", 0, 5, states_47, "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000"}, {304, "suite", 0, 5, states_48, - "\344\373\126\373\343\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\344\373\325\376\270\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {305, "namedexpr_test", 0, 4, states_49, - "\240\173\002\120\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {306, "test", 0, 6, states_50, - "\240\173\002\120\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {307, "test_nocond", 0, 2, states_51, - "\240\173\002\120\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {308, "lambdef", 0, 5, states_52, - "\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {309, "lambdef_nocond", 0, 5, states_53, - "\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {310, "or_test", 0, 2, states_54, - "\240\173\002\100\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\020\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {311, "and_test", 0, 2, states_55, - "\240\173\002\100\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\020\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {312, "not_test", 0, 3, states_56, - "\240\173\002\100\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\020\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {313, "comparison", 0, 2, states_57, - "\240\173\002\000\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\000\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {314, "comp_op", 0, 4, states_58, - "\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\004\000\340\017\000\000\000\000"}, + "\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\004\000\340\017\000\000\000\000"}, {315, "star_expr", 0, 3, states_59, "\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {316, "expr", 0, 2, states_60, - "\240\173\002\000\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\000\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {317, "xor_expr", 0, 2, states_61, - "\240\173\002\000\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\000\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {318, "and_expr", 0, 2, states_62, - "\240\173\002\000\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\000\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {319, "shift_expr", 0, 2, states_63, - "\240\173\002\000\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\000\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {320, "arith_expr", 0, 2, states_64, - "\240\173\002\000\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\000\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {321, "term", 0, 2, states_65, - "\240\173\002\000\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\000\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {322, "factor", 0, 3, states_66, - "\240\173\002\000\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\000\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {323, "power", 0, 4, states_67, - "\040\172\002\000\100\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\040\172\000\000\220\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {324, "atom_expr", 0, 3, states_68, - "\040\172\002\000\100\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\040\172\000\000\220\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {325, "atom", 0, 9, states_69, - "\040\172\000\000\100\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\040\172\000\000\020\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {326, "testlist_comp", 0, 5, states_70, - "\340\173\002\120\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\173\000\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {327, "trailer", 0, 7, states_71, "\040\100\000\000\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000"}, {328, "subscriptlist", 0, 3, states_72, - "\240\173\002\120\300\007\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\024\260\007\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {329, "subscript", 0, 5, states_73, - "\240\173\002\120\300\007\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\024\260\007\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {330, "sliceop", 0, 3, states_74, "\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {331, "exprlist", 0, 3, states_75, - "\340\173\002\000\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\173\000\000\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {332, "testlist", 0, 3, states_76, - "\240\173\002\120\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\240\173\000\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {333, "dictorsetmaker", 0, 14, states_77, - "\340\173\002\120\300\007\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\173\000\024\260\007\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {334, "classdef", 0, 8, states_78, - "\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {335, "arglist", 0, 3, states_79, - "\340\173\002\120\300\007\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\173\000\024\260\007\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {336, "argument", 0, 4, states_80, - "\340\173\002\120\300\007\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\173\000\024\260\007\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {337, "comp_iter", 0, 2, states_81, - "\000\000\201\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\001\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {338, "sync_comp_for", 0, 6, states_82, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {339, "comp_for", 0, 3, states_83, - "\000\000\201\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {340, "comp_if", 0, 4, states_84, - "\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {341, "encoding_decl", 0, 2, states_85, "\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {342, "yield_expr", 0, 3, states_86, - "\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {343, "yield_arg", 0, 3, states_87, - "\340\173\002\121\300\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\173\100\024\260\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {344, "func_body_suite", 0, 7, states_88, - "\344\373\126\373\343\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\344\373\325\376\270\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {345, "func_type_input", 0, 3, states_89, "\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {346, "func_type", 0, 6, states_90, "\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {347, "typelist", 0, 11, states_91, - "\340\173\002\120\300\007\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\340\173\000\024\260\007\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, }; static label labels[183] = { {0, "EMPTY"}, @@ -2312,8 +2312,6 @@ static label labels[183] = { {1, "True"}, {9, 0}, {1, "assert"}, - {1, "async"}, - {1, "await"}, {1, "break"}, {1, "class"}, {1, "continue"}, @@ -2336,6 +2334,8 @@ static label labels[183] = { {1, "yield"}, {25, 0}, {31, 0}, + {56, 0}, + {55, 0}, {1, 0}, {2, 0}, {3, 0}, @@ -2357,7 +2357,7 @@ static label labels[183] = { {51, 0}, {11, 0}, {306, 0}, - {56, 0}, + {58, 0}, {344, 0}, {265, 0}, {35, 0}, diff --git a/Python/pythonrun.c b/Python/pythonrun.c index ae6778d8a5..b763c99da0 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1200,6 +1200,8 @@ PyParser_ASTFromStringObject(const char *s, PyObject *filename, int start, PyCompilerFlags localflags; perrdetail err; int iflags = PARSER_FLAGS(flags); + if (flags->cf_feature_version < 7) + iflags |= PyPARSE_ASYNC_HACKS; node *n = PyParser_ParseStringObject(s, filename, &_PyParser_Grammar, start, &err, |
