From 495da292255b92dd73758fdd0e4c7d27d82b1e57 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 7 Mar 2019 12:38:08 -0800 Subject: bpo-35975: Support parsing earlier minor versions of Python 3 (GH-12086) This adds a `feature_version` flag to `ast.parse()` (documented) and `compile()` (hidden) that allow tweaking the parser to support older versions of the grammar. In particular if `feature_version` is 5 or 6, the hacks for the `async` and `await` keyword from PEP 492 are reinstated. (For 7 or higher, these are unconditionally treated as keywords, but they are still special tokens rather than `NAME` tokens that the parser driver recognizes.) https://bugs.python.org/issue35975 --- Python/Python-ast.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'Python/Python-ast.c') diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 92ec157571..0411f9f07f 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -8899,6 +8899,11 @@ PyObject* PyAST_mod2obj(mod_ty t) /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) +{ + return PyAST_obj2mod_ex(ast, arena, mode, PY_MINOR_VERSION); +} + +mod_ty PyAST_obj2mod_ex(PyObject* ast, PyArena* arena, int mode, int feature_version) { mod_ty res; PyObject *req_type[3]; -- cgit v1.2.1 From 10f8ce66884cd7fee2372b8dae08ca8132091574 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 13 Mar 2019 13:00:46 -0700 Subject: bpo-36280: Add Constant.kind field (GH-12295) The value is a string for string and byte literals, None otherwise. It is 'u' for u"..." literals, 'b' for b"..." literals, '' for "..." literals. The 'r' (raw) prefix is ignored. Does not apply to f-strings. This appears sufficient to make mypy capable of using the stdlib ast module instead of typed_ast (assuming a mypy patch I'm working on). WIP: I need to make the tests pass. @ilevkivskyi @serhiy-storchaka https://bugs.python.org/issue36280 --- Python/Python-ast.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) (limited to 'Python/Python-ast.c') diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 0411f9f07f..d0416eb639 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -324,8 +324,10 @@ static char *JoinedStr_fields[]={ "values", }; static PyTypeObject *Constant_type; +_Py_IDENTIFIER(kind); static char *Constant_fields[]={ "value", + "kind", }; static PyTypeObject *Attribute_type; _Py_IDENTIFIER(attr); @@ -950,7 +952,7 @@ static int init_types(void) if (!FormattedValue_type) return 0; JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1); if (!JoinedStr_type) return 0; - Constant_type = make_type("Constant", expr_type, Constant_fields, 1); + Constant_type = make_type("Constant", expr_type, Constant_fields, 2); if (!Constant_type) return 0; Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3); if (!Attribute_type) return 0; @@ -2287,8 +2289,8 @@ JoinedStr(asdl_seq * values, int lineno, int col_offset, int end_lineno, int } expr_ty -Constant(constant value, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena) +Constant(constant value, string kind, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -2301,6 +2303,7 @@ Constant(constant value, int lineno, int col_offset, int end_lineno, int return NULL; p->kind = Constant_kind; p->v.Constant.value = value; + p->v.Constant.kind = kind; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -3507,6 +3510,11 @@ ast2obj_expr(void* _o) if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.Constant.kind); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_kind, value) == -1) + goto failed; + Py_DECREF(value); break; case Attribute_kind: result = PyType_GenericNew(Attribute_type, NULL, NULL); @@ -7224,6 +7232,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } if (isinstance) { constant value; + string kind; if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) { return 1; @@ -7238,8 +7247,21 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Constant(value, lineno, col_offset, end_lineno, end_col_offset, - arena); + if (_PyObject_LookupAttrId(obj, &PyId_kind, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + kind = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &kind, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = Constant(value, kind, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } -- cgit v1.2.1