bpo-35975: Support parsing earlier minor versions of Python 3 (GH-12086)

This adds a `feature_version` flag to `ast.parse()` (documented) and `compile()` (hidden) that allow tweaking the parser to support older versions of the grammar. In particular if `feature_version` is 5 or 6, the hacks for the `async` and `await` keyword from PEP 492 are reinstated. (For 7 or higher, these are unconditionally treated as keywords, but they are still special tokens rather than `NAME` tokens that the parser driver recognizes.)



https://bugs.python.org/issue35975

1 files changed, 9 insertions, 2 deletions

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index a19b8b8ddc..7a2b259cbd 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -745,6 +745,7 @@ compile as builtin_compile
     flags: int = 0
     dont_inherit: bool(accept={int}) = False
     optimize: int = -1
+    feature_version: int = -1
 
 Compile source into a code object that can be executed by exec() or eval().
 
@@ -763,8 +764,8 @@ in addition to any features explicitly specified.
 static PyObject *
 builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
                      const char *mode, int flags, int dont_inherit,
-                     int optimize)
-/*[clinic end generated code: output=1fa176e33452bb63 input=0ff726f595eb9fcd]*/
+                     int optimize, int feature_version)
+/*[clinic end generated code: output=b0c09c84f116d3d7 input=5fcc30651a6acaa9]*/
 {
     PyObject *source_copy;
     const char *str;
@@ -775,6 +776,10 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
     PyObject *result;
 
     cf.cf_flags = flags | PyCF_SOURCE_IS_UTF8;
+    cf.cf_feature_version = PY_MINOR_VERSION;
+    if (feature_version >= 0 && (flags & PyCF_ONLY_AST)) {
+        cf.cf_feature_version = feature_version;
+    }
 
     if (flags &
         ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST | PyCF_TYPE_COMMENTS))
@@ -981,6 +986,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals,
     }
 
     cf.cf_flags = PyCF_SOURCE_IS_UTF8;
+    cf.cf_feature_version = PY_MINOR_VERSION;
     str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy);
     if (str == NULL)
         return NULL;
@@ -1068,6 +1074,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals,
         const char *str;
         PyCompilerFlags cf;
         cf.cf_flags = PyCF_SOURCE_IS_UTF8;
+        cf.cf_feature_version = PY_MINOR_VERSION;
         str = source_as_string(source, "exec",
                                        "string, bytes or code", &cf,
                                        &source_copy);