bpo-35975: Support parsing earlier minor versions of Python 3 (GH-12086)

This adds a `feature_version` flag to `ast.parse()` (documented) and `compile()` (hidden) that allow tweaking the parser to support older versions of the grammar. In particular if `feature_version` is 5 or 6, the hacks for the `async` and `await` keyword from PEP 492 are reinstated. (For 7 or higher, these are unconditionally treated as keywords, but they are still special tokens rather than `NAME` tokens that the parser driver recognizes.)



https://bugs.python.org/issue35975

4 files changed, 9 insertions, 4 deletions

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 52ba127..8265bed 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -703,6 +703,7 @@ type_ignore_ty _Py_TypeIgnore(int lineno, PyArena *arena);
 
 PyObject* PyAST_mod2obj(mod_ty t);
 mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);
+mod_ty PyAST_obj2mod_ex(PyObject* ast, PyArena* arena, int mode, int feature_version);
 int PyAST_Check(PyObject* obj);
 
 #ifdef __cplusplus
diff --git a/Include/compile.h b/Include/compile.h
index d0bbed8..1370867 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -27,6 +27,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
 #ifndef Py_LIMITED_API
 typedef struct {
     int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
+    int cf_feature_version;  /* minor Python version (PyCF_ONLY_AST) */
 } PyCompilerFlags;
 #endif
 
diff --git a/Include/parsetok.h b/Include/parsetok.h
index be49f00..935d733 100644
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -35,6 +35,7 @@ typedef struct {
 #define PyPARSE_IGNORE_COOKIE 0x0010
 #define PyPARSE_BARRY_AS_BDFL 0x0020
 #define PyPARSE_TYPE_COMMENTS 0x0040
+#define PyPARSE_ASYNC_HACKS   0x0080
 
 PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
                                               perrdetail *);
diff --git a/Include/token.h b/Include/token.h
index ef6bf96..e08708b 100644
--- a/Include/token.h
+++ b/Include/token.h
@@ -65,10 +65,12 @@ extern "C" {
 #define ELLIPSIS        52
 #define COLONEQUAL      53
 #define OP              54
-#define TYPE_IGNORE     55
-#define TYPE_COMMENT    56
-#define ERRORTOKEN      57
-#define N_TOKENS        61
+#define AWAIT           55
+#define ASYNC           56
+#define TYPE_IGNORE     57
+#define TYPE_COMMENT    58
+#define ERRORTOKEN      59
+#define N_TOKENS        63
 #define NT_OFFSET       256
 
 /* Special definitions for cooperation with parser */