diff options
author | Pablo Galindo <Pablogsal@gmail.com> | 2020-04-22 22:29:27 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-04-22 22:29:27 (GMT) |
commit | c5fc15685202cda73f7c3f5c6f299b0945f58508 (patch) | |
tree | 7624ae45b95f2812e801c5879bdbdcb796f570a6 /Python | |
parent | a81849b0315277bb3937271174aaaa5059c0b445 (diff) | |
download | cpython-c5fc15685202cda73f7c3f5c6f299b0945f58508.zip cpython-c5fc15685202cda73f7c3f5c6f299b0945f58508.tar.gz cpython-c5fc15685202cda73f7c3f5c6f299b0945f58508.tar.bz2 |
bpo-40334: PEP 617 implementation: New PEG parser for CPython (GH-19503)
Co-authored-by: Guido van Rossum <guido@python.org>
Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
Diffstat (limited to 'Python')
-rw-r--r-- | Python/ast_opt.c | 3 | ||||
-rw-r--r-- | Python/bltinmodule.c | 5 | ||||
-rw-r--r-- | Python/compile.c | 67 | ||||
-rw-r--r-- | Python/importlib.h | 89 | ||||
-rw-r--r-- | Python/importlib_external.h | 7 | ||||
-rw-r--r-- | Python/initconfig.c | 10 | ||||
-rw-r--r-- | Python/pythonrun.c | 54 | ||||
-rw-r--r-- | Python/sysmodule.c | 4 |
8 files changed, 182 insertions, 57 deletions
diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 1766321..ff786d6 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -563,7 +563,8 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL(fold_tuple, expr_ty, node_); break; case Name_kind: - if (_PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) { + if (node_->v.Name.ctx == Load && + _PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) { return make_const(node_, PyBool_FromLong(!state->optimize), ctx_); } break; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 22ee596..1888335 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -816,7 +816,12 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (str == NULL) goto error; + int current_use_peg = PyInterpreterState_Get()->config.use_peg; + if (flags & PyCF_TYPE_COMMENTS || feature_version >= 0) { + PyInterpreterState_Get()->config.use_peg = 0; + } result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); + PyInterpreterState_Get()->config.use_peg = current_use_peg; Py_XDECREF(source_copy); goto finally; diff --git a/Python/compile.c b/Python/compile.c index 54e6516..3c21fba 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2153,6 +2153,55 @@ compiler_default_arguments(struct compiler *c, arguments_ty args) } static int +forbidden_name(struct compiler *c, identifier name, expr_context_ty ctx) +{ + + if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) { + compiler_error(c, "cannot assign to __debug__"); + return 1; + } + return 0; +} + +static int +compiler_check_debug_one_arg(struct compiler *c, arg_ty arg) +{ + if (arg != NULL) { + if (forbidden_name(c, arg->arg, Store)) + return 0; + } + return 1; +} + +static int +compiler_check_debug_args_seq(struct compiler *c, asdl_seq *args) +{ + if (args != NULL) { + for (int i = 0, n = asdl_seq_LEN(args); i < n; i++) { + if (!compiler_check_debug_one_arg(c, asdl_seq_GET(args, i))) + return 0; + } + } + return 1; +} + +static int +compiler_check_debug_args(struct compiler *c, arguments_ty args) +{ + if (!compiler_check_debug_args_seq(c, args->posonlyargs)) + return 0; + if (!compiler_check_debug_args_seq(c, args->args)) + return 0; + if (!compiler_check_debug_one_arg(c, args->vararg)) + return 0; + if (!compiler_check_debug_args_seq(c, args->kwonlyargs)) + return 0; + if (!compiler_check_debug_one_arg(c, args->kwarg)) + return 0; + return 1; +} + +static int compiler_function(struct compiler *c, stmt_ty s, int is_async) { PyCodeObject *co; @@ -2189,6 +2238,9 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) scope_type = COMPILER_SCOPE_FUNCTION; } + if (!compiler_check_debug_args(c, args)) + return 0; + if (!compiler_decorators(c, decos)) return 0; @@ -2596,6 +2648,9 @@ compiler_lambda(struct compiler *c, expr_ty e) arguments_ty args = e->v.Lambda.args; assert(e->kind == Lambda_kind); + if (!compiler_check_debug_args(c, args)) + return 0; + if (!name) { name = PyUnicode_InternFromString("<lambda>"); if (!name) @@ -3505,6 +3560,9 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) !_PyUnicode_EqualToASCIIString(name, "True") && !_PyUnicode_EqualToASCIIString(name, "False")); + if (forbidden_name(c, name, ctx)) + return 0; + mangled = _Py_Mangle(c->u->u_private, name); if (!mangled) return 0; @@ -4056,6 +4114,9 @@ validate_keywords(struct compiler *c, asdl_seq *keywords) if (key->arg == NULL) { continue; } + if (forbidden_name(c, key->arg, Store)) { + return -1; + } for (Py_ssize_t j = i + 1; j < nkeywords; j++) { keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j)); if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) { @@ -5013,6 +5074,8 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP_NAME(c, LOAD_ATTR, e->v.Attribute.attr, names); break; case Store: + if (forbidden_name(c, e->v.Attribute.attr, e->v.Attribute.ctx)) + return 0; ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names); break; case Del: @@ -5183,6 +5246,8 @@ compiler_annassign(struct compiler *c, stmt_ty s) } switch (targ->kind) { case Name_kind: + if (forbidden_name(c, targ->v.Name.id, Store)) + return 0; /* If we have a simple name in a module or class, store annotation. */ if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || @@ -5200,6 +5265,8 @@ compiler_annassign(struct compiler *c, stmt_ty s) } break; case Attribute_kind: + if (forbidden_name(c, targ->v.Attribute.attr, Store)) + return 0; if (!s->v.AnnAssign.value && !check_ann_expr(c, targ->v.Attribute.value)) { return 0; diff --git a/Python/importlib.h b/Python/importlib.h index 4bd8b62..59e0272 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1594,50 +1594,51 @@ const unsigned char _Py_M__importlib_bootstrap[] = { 0,218,1,120,90,5,119,104,101,114,101,90,9,102,114,111, 109,95,110,97,109,101,90,3,101,120,99,114,10,0,0,0, 114,10,0,0,0,114,11,0,0,0,114,215,0,0,0,9, - 4,0,0,115,44,0,0,0,0,10,8,1,10,1,4,1, - 12,2,4,1,28,2,8,1,14,1,10,1,2,255,8,2, - 10,1,14,1,2,1,14,1,14,4,10,1,16,255,2,2, - 12,1,26,1,114,215,0,0,0,99,1,0,0,0,0,0, - 0,0,0,0,0,0,3,0,0,0,6,0,0,0,67,0, - 0,0,115,146,0,0,0,124,0,160,0,100,1,161,1,125, - 1,124,0,160,0,100,2,161,1,125,2,124,1,100,3,117, - 1,114,82,124,2,100,3,117,1,114,78,124,1,124,2,106, - 1,107,3,114,78,116,2,106,3,100,4,124,1,155,2,100, - 5,124,2,106,1,155,2,100,6,157,5,116,4,100,7,100, - 8,141,3,1,0,124,1,83,0,124,2,100,3,117,1,114, - 96,124,2,106,1,83,0,116,2,106,3,100,9,116,4,100, - 7,100,8,141,3,1,0,124,0,100,10,25,0,125,1,100, - 11,124,0,118,1,114,142,124,1,160,5,100,12,161,1,100, - 13,25,0,125,1,124,1,83,0,41,14,122,167,67,97,108, - 99,117,108,97,116,101,32,119,104,97,116,32,95,95,112,97, - 99,107,97,103,101,95,95,32,115,104,111,117,108,100,32,98, - 101,46,10,10,32,32,32,32,95,95,112,97,99,107,97,103, - 101,95,95,32,105,115,32,110,111,116,32,103,117,97,114,97, - 110,116,101,101,100,32,116,111,32,98,101,32,100,101,102,105, - 110,101,100,32,111,114,32,99,111,117,108,100,32,98,101,32, - 115,101,116,32,116,111,32,78,111,110,101,10,32,32,32,32, - 116,111,32,114,101,112,114,101,115,101,110,116,32,116,104,97, - 116,32,105,116,115,32,112,114,111,112,101,114,32,118,97,108, - 117,101,32,105,115,32,117,110,107,110,111,119,110,46,10,10, - 32,32,32,32,114,146,0,0,0,114,106,0,0,0,78,122, - 32,95,95,112,97,99,107,97,103,101,95,95,32,33,61,32, - 95,95,115,112,101,99,95,95,46,112,97,114,101,110,116,32, - 40,122,4,32,33,61,32,250,1,41,233,3,0,0,0,41, - 1,90,10,115,116,97,99,107,108,101,118,101,108,122,89,99, - 97,110,39,116,32,114,101,115,111,108,118,101,32,112,97,99, - 107,97,103,101,32,102,114,111,109,32,95,95,115,112,101,99, - 95,95,32,111,114,32,95,95,112,97,99,107,97,103,101,95, - 95,44,32,102,97,108,108,105,110,103,32,98,97,99,107,32, - 111,110,32,95,95,110,97,109,101,95,95,32,97,110,100,32, - 95,95,112,97,116,104,95,95,114,1,0,0,0,114,142,0, - 0,0,114,129,0,0,0,114,22,0,0,0,41,6,114,35, - 0,0,0,114,131,0,0,0,114,193,0,0,0,114,194,0, - 0,0,114,195,0,0,0,114,130,0,0,0,41,3,218,7, - 103,108,111,98,97,108,115,114,187,0,0,0,114,96,0,0, - 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,17,95,99,97,108,99,95,95,95,112,97,99,107,97,103, - 101,95,95,46,4,0,0,115,34,0,0,0,0,7,10,1, - 10,1,8,1,18,1,22,2,4,254,6,3,4,1,8,1, + 4,0,0,115,52,0,0,0,0,10,8,1,10,1,4,1, + 12,2,4,1,4,1,2,255,4,1,8,255,10,2,8,1, + 14,1,10,1,2,255,8,2,10,1,14,1,2,1,14,1, + 14,4,10,1,16,255,2,2,12,1,26,1,114,215,0,0, + 0,99,1,0,0,0,0,0,0,0,0,0,0,0,3,0, + 0,0,6,0,0,0,67,0,0,0,115,146,0,0,0,124, + 0,160,0,100,1,161,1,125,1,124,0,160,0,100,2,161, + 1,125,2,124,1,100,3,117,1,114,82,124,2,100,3,117, + 1,114,78,124,1,124,2,106,1,107,3,114,78,116,2,106, + 3,100,4,124,1,155,2,100,5,124,2,106,1,155,2,100, + 6,157,5,116,4,100,7,100,8,141,3,1,0,124,1,83, + 0,124,2,100,3,117,1,114,96,124,2,106,1,83,0,116, + 2,106,3,100,9,116,4,100,7,100,8,141,3,1,0,124, + 0,100,10,25,0,125,1,100,11,124,0,118,1,114,142,124, + 1,160,5,100,12,161,1,100,13,25,0,125,1,124,1,83, + 0,41,14,122,167,67,97,108,99,117,108,97,116,101,32,119, + 104,97,116,32,95,95,112,97,99,107,97,103,101,95,95,32, + 115,104,111,117,108,100,32,98,101,46,10,10,32,32,32,32, + 95,95,112,97,99,107,97,103,101,95,95,32,105,115,32,110, + 111,116,32,103,117,97,114,97,110,116,101,101,100,32,116,111, + 32,98,101,32,100,101,102,105,110,101,100,32,111,114,32,99, + 111,117,108,100,32,98,101,32,115,101,116,32,116,111,32,78, + 111,110,101,10,32,32,32,32,116,111,32,114,101,112,114,101, + 115,101,110,116,32,116,104,97,116,32,105,116,115,32,112,114, + 111,112,101,114,32,118,97,108,117,101,32,105,115,32,117,110, + 107,110,111,119,110,46,10,10,32,32,32,32,114,146,0,0, + 0,114,106,0,0,0,78,122,32,95,95,112,97,99,107,97, + 103,101,95,95,32,33,61,32,95,95,115,112,101,99,95,95, + 46,112,97,114,101,110,116,32,40,122,4,32,33,61,32,250, + 1,41,233,3,0,0,0,41,1,90,10,115,116,97,99,107, + 108,101,118,101,108,122,89,99,97,110,39,116,32,114,101,115, + 111,108,118,101,32,112,97,99,107,97,103,101,32,102,114,111, + 109,32,95,95,115,112,101,99,95,95,32,111,114,32,95,95, + 112,97,99,107,97,103,101,95,95,44,32,102,97,108,108,105, + 110,103,32,98,97,99,107,32,111,110,32,95,95,110,97,109, + 101,95,95,32,97,110,100,32,95,95,112,97,116,104,95,95, + 114,1,0,0,0,114,142,0,0,0,114,129,0,0,0,114, + 22,0,0,0,41,6,114,35,0,0,0,114,131,0,0,0, + 114,193,0,0,0,114,194,0,0,0,114,195,0,0,0,114, + 130,0,0,0,41,3,218,7,103,108,111,98,97,108,115,114, + 187,0,0,0,114,96,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,218,17,95,99,97,108,99,95, + 95,95,112,97,99,107,97,103,101,95,95,46,4,0,0,115, + 42,0,0,0,0,7,10,1,10,1,8,1,18,1,6,1, + 2,255,4,1,4,255,6,2,4,254,6,3,4,1,8,1, 6,2,6,2,4,254,6,3,8,1,8,1,14,1,114,221, 0,0,0,114,10,0,0,0,99,5,0,0,0,0,0,0, 0,0,0,0,0,9,0,0,0,5,0,0,0,67,0,0, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 9618f9f..dd23742 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -481,10 +481,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = { 108,101,118,101,108,90,13,98,97,115,101,95,102,105,108,101, 110,97,109,101,114,5,0,0,0,114,5,0,0,0,114,8, 0,0,0,218,17,115,111,117,114,99,101,95,102,114,111,109, - 95,99,97,99,104,101,116,1,0,0,115,52,0,0,0,0, + 95,99,97,99,104,101,116,1,0,0,115,68,0,0,0,0, 9,12,1,8,1,10,1,12,1,4,1,10,1,12,1,14, - 1,16,1,4,1,4,1,12,1,8,1,18,2,10,1,8, - 1,16,1,10,1,16,1,10,1,14,2,16,1,10,1,16, + 1,16,1,4,1,4,1,12,1,8,1,2,1,2,255,4, + 1,2,255,8,2,10,1,8,1,16,1,10,1,16,1,10, + 1,4,1,2,255,8,2,16,1,10,1,4,1,2,255,10, 2,14,1,114,102,0,0,0,99,1,0,0,0,0,0,0, 0,0,0,0,0,5,0,0,0,9,0,0,0,67,0,0, 0,115,124,0,0,0,116,0,124,0,131,1,100,1,107,2, diff --git a/Python/initconfig.c b/Python/initconfig.c index c313d91..7662d61 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -68,6 +68,7 @@ static const char usage_3[] = "\ -X opt : set implementation-specific option. The following options are available:\n\ \n\ -X faulthandler: enable faulthandler\n\ + -X oldparser: enable the traditional LL(1) parser; also PYTHONOLDPARSER\n\ -X showrefcount: output the total reference count and number of used\n\ memory blocks when the program finishes or after each statement in the\n\ interactive interpreter. This only works on debug builds\n\ @@ -634,6 +635,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = -1; #endif + config->use_peg = 1; } @@ -791,6 +793,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_ATTR(isolated); COPY_ATTR(use_environment); COPY_ATTR(dev_mode); + COPY_ATTR(use_peg); COPY_ATTR(install_signal_handlers); COPY_ATTR(use_hash_seed); COPY_ATTR(hash_seed); @@ -894,6 +897,7 @@ config_as_dict(const PyConfig *config) SET_ITEM_INT(isolated); SET_ITEM_INT(use_environment); SET_ITEM_INT(dev_mode); + SET_ITEM_INT(use_peg); SET_ITEM_INT(install_signal_handlers); SET_ITEM_INT(use_hash_seed); SET_ITEM_UINT(hash_seed); @@ -1428,6 +1432,11 @@ config_read_complex_options(PyConfig *config) config->import_time = 1; } + if (config_get_env(config, "PYTHONOLDPARSER") + || config_get_xoption(config, L"oldparser")) { + config->use_peg = 0; + } + PyStatus status; if (config->tracemalloc < 0) { status = config_init_tracemalloc(config); @@ -2507,6 +2516,7 @@ PyConfig_Read(PyConfig *config) assert(config->isolated >= 0); assert(config->use_environment >= 0); assert(config->dev_mode >= 0); + assert(config->use_peg >= 0); assert(config->install_signal_handlers >= 0); assert(config->use_hash_seed >= 0); assert(config->faulthandler >= 0); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 0a25ebc..6199f0c 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -29,6 +29,8 @@ #include "ast.h" // PyAST_FromNodeObject() #include "marshal.h" // PyMarshal_ReadLongFromFile() +#include <pegen_interface.h> // PyPegen_ASTFrom* + #ifdef MS_WINDOWS # include "malloc.h" // alloca() #endif @@ -183,6 +185,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, PyArena *arena; const char *ps1 = "", *ps2 = "", *enc = NULL; int errcode = 0; + int use_peg = _PyInterpreterState_GET()->config.use_peg; _Py_IDENTIFIER(encoding); _Py_IDENTIFIER(__main__); @@ -235,9 +238,17 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, Py_XDECREF(oenc); return -1; } - mod = PyParser_ASTFromFileObject(fp, filename, enc, - Py_single_input, ps1, ps2, - flags, &errcode, arena); + + if (use_peg) { + mod = PyPegen_ASTFromFileObject(fp, filename, Py_single_input, + enc, ps1, ps2, &errcode, arena); + } + else { + mod = PyParser_ASTFromFileObject(fp, filename, enc, + Py_single_input, ps1, ps2, + flags, &errcode, arena); + } + Py_XDECREF(v); Py_XDECREF(w); Py_XDECREF(oenc); @@ -1019,6 +1030,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, mod_ty mod; PyArena *arena; PyObject *filename; + int use_peg = _PyInterpreterState_GET()->config.use_peg; filename = _PyUnicode_FromId(&PyId_string); /* borrowed */ if (filename == NULL) @@ -1028,7 +1040,13 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, if (arena == NULL) return NULL; - mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + if (use_peg) { + mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena); + } + else { + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + } + if (mod != NULL) ret = run_mod(mod, filename, globals, locals, flags, arena); PyArena_Free(arena); @@ -1043,6 +1061,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa mod_ty mod; PyArena *arena = NULL; PyObject *filename; + int use_peg = _PyInterpreterState_GET()->config.use_peg; filename = PyUnicode_DecodeFSDefault(filename_str); if (filename == NULL) @@ -1052,8 +1071,15 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa if (arena == NULL) goto exit; - mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0, - flags, NULL, arena); + if (use_peg) { + mod = PyPegen_ASTFromFileObject(fp, filename, start, NULL, NULL, NULL, + NULL, arena); + } + else { + mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0, + flags, NULL, arena); + } + if (closeit) fclose(fp); if (mod == NULL) { @@ -1196,11 +1222,17 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, { PyCodeObject *co; mod_ty mod; + int use_peg = _PyInterpreterState_GET()->config.use_peg; PyArena *arena = PyArena_New(); if (arena == NULL) return NULL; - mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + if (use_peg) { + mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena); + } + else { + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + } if (mod == NULL) { PyArena_Free(arena); return NULL; @@ -1297,13 +1329,19 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, Py { struct symtable *st; mod_ty mod; + int use_peg = _PyInterpreterState_GET()->config.use_peg; PyArena *arena; arena = PyArena_New(); if (arena == NULL) return NULL; - mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + if (use_peg) { + mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena); + } + else { + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + } if (mod == NULL) { PyArena_Free(arena); return NULL; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 92ea5e7..cf3ddff 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2427,6 +2427,7 @@ static PyStructSequence_Field flags_fields[] = { {"inspect", "-i"}, {"interactive", "-i"}, {"optimize", "-O or -OO"}, + {"use_peg", "-p old or -p new"}, {"dont_write_bytecode", "-B"}, {"no_user_site", "-s"}, {"no_site", "-S"}, @@ -2447,7 +2448,7 @@ static PyStructSequence_Desc flags_desc = { "sys.flags", /* name */ flags__doc__, /* doc */ flags_fields, /* fields */ - 15 + 16 }; static PyObject* @@ -2470,6 +2471,7 @@ make_flags(PyThreadState *tstate) SetFlag(config->inspect); SetFlag(config->interactive); SetFlag(config->optimization_level); + SetFlag(config->use_peg); SetFlag(!config->write_bytecode); SetFlag(!config->user_site_directory); SetFlag(!config->site_import); |