diff options
Diffstat (limited to 'Python/symtable.c')
| -rw-r--r-- | Python/symtable.c | 1488 |
1 files changed, 573 insertions, 915 deletions
diff --git a/Python/symtable.c b/Python/symtable.c index b871358..23aeaaa 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1,51 +1,25 @@ #include "Python.h" -#include "pycore_pystate.h" +#include "Python-ast.h" +#include "code.h" #include "symtable.h" -#undef Yield /* undefine macro conflicting with <winbase.h> */ #include "structmember.h" /* error strings used for warnings */ -#define GLOBAL_PARAM \ -"name '%U' is parameter and global" - -#define NONLOCAL_PARAM \ -"name '%U' is parameter and nonlocal" - #define GLOBAL_AFTER_ASSIGN \ -"name '%U' is assigned to before global declaration" - -#define NONLOCAL_AFTER_ASSIGN \ -"name '%U' is assigned to before nonlocal declaration" +"name '%.400s' is assigned to before global declaration" #define GLOBAL_AFTER_USE \ -"name '%U' is used prior to global declaration" - -#define NONLOCAL_AFTER_USE \ -"name '%U' is used prior to nonlocal declaration" - -#define GLOBAL_ANNOT \ -"annotated name '%U' can't be global" - -#define NONLOCAL_ANNOT \ -"annotated name '%U' can't be nonlocal" +"name '%.400s' is used prior to global declaration" #define IMPORT_STAR_WARNING "import * only allowed at module level" -#define NAMED_EXPR_COMP_IN_CLASS \ -"assignment expression within a comprehension cannot be used in a class body" +#define RETURN_VAL_IN_GENERATOR \ + "'return' with argument inside generator" -#define NAMED_EXPR_COMP_CONFLICT \ -"assignment expression cannot rebind comprehension iteration variable '%U'" - -#define NAMED_EXPR_COMP_INNER_LOOP_CONFLICT \ -"comprehension inner loop cannot rebind assignment expression target '%U'" - -#define NAMED_EXPR_COMP_ITER_EXPR \ -"assignment expression cannot be used in a comprehension iterable expression" static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, - void *key, int lineno, int col_offset) + void *key, int lineno) { PySTEntryObject *ste = NULL; PyObject *k = NULL; @@ -61,24 +35,34 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_table = st; ste->ste_id = k; /* ste owns reference to k */ - Py_INCREF(name); ste->ste_name = name; + Py_INCREF(name); ste->ste_symbols = NULL; ste->ste_varnames = NULL; ste->ste_children = NULL; - ste->ste_directives = NULL; + ste->ste_symbols = PyDict_New(); + if (ste->ste_symbols == NULL) + goto fail; + + ste->ste_varnames = PyList_New(0); + if (ste->ste_varnames == NULL) + goto fail; + + ste->ste_children = PyList_New(0); + if (ste->ste_children == NULL) + goto fail; ste->ste_type = block; + ste->ste_unoptimized = 0; ste->ste_nested = 0; ste->ste_free = 0; ste->ste_varargs = 0; ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; - ste->ste_opt_col_offset = 0; + ste->ste_tmpname = 0; ste->ste_lineno = lineno; - ste->ste_col_offset = col_offset; if (st->st_cur != NULL && (st->st_cur->ste_nested || @@ -86,22 +70,9 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_nested = 1; ste->ste_child_free = 0; ste->ste_generator = 0; - ste->ste_coroutine = 0; - ste->ste_comprehension = 0; ste->ste_returns_value = 0; - ste->ste_needs_class_closure = 0; - ste->ste_comp_iter_target = 0; - ste->ste_comp_iter_expr = 0; - - ste->ste_symbols = PyDict_New(); - ste->ste_varnames = PyList_New(0); - ste->ste_children = PyList_New(0); - if (ste->ste_symbols == NULL - || ste->ste_varnames == NULL - || ste->ste_children == NULL) - goto fail; - if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0) + if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0) goto fail; return ste; @@ -113,9 +84,13 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, static PyObject * ste_repr(PySTEntryObject *ste) { - return PyUnicode_FromFormat("<symtable entry %U(%ld), line %d>", - ste->ste_name, - PyLong_AS_LONG(ste->ste_id), ste->ste_lineno); + char buf[256]; + + PyOS_snprintf(buf, sizeof(buf), + "<symtable entry %.100s(%ld), line %d>", + PyString_AS_STRING(ste->ste_name), + PyInt_AS_LONG(ste->ste_id), ste->ste_lineno); + return PyString_FromString(buf); } static void @@ -127,7 +102,6 @@ ste_dealloc(PySTEntryObject *ste) Py_XDECREF(ste->ste_symbols); Py_XDECREF(ste->ste_varnames); Py_XDECREF(ste->ste_children); - Py_XDECREF(ste->ste_directives); PyObject_Del(ste); } @@ -139,6 +113,7 @@ static PyMemberDef ste_memberlist[] = { {"symbols", T_OBJECT, OFF(ste_symbols), READONLY}, {"varnames", T_OBJECT, OFF(ste_varnames), READONLY}, {"children", T_OBJECT, OFF(ste_children), READONLY}, + {"optimized",T_INT, OFF(ste_unoptimized), READONLY}, {"nested", T_INT, OFF(ste_nested), READONLY}, {"type", T_INT, OFF(ste_type), READONLY}, {"lineno", T_INT, OFF(ste_lineno), READONLY}, @@ -151,10 +126,10 @@ PyTypeObject PySTEntry_Type = { sizeof(PySTEntryObject), 0, (destructor)ste_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ + 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ - 0, /* tp_as_async */ + 0, /* tp_compare */ (reprfunc)ste_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ @@ -187,38 +162,36 @@ PyTypeObject PySTEntry_Type = { }; static int symtable_analyze(struct symtable *st); +static int symtable_warn(struct symtable *st, + PyObject *warn, const char *msg, int lineno); static int symtable_enter_block(struct symtable *st, identifier name, - _Py_block_ty block, void *ast, int lineno, - int col_offset); + _Py_block_ty block, void *ast, int lineno); static int symtable_exit_block(struct symtable *st, void *ast); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); +static int symtable_visit_listcomp(struct symtable *st, expr_ty e); static int symtable_visit_genexp(struct symtable *st, expr_ty s); -static int symtable_visit_listcomp(struct symtable *st, expr_ty s); -static int symtable_visit_setcomp(struct symtable *st, expr_ty s); -static int symtable_visit_dictcomp(struct symtable *st, expr_ty s); +static int symtable_visit_setcomp(struct symtable *st, expr_ty e); +static int symtable_visit_dictcomp(struct symtable *st, expr_ty e); static int symtable_visit_arguments(struct symtable *st, arguments_ty); static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_slice(struct symtable *st, slice_ty); -static int symtable_visit_params(struct symtable *st, asdl_seq *args); -static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args); +static int symtable_visit_params(struct symtable *st, asdl_seq *args, int top); +static int symtable_visit_params_nested(struct symtable *st, asdl_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); -static int symtable_visit_annotations(struct symtable *st, stmt_ty s, arguments_ty, expr_ty); -static int symtable_visit_withitem(struct symtable *st, withitem_ty item); -static identifier top = NULL, lambda = NULL, genexpr = NULL, - listcomp = NULL, setcomp = NULL, dictcomp = NULL, - __class__ = NULL; +static identifier top = NULL, lambda = NULL, genexpr = NULL, setcomp = NULL, + dictcomp = NULL; #define GET_IDENTIFIER(VAR) \ - ((VAR) ? (VAR) : ((VAR) = PyUnicode_InternFromString(# VAR))) + ((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR))) #define DUPLICATE_ARGUMENT \ -"duplicate argument '%U' in function definition" +"duplicate argument '%s' in function definition" static struct symtable * symtable_new(void) @@ -232,11 +205,11 @@ symtable_new(void) } st->st_filename = NULL; - st->st_blocks = NULL; + st->st_symbols = NULL; if ((st->st_stack = PyList_New(0)) == NULL) goto fail; - if ((st->st_blocks = PyDict_New()) == NULL) + if ((st->st_symbols = PyDict_New()) == NULL) goto fail; st->st_cur = NULL; st->st_private = NULL; @@ -246,59 +219,26 @@ symtable_new(void) return NULL; } -/* When compiling the use of C stack is probably going to be a lot - lighter than when executing Python code but still can overflow - and causing a Python crash if not checked (e.g. eval("()"*300000)). - Using the current recursion limit for the compiler seems too - restrictive (it caused at least one test to fail) so a factor is - used to allow deeper recursion when compiling an expression. - - Using a scaling factor means this should automatically adjust when - the recursion limit is adjusted for small or large C stack allocations. -*/ -#define COMPILER_STACK_FRAME_SCALE 3 - struct symtable * -PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future) +PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) { struct symtable *st = symtable_new(); asdl_seq *seq; int i; - PyThreadState *tstate; - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; if (st == NULL) - return NULL; - if (filename == NULL) { - PySymtable_Free(st); - return NULL; - } - Py_INCREF(filename); + return st; st->st_filename = filename; st->st_future = future; - - /* Setup recursion depth check counters */ - tstate = _PyThreadState_GET(); - if (!tstate) { - PySymtable_Free(st); - return NULL; - } - /* Be careful here to prevent overflow. */ - starting_recursion_depth = (tstate->recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - tstate->recursion_depth * COMPILER_STACK_FRAME_SCALE : tstate->recursion_depth; - st->recursion_depth = starting_recursion_depth; - st->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; - - /* Make the initial symbol information gathering pass */ if (!GET_IDENTIFIER(top) || - !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0, 0)) { + !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0)) { PySymtable_Free(st); return NULL; } st->st_top = st->st_cur; + st->st_cur->ste_unoptimized = OPT_TOPLEVEL; + /* Any other top-level initialization? */ switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; @@ -322,24 +262,11 @@ PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future) PyErr_SetString(PyExc_RuntimeError, "this compiler does not handle Suites"); goto error; - case FunctionType_kind: - PyErr_SetString(PyExc_RuntimeError, - "this compiler does not handle FunctionTypes"); - goto error; } if (!symtable_exit_block(st, (void *)mod)) { PySymtable_Free(st); return NULL; } - /* Check that the recursion depth counting balanced correctly */ - if (st->recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "symtable analysis recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, st->recursion_depth); - PySymtable_Free(st); - return NULL; - } - /* Make the second symbol analysis pass */ if (symtable_analyze(st)) return st; PySymtable_Free(st); @@ -350,24 +277,10 @@ PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future) return NULL; } -struct symtable * -PySymtable_Build(mod_ty mod, const char *filename_str, PyFutureFeatures *future) -{ - PyObject *filename; - struct symtable *st; - filename = PyUnicode_DecodeFSDefault(filename_str); - if (filename == NULL) - return NULL; - st = PySymtable_BuildObject(mod, filename, future); - Py_DECREF(filename); - return st; -} - void PySymtable_Free(struct symtable *st) { - Py_XDECREF(st->st_filename); - Py_XDECREF(st->st_blocks); + Py_XDECREF(st->st_symbols); Py_XDECREF(st->st_stack); PyMem_Free((void *)st); } @@ -380,12 +293,12 @@ PySymtable_Lookup(struct symtable *st, void *key) k = PyLong_FromVoidPtr(key); if (k == NULL) return NULL; - v = PyDict_GetItemWithError(st->st_blocks, k); + v = PyDict_GetItem(st->st_symbols, k); if (v) { assert(PySTEntry_Check(v)); Py_INCREF(v); } - else if (!PyErr_Occurred()) { + else { PyErr_SetString(PyExc_KeyError, "unknown symbol table entry"); } @@ -394,55 +307,25 @@ PySymtable_Lookup(struct symtable *st, void *key) return (PySTEntryObject *)v; } -static long -_PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) +int +PyST_GetScope(PySTEntryObject *ste, PyObject *name) { PyObject *v = PyDict_GetItem(ste->ste_symbols, name); if (!v) return 0; - assert(PyLong_Check(v)); - return PyLong_AS_LONG(v); -} - -int -PyST_GetScope(PySTEntryObject *ste, PyObject *name) -{ - long symbol = _PyST_GetSymbol(ste, name); - return (symbol >> SCOPE_OFFSET) & SCOPE_MASK; -} - -static int -error_at_directive(PySTEntryObject *ste, PyObject *name) -{ - Py_ssize_t i; - PyObject *data; - assert(ste->ste_directives); - for (i = 0; i < PyList_GET_SIZE(ste->ste_directives); i++) { - data = PyList_GET_ITEM(ste->ste_directives, i); - assert(PyTuple_CheckExact(data)); - assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(data, 0))); - if (PyUnicode_Compare(PyTuple_GET_ITEM(data, 0), name) == 0) { - PyErr_SyntaxLocationObject(ste->ste_table->st_filename, - PyLong_AsLong(PyTuple_GET_ITEM(data, 1)), - PyLong_AsLong(PyTuple_GET_ITEM(data, 2)) + 1); - - return 0; - } - } - PyErr_SetString(PyExc_RuntimeError, - "BUG: internal directive bookkeeping broken"); - return 0; + assert(PyInt_Check(v)); + return (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; } /* Analyze raw symbol information to determine scope of each name. - The next several functions are helpers for symtable_analyze(), + The next several functions are helpers for PySymtable_Analyze(), which determines whether a name is local, global, or free. In addition, it determines which local variables are cell variables; they provide bindings that are used for free variables in enclosed blocks. - There are also two kinds of global variables, implicit and explicit. An + There are also two kinds of free variables, implicit and explicit. An explicit global is declared with the global statement. An implicit global is a free variable for which the compiler has found no binding in an enclosing function scope. The implicit global is either a global @@ -452,36 +335,28 @@ error_at_directive(PySTEntryObject *ste, PyObject *name) is treated as a local. The symbol table requires two passes to determine the scope of each name. - The first pass collects raw facts from the AST via the symtable_visit_* - functions: the name is a parameter here, the name is used but not defined - here, etc. The second pass analyzes these facts during a pass over the - PySTEntryObjects created during pass 1. + The first pass collects raw facts from the AST: the name is a parameter + here, the name is used by not defined here, etc. The second pass analyzes + these facts during a pass over the PySTEntryObjects created during pass 1. When a function is entered during the second pass, the parent passes the set of all name bindings visible to its children. These bindings - are used to determine if non-local variables are free or implicit globals. - Names which are explicitly declared nonlocal must exist in this set of - visible names - if they do not, a syntax error is raised. After doing - the local analysis, it analyzes each of its child blocks using an - updated set of name bindings. - - The children update the free variable set. If a local variable is added to - the free variable set by the child, the variable is marked as a cell. The - function object being defined must provide runtime storage for the variable - that may outlive the function's frame. Cell variables are removed from the - free set before the analyze function returns to its parent. - - During analysis, the names are: - symbols: dict mapping from symbol names to flag values (including offset scope values) - scopes: dict mapping from symbol names to scope values (no offset) - local: set of all symbol names local to the current scope - bound: set of all symbol names local to a containing function scope - free: set of all symbol names referenced but not bound in child scopes - global: set of all symbol names explicitly declared as global + are used to determine if the variable is free or an implicit global. + After doing the local analysis, it analyzes each of its child blocks + using an updated set of name bindings. + + The children update the free variable set. If a local variable is free + in a child, the variable is marked as a cell. The current function must + provide runtime storage for the variable that may outlive the function's + frame. Cell variables are removed from the free set before the analyze + function returns to its parent. + + The sets of bound and free variables are implemented as dictionaries + mapping strings to None. */ #define SET_SCOPE(DICT, NAME, I) { \ - PyObject *o = PyLong_FromLong(I); \ + PyObject *o = PyInt_FromLong(I); \ if (!o) \ return 0; \ if (PyDict_SetItem((DICT), (NAME), o) < 0) { \ @@ -499,47 +374,37 @@ error_at_directive(PySTEntryObject *ste, PyObject *name) */ static int -analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, +analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags, PyObject *bound, PyObject *local, PyObject *free, PyObject *global) { if (flags & DEF_GLOBAL) { - if (flags & DEF_NONLOCAL) { + if (flags & DEF_PARAM) { PyErr_Format(PyExc_SyntaxError, - "name '%U' is nonlocal and global", - name); - return error_at_directive(ste, name); - } - SET_SCOPE(scopes, name, GLOBAL_EXPLICIT); - if (PySet_Add(global, name) < 0) - return 0; - if (bound && (PySet_Discard(bound, name) < 0)) + "name '%s' is local and global", + PyString_AS_STRING(name)); + PyErr_SyntaxLocation(ste->ste_table->st_filename, + ste->ste_lineno); + return 0; - return 1; - } - if (flags & DEF_NONLOCAL) { - if (!bound) { - PyErr_Format(PyExc_SyntaxError, - "nonlocal declaration not allowed at module level"); - return error_at_directive(ste, name); } - if (!PySet_Contains(bound, name)) { - PyErr_Format(PyExc_SyntaxError, - "no binding for nonlocal '%U' found", - name); - - return error_at_directive(ste, name); + SET_SCOPE(dict, name, GLOBAL_EXPLICIT); + if (PyDict_SetItem(global, name, Py_None) < 0) + return 0; + if (bound && PyDict_GetItem(bound, name)) { + if (PyDict_DelItem(bound, name) < 0) + return 0; } - SET_SCOPE(scopes, name, FREE); - ste->ste_free = 1; - return PySet_Add(free, name) >= 0; + return 1; } if (flags & DEF_BOUND) { - SET_SCOPE(scopes, name, LOCAL); - if (PySet_Add(local, name) < 0) - return 0; - if (PySet_Discard(global, name) < 0) + SET_SCOPE(dict, name, LOCAL); + if (PyDict_SetItem(local, name, Py_None) < 0) return 0; + if (PyDict_GetItem(global, name)) { + if (PyDict_DelItem(global, name) < 0) + return 0; + } return 1; } /* If an enclosing block has a binding for this name, it @@ -547,22 +412,30 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, Note that having a non-NULL bound implies that the block is nested. */ - if (bound && PySet_Contains(bound, name)) { - SET_SCOPE(scopes, name, FREE); + if (bound && PyDict_GetItem(bound, name)) { + SET_SCOPE(dict, name, FREE); ste->ste_free = 1; - return PySet_Add(free, name) >= 0; + if (PyDict_SetItem(free, name, Py_None) < 0) + return 0; + return 1; } /* If a parent has a global statement, then call it global explicit? It could also be global implicit. */ - if (global && PySet_Contains(global, name)) { - SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); + else if (global && PyDict_GetItem(global, name)) { + SET_SCOPE(dict, name, GLOBAL_IMPLICIT); return 1; } - if (ste->ste_nested) - ste->ste_free = 1; - SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); - return 1; + else { + if (ste->ste_nested) + ste->ste_free = 1; + SET_SCOPE(dict, name, GLOBAL_IMPLICIT); + return 1; + } + /* Should never get here. */ + PyErr_Format(PyExc_SystemError, "failed to set scope for %s", + PyString_AS_STRING(name)); + return 0; } #undef SET_SCOPE @@ -576,142 +449,153 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, */ static int -analyze_cells(PyObject *scopes, PyObject *free) +analyze_cells(PyObject *scope, PyObject *free) { - PyObject *name, *v, *v_cell; + PyObject *name, *v, *w; int success = 0; Py_ssize_t pos = 0; - v_cell = PyLong_FromLong(CELL); - if (!v_cell) + w = PyInt_FromLong(CELL); + if (!w) return 0; - while (PyDict_Next(scopes, &pos, &name, &v)) { - long scope; - assert(PyLong_Check(v)); - scope = PyLong_AS_LONG(v); - if (scope != LOCAL) + while (PyDict_Next(scope, &pos, &name, &v)) { + long flags; + assert(PyInt_Check(v)); + flags = PyInt_AS_LONG(v); + if (flags != LOCAL) continue; - if (!PySet_Contains(free, name)) + if (!PyDict_GetItem(free, name)) continue; /* Replace LOCAL with CELL for this name, and remove from free. It is safe to replace the value of name in the dict, because it will not cause a resize. */ - if (PyDict_SetItem(scopes, name, v_cell) < 0) + if (PyDict_SetItem(scope, name, w) < 0) goto error; - if (PySet_Discard(free, name) < 0) + if (PyDict_DelItem(free, name) < 0) goto error; } success = 1; error: - Py_DECREF(v_cell); + Py_DECREF(w); return success; } +/* Check for illegal statements in unoptimized namespaces */ static int -drop_class_free(PySTEntryObject *ste, PyObject *free) -{ - int res; - if (!GET_IDENTIFIER(__class__)) - return 0; - res = PySet_Discard(free, __class__); - if (res < 0) - return 0; - if (res) - ste->ste_needs_class_closure = 1; - return 1; +check_unoptimized(const PySTEntryObject* ste) { + char buf[300]; + const char* trailer; + + if (ste->ste_type != FunctionBlock || !ste->ste_unoptimized + || !(ste->ste_free || ste->ste_child_free)) + return 1; + + trailer = (ste->ste_child_free ? + "contains a nested function with free variables" : + "is a nested function"); + + switch (ste->ste_unoptimized) { + case OPT_TOPLEVEL: /* exec / import * at top-level is fine */ + case OPT_EXEC: /* qualified exec is fine */ + return 1; + case OPT_IMPORT_STAR: + PyOS_snprintf(buf, sizeof(buf), + "import * is not allowed in function '%.100s' " + "because it %s", + PyString_AS_STRING(ste->ste_name), trailer); + break; + case OPT_BARE_EXEC: + PyOS_snprintf(buf, sizeof(buf), + "unqualified exec is not allowed in function " + "'%.100s' because it %s", + PyString_AS_STRING(ste->ste_name), trailer); + break; + default: + PyOS_snprintf(buf, sizeof(buf), + "function '%.100s' uses import * and bare exec, " + "which are illegal because it %s", + PyString_AS_STRING(ste->ste_name), trailer); + break; + } + + PyErr_SetString(PyExc_SyntaxError, buf); + PyErr_SyntaxLocation(ste->ste_table->st_filename, + ste->ste_opt_lineno); + return 0; } -/* Enter the final scope information into the ste_symbols dict. +/* Enter the final scope information into the st_symbols dict. * * All arguments are dicts. Modifies symbols, others are read-only. */ static int -update_symbols(PyObject *symbols, PyObject *scopes, +update_symbols(PyObject *symbols, PyObject *scope, PyObject *bound, PyObject *free, int classflag) { - PyObject *name = NULL, *itr = NULL; - PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL; + PyObject *name, *v, *u, *w, *free_value = NULL; Py_ssize_t pos = 0; - /* Update scope information for all symbols in this scope */ while (PyDict_Next(symbols, &pos, &name, &v)) { - long scope, flags; - assert(PyLong_Check(v)); - flags = PyLong_AS_LONG(v); - v_scope = PyDict_GetItem(scopes, name); - assert(v_scope && PyLong_Check(v_scope)); - scope = PyLong_AS_LONG(v_scope); - flags |= (scope << SCOPE_OFFSET); - v_new = PyLong_FromLong(flags); - if (!v_new) + long i, flags; + assert(PyInt_Check(v)); + flags = PyInt_AS_LONG(v); + w = PyDict_GetItem(scope, name); + assert(w && PyInt_Check(w)); + i = PyInt_AS_LONG(w); + flags |= (i << SCOPE_OFF); + u = PyInt_FromLong(flags); + if (!u) return 0; - if (PyDict_SetItem(symbols, name, v_new) < 0) { - Py_DECREF(v_new); + if (PyDict_SetItem(symbols, name, u) < 0) { + Py_DECREF(u); return 0; } - Py_DECREF(v_new); + Py_DECREF(u); } - /* Record not yet resolved free variables from children (if any) */ - v_free = PyLong_FromLong(FREE << SCOPE_OFFSET); - if (!v_free) - return 0; - - itr = PyObject_GetIter(free); - if (itr == NULL) { - Py_DECREF(v_free); + free_value = PyInt_FromLong(FREE << SCOPE_OFF); + if (!free_value) return 0; - } - while ((name = PyIter_Next(itr))) { - v = PyDict_GetItemWithError(symbols, name); + /* add a free variable when it's only use is for creating a closure */ + pos = 0; + while (PyDict_Next(free, &pos, &name, &v)) { + PyObject *o = PyDict_GetItem(symbols, name); - /* Handle symbol that already exists in this scope */ - if (v) { - /* Handle a free variable in a method of + if (o) { + /* It could be a free variable in a method of the class that has the same name as a local or global in the class scope. */ if (classflag && - PyLong_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) { - long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS; - v_new = PyLong_FromLong(flags); - if (!v_new) { - goto error; + PyInt_AS_LONG(o) & (DEF_BOUND | DEF_GLOBAL)) { + long i = PyInt_AS_LONG(o) | DEF_FREE_CLASS; + o = PyInt_FromLong(i); + if (!o) { + Py_DECREF(free_value); + return 0; } - if (PyDict_SetItem(symbols, name, v_new) < 0) { - Py_DECREF(v_new); - goto error; + if (PyDict_SetItem(symbols, name, o) < 0) { + Py_DECREF(o); + Py_DECREF(free_value); + return 0; } - Py_DECREF(v_new); + Py_DECREF(o); } - /* It's a cell, or already free in this scope */ - Py_DECREF(name); + /* else it's not free, probably a cell */ continue; } - else if (PyErr_Occurred()) { - goto error; - } - /* Handle global symbol */ - if (bound && !PySet_Contains(bound, name)) { - Py_DECREF(name); + if (!PyDict_GetItem(bound, name)) continue; /* it's a global */ + + if (PyDict_SetItem(symbols, name, free_value) < 0) { + Py_DECREF(free_value); + return 0; } - /* Propagate new free symbol up the lexical stack */ - if (PyDict_SetItem(symbols, name, v_free) < 0) { - goto error; - } - Py_DECREF(name); } - Py_DECREF(itr); - Py_DECREF(v_free); + Py_DECREF(free_value); return 1; -error: - Py_XDECREF(v_free); - Py_XDECREF(itr); - Py_XDECREF(name); - return 0; } /* Make final symbol table decisions for block of ste. @@ -742,17 +626,17 @@ static int analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PyObject *global) { - PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; - PyObject *newglobal = NULL, *newfree = NULL, *allfree = NULL; - PyObject *temp; + PyObject *name, *v, *local = NULL, *scope = NULL; + PyObject *newbound = NULL, *newglobal = NULL; + PyObject *newfree = NULL, *allfree = NULL; int i, success = 0; Py_ssize_t pos = 0; - local = PySet_New(NULL); /* collect new names bound in block */ + local = PyDict_New(); /* collect new names bound in block */ if (!local) goto error; - scopes = PyDict_New(); /* collect scopes defined for each name */ - if (!scopes) + scope = PyDict_New(); /* collect scopes defined for each name */ + if (!scope) goto error; /* Allocate new global and bound variable dictionaries. These @@ -766,80 +650,51 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, /* TODO(jhylton): Package these dicts in a struct so that we can write reasonable helper functions? */ - newglobal = PySet_New(NULL); + newglobal = PyDict_New(); if (!newglobal) goto error; - newfree = PySet_New(NULL); - if (!newfree) - goto error; - newbound = PySet_New(NULL); + newbound = PyDict_New(); if (!newbound) goto error; + newfree = PyDict_New(); + if (!newfree) + goto error; - /* Class namespace has no effect on names visible in - nested functions, so populate the global and bound - sets to be passed to child blocks before analyzing - this one. - */ if (ste->ste_type == ClassBlock) { - /* Pass down known globals */ - temp = PyNumber_InPlaceOr(newglobal, global); - if (!temp) + if (PyDict_Update(newglobal, global) < 0) goto error; - Py_DECREF(temp); - /* Pass down previously bound symbols */ - if (bound) { - temp = PyNumber_InPlaceOr(newbound, bound); - if (!temp) + if (bound) + if (PyDict_Update(newbound, bound) < 0) goto error; - Py_DECREF(temp); - } } while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { - long flags = PyLong_AS_LONG(v); - if (!analyze_name(ste, scopes, name, flags, + long flags = PyInt_AS_LONG(v); + if (!analyze_name(ste, scope, name, flags, bound, local, free, global)) goto error; } - /* Populate global and bound sets to be passed to children. */ if (ste->ste_type != ClassBlock) { - /* Add function locals to bound set */ if (ste->ste_type == FunctionBlock) { - temp = PyNumber_InPlaceOr(newbound, local); - if (!temp) + if (PyDict_Update(newbound, local) < 0) goto error; - Py_DECREF(temp); } - /* Pass down previously bound symbols */ if (bound) { - temp = PyNumber_InPlaceOr(newbound, bound); - if (!temp) + if (PyDict_Update(newbound, bound) < 0) goto error; - Py_DECREF(temp); } - /* Pass down known globals */ - temp = PyNumber_InPlaceOr(newglobal, global); - if (!temp) - goto error; - Py_DECREF(temp); - } - else { - /* Special-case __class__ */ - if (!GET_IDENTIFIER(__class__)) - goto error; - if (PySet_Add(newbound, __class__) < 0) + if (PyDict_Update(newglobal, global) < 0) goto error; } - /* Recursively call analyze_child_block() on each child block. + /* Recursively call analyze_block() on each child block. newbound, newglobal now contain the names visible in nested blocks. The free variables in the children will be collected in allfree. */ - allfree = PySet_New(NULL); + allfree = PyDict_New(); if (!allfree) goto error; for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) { @@ -850,34 +705,26 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, if (!analyze_child_block(entry, newbound, newfree, newglobal, allfree)) goto error; - /* Check if any children have free variables */ if (entry->ste_free || entry->ste_child_free) ste->ste_child_free = 1; } - temp = PyNumber_InPlaceOr(newfree, allfree); - if (!temp) - goto error; - Py_DECREF(temp); - - /* Check if any local variables must be converted to cell variables */ - if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree)) + if (PyDict_Update(newfree, allfree) < 0) goto error; - else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) + if (ste->ste_type == FunctionBlock && !analyze_cells(scope, newfree)) goto error; - /* Records the results of the analysis in the symbol table entry */ - if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, + if (!update_symbols(ste->ste_symbols, scope, bound, newfree, ste->ste_type == ClassBlock)) goto error; + if (!check_unoptimized(ste)) + goto error; - temp = PyNumber_InPlaceOr(free, newfree); - if (!temp) + if (PyDict_Update(free, newfree) < 0) goto error; - Py_DECREF(temp); success = 1; error: - Py_XDECREF(scopes); Py_XDECREF(local); + Py_XDECREF(scope); Py_XDECREF(newbound); Py_XDECREF(newglobal); Py_XDECREF(newfree); @@ -892,7 +739,6 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, PyObject *global, PyObject* child_free) { PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL; - PyObject *temp; /* Copy the bound and global dictionaries. @@ -901,22 +747,26 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, dictionaries. */ - temp_bound = PySet_New(bound); + temp_bound = PyDict_New(); if (!temp_bound) goto error; - temp_free = PySet_New(free); + if (PyDict_Update(temp_bound, bound) < 0) + goto error; + temp_free = PyDict_New(); if (!temp_free) goto error; - temp_global = PySet_New(global); + if (PyDict_Update(temp_free, free) < 0) + goto error; + temp_global = PyDict_New(); if (!temp_global) goto error; + if (PyDict_Update(temp_global, global) < 0) + goto error; if (!analyze_block(entry, temp_bound, temp_free, temp_global)) goto error; - temp = PyNumber_InPlaceOr(child_free, temp_free); - if (!temp) + if (PyDict_Update(child_free, temp_free) < 0) goto error; - Py_DECREF(temp); Py_DECREF(temp_bound); Py_DECREF(temp_free); Py_DECREF(temp_global); @@ -934,10 +784,10 @@ symtable_analyze(struct symtable *st) PyObject *free, *global; int r; - free = PySet_New(NULL); + free = PyDict_New(); if (!free) return 0; - global = PySet_New(NULL); + global = PyDict_New(); if (!global) { Py_DECREF(free); return 0; @@ -948,6 +798,26 @@ symtable_analyze(struct symtable *st) return r; } + +static int +symtable_warn(struct symtable *st, PyObject *warn, const char *msg, int lineno) +{ + if (lineno < 0) { + lineno = st->st_cur->ste_lineno; + } + if (PyErr_WarnExplicit(warn, msg, st->st_filename, lineno, NULL, NULL) < 0) { + if (PyErr_ExceptionMatches(warn)) { + /* Replace the warning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); + PyErr_SetString(PyExc_SyntaxError, msg); + PyErr_SyntaxLocation(st->st_filename, lineno); + } + return 0; + } + return 1; +} + /* symtable_enter_block() gets a reference via ste_new. This reference is released when the block is exited, via the DECREF in symtable_exit_block(). @@ -956,47 +826,43 @@ symtable_analyze(struct symtable *st) static int symtable_exit_block(struct symtable *st, void *ast) { - Py_ssize_t size; - - st->st_cur = NULL; - size = PyList_GET_SIZE(st->st_stack); - if (size) { - if (PyList_SetSlice(st->st_stack, size - 1, size, NULL) < 0) + Py_ssize_t end; + + Py_CLEAR(st->st_cur); + end = PyList_GET_SIZE(st->st_stack) - 1; + if (end >= 0) { + st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, + end); + if (st->st_cur == NULL) + return 0; + Py_INCREF(st->st_cur); + if (PySequence_DelItem(st->st_stack, end) < 0) return 0; - if (--size) - st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, size - 1); } return 1; } static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, - void *ast, int lineno, int col_offset) + void *ast, int lineno) { - PySTEntryObject *prev = NULL, *ste; + PySTEntryObject *prev = NULL; - ste = ste_new(st, name, block, ast, lineno, col_offset); - if (ste == NULL) - return 0; - if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) { - Py_DECREF(ste); - return 0; - } - prev = st->st_cur; - /* bpo-37757: For now, disallow *all* assignment expressions in the - * outermost iterator expression of a comprehension, even those inside - * a nested comprehension or a lambda expression. - */ - if (prev) { - ste->ste_comp_iter_expr = prev->ste_comp_iter_expr; + if (st->st_cur) { + prev = st->st_cur; + if (PyList_Append(st->st_stack, (PyObject *)st->st_cur) < 0) { + return 0; + } + Py_DECREF(st->st_cur); } - /* The entry is owned by the stack. Borrow it for st_cur. */ - Py_DECREF(ste); - st->st_cur = ste; + st->st_cur = ste_new(st, name, block, ast, lineno); + if (st->st_cur == NULL) + return 0; if (block == ModuleBlock) st->st_global = st->st_cur->ste_symbols; if (prev) { - if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) { + if (PyList_Append(prev->ste_children, + (PyObject *)st->st_cur) < 0) { return 0; } } @@ -1006,61 +872,42 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, static long symtable_lookup(struct symtable *st, PyObject *name) { + PyObject *o; PyObject *mangled = _Py_Mangle(st->st_private, name); if (!mangled) return 0; - long ret = _PyST_GetSymbol(st->st_cur, mangled); + o = PyDict_GetItem(st->st_cur->ste_symbols, mangled); Py_DECREF(mangled); - return ret; + if (!o) + return 0; + return PyInt_AsLong(o); } static int -symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste) +symtable_add_def(struct symtable *st, PyObject *name, int flag) { PyObject *o; PyObject *dict; long val; PyObject *mangled = _Py_Mangle(st->st_private, name); - if (!mangled) return 0; - dict = ste->ste_symbols; - if ((o = PyDict_GetItemWithError(dict, mangled))) { - val = PyLong_AS_LONG(o); + dict = st->st_cur->ste_symbols; + if ((o = PyDict_GetItem(dict, mangled))) { + val = PyInt_AS_LONG(o); if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { /* Is it better to use 'mangled' or 'name' here? */ - PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name); - PyErr_SyntaxLocationObject(st->st_filename, - ste->ste_lineno, - ste->ste_col_offset + 1); + PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, + PyString_AsString(name)); + PyErr_SyntaxLocation(st->st_filename, + st->st_cur->ste_lineno); goto error; } val |= flag; - } - else if (PyErr_Occurred()) { - goto error; - } - else { + } else val = flag; - } - if (ste->ste_comp_iter_target) { - /* This name is an iteration variable in a comprehension, - * so check for a binding conflict with any named expressions. - * Otherwise, mark it as an iteration variable so subsequent - * named expressions can check for conflicts. - */ - if (val & (DEF_GLOBAL | DEF_NONLOCAL)) { - PyErr_Format(PyExc_SyntaxError, - NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name); - PyErr_SyntaxLocationObject(st->st_filename, - ste->ste_lineno, - ste->ste_col_offset + 1); - goto error; - } - val |= DEF_COMP_ITER; - } - o = PyLong_FromLong(val); + o = PyInt_FromLong(val); if (o == NULL) goto error; if (PyDict_SetItem(dict, mangled, o) < 0) { @@ -1070,16 +917,16 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s Py_DECREF(o); if (flag & DEF_PARAM) { - if (PyList_Append(ste->ste_varnames, mangled) < 0) + if (PyList_Append(st->st_cur->ste_varnames, mangled) < 0) goto error; } else if (flag & DEF_GLOBAL) { /* XXX need to update DEF_GLOBAL for other flags too; perhaps only DEF_FREE_GLOBAL */ val = flag; if ((o = PyDict_GetItem(st->st_global, mangled))) { - val |= PyLong_AS_LONG(o); + val |= PyInt_AS_LONG(o); } - o = PyLong_FromLong(val); + o = PyInt_FromLong(val); if (o == NULL) goto error; if (PyDict_SetItem(st->st_global, mangled, o) < 0) { @@ -1096,28 +943,23 @@ error: return 0; } -static int -symtable_add_def(struct symtable *st, PyObject *name, int flag) { - return symtable_add_def_helper(st, name, flag, st->st_cur); -} - /* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument. They use the ASDL name to synthesize the name of the C type and the visit function. VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is useful if the first node in the sequence requires special treatment. - - VISIT_QUIT macro returns the specified value exiting from the function but - first adjusts current recursion counter depth. */ -#define VISIT_QUIT(ST, X) \ - return --(ST)->recursion_depth,(X) - #define VISIT(ST, TYPE, V) \ if (!symtable_visit_ ## TYPE((ST), (V))) \ - VISIT_QUIT((ST), 0); + return 0; + +#define VISIT_IN_BLOCK(ST, TYPE, V, S) \ + if (!symtable_visit_ ## TYPE((ST), (V))) { \ + symtable_exit_block((ST), (S)); \ + return 0; \ + } #define VISIT_SEQ(ST, TYPE, SEQ) { \ int i; \ @@ -1125,106 +967,92 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) { for (i = 0; i < asdl_seq_LEN(seq); i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ + return 0; \ } \ } -#define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \ +#define VISIT_SEQ_IN_BLOCK(ST, TYPE, SEQ, S) { \ int i; \ asdl_seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = (START); i < asdl_seq_LEN(seq); i++) { \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ + if (!symtable_visit_ ## TYPE((ST), elt)) { \ + symtable_exit_block((ST), (S)); \ + return 0; \ + } \ } \ } -#define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) { \ - int i = 0; \ +#define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \ + int i; \ asdl_seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + for (i = (START); i < asdl_seq_LEN(seq); i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!elt) continue; /* can be NULL */ \ if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ + return 0; \ } \ } -static int -symtable_record_directive(struct symtable *st, identifier name, int lineno, int col_offset) -{ - PyObject *data, *mangled; - int res; - if (!st->st_cur->ste_directives) { - st->st_cur->ste_directives = PyList_New(0); - if (!st->st_cur->ste_directives) - return 0; - } - mangled = _Py_Mangle(st->st_private, name); - if (!mangled) - return 0; - data = Py_BuildValue("(Nii)", mangled, lineno, col_offset); - if (!data) - return 0; - res = PyList_Append(st->st_cur->ste_directives, data); - Py_DECREF(data); - return res == 0; +#define VISIT_SEQ_TAIL_IN_BLOCK(ST, TYPE, SEQ, START, S) { \ + int i; \ + asdl_seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = (START); i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt)) { \ + symtable_exit_block((ST), (S)); \ + return 0; \ + } \ + } \ } - static int symtable_visit_stmt(struct symtable *st, stmt_ty s) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); - } switch (s->kind) { case FunctionDef_kind: if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL)) - VISIT_QUIT(st, 0); + return 0; if (s->v.FunctionDef.args->defaults) VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults); - if (s->v.FunctionDef.args->kw_defaults) - VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults); - if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, - s->v.FunctionDef.returns)) - VISIT_QUIT(st, 0); if (s->v.FunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list); if (!symtable_enter_block(st, s->v.FunctionDef.name, - FunctionBlock, (void *)s, s->lineno, - s->col_offset)) - VISIT_QUIT(st, 0); - VISIT(st, arguments, s->v.FunctionDef.args); - VISIT_SEQ(st, stmt, s->v.FunctionDef.body); + FunctionBlock, (void *)s, s->lineno)) + return 0; + VISIT_IN_BLOCK(st, arguments, s->v.FunctionDef.args, s); + VISIT_SEQ_IN_BLOCK(st, stmt, s->v.FunctionDef.body, s); if (!symtable_exit_block(st, s)) - VISIT_QUIT(st, 0); + return 0; break; case ClassDef_kind: { PyObject *tmp; if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL)) - VISIT_QUIT(st, 0); + return 0; VISIT_SEQ(st, expr, s->v.ClassDef.bases); - VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); if (s->v.ClassDef.decorator_list) VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, - (void *)s, s->lineno, s->col_offset)) - VISIT_QUIT(st, 0); + (void *)s, s->lineno)) + return 0; tmp = st->st_private; st->st_private = s->v.ClassDef.name; - VISIT_SEQ(st, stmt, s->v.ClassDef.body); + VISIT_SEQ_IN_BLOCK(st, stmt, s->v.ClassDef.body, s); st->st_private = tmp; if (!symtable_exit_block(st, s)) - VISIT_QUIT(st, 0); + return 0; break; } case Return_kind: if (s->v.Return.value) { VISIT(st, expr, s->v.Return.value); st->st_cur->ste_returns_value = 1; + if (st->st_cur->ste_generator) { + PyErr_SetString(PyExc_SyntaxError, + RETURN_VAL_IN_GENERATOR); + PyErr_SyntaxLocation(st->st_filename, + s->lineno); + return 0; + } } break; case Delete_kind: @@ -1234,48 +1062,15 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, expr, s->v.Assign.targets); VISIT(st, expr, s->v.Assign.value); break; - case AnnAssign_kind: - if (s->v.AnnAssign.target->kind == Name_kind) { - expr_ty e_name = s->v.AnnAssign.target; - long cur = symtable_lookup(st, e_name->v.Name.id); - if (cur < 0) { - VISIT_QUIT(st, 0); - } - if ((cur & (DEF_GLOBAL | DEF_NONLOCAL)) - && (st->st_cur->ste_symbols != st->st_global) - && s->v.AnnAssign.simple) { - PyErr_Format(PyExc_SyntaxError, - cur & DEF_GLOBAL ? GLOBAL_ANNOT : NONLOCAL_ANNOT, - e_name->v.Name.id); - PyErr_SyntaxLocationObject(st->st_filename, - s->lineno, - s->col_offset + 1); - VISIT_QUIT(st, 0); - } - if (s->v.AnnAssign.simple && - !symtable_add_def(st, e_name->v.Name.id, - DEF_ANNOT | DEF_LOCAL)) { - VISIT_QUIT(st, 0); - } - else { - if (s->v.AnnAssign.value - && !symtable_add_def(st, e_name->v.Name.id, DEF_LOCAL)) { - VISIT_QUIT(st, 0); - } - } - } - else { - VISIT(st, expr, s->v.AnnAssign.target); - } - VISIT(st, expr, s->v.AnnAssign.annotation); - if (s->v.AnnAssign.value) { - VISIT(st, expr, s->v.AnnAssign.value); - } - break; case AugAssign_kind: VISIT(st, expr, s->v.AugAssign.target); VISIT(st, expr, s->v.AugAssign.value); break; + case Print_kind: + if (s->v.Print.dest) + VISIT(st, expr, s->v.Print.dest); + VISIT_SEQ(st, expr, s->v.Print.values); + break; case For_kind: VISIT(st, expr, s->v.For.target); VISIT(st, expr, s->v.For.iter); @@ -1297,18 +1092,23 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, stmt, s->v.If.orelse); break; case Raise_kind: - if (s->v.Raise.exc) { - VISIT(st, expr, s->v.Raise.exc); - if (s->v.Raise.cause) { - VISIT(st, expr, s->v.Raise.cause); + if (s->v.Raise.type) { + VISIT(st, expr, s->v.Raise.type); + if (s->v.Raise.inst) { + VISIT(st, expr, s->v.Raise.inst); + if (s->v.Raise.tback) + VISIT(st, expr, s->v.Raise.tback); } } break; - case Try_kind: - VISIT_SEQ(st, stmt, s->v.Try.body); - VISIT_SEQ(st, stmt, s->v.Try.orelse); - VISIT_SEQ(st, excepthandler, s->v.Try.handlers); - VISIT_SEQ(st, stmt, s->v.Try.finalbody); + case TryExcept_kind: + VISIT_SEQ(st, stmt, s->v.TryExcept.body); + VISIT_SEQ(st, stmt, s->v.TryExcept.orelse); + VISIT_SEQ(st, excepthandler, s->v.TryExcept.handlers); + break; + case TryFinally_kind: + VISIT_SEQ(st, stmt, s->v.TryFinally.body); + VISIT_SEQ(st, stmt, s->v.TryFinally.finalbody); break; case Assert_kind: VISIT(st, expr, s->v.Assert.test); @@ -1317,72 +1117,55 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case Import_kind: VISIT_SEQ(st, alias, s->v.Import.names); + /* XXX Don't have the lineno available inside + visit_alias */ + if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) + st->st_cur->ste_opt_lineno = s->lineno; break; case ImportFrom_kind: VISIT_SEQ(st, alias, s->v.ImportFrom.names); + /* XXX Don't have the lineno available inside + visit_alias */ + if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) + st->st_cur->ste_opt_lineno = s->lineno; + break; + case Exec_kind: + VISIT(st, expr, s->v.Exec.body); + if (!st->st_cur->ste_opt_lineno) + st->st_cur->ste_opt_lineno = s->lineno; + if (s->v.Exec.globals) { + st->st_cur->ste_unoptimized |= OPT_EXEC; + VISIT(st, expr, s->v.Exec.globals); + if (s->v.Exec.locals) + VISIT(st, expr, s->v.Exec.locals); + } else { + st->st_cur->ste_unoptimized |= OPT_BARE_EXEC; + } break; case Global_kind: { int i; asdl_seq *seq = s->v.Global.names; for (i = 0; i < asdl_seq_LEN(seq); i++) { identifier name = (identifier)asdl_seq_GET(seq, i); + char *c_name = PyString_AS_STRING(name); long cur = symtable_lookup(st, name); if (cur < 0) - VISIT_QUIT(st, 0); - if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) { - const char* msg; - if (cur & DEF_PARAM) { - msg = GLOBAL_PARAM; - } else if (cur & USE) { - msg = GLOBAL_AFTER_USE; - } else if (cur & DEF_ANNOT) { - msg = GLOBAL_ANNOT; - } else { /* DEF_LOCAL */ - msg = GLOBAL_AFTER_ASSIGN; - } - PyErr_Format(PyExc_SyntaxError, - msg, name); - PyErr_SyntaxLocationObject(st->st_filename, - s->lineno, - s->col_offset + 1); - VISIT_QUIT(st, 0); + return 0; + if (cur & (DEF_LOCAL | USE)) { + char buf[256]; + if (cur & DEF_LOCAL) + PyOS_snprintf(buf, sizeof(buf), + GLOBAL_AFTER_ASSIGN, + c_name); + else + PyOS_snprintf(buf, sizeof(buf), + GLOBAL_AFTER_USE, + c_name); + if (!symtable_warn(st, PyExc_SyntaxWarning, buf, s->lineno)) + return 0; } if (!symtable_add_def(st, name, DEF_GLOBAL)) - VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, name, s->lineno, s->col_offset)) - VISIT_QUIT(st, 0); - } - break; - } - case Nonlocal_kind: { - int i; - asdl_seq *seq = s->v.Nonlocal.names; - for (i = 0; i < asdl_seq_LEN(seq); i++) { - identifier name = (identifier)asdl_seq_GET(seq, i); - long cur = symtable_lookup(st, name); - if (cur < 0) - VISIT_QUIT(st, 0); - if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) { - const char* msg; - if (cur & DEF_PARAM) { - msg = NONLOCAL_PARAM; - } else if (cur & USE) { - msg = NONLOCAL_AFTER_USE; - } else if (cur & DEF_ANNOT) { - msg = NONLOCAL_ANNOT; - } else { /* DEF_LOCAL */ - msg = NONLOCAL_AFTER_ASSIGN; - } - PyErr_Format(PyExc_SyntaxError, msg, name); - PyErr_SyntaxLocationObject(st->st_filename, - s->lineno, - s->col_offset + 1); - VISIT_QUIT(st, 0); - } - if (!symtable_add_def(st, name, DEF_NONLOCAL)) - VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, name, s->lineno, s->col_offset)) - VISIT_QUIT(st, 0); + return 0; } break; } @@ -1395,153 +1178,20 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) /* nothing to do here */ break; case With_kind: - VISIT_SEQ(st, withitem, s->v.With.items); + VISIT(st, expr, s->v.With.context_expr); + if (s->v.With.optional_vars) { + VISIT(st, expr, s->v.With.optional_vars); + } VISIT_SEQ(st, stmt, s->v.With.body); break; - case AsyncFunctionDef_kind: - if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL)) - VISIT_QUIT(st, 0); - if (s->v.AsyncFunctionDef.args->defaults) - VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults); - if (s->v.AsyncFunctionDef.args->kw_defaults) - VISIT_SEQ_WITH_NULL(st, expr, - s->v.AsyncFunctionDef.args->kw_defaults); - if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, - s->v.AsyncFunctionDef.returns)) - VISIT_QUIT(st, 0); - if (s->v.AsyncFunctionDef.decorator_list) - VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list); - if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name, - FunctionBlock, (void *)s, s->lineno, - s->col_offset)) - VISIT_QUIT(st, 0); - st->st_cur->ste_coroutine = 1; - VISIT(st, arguments, s->v.AsyncFunctionDef.args); - VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); - if (!symtable_exit_block(st, s)) - VISIT_QUIT(st, 0); - break; - case AsyncWith_kind: - VISIT_SEQ(st, withitem, s->v.AsyncWith.items); - VISIT_SEQ(st, stmt, s->v.AsyncWith.body); - break; - case AsyncFor_kind: - VISIT(st, expr, s->v.AsyncFor.target); - VISIT(st, expr, s->v.AsyncFor.iter); - VISIT_SEQ(st, stmt, s->v.AsyncFor.body); - if (s->v.AsyncFor.orelse) - VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse); - break; - } - VISIT_QUIT(st, 1); -} - -static int -symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) -{ - assert(st->st_stack); - assert(e->kind == Name_kind); - - PyObject *target_name = e->v.Name.id; - Py_ssize_t i, size; - struct _symtable_entry *ste; - size = PyList_GET_SIZE(st->st_stack); - assert(size); - - /* Iterate over the stack in reverse and add to the nearest adequate scope */ - for (i = size - 1; i >= 0; i--) { - ste = (struct _symtable_entry *) PyList_GET_ITEM(st->st_stack, i); - - /* If we find a comprehension scope, check for a target - * binding conflict with iteration variables, otherwise skip it - */ - if (ste->ste_comprehension) { - long target_in_scope = _PyST_GetSymbol(ste, target_name); - if (target_in_scope & DEF_COMP_ITER) { - PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name); - PyErr_SyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset); - VISIT_QUIT(st, 0); - } - continue; - } - - /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */ - if (ste->ste_type == FunctionBlock) { - long target_in_scope = _PyST_GetSymbol(ste, target_name); - if (target_in_scope & DEF_GLOBAL) { - if (!symtable_add_def(st, target_name, DEF_GLOBAL)) - VISIT_QUIT(st, 0); - } else { - if (!symtable_add_def(st, target_name, DEF_NONLOCAL)) - VISIT_QUIT(st, 0); - } - if (!symtable_record_directive(st, target_name, e->lineno, e->col_offset)) - VISIT_QUIT(st, 0); - - return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste); - } - /* If we find a ModuleBlock entry, add as GLOBAL */ - if (ste->ste_type == ModuleBlock) { - if (!symtable_add_def(st, target_name, DEF_GLOBAL)) - VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, target_name, e->lineno, e->col_offset)) - VISIT_QUIT(st, 0); - - return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste); - } - /* Disallow usage in ClassBlock */ - if (ste->ste_type == ClassBlock) { - PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS); - PyErr_SyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset); - VISIT_QUIT(st, 0); - } - } - - /* We should always find either a FunctionBlock, ModuleBlock or ClassBlock - and should never fall to this case - */ - assert(0); - return 0; -} - -static int -symtable_handle_namedexpr(struct symtable *st, expr_ty e) -{ - if (st->st_cur->ste_comp_iter_expr > 0) { - /* Assignment isn't allowed in a comprehension iterable expression */ - PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR); - PyErr_SyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset); - return 0; } - if (st->st_cur->ste_comprehension) { - /* Inside a comprehension body, so find the right target scope */ - if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target)) - return 0; - } - VISIT(st, expr, e->v.NamedExpr.value); - VISIT(st, expr, e->v.NamedExpr.target); return 1; } static int symtable_visit_expr(struct symtable *st, expr_ty e) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); - } switch (e->kind) { - case NamedExpr_kind: - if(!symtable_handle_namedexpr(st, e)) - VISIT_QUIT(st, 0); - break; case BoolOp_kind: VISIT_SEQ(st, expr, e->v.BoolOp.values); break; @@ -1554,19 +1204,16 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case Lambda_kind: { if (!GET_IDENTIFIER(lambda)) - VISIT_QUIT(st, 0); + return 0; if (e->v.Lambda.args->defaults) VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); - if (e->v.Lambda.args->kw_defaults) - VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults); if (!symtable_enter_block(st, lambda, - FunctionBlock, (void *)e, e->lineno, - e->col_offset)) - VISIT_QUIT(st, 0); - VISIT(st, arguments, e->v.Lambda.args); - VISIT(st, expr, e->v.Lambda.body); + FunctionBlock, (void *)e, e->lineno)) + return 0; + VISIT_IN_BLOCK(st, arguments, e->v.Lambda.args, (void*)e); + VISIT_IN_BLOCK(st, expr, e->v.Lambda.body, (void*)e); if (!symtable_exit_block(st, (void *)e)) - VISIT_QUIT(st, 0); + return 0; break; } case IfExp_kind: @@ -1575,40 +1222,39 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.IfExp.orelse); break; case Dict_kind: - VISIT_SEQ_WITH_NULL(st, expr, e->v.Dict.keys); + VISIT_SEQ(st, expr, e->v.Dict.keys); VISIT_SEQ(st, expr, e->v.Dict.values); break; case Set_kind: VISIT_SEQ(st, expr, e->v.Set.elts); break; - case GeneratorExp_kind: - if (!symtable_visit_genexp(st, e)) - VISIT_QUIT(st, 0); - break; case ListComp_kind: if (!symtable_visit_listcomp(st, e)) - VISIT_QUIT(st, 0); + return 0; + break; + case GeneratorExp_kind: + if (!symtable_visit_genexp(st, e)) + return 0; break; case SetComp_kind: if (!symtable_visit_setcomp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case DictComp_kind: if (!symtable_visit_dictcomp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case Yield_kind: if (e->v.Yield.value) VISIT(st, expr, e->v.Yield.value); st->st_cur->ste_generator = 1; - break; - case YieldFrom_kind: - VISIT(st, expr, e->v.YieldFrom.value); - st->st_cur->ste_generator = 1; - break; - case Await_kind: - VISIT(st, expr, e->v.Await.value); - st->st_cur->ste_coroutine = 1; + if (st->st_cur->ste_returns_value) { + PyErr_SetString(PyExc_SyntaxError, + RETURN_VAL_IN_GENERATOR); + PyErr_SyntaxLocation(st->st_filename, + e->lineno); + return 0; + } break; case Compare_kind: VISIT(st, expr, e->v.Compare.left); @@ -1617,17 +1263,17 @@ symtable_visit_expr(struct symtable *st, expr_ty e) case Call_kind: VISIT(st, expr, e->v.Call.func); VISIT_SEQ(st, expr, e->v.Call.args); - VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords); - break; - case FormattedValue_kind: - VISIT(st, expr, e->v.FormattedValue.value); - if (e->v.FormattedValue.format_spec) - VISIT(st, expr, e->v.FormattedValue.format_spec); + VISIT_SEQ(st, keyword, e->v.Call.keywords); + if (e->v.Call.starargs) + VISIT(st, expr, e->v.Call.starargs); + if (e->v.Call.kwargs) + VISIT(st, expr, e->v.Call.kwargs); break; - case JoinedStr_kind: - VISIT_SEQ(st, expr, e->v.JoinedStr.values); + case Repr_kind: + VISIT(st, expr, e->v.Repr.value); break; - case Constant_kind: + case Num_kind: + case Str_kind: /* Nothing to do here. */ break; /* The following exprs can be assignment targets. */ @@ -1638,21 +1284,10 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.Subscript.value); VISIT(st, slice, e->v.Subscript.slice); break; - case Starred_kind: - VISIT(st, expr, e->v.Starred.value); - break; case Name_kind: if (!symtable_add_def(st, e->v.Name.id, e->v.Name.ctx == Load ? USE : DEF_LOCAL)) - VISIT_QUIT(st, 0); - /* Special-case super: it counts as a use of __class__ */ - if (e->v.Name.ctx == Load && - st->st_cur->ste_type == FunctionBlock && - _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) { - if (!GET_IDENTIFIER(__class__) || - !symtable_add_def(st, __class__, USE)) - VISIT_QUIT(st, 0); - } + return 0; break; /* child nodes of List and Tuple will have expr_context set */ case List_kind: @@ -1662,13 +1297,13 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Tuple.elts); break; } - VISIT_QUIT(st, 1); + return 1; } static int symtable_implicit_arg(struct symtable *st, int pos) { - PyObject *id = PyUnicode_FromFormat(".%d", pos); + PyObject *id = PyString_FromFormat(".%d", pos); if (id == NULL) return 0; if (!symtable_add_def(st, id, DEF_PARAM)) { @@ -1680,16 +1315,37 @@ symtable_implicit_arg(struct symtable *st, int pos) } static int -symtable_visit_params(struct symtable *st, asdl_seq *args) +symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel) { int i; - if (!args) - return -1; - + /* go through all the toplevel arguments first */ for (i = 0; i < asdl_seq_LEN(args); i++) { - arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - if (!symtable_add_def(st, arg->arg, DEF_PARAM)) + expr_ty arg = (expr_ty)asdl_seq_GET(args, i); + if (arg->kind == Name_kind) { + assert(arg->v.Name.ctx == Param || + (arg->v.Name.ctx == Store && !toplevel)); + if (!symtable_add_def(st, arg->v.Name.id, DEF_PARAM)) + return 0; + } + else if (arg->kind == Tuple_kind) { + assert(arg->v.Tuple.ctx == Store); + if (toplevel) { + if (!symtable_implicit_arg(st, i)) + return 0; + } + } + else { + PyErr_SetString(PyExc_SyntaxError, + "invalid expression in parameter list"); + PyErr_SyntaxLocation(st->st_filename, + st->st_cur->ste_lineno); + return 0; + } + } + + if (!toplevel) { + if (!symtable_visit_params_nested(st, args)) return 0; } @@ -1697,61 +1353,39 @@ symtable_visit_params(struct symtable *st, asdl_seq *args) } static int -symtable_visit_argannotations(struct symtable *st, asdl_seq *args) +symtable_visit_params_nested(struct symtable *st, asdl_seq *args) { int i; - - if (!args) - return -1; - for (i = 0; i < asdl_seq_LEN(args); i++) { - arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - if (arg->annotation) - VISIT(st, expr, arg->annotation); + expr_ty arg = (expr_ty)asdl_seq_GET(args, i); + if (arg->kind == Tuple_kind && + !symtable_visit_params(st, arg->v.Tuple.elts, 0)) + return 0; } return 1; } static int -symtable_visit_annotations(struct symtable *st, stmt_ty s, - arguments_ty a, expr_ty returns) -{ - if (a->args && !symtable_visit_argannotations(st, a->args)) - return 0; - if (a->vararg && a->vararg->annotation) - VISIT(st, expr, a->vararg->annotation); - if (a->kwarg && a->kwarg->annotation) - VISIT(st, expr, a->kwarg->annotation); - if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) - return 0; - if (returns) - VISIT(st, expr, returns); - return 1; -} - -static int symtable_visit_arguments(struct symtable *st, arguments_ty a) { /* skip default arguments inside function block XXX should ast be different? */ - if (a->posonlyargs && !symtable_visit_params(st, a->posonlyargs)) - return 0; - if (a->args && !symtable_visit_params(st, a->args)) - return 0; - if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs)) + if (a->args && !symtable_visit_params(st, a->args, 1)) return 0; if (a->vararg) { - if (!symtable_add_def(st, a->vararg->arg, DEF_PARAM)) + if (!symtable_add_def(st, a->vararg, DEF_PARAM)) return 0; st->st_cur->ste_varargs = 1; } if (a->kwarg) { - if (!symtable_add_def(st, a->kwarg->arg, DEF_PARAM)) + if (!symtable_add_def(st, a->kwarg, DEF_PARAM)) return 0; st->st_cur->ste_varkeywords = 1; } + if (a->args && !symtable_visit_params_nested(st, a->args)) + return 0; return 1; } @@ -1762,22 +1396,11 @@ symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh) if (eh->v.ExceptHandler.type) VISIT(st, expr, eh->v.ExceptHandler.type); if (eh->v.ExceptHandler.name) - if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL)) - return 0; + VISIT(st, expr, eh->v.ExceptHandler.name); VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body); return 1; } -static int -symtable_visit_withitem(struct symtable *st, withitem_ty item) -{ - VISIT(st, expr, item->context_expr); - if (item->optional_vars) { - VISIT(st, expr, item->optional_vars); - } - return 1; -} - static int symtable_visit_alias(struct symtable *st, alias_ty a) @@ -1788,10 +1411,10 @@ symtable_visit_alias(struct symtable *st, alias_ty a) */ PyObject *store_name; PyObject *name = (a->asname == NULL) ? a->name : a->asname; - Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, - PyUnicode_GET_LENGTH(name), 1); - if (dot != -1) { - store_name = PyUnicode_Substring(name, 0, dot); + const char *base = PyString_AS_STRING(name); + char *dot = strchr(base, '.'); + if (dot) { + store_name = PyString_FromStringAndSize(base, dot - base); if (!store_name) return 0; } @@ -1799,20 +1422,19 @@ symtable_visit_alias(struct symtable *st, alias_ty a) store_name = name; Py_INCREF(store_name); } - if (!_PyUnicode_EqualToASCIIString(name, "*")) { + if (strcmp(PyString_AS_STRING(name), "*")) { int r = symtable_add_def(st, store_name, DEF_IMPORT); Py_DECREF(store_name); return r; } else { - if (st->st_cur->ste_type != ModuleBlock) { - int lineno = st->st_cur->ste_lineno; - int col_offset = st->st_cur->ste_col_offset; - PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING); - PyErr_SyntaxLocationObject(st->st_filename, lineno, col_offset + 1); + if (st->st_cur->ste_type != ModuleBlock && + !symtable_warn(st, PyExc_SyntaxWarning, IMPORT_STAR_WARNING, -1)) + { Py_DECREF(store_name); return 0; } + st->st_cur->ste_unoptimized |= OPT_IMPORT_STAR; Py_DECREF(store_name); return 1; } @@ -1822,16 +1444,9 @@ symtable_visit_alias(struct symtable *st, alias_ty a) static int symtable_visit_comprehension(struct symtable *st, comprehension_ty lc) { - st->st_cur->ste_comp_iter_target = 1; VISIT(st, expr, lc->target); - st->st_cur->ste_comp_iter_target = 0; - st->st_cur->ste_comp_iter_expr++; VISIT(st, expr, lc->iter); - st->st_cur->ste_comp_iter_expr--; VISIT_SEQ(st, expr, lc->ifs); - if (lc->is_async) { - st->st_cur->ste_coroutine = 1; - } return 1; } @@ -1862,78 +1477,121 @@ symtable_visit_slice(struct symtable *st, slice_ty s) case Index_kind: VISIT(st, expr, s->v.Index.value) break; + case Ellipsis_kind: + break; } return 1; } static int +symtable_new_tmpname(struct symtable *st) +{ + char tmpname[256]; + identifier tmp; + + PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", + ++st->st_cur->ste_tmpname); + tmp = PyString_InternFromString(tmpname); + if (!tmp) + return 0; + if (!symtable_add_def(st, tmp, DEF_LOCAL)) + return 0; + Py_DECREF(tmp); + return 1; +} + +static int symtable_handle_comprehension(struct symtable *st, expr_ty e, identifier scope_name, asdl_seq *generators, expr_ty elt, expr_ty value) { int is_generator = (e->kind == GeneratorExp_kind); + int needs_tmp = !is_generator; comprehension_ty outermost = ((comprehension_ty) asdl_seq_GET(generators, 0)); /* Outermost iterator is evaluated in current scope */ - st->st_cur->ste_comp_iter_expr++; VISIT(st, expr, outermost->iter); - st->st_cur->ste_comp_iter_expr--; /* Create comprehension scope for the rest */ if (!scope_name || - !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, - e->lineno, e->col_offset)) { + !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) { return 0; } - if (outermost->is_async) { - st->st_cur->ste_coroutine = 1; - } - st->st_cur->ste_comprehension = 1; - + /* In order to check for yield expressions under '-3', we clear + the generator flag, and restore it at the end */ + is_generator |= st->st_cur->ste_generator; + st->st_cur->ste_generator = 0; /* Outermost iter is received as an argument */ if (!symtable_implicit_arg(st, 0)) { symtable_exit_block(st, (void *)e); return 0; } - /* Visit iteration variable target, and mark them as such */ - st->st_cur->ste_comp_iter_target = 1; - VISIT(st, expr, outermost->target); - st->st_cur->ste_comp_iter_target = 0; - /* Visit the rest of the comprehension body */ - VISIT_SEQ(st, expr, outermost->ifs); - VISIT_SEQ_TAIL(st, comprehension, generators, 1); + /* Allocate temporary name if needed */ + if (needs_tmp && !symtable_new_tmpname(st)) { + symtable_exit_block(st, (void *)e); + return 0; + } + VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e); + VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e); + VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension, + generators, 1, (void*)e); if (value) - VISIT(st, expr, value); - VISIT(st, expr, elt); - if (st->st_cur->ste_generator) { - PyErr_SetString(PyExc_SyntaxError, - (e->kind == ListComp_kind) ? "'yield' inside list comprehension" : + VISIT_IN_BLOCK(st, expr, value, (void*)e); + VISIT_IN_BLOCK(st, expr, elt, (void*)e); + if (Py_Py3kWarningFlag && st->st_cur->ste_generator) { + const char *msg = ( (e->kind == SetComp_kind) ? "'yield' inside set comprehension" : (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" : "'yield' inside generator expression"); - PyErr_SyntaxLocationObject(st->st_filename, - st->st_cur->ste_lineno, - st->st_cur->ste_col_offset + 1); - symtable_exit_block(st, (void *)e); - return 0; + if (!symtable_warn(st, PyExc_DeprecationWarning, msg, -1)) { + symtable_exit_block(st, (void *)e); + return 0; + } } - st->st_cur->ste_generator = is_generator; + st->st_cur->ste_generator |= is_generator; return symtable_exit_block(st, (void *)e); } static int -symtable_visit_genexp(struct symtable *st, expr_ty e) +symtable_visit_listcomp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr), - e->v.GeneratorExp.generators, - e->v.GeneratorExp.elt, NULL); + asdl_seq *generators = e->v.ListComp.generators; + int i, is_generator; + /* In order to check for yield expressions under '-3', we clear + the generator flag, and restore it at the end */ + is_generator = st->st_cur->ste_generator; + st->st_cur->ste_generator = 0; + VISIT(st, expr, e->v.ListComp.elt); + for (i = 0; i < asdl_seq_LEN(generators); i++) { + comprehension_ty lc = (comprehension_ty)asdl_seq_GET(generators, i); + VISIT(st, expr, lc->target); + if (i == 0 && !st->st_cur->ste_generator) { + /* 'yield' in the outermost iterator doesn't cause a warning */ + VISIT(st, expr, lc->iter); + is_generator |= st->st_cur->ste_generator; + st->st_cur->ste_generator = 0; + } + else { + VISIT(st, expr, lc->iter); + } + VISIT_SEQ(st, expr, lc->ifs); + } + + if (Py_Py3kWarningFlag && st->st_cur->ste_generator) { + const char *msg = "'yield' inside list comprehension"; + if (!symtable_warn(st, PyExc_DeprecationWarning, msg, -1)) { + return 0; + } + } + st->st_cur->ste_generator |= is_generator; + return 1; } static int -symtable_visit_listcomp(struct symtable *st, expr_ty e) +symtable_visit_genexp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, GET_IDENTIFIER(listcomp), - e->v.ListComp.generators, - e->v.ListComp.elt, NULL); + return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr), + e->v.GeneratorExp.generators, + e->v.GeneratorExp.elt, NULL); } static int |