diff options
Diffstat (limited to 'Python/symtable.c')
| -rw-r--r-- | Python/symtable.c | 976 |
1 files changed, 559 insertions, 417 deletions
diff --git a/Python/symtable.c b/Python/symtable.c index 644d9c5083..55898f99bf 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -8,18 +8,20 @@ #define GLOBAL_AFTER_ASSIGN \ "name '%.400s' is assigned to before global declaration" +#define NONLOCAL_AFTER_ASSIGN \ +"name '%.400s' is assigned to before nonlocal declaration" + #define GLOBAL_AFTER_USE \ "name '%.400s' is used prior to global declaration" -#define IMPORT_STAR_WARNING "import * only allowed at module level" - -#define RETURN_VAL_IN_GENERATOR \ - "'return' with argument inside generator" +#define NONLOCAL_AFTER_USE \ +"name '%.400s' is used prior to nonlocal declaration" +#define IMPORT_STAR_WARNING "import * only allowed at module level" static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, - void *key, int lineno) + void *key, int lineno, int col_offset) { PySTEntryObject *ste = NULL; PyObject *k = NULL; @@ -61,8 +63,10 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_varargs = 0; ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; + ste->ste_opt_col_offset = 0; ste->ste_tmpname = 0; ste->ste_lineno = lineno; + ste->ste_col_offset = col_offset; if (st->st_cur != NULL && (st->st_cur->ste_nested || @@ -72,7 +76,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_generator = 0; ste->ste_returns_value = 0; - if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0) + if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0) goto fail; return ste; @@ -84,13 +88,9 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, static PyObject * ste_repr(PySTEntryObject *ste) { - char buf[256]; - - PyOS_snprintf(buf, sizeof(buf), - "<symtable entry %.100s(%ld), line %d>", - PyString_AS_STRING(ste->ste_name), - PyInt_AS_LONG(ste->ste_id), ste->ste_lineno); - return PyString_FromString(buf); + return PyUnicode_FromFormat("<symtable entry %U(%ld), line %d>", + ste->ste_name, + PyLong_AS_LONG(ste->ste_id), ste->ste_lineno); } static void @@ -129,7 +129,7 @@ PyTypeObject PySTEntry_Type = { 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ - 0, /* tp_compare */ + 0, /* tp_reserved */ (reprfunc)ste_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ @@ -164,32 +164,37 @@ PyTypeObject PySTEntry_Type = { static int symtable_analyze(struct symtable *st); static int symtable_warn(struct symtable *st, char *msg, int lineno); static int symtable_enter_block(struct symtable *st, identifier name, - _Py_block_ty block, void *ast, int lineno); + _Py_block_ty block, void *ast, int lineno, + int col_offset); static int symtable_exit_block(struct symtable *st, void *ast); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s); -static int symtable_visit_setcomp(struct symtable *st, expr_ty e); -static int symtable_visit_dictcomp(struct symtable *st, expr_ty e); +static int symtable_visit_listcomp(struct symtable *st, expr_ty s); +static int symtable_visit_setcomp(struct symtable *st, expr_ty s); +static int symtable_visit_dictcomp(struct symtable *st, expr_ty s); static int symtable_visit_arguments(struct symtable *st, arguments_ty); static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_slice(struct symtable *st, slice_ty); -static int symtable_visit_params(struct symtable *st, asdl_seq *args, int top); -static int symtable_visit_params_nested(struct symtable *st, asdl_seq *args); +static int symtable_visit_params(struct symtable *st, asdl_seq *args); +static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); +static int symtable_visit_annotations(struct symtable *st, stmt_ty s); +static int symtable_visit_withitem(struct symtable *st, withitem_ty item); -static identifier top = NULL, lambda = NULL, genexpr = NULL, setcomp = NULL, - dictcomp = NULL; +static identifier top = NULL, lambda = NULL, genexpr = NULL, + listcomp = NULL, setcomp = NULL, dictcomp = NULL, + __class__ = NULL, __locals__ = NULL; #define GET_IDENTIFIER(VAR) \ - ((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR))) + ((VAR) ? (VAR) : ((VAR) = PyUnicode_InternFromString(# VAR))) #define DUPLICATE_ARGUMENT \ -"duplicate argument '%s' in function definition" +"duplicate argument '%U' in function definition" static struct symtable * symtable_new(void) @@ -201,11 +206,11 @@ symtable_new(void) return NULL; st->st_filename = NULL; - st->st_symbols = NULL; + st->st_blocks = NULL; if ((st->st_stack = PyList_New(0)) == NULL) goto fail; - if ((st->st_symbols = PyDict_New()) == NULL) + if ((st->st_blocks = PyDict_New()) == NULL) goto fail; st->st_cur = NULL; st->st_private = NULL; @@ -215,26 +220,49 @@ symtable_new(void) return NULL; } +/* When compiling the use of C stack is probably going to be a lot + lighter than when executing Python code but still can overflow + and causing a Python crash if not checked (e.g. eval("()"*300000)). + Using the current recursion limit for the compiler seems too + restrictive (it caused at least one test to fail) so a factor is + used to allow deeper recursion when compiling an expression. + + Using a scaling factor means this should automatically adjust when + the recursion limit is adjusted for small or large C stack allocations. +*/ +#define COMPILER_STACK_FRAME_SCALE 3 + struct symtable * PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) { struct symtable *st = symtable_new(); asdl_seq *seq; int i; + PyThreadState *tstate; if (st == NULL) return st; st->st_filename = filename; st->st_future = future; + + /* Setup recursion depth check counters */ + tstate = PyThreadState_GET(); + if (!tstate) { + PySymtable_Free(st); + return NULL; + } + st->recursion_depth = tstate->recursion_depth * COMPILER_STACK_FRAME_SCALE; + st->recursion_limit = Py_GetRecursionLimit() * COMPILER_STACK_FRAME_SCALE; + + /* Make the initial symbol information gathering pass */ if (!GET_IDENTIFIER(top) || - !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0)) { + !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0, 0)) { PySymtable_Free(st); return NULL; } st->st_top = st->st_cur; st->st_cur->ste_unoptimized = OPT_TOPLEVEL; - /* Any other top-level initialization? */ switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; @@ -263,6 +291,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) PySymtable_Free(st); return NULL; } + /* Make the second symbol analysis pass */ if (symtable_analyze(st)) return st; PySymtable_Free(st); @@ -276,7 +305,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) void PySymtable_Free(struct symtable *st) { - Py_XDECREF(st->st_symbols); + Py_XDECREF(st->st_blocks); Py_XDECREF(st->st_stack); PyMem_Free((void *)st); } @@ -289,7 +318,7 @@ PySymtable_Lookup(struct symtable *st, void *key) k = PyLong_FromVoidPtr(key); if (k == NULL) return NULL; - v = PyDict_GetItem(st->st_symbols, k); + v = PyDict_GetItem(st->st_blocks, k); if (v) { assert(PySTEntry_Check(v)); Py_INCREF(v); @@ -309,19 +338,19 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name) PyObject *v = PyDict_GetItem(ste->ste_symbols, name); if (!v) return 0; - assert(PyInt_Check(v)); - return (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; + assert(PyLong_Check(v)); + return (PyLong_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK; } /* Analyze raw symbol information to determine scope of each name. - The next several functions are helpers for PySymtable_Analyze(), + The next several functions are helpers for symtable_analyze(), which determines whether a name is local, global, or free. In addition, it determines which local variables are cell variables; they provide bindings that are used for free variables in enclosed blocks. - There are also two kinds of free variables, implicit and explicit. An + There are also two kinds of global variables, implicit and explicit. An explicit global is declared with the global statement. An implicit global is a free variable for which the compiler has found no binding in an enclosing function scope. The implicit global is either a global @@ -331,28 +360,36 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name) is treated as a local. The symbol table requires two passes to determine the scope of each name. - The first pass collects raw facts from the AST: the name is a parameter - here, the name is used by not defined here, etc. The second pass analyzes - these facts during a pass over the PySTEntryObjects created during pass 1. + The first pass collects raw facts from the AST via the symtable_visit_* + functions: the name is a parameter here, the name is used but not defined + here, etc. The second pass analyzes these facts during a pass over the + PySTEntryObjects created during pass 1. When a function is entered during the second pass, the parent passes the set of all name bindings visible to its children. These bindings - are used to determine if the variable is free or an implicit global. - After doing the local analysis, it analyzes each of its child blocks - using an updated set of name bindings. - - The children update the free variable set. If a local variable is free - in a child, the variable is marked as a cell. The current function must - provide runtime storage for the variable that may outlive the function's - frame. Cell variables are removed from the free set before the analyze - function returns to its parent. - - The sets of bound and free variables are implemented as dictionaries - mapping strings to None. + are used to determine if non-local variables are free or implicit globals. + Names which are explicitly declared nonlocal must exist in this set of + visible names - if they do not, a syntax error is raised. After doing + the local analysis, it analyzes each of its child blocks using an + updated set of name bindings. + + The children update the free variable set. If a local variable is added to + the free variable set by the child, the variable is marked as a cell. The + function object being defined must provide runtime storage for the variable + that may outlive the function's frame. Cell variables are removed from the + free set before the analyze function returns to its parent. + + During analysis, the names are: + symbols: dict mapping from symbol names to flag values (including offset scope values) + scopes: dict mapping from symbol names to scope values (no offset) + local: set of all symbol names local to the current scope + bound: set of all symbol names local to a containing function scope + free: set of all symbol names referenced but not bound in child scopes + global: set of all symbol names explicitly declared as global */ #define SET_SCOPE(DICT, NAME, I) { \ - PyObject *o = PyInt_FromLong(I); \ + PyObject *o = PyLong_FromLong(I); \ if (!o) \ return 0; \ if (PyDict_SetItem((DICT), (NAME), o) < 0) { \ @@ -370,37 +407,62 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name) */ static int -analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags, +analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, PyObject *bound, PyObject *local, PyObject *free, PyObject *global) { if (flags & DEF_GLOBAL) { if (flags & DEF_PARAM) { PyErr_Format(PyExc_SyntaxError, - "name '%s' is local and global", - PyString_AS_STRING(name)); - PyErr_SyntaxLocation(ste->ste_table->st_filename, - ste->ste_lineno); + "name '%U' is parameter and global", + name); + PyErr_SyntaxLocationEx(ste->ste_table->st_filename, + ste->ste_lineno, ste->ste_col_offset); return 0; } - SET_SCOPE(dict, name, GLOBAL_EXPLICIT); - if (PyDict_SetItem(global, name, Py_None) < 0) + if (flags & DEF_NONLOCAL) { + PyErr_Format(PyExc_SyntaxError, + "name '%U' is nonlocal and global", + name); return 0; - if (bound && PyDict_GetItem(bound, name)) { - if (PyDict_DelItem(bound, name) < 0) - return 0; } + SET_SCOPE(scopes, name, GLOBAL_EXPLICIT); + if (PySet_Add(global, name) < 0) + return 0; + if (bound && (PySet_Discard(bound, name) < 0)) + return 0; return 1; } - if (flags & DEF_BOUND) { - SET_SCOPE(dict, name, LOCAL); - if (PyDict_SetItem(local, name, Py_None) < 0) + if (flags & DEF_NONLOCAL) { + if (flags & DEF_PARAM) { + PyErr_Format(PyExc_SyntaxError, + "name '%U' is parameter and nonlocal", + name); + return 0; + } + if (!bound) { + PyErr_Format(PyExc_SyntaxError, + "nonlocal declaration not allowed at module level"); return 0; - if (PyDict_GetItem(global, name)) { - if (PyDict_DelItem(global, name) < 0) - return 0; } + if (!PySet_Contains(bound, name)) { + PyErr_Format(PyExc_SyntaxError, + "no binding for nonlocal '%U' found", + name); + + return 0; + } + SET_SCOPE(scopes, name, FREE); + ste->ste_free = 1; + return PySet_Add(free, name) >= 0; + } + if (flags & DEF_BOUND) { + SET_SCOPE(scopes, name, LOCAL); + if (PySet_Add(local, name) < 0) + return 0; + if (PySet_Discard(global, name) < 0) + return 0; return 1; } /* If an enclosing block has a binding for this name, it @@ -408,30 +470,22 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags, Note that having a non-NULL bound implies that the block is nested. */ - if (bound && PyDict_GetItem(bound, name)) { - SET_SCOPE(dict, name, FREE); + if (bound && PySet_Contains(bound, name)) { + SET_SCOPE(scopes, name, FREE); ste->ste_free = 1; - if (PyDict_SetItem(free, name, Py_None) < 0) - return 0; - return 1; + return PySet_Add(free, name) >= 0; } /* If a parent has a global statement, then call it global explicit? It could also be global implicit. */ - else if (global && PyDict_GetItem(global, name)) { - SET_SCOPE(dict, name, GLOBAL_IMPLICIT); - return 1; - } - else { - if (ste->ste_nested) - ste->ste_free = 1; - SET_SCOPE(dict, name, GLOBAL_IMPLICIT); + if (global && PySet_Contains(global, name)) { + SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); return 1; } - /* Should never get here. */ - PyErr_Format(PyExc_SystemError, "failed to set scope for %s", - PyString_AS_STRING(name)); - return 0; + if (ste->ste_nested) + ste->ste_free = 1; + SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); + return 1; } #undef SET_SCOPE @@ -442,45 +496,50 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags, Note that the current block's free variables are included in free. That's safe because no name can be free and local in the same scope. + + The 'restricted' argument may be set to a string to restrict the analysis + to the one variable whose name equals that string (e.g. "__class__"). */ static int -analyze_cells(PyObject *scope, PyObject *free) +analyze_cells(PyObject *scopes, PyObject *free, const char *restricted) { - PyObject *name, *v, *w; + PyObject *name, *v, *v_cell; int success = 0; Py_ssize_t pos = 0; - w = PyInt_FromLong(CELL); - if (!w) + v_cell = PyLong_FromLong(CELL); + if (!v_cell) return 0; - while (PyDict_Next(scope, &pos, &name, &v)) { - long flags; - assert(PyInt_Check(v)); - flags = PyInt_AS_LONG(v); - if (flags != LOCAL) + while (PyDict_Next(scopes, &pos, &name, &v)) { + long scope; + assert(PyLong_Check(v)); + scope = PyLong_AS_LONG(v); + if (scope != LOCAL) continue; - if (!PyDict_GetItem(free, name)) + if (!PySet_Contains(free, name)) + continue; + if (restricted != NULL && + PyUnicode_CompareWithASCIIString(name, restricted)) continue; /* Replace LOCAL with CELL for this name, and remove from free. It is safe to replace the value of name in the dict, because it will not cause a resize. */ - if (PyDict_SetItem(scope, name, w) < 0) + if (PyDict_SetItem(scopes, name, v_cell) < 0) goto error; - if (!PyDict_DelItem(free, name) < 0) + if (PySet_Discard(free, name) < 0) goto error; } success = 1; error: - Py_DECREF(w); + Py_DECREF(v_cell); return success; } /* Check for illegal statements in unoptimized namespaces */ static int check_unoptimized(const PySTEntryObject* ste) { - char buf[300]; const char* trailer; if (ste->ste_type != FunctionBlock || !ste->ste_unoptimized @@ -492,106 +551,105 @@ check_unoptimized(const PySTEntryObject* ste) { "is a nested function"); switch (ste->ste_unoptimized) { - case OPT_TOPLEVEL: /* exec / import * at top-level is fine */ - case OPT_EXEC: /* qualified exec is fine */ + case OPT_TOPLEVEL: /* import * at top-level is fine */ return 1; case OPT_IMPORT_STAR: - PyOS_snprintf(buf, sizeof(buf), - "import * is not allowed in function '%.100s' " - "because it %s", - PyString_AS_STRING(ste->ste_name), trailer); - break; - case OPT_BARE_EXEC: - PyOS_snprintf(buf, sizeof(buf), - "unqualified exec is not allowed in function " - "'%.100s' it %s", - PyString_AS_STRING(ste->ste_name), trailer); - break; - default: - PyOS_snprintf(buf, sizeof(buf), - "function '%.100s' uses import * and bare exec, " - "which are illegal because it %s", - PyString_AS_STRING(ste->ste_name), trailer); + PyErr_Format(PyExc_SyntaxError, + "import * is not allowed in function '%U' because it %s", + ste->ste_name, trailer); break; } - PyErr_SetString(PyExc_SyntaxError, buf); - PyErr_SyntaxLocation(ste->ste_table->st_filename, - ste->ste_opt_lineno); + PyErr_SyntaxLocationEx(ste->ste_table->st_filename, ste->ste_opt_lineno, + ste->ste_opt_col_offset); return 0; } -/* Enter the final scope information into the st_symbols dict. +/* Enter the final scope information into the ste_symbols dict. * * All arguments are dicts. Modifies symbols, others are read-only. */ static int -update_symbols(PyObject *symbols, PyObject *scope, +update_symbols(PyObject *symbols, PyObject *scopes, PyObject *bound, PyObject *free, int classflag) { - PyObject *name, *v, *u, *w, *free_value = NULL; + PyObject *name = NULL, *itr = NULL; + PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL; Py_ssize_t pos = 0; + /* Update scope information for all symbols in this scope */ while (PyDict_Next(symbols, &pos, &name, &v)) { - long i, flags; - assert(PyInt_Check(v)); - flags = PyInt_AS_LONG(v); - w = PyDict_GetItem(scope, name); - assert(w && PyInt_Check(w)); - i = PyInt_AS_LONG(w); - flags |= (i << SCOPE_OFF); - u = PyInt_FromLong(flags); - if (!u) + long scope, flags; + assert(PyLong_Check(v)); + flags = PyLong_AS_LONG(v); + v_scope = PyDict_GetItem(scopes, name); + assert(v_scope && PyLong_Check(v_scope)); + scope = PyLong_AS_LONG(v_scope); + flags |= (scope << SCOPE_OFFSET); + v_new = PyLong_FromLong(flags); + if (!v_new) return 0; - if (PyDict_SetItem(symbols, name, u) < 0) { - Py_DECREF(u); + if (PyDict_SetItem(symbols, name, v_new) < 0) { + Py_DECREF(v_new); return 0; } - Py_DECREF(u); + Py_DECREF(v_new); } - free_value = PyInt_FromLong(FREE << SCOPE_OFF); - if (!free_value) + /* Record not yet resolved free variables from children (if any) */ + v_free = PyLong_FromLong(FREE << SCOPE_OFFSET); + if (!v_free) return 0; - /* add a free variable when it's only use is for creating a closure */ - pos = 0; - while (PyDict_Next(free, &pos, &name, &v)) { - PyObject *o = PyDict_GetItem(symbols, name); + itr = PyObject_GetIter(free); + if (!itr) + goto error; + + while ((name = PyIter_Next(itr))) { + v = PyDict_GetItem(symbols, name); - if (o) { - /* It could be a free variable in a method of + /* Handle symbol that already exists in this scope */ + if (v) { + /* Handle a free variable in a method of the class that has the same name as a local or global in the class scope. */ if (classflag && - PyInt_AS_LONG(o) & (DEF_BOUND | DEF_GLOBAL)) { - long i = PyInt_AS_LONG(o) | DEF_FREE_CLASS; - o = PyInt_FromLong(i); - if (!o) { - Py_DECREF(free_value); - return 0; + PyLong_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) { + long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS; + v_new = PyLong_FromLong(flags); + if (!v_new) { + goto error; } - if (PyDict_SetItem(symbols, name, o) < 0) { - Py_DECREF(o); - Py_DECREF(free_value); - return 0; + if (PyDict_SetItem(symbols, name, v_new) < 0) { + Py_DECREF(v_new); + goto error; } - Py_DECREF(o); + Py_DECREF(v_new); } - /* else it's not free, probably a cell */ + /* It's a cell, or already free in this scope */ + Py_DECREF(name); continue; } - if (!PyDict_GetItem(bound, name)) + /* Handle global symbol */ + if (!PySet_Contains(bound, name)) { + Py_DECREF(name); continue; /* it's a global */ - - if (PyDict_SetItem(symbols, name, free_value) < 0) { - Py_DECREF(free_value); - return 0; } + /* Propagate new free symbol up the lexical stack */ + if (PyDict_SetItem(symbols, name, v_free) < 0) { + goto error; + } + Py_DECREF(name); } - Py_DECREF(free_value); + Py_DECREF(itr); + Py_DECREF(v_free); return 1; +error: + Py_XDECREF(v_free); + Py_XDECREF(itr); + Py_XDECREF(name); + return 0; } /* Make final symbol table decisions for block of ste. @@ -622,17 +680,17 @@ static int analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PyObject *global) { - PyObject *name, *v, *local = NULL, *scope = NULL; - PyObject *newbound = NULL, *newglobal = NULL; - PyObject *newfree = NULL, *allfree = NULL; + PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; + PyObject *newglobal = NULL, *newfree = NULL, *allfree = NULL; + PyObject *temp; int i, success = 0; Py_ssize_t pos = 0; - local = PyDict_New(); /* collect new names bound in block */ + local = PySet_New(NULL); /* collect new names bound in block */ if (!local) goto error; - scope = PyDict_New(); /* collect scopes defined for each name */ - if (!scope) + scopes = PyDict_New(); /* collect scopes defined for each name */ + if (!scopes) goto error; /* Allocate new global and bound variable dictionaries. These @@ -646,51 +704,81 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, /* TODO(jhylton): Package these dicts in a struct so that we can write reasonable helper functions? */ - newglobal = PyDict_New(); + newglobal = PySet_New(NULL); if (!newglobal) goto error; - newbound = PyDict_New(); - if (!newbound) - goto error; - newfree = PyDict_New(); + newfree = PySet_New(NULL); if (!newfree) goto error; + newbound = PySet_New(NULL); + if (!newbound) + goto error; + /* Class namespace has no effect on names visible in + nested functions, so populate the global and bound + sets to be passed to child blocks before analyzing + this one. + */ if (ste->ste_type == ClassBlock) { - if (PyDict_Update(newglobal, global) < 0) + /* Pass down known globals */ + temp = PyNumber_InPlaceOr(newglobal, global); + if (!temp) goto error; - if (bound) - if (PyDict_Update(newbound, bound) < 0) + Py_DECREF(temp); + /* Pass down previously bound symbols */ + if (bound) { + temp = PyNumber_InPlaceOr(newbound, bound); + if (!temp) goto error; + Py_DECREF(temp); + } } while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { - long flags = PyInt_AS_LONG(v); - if (!analyze_name(ste, scope, name, flags, + long flags = PyLong_AS_LONG(v); + if (!analyze_name(ste, scopes, name, flags, bound, local, free, global)) goto error; } + /* Populate global and bound sets to be passed to children. */ if (ste->ste_type != ClassBlock) { + /* Add function locals to bound set */ if (ste->ste_type == FunctionBlock) { - if (PyDict_Update(newbound, local) < 0) + temp = PyNumber_InPlaceOr(newbound, local); + if (!temp) goto error; + Py_DECREF(temp); } + /* Pass down previously bound symbols */ if (bound) { - if (PyDict_Update(newbound, bound) < 0) + temp = PyNumber_InPlaceOr(newbound, bound); + if (!temp) goto error; + Py_DECREF(temp); } - if (PyDict_Update(newglobal, global) < 0) + /* Pass down known globals */ + temp = PyNumber_InPlaceOr(newglobal, global); + if (!temp) + goto error; + Py_DECREF(temp); + } + else { + /* Special-case __class__ */ + if (!GET_IDENTIFIER(__class__)) + goto error; + assert(PySet_Contains(local, __class__) == 1); + if (PySet_Add(newbound, __class__) < 0) goto error; } - /* Recursively call analyze_block() on each child block. + /* Recursively call analyze_child_block() on each child block. newbound, newglobal now contain the names visible in nested blocks. The free variables in the children will be collected in allfree. */ - allfree = PyDict_New(); + allfree = PySet_New(NULL); if (!allfree) goto error; for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) { @@ -701,26 +789,38 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, if (!analyze_child_block(entry, newbound, newfree, newglobal, allfree)) goto error; + /* Check if any children have free variables */ if (entry->ste_free || entry->ste_child_free) ste->ste_child_free = 1; } - if (PyDict_Update(newfree, allfree) < 0) + temp = PyNumber_InPlaceOr(newfree, allfree); + if (!temp) goto error; - if (ste->ste_type == FunctionBlock && !analyze_cells(scope, newfree)) + Py_DECREF(temp); + + /* Check if any local variables must be converted to cell variables */ + if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree, + NULL)) + goto error; + else if (ste->ste_type == ClassBlock && !analyze_cells(scopes, newfree, + "__class__")) goto error; - if (!update_symbols(ste->ste_symbols, scope, bound, newfree, + /* Records the results of the analysis in the symbol table entry */ + if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, ste->ste_type == ClassBlock)) goto error; if (!check_unoptimized(ste)) goto error; - if (PyDict_Update(free, newfree) < 0) + temp = PyNumber_InPlaceOr(free, newfree); + if (!temp) goto error; + Py_DECREF(temp); success = 1; error: + Py_XDECREF(scopes); Py_XDECREF(local); - Py_XDECREF(scope); Py_XDECREF(newbound); Py_XDECREF(newglobal); Py_XDECREF(newfree); @@ -735,6 +835,7 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, PyObject *global, PyObject* child_free) { PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL; + PyObject *temp; /* Copy the bound and global dictionaries. @@ -743,26 +844,22 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, dictionaries. */ - temp_bound = PyDict_New(); + temp_bound = PySet_New(bound); if (!temp_bound) goto error; - if (PyDict_Update(temp_bound, bound) < 0) - goto error; - temp_free = PyDict_New(); + temp_free = PySet_New(free); if (!temp_free) goto error; - if (PyDict_Update(temp_free, free) < 0) - goto error; - temp_global = PyDict_New(); + temp_global = PySet_New(global); if (!temp_global) goto error; - if (PyDict_Update(temp_global, global) < 0) - goto error; if (!analyze_block(entry, temp_bound, temp_free, temp_global)) goto error; - if (PyDict_Update(child_free, temp_free) < 0) + temp = PyNumber_InPlaceOr(child_free, temp_free); + if (!temp) goto error; + Py_DECREF(temp); Py_DECREF(temp_bound); Py_DECREF(temp_free); Py_DECREF(temp_global); @@ -780,10 +877,10 @@ symtable_analyze(struct symtable *st) PyObject *free, *global; int r; - free = PyDict_New(); + free = PySet_New(NULL); if (!free) return 0; - global = PyDict_New(); + global = PySet_New(NULL); if (!global) { Py_DECREF(free); return 0; @@ -802,8 +899,8 @@ symtable_warn(struct symtable *st, char *msg, int lineno) lineno, NULL, NULL) < 0) { if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { PyErr_SetString(PyExc_SyntaxError, msg); - PyErr_SyntaxLocation(st->st_filename, - st->st_cur->ste_lineno); + PyErr_SyntaxLocationEx(st->st_filename, st->st_cur->ste_lineno, + st->st_cur->ste_col_offset); } return 0; } @@ -818,43 +915,40 @@ symtable_warn(struct symtable *st, char *msg, int lineno) static int symtable_exit_block(struct symtable *st, void *ast) { - Py_ssize_t end; - - Py_CLEAR(st->st_cur); - end = PyList_GET_SIZE(st->st_stack) - 1; - if (end >= 0) { - st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, - end); - if (st->st_cur == NULL) - return 0; - Py_INCREF(st->st_cur); - if (PySequence_DelItem(st->st_stack, end) < 0) + Py_ssize_t size; + + st->st_cur = NULL; + size = PyList_GET_SIZE(st->st_stack); + if (size) { + if (PyList_SetSlice(st->st_stack, size - 1, size, NULL) < 0) return 0; + if (--size) + st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, size - 1); } return 1; } static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, - void *ast, int lineno) + void *ast, int lineno, int col_offset) { - PySTEntryObject *prev = NULL; + PySTEntryObject *prev = NULL, *ste; - if (st->st_cur) { - prev = st->st_cur; - if (PyList_Append(st->st_stack, (PyObject *)st->st_cur) < 0) { - return 0; - } - Py_DECREF(st->st_cur); - } - st->st_cur = ste_new(st, name, block, ast, lineno); - if (st->st_cur == NULL) + ste = ste_new(st, name, block, ast, lineno, col_offset); + if (ste == NULL) return 0; + if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) { + Py_DECREF(ste); + return 0; + } + prev = st->st_cur; + /* The entry is owned by the stack. Borrow it for st_cur. */ + Py_DECREF(ste); + st->st_cur = ste; if (block == ModuleBlock) st->st_global = st->st_cur->ste_symbols; if (prev) { - if (PyList_Append(prev->ste_children, - (PyObject *)st->st_cur) < 0) { + if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) { return 0; } } @@ -872,7 +966,7 @@ symtable_lookup(struct symtable *st, PyObject *name) Py_DECREF(mangled); if (!o) return 0; - return PyInt_AsLong(o); + return PyLong_AsLong(o); } static int @@ -883,23 +977,24 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) long val; PyObject *mangled = _Py_Mangle(st->st_private, name); + if (!mangled) return 0; dict = st->st_cur->ste_symbols; if ((o = PyDict_GetItem(dict, mangled))) { - val = PyInt_AS_LONG(o); + val = PyLong_AS_LONG(o); if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { /* Is it better to use 'mangled' or 'name' here? */ - PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, - PyString_AsString(name)); - PyErr_SyntaxLocation(st->st_filename, - st->st_cur->ste_lineno); + PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name); + PyErr_SyntaxLocationEx(st->st_filename, + st->st_cur->ste_lineno, + st->st_cur->ste_col_offset); goto error; } val |= flag; } else val = flag; - o = PyInt_FromLong(val); + o = PyLong_FromLong(val); if (o == NULL) goto error; if (PyDict_SetItem(dict, mangled, o) < 0) { @@ -916,9 +1011,9 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) perhaps only DEF_FREE_GLOBAL */ val = flag; if ((o = PyDict_GetItem(st->st_global, mangled))) { - val |= PyInt_AS_LONG(o); + val |= PyLong_AS_LONG(o); } - o = PyInt_FromLong(val); + o = PyLong_FromLong(val); if (o == NULL) goto error; if (PyDict_SetItem(st->st_global, mangled, o) < 0) { @@ -941,17 +1036,17 @@ error: VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is useful if the first node in the sequence requires special treatment. + + VISIT_QUIT macro returns the specified value exiting from the function but + first adjusts current recursion counter depth. */ +#define VISIT_QUIT(ST, X) \ + return --(ST)->recursion_depth,(X) + #define VISIT(ST, TYPE, V) \ if (!symtable_visit_ ## TYPE((ST), (V))) \ - return 0; - -#define VISIT_IN_BLOCK(ST, TYPE, V, S) \ - if (!symtable_visit_ ## TYPE((ST), (V))) { \ - symtable_exit_block((ST), (S)); \ - return 0; \ - } + VISIT_QUIT((ST), 0); #define VISIT_SEQ(ST, TYPE, SEQ) { \ int i; \ @@ -959,19 +1054,7 @@ error: for (i = 0; i < asdl_seq_LEN(seq); i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ if (!symtable_visit_ ## TYPE((ST), elt)) \ - return 0; \ - } \ -} - -#define VISIT_SEQ_IN_BLOCK(ST, TYPE, SEQ, S) { \ - int i; \ - asdl_seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) { \ - symtable_exit_block((ST), (S)); \ - return 0; \ - } \ + VISIT_QUIT((ST), 0); \ } \ } @@ -981,70 +1064,103 @@ error: for (i = (START); i < asdl_seq_LEN(seq); i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ if (!symtable_visit_ ## TYPE((ST), elt)) \ - return 0; \ + VISIT_QUIT((ST), 0); \ } \ } -#define VISIT_SEQ_TAIL_IN_BLOCK(ST, TYPE, SEQ, START, S) { \ - int i; \ - asdl_seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = (START); i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) { \ - symtable_exit_block((ST), (S)); \ - return 0; \ - } \ +#define VISIT_KWONLYDEFAULTS(ST, KW_DEFAULTS) { \ + int i = 0; \ + asdl_seq *seq = (KW_DEFAULTS); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + expr_ty elt = (expr_ty)asdl_seq_GET(seq, i); \ + if (!elt) continue; /* can be NULL */ \ + if (!symtable_visit_expr((ST), elt)) \ + VISIT_QUIT((ST), 0); \ } \ } static int +symtable_new_tmpname(struct symtable *st) +{ + char tmpname[256]; + identifier tmp; + + PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", + ++st->st_cur->ste_tmpname); + tmp = PyUnicode_InternFromString(tmpname); + if (!tmp) + return 0; + if (!symtable_add_def(st, tmp, DEF_LOCAL)) + return 0; + Py_DECREF(tmp); + return 1; +} + + +static int symtable_visit_stmt(struct symtable *st, stmt_ty s) { + if (++st->recursion_depth > st->recursion_limit) { + PyErr_SetString(PyExc_RuntimeError, + "maximum recursion depth exceeded during compilation"); + VISIT_QUIT(st, 0); + } switch (s->kind) { case FunctionDef_kind: if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL)) - return 0; + VISIT_QUIT(st, 0); if (s->v.FunctionDef.args->defaults) VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults); + if (s->v.FunctionDef.args->kw_defaults) + VISIT_KWONLYDEFAULTS(st, + s->v.FunctionDef.args->kw_defaults); + if (!symtable_visit_annotations(st, s)) + VISIT_QUIT(st, 0); if (s->v.FunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list); if (!symtable_enter_block(st, s->v.FunctionDef.name, - FunctionBlock, (void *)s, s->lineno)) - return 0; - VISIT_IN_BLOCK(st, arguments, s->v.FunctionDef.args, s); - VISIT_SEQ_IN_BLOCK(st, stmt, s->v.FunctionDef.body, s); + FunctionBlock, (void *)s, s->lineno, + s->col_offset)) + VISIT_QUIT(st, 0); + VISIT(st, arguments, s->v.FunctionDef.args); + VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st, s)) - return 0; + VISIT_QUIT(st, 0); break; case ClassDef_kind: { PyObject *tmp; if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL)) - return 0; + VISIT_QUIT(st, 0); VISIT_SEQ(st, expr, s->v.ClassDef.bases); + VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); + if (s->v.ClassDef.starargs) + VISIT(st, expr, s->v.ClassDef.starargs); + if (s->v.ClassDef.kwargs) + VISIT(st, expr, s->v.ClassDef.kwargs); if (s->v.ClassDef.decorator_list) VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, - (void *)s, s->lineno)) - return 0; + (void *)s, s->lineno, s->col_offset)) + VISIT_QUIT(st, 0); + if (!GET_IDENTIFIER(__class__) || + !symtable_add_def(st, __class__, DEF_LOCAL) || + !GET_IDENTIFIER(__locals__) || + !symtable_add_def(st, __locals__, DEF_PARAM)) { + symtable_exit_block(st, s); + VISIT_QUIT(st, 0); + } tmp = st->st_private; st->st_private = s->v.ClassDef.name; - VISIT_SEQ_IN_BLOCK(st, stmt, s->v.ClassDef.body, s); + VISIT_SEQ(st, stmt, s->v.ClassDef.body); st->st_private = tmp; if (!symtable_exit_block(st, s)) - return 0; + VISIT_QUIT(st, 0); break; } case Return_kind: if (s->v.Return.value) { VISIT(st, expr, s->v.Return.value); st->st_cur->ste_returns_value = 1; - if (st->st_cur->ste_generator) { - PyErr_SetString(PyExc_SyntaxError, - RETURN_VAL_IN_GENERATOR); - PyErr_SyntaxLocation(st->st_filename, - s->lineno); - return 0; - } } break; case Delete_kind: @@ -1058,11 +1174,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.AugAssign.target); VISIT(st, expr, s->v.AugAssign.value); break; - case Print_kind: - if (s->v.Print.dest) - VISIT(st, expr, s->v.Print.dest); - VISIT_SEQ(st, expr, s->v.Print.values); - break; case For_kind: VISIT(st, expr, s->v.For.target); VISIT(st, expr, s->v.For.iter); @@ -1084,23 +1195,18 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, stmt, s->v.If.orelse); break; case Raise_kind: - if (s->v.Raise.type) { - VISIT(st, expr, s->v.Raise.type); - if (s->v.Raise.inst) { - VISIT(st, expr, s->v.Raise.inst); - if (s->v.Raise.tback) - VISIT(st, expr, s->v.Raise.tback); + if (s->v.Raise.exc) { + VISIT(st, expr, s->v.Raise.exc); + if (s->v.Raise.cause) { + VISIT(st, expr, s->v.Raise.cause); } } break; - case TryExcept_kind: - VISIT_SEQ(st, stmt, s->v.TryExcept.body); - VISIT_SEQ(st, stmt, s->v.TryExcept.orelse); - VISIT_SEQ(st, excepthandler, s->v.TryExcept.handlers); - break; - case TryFinally_kind: - VISIT_SEQ(st, stmt, s->v.TryFinally.body); - VISIT_SEQ(st, stmt, s->v.TryFinally.finalbody); + case Try_kind: + VISIT_SEQ(st, stmt, s->v.Try.body); + VISIT_SEQ(st, stmt, s->v.Try.orelse); + VISIT_SEQ(st, excepthandler, s->v.Try.handlers); + VISIT_SEQ(st, stmt, s->v.Try.finalbody); break; case Assert_kind: VISIT(st, expr, s->v.Assert.test); @@ -1111,27 +1217,18 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, alias, s->v.Import.names); /* XXX Don't have the lineno available inside visit_alias */ - if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) + if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) { st->st_cur->ste_opt_lineno = s->lineno; + st->st_cur->ste_opt_col_offset = s->col_offset; + } break; case ImportFrom_kind: VISIT_SEQ(st, alias, s->v.ImportFrom.names); /* XXX Don't have the lineno available inside visit_alias */ - if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) - st->st_cur->ste_opt_lineno = s->lineno; - break; - case Exec_kind: - VISIT(st, expr, s->v.Exec.body); - if (!st->st_cur->ste_opt_lineno) + if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) { st->st_cur->ste_opt_lineno = s->lineno; - if (s->v.Exec.globals) { - st->st_cur->ste_unoptimized |= OPT_EXEC; - VISIT(st, expr, s->v.Exec.globals); - if (s->v.Exec.locals) - VISIT(st, expr, s->v.Exec.locals); - } else { - st->st_cur->ste_unoptimized |= OPT_BARE_EXEC; + st->st_cur->ste_opt_col_offset = s->col_offset; } break; case Global_kind: { @@ -1139,12 +1236,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) asdl_seq *seq = s->v.Global.names; for (i = 0; i < asdl_seq_LEN(seq); i++) { identifier name = (identifier)asdl_seq_GET(seq, i); - char *c_name = PyString_AS_STRING(name); long cur = symtable_lookup(st, name); if (cur < 0) - return 0; + VISIT_QUIT(st, 0); if (cur & (DEF_LOCAL | USE)) { char buf[256]; + char *c_name = _PyUnicode_AsString(name); + if (!c_name) + return 0; if (cur & DEF_LOCAL) PyOS_snprintf(buf, sizeof(buf), GLOBAL_AFTER_ASSIGN, @@ -1154,10 +1253,39 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) GLOBAL_AFTER_USE, c_name); if (!symtable_warn(st, buf, s->lineno)) - return 0; + VISIT_QUIT(st, 0); } if (!symtable_add_def(st, name, DEF_GLOBAL)) - return 0; + VISIT_QUIT(st, 0); + } + break; + } + case Nonlocal_kind: { + int i; + asdl_seq *seq = s->v.Nonlocal.names; + for (i = 0; i < asdl_seq_LEN(seq); i++) { + identifier name = (identifier)asdl_seq_GET(seq, i); + long cur = symtable_lookup(st, name); + if (cur < 0) + VISIT_QUIT(st, 0); + if (cur & (DEF_LOCAL | USE)) { + char buf[256]; + char *c_name = _PyUnicode_AsString(name); + if (!c_name) + return 0; + if (cur & DEF_LOCAL) + PyOS_snprintf(buf, sizeof(buf), + NONLOCAL_AFTER_ASSIGN, + c_name); + else + PyOS_snprintf(buf, sizeof(buf), + NONLOCAL_AFTER_USE, + c_name); + if (!symtable_warn(st, buf, s->lineno)) + VISIT_QUIT(st, 0); + } + if (!symtable_add_def(st, name, DEF_NONLOCAL)) + VISIT_QUIT(st, 0); } break; } @@ -1170,19 +1298,21 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) /* nothing to do here */ break; case With_kind: - VISIT(st, expr, s->v.With.context_expr); - if (s->v.With.optional_vars) { - VISIT(st, expr, s->v.With.optional_vars); - } + VISIT_SEQ(st, withitem, s->v.With.items); VISIT_SEQ(st, stmt, s->v.With.body); break; } - return 1; + VISIT_QUIT(st, 1); } static int symtable_visit_expr(struct symtable *st, expr_ty e) { + if (++st->recursion_depth > st->recursion_limit) { + PyErr_SetString(PyExc_RuntimeError, + "maximum recursion depth exceeded during compilation"); + VISIT_QUIT(st, 0); + } switch (e->kind) { case BoolOp_kind: VISIT_SEQ(st, expr, e->v.BoolOp.values); @@ -1196,16 +1326,20 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case Lambda_kind: { if (!GET_IDENTIFIER(lambda)) - return 0; + VISIT_QUIT(st, 0); if (e->v.Lambda.args->defaults) VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); + if (e->v.Lambda.args->kw_defaults) + VISIT_KWONLYDEFAULTS(st, + e->v.Lambda.args->kw_defaults); if (!symtable_enter_block(st, lambda, - FunctionBlock, (void *)e, e->lineno)) - return 0; - VISIT_IN_BLOCK(st, arguments, e->v.Lambda.args, (void*)e); - VISIT_IN_BLOCK(st, expr, e->v.Lambda.body, (void*)e); + FunctionBlock, (void *)e, e->lineno, + e->col_offset)) + VISIT_QUIT(st, 0); + VISIT(st, arguments, e->v.Lambda.args); + VISIT(st, expr, e->v.Lambda.body); if (!symtable_exit_block(st, (void *)e)) - return 0; + VISIT_QUIT(st, 0); break; } case IfExp_kind: @@ -1220,33 +1354,30 @@ symtable_visit_expr(struct symtable *st, expr_ty e) case Set_kind: VISIT_SEQ(st, expr, e->v.Set.elts); break; - case ListComp_kind: - VISIT(st, expr, e->v.ListComp.elt); - VISIT_SEQ(st, comprehension, e->v.ListComp.generators); - break; case GeneratorExp_kind: if (!symtable_visit_genexp(st, e)) - return 0; + VISIT_QUIT(st, 0); + break; + case ListComp_kind: + if (!symtable_visit_listcomp(st, e)) + VISIT_QUIT(st, 0); break; case SetComp_kind: if (!symtable_visit_setcomp(st, e)) - return 0; + VISIT_QUIT(st, 0); break; case DictComp_kind: if (!symtable_visit_dictcomp(st, e)) - return 0; + VISIT_QUIT(st, 0); break; case Yield_kind: if (e->v.Yield.value) VISIT(st, expr, e->v.Yield.value); st->st_cur->ste_generator = 1; - if (st->st_cur->ste_returns_value) { - PyErr_SetString(PyExc_SyntaxError, - RETURN_VAL_IN_GENERATOR); - PyErr_SyntaxLocation(st->st_filename, - e->lineno); - return 0; - } + break; + case YieldFrom_kind: + VISIT(st, expr, e->v.YieldFrom.value); + st->st_cur->ste_generator = 1; break; case Compare_kind: VISIT(st, expr, e->v.Compare.left); @@ -1261,11 +1392,10 @@ symtable_visit_expr(struct symtable *st, expr_ty e) if (e->v.Call.kwargs) VISIT(st, expr, e->v.Call.kwargs); break; - case Repr_kind: - VISIT(st, expr, e->v.Repr.value); - break; case Num_kind: case Str_kind: + case Bytes_kind: + case Ellipsis_kind: /* Nothing to do here. */ break; /* The following exprs can be assignment targets. */ @@ -1276,10 +1406,21 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.Subscript.value); VISIT(st, slice, e->v.Subscript.slice); break; + case Starred_kind: + VISIT(st, expr, e->v.Starred.value); + break; case Name_kind: if (!symtable_add_def(st, e->v.Name.id, e->v.Name.ctx == Load ? USE : DEF_LOCAL)) - return 0; + VISIT_QUIT(st, 0); + /* Special-case super: it counts as a use of __class__ */ + if (e->v.Name.ctx == Load && + st->st_cur->ste_type == FunctionBlock && + !PyUnicode_CompareWithASCIIString(e->v.Name.id, "super")) { + if (!GET_IDENTIFIER(__class__) || + !symtable_add_def(st, __class__, USE)) + VISIT_QUIT(st, 0); + } break; /* child nodes of List and Tuple will have expr_context set */ case List_kind: @@ -1289,13 +1430,13 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Tuple.elts); break; } - return 1; + VISIT_QUIT(st, 1); } static int symtable_implicit_arg(struct symtable *st, int pos) { - PyObject *id = PyString_FromFormat(".%d", pos); + PyObject *id = PyUnicode_FromFormat(".%d", pos); if (id == NULL) return 0; if (!symtable_add_def(st, id, DEF_PARAM)) { @@ -1307,37 +1448,16 @@ symtable_implicit_arg(struct symtable *st, int pos) } static int -symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel) +symtable_visit_params(struct symtable *st, asdl_seq *args) { int i; - /* go through all the toplevel arguments first */ - for (i = 0; i < asdl_seq_LEN(args); i++) { - expr_ty arg = (expr_ty)asdl_seq_GET(args, i); - if (arg->kind == Name_kind) { - assert(arg->v.Name.ctx == Param || - (arg->v.Name.ctx == Store && !toplevel)); - if (!symtable_add_def(st, arg->v.Name.id, DEF_PARAM)) - return 0; - } - else if (arg->kind == Tuple_kind) { - assert(arg->v.Tuple.ctx == Store); - if (toplevel) { - if (!symtable_implicit_arg(st, i)) - return 0; - } - } - else { - PyErr_SetString(PyExc_SyntaxError, - "invalid expression in parameter list"); - PyErr_SyntaxLocation(st->st_filename, - st->st_cur->ste_lineno); - return 0; - } - } + if (!args) + return -1; - if (!toplevel) { - if (!symtable_visit_params_nested(st, args)) + for (i = 0; i < asdl_seq_LEN(args); i++) { + arg_ty arg = (arg_ty)asdl_seq_GET(args, i); + if (!symtable_add_def(st, arg->arg, DEF_PARAM)) return 0; } @@ -1345,26 +1465,49 @@ symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel) } static int -symtable_visit_params_nested(struct symtable *st, asdl_seq *args) +symtable_visit_argannotations(struct symtable *st, asdl_seq *args) { int i; + + if (!args) + return -1; + for (i = 0; i < asdl_seq_LEN(args); i++) { - expr_ty arg = (expr_ty)asdl_seq_GET(args, i); - if (arg->kind == Tuple_kind && - !symtable_visit_params(st, arg->v.Tuple.elts, 0)) - return 0; + arg_ty arg = (arg_ty)asdl_seq_GET(args, i); + if (arg->annotation) + VISIT(st, expr, arg->annotation); } return 1; } static int +symtable_visit_annotations(struct symtable *st, stmt_ty s) +{ + arguments_ty a = s->v.FunctionDef.args; + + if (a->args && !symtable_visit_argannotations(st, a->args)) + return 0; + if (a->varargannotation) + VISIT(st, expr, a->varargannotation); + if (a->kwargannotation) + VISIT(st, expr, a->kwargannotation); + if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) + return 0; + if (s->v.FunctionDef.returns) + VISIT(st, expr, s->v.FunctionDef.returns); + return 1; +} + +static int symtable_visit_arguments(struct symtable *st, arguments_ty a) { /* skip default arguments inside function block XXX should ast be different? */ - if (a->args && !symtable_visit_params(st, a->args, 1)) + if (a->args && !symtable_visit_params(st, a->args)) + return 0; + if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs)) return 0; if (a->vararg) { if (!symtable_add_def(st, a->vararg, DEF_PARAM)) @@ -1376,8 +1519,6 @@ symtable_visit_arguments(struct symtable *st, arguments_ty a) return 0; st->st_cur->ste_varkeywords = 1; } - if (a->args && !symtable_visit_params_nested(st, a->args)) - return 0; return 1; } @@ -1388,11 +1529,22 @@ symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh) if (eh->v.ExceptHandler.type) VISIT(st, expr, eh->v.ExceptHandler.type); if (eh->v.ExceptHandler.name) - VISIT(st, expr, eh->v.ExceptHandler.name); + if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL)) + return 0; VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body); return 1; } +static int +symtable_visit_withitem(struct symtable *st, withitem_ty item) +{ + VISIT(st, expr, item->context_expr); + if (item->optional_vars) { + VISIT(st, expr, item->optional_vars); + } + return 1; +} + static int symtable_visit_alias(struct symtable *st, alias_ty a) @@ -1403,10 +1555,10 @@ symtable_visit_alias(struct symtable *st, alias_ty a) */ PyObject *store_name; PyObject *name = (a->asname == NULL) ? a->name : a->asname; - const char *base = PyString_AS_STRING(name); - char *dot = strchr(base, '.'); - if (dot) { - store_name = PyString_FromStringAndSize(base, dot - base); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, + PyUnicode_GET_LENGTH(name), 1); + if (dot != -1) { + store_name = PyUnicode_Substring(name, 0, dot); if (!store_name) return 0; } @@ -1414,19 +1566,20 @@ symtable_visit_alias(struct symtable *st, alias_ty a) store_name = name; Py_INCREF(store_name); } - if (strcmp(PyString_AS_STRING(name), "*")) { + if (PyUnicode_CompareWithASCIIString(name, "*")) { int r = symtable_add_def(st, store_name, DEF_IMPORT); Py_DECREF(store_name); return r; } else { if (st->st_cur->ste_type != ModuleBlock) { - int lineno = st->st_cur->ste_lineno; - if (!symtable_warn(st, IMPORT_STAR_WARNING, lineno)) { + int lineno = st->st_cur->ste_lineno; + int col_offset = st->st_cur->ste_col_offset; + PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING); + PyErr_SyntaxLocationEx(st->st_filename, lineno, col_offset); Py_DECREF(store_name); return 0; } - } st->st_cur->ste_unoptimized |= OPT_IMPORT_STAR; Py_DECREF(store_name); return 1; @@ -1470,30 +1623,11 @@ symtable_visit_slice(struct symtable *st, slice_ty s) case Index_kind: VISIT(st, expr, s->v.Index.value) break; - case Ellipsis_kind: - break; } return 1; } static int -symtable_new_tmpname(struct symtable *st) -{ - char tmpname[256]; - identifier tmp; - - PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", - ++st->st_cur->ste_tmpname); - tmp = PyString_InternFromString(tmpname); - if (!tmp) - return 0; - if (!symtable_add_def(st, tmp, DEF_LOCAL)) - return 0; - Py_DECREF(tmp); - return 1; -} - -static int symtable_handle_comprehension(struct symtable *st, expr_ty e, identifier scope_name, asdl_seq *generators, expr_ty elt, expr_ty value) @@ -1506,7 +1640,8 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, VISIT(st, expr, outermost->iter); /* Create comprehension scope for the rest */ if (!scope_name || - !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) { + !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, + e->lineno, e->col_offset)) { return 0; } st->st_cur->ste_generator = is_generator; @@ -1520,13 +1655,12 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, symtable_exit_block(st, (void *)e); return 0; } - VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e); - VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e); - VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension, - generators, 1, (void*)e); + VISIT(st, expr, outermost->target); + VISIT_SEQ(st, expr, outermost->ifs); + VISIT_SEQ_TAIL(st, comprehension, generators, 1); if (value) - VISIT_IN_BLOCK(st, expr, value, (void*)e); - VISIT_IN_BLOCK(st, expr, elt, (void*)e); + VISIT(st, expr, value); + VISIT(st, expr, elt); return symtable_exit_block(st, (void *)e); } @@ -1539,6 +1673,14 @@ symtable_visit_genexp(struct symtable *st, expr_ty e) } static int +symtable_visit_listcomp(struct symtable *st, expr_ty e) +{ + return symtable_handle_comprehension(st, e, GET_IDENTIFIER(listcomp), + e->v.ListComp.generators, + e->v.ListComp.elt, NULL); +} + +static int symtable_visit_setcomp(struct symtable *st, expr_ty e) { return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp), |