From 55e00f279f6e662f0f7cfc53398fa854d9666e09 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 17 Aug 2008 18:02:44 +0000 Subject: Merged revisions 65715,65724,65726,65732,65736-65739,65775 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r65715 | benjamin.peterson | 2008-08-16 16:04:16 -0500 (Sat, 16 Aug 2008) | 1 line add some documentation for symtable ........ r65724 | benjamin.peterson | 2008-08-16 17:11:33 -0500 (Sat, 16 Aug 2008) | 2 lines include filename and line number in SyntaxError ........ r65726 | georg.brandl | 2008-08-16 17:37:05 -0500 (Sat, 16 Aug 2008) | 2 lines Review symtable docs. ........ r65732 | benjamin.peterson | 2008-08-16 18:29:40 -0500 (Sat, 16 Aug 2008) | 1 line PySTEntry's constructor is static; there's no point in a fancy API name ........ r65736 | benjamin.peterson | 2008-08-16 20:09:17 -0500 (Sat, 16 Aug 2008) | 1 line expose PySTEntry.nested so the symtable module will work ........ r65737 | benjamin.peterson | 2008-08-16 20:17:15 -0500 (Sat, 16 Aug 2008) | 1 line a few improvements ........ r65738 | benjamin.peterson | 2008-08-16 20:27:30 -0500 (Sat, 16 Aug 2008) | 1 line fix compile errors ........ r65739 | benjamin.peterson | 2008-08-16 21:23:43 -0500 (Sat, 16 Aug 2008) | 1 line uhh PySTEntry->ste_unoptimized has to be exposed too ........ r65775 | benjamin.peterson | 2008-08-17 12:13:26 -0500 (Sun, 17 Aug 2008) | 5 lines get the symtable module back in working order - Fix broken functions - Add (hopefully) extensive tests - Modernize a little ........ --- Doc/library/language.rst | 1 + Doc/library/symtable.rst | 187 ++++++++++++++++++++++++++++++++++++++++++++++ Include/symtable.h | 2 +- Lib/symtable.py | 53 +++++++------ Lib/test/test_symtable.py | 170 ++++++++++++++++++++++++++++++++++++----- Modules/symtablemodule.c | 4 + Python/symtable.c | 13 +++- 7 files changed, 377 insertions(+), 53 deletions(-) create mode 100644 Doc/library/symtable.rst diff --git a/Doc/library/language.rst b/Doc/library/language.rst index bcf9ac0..3cc4b82 100644 --- a/Doc/library/language.rst +++ b/Doc/library/language.rst @@ -16,6 +16,7 @@ These modules include: parser.rst ast.rst + symtable.rst symbol.rst token.rst keyword.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst new file mode 100644 index 0000000..0082287 --- /dev/null +++ b/Doc/library/symtable.rst @@ -0,0 +1,187 @@ +:mod:`symtable` --- Access to the compiler's symbol tables +========================================================== + +.. module:: symtable + :synopsis: Interface to the compiler's internal symbol tables. + +.. moduleauthor:: Jeremy Hylton +.. sectionauthor:: Benjamin Peterson + + +Symbol tables are generated by the compiler from AST just before bytecode is +generated. The symbol table is responsible for calculating the scope of every +identifier in the code. :mod:`symtable` provides an interface to examine these +tables. + + +Generating Symbol Tables +------------------------ + +.. function:: symtable(code, filename, compile_type) + + Return the toplevel :class:`SymbolTable` for the Python source *code*. + *filename* is the name of the file containing the code. *compile_type* is + like the *mode* argument to :func:`compile`. + + +Examining Symbol Tables +----------------------- + +.. class:: SymbolTable + + A namespace table for a block. The constructor is not public. + + .. method:: get_type() + + Return the type of the symbol table. Possible values are ``'class'``, + ``'module'``, and ``'function'``. + + .. method:: get_id() + + Return the table's identifier. + + .. method:: get_name() + + Return the table's name. This is the name of the class if the table is + for a class, the name of the function if the table is for a function, or + ``'top'`` if the table is global (:meth:`get_type` returns ``'module'``). + + .. method:: get_lineno() + + Return the number of the first line in the block this table represents. + + .. method:: is_optimized() + + Return ``True`` if the locals in this table can be optimized. + + .. method:: is_nested() + + Return ``True`` if the block is a nested class or function. + + .. method:: has_children() + + Return ``True`` if the block has nested namespaces within it. These can + be obtained with :meth:`get_children`. + + .. method:: has_exec() + + Return ``True`` if the block uses ``exec``. + + .. method:: has_import_start() + + Return ``True`` if the block uses a starred from-import. + + .. method:: get_identifiers() + + Return a list of names of symbols in this table. + + .. method:: lookup(name) + + Lookup *name* in the table and return a :class:`Symbol` instance. + + .. method:: get_symbols() + + Return a list of :class:`Symbol` instances for names in the table. + + .. method:: get_children() + + Return a list of the nested symbol tables. + + +.. class:: Function + + A namespace for a function or method. This class inherits + :class:`SymbolTable`. + + .. method:: get_parameters() + + Return a tuple containing names of parameters to this function. + + .. method:: get_locals() + + Return a tuple containing names of locals in this function. + + .. method:: get_globals() + + Return a tuple containing names of globals in this function. + + .. method:: get_frees() + + Return a tuple containing names of free variables in this function. + + +.. class:: Class + + A namespace of a class. This class inherits :class:`SymbolTable`. + + .. method:: get_methods() + + Return a tuple containing the names of methods declared in the class. + + +.. class:: Symbol + + An entry in a :class:`SymbolTable` corresponding to an identifier in the + source. The constructor is not public. + + .. method:: get_name() + + Return the symbol's name. + + .. method:: is_referenced() + + Return ``True`` if the symbol is used in its block. + + .. method:: is_imported() + + Return ``True`` if the symbol is created from an import statement. + + .. method:: is_parameter() + + Return ``True`` if the symbol is a parameter. + + .. method:: is_global() + + Return ``True`` if the symbol is global. + + .. method:: is_vararg() + + Return ``True`` if the symbol is a star arg (receives varargs). + + .. method:: is_kewordarg() + + Return ``True`` if the symbol is a two-star arg (receives keyword + arguments). + + .. method:: is_local() + + Return ``True`` if the symbol is local to its block. + + .. method:: is_free() + + Return ``True`` if the symbol is referenced in its block, but not assigned + to. + + .. method:: is_assigned() + + Return ``True`` if the symbol is assigned to in its block. + + .. method:: is_namespace() + + Return ``True`` if name binding introduces new namespace. + + If the name is used as the target of a function or class statement, this + will be true. + + Note that a single name can be bound to multiple objects. If the result + is ``True``, the name may also be bound to other objects, like an int or + list, that does not introduce a new namespace. + + .. method:: get_namespaces() + + Return a list of namespaces bound to this name. + + .. method:: get_namespace() + + Return the namespace bound to this name. If more than one namespace is + bound, a :exc:`ValueError` is raised. diff --git a/Include/symtable.h b/Include/symtable.h index 8c1dc41..b07d77c 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -36,7 +36,7 @@ typedef struct _symtable_entry { PyObject *ste_children; /* list of child blocks */ _Py_block_ty ste_type; /* module, class, or function */ int ste_unoptimized; /* false if namespace is optimized */ - unsigned ste_nested : 1; /* true if block is nested */ + int ste_nested; /* true if block is nested */ unsigned ste_free : 1; /* true if block has free variables */ unsigned ste_child_free : 1; /* true if a child block has free vars, including free refs to globals */ diff --git a/Lib/symtable.py b/Lib/symtable.py index 089c8dc..ff332f2 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -1,10 +1,11 @@ """Interface to the compiler's internal symbol tables""" import _symtable -from _symtable import USE, DEF_GLOBAL, DEF_LOCAL, DEF_PARAM, \ - DEF_STAR, DEF_DOUBLESTAR, DEF_INTUPLE, DEF_FREE, \ - DEF_FREE_GLOBAL, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND, \ - OPT_IMPORT_STAR +from _symtable import (USE, DEF_GLOBAL, DEF_LOCAL, DEF_PARAM, + DEF_STAR, DEF_DOUBLESTAR, DEF_INTUPLE, DEF_FREE, + DEF_FREE_GLOBAL, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND, + OPT_IMPORT_STAR, SCOPE_OFF, SCOPE_MASK, FREE, + GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) import weakref @@ -38,15 +39,9 @@ class SymbolTableFactory: newSymbolTable = SymbolTableFactory() -def is_free(flags): - if (flags & (USE | DEF_FREE)) \ - and (flags & (DEF_LOCAL | DEF_PARAM | DEF_GLOBAL)): - return True - if flags & DEF_FREE_CLASS: - return True - return False -class SymbolTable: +class SymbolTable(object): + def __init__(self, raw_table, filename): self._table = raw_table self._filename = filename @@ -59,10 +54,11 @@ class SymbolTable: kind = "%s " % self.__class__.__name__ if self._table.name == "global": - return "<%sSymbolTable for module %s>" % (kind, self._filename) + return "<{0}SymbolTable for module {1}>".format(kind, self._filename) else: - return "<%sSymbolTable for %s in %s>" % (kind, self._table.name, - self._filename) + return "<{0}SymbolTable for {1} in {2}>".format(kind, + self._table.name, + self._filename) def get_type(self): if self._table.type == _symtable.TYPE_MODULE: @@ -72,7 +68,7 @@ class SymbolTable: if self._table.type == _symtable.TYPE_CLASS: return "class" assert self._table.type in (1, 2, 3), \ - "unexpected type: %s" % self._table.type + "unexpected type: {0}".format(self._table.type) def get_id(self): return self._table.id @@ -124,6 +120,7 @@ class SymbolTable: return [newSymbolTable(st, self._filename) for st in self._table.children] + class Function(SymbolTable): # Default values for instance variables @@ -148,15 +145,18 @@ class Function(SymbolTable): def get_globals(self): if self.__globals is None: - glob = DEF_GLOBAL | DEF_FREE_GLOBAL - self.__globals = self.__idents_matching(lambda x:x & glob) + glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) + test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob + self.__globals = self.__idents_matching(test) return self.__globals def get_frees(self): if self.__frees is None: + is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE self.__frees = self.__idents_matching(is_free) return self.__frees + class Class(SymbolTable): __methods = None @@ -169,14 +169,17 @@ class Class(SymbolTable): self.__methods = tuple(d) return self.__methods -class Symbol: + +class Symbol(object): + def __init__(self, name, flags, namespaces=None): self.__name = name self.__flags = flags + self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope() self.__namespaces = namespaces or () def __repr__(self): - return "" % self.__name + return "".format(self.__name) def get_name(self): return self.__name @@ -188,8 +191,7 @@ class Symbol: return bool(self.__flags & DEF_PARAM) def is_global(self): - return bool((self.__flags & DEF_GLOBAL) - or (self.__flags & DEF_FREE_GLOBAL)) + return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)) def is_vararg(self): return bool(self.__flags & DEF_STAR) @@ -201,12 +203,7 @@ class Symbol: return bool(self.__flags & DEF_BOUND) def is_free(self): - if (self.__flags & (USE | DEF_FREE)) \ - and (self.__flags & (DEF_LOCAL | DEF_PARAM | DEF_GLOBAL)): - return True - if self.__flags & DEF_FREE_CLASS: - return True - return False + return bool(self.__scope == FREE) def is_imported(self): return bool(self.__flags & DEF_IMPORT) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 77115e0..7e0206d 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -1,31 +1,161 @@ -from test import support - +""" +Test the API of the symtable module. +""" import symtable import unittest +from test import support + + +TEST_CODE = """ +import sys + +glob = 42 + +class Mine: + instance_var = 24 + def a_method(p1, p2): + pass + +def spam(a, b, *var, **kw): + global bar + bar = 47 + x = 23 + glob + def internal(): + return x + return internal -## XXX -## Test disabled because symtable module needs to be rewritten for new compiler +def foo(): + pass -##vereq(symbols[0].name, "global") -##vereq(len([ste for ste in symbols.values() if ste.name == "f"]), 1) +def namespace_test(): pass +def namespace_test(): pass +""" + + +def find_block(block, name): + for ch in block.get_children(): + if ch.get_name() == name: + return ch -### Bug tickler: SyntaxError file name correct whether error raised -### while parsing or building symbol table. -##def checkfilename(brokencode): -## try: -## _symtable.symtable(brokencode, "spam", "exec") -## except SyntaxError, e: -## vereq(e.filename, "spam") -## else: -## raise TestFailed("no SyntaxError for %r" % (brokencode,)) -##checkfilename("def f(x): foo)(") # parse-time -##checkfilename("def f(x): global x") # symtable-build-time class SymtableTest(unittest.TestCase): - def test_invalid_args(self): - self.assertRaises(TypeError, symtable.symtable, "42") - self.assertRaises(ValueError, symtable.symtable, "42", "?", "") + + top = symtable.symtable(TEST_CODE, "?", "exec") + # These correspond to scopes in TEST_CODE + Mine = find_block(top, "Mine") + a_method = find_block(Mine, "a_method") + spam = find_block(top, "spam") + internal = find_block(spam, "internal") + foo = find_block(top, "foo") + + def test_type(self): + self.assertEqual(self.top.get_type(), "module") + self.assertEqual(self.Mine.get_type(), "class") + self.assertEqual(self.a_method.get_type(), "function") + self.assertEqual(self.spam.get_type(), "function") + self.assertEqual(self.internal.get_type(), "function") + + def test_optimized(self): + self.assertFalse(self.top.is_optimized()) + self.assertFalse(self.top.has_exec()) + + self.assertTrue(self.spam.is_optimized()) + + def test_nested(self): + self.assertFalse(self.top.is_nested()) + self.assertFalse(self.Mine.is_nested()) + self.assertFalse(self.spam.is_nested()) + self.assertTrue(self.internal.is_nested()) + + def test_children(self): + self.assertTrue(self.top.has_children()) + self.assertTrue(self.Mine.has_children()) + self.assertFalse(self.foo.has_children()) + + def test_lineno(self): + self.assertEqual(self.top.get_lineno(), 0) + self.assertEqual(self.spam.get_lineno(), 11) + + def test_function_info(self): + func = self.spam + self.assertEqual(func.get_parameters(), ("a", "b", "kw", "var")) + self.assertEqual(func.get_locals(), + ("a", "b", "bar", "glob", "internal", "kw", "var", "x")) + self.assertEqual(func.get_globals(), ("bar", "glob")) + self.assertEqual(self.internal.get_frees(), ("x",)) + + def test_globals(self): + self.assertTrue(self.spam.lookup("glob").is_global()) + self.assertTrue(self.spam.lookup("bar").is_global()) + self.assertFalse(self.internal.lookup("x").is_global()) + self.assertFalse(self.Mine.lookup("instance_var").is_global()) + + def test_local(self): + self.assertTrue(self.spam.lookup("x").is_local()) + self.assertFalse(self.internal.lookup("x").is_local()) + + def test_referenced(self): + self.assertTrue(self.internal.lookup("x").is_referenced()) + self.assertTrue(self.spam.lookup("internal").is_referenced()) + self.assertFalse(self.spam.lookup("x").is_referenced()) + + def test_parameters(self): + for sym in ("a", "var", "kw"): + self.assertTrue(self.spam.lookup(sym).is_parameter()) + self.assertFalse(self.spam.lookup("x").is_parameter()) + + def test_symbol_lookup(self): + self.assertEqual(len(self.top.get_identifiers()), + len(self.top.get_symbols())) + + self.assertRaises(KeyError, self.top.lookup, "not_here") + + def test_namespaces(self): + self.assertTrue(self.top.lookup("Mine").is_namespace()) + self.assertTrue(self.Mine.lookup("a_method").is_namespace()) + self.assertTrue(self.top.lookup("spam").is_namespace()) + self.assertTrue(self.spam.lookup("internal").is_namespace()) + self.assertTrue(self.top.lookup("namespace_test").is_namespace()) + self.assertFalse(self.spam.lookup("x").is_namespace()) + + self.assert_(self.top.lookup("spam").get_namespace() is self.spam) + ns_test = self.top.lookup("namespace_test") + self.assertEqual(len(ns_test.get_namespaces()), 2) + self.assertRaises(ValueError, ns_test.get_namespace) + + def test_assigned(self): + self.assertTrue(self.spam.lookup("x").is_assigned()) + self.assertTrue(self.spam.lookup("bar").is_assigned()) + self.assertTrue(self.top.lookup("spam").is_assigned()) + self.assertTrue(self.Mine.lookup("a_method").is_assigned()) + self.assertFalse(self.internal.lookup("x").is_assigned()) + + def test_imported(self): + self.assertTrue(self.top.lookup("sys").is_imported()) + + def test_name(self): + self.assertEqual(self.top.get_name(), "top") + self.assertEqual(self.spam.get_name(), "spam") + self.assertEqual(self.spam.lookup("x").get_name(), "x") + self.assertEqual(self.Mine.get_name(), "Mine") + + def test_class_info(self): + self.assertEqual(self.Mine.get_methods(), ('a_method',)) + + def test_filename_correct(self): + ### Bug tickler: SyntaxError file name correct whether error raised + ### while parsing or building symbol table. + def checkfilename(brokencode): + try: + symtable.symtable(brokencode, "spam", "exec") + except SyntaxError as e: + self.assertEqual(e.filename, "spam") + else: + self.fail("no SyntaxError for %r" % (brokencode,)) + checkfilename("def f(x): foo)(") # parse-time + checkfilename("def f(x): global x") # symtable-build-time def test_eval(self): symbols = symtable.symtable("42", "?", "eval") diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index e3c9e51..3388d63 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -92,6 +92,10 @@ PyInit__symtable(void) PyModule_AddIntConstant(m, "GLOBAL_IMPLICIT", GLOBAL_IMPLICIT); PyModule_AddIntConstant(m, "FREE", FREE); PyModule_AddIntConstant(m, "CELL", CELL); + + PyModule_AddIntConstant(m, "SCOPE_OFF", SCOPE_OFFSET); + PyModule_AddIntConstant(m, "SCOPE_MASK", SCOPE_MASK); + if (PyErr_Occurred()) { Py_DECREF(m); m = 0; diff --git a/Python/symtable.c b/Python/symtable.c index 92432ab..e3bb270 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -22,9 +22,9 @@ #define RETURN_VAL_IN_GENERATOR \ "'return' with argument inside generator" -/* XXX(nnorwitz): change name since static? */ + static PySTEntryObject * -PySTEntry_New(struct symtable *st, identifier name, _Py_block_ty block, +ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, int lineno) { PySTEntryObject *ste = NULL; @@ -114,6 +114,8 @@ static PyMemberDef ste_memberlist[] = { {"symbols", T_OBJECT, OFF(ste_symbols), READONLY}, {"varnames", T_OBJECT, OFF(ste_varnames), READONLY}, {"children", T_OBJECT, OFF(ste_children), READONLY}, + {"optimized",T_INT, OFF(ste_unoptimized), READONLY}, + {"nested", T_INT, OFF(ste_nested), READONLY}, {"type", T_INT, OFF(ste_type), READONLY}, {"lineno", T_INT, OFF(ste_lineno), READONLY}, {NULL} @@ -388,6 +390,9 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, PyErr_Format(PyExc_SyntaxError, "name '%U' is parameter and global", name); + PyErr_SyntaxLocation(ste->ste_table->st_filename, + ste->ste_lineno); + return 0; } if (flags & DEF_NONLOCAL) { @@ -788,7 +793,7 @@ symtable_warn(struct symtable *st, char *msg, int lineno) return 1; } -/* symtable_enter_block() gets a reference via PySTEntry_New(). +/* symtable_enter_block() gets a reference via ste_new. This reference is released when the block is exited, via the DECREF in symtable_exit_block(). */ @@ -825,7 +830,7 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, } Py_DECREF(st->st_cur); } - st->st_cur = PySTEntry_New(st, name, block, ast, lineno); + st->st_cur = ste_new(st, name, block, ast, lineno); if (st->st_cur == NULL) return 0; if (name == GET_IDENTIFIER(top)) -- cgit v0.12