diff options
Diffstat (limited to 'Lib')
| -rw-r--r-- | Lib/AST.py | 31 | ||||
| -rwxr-xr-x | Lib/symbol.py | 84 | ||||
| -rwxr-xr-x | Lib/token.py | 79 |
3 files changed, 155 insertions, 39 deletions
@@ -1,13 +1,13 @@ """Object-oriented interface to the parser module. -This module exports three classes which together provide an interface +This module exports four classes which together provide an interface to the parser module. Together, the three classes represent two ways to create parsed representations of Python source and the two starting data types (source text and tuple representations). Each class provides interfaces which are identical other than the constructors. The constructors are described in detail in the documentation for each class and the remaining, shared portion of the interface is documented -below. Briefly, the three classes provided are: +below. Briefly, the classes provided are: AST Defines the primary interface to the AST objects and supports creation @@ -23,6 +23,9 @@ FileSuiteAST Convenience subclass of the `SuiteAST' class; loads source text of the suite from an external file. +Common Methods +-------------- + Aside from the constructors, several methods are provided to allow access to the various interpretations of the parse tree and to check conditions of the construct represented by the parse tree. @@ -68,8 +71,8 @@ class AST: This base class provides all of the query methods for subclass objects defined in this module. """ - _p = __import__('parser') # import internally to avoid - # namespace pollution at the + import parser # import internally to avoid + _p = parser # namespace pollution at the # top level _text = None _code = None @@ -84,7 +87,8 @@ class AST: The tuple tree to convert. The tuple-tree may represent either an expression or a suite; the - type will be determined automatically. + type will be determined automatically. Line number information may + optionally be present for any subset of the terminal tokens. """ if type(tuple) is not type(()): raise TypeError, 'Base AST class requires tuple parameter.' @@ -93,11 +97,24 @@ class AST: self._ast = self._p.tuple2ast(tuple) self._type = (self._p.isexpr(self._ast) and 'expression') or 'suite' - def tuple(self): + def list(self, line_info = 0): + """Returns a fresh list representing the parse tree. + + line_info + If true, includes line number information for terminal tokens in + the output data structure, + """ + return self._p.ast2list(self._ast, line_info) + + def tuple(self, line_info = 0): """Returns the tuple representing the parse tree. + + line_info + If true, includes line number information for terminal tokens in + the output data structure, """ if self._tupl is None: - self._tupl = self._p.ast2tuple(self._ast) + self._tupl = self._p.ast2tuple(self._ast, line_info) return self._tupl def code(self): diff --git a/Lib/symbol.py b/Lib/symbol.py index 36f178a..6d925ea 100755 --- a/Lib/symbol.py +++ b/Lib/symbol.py @@ -1,5 +1,18 @@ -# Non-terminal symbols of Python grammar (from "graminit.h") +#! /usr/bin/env python +# +# Non-terminal symbols of Python grammar (from "graminit.h") +# +# This file is automatically generated; please don't muck it up! +# +# To update the symbols in this file, 'cd' to the top directory of +# the python source tree after building the interpreter and run: +# +# PYTHONPATH=Lib:Modules ./python Lib/symbol.py +# +# (this path allows the import of string.py, token.py, and regexmodule.so +# for a site with no installation in place) +#--start constants-- single_input = 256 file_input = 257 eval_input = 258 @@ -23,39 +36,40 @@ raise_stmt = 275 import_stmt = 276 dotted_name = 277 global_stmt = 278 -access_stmt = 279 -accesstype = 280 -exec_stmt = 281 -compound_stmt = 282 -if_stmt = 283 -while_stmt = 284 -for_stmt = 285 -try_stmt = 286 -except_clause = 287 -suite = 288 -test = 289 -and_test = 290 -not_test = 291 -comparison = 292 -comp_op = 293 -expr = 294 -xor_expr = 295 -and_expr = 296 -shift_expr = 297 -arith_expr = 298 -term = 299 -factor = 300 -power = 301 -atom = 302 -lambdef = 303 -trailer = 304 -subscript = 305 +exec_stmt = 279 +compound_stmt = 280 +if_stmt = 281 +while_stmt = 282 +for_stmt = 283 +try_stmt = 284 +except_clause = 285 +suite = 286 +test = 287 +and_test = 288 +not_test = 289 +comparison = 290 +comp_op = 291 +expr = 292 +xor_expr = 293 +and_expr = 294 +shift_expr = 295 +arith_expr = 296 +term = 297 +factor = 298 +power = 299 +atom = 300 +lambdef = 301 +trailer = 302 +subscriptlist = 303 +subscript = 304 +sliceop = 305 exprlist = 306 testlist = 307 dictmaker = 308 classdef = 309 arglist = 310 argument = 311 +#--end constants-- names = dir() sym_name = {} @@ -63,3 +77,17 @@ for name in names: number = eval(name) if type(number) is type(0): sym_name[number] = name + + +def main(): + import sys + import token + if len(sys.argv) == 1: + sys.argv = sys.argv + ["Include/graminit.h", "Lib/symbol.py"] + token.main() + +if __name__ == "__main__": + main() + +# +# end of file diff --git a/Lib/token.py b/Lib/token.py index 527df70..61228e5 100755 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,5 +1,18 @@ -# Tokens (from "token.h") +#! /usr/bin/env python +# +# Tokens (from "token.h") +# +# This file is automatically generated; please don't muck it up! +# +# To update the symbols in this file, 'cd' to the top directory of +# the python source tree after building the interpreter and run: +# +# PYTHONPATH=./Lib ./python Lib/token.py +# +# (this path allows the import of string.py and regexmodule.so +# for a site with no installation in place) +#--start constants-- ENDMARKER = 0 NAME = 1 NUMBER = 2 @@ -39,6 +52,9 @@ RIGHTSHIFT = 35 DOUBLESTAR = 36 OP = 37 ERRORTOKEN = 38 +N_TOKENS = 39 +NT_OFFSET = 256 +#--end constants-- names = dir() tok_name = {} @@ -47,9 +63,6 @@ for name in names: if type(number) is type(0): tok_name[number] = name -N_TOKENS = 39 # Number of tokens including ERRORTOKEN -NT_OFFSET = 256 # Start of non-terminal symbols - def ISTERMINAL(x): return x < NT_OFFSET @@ -58,3 +71,61 @@ def ISNONTERMINAL(x): def ISEOF(x): return x == ENDMARKER + + +def main(): + import regex + import string + import sys + args = sys.argv[1:] + inFileName = args and args[0] or "Include/token.h" + outFileName = "Lib/token.py" + if len(args) > 1: + outFileName = args[1] + try: + fp = open(inFileName) + except IOError, err: + sys.stdout.write("I/O error: %s\n" % str(err)) + sys.exit(1) + lines = string.splitfields(fp.read(), "\n") + fp.close() + re = regex.compile( + "#define[ \t][ \t]*\([A-Z][A-Z_]*\)[ \t][ \t]*\([0-9][0-9]*\)", + regex.casefold) + tokens = {} + for line in lines: + if re.match(line) > -1: + name, val = re.group(1, 2) + val = string.atoi(val) + tokens[val] = name # reverse so we can sort them... + keys = tokens.keys() + keys.sort() + # load the output skeleton from the target: + try: + fp = open(outFileName) + except IOError, err: + sys.stderr.write("I/O error: %s\n" % str(err)) + sys.exit(2) + format = string.splitfields(fp.read(), "\n") + fp.close() + try: + start = format.index("#--start constants--") + 1 + end = format.index("#--end constants--") + except ValueError: + sys.stderr.write("target does not contain format markers") + sys.exit(3) + lines = [] + for val in keys: + lines.append("%s = %d" % (tokens[val], val)) + format[start:end] = lines + try: + fp = open(outFileName, 'w') + except IOError, err: + sys.stderr.write("I/O error: %s\n" % str(err)) + sys.exit(4) + fp.write(string.joinfields(format, "\n")) + fp.close() + + +if __name__ == "__main__": + main() |