diff options
Diffstat (limited to 'Demo/parser')
| -rw-r--r-- | Demo/parser/FILES | 6 | ||||
| -rw-r--r-- | Demo/parser/README | 31 | ||||
| -rw-r--r-- | Demo/parser/docstring.py | 2 | ||||
| -rw-r--r-- | Demo/parser/example.py | 190 | ||||
| -rw-r--r-- | Demo/parser/simple.py | 1 | ||||
| -rw-r--r-- | Demo/parser/source.py | 27 | ||||
| -rwxr-xr-x | Demo/parser/test_parser.py | 48 | ||||
| -rw-r--r-- | Demo/parser/test_unparse.py | 240 | ||||
| -rw-r--r-- | Demo/parser/texipre.dat | 100 | ||||
| -rw-r--r-- | Demo/parser/unparse.py | 600 |
10 files changed, 0 insertions, 1245 deletions
diff --git a/Demo/parser/FILES b/Demo/parser/FILES deleted file mode 100644 index 1ff59a3..0000000 --- a/Demo/parser/FILES +++ /dev/null @@ -1,6 +0,0 @@ -Demo/parser -Doc/libparser.tex -Lib/AST.py -Lib/symbol.py -Lib/token.py -Modules/parsermodule.c diff --git a/Demo/parser/README b/Demo/parser/README deleted file mode 100644 index a576d33..0000000 --- a/Demo/parser/README +++ /dev/null @@ -1,31 +0,0 @@ -These files are from the large example of using the `parser' module. Refer -to the Python Library Reference for more information. - -It also contains examples for the AST parser. - -Files: ------- - - FILES -- list of files associated with the parser module. - - README -- this file. - - example.py -- module that uses the `parser' module to extract - information from the parse tree of Python source - code. - - docstring.py -- sample source file containing only a module docstring. - - simple.py -- sample source containing a "short form" definition. - - source.py -- sample source code used to demonstrate ability to - handle nested constructs easily using the functions - and classes in example.py. - - test_parser.py program to put the parser module through its paces. - - unparse.py AST (2.5) based example to recreate source code - from an AST. This is incomplete; contributions - are welcome. - -Enjoy! diff --git a/Demo/parser/docstring.py b/Demo/parser/docstring.py deleted file mode 100644 index 45a261b..0000000 --- a/Demo/parser/docstring.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Some documentation. -""" diff --git a/Demo/parser/example.py b/Demo/parser/example.py deleted file mode 100644 index c2f0883..0000000 --- a/Demo/parser/example.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Simple code to extract class & function docstrings from a module. - -This code is used as an example in the library reference manual in the -section on using the parser module. Refer to the manual for a thorough -discussion of the operation of this code. -""" - -import os -import parser -import symbol -import token -import types - -from types import ListType, TupleType - - -def get_docs(fileName): - """Retrieve information from the parse tree of a source file. - - fileName - Name of the file to read Python source code from. - """ - source = open(fileName).read() - basename = os.path.basename(os.path.splitext(fileName)[0]) - ast = parser.suite(source) - return ModuleInfo(ast.totuple(), basename) - - -class SuiteInfoBase: - _docstring = '' - _name = '' - - def __init__(self, tree = None): - self._class_info = {} - self._function_info = {} - if tree: - self._extract_info(tree) - - def _extract_info(self, tree): - # extract docstring - if len(tree) == 2: - found, vars = match(DOCSTRING_STMT_PATTERN[1], tree[1]) - else: - found, vars = match(DOCSTRING_STMT_PATTERN, tree[3]) - if found: - self._docstring = eval(vars['docstring']) - # discover inner definitions - for node in tree[1:]: - found, vars = match(COMPOUND_STMT_PATTERN, node) - if found: - cstmt = vars['compound'] - if cstmt[0] == symbol.funcdef: - name = cstmt[2][1] - self._function_info[name] = FunctionInfo(cstmt) - elif cstmt[0] == symbol.classdef: - name = cstmt[2][1] - self._class_info[name] = ClassInfo(cstmt) - - def get_docstring(self): - return self._docstring - - def get_name(self): - return self._name - - def get_class_names(self): - return list(self._class_info.keys()) - - def get_class_info(self, name): - return self._class_info[name] - - def __getitem__(self, name): - try: - return self._class_info[name] - except KeyError: - return self._function_info[name] - - -class SuiteFuncInfo: - # Mixin class providing access to function names and info. - - def get_function_names(self): - return list(self._function_info.keys()) - - def get_function_info(self, name): - return self._function_info[name] - - -class FunctionInfo(SuiteInfoBase, SuiteFuncInfo): - def __init__(self, tree = None): - self._name = tree[2][1] - SuiteInfoBase.__init__(self, tree and tree[-1] or None) - - -class ClassInfo(SuiteInfoBase): - def __init__(self, tree = None): - self._name = tree[2][1] - SuiteInfoBase.__init__(self, tree and tree[-1] or None) - - def get_method_names(self): - return list(self._function_info.keys()) - - def get_method_info(self, name): - return self._function_info[name] - - -class ModuleInfo(SuiteInfoBase, SuiteFuncInfo): - def __init__(self, tree = None, name = "<string>"): - self._name = name - SuiteInfoBase.__init__(self, tree) - if tree: - found, vars = match(DOCSTRING_STMT_PATTERN, tree[1]) - if found: - self._docstring = vars["docstring"] - - -def match(pattern, data, vars=None): - """Match `data' to `pattern', with variable extraction. - - pattern - Pattern to match against, possibly containing variables. - - data - Data to be checked and against which variables are extracted. - - vars - Dictionary of variables which have already been found. If not - provided, an empty dictionary is created. - - The `pattern' value may contain variables of the form ['varname'] which - are allowed to match anything. The value that is matched is returned as - part of a dictionary which maps 'varname' to the matched value. 'varname' - is not required to be a string object, but using strings makes patterns - and the code which uses them more readable. - - This function returns two values: a boolean indicating whether a match - was found and a dictionary mapping variable names to their associated - values. - """ - if vars is None: - vars = {} - if type(pattern) is ListType: # 'variables' are ['varname'] - vars[pattern[0]] = data - return 1, vars - if type(pattern) is not TupleType: - return (pattern == data), vars - if len(data) != len(pattern): - return 0, vars - for pattern, data in map(None, pattern, data): - same, vars = match(pattern, data, vars) - if not same: - break - return same, vars - - -# This pattern identifies compound statements, allowing them to be readily -# differentiated from simple statements. -# -COMPOUND_STMT_PATTERN = ( - symbol.stmt, - (symbol.compound_stmt, ['compound']) - ) - - -# This pattern will match a 'stmt' node which *might* represent a docstring; -# docstrings require that the statement which provides the docstring be the -# first statement in the class or function, which this pattern does not check. -# -DOCSTRING_STMT_PATTERN = ( - symbol.stmt, - (symbol.simple_stmt, - (symbol.small_stmt, - (symbol.expr_stmt, - (symbol.testlist, - (symbol.test, - (symbol.and_test, - (symbol.not_test, - (symbol.comparison, - (symbol.expr, - (symbol.xor_expr, - (symbol.and_expr, - (symbol.shift_expr, - (symbol.arith_expr, - (symbol.term, - (symbol.factor, - (symbol.power, - (symbol.atom, - (token.STRING, ['docstring']) - )))))))))))))))), - (token.NEWLINE, '') - )) diff --git a/Demo/parser/simple.py b/Demo/parser/simple.py deleted file mode 100644 index 184e2fe..0000000 --- a/Demo/parser/simple.py +++ /dev/null @@ -1 +0,0 @@ -def f(): "maybe a docstring" diff --git a/Demo/parser/source.py b/Demo/parser/source.py deleted file mode 100644 index b900628..0000000 --- a/Demo/parser/source.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Exmaple file to be parsed for the parsermodule example. - -The classes and functions in this module exist only to exhibit the ability -of the handling information extraction from nested definitions using parse -trees. They shouldn't interest you otherwise! -""" - -class Simple: - "This class does very little." - - def method(self): - "This method does almost nothing." - return 1 - - class Nested: - "This is a nested class." - - def nested_method(self): - "Method of Nested class." - def nested_function(): - "Function in method of Nested class." - pass - return nested_function - -def function(): - "This function lives at the module level." - return 0 diff --git a/Demo/parser/test_parser.py b/Demo/parser/test_parser.py deleted file mode 100755 index ffa6630..0000000 --- a/Demo/parser/test_parser.py +++ /dev/null @@ -1,48 +0,0 @@ -#! /usr/bin/env python3 -# (Force the script to use the latest build.) -# -# test_parser.py - -import parser, traceback - -_numFailed = 0 - -def testChunk(t, fileName): - global _numFailed - print('----', fileName, end=' ') - try: - st = parser.suite(t) - tup = parser.st2tuple(st) - # this discards the first ST; a huge memory savings when running - # against a large source file like Tkinter.py. - st = None - new = parser.tuple2st(tup) - except parser.ParserError as err: - print() - print('parser module raised exception on input file', fileName + ':') - traceback.print_exc() - _numFailed = _numFailed + 1 - else: - if tup != parser.st2tuple(new): - print() - print('parser module failed on input file', fileName) - _numFailed = _numFailed + 1 - else: - print('o.k.') - -def testFile(fileName): - t = open(fileName).read() - testChunk(t, fileName) - -def test(): - import sys - args = sys.argv[1:] - if not args: - import glob - args = glob.glob("*.py") - args.sort() - list(map(testFile, args)) - sys.exit(_numFailed != 0) - -if __name__ == '__main__': - test() diff --git a/Demo/parser/test_unparse.py b/Demo/parser/test_unparse.py deleted file mode 100644 index d457523..0000000 --- a/Demo/parser/test_unparse.py +++ /dev/null @@ -1,240 +0,0 @@ -import unittest -import test.support -import io -import os -import tokenize -import ast -import unparse - -def read_pyfile(filename): - """Read and return the contents of a Python source file (as a - string), taking into account the file encoding.""" - with open(filename, "rb") as pyfile: - encoding = tokenize.detect_encoding(pyfile.readline)[0] - with open(filename, "r", encoding=encoding) as pyfile: - source = pyfile.read() - return source - -for_else = """\ -def f(): - for x in range(10): - break - else: - y = 2 - z = 3 -""" - -while_else = """\ -def g(): - while True: - break - else: - y = 2 - z = 3 -""" - -relative_import = """\ -from . import fred -from .. import barney -from .australia import shrimp as prawns -""" - -nonlocal_ex = """\ -def f(): - x = 1 - def g(): - nonlocal x - x = 2 - y = 7 - def h(): - nonlocal x, y -""" - -# also acts as test for 'except ... as ...' -raise_from = """\ -try: - 1 / 0 -except ZeroDivisionError as e: - raise ArithmeticError from e -""" - -class_decorator = """\ -@f1(arg) -@f2 -class Foo: pass -""" - -elif1 = """\ -if cond1: - suite1 -elif cond2: - suite2 -else: - suite3 -""" - -elif2 = """\ -if cond1: - suite1 -elif cond2: - suite2 -""" - -try_except_finally = """\ -try: - suite1 -except ex1: - suite2 -except ex2: - suite3 -else: - suite4 -finally: - suite5 -""" - -class ASTTestCase(unittest.TestCase): - def assertASTEqual(self, ast1, ast2): - self.assertEqual(ast.dump(ast1), ast.dump(ast2)) - - def check_roundtrip(self, code1, filename="internal"): - ast1 = compile(code1, filename, "exec", ast.PyCF_ONLY_AST) - unparse_buffer = io.StringIO() - unparse.Unparser(ast1, unparse_buffer) - code2 = unparse_buffer.getvalue() - ast2 = compile(code2, filename, "exec", ast.PyCF_ONLY_AST) - self.assertASTEqual(ast1, ast2) - -class UnparseTestCase(ASTTestCase): - # Tests for specific bugs found in earlier versions of unparse - - def test_del_statement(self): - self.check_roundtrip("del x, y, z") - - def test_shifts(self): - self.check_roundtrip("45 << 2") - self.check_roundtrip("13 >> 7") - - def test_for_else(self): - self.check_roundtrip(for_else) - - def test_while_else(self): - self.check_roundtrip(while_else) - - def test_unary_parens(self): - self.check_roundtrip("(-1)**7") - self.check_roundtrip("(-1.)**8") - self.check_roundtrip("(-1j)**6") - self.check_roundtrip("not True or False") - self.check_roundtrip("True or not False") - - def test_integer_parens(self): - self.check_roundtrip("3 .__abs__()") - - def test_huge_float(self): - self.check_roundtrip("1e1000") - self.check_roundtrip("-1e1000") - self.check_roundtrip("1e1000j") - self.check_roundtrip("-1e1000j") - - def test_min_int(self): - self.check_roundtrip(str(-2**31)) - self.check_roundtrip(str(-2**63)) - - def test_imaginary_literals(self): - self.check_roundtrip("7j") - self.check_roundtrip("-7j") - self.check_roundtrip("0j") - self.check_roundtrip("-0j") - - def test_lambda_parentheses(self): - self.check_roundtrip("(lambda: int)()") - - def test_chained_comparisons(self): - self.check_roundtrip("1 < 4 <= 5") - self.check_roundtrip("a is b is c is not d") - - def test_function_arguments(self): - self.check_roundtrip("def f(): pass") - self.check_roundtrip("def f(a): pass") - self.check_roundtrip("def f(b = 2): pass") - self.check_roundtrip("def f(a, b): pass") - self.check_roundtrip("def f(a, b = 2): pass") - self.check_roundtrip("def f(a = 5, b = 2): pass") - self.check_roundtrip("def f(*, a = 1, b = 2): pass") - self.check_roundtrip("def f(*, a = 1, b): pass") - self.check_roundtrip("def f(*, a, b = 2): pass") - self.check_roundtrip("def f(a, b = None, *, c, **kwds): pass") - self.check_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass") - self.check_roundtrip("def f(*args, **kwargs): pass") - - def test_relative_import(self): - self.check_roundtrip(relative_import) - - def test_nonlocal(self): - self.check_roundtrip(nonlocal_ex) - - def test_raise_from(self): - self.check_roundtrip(raise_from) - - def test_bytes(self): - self.check_roundtrip("b'123'") - - def test_annotations(self): - self.check_roundtrip("def f(a : int): pass") - self.check_roundtrip("def f(a: int = 5): pass") - self.check_roundtrip("def f(*args: [int]): pass") - self.check_roundtrip("def f(**kwargs: dict): pass") - self.check_roundtrip("def f() -> None: pass") - - def test_set_literal(self): - self.check_roundtrip("{'a', 'b', 'c'}") - - def test_set_comprehension(self): - self.check_roundtrip("{x for x in range(5)}") - - def test_dict_comprehension(self): - self.check_roundtrip("{x: x*x for x in range(10)}") - - def test_class_decorators(self): - self.check_roundtrip(class_decorator) - - def test_class_definition(self): - self.check_roundtrip("class A(metaclass=type, *[], **{}): pass") - - def test_elifs(self): - self.check_roundtrip(elif1) - self.check_roundtrip(elif2) - - def test_try_except_finally(self): - self.check_roundtrip(try_except_finally) - -class DirectoryTestCase(ASTTestCase): - """Test roundtrip behaviour on all files in Lib and Lib/test.""" - - # test directories, relative to the root of the distribution - test_directories = 'Lib', os.path.join('Lib', 'test') - - def test_files(self): - # get names of files to test - dist_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) - - names = [] - for d in self.test_directories: - test_dir = os.path.join(dist_dir, d) - for n in os.listdir(test_dir): - if n.endswith('.py') and not n.startswith('bad'): - names.append(os.path.join(test_dir, n)) - - for filename in names: - if test.support.verbose: - print('Testing %s' % filename) - source = read_pyfile(filename) - self.check_roundtrip(source) - - -def test_main(): - test.support.run_unittest(UnparseTestCase, DirectoryTestCase) - -if __name__ == '__main__': - test_main() diff --git a/Demo/parser/texipre.dat b/Demo/parser/texipre.dat deleted file mode 100644 index 8ad03a6..0000000 --- a/Demo/parser/texipre.dat +++ /dev/null @@ -1,100 +0,0 @@ -\input texinfo @c -*-texinfo-*- -@c %**start of header -@setfilename parser.info -@settitle Python Parser Module Reference -@setchapternewpage odd -@footnotestyle end -@c %**end of header - -@ifinfo -This file describes the interfaces -published by the optional @code{parser} module and gives examples of -how they may be used. It contains the same text as the chapter on the -@code{parser} module in the @cite{Python Library Reference}, but is -presented as a separate document. - -Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and -Virginia Polytechnic Institute and State University, Blacksburg, -Virginia, USA. Portions of the software copyright 1991-1995 by -Stichting Mathematisch Centrum, Amsterdam, The Netherlands. Copying is -permitted under the terms associated with the main Python distribution, -with the additional restriction that this additional notice be included -and maintained on all distributed copies. - - All Rights Reserved - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the names of Fred L. Drake, Jr. and -Virginia Polytechnic Institute and State University not be used in -advertising or publicity pertaining to distribution of the software -without specific, written prior permission. - -FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE -UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO -EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND -STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL -DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR -PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. -@end ifinfo - -@titlepage -@title Python Parser Module Reference -@author Fred L. Drake, Jr. - -@c The following two commands start the copyright page. -@page -@vskip 0pt plus 1filll -Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and -Virginia Polytechnic Institute and State University, Blacksburg, -Virginia, USA. Portions of the software copyright 1991-1995 by -Stichting Mathematisch Centrum, Amsterdam, The Netherlands. Copying is -permitted under the terms associated with the main Python distribution, -with the additional restriction that this additional notice be included -and maintained on all distributed copies. - -@center All Rights Reserved - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the names of Fred L. Drake, Jr. and -Virginia Polytechnic Institute and State University not be used in -advertising or publicity pertaining to distribution of the software -without specific, written prior permission. - -FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE -UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO -EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND -STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL -DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR -PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. -@end titlepage - - -@node Top, Overview, (dir), (dir) -@top The Python Parser Module - -@ifinfo -This file describes the interfaces -published by the optional @code{parser} module and gives examples of -how they may be used. It contains the same text as the chapter on the -@code{parser} module in the @cite{Python Library Reference}, but is -presented as a separate document. - -This version corresponds to Python version 1.4 (1 Sept. 1996). - -@end ifinfo - -@c placeholder for the master menu -- patched by texinfo-all-menus-update -@menu -@end menu diff --git a/Demo/parser/unparse.py b/Demo/parser/unparse.py deleted file mode 100644 index e96ef54..0000000 --- a/Demo/parser/unparse.py +++ /dev/null @@ -1,600 +0,0 @@ -"Usage: unparse.py <path to source file>" -import sys -import math -import ast -import tokenize -import io -import os - -# Large float and imaginary literals get turned into infinities in the AST. -# We unparse those infinities to INFSTR. -INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) - -def interleave(inter, f, seq): - """Call f on each item in seq, calling inter() in between. - """ - seq = iter(seq) - try: - f(next(seq)) - except StopIteration: - pass - else: - for x in seq: - inter() - f(x) - -class Unparser: - """Methods in this class recursively traverse an AST and - output source code for the abstract syntax; original formatting - is disregarded. """ - - def __init__(self, tree, file = sys.stdout): - """Unparser(tree, file=sys.stdout) -> None. - Print the source for tree to file.""" - self.f = file - self._indent = 0 - self.dispatch(tree) - print("", file=self.f) - self.f.flush() - - def fill(self, text = ""): - "Indent a piece of text, according to the current indentation level" - self.f.write("\n"+" "*self._indent + text) - - def write(self, text): - "Append a piece of text to the current line." - self.f.write(text) - - def enter(self): - "Print ':', and increase the indentation." - self.write(":") - self._indent += 1 - - def leave(self): - "Decrease the indentation level." - self._indent -= 1 - - def dispatch(self, tree): - "Dispatcher function, dispatching tree type T to method _T." - if isinstance(tree, list): - for t in tree: - self.dispatch(t) - return - meth = getattr(self, "_"+tree.__class__.__name__) - meth(tree) - - - ############### Unparsing methods ###################### - # There should be one method per concrete grammar type # - # Constructors should be grouped by sum type. Ideally, # - # this would follow the order in the grammar, but # - # currently doesn't. # - ######################################################## - - def _Module(self, tree): - for stmt in tree.body: - self.dispatch(stmt) - - # stmt - def _Expr(self, tree): - self.fill() - self.dispatch(tree.value) - - def _Import(self, t): - self.fill("import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _ImportFrom(self, t): - self.fill("from ") - self.write("." * t.level) - if t.module: - self.write(t.module) - self.write(" import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _Assign(self, t): - self.fill() - for target in t.targets: - self.dispatch(target) - self.write(" = ") - self.dispatch(t.value) - - def _AugAssign(self, t): - self.fill() - self.dispatch(t.target) - self.write(" "+self.binop[t.op.__class__.__name__]+"= ") - self.dispatch(t.value) - - def _Return(self, t): - self.fill("return") - if t.value: - self.write(" ") - self.dispatch(t.value) - - def _Pass(self, t): - self.fill("pass") - - def _Break(self, t): - self.fill("break") - - def _Continue(self, t): - self.fill("continue") - - def _Delete(self, t): - self.fill("del ") - interleave(lambda: self.write(", "), self.dispatch, t.targets) - - def _Assert(self, t): - self.fill("assert ") - self.dispatch(t.test) - if t.msg: - self.write(", ") - self.dispatch(t.msg) - - def _Global(self, t): - self.fill("global ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Nonlocal(self, t): - self.fill("nonlocal ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Yield(self, t): - self.write("(") - self.write("yield") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _Raise(self, t): - self.fill("raise") - if not t.exc: - assert not t.cause - return - self.write(" ") - self.dispatch(t.exc) - if t.cause: - self.write(" from ") - self.dispatch(t.cause) - - def _TryExcept(self, t): - self.fill("try") - self.enter() - self.dispatch(t.body) - self.leave() - - for ex in t.handlers: - self.dispatch(ex) - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _TryFinally(self, t): - if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept): - # try-except-finally - self.dispatch(t.body) - else: - self.fill("try") - self.enter() - self.dispatch(t.body) - self.leave() - - self.fill("finally") - self.enter() - self.dispatch(t.finalbody) - self.leave() - - def _ExceptHandler(self, t): - self.fill("except") - if t.type: - self.write(" ") - self.dispatch(t.type) - if t.name: - self.write(" as ") - self.write(t.name) - self.enter() - self.dispatch(t.body) - self.leave() - - def _ClassDef(self, t): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - self.fill("class "+t.name) - self.write("(") - comma = False - for e in t.bases: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - if t.starargs: - if comma: self.write(", ") - else: comma = True - self.write("*") - self.dispatch(t.starargs) - if t.kwargs: - if comma: self.write(", ") - else: comma = True - self.write("**") - self.dispatch(t.kwargs) - self.write(")") - - self.enter() - self.dispatch(t.body) - self.leave() - - def _FunctionDef(self, t): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - self.fill("def "+t.name + "(") - self.dispatch(t.args) - self.write(")") - if t.returns: - self.write(" -> ") - self.dispatch(t.returns) - self.enter() - self.dispatch(t.body) - self.leave() - - def _For(self, t): - self.fill("for ") - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _If(self, t): - self.fill("if ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # collapse nested ifs into equivalent elifs. - while (t.orelse and len(t.orelse) == 1 and - isinstance(t.orelse[0], ast.If)): - t = t.orelse[0] - self.fill("elif ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # final else - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _While(self, t): - self.fill("while ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _With(self, t): - self.fill("with ") - self.dispatch(t.context_expr) - if t.optional_vars: - self.write(" as ") - self.dispatch(t.optional_vars) - self.enter() - self.dispatch(t.body) - self.leave() - - # expr - def _Bytes(self, t): - self.write(repr(t.s)) - - def _Str(self, tree): - self.write(repr(tree.s)) - - def _Name(self, t): - self.write(t.id) - - def _Num(self, t): - # Substitute overflowing decimal literal for AST infinities. - self.write(repr(t.n).replace("inf", INFSTR)) - - def _List(self, t): - self.write("[") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("]") - - def _ListComp(self, t): - self.write("[") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("]") - - def _GeneratorExp(self, t): - self.write("(") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write(")") - - def _SetComp(self, t): - self.write("{") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _DictComp(self, t): - self.write("{") - self.dispatch(t.key) - self.write(": ") - self.dispatch(t.value) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _comprehension(self, t): - self.write(" for ") - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - for if_clause in t.ifs: - self.write(" if ") - self.dispatch(if_clause) - - def _IfExp(self, t): - self.write("(") - self.dispatch(t.body) - self.write(" if ") - self.dispatch(t.test) - self.write(" else ") - self.dispatch(t.orelse) - self.write(")") - - def _Set(self, t): - assert(t.elts) # should be at least one element - self.write("{") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("}") - - def _Dict(self, t): - self.write("{") - def write_pair(pair): - (k, v) = pair - self.dispatch(k) - self.write(": ") - self.dispatch(v) - interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values)) - self.write("}") - - def _Tuple(self, t): - self.write("(") - if len(t.elts) == 1: - (elt,) = t.elts - self.dispatch(elt) - self.write(",") - else: - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write(")") - - unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} - def _UnaryOp(self, t): - self.write("(") - self.write(self.unop[t.op.__class__.__name__]) - self.write(" ") - self.dispatch(t.operand) - self.write(")") - - binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%", - "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", - "FloorDiv":"//", "Pow": "**"} - def _BinOp(self, t): - self.write("(") - self.dispatch(t.left) - self.write(" " + self.binop[t.op.__class__.__name__] + " ") - self.dispatch(t.right) - self.write(")") - - cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", - "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} - def _Compare(self, t): - self.write("(") - self.dispatch(t.left) - for o, e in zip(t.ops, t.comparators): - self.write(" " + self.cmpops[o.__class__.__name__] + " ") - self.dispatch(e) - self.write(")") - - boolops = {ast.And: 'and', ast.Or: 'or'} - def _BoolOp(self, t): - self.write("(") - s = " %s " % self.boolops[t.op.__class__] - interleave(lambda: self.write(s), self.dispatch, t.values) - self.write(")") - - def _Attribute(self,t): - self.dispatch(t.value) - # Special case: 3.__abs__() is a syntax error, so if t.value - # is an integer literal then we need to either parenthesize - # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): - self.write(" ") - self.write(".") - self.write(t.attr) - - def _Call(self, t): - self.dispatch(t.func) - self.write("(") - comma = False - for e in t.args: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - if t.starargs: - if comma: self.write(", ") - else: comma = True - self.write("*") - self.dispatch(t.starargs) - if t.kwargs: - if comma: self.write(", ") - else: comma = True - self.write("**") - self.dispatch(t.kwargs) - self.write(")") - - def _Subscript(self, t): - self.dispatch(t.value) - self.write("[") - self.dispatch(t.slice) - self.write("]") - - # slice - def _Ellipsis(self, t): - self.write("...") - - def _Index(self, t): - self.dispatch(t.value) - - def _Slice(self, t): - if t.lower: - self.dispatch(t.lower) - self.write(":") - if t.upper: - self.dispatch(t.upper) - if t.step: - self.write(":") - self.dispatch(t.step) - - def _ExtSlice(self, t): - interleave(lambda: self.write(', '), self.dispatch, t.dims) - - # argument - def _arg(self, t): - self.write(t.arg) - if t.annotation: - self.write(": ") - self.dispatch(t.annotation) - - # others - def _arguments(self, t): - first = True - # normal arguments - defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults - for a, d in zip(t.args, defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a) - if d: - self.write("=") - self.dispatch(d) - - # varargs, or bare '*' if no varargs but keyword-only arguments present - if t.vararg or t.kwonlyargs: - if first:first = False - else: self.write(", ") - self.write("*") - if t.vararg: - self.write(t.vararg) - if t.varargannotation: - self.write(": ") - self.dispatch(t.varargannotation) - - # keyword-only arguments - if t.kwonlyargs: - for a, d in zip(t.kwonlyargs, t.kw_defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a), - if d: - self.write("=") - self.dispatch(d) - - # kwargs - if t.kwarg: - if first:first = False - else: self.write(", ") - self.write("**"+t.kwarg) - if t.kwargannotation: - self.write(": ") - self.dispatch(t.kwargannotation) - - def _keyword(self, t): - self.write(t.arg) - self.write("=") - self.dispatch(t.value) - - def _Lambda(self, t): - self.write("(") - self.write("lambda ") - self.dispatch(t.args) - self.write(": ") - self.dispatch(t.body) - self.write(")") - - def _alias(self, t): - self.write(t.name) - if t.asname: - self.write(" as "+t.asname) - -def roundtrip(filename, output=sys.stdout): - with open(filename, "rb") as pyfile: - encoding = tokenize.detect_encoding(pyfile.readline)[0] - with open(filename, "r", encoding=encoding) as pyfile: - source = pyfile.read() - tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) - Unparser(tree, output) - - - -def testdir(a): - try: - names = [n for n in os.listdir(a) if n.endswith('.py')] - except OSError: - print("Directory not readable: %s" % a, file=sys.stderr) - else: - for n in names: - fullname = os.path.join(a, n) - if os.path.isfile(fullname): - output = io.StringIO() - print('Testing %s' % fullname) - try: - roundtrip(fullname, output) - except Exception as e: - print(' Failed to compile, exception is %s' % repr(e)) - elif os.path.isdir(fullname): - testdir(fullname) - -def main(args): - if args[0] == '--testdir': - for a in args[1:]: - testdir(a) - else: - for a in args: - roundtrip(a) - -if __name__=='__main__': - main(sys.argv[1:]) |