diff options
author | Mark Dickinson <dickinsm@gmail.com> | 2010-06-30 14:19:56 (GMT) |
---|---|---|
committer | Mark Dickinson <dickinsm@gmail.com> | 2010-06-30 14:19:56 (GMT) |
commit | 20379135076182fc185fee15f918df6d44924d4c (patch) | |
tree | 99990048d5b1bcaf06961392a40d5bcfb9ba7a30 /Demo | |
parent | 4aa5f6f1f2b080bf1f17cfb97edb2ed9483e2e0d (diff) | |
download | cpython-20379135076182fc185fee15f918df6d44924d4c.zip cpython-20379135076182fc185fee15f918df6d44924d4c.tar.gz cpython-20379135076182fc185fee15f918df6d44924d4c.tar.bz2 |
Update Demo/parser directory; backport unparse fixes from py3k.
Diffstat (limited to 'Demo')
-rw-r--r-- | Demo/parser/README | 29 | ||||
-rwxr-xr-x | Demo/parser/test_parser.py | 12 | ||||
-rw-r--r-- | Demo/parser/test_unparse.py | 172 | ||||
-rw-r--r-- | Demo/parser/texipre.dat | 100 | ||||
-rw-r--r-- | Demo/parser/unparse.py | 154 |
5 files changed, 297 insertions, 170 deletions
diff --git a/Demo/parser/README b/Demo/parser/README index a576d33..fd5eb25 100644 --- a/Demo/parser/README +++ b/Demo/parser/README @@ -6,26 +6,27 @@ It also contains examples for the AST parser. Files: ------ - FILES -- list of files associated with the parser module. + FILES -- list of files associated with the parser module. - README -- this file. + README -- this file. - example.py -- module that uses the `parser' module to extract - information from the parse tree of Python source - code. + docstring.py -- sample source file containing only a module docstring. - docstring.py -- sample source file containing only a module docstring. + example.py -- module that uses the `parser' module to extract + information from the parse tree of Python source + code. - simple.py -- sample source containing a "short form" definition. + simple.py -- sample source containing a "short form" definition. - source.py -- sample source code used to demonstrate ability to - handle nested constructs easily using the functions - and classes in example.py. + source.py -- sample source code used to demonstrate ability to + handle nested constructs easily using the functions + and classes in example.py. - test_parser.py program to put the parser module through its paces. + test_parser.py program to put the parser module through its paces. - unparse.py AST (2.5) based example to recreate source code - from an AST. This is incomplete; contributions - are welcome. + test_unparse.py tests for the unparse module + + unparse.py AST (2.7) based example to recreate source code + from an AST. Enjoy! diff --git a/Demo/parser/test_parser.py b/Demo/parser/test_parser.py index be39bca..db4e6fe 100755 --- a/Demo/parser/test_parser.py +++ b/Demo/parser/test_parser.py @@ -11,19 +11,19 @@ def testChunk(t, fileName): global _numFailed print '----', fileName, try: - ast = parser.suite(t) - tup = parser.ast2tuple(ast) - # this discards the first AST; a huge memory savings when running + st = parser.suite(t) + tup = parser.st2tuple(st) + # this discards the first ST; a huge memory savings when running # against a large source file like Tkinter.py. - ast = None - new = parser.tuple2ast(tup) + st = None + new = parser.tuple2st(tup) except parser.ParserError, err: print print 'parser module raised exception on input file', fileName + ':' traceback.print_exc() _numFailed = _numFailed + 1 else: - if tup != parser.ast2tuple(new): + if tup != parser.st2tuple(new): print print 'parser module failed on input file', fileName _numFailed = _numFailed + 1 diff --git a/Demo/parser/test_unparse.py b/Demo/parser/test_unparse.py index 857b999..2ecfd37 100644 --- a/Demo/parser/test_unparse.py +++ b/Demo/parser/test_unparse.py @@ -1,12 +1,20 @@ import unittest from test import test_support - import cStringIO +import sys +import os +import tokenize import ast -import _ast import unparse -forelse = """\ +def read_pyfile(filename): + """Read and return the contents of a Python source file (as a + string), taking into account the file encoding.""" + with open(filename, "r") as pyfile: + source = pyfile.read() + return source + +for_else = """\ def f(): for x in range(10): break @@ -15,7 +23,7 @@ def f(): z = 3 """ -whileelse = """\ +while_else = """\ def g(): while True: break @@ -24,16 +32,63 @@ def g(): z = 3 """ -class UnparseTestCase(unittest.TestCase): - # Tests for specific bugs found in earlier versions of unparse +relative_import = """\ +from . import fred +from .. import barney +from .australia import shrimp as prawns +""" + +class_decorator = """\ +@f1(arg) +@f2 +class Foo: pass +""" + +elif1 = """\ +if cond1: + suite1 +elif cond2: + suite2 +else: + suite3 +""" + +elif2 = """\ +if cond1: + suite1 +elif cond2: + suite2 +""" + +try_except_finally = """\ +try: + suite1 +except ex1: + suite2 +except ex2: + suite3 +else: + suite4 +finally: + suite5 +""" + +class ASTTestCase(unittest.TestCase): + def assertASTEqual(self, ast1, ast2): + dump1 = ast.dump(ast1) + dump2 = ast.dump(ast2) + self.assertEqual(ast.dump(ast1), ast.dump(ast2)) def check_roundtrip(self, code1, filename="internal"): - ast1 = compile(code1, filename, "exec", _ast.PyCF_ONLY_AST) + ast1 = compile(code1, filename, "exec", ast.PyCF_ONLY_AST) unparse_buffer = cStringIO.StringIO() unparse.Unparser(ast1, unparse_buffer) code2 = unparse_buffer.getvalue() - ast2 = compile(code2, filename, "exec", _ast.PyCF_ONLY_AST) - self.assertEqual(ast.dump(ast1), ast.dump(ast2)) + ast2 = compile(code2, filename, "exec", ast.PyCF_ONLY_AST) + self.assertASTEqual(ast1, ast2) + +class UnparseTestCase(ASTTestCase): + # Tests for specific bugs found in earlier versions of unparse def test_del_statement(self): self.check_roundtrip("del x, y, z") @@ -43,23 +98,116 @@ class UnparseTestCase(unittest.TestCase): self.check_roundtrip("13 >> 7") def test_for_else(self): - self.check_roundtrip(forelse) + self.check_roundtrip(for_else) def test_while_else(self): - self.check_roundtrip(whileelse) + self.check_roundtrip(while_else) def test_unary_parens(self): self.check_roundtrip("(-1)**7") + self.check_roundtrip("(-1.)**8") + self.check_roundtrip("(-1j)**6") self.check_roundtrip("not True or False") self.check_roundtrip("True or not False") + def test_integer_parens(self): + self.check_roundtrip("3 .__abs__()") + + def test_huge_float(self): + self.check_roundtrip("1e1000") + self.check_roundtrip("-1e1000") + self.check_roundtrip("1e1000j") + self.check_roundtrip("-1e1000j") + + def test_min_int(self): + self.check_roundtrip(str(-sys.maxint-1)) + self.check_roundtrip("-(%s)" % (sys.maxint + 1)) + + def test_imaginary_literals(self): + self.check_roundtrip("7j") + self.check_roundtrip("-7j") + self.check_roundtrip("-(7j)") + self.check_roundtrip("0j") + self.check_roundtrip("-0j") + self.check_roundtrip("-(0j)") + + def test_negative_zero(self): + self.check_roundtrip("-0") + self.check_roundtrip("-(0)") + self.check_roundtrip("-0b0") + self.check_roundtrip("-(0b0)") + self.check_roundtrip("-0o0") + self.check_roundtrip("-(0o0)") + self.check_roundtrip("-0x0") + self.check_roundtrip("-(0x0)") + + def test_lambda_parentheses(self): + self.check_roundtrip("(lambda: int)()") + def test_chained_comparisons(self): self.check_roundtrip("1 < 4 <= 5") self.check_roundtrip("a is b is c is not d") + def test_function_arguments(self): + self.check_roundtrip("def f(): pass") + self.check_roundtrip("def f(a): pass") + self.check_roundtrip("def f(b = 2): pass") + self.check_roundtrip("def f(a, b): pass") + self.check_roundtrip("def f(a, b = 2): pass") + self.check_roundtrip("def f(a = 5, b = 2): pass") + self.check_roundtrip("def f(*args, **kwargs): pass") + + def test_relative_import(self): + self.check_roundtrip(relative_import) + + def test_bytes(self): + self.check_roundtrip("b'123'") + + def test_set_literal(self): + self.check_roundtrip("{'a', 'b', 'c'}") + + def test_set_comprehension(self): + self.check_roundtrip("{x for x in range(5)}") + + def test_dict_comprehension(self): + self.check_roundtrip("{x: x*x for x in range(10)}") + + def test_class_decorators(self): + self.check_roundtrip(class_decorator) + + def test_elifs(self): + self.check_roundtrip(elif1) + self.check_roundtrip(elif2) + + def test_try_except_finally(self): + self.check_roundtrip(try_except_finally) + +class DirectoryTestCase(ASTTestCase): + """Test roundtrip behaviour on all files in Lib and Lib/test.""" + + # test directories, relative to the root of the distribution + test_directories = 'Lib', os.path.join('Lib', 'test') + + def test_files(self): + # get names of files to test + dist_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) + + names = [] + for d in self.test_directories: + test_dir = os.path.join(dist_dir, d) + for n in os.listdir(test_dir): + if n.endswith('.py') and not n.startswith('bad'): + names.append(os.path.join(test_dir, n)) + + for filename in names: + if test_support.verbose: + print('Testing %s' % filename) + source = read_pyfile(filename) + self.check_roundtrip(source) + def test_main(): - test_support.run_unittest(UnparseTestCase) + test_support.run_unittest(UnparseTestCase, DirectoryTestCase) if __name__ == '__main__': test_main() diff --git a/Demo/parser/texipre.dat b/Demo/parser/texipre.dat deleted file mode 100644 index 8ad03a6..0000000 --- a/Demo/parser/texipre.dat +++ /dev/null @@ -1,100 +0,0 @@ -\input texinfo @c -*-texinfo-*- -@c %**start of header -@setfilename parser.info -@settitle Python Parser Module Reference -@setchapternewpage odd -@footnotestyle end -@c %**end of header - -@ifinfo -This file describes the interfaces -published by the optional @code{parser} module and gives examples of -how they may be used. It contains the same text as the chapter on the -@code{parser} module in the @cite{Python Library Reference}, but is -presented as a separate document. - -Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and -Virginia Polytechnic Institute and State University, Blacksburg, -Virginia, USA. Portions of the software copyright 1991-1995 by -Stichting Mathematisch Centrum, Amsterdam, The Netherlands. Copying is -permitted under the terms associated with the main Python distribution, -with the additional restriction that this additional notice be included -and maintained on all distributed copies. - - All Rights Reserved - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the names of Fred L. Drake, Jr. and -Virginia Polytechnic Institute and State University not be used in -advertising or publicity pertaining to distribution of the software -without specific, written prior permission. - -FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE -UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO -EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND -STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL -DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR -PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. -@end ifinfo - -@titlepage -@title Python Parser Module Reference -@author Fred L. Drake, Jr. - -@c The following two commands start the copyright page. -@page -@vskip 0pt plus 1filll -Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and -Virginia Polytechnic Institute and State University, Blacksburg, -Virginia, USA. Portions of the software copyright 1991-1995 by -Stichting Mathematisch Centrum, Amsterdam, The Netherlands. Copying is -permitted under the terms associated with the main Python distribution, -with the additional restriction that this additional notice be included -and maintained on all distributed copies. - -@center All Rights Reserved - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the names of Fred L. Drake, Jr. and -Virginia Polytechnic Institute and State University not be used in -advertising or publicity pertaining to distribution of the software -without specific, written prior permission. - -FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE -UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO -EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND -STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL -DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR -PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. -@end titlepage - - -@node Top, Overview, (dir), (dir) -@top The Python Parser Module - -@ifinfo -This file describes the interfaces -published by the optional @code{parser} module and gives examples of -how they may be used. It contains the same text as the chapter on the -@code{parser} module in the @cite{Python Library Reference}, but is -presented as a separate document. - -This version corresponds to Python version 1.4 (1 Sept. 1996). - -@end ifinfo - -@c placeholder for the master menu -- patched by texinfo-all-menus-update -@menu -@end menu diff --git a/Demo/parser/unparse.py b/Demo/parser/unparse.py index 98aeb32..1b1ff29 100644 --- a/Demo/parser/unparse.py +++ b/Demo/parser/unparse.py @@ -1,15 +1,19 @@ "Usage: unparse.py <path to source file>" import sys -import _ast +import ast import cStringIO import os +# Large float and imaginary literals get turned into infinities in the AST. +# We unparse those infinities to INFSTR. +INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) + def interleave(inter, f, seq): """Call f on each item in seq, calling inter() in between. """ seq = iter(seq) try: - f(seq.next()) + f(next(seq)) except StopIteration: pass else: @@ -20,15 +24,16 @@ def interleave(inter, f, seq): class Unparser: """Methods in this class recursively traverse an AST and output source code for the abstract syntax; original formatting - is disregarged. """ + is disregarded. """ def __init__(self, tree, file = sys.stdout): """Unparser(tree, file=sys.stdout) -> None. Print the source for tree to file.""" self.f = file + self.future_imports = [] self._indent = 0 self.dispatch(tree) - print >>self.f,"" + self.f.write("") self.f.flush() def fill(self, text = ""): @@ -79,11 +84,16 @@ class Unparser: interleave(lambda: self.write(", "), self.dispatch, t.names) def _ImportFrom(self, t): + # A from __future__ import may affect unparsing, so record it. + if t.module and t.module == '__future__': + self.future_imports.extend(n.name for n in t.names) + self.fill("from ") - self.write(t.module) + self.write("." * t.level) + if t.module: + self.write(t.module) self.write(" import ") interleave(lambda: self.write(", "), self.dispatch, t.names) - # XXX(jpe) what is level for? def _Assign(self, t): self.fill() @@ -186,10 +196,14 @@ class Unparser: self.leave() def _TryFinally(self, t): - self.fill("try") - self.enter() - self.dispatch(t.body) - self.leave() + if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept): + # try-except-finally + self.dispatch(t.body) + else: + self.fill("try") + self.enter() + self.dispatch(t.body) + self.leave() self.fill("finally") self.enter() @@ -202,7 +216,7 @@ class Unparser: self.write(" ") self.dispatch(t.type) if t.name: - self.write(", ") + self.write(" as ") self.dispatch(t.name) self.enter() self.dispatch(t.body) @@ -210,6 +224,9 @@ class Unparser: def _ClassDef(self, t): self.write("\n") + for deco in t.decorator_list: + self.fill("@") + self.dispatch(deco) self.fill("class "+t.name) if t.bases: self.write("(") @@ -251,9 +268,18 @@ class Unparser: self.fill("if ") self.dispatch(t.test) self.enter() - # XXX elif? self.dispatch(t.body) self.leave() + # collapse nested ifs into equivalent elifs. + while (t.orelse and len(t.orelse) == 1 and + isinstance(t.orelse[0], ast.If)): + t = t.orelse[0] + self.fill("elif ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + # final else if t.orelse: self.fill("else") self.enter() @@ -284,7 +310,17 @@ class Unparser: # expr def _Str(self, tree): - self.write(repr(tree.s)) + # if from __future__ import unicode_literals is in effect, + # then we want to output string literals using a 'b' prefix + # and unicode literals with no prefix. + if "unicode_literals" not in self.future_imports: + self.write(repr(tree.s)) + elif isinstance(tree.s, str): + self.write("b" + repr(tree.s)) + elif isinstance(tree.s, unicode): + self.write(repr(tree.s).lstrip("u")) + else: + assert False, "shouldn't get here" def _Name(self, t): self.write(t.id) @@ -295,16 +331,14 @@ class Unparser: self.write("`") def _Num(self, t): - # There are no negative numeric literals in Python; however, - # some optimizations produce a negative Num in the AST. Add - # parentheses to avoid turning (-1)**2 into -1**2. - strnum = repr(t.n) - if strnum.startswith("-"): + repr_n = repr(t.n) + # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2. + if repr_n.startswith("-"): self.write("(") - self.write(strnum) + # Substitute overflowing decimal literal for AST infinities. + self.write(repr_n.replace("inf", INFSTR)) + if repr_n.startswith("-"): self.write(")") - else: - self.write(repr(t.n)) def _List(self, t): self.write("[") @@ -325,6 +359,22 @@ class Unparser: self.dispatch(gen) self.write(")") + def _SetComp(self, t): + self.write("{") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write("}") + + def _DictComp(self, t): + self.write("{") + self.dispatch(t.key) + self.write(": ") + self.dispatch(t.value) + for gen in t.generators: + self.dispatch(gen) + self.write("}") + def _comprehension(self, t): self.write(" for ") self.dispatch(t.target) @@ -343,13 +393,20 @@ class Unparser: self.dispatch(t.orelse) self.write(")") + def _Set(self, t): + assert(t.elts) # should be at least one element + self.write("{") + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write("}") + def _Dict(self, t): self.write("{") - def writem((k, v)): + def write_pair(pair): + (k, v) = pair self.dispatch(k) self.write(": ") self.dispatch(v) - interleave(lambda: self.write(", "), writem, zip(t.keys, t.values)) + interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values)) self.write("}") def _Tuple(self, t): @@ -367,7 +424,17 @@ class Unparser: self.write("(") self.write(self.unop[t.op.__class__.__name__]) self.write(" ") - self.dispatch(t.operand) + # If we're applying unary minus to a number, parenthesize the number. + # This is necessary: -2147483648 is different from -(2147483648) on + # a 32-bit machine (the first is an int, the second a long), and + # -7j is different from -(7j). (The first has real part 0.0, the second + # has real part -0.0.) + if isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num): + self.write("(") + self.dispatch(t.operand) + self.write(")") + else: + self.dispatch(t.operand) self.write(")") binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%", @@ -390,7 +457,7 @@ class Unparser: self.dispatch(e) self.write(")") - boolops = {_ast.And: 'and', _ast.Or: 'or'} + boolops = {ast.And: 'and', ast.Or: 'or'} def _BoolOp(self, t): self.write("(") s = " %s " % self.boolops[t.op.__class__] @@ -399,6 +466,11 @@ class Unparser: def _Attribute(self,t): self.dispatch(t.value) + # Special case: 3.__abs__() is a syntax error, so if t.value + # is an integer literal then we need to either parenthesize + # it or add an extra space to get 3 .__abs__(). + if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): + self.write(" ") self.write(".") self.write(t.attr) @@ -455,21 +527,24 @@ class Unparser: # others def _arguments(self, t): first = True - nonDef = len(t.args)-len(t.defaults) - for a in t.args[0:nonDef]: - if first:first = False - else: self.write(", ") - self.dispatch(a) - for a,d in zip(t.args[nonDef:], t.defaults): + # normal arguments + defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults + for a,d in zip(t.args, defaults): if first:first = False else: self.write(", ") self.dispatch(a), - self.write("=") - self.dispatch(d) + if d: + self.write("=") + self.dispatch(d) + + # varargs if t.vararg: if first:first = False else: self.write(", ") - self.write("*"+t.vararg) + self.write("*") + self.write(t.vararg) + + # kwargs if t.kwarg: if first:first = False else: self.write(", ") @@ -481,10 +556,12 @@ class Unparser: self.dispatch(t.value) def _Lambda(self, t): + self.write("(") self.write("lambda ") self.dispatch(t.args) self.write(": ") self.dispatch(t.body) + self.write(")") def _alias(self, t): self.write(t.name) @@ -492,8 +569,9 @@ class Unparser: self.write(" as "+t.asname) def roundtrip(filename, output=sys.stdout): - source = open(filename).read() - tree = compile(source, filename, "exec", _ast.PyCF_ONLY_AST) + with open(filename, "r") as pyfile: + source = pyfile.read() + tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) Unparser(tree, output) @@ -502,7 +580,7 @@ def testdir(a): try: names = [n for n in os.listdir(a) if n.endswith('.py')] except OSError: - print >> sys.stderr, "Directory not readable: %s" % a + sys.stderr.write("Directory not readable: %s" % a) else: for n in names: fullname = os.path.join(a, n) @@ -511,7 +589,7 @@ def testdir(a): print 'Testing %s' % fullname try: roundtrip(fullname, output) - except Exception, e: + except Exception as e: print ' Failed to compile, exception is %s' % repr(e) elif os.path.isdir(fullname): testdir(fullname) |