summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Lib/compiler/ast.py61
-rw-r--r--Lib/compiler/pyassem.py10
-rw-r--r--Lib/compiler/pycodegen.py278
-rw-r--r--Lib/compiler/transformer.py207
-rw-r--r--Tools/compiler/compiler/ast.py61
-rw-r--r--Tools/compiler/compiler/pyassem.py10
-rw-r--r--Tools/compiler/compiler/pycodegen.py278
-rw-r--r--Tools/compiler/compiler/transformer.py207
8 files changed, 926 insertions, 186 deletions
diff --git a/Lib/compiler/ast.py b/Lib/compiler/ast.py
index a3e51b7..5b0a06a 100644
--- a/Lib/compiler/ast.py
+++ b/Lib/compiler/ast.py
@@ -279,22 +279,24 @@ class Const(Node):
class Print(Node):
nodes['print'] = 'Print'
- def __init__(self, nodes):
+ def __init__(self, nodes, dest):
self.nodes = nodes
- self._children = ('print', nodes)
+ self.dest = dest
+ self._children = ('print', nodes, dest)
def __repr__(self):
- return "Print(%s)" % self._children[1:]
+ return "Print(%s, %s)" % (self._children[1:-1], self._children[-1])
class Printnl(Node):
nodes['printnl'] = 'Printnl'
- def __init__(self, nodes):
+ def __init__(self, nodes, dest):
self.nodes = nodes
- self._children = ('printnl', nodes)
+ self.dest = dest
+ self._children = ('printnl', nodes, dest)
def __repr__(self):
- return "Printnl(%s)" % self._children[1:]
+ return "Printnl(%s, %s)" % (self._children[1:-1], self._children[-1])
class Discard(Node):
nodes['discard'] = 'Discard'
@@ -306,6 +308,18 @@ class Discard(Node):
def __repr__(self):
return "Discard(%s)" % self._children[1:]
+class AugAssign(Node):
+ nodes['augassign'] = 'AugAssign'
+
+ def __init__(self, node, op, expr):
+ self.node = node
+ self.op = op
+ self.expr = expr
+ self._children = ('augassign', node, op, expr)
+
+ def __repr__(self):
+ return "AugAssign(%s)" % str(self._children[1:])
+
class Assign(Node):
nodes['assign'] = 'Assign'
@@ -360,6 +374,41 @@ class AssAttr(Node):
def __repr__(self):
return "AssAttr(%s,%s,%s)" % self._children[1:]
+class ListComp(Node):
+ nodes['listcomp'] = 'ListComp'
+
+ def __init__(self, expr, quals):
+ self.expr = expr
+ self.quals = quals
+ self._children = ('listcomp', expr, quals)
+
+ def __repr__(self):
+ return "ListComp(%s, %s)" % self._children[1:]
+
+class ListCompFor(Node):
+ nodes['listcomp_for'] = 'ListCompFor'
+
+ # transformer fills in ifs after node is created
+
+ def __init__(self, assign, list, ifs):
+ self.assign = assign
+ self.list = list
+ self.ifs = ifs
+ self._children = ('listcomp_for', assign, list, ifs)
+
+ def __repr__(self):
+ return "ListCompFor(%s, %s, %s)" % self._children[1:]
+
+class ListCompIf(Node):
+ nodes['listcomp_if'] = 'ListCompIf'
+
+ def __init__(self, test):
+ self.test = test
+ self._children = ('listcomp_if', test)
+
+ def __repr__(self):
+ return "ListCompIf(%s)" % self._children[1:]
+
class List(Node):
nodes['list'] = 'List'
diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py
index 3411273..dcc8bc0 100644
--- a/Lib/compiler/pyassem.py
+++ b/Lib/compiler/pyassem.py
@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph):
def _lookupName(self, name, list):
"""Return index of name in list, appending if necessary"""
- if name in list:
- i = list.index(name)
+ found = None
+ t = type(name)
+ for i in range(len(list)):
+ # must do a comparison on type first to prevent UnicodeErrors
+ if t == type(list[i]) and list[i] == name:
+ found = 1
+ break
+ if found:
# this is cheap, but incorrect in some cases, e.g 2 vs. 2L
if type(name) == type(list[i]):
return i
diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py
index 2888729..bf54c32 100644
--- a/Lib/compiler/pycodegen.py
+++ b/Lib/compiler/pycodegen.py
@@ -1,8 +1,10 @@
+import imp
import os
import marshal
import stat
import string
import struct
+import sys
import types
from cStringIO import StringIO
@@ -10,6 +12,12 @@ from compiler import ast, parse, walk
from compiler import pyassem, misc
from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg
+# Do we have Python 1.x or Python 2.x?
+try:
+ VERSION = sys.version_info[0]
+except AttributeError:
+ VERSION = 1
+
callfunc_opcode_info = {
# (Have *args, Have **args) : opcode
(0,0) : "CALL_FUNCTION",
@@ -18,12 +26,12 @@ callfunc_opcode_info = {
(1,1) : "CALL_FUNCTION_VAR_KW",
}
-def compile(filename):
+def compile(filename, display=0):
f = open(filename)
buf = f.read()
f.close()
mod = Module(buf, filename)
- mod.compile()
+ mod.compile(display)
f = open(filename + "c", "wb")
mod.dump(f)
f.close()
@@ -34,28 +42,30 @@ class Module:
self.source = source
self.code = None
- def compile(self):
+ def compile(self, display=0):
ast = parse(self.source)
root, filename = os.path.split(self.filename)
gen = ModuleCodeGenerator(filename)
walk(ast, gen, 1)
+ if display:
+ import pprint
+ print pprint.pprint(ast)
self.code = gen.getCode()
def dump(self, f):
f.write(self.getPycHeader())
marshal.dump(self.code, f)
- MAGIC = (50823 | (ord('\r')<<16) | (ord('\n')<<24))
+ MAGIC = imp.get_magic()
def getPycHeader(self):
# compile.c uses marshal to write a long directly, with
# calling the interface that would also generate a 1-byte code
# to indicate the type of the value. simplest way to get the
# same effect is to call marshal and then skip the code.
- magic = marshal.dumps(self.MAGIC)[1:]
mtime = os.stat(self.filename)[stat.ST_MTIME]
mtime = struct.pack('i', mtime)
- return magic + mtime
+ return self.MAGIC + mtime
class CodeGenerator:
@@ -63,7 +73,7 @@ class CodeGenerator:
def __init__(self, filename):
## Subclasses must define a constructor that intializes self.graph
-## before calling this init function
+## before calling this init function, e.g.
## self.graph = pyassem.PyFlowGraph()
self.filename = filename
self.locals = misc.Stack()
@@ -142,7 +152,6 @@ class CodeGenerator:
def visitLambda(self, node):
self._visitFuncOrLambda(node, isLambda=1)
-## self.storeName("<lambda>")
def _visitFuncOrLambda(self, node, isLambda):
gen = FunctionCodeGenerator(node, self.filename, isLambda)
@@ -180,10 +189,6 @@ class CodeGenerator:
test, suite = node.tests[i]
self.set_lineno(test)
self.visit(test)
-## if i == numtests - 1 and not node.else_:
-## nextTest = end
-## else:
-## nextTest = self.newBlock()
nextTest = self.newBlock()
self.emit('JUMP_IF_FALSE', nextTest)
self.nextBlock()
@@ -304,6 +309,70 @@ class CodeGenerator:
self.emit('POP_TOP')
self.nextBlock(end)
+ # list comprehensions
+ __list_count = 0
+
+ def visitListComp(self, node):
+ # XXX would it be easier to transform the AST into the form it
+ # would have if the list comp were expressed as a series of
+ # for and if stmts and an explicit append?
+ self.set_lineno(node)
+ # setup list
+ append = "$append%d" % self.__list_count
+ self.__list_count = self.__list_count + 1
+ self.emit('BUILD_LIST', 0)
+ self.emit('DUP_TOP')
+ self.emit('LOAD_ATTR', 'append')
+ self.storeName(append)
+ l = len(node.quals)
+ stack = []
+ for i, for_ in zip(range(l), node.quals):
+ start, anchor = self.visit(for_)
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor))
+
+ self.loadName(append)
+ self.visit(node.expr)
+ self.emit('CALL_FUNCTION', 1)
+ self.emit('POP_TOP')
+
+ for start, cont, anchor in stack:
+ if cont:
+ skip_one = self.newBlock()
+ self.emit('JUMP_FORWARD', skip_one)
+ self.nextBlock(cont)
+ self.emit('POP_TOP')
+ self.nextBlock(skip_one)
+ self.emit('JUMP_ABSOLUTE', start)
+ self.nextBlock(anchor)
+ self.delName(append)
+
+ self.__list_count = self.__list_count - 1
+
+ def visitListCompFor(self, node):
+ self.set_lineno(node)
+ start = self.newBlock()
+ anchor = self.newBlock()
+
+ self.visit(node.list)
+ self.visit(ast.Const(0))
+ self.emit('SET_LINENO', node.lineno)
+ self.nextBlock(start)
+ self.emit('FOR_LOOP', anchor)
+ self.visit(node.assign)
+ return start, anchor
+
+ def visitListCompIf(self, node, branch):
+ self.set_lineno(node)
+ self.visit(node.test)
+ self.emit('JUMP_IF_FALSE', branch)
+ self.newBlock()
+ self.emit('POP_TOP')
+
# exception related
def visitAssert(self, node):
@@ -397,10 +466,6 @@ class CodeGenerator:
# misc
-## def visitStmt(self, node):
-## # nothing to do except walk the children
-## pass
-
def visitDiscard(self, node):
self.visit(node.expr)
self.emit('POP_TOP')
@@ -426,27 +491,32 @@ class CodeGenerator:
def visitImport(self, node):
self.set_lineno(node)
for name, alias in node.names:
- self.emit('LOAD_CONST', None)
+ if VERSION > 1:
+ self.emit('LOAD_CONST', None)
self.emit('IMPORT_NAME', name)
- self._resolveDots(name)
- self.storeName(alias or name)
+ mod = string.split(name, ".")[0]
+ self.storeName(alias or mod)
def visitFrom(self, node):
self.set_lineno(node)
fromlist = map(lambda (name, alias): name, node.names)
- self.emit('LOAD_CONST', tuple(fromlist))
+ if VERSION > 1:
+ self.emit('LOAD_CONST', tuple(fromlist))
self.emit('IMPORT_NAME', node.modname)
for name, alias in node.names:
- if name == '*':
- self.namespace = 0
- self.emit('IMPORT_STAR')
- # There can only be one name w/ from ... import *
- assert len(node.names) == 1
- return
+ if VERSION > 1:
+ if name == '*':
+ self.namespace = 0
+ self.emit('IMPORT_STAR')
+ # There can only be one name w/ from ... import *
+ assert len(node.names) == 1
+ return
+ else:
+ self.emit('IMPORT_FROM', name)
+ self._resolveDots(name)
+ self.storeName(alias or name)
else:
self.emit('IMPORT_FROM', name)
- self._resolveDots(name)
- self.storeName(alias or name)
self.emit('POP_TOP')
def _resolveDots(self, name):
@@ -491,13 +561,85 @@ class CodeGenerator:
print "warning: unexpected flags:", node.flags
print node
- def visitAssTuple(self, node):
+ def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
if findOp(node) != 'OP_DELETE':
- self.emit('UNPACK_SEQUENCE', len(node.nodes))
+ self.emit(op, len(node.nodes))
for child in node.nodes:
self.visit(child)
- visitAssList = visitAssTuple
+ if VERSION > 1:
+ visitAssTuple = _visitAssSequence
+ visitAssList = _visitAssSequence
+ else:
+ def visitAssTuple(self, node):
+ self._visitAssSequence(node, 'UNPACK_TUPLE')
+
+ def visitAssList(self, node):
+ self._visitAssSequence(node, 'UNPACK_LIST')
+
+ # augmented assignment
+
+ def visitAugAssign(self, node):
+ aug_node = wrap_aug(node.node)
+ self.visit(aug_node, "load")
+ self.visit(node.expr)
+ self.emit(self._augmented_opcode[node.op])
+ self.visit(aug_node, "store")
+
+ _augmented_opcode = {
+ '+=' : 'INPLACE_ADD',
+ '-=' : 'INPLACE_SUBTRACT',
+ '*=' : 'INPLACE_MULTIPLY',
+ '/=' : 'INPLACE_DIVIDE',
+ '%=' : 'INPLACE_MODULO',
+ '**=': 'INPLACE_POWER',
+ '>>=': 'INPLACE_RSHIFT',
+ '<<=': 'INPLACE_LSHIFT',
+ '&=' : 'INPLACE_AND',
+ '^=' : 'INPLACE_XOR',
+ '|=' : 'INPLACE_OR',
+ }
+
+ def visitAugName(self, node, mode):
+ if mode == "load":
+ self.loadName(node.name)
+ elif mode == "store":
+ self.storeName(node.name)
+
+ def visitAugGetattr(self, node, mode):
+ if mode == "load":
+ self.visit(node.expr)
+ self.emit('DUP_TOP')
+ self.emit('LOAD_ATTR', node.attrname)
+ elif mode == "store":
+ self.emit('ROT_TWO')
+ self.emit('STORE_ATTR', node.attrname)
+
+ def visitAugSlice(self, node, mode):
+ if mode == "load":
+ self.visitSlice(node, 1)
+ elif mode == "store":
+ slice = 0
+ if node.lower:
+ slice = slice | 1
+ if node.upper:
+ slice = slice | 2
+ if slice == 0:
+ self.emit('ROT_TWO')
+ elif slice == 3:
+ self.emit('ROT_FOUR')
+ else:
+ self.emit('ROT_THREE')
+ self.emit('STORE_SLICE+%d' % slice)
+
+ def visitAugSubscript(self, node, mode):
+ if len(node.subs) > 1:
+ raise SyntaxError, "augmented assignment to tuple is not possible"
+ if mode == "load":
+ self.visitSubscript(node, 1)
+ elif mode == "store":
+ self.emit('ROT_THREE')
+ self.emit('STORE_SUBSCR')
def visitExec(self, node):
self.visit(node.expr)
@@ -533,13 +675,24 @@ class CodeGenerator:
def visitPrint(self, node):
self.set_lineno(node)
+ if node.dest:
+ self.visit(node.dest)
for child in node.nodes:
+ if node.dest:
+ self.emit('DUP_TOP')
self.visit(child)
- self.emit('PRINT_ITEM')
+ if node.dest:
+ self.emit('ROT_TWO')
+ self.emit('PRINT_ITEM_TO')
+ else:
+ self.emit('PRINT_ITEM')
def visitPrintnl(self, node):
self.visitPrint(node)
- self.emit('PRINT_NEWLINE')
+ if node.dest:
+ self.emit('PRINT_NEWLINE_TO')
+ else:
+ self.emit('PRINT_NEWLINE')
def visitReturn(self, node):
self.set_lineno(node)
@@ -548,7 +701,8 @@ class CodeGenerator:
# slice and subscript stuff
- def visitSlice(self, node):
+ def visitSlice(self, node, aug_flag=None):
+ # aug_flag is used by visitAugSlice
self.visit(node.expr)
slice = 0
if node.lower:
@@ -557,6 +711,13 @@ class CodeGenerator:
if node.upper:
self.visit(node.upper)
slice = slice | 2
+ if aug_flag:
+ if slice == 0:
+ self.emit('DUP_TOP')
+ elif slice == 3:
+ self.emit('DUP_TOPX', 3)
+ else:
+ self.emit('DUP_TOPX', 2)
if node.flags == 'OP_APPLY':
self.emit('SLICE+%d' % slice)
elif node.flags == 'OP_ASSIGN':
@@ -567,10 +728,12 @@ class CodeGenerator:
print "weird slice", node.flags
raise
- def visitSubscript(self, node):
+ def visitSubscript(self, node, aug_flag=None):
self.visit(node.expr)
for sub in node.subs:
self.visit(sub)
+ if aug_flag:
+ self.emit('DUP_TOPX', 2)
if len(node.subs) > 1:
self.emit('BUILD_TUPLE', len(node.subs))
if node.flags == 'OP_APPLY':
@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator):
self.unpackSequence(arg)
def unpackSequence(self, tup):
- self.emit('UNPACK_SEQUENCE', len(tup))
+ if VERSION > 1:
+ self.emit('UNPACK_SEQUENCE', len(tup))
+ else:
+ self.emit('UNPACK_TUPLE', len(tup))
for elt in tup:
if type(elt) == types.TupleType:
self.unpackSequence(elt)
@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator):
self.emit('LOAD_LOCALS')
self.emit('RETURN_VALUE')
-
def generateArgList(arglist):
"""Generate an arg list marking TupleArgs"""
args = []
@@ -838,6 +1003,45 @@ class OpFinder:
elif self.op != node.flags:
raise ValueError, "mixed ops in stmt"
+class Delegator:
+ """Base class to support delegation for augmented assignment nodes
+
+ To generator code for augmented assignments, we use the following
+ wrapper classes. In visitAugAssign, the left-hand expression node
+ is visited twice. The first time the visit uses the normal method
+ for that node . The second time the visit uses a different method
+ that generates the appropriate code to perform the assignment.
+ These delegator classes wrap the original AST nodes in order to
+ support the variant visit methods.
+ """
+ def __init__(self, obj):
+ self.obj = obj
+
+ def __getattr__(self, attr):
+ return getattr(self.obj, attr)
+
+class AugGetattr(Delegator):
+ pass
+
+class AugName(Delegator):
+ pass
+
+class AugSlice(Delegator):
+ pass
+
+class AugSubscript(Delegator):
+ pass
+
+wrapper = {
+ ast.Getattr: AugGetattr,
+ ast.Name: AugName,
+ ast.Slice: AugSlice,
+ ast.Subscript: AugSubscript,
+ }
+
+def wrap_aug(node):
+ return wrapper[node.__class__](node)
+
if __name__ == "__main__":
import sys
diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py
index c8a8518..a97798d 100644
--- a/Lib/compiler/transformer.py
+++ b/Lib/compiler/transformer.py
@@ -1,19 +1,3 @@
-#
-# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
-#
-# This module is provided under a BSD-ish license. See
-# http://www.opensource.org/licenses/bsd-license.html
-# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
-#
-#
-# Written by Greg Stein (gstein@lyra.org)
-# and Bill Tutt (rassilon@lima.mudlib.org)
-# February 1997.
-#
-# Support for ast.Node subclasses written and other revisions by
-# Jeremy Hylton (jeremy@beopen.com)
-#
-
"""Parse tree transformation module.
Transforms Python source code into an abstract syntax tree (AST)
@@ -24,7 +8,21 @@ parse(buf) -> AST
parseFile(path) -> AST
"""
+# Original version written by Greg Stein (gstein@lyra.org)
+# and Bill Tutt (rassilon@lima.mudlib.org)
+# February 1997.
#
+# Modifications and improvements for Python 2.0 by Jeremy Hylton and
+# Mark Hammond
+
+# Portions of this file are:
+# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
+#
+# This module is provided under a BSD-ish license. See
+# http://www.opensource.org/licenses/bsd-license.html
+# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
+
+
# The output tree has the following nodes:
#
# Source Python line #'s appear at the end of each of all of these nodes
@@ -49,9 +47,10 @@ parseFile(path) -> AST
# tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode
# return: valueNode
# const: value
-# print: [ node1, ..., nodeN ]
-# printnl: [ node1, ..., nodeN ]
+# print: [ node1, ..., nodeN ] [, dest]
+# printnl: [ node1, ..., nodeN ] [, dest]
# discard: exprNode
+# augassign: node, op, expr
# assign: [ node1, ..., nodeN ], exprNode
# ass_tuple: [ node1, ..., nodeN ]
# ass_list: [ node1, ..., nodeN ]
@@ -97,12 +96,12 @@ parseFile(path) -> AST
import ast
import parser
+# Care must be taken to use only symbols and tokens defined in Python
+# 1.5.2 for code branches executed in 1.5.2
import symbol
import token
import string
-import pprint
-
error = 'walker.error'
from consts import CO_VARARGS, CO_VARKEYWORDS
@@ -328,27 +327,44 @@ class Transformer:
#
def expr_stmt(self, nodelist):
- # testlist ('=' testlist)*
+ # augassign testlist | testlist ('=' testlist)*
exprNode = self.com_node(nodelist[-1])
if len(nodelist) == 1:
return Node('discard', exprNode)
- nodes = [ ]
- for i in range(0, len(nodelist) - 2, 2):
- nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
- n = Node('assign', nodes, exprNode)
- n.lineno = nodelist[1][2]
+ if nodelist[1][0] == token.EQUAL:
+ nodes = [ ]
+ for i in range(0, len(nodelist) - 2, 2):
+ nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
+ n = Node('assign', nodes, exprNode)
+ n.lineno = nodelist[1][2]
+ else:
+ lval = self.com_augassign(nodelist[0])
+ op = self.com_augassign_op(nodelist[1])
+ n = Node('augassign', lval, op[1], exprNode)
+ n.lineno = op[2]
return n
def print_stmt(self, nodelist):
- # print: (test ',')* [test]
+ # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
items = [ ]
- for i in range(1, len(nodelist), 2):
+ if len(nodelist) == 1:
+ start = 1
+ dest = None
+ elif nodelist[1][0] == token.RIGHTSHIFT:
+ assert len(nodelist) == 3 \
+ or nodelist[3][0] == token.COMMA
+ dest = self.com_node(nodelist[2])
+ start = 4
+ else:
+ dest = None
+ start = 1
+ for i in range(start, len(nodelist), 2):
items.append(self.com_node(nodelist[i]))
if nodelist[-1][0] == token.COMMA:
- n = Node('print', items)
+ n = Node('print', items, dest)
n.lineno = nodelist[0][2]
return n
- n = Node('printnl', items)
+ n = Node('printnl', items, dest)
n.lineno = nodelist[0][2]
return n
@@ -405,17 +421,24 @@ class Transformer:
# import_stmt: 'import' dotted_as_name (',' dotted_as_name)* |
# from: 'from' dotted_name 'import'
# ('*' | import_as_name (',' import_as_name)*)
- names = []
- is_as = 0
if nodelist[0][1] == 'from':
- for i in range(3, len(nodelist), 2):
- names.append(self.com_import_as_name(nodelist[i][1]))
+ names = []
+ if nodelist[3][0] == token.NAME:
+ for i in range(3, len(nodelist), 2):
+ names.append((nodelist[i][1], None))
+ else:
+ for i in range(3, len(nodelist), 2):
+ names.append(self.com_import_as_name(nodelist[i][1]))
n = Node('from', self.com_dotted_name(nodelist[1]), names)
n.lineno = nodelist[0][2]
return n
- for i in range(1, len(nodelist), 2):
- names.append(self.com_dotted_as_name(nodelist[i]))
+ if nodelist[1][0] == symbol.dotted_name:
+ names = [(self.com_dotted_name(nodelist[1][1:]), None)]
+ else:
+ names = []
+ for i in range(1, len(nodelist), 2):
+ names.append(self.com_dotted_as_name(nodelist[i]))
n = Node('import', names)
n.lineno = nodelist[0][2]
return n
@@ -737,7 +760,7 @@ class Transformer:
return Node('discard', Node('const', None))
if node[0] not in _legal_node_types:
- raise error, 'illegal node passed to com_node: %s' % node[0]
+ raise error, 'illegal node passed to com_node: %s' % `node`
# print "dispatch", self._dispatch[node[0]].__name__, node
return self._dispatch[node[0]](node[1:])
@@ -818,11 +841,14 @@ class Transformer:
def com_dotted_as_name(self, node):
dot = self.com_dotted_name(node[1])
- if len(node) == 2:
+ if len(node) <= 2:
return dot, None
- assert node[2][1] == 'as'
- assert node[3][0] == token.NAME
- return dot, node[3][1]
+ if node[0] == symbol.dotted_name:
+ pass
+ else:
+ assert node[2][1] == 'as'
+ assert node[3][0] == token.NAME
+ return dot, node[3][1]
def com_import_as_name(self, node):
if node == '*':
@@ -872,6 +898,20 @@ class Transformer:
n.lineno = nodelist[0][2]
return n
+ def com_augassign_op(self, node):
+ assert node[0] == symbol.augassign
+ return node[1]
+
+ def com_augassign(self, node):
+ """Return node suitable for lvalue of augmented assignment
+
+ Names, slices, and attributes are the only allowable nodes.
+ """
+ l = self.com_node(node)
+ if l[0] in ('name', 'slice', 'subscript', 'getattr'):
+ return l
+ raise SyntaxError, "can't assign to %s" % l[0]
+
def com_assign(self, node, assigning):
# return a node suitable for use as an "lvalue"
# loop to avoid trivial recursion
@@ -955,7 +995,6 @@ class Transformer:
return Node(type, items)
def com_stmt(self, node):
- #pprint.pprint(node)
result = self.com_node(node)
try:
result[0]
@@ -976,17 +1015,71 @@ class Transformer:
else:
stmts.append(result)
- def com_list_constructor(self, nodelist):
- values = [ ]
- for i in range(1, len(nodelist), 2):
- values.append(self.com_node(nodelist[i]))
- return Node('list', values)
+ if hasattr(symbol, 'list_for'):
+ def com_list_constructor(self, nodelist):
+ # listmaker: test ( list_for | (',' test)* [','] )
+ values = [ ]
+ for i in range(1, len(nodelist)):
+ if nodelist[i][0] == symbol.list_for:
+ assert len(nodelist[i:]) == 1
+ return self.com_list_comprehension(values[0],
+ nodelist[i])
+ elif nodelist[i][0] == token.COMMA:
+ continue
+ values.append(self.com_node(nodelist[i]))
+ return Node('list', values)
+
+ def com_list_comprehension(self, expr, node):
+ # list_iter: list_for | list_if
+ # list_for: 'for' exprlist 'in' testlist [list_iter]
+ # list_if: 'if' test [list_iter]
+ lineno = node[1][2]
+ fors = []
+ while node:
+ if node[1][1] == 'for':
+ assignNode = self.com_assign(node[2], OP_ASSIGN)
+ listNode = self.com_node(node[4])
+ newfor = Node('listcomp_for', assignNode,
+ listNode, [])
+ newfor.lineno = node[1][2]
+ fors.append(newfor)
+ if len(node) == 5:
+ node = None
+ else:
+ node = self.com_list_iter(node[5])
+ elif node[1][1] == 'if':
+ test = self.com_node(node[2])
+ newif = Node('listcomp_if', test)
+ newif.lineno = node[1][2]
+ newfor.ifs.append(newif)
+ if len(node) == 3:
+ node = None
+ else:
+ node = self.com_list_iter(node[3])
+ else:
+ raise SyntaxError, \
+ ("unexpected list comprehension element: %s %d"
+ % (node, lineno))
+ n = Node('listcomp', expr, fors)
+ n.lineno = lineno
+ return n
+
+ def com_list_iter(self, node):
+ assert node[0] == symbol.list_iter
+ return node[1]
+ else:
+ def com_list_constructor(self, nodelist):
+ values = [ ]
+ for i in range(1, len(nodelist), 2):
+ values.append(self.com_node(nodelist[i]))
+ return Node('list', values)
def com_dictmaker(self, nodelist):
# dictmaker: test ':' test (',' test ':' value)* [',']
items = [ ]
for i in range(1, len(nodelist), 4):
- items.append((self.com_node(nodelist[i]), self.com_node(nodelist[i+2])))
+ items.append((self.com_node(nodelist[i]),
+ self.com_node(nodelist[i+2])))
return Node('dict', items)
def com_apply_trailer(self, primaryNode, nodelist):
@@ -1250,3 +1343,21 @@ _assign_types = [
symbol.term,
symbol.factor,
]
+
+import types
+_names = {}
+for k, v in symbol.sym_name.items():
+ _names[k] = v
+for k, v in token.tok_name.items():
+ _names[k] = v
+
+def debug_tree(tree):
+ l = []
+ for elt in tree:
+ if type(elt) == types.IntType:
+ l.append(_names.get(elt, elt))
+ elif type(elt) == types.StringType:
+ l.append(elt)
+ else:
+ l.append(debug_tree(elt))
+ return l
diff --git a/Tools/compiler/compiler/ast.py b/Tools/compiler/compiler/ast.py
index a3e51b7..5b0a06a 100644
--- a/Tools/compiler/compiler/ast.py
+++ b/Tools/compiler/compiler/ast.py
@@ -279,22 +279,24 @@ class Const(Node):
class Print(Node):
nodes['print'] = 'Print'
- def __init__(self, nodes):
+ def __init__(self, nodes, dest):
self.nodes = nodes
- self._children = ('print', nodes)
+ self.dest = dest
+ self._children = ('print', nodes, dest)
def __repr__(self):
- return "Print(%s)" % self._children[1:]
+ return "Print(%s, %s)" % (self._children[1:-1], self._children[-1])
class Printnl(Node):
nodes['printnl'] = 'Printnl'
- def __init__(self, nodes):
+ def __init__(self, nodes, dest):
self.nodes = nodes
- self._children = ('printnl', nodes)
+ self.dest = dest
+ self._children = ('printnl', nodes, dest)
def __repr__(self):
- return "Printnl(%s)" % self._children[1:]
+ return "Printnl(%s, %s)" % (self._children[1:-1], self._children[-1])
class Discard(Node):
nodes['discard'] = 'Discard'
@@ -306,6 +308,18 @@ class Discard(Node):
def __repr__(self):
return "Discard(%s)" % self._children[1:]
+class AugAssign(Node):
+ nodes['augassign'] = 'AugAssign'
+
+ def __init__(self, node, op, expr):
+ self.node = node
+ self.op = op
+ self.expr = expr
+ self._children = ('augassign', node, op, expr)
+
+ def __repr__(self):
+ return "AugAssign(%s)" % str(self._children[1:])
+
class Assign(Node):
nodes['assign'] = 'Assign'
@@ -360,6 +374,41 @@ class AssAttr(Node):
def __repr__(self):
return "AssAttr(%s,%s,%s)" % self._children[1:]
+class ListComp(Node):
+ nodes['listcomp'] = 'ListComp'
+
+ def __init__(self, expr, quals):
+ self.expr = expr
+ self.quals = quals
+ self._children = ('listcomp', expr, quals)
+
+ def __repr__(self):
+ return "ListComp(%s, %s)" % self._children[1:]
+
+class ListCompFor(Node):
+ nodes['listcomp_for'] = 'ListCompFor'
+
+ # transformer fills in ifs after node is created
+
+ def __init__(self, assign, list, ifs):
+ self.assign = assign
+ self.list = list
+ self.ifs = ifs
+ self._children = ('listcomp_for', assign, list, ifs)
+
+ def __repr__(self):
+ return "ListCompFor(%s, %s, %s)" % self._children[1:]
+
+class ListCompIf(Node):
+ nodes['listcomp_if'] = 'ListCompIf'
+
+ def __init__(self, test):
+ self.test = test
+ self._children = ('listcomp_if', test)
+
+ def __repr__(self):
+ return "ListCompIf(%s)" % self._children[1:]
+
class List(Node):
nodes['list'] = 'List'
diff --git a/Tools/compiler/compiler/pyassem.py b/Tools/compiler/compiler/pyassem.py
index 3411273..dcc8bc0 100644
--- a/Tools/compiler/compiler/pyassem.py
+++ b/Tools/compiler/compiler/pyassem.py
@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph):
def _lookupName(self, name, list):
"""Return index of name in list, appending if necessary"""
- if name in list:
- i = list.index(name)
+ found = None
+ t = type(name)
+ for i in range(len(list)):
+ # must do a comparison on type first to prevent UnicodeErrors
+ if t == type(list[i]) and list[i] == name:
+ found = 1
+ break
+ if found:
# this is cheap, but incorrect in some cases, e.g 2 vs. 2L
if type(name) == type(list[i]):
return i
diff --git a/Tools/compiler/compiler/pycodegen.py b/Tools/compiler/compiler/pycodegen.py
index 2888729..bf54c32 100644
--- a/Tools/compiler/compiler/pycodegen.py
+++ b/Tools/compiler/compiler/pycodegen.py
@@ -1,8 +1,10 @@
+import imp
import os
import marshal
import stat
import string
import struct
+import sys
import types
from cStringIO import StringIO
@@ -10,6 +12,12 @@ from compiler import ast, parse, walk
from compiler import pyassem, misc
from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg
+# Do we have Python 1.x or Python 2.x?
+try:
+ VERSION = sys.version_info[0]
+except AttributeError:
+ VERSION = 1
+
callfunc_opcode_info = {
# (Have *args, Have **args) : opcode
(0,0) : "CALL_FUNCTION",
@@ -18,12 +26,12 @@ callfunc_opcode_info = {
(1,1) : "CALL_FUNCTION_VAR_KW",
}
-def compile(filename):
+def compile(filename, display=0):
f = open(filename)
buf = f.read()
f.close()
mod = Module(buf, filename)
- mod.compile()
+ mod.compile(display)
f = open(filename + "c", "wb")
mod.dump(f)
f.close()
@@ -34,28 +42,30 @@ class Module:
self.source = source
self.code = None
- def compile(self):
+ def compile(self, display=0):
ast = parse(self.source)
root, filename = os.path.split(self.filename)
gen = ModuleCodeGenerator(filename)
walk(ast, gen, 1)
+ if display:
+ import pprint
+ print pprint.pprint(ast)
self.code = gen.getCode()
def dump(self, f):
f.write(self.getPycHeader())
marshal.dump(self.code, f)
- MAGIC = (50823 | (ord('\r')<<16) | (ord('\n')<<24))
+ MAGIC = imp.get_magic()
def getPycHeader(self):
# compile.c uses marshal to write a long directly, with
# calling the interface that would also generate a 1-byte code
# to indicate the type of the value. simplest way to get the
# same effect is to call marshal and then skip the code.
- magic = marshal.dumps(self.MAGIC)[1:]
mtime = os.stat(self.filename)[stat.ST_MTIME]
mtime = struct.pack('i', mtime)
- return magic + mtime
+ return self.MAGIC + mtime
class CodeGenerator:
@@ -63,7 +73,7 @@ class CodeGenerator:
def __init__(self, filename):
## Subclasses must define a constructor that intializes self.graph
-## before calling this init function
+## before calling this init function, e.g.
## self.graph = pyassem.PyFlowGraph()
self.filename = filename
self.locals = misc.Stack()
@@ -142,7 +152,6 @@ class CodeGenerator:
def visitLambda(self, node):
self._visitFuncOrLambda(node, isLambda=1)
-## self.storeName("<lambda>")
def _visitFuncOrLambda(self, node, isLambda):
gen = FunctionCodeGenerator(node, self.filename, isLambda)
@@ -180,10 +189,6 @@ class CodeGenerator:
test, suite = node.tests[i]
self.set_lineno(test)
self.visit(test)
-## if i == numtests - 1 and not node.else_:
-## nextTest = end
-## else:
-## nextTest = self.newBlock()
nextTest = self.newBlock()
self.emit('JUMP_IF_FALSE', nextTest)
self.nextBlock()
@@ -304,6 +309,70 @@ class CodeGenerator:
self.emit('POP_TOP')
self.nextBlock(end)
+ # list comprehensions
+ __list_count = 0
+
+ def visitListComp(self, node):
+ # XXX would it be easier to transform the AST into the form it
+ # would have if the list comp were expressed as a series of
+ # for and if stmts and an explicit append?
+ self.set_lineno(node)
+ # setup list
+ append = "$append%d" % self.__list_count
+ self.__list_count = self.__list_count + 1
+ self.emit('BUILD_LIST', 0)
+ self.emit('DUP_TOP')
+ self.emit('LOAD_ATTR', 'append')
+ self.storeName(append)
+ l = len(node.quals)
+ stack = []
+ for i, for_ in zip(range(l), node.quals):
+ start, anchor = self.visit(for_)
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor))
+
+ self.loadName(append)
+ self.visit(node.expr)
+ self.emit('CALL_FUNCTION', 1)
+ self.emit('POP_TOP')
+
+ for start, cont, anchor in stack:
+ if cont:
+ skip_one = self.newBlock()
+ self.emit('JUMP_FORWARD', skip_one)
+ self.nextBlock(cont)
+ self.emit('POP_TOP')
+ self.nextBlock(skip_one)
+ self.emit('JUMP_ABSOLUTE', start)
+ self.nextBlock(anchor)
+ self.delName(append)
+
+ self.__list_count = self.__list_count - 1
+
+ def visitListCompFor(self, node):
+ self.set_lineno(node)
+ start = self.newBlock()
+ anchor = self.newBlock()
+
+ self.visit(node.list)
+ self.visit(ast.Const(0))
+ self.emit('SET_LINENO', node.lineno)
+ self.nextBlock(start)
+ self.emit('FOR_LOOP', anchor)
+ self.visit(node.assign)
+ return start, anchor
+
+ def visitListCompIf(self, node, branch):
+ self.set_lineno(node)
+ self.visit(node.test)
+ self.emit('JUMP_IF_FALSE', branch)
+ self.newBlock()
+ self.emit('POP_TOP')
+
# exception related
def visitAssert(self, node):
@@ -397,10 +466,6 @@ class CodeGenerator:
# misc
-## def visitStmt(self, node):
-## # nothing to do except walk the children
-## pass
-
def visitDiscard(self, node):
self.visit(node.expr)
self.emit('POP_TOP')
@@ -426,27 +491,32 @@ class CodeGenerator:
def visitImport(self, node):
self.set_lineno(node)
for name, alias in node.names:
- self.emit('LOAD_CONST', None)
+ if VERSION > 1:
+ self.emit('LOAD_CONST', None)
self.emit('IMPORT_NAME', name)
- self._resolveDots(name)
- self.storeName(alias or name)
+ mod = string.split(name, ".")[0]
+ self.storeName(alias or mod)
def visitFrom(self, node):
self.set_lineno(node)
fromlist = map(lambda (name, alias): name, node.names)
- self.emit('LOAD_CONST', tuple(fromlist))
+ if VERSION > 1:
+ self.emit('LOAD_CONST', tuple(fromlist))
self.emit('IMPORT_NAME', node.modname)
for name, alias in node.names:
- if name == '*':
- self.namespace = 0
- self.emit('IMPORT_STAR')
- # There can only be one name w/ from ... import *
- assert len(node.names) == 1
- return
+ if VERSION > 1:
+ if name == '*':
+ self.namespace = 0
+ self.emit('IMPORT_STAR')
+ # There can only be one name w/ from ... import *
+ assert len(node.names) == 1
+ return
+ else:
+ self.emit('IMPORT_FROM', name)
+ self._resolveDots(name)
+ self.storeName(alias or name)
else:
self.emit('IMPORT_FROM', name)
- self._resolveDots(name)
- self.storeName(alias or name)
self.emit('POP_TOP')
def _resolveDots(self, name):
@@ -491,13 +561,85 @@ class CodeGenerator:
print "warning: unexpected flags:", node.flags
print node
- def visitAssTuple(self, node):
+ def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
if findOp(node) != 'OP_DELETE':
- self.emit('UNPACK_SEQUENCE', len(node.nodes))
+ self.emit(op, len(node.nodes))
for child in node.nodes:
self.visit(child)
- visitAssList = visitAssTuple
+ if VERSION > 1:
+ visitAssTuple = _visitAssSequence
+ visitAssList = _visitAssSequence
+ else:
+ def visitAssTuple(self, node):
+ self._visitAssSequence(node, 'UNPACK_TUPLE')
+
+ def visitAssList(self, node):
+ self._visitAssSequence(node, 'UNPACK_LIST')
+
+ # augmented assignment
+
+ def visitAugAssign(self, node):
+ aug_node = wrap_aug(node.node)
+ self.visit(aug_node, "load")
+ self.visit(node.expr)
+ self.emit(self._augmented_opcode[node.op])
+ self.visit(aug_node, "store")
+
+ _augmented_opcode = {
+ '+=' : 'INPLACE_ADD',
+ '-=' : 'INPLACE_SUBTRACT',
+ '*=' : 'INPLACE_MULTIPLY',
+ '/=' : 'INPLACE_DIVIDE',
+ '%=' : 'INPLACE_MODULO',
+ '**=': 'INPLACE_POWER',
+ '>>=': 'INPLACE_RSHIFT',
+ '<<=': 'INPLACE_LSHIFT',
+ '&=' : 'INPLACE_AND',
+ '^=' : 'INPLACE_XOR',
+ '|=' : 'INPLACE_OR',
+ }
+
+ def visitAugName(self, node, mode):
+ if mode == "load":
+ self.loadName(node.name)
+ elif mode == "store":
+ self.storeName(node.name)
+
+ def visitAugGetattr(self, node, mode):
+ if mode == "load":
+ self.visit(node.expr)
+ self.emit('DUP_TOP')
+ self.emit('LOAD_ATTR', node.attrname)
+ elif mode == "store":
+ self.emit('ROT_TWO')
+ self.emit('STORE_ATTR', node.attrname)
+
+ def visitAugSlice(self, node, mode):
+ if mode == "load":
+ self.visitSlice(node, 1)
+ elif mode == "store":
+ slice = 0
+ if node.lower:
+ slice = slice | 1
+ if node.upper:
+ slice = slice | 2
+ if slice == 0:
+ self.emit('ROT_TWO')
+ elif slice == 3:
+ self.emit('ROT_FOUR')
+ else:
+ self.emit('ROT_THREE')
+ self.emit('STORE_SLICE+%d' % slice)
+
+ def visitAugSubscript(self, node, mode):
+ if len(node.subs) > 1:
+ raise SyntaxError, "augmented assignment to tuple is not possible"
+ if mode == "load":
+ self.visitSubscript(node, 1)
+ elif mode == "store":
+ self.emit('ROT_THREE')
+ self.emit('STORE_SUBSCR')
def visitExec(self, node):
self.visit(node.expr)
@@ -533,13 +675,24 @@ class CodeGenerator:
def visitPrint(self, node):
self.set_lineno(node)
+ if node.dest:
+ self.visit(node.dest)
for child in node.nodes:
+ if node.dest:
+ self.emit('DUP_TOP')
self.visit(child)
- self.emit('PRINT_ITEM')
+ if node.dest:
+ self.emit('ROT_TWO')
+ self.emit('PRINT_ITEM_TO')
+ else:
+ self.emit('PRINT_ITEM')
def visitPrintnl(self, node):
self.visitPrint(node)
- self.emit('PRINT_NEWLINE')
+ if node.dest:
+ self.emit('PRINT_NEWLINE_TO')
+ else:
+ self.emit('PRINT_NEWLINE')
def visitReturn(self, node):
self.set_lineno(node)
@@ -548,7 +701,8 @@ class CodeGenerator:
# slice and subscript stuff
- def visitSlice(self, node):
+ def visitSlice(self, node, aug_flag=None):
+ # aug_flag is used by visitAugSlice
self.visit(node.expr)
slice = 0
if node.lower:
@@ -557,6 +711,13 @@ class CodeGenerator:
if node.upper:
self.visit(node.upper)
slice = slice | 2
+ if aug_flag:
+ if slice == 0:
+ self.emit('DUP_TOP')
+ elif slice == 3:
+ self.emit('DUP_TOPX', 3)
+ else:
+ self.emit('DUP_TOPX', 2)
if node.flags == 'OP_APPLY':
self.emit('SLICE+%d' % slice)
elif node.flags == 'OP_ASSIGN':
@@ -567,10 +728,12 @@ class CodeGenerator:
print "weird slice", node.flags
raise
- def visitSubscript(self, node):
+ def visitSubscript(self, node, aug_flag=None):
self.visit(node.expr)
for sub in node.subs:
self.visit(sub)
+ if aug_flag:
+ self.emit('DUP_TOPX', 2)
if len(node.subs) > 1:
self.emit('BUILD_TUPLE', len(node.subs))
if node.flags == 'OP_APPLY':
@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator):
self.unpackSequence(arg)
def unpackSequence(self, tup):
- self.emit('UNPACK_SEQUENCE', len(tup))
+ if VERSION > 1:
+ self.emit('UNPACK_SEQUENCE', len(tup))
+ else:
+ self.emit('UNPACK_TUPLE', len(tup))
for elt in tup:
if type(elt) == types.TupleType:
self.unpackSequence(elt)
@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator):
self.emit('LOAD_LOCALS')
self.emit('RETURN_VALUE')
-
def generateArgList(arglist):
"""Generate an arg list marking TupleArgs"""
args = []
@@ -838,6 +1003,45 @@ class OpFinder:
elif self.op != node.flags:
raise ValueError, "mixed ops in stmt"
+class Delegator:
+ """Base class to support delegation for augmented assignment nodes
+
+ To generator code for augmented assignments, we use the following
+ wrapper classes. In visitAugAssign, the left-hand expression node
+ is visited twice. The first time the visit uses the normal method
+ for that node . The second time the visit uses a different method
+ that generates the appropriate code to perform the assignment.
+ These delegator classes wrap the original AST nodes in order to
+ support the variant visit methods.
+ """
+ def __init__(self, obj):
+ self.obj = obj
+
+ def __getattr__(self, attr):
+ return getattr(self.obj, attr)
+
+class AugGetattr(Delegator):
+ pass
+
+class AugName(Delegator):
+ pass
+
+class AugSlice(Delegator):
+ pass
+
+class AugSubscript(Delegator):
+ pass
+
+wrapper = {
+ ast.Getattr: AugGetattr,
+ ast.Name: AugName,
+ ast.Slice: AugSlice,
+ ast.Subscript: AugSubscript,
+ }
+
+def wrap_aug(node):
+ return wrapper[node.__class__](node)
+
if __name__ == "__main__":
import sys
diff --git a/Tools/compiler/compiler/transformer.py b/Tools/compiler/compiler/transformer.py
index c8a8518..a97798d 100644
--- a/Tools/compiler/compiler/transformer.py
+++ b/Tools/compiler/compiler/transformer.py
@@ -1,19 +1,3 @@
-#
-# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
-#
-# This module is provided under a BSD-ish license. See
-# http://www.opensource.org/licenses/bsd-license.html
-# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
-#
-#
-# Written by Greg Stein (gstein@lyra.org)
-# and Bill Tutt (rassilon@lima.mudlib.org)
-# February 1997.
-#
-# Support for ast.Node subclasses written and other revisions by
-# Jeremy Hylton (jeremy@beopen.com)
-#
-
"""Parse tree transformation module.
Transforms Python source code into an abstract syntax tree (AST)
@@ -24,7 +8,21 @@ parse(buf) -> AST
parseFile(path) -> AST
"""
+# Original version written by Greg Stein (gstein@lyra.org)
+# and Bill Tutt (rassilon@lima.mudlib.org)
+# February 1997.
#
+# Modifications and improvements for Python 2.0 by Jeremy Hylton and
+# Mark Hammond
+
+# Portions of this file are:
+# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
+#
+# This module is provided under a BSD-ish license. See
+# http://www.opensource.org/licenses/bsd-license.html
+# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
+
+
# The output tree has the following nodes:
#
# Source Python line #'s appear at the end of each of all of these nodes
@@ -49,9 +47,10 @@ parseFile(path) -> AST
# tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode
# return: valueNode
# const: value
-# print: [ node1, ..., nodeN ]
-# printnl: [ node1, ..., nodeN ]
+# print: [ node1, ..., nodeN ] [, dest]
+# printnl: [ node1, ..., nodeN ] [, dest]
# discard: exprNode
+# augassign: node, op, expr
# assign: [ node1, ..., nodeN ], exprNode
# ass_tuple: [ node1, ..., nodeN ]
# ass_list: [ node1, ..., nodeN ]
@@ -97,12 +96,12 @@ parseFile(path) -> AST
import ast
import parser
+# Care must be taken to use only symbols and tokens defined in Python
+# 1.5.2 for code branches executed in 1.5.2
import symbol
import token
import string
-import pprint
-
error = 'walker.error'
from consts import CO_VARARGS, CO_VARKEYWORDS
@@ -328,27 +327,44 @@ class Transformer:
#
def expr_stmt(self, nodelist):
- # testlist ('=' testlist)*
+ # augassign testlist | testlist ('=' testlist)*
exprNode = self.com_node(nodelist[-1])
if len(nodelist) == 1:
return Node('discard', exprNode)
- nodes = [ ]
- for i in range(0, len(nodelist) - 2, 2):
- nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
- n = Node('assign', nodes, exprNode)
- n.lineno = nodelist[1][2]
+ if nodelist[1][0] == token.EQUAL:
+ nodes = [ ]
+ for i in range(0, len(nodelist) - 2, 2):
+ nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
+ n = Node('assign', nodes, exprNode)
+ n.lineno = nodelist[1][2]
+ else:
+ lval = self.com_augassign(nodelist[0])
+ op = self.com_augassign_op(nodelist[1])
+ n = Node('augassign', lval, op[1], exprNode)
+ n.lineno = op[2]
return n
def print_stmt(self, nodelist):
- # print: (test ',')* [test]
+ # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
items = [ ]
- for i in range(1, len(nodelist), 2):
+ if len(nodelist) == 1:
+ start = 1
+ dest = None
+ elif nodelist[1][0] == token.RIGHTSHIFT:
+ assert len(nodelist) == 3 \
+ or nodelist[3][0] == token.COMMA
+ dest = self.com_node(nodelist[2])
+ start = 4
+ else:
+ dest = None
+ start = 1
+ for i in range(start, len(nodelist), 2):
items.append(self.com_node(nodelist[i]))
if nodelist[-1][0] == token.COMMA:
- n = Node('print', items)
+ n = Node('print', items, dest)
n.lineno = nodelist[0][2]
return n
- n = Node('printnl', items)
+ n = Node('printnl', items, dest)
n.lineno = nodelist[0][2]
return n
@@ -405,17 +421,24 @@ class Transformer:
# import_stmt: 'import' dotted_as_name (',' dotted_as_name)* |
# from: 'from' dotted_name 'import'
# ('*' | import_as_name (',' import_as_name)*)
- names = []
- is_as = 0
if nodelist[0][1] == 'from':
- for i in range(3, len(nodelist), 2):
- names.append(self.com_import_as_name(nodelist[i][1]))
+ names = []
+ if nodelist[3][0] == token.NAME:
+ for i in range(3, len(nodelist), 2):
+ names.append((nodelist[i][1], None))
+ else:
+ for i in range(3, len(nodelist), 2):
+ names.append(self.com_import_as_name(nodelist[i][1]))
n = Node('from', self.com_dotted_name(nodelist[1]), names)
n.lineno = nodelist[0][2]
return n
- for i in range(1, len(nodelist), 2):
- names.append(self.com_dotted_as_name(nodelist[i]))
+ if nodelist[1][0] == symbol.dotted_name:
+ names = [(self.com_dotted_name(nodelist[1][1:]), None)]
+ else:
+ names = []
+ for i in range(1, len(nodelist), 2):
+ names.append(self.com_dotted_as_name(nodelist[i]))
n = Node('import', names)
n.lineno = nodelist[0][2]
return n
@@ -737,7 +760,7 @@ class Transformer:
return Node('discard', Node('const', None))
if node[0] not in _legal_node_types:
- raise error, 'illegal node passed to com_node: %s' % node[0]
+ raise error, 'illegal node passed to com_node: %s' % `node`
# print "dispatch", self._dispatch[node[0]].__name__, node
return self._dispatch[node[0]](node[1:])
@@ -818,11 +841,14 @@ class Transformer:
def com_dotted_as_name(self, node):
dot = self.com_dotted_name(node[1])
- if len(node) == 2:
+ if len(node) <= 2:
return dot, None
- assert node[2][1] == 'as'
- assert node[3][0] == token.NAME
- return dot, node[3][1]
+ if node[0] == symbol.dotted_name:
+ pass
+ else:
+ assert node[2][1] == 'as'
+ assert node[3][0] == token.NAME
+ return dot, node[3][1]
def com_import_as_name(self, node):
if node == '*':
@@ -872,6 +898,20 @@ class Transformer:
n.lineno = nodelist[0][2]
return n
+ def com_augassign_op(self, node):
+ assert node[0] == symbol.augassign
+ return node[1]
+
+ def com_augassign(self, node):
+ """Return node suitable for lvalue of augmented assignment
+
+ Names, slices, and attributes are the only allowable nodes.
+ """
+ l = self.com_node(node)
+ if l[0] in ('name', 'slice', 'subscript', 'getattr'):
+ return l
+ raise SyntaxError, "can't assign to %s" % l[0]
+
def com_assign(self, node, assigning):
# return a node suitable for use as an "lvalue"
# loop to avoid trivial recursion
@@ -955,7 +995,6 @@ class Transformer:
return Node(type, items)
def com_stmt(self, node):
- #pprint.pprint(node)
result = self.com_node(node)
try:
result[0]
@@ -976,17 +1015,71 @@ class Transformer:
else:
stmts.append(result)
- def com_list_constructor(self, nodelist):
- values = [ ]
- for i in range(1, len(nodelist), 2):
- values.append(self.com_node(nodelist[i]))
- return Node('list', values)
+ if hasattr(symbol, 'list_for'):
+ def com_list_constructor(self, nodelist):
+ # listmaker: test ( list_for | (',' test)* [','] )
+ values = [ ]
+ for i in range(1, len(nodelist)):
+ if nodelist[i][0] == symbol.list_for:
+ assert len(nodelist[i:]) == 1
+ return self.com_list_comprehension(values[0],
+ nodelist[i])
+ elif nodelist[i][0] == token.COMMA:
+ continue
+ values.append(self.com_node(nodelist[i]))
+ return Node('list', values)
+
+ def com_list_comprehension(self, expr, node):
+ # list_iter: list_for | list_if
+ # list_for: 'for' exprlist 'in' testlist [list_iter]
+ # list_if: 'if' test [list_iter]
+ lineno = node[1][2]
+ fors = []
+ while node:
+ if node[1][1] == 'for':
+ assignNode = self.com_assign(node[2], OP_ASSIGN)
+ listNode = self.com_node(node[4])
+ newfor = Node('listcomp_for', assignNode,
+ listNode, [])
+ newfor.lineno = node[1][2]
+ fors.append(newfor)
+ if len(node) == 5:
+ node = None
+ else:
+ node = self.com_list_iter(node[5])
+ elif node[1][1] == 'if':
+ test = self.com_node(node[2])
+ newif = Node('listcomp_if', test)
+ newif.lineno = node[1][2]
+ newfor.ifs.append(newif)
+ if len(node) == 3:
+ node = None
+ else:
+ node = self.com_list_iter(node[3])
+ else:
+ raise SyntaxError, \
+ ("unexpected list comprehension element: %s %d"
+ % (node, lineno))
+ n = Node('listcomp', expr, fors)
+ n.lineno = lineno
+ return n
+
+ def com_list_iter(self, node):
+ assert node[0] == symbol.list_iter
+ return node[1]
+ else:
+ def com_list_constructor(self, nodelist):
+ values = [ ]
+ for i in range(1, len(nodelist), 2):
+ values.append(self.com_node(nodelist[i]))
+ return Node('list', values)
def com_dictmaker(self, nodelist):
# dictmaker: test ':' test (',' test ':' value)* [',']
items = [ ]
for i in range(1, len(nodelist), 4):
- items.append((self.com_node(nodelist[i]), self.com_node(nodelist[i+2])))
+ items.append((self.com_node(nodelist[i]),
+ self.com_node(nodelist[i+2])))
return Node('dict', items)
def com_apply_trailer(self, primaryNode, nodelist):
@@ -1250,3 +1343,21 @@ _assign_types = [
symbol.term,
symbol.factor,
]
+
+import types
+_names = {}
+for k, v in symbol.sym_name.items():
+ _names[k] = v
+for k, v in token.tok_name.items():
+ _names[k] = v
+
+def debug_tree(tree):
+ l = []
+ for elt in tree:
+ if type(elt) == types.IntType:
+ l.append(_names.get(elt, elt))
+ elif type(elt) == types.StringType:
+ l.append(elt)
+ else:
+ l.append(debug_tree(elt))
+ return l
generated by cgit v0.12 at 2024-11-12 04:17:54 (GMT)