complete rewrite

code generator uses flowgraph as intermediate representation.  the old
    rep uses a list with explicit "StackRefs" to indicate the target
    of jumps.

pyassem converts flowgraph to bytecode, breaks up individual steps of
    generating bytecode

1 files changed, 490 insertions, 517 deletions

diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py
index b2d55d9..2e98d4e 100644
--- a/Lib/compiler/pycodegen.py
+++ b/Lib/compiler/pycodegen.py
@@ -1,132 +1,84 @@
-"""Python bytecode generator
-
-Currently contains generic ASTVisitor code, a LocalNameFinder, and a
-CodeGenerator.  Eventually, this will get split into the ASTVisitor as
-a generic tool and CodeGenerator as a specific tool.
-"""
-
-from compiler import parseFile, ast, visitor, walk, parse
-from pyassem import StackRef, PyAssembler, TupleArg
-import dis
-import misc
-import marshal
-import new
-import string
-import sys
 import os
+import marshal
 import stat
 import struct
 import types
+from cStringIO import StringIO
 
-class CodeGenerator:
-    """Generate bytecode for the Python VM"""
-
-    OPTIMIZED = 1
-
-    # XXX should clean up initialization and generateXXX funcs
-    def __init__(self, filename="<?>"):
-        self.filename = filename
-        self.code = PyAssembler()
-        self.code.setFlags(0)
-        self.locals = misc.Stack()
-        self.loops = misc.Stack()
-        self.namespace = 0
-        self.curStack = 0
-        self.maxStack = 0
-
-    def emit(self, *args):
-        # XXX could just use self.emit = self.code.emit
-        apply(self.code.emit, args)
-
-    def _generateFunctionOrLambdaCode(self, func):
-        self.name = func.name
- 
-        # keep a lookout for 'def foo((x,y)):'
-        args, hasTupleArg = self.generateArglist(func.argnames)
-        
-        self.code = PyAssembler(args=args, name=func.name,
-                                filename=self.filename)
-        self.namespace = self.OPTIMIZED
-        if func.varargs:
-            self.code.setVarArgs()
-        if func.kwargs:
-            self.code.setKWArgs()
-        lnf = walk(func.code, LocalNameFinder(args), 0)
-        self.locals.push(lnf.getLocals())
-        self.emit('SET_LINENO', func.lineno)
-        if hasTupleArg:
-            self.generateArgUnpack(func.argnames)
-        walk(func.code, self)
+from compiler import ast, parse, walk
+from compiler import pyassem, misc
+from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, TupleArg
 
-    def generateArglist(self, arglist):
-        args = []
-        extra = []
-        count = 0
-        for elt in arglist:
-            if type(elt) == types.StringType:
-                args.append(elt)
-            elif type(elt) == types.TupleType:
-                args.append(TupleArg(count, elt))
-                count = count + 1
-                extra.extend(misc.flatten(elt))
-            else:
-                raise ValueError, "unexpect argument type:", elt
-        return args + extra, count
+def compile(filename):
+    f = open(filename)
+    buf = f.read()
+    f.close()
+    mod = Module(buf, filename)
+    mod.compile()
+    f = open(filename + "c", "wb")
+    mod.dump(f)
+    f.close()
 
-    def generateArgUnpack(self, args):
-        count = 0
-        for arg in args:
-            if type(arg) == types.TupleType:
-                self.emit('LOAD_FAST', '.nested%d' % count)
-                count = count + 1
-                self.unpackTuple(arg)
-                        
-    def unpackTuple(self, tup):
-        self.emit('UNPACK_TUPLE', len(tup))
-        for elt in tup:
-            if type(elt) == types.TupleType:
-                self.unpackTuple(elt)
-            else:
-                self.emit('STORE_FAST', elt)
+class Module:
+    def __init__(self, source, filename):
+	self.filename = filename
+	self.source = source
+	self.code = None
 
-    def generateFunctionCode(self, func):
-        """Generate code for a function body"""
-        self._generateFunctionOrLambdaCode(func)
-        self.emit('LOAD_CONST', None)
-        self.emit('RETURN_VALUE')
+    def compile(self):
+	ast = parse(self.source)
+        root, filename = os.path.split(self.filename)
+	gen = ModuleCodeGenerator(filename)
+	walk(ast, gen, 1)
+	self.code = gen.getCode()
 
-    def generateLambdaCode(self, func):
-        self._generateFunctionOrLambdaCode(func)
-        self.emit('RETURN_VALUE')
+    def dump(self, f):
+	f.write(self.getPycHeader())
+	marshal.dump(self.code, f)
 
-    def generateClassCode(self, klass):
-        self.code = PyAssembler(name=klass.name,
-                                 filename=self.filename)
-        self.emit('SET_LINENO', klass.lineno)
-        lnf = walk(klass.code, LocalNameFinder(), 0)
-        self.locals.push(lnf.getLocals())
-        walk(klass.code, self)
-        self.emit('LOAD_LOCALS')
-        self.emit('RETURN_VALUE')
+    MAGIC = (20121 | (ord('\r')<<16) | (ord('\n')<<24))
+
+    def getPycHeader(self):
+	# compile.c uses marshal to write a long directly, with
+	# calling the interface that would also generate a 1-byte code
+	# to indicate the type of the value.  simplest way to get the
+	# same effect is to call marshal and then skip the code.
+	magic = marshal.dumps(self.MAGIC)[1:]
+	mtime = os.stat(self.filename)[stat.ST_MTIME]
+	mtime = struct.pack('i', mtime)
+	return magic + mtime
+
+class CodeGenerator:
+
+    optimized = 0 # is namespace access optimized?
+
+    def __init__(self, filename):
+## Subclasses must define a constructor that intializes self.graph
+## before calling this init function
+## 	self.graph = pyassem.PyFlowGraph()
+	self.filename = filename
+	self.locals = misc.Stack()
+	self.loops = misc.Stack()
+	self.curStack = 0
+	self.maxStack = 0
+	self._setupGraphDelegation()
+
+    def _setupGraphDelegation(self):
+	self.emit = self.graph.emit
+	self.newBlock = self.graph.newBlock
+	self.startBlock = self.graph.startBlock
+	self.nextBlock = self.graph.nextBlock
+	self.setDocstring = self.graph.setDocstring
 
-    def asConst(self):
-        """Create a Python code object."""
-        if self.namespace == self.OPTIMIZED:
-            self.code.setOptimized()
-        return self.code.makeCodeObject()
+    def getCode(self):
+	"""Return a code object"""
+	return self.graph.getCode()
+
+    # Next five methods handle name access
 
     def isLocalName(self, name):
         return self.locals.top().has_elt(name)
 
-    def _nameOp(self, prefix, name):
-        if self.isLocalName(name):
-            if self.namespace == self.OPTIMIZED:
-                self.emit(prefix + '_FAST', name)
-            else:
-                self.emit(prefix + '_NAME', name)
-        else:
-            self.emit(prefix + '_GLOBAL', name)
-
     def storeName(self, name):
         self._nameOp('STORE', name)
 
@@ -136,195 +88,237 @@ class CodeGenerator:
     def delName(self, name):
         self._nameOp('DELETE', name)
 
-    def visitNULL(self, node):
-        """Method exists only to stop warning in -v mode"""
-        pass
-
-    visitStmt = visitNULL
-    visitGlobal = visitNULL
-
-    def visitDiscard(self, node):
-        self.visit(node.expr)
-        self.emit('POP_TOP')
-        return 1
+    def _nameOp(self, prefix, name):
+	if not self.optimized:
+	    self.emit(prefix + '_NAME', name)
+	    return
+        if self.isLocalName(name):
+	    self.emit(prefix + '_FAST', name)
+        else:
+            self.emit(prefix + '_GLOBAL', name)
 
-    def visitPass(self, node):
-        self.emit('SET_LINENO', node.lineno)
+    # The first few visitor methods handle nodes that generator new
+    # code objects 
 
     def visitModule(self, node):
-        lnf = walk(node.node, LocalNameFinder(), 0)
-        self.locals.push(lnf.getLocals())
-        self.visit(node.node)
-        self.emit('LOAD_CONST', None)
-        self.emit('RETURN_VALUE')
-        return 1
+	lnf = walk(node.node, LocalNameFinder(), 0)
+	self.locals.push(lnf.getLocals())
+	self.setDocstring(node.doc)
+	self.visit(node.node)
+	self.emit('LOAD_CONST', None)
+	self.emit('RETURN_VALUE')
 
-    def visitImport(self, node):
-        self.emit('SET_LINENO', node.lineno)
-        for name in node.names:
-            self.emit('IMPORT_NAME', name)
-            self.storeName(name)
+    def visitFunction(self, node):
+	self._visitFuncOrLambda(node, isLambda=0)
+	self.storeName(node.name)
 
-    def visitFrom(self, node):
-        self.emit('SET_LINENO', node.lineno)
-        self.emit('IMPORT_NAME', node.modname)
-        for name in node.names:
-            if name == '*':
-                self.namespace = 0
-            self.emit('IMPORT_FROM', name)
-        self.emit('POP_TOP')
+    def visitLambda(self, node):
+	self._visitFuncOrLambda(node, isLambda=1)
+##	self.storeName("<lambda>")
+
+    def _visitFuncOrLambda(self, node, isLambda):
+	gen = FunctionCodeGenerator(node, self.filename, isLambda)
+	walk(node.code, gen)
+	gen.finish()
+	self.emit('SET_LINENO', node.lineno)
+        for default in node.defaults:
+            self.visit(default)
+        self.emit('LOAD_CONST', gen.getCode())
+        self.emit('MAKE_FUNCTION', len(node.defaults))
 
     def visitClass(self, node):
+        gen = ClassCodeGenerator(node, self.filename)
+        walk(node.code, gen)
+        gen.finish()
         self.emit('SET_LINENO', node.lineno)
         self.emit('LOAD_CONST', node.name)
         for base in node.bases:
             self.visit(base)
         self.emit('BUILD_TUPLE', len(node.bases))
-        classBody = CodeGenerator(self.filename)
-        classBody.generateClassCode(node)
-        self.emit('LOAD_CONST', classBody)
+        self.emit('LOAD_CONST', gen.getCode())
         self.emit('MAKE_FUNCTION', 0)
         self.emit('CALL_FUNCTION', 0)
         self.emit('BUILD_CLASS')
         self.storeName(node.name)
-        return 1
 
-    def _visitFuncOrLambda(self, node, kind):
-        """Code common to Function and Lambda nodes"""
-        codeBody = CodeGenerator(self.filename)
-        getattr(codeBody, 'generate%sCode' % kind)(node)
-        self.emit('SET_LINENO', node.lineno)
-        for default in node.defaults:
-            self.visit(default)
-        self.emit('LOAD_CONST', codeBody)
-        self.emit('MAKE_FUNCTION', len(node.defaults))
+    # The rest are standard visitor methods
 
-    def visitFunction(self, node):
-        self._visitFuncOrLambda(node, 'Function')
-        self.storeName(node.name)
-        return 1
+    # The next few implement control-flow statements
 
-    def visitLambda(self, node):
-        node.name = '<lambda>'
-        self._visitFuncOrLambda(node, 'Lambda')
-        return 1
+    def visitIf(self, node):
+	end = self.newBlock()
+	numtests = len(node.tests)
+	for i in range(numtests):
+	    test, suite = node.tests[i]
+	    if hasattr(test, 'lineno'):
+		self.emit('SET_LINENO', test.lineno)
+	    self.visit(test)
+## 	    if i == numtests - 1 and not node.else_:
+## 		nextTest = end
+## 	    else:
+## 		nextTest = self.newBlock()
+	    nextTest = self.newBlock()
+	    self.emit('JUMP_IF_FALSE', nextTest)
+	    self.nextBlock()
+	    self.emit('POP_TOP')
+	    self.visit(suite)
+	    self.emit('JUMP_FORWARD', end)
+	    self.nextBlock(nextTest)
+	    self.emit('POP_TOP')
+	if node.else_:
+	    self.visit(node.else_)
+	self.nextBlock(end)
 
-    def visitCallFunc(self, node):
-        pos = 0
-        kw = 0
-        if hasattr(node, 'lineno'):
-            self.emit('SET_LINENO', node.lineno)
-        self.visit(node.node)
-        for arg in node.args:
-            self.visit(arg)
-            if isinstance(arg, ast.Keyword):
-                kw = kw + 1
-            else:
-                pos = pos + 1
-        self.emit('CALL_FUNCTION', kw << 8 | pos)
-        return 1
+    def visitWhile(self, node):
+	self.emit('SET_LINENO', node.lineno)
 
-    def visitKeyword(self, node):
-        self.emit('LOAD_CONST', node.name)
-        self.visit(node.expr)
-        return 1
+	loop = self.newBlock()
+	else_ = self.newBlock()
 
-    def visitIf(self, node):
-        after = StackRef()
-        for test, suite in node.tests:
-            if hasattr(test, 'lineno'):
-                self.emit('SET_LINENO', test.lineno)
-            else:
-                print "warning", "no line number"
-            self.visit(test)
-            dest = StackRef()
-            self.emit('JUMP_IF_FALSE', dest)
-            self.emit('POP_TOP')
-            self.visit(suite)
-            self.emit('JUMP_FORWARD', after)
-            dest.bind(self.code.getCurInst())
-            self.emit('POP_TOP')
-        if node.else_:
-            self.visit(node.else_)
-        after.bind(self.code.getCurInst())
-        return 1
+	after = self.newBlock()
+	self.emit('SETUP_LOOP', after)
+
+	self.nextBlock(loop)
+	self.loops.push(loop)
+
+	self.emit('SET_LINENO', node.lineno)
+	self.visit(node.test)
+	self.emit('JUMP_IF_FALSE', else_ or after)
 
-    def startLoop(self):
-        l = Loop()
-        self.loops.push(l)
-        self.emit('SETUP_LOOP', l.extentAnchor)
-        return l
+	self.nextBlock()
+	self.emit('POP_TOP')
+	self.visit(node.body)
+	self.emit('JUMP_ABSOLUTE', loop)
 
-    def finishLoop(self):
-        l = self.loops.pop()
-        i = self.code.getCurInst()
-        l.extentAnchor.bind(self.code.getCurInst())
+	self.startBlock(else_) # or just the POPs if not else clause
+	self.emit('POP_TOP')
+	self.emit('POP_BLOCK')
+	if node.else_:
+	    self.visit(node.else_)
+	self.loops.pop()
+	self.nextBlock(after)
 
     def visitFor(self, node):
-        # three refs needed
-        anchor = StackRef()
+	start = self.newBlock()
+        anchor = self.newBlock()
+	after = self.newBlock()
+        self.loops.push(start)
 
         self.emit('SET_LINENO', node.lineno)
-        l = self.startLoop()
+	self.emit('SETUP_LOOP', after)
         self.visit(node.list)
         self.visit(ast.Const(0))
-        l.startAnchor.bind(self.code.getCurInst())
+	self.nextBlock(start)
         self.emit('SET_LINENO', node.lineno)
         self.emit('FOR_LOOP', anchor)
         self.visit(node.assign)
         self.visit(node.body)
-        self.emit('JUMP_ABSOLUTE', l.startAnchor)
-        anchor.bind(self.code.getCurInst())
+        self.emit('JUMP_ABSOLUTE', start)
+	self.nextBlock(anchor)
         self.emit('POP_BLOCK')
         if node.else_:
             self.visit(node.else_)
-        self.finishLoop()
-        return 1
-
-    def visitWhile(self, node):
-        self.emit('SET_LINENO', node.lineno)
-        l = self.startLoop()
-        if node.else_:
-            lElse = StackRef()
-        else:
-            lElse = l.breakAnchor
-        l.startAnchor.bind(self.code.getCurInst())
-        if hasattr(node.test, 'lineno'):
-            self.emit('SET_LINENO', node.test.lineno)
-        self.visit(node.test)
-        self.emit('JUMP_IF_FALSE', lElse)
-        self.emit('POP_TOP')
-        self.visit(node.body)
-        self.emit('JUMP_ABSOLUTE', l.startAnchor)
-        # note that lElse may be an alias for l.breakAnchor
-        lElse.bind(self.code.getCurInst())
-        self.emit('POP_TOP')
-        self.emit('POP_BLOCK')
-        if node.else_:
-            self.visit(node.else_)
-        self.finishLoop()
-        return 1
+	self.loops.pop()
+	self.nextBlock(after)
 
     def visitBreak(self, node):
-        if not self.loops:
-            raise SyntaxError, "'break' outside loop"
-        self.emit('SET_LINENO', node.lineno)
-        self.emit('BREAK_LOOP')
+	if not self.loops:
+	    raise SyntaxError, "'break' outside loop (%s, %d)" % \
+		  (self.filename, node.lineno)
+	self.emit('SET_LINENO', node.lineno)
+	self.emit('BREAK_LOOP')
 
     def visitContinue(self, node):
         if not self.loops:
-            raise SyntaxError, "'continue' outside loop"
+            raise SyntaxError, "'continue' outside loop (%s, %d)" % \
+		  (self.filename, node.lineno)
         l = self.loops.top()
         self.emit('SET_LINENO', node.lineno)
-        self.emit('JUMP_ABSOLUTE', l.startAnchor)
+        self.emit('JUMP_ABSOLUTE', l)
+	self.nextBlock()
+
+    def visitTest(self, node, jump):
+	end = self.newBlock()
+        for child in node.nodes[:-1]:
+            self.visit(child)
+            self.emit(jump, end)
+	    self.nextBlock()
+            self.emit('POP_TOP')
+        self.visit(node.nodes[-1])
+	self.nextBlock(end)
+
+    def visitAnd(self, node):
+	self.visitTest(node, 'JUMP_IF_FALSE')
+
+    def visitOr(self, node):
+	self.visitTest(node, 'JUMP_IF_TRUE')
+
+    def visitCompare(self, node):
+	self.visit(node.expr)
+	cleanup = self.newBlock()
+	for op, code in node.ops[:-1]:
+	    self.visit(code)
+	    self.emit('DUP_TOP')
+	    self.emit('ROT_THREE')
+	    self.emit('COMPARE_OP', op)
+	    self.emit('JUMP_IF_FALSE', cleanup)
+	    self.nextBlock()
+	    self.emit('POP_TOP')
+	# now do the last comparison
+	if node.ops:
+	    op, code = node.ops[-1]
+	    self.visit(code)
+	    self.emit('COMPARE_OP', op)
+	if len(node.ops) > 1:
+	    end = self.newBlock()
+	    self.emit('JUMP_FORWARD', end)
+	    self.nextBlock(cleanup)
+	    self.emit('ROT_TWO')
+	    self.emit('POP_TOP')
+	    self.nextBlock(end)
+
+    # exception related
+
+    def visitAssert(self, node):
+	# XXX would be interesting to implement this via a
+	# transformation of the AST before this stage
+	end = self.newBlock()
+	self.emit('SET_LINENO', node.lineno)
+        # XXX __debug__ and AssertionError appear to be special cases
+        # -- they are always loaded as globals even if there are local
+        # names.  I guess this is a sort of renaming op.
+	self.emit('LOAD_GLOBAL', '__debug__')
+	self.emit('JUMP_IF_FALSE', end)
+	self.nextBlock()
+	self.emit('POP_TOP')
+	self.visit(node.test)
+	self.emit('JUMP_IF_TRUE', end)
+	self.nextBlock()
+	self.emit('LOAD_GLOBAL', 'AssertionError')
+	self.visit(node.fail)
+	self.emit('RAISE_VARARGS', 2)
+	self.nextBlock(end)
+	self.emit('POP_TOP')
+
+    def visitRaise(self, node):
+        self.emit('SET_LINENO', node.lineno)
+        n = 0
+        if node.expr1:
+            self.visit(node.expr1)
+            n = n + 1
+        if node.expr2:
+            self.visit(node.expr2)
+            n = n + 1
+        if node.expr3:
+            self.visit(node.expr3)
+            n = n + 1
+        self.emit('RAISE_VARARGS', n)
 
     def visitTryExcept(self, node):
-        # XXX need to figure out exactly what is on the stack when an
-        # exception is raised and the first handler is checked
-        handlers = StackRef()
-        end = StackRef()
+        handlers = self.newBlock()
+        end = self.newBlock()
         if node.else_:
-            lElse = StackRef()
+            lElse = self.newBlock()
         else:
             lElse = end
         self.emit('SET_LINENO', node.lineno)
@@ -332,7 +326,7 @@ class CodeGenerator:
         self.visit(node.body)
         self.emit('POP_BLOCK')
         self.emit('JUMP_FORWARD', lElse)
-        handlers.bind(self.code.getCurInst())
+	self.nextBlock(handlers)
         
         last = len(node.handlers) - 1
         for i in range(len(node.handlers)):
@@ -342,9 +336,10 @@ class CodeGenerator:
             if expr:
                 self.emit('DUP_TOP')
                 self.visit(expr)
-                self.emit('COMPARE_OP', "exception match")
-                next = StackRef()
+                self.emit('COMPARE_OP', 'exception match')
+                next = self.newBlock()
                 self.emit('JUMP_IF_FALSE', next)
+		self.nextBlock()
                 self.emit('POP_TOP')
             self.emit('POP_TOP')
             if target:
@@ -355,132 +350,72 @@ class CodeGenerator:
             self.visit(body)
             self.emit('JUMP_FORWARD', end)
             if expr:
-                next.bind(self.code.getCurInst())
+		self.nextBlock(next)
             self.emit('POP_TOP')
         self.emit('END_FINALLY')
         if node.else_:
-            lElse.bind(self.code.getCurInst())
+	    self.nextBlock(lElse)
             self.visit(node.else_)
-        end.bind(self.code.getCurInst())
-        return 1
+	self.nextBlock(end)
     
     def visitTryFinally(self, node):
-        final = StackRef()
+        final = self.newBlock()
         self.emit('SET_LINENO', node.lineno)
         self.emit('SETUP_FINALLY', final)
         self.visit(node.body)
         self.emit('POP_BLOCK')
         self.emit('LOAD_CONST', None)
-        final.bind(self.code.getCurInst())
+	self.nextBlock(final)
         self.visit(node.final)
         self.emit('END_FINALLY')
-        return 1
 
-    def visitCompare(self, node):
-        """Comment from compile.c follows:
-
-        The following code is generated for all but the last
-        comparison in a chain:
-           
-        label:  on stack:       opcode:         jump to:
-           
-                a               <code to load b>
-                a, b            DUP_TOP
-                a, b, b         ROT_THREE
-                b, a, b         COMPARE_OP
-                b, 0-or-1       JUMP_IF_FALSE   L1
-                b, 1            POP_TOP
-                b               
-        
-        We are now ready to repeat this sequence for the next
-        comparison in the chain.
-           
-        For the last we generate:
-           
-                b               <code to load c>
-                b, c            COMPARE_OP
-                0-or-1          
-           
-        If there were any jumps to L1 (i.e., there was more than one
-        comparison), we generate:
-           
-                0-or-1          JUMP_FORWARD    L2
-           L1:  b, 0            ROT_TWO
-                0, b            POP_TOP
-                0
-           L2:  0-or-1
-        """
-        self.visit(node.expr)
-        # if refs are never emitted, subsequent bind call has no effect
-        l1 = StackRef()
-        l2 = StackRef()
-        for op, code in node.ops[:-1]:
-            # emit every comparison except the last
-            self.visit(code)
-            self.emit('DUP_TOP')
-            self.emit('ROT_THREE')
-            self.emit('COMPARE_OP', op)
-            # dupTop and compareOp cancel stack effect
-            self.emit('JUMP_IF_FALSE', l1)
-            self.emit('POP_TOP')
-        if node.ops:
-            # emit the last comparison
-            op, code = node.ops[-1]
-            self.visit(code)
-            self.emit('COMPARE_OP', op)
-        if len(node.ops) > 1:
-            self.emit('JUMP_FORWARD', l2)
-            l1.bind(self.code.getCurInst())
-            self.emit('ROT_TWO')
-            self.emit('POP_TOP')
-            l2.bind(self.code.getCurInst())
-        return 1
+    # misc
 
-    def visitGetattr(self, node):
-        self.visit(node.expr)
-        self.emit('LOAD_ATTR', node.attrname)
-        return 1
+##     def visitStmt(self, node):
+## 	# nothing to do except walk the children
+## 	pass
 
-    def visitSubscript(self, node):
+    def visitDiscard(self, node):
         self.visit(node.expr)
-        for sub in node.subs:
-            self.visit(sub)
-        if len(node.subs) > 1:
-            self.emit('BUILD_TUPLE', len(node.subs))
-        if node.flags == 'OP_APPLY':
-            self.emit('BINARY_SUBSCR')
-        elif node.flags == 'OP_ASSIGN':
-            self.emit('STORE_SUBSCR')
-        elif node.flags == 'OP_DELETE':
-            self.emit('DELETE_SUBSCR')
-        return 1
+        self.emit('POP_TOP')
 
-    def visitSlice(self, node):
+    def visitConst(self, node):
+        self.emit('LOAD_CONST', node.value)
+
+    def visitKeyword(self, node):
+	self.emit('LOAD_CONST', node.name)
+	self.visit(node.expr)
+
+    def visitGlobal(self, node):
+        # no code to generate
+	pass
+
+    def visitName(self, node):
+        self.loadName(node.name)
+        
+    def visitPass(self, node):
+        self.emit('SET_LINENO', node.lineno)
+
+    def visitImport(self, node):
+        self.emit('SET_LINENO', node.lineno)
+        for name in node.names:
+            self.emit('IMPORT_NAME', name)
+            self.storeName(name)
+
+    def visitFrom(self, node):
+        self.emit('SET_LINENO', node.lineno)
+        self.emit('IMPORT_NAME', node.modname)
+        for name in node.names:
+            if name == '*':
+                self.namespace = 0
+            self.emit('IMPORT_FROM', name)
+        self.emit('POP_TOP')
+
+    def visitGetattr(self, node):
         self.visit(node.expr)
-        slice = 0
-        if node.lower:
-            self.visit(node.lower)
-            slice = slice | 1
-        if node.upper:
-            self.visit(node.upper)
-            slice = slice | 2
-        if node.flags == 'OP_APPLY':
-            self.emit('SLICE+%d' % slice)
-        elif node.flags == 'OP_ASSIGN':
-            self.emit('STORE_SLICE+%d' % slice)
-        elif node.flags == 'OP_DELETE':
-            self.emit('DELETE_SLICE+%d' % slice)
-        else:
-            print "weird slice", node.flags
-            raise
-        return 1
+        self.emit('LOAD_ATTR', node.attrname)
 
-    def visitSliceobj(self, node):
-        for child in node.nodes:
-            print child
-            self.visit(child)
-        self.emit('BUILD_SLICE', len(node.nodes))
-        return 1
+    # next five implement assignments
 
     def visitAssign(self, node):
         self.emit('SET_LINENO', node.lineno)
@@ -492,7 +427,6 @@ class CodeGenerator:
                 self.emit('DUP_TOP')
             if isinstance(elt, ast.Node):
                 self.visit(elt)
-        return 1
 
     def visitAssName(self, node):
         if node.flags == 'OP_ASSIGN':
@@ -501,7 +435,6 @@ class CodeGenerator:
             self.delName(node.name)
         else:
             print "oops", node.flags
-        return 1
 
     def visitAssAttr(self, node):
         self.visit(node.expr)
@@ -512,24 +445,96 @@ class CodeGenerator:
         else:
             print "warning: unexpected flags:", node.flags
             print node
-        return 1
 
     def visitAssTuple(self, node):
         if findOp(node) != 'OP_DELETE':
             self.emit('UNPACK_TUPLE', len(node.nodes))
         for child in node.nodes:
             self.visit(child)
-        return 1
 
     visitAssList = visitAssTuple
 
+    def visitExec(self, node):
+        self.visit(node.expr)
+        if node.locals is None:
+            self.emit('LOAD_CONST', None)
+        else:
+            self.visit(node.locals)
+        if node.globals is None:
+            self.emit('DUP_TOP')
+        else:
+            self.visit(node.globals)
+        self.emit('EXEC_STMT')
+
+    def visitCallFunc(self, node):
+        pos = 0
+        kw = 0
+        if hasattr(node, 'lineno'):
+            self.emit('SET_LINENO', node.lineno)
+        self.visit(node.node)
+        for arg in node.args:
+            self.visit(arg)
+            if isinstance(arg, ast.Keyword):
+                kw = kw + 1
+            else:
+                pos = pos + 1
+        self.emit('CALL_FUNCTION', kw << 8 | pos)
+
+    def visitPrint(self, node):
+        self.emit('SET_LINENO', node.lineno)
+        for child in node.nodes:
+            self.visit(child)
+            self.emit('PRINT_ITEM')
+
+    def visitPrintnl(self, node):
+        self.visitPrint(node)
+        self.emit('PRINT_NEWLINE')
+
+    def visitReturn(self, node):
+        self.emit('SET_LINENO', node.lineno)
+        self.visit(node.value)
+        self.emit('RETURN_VALUE')
+
+    # slice and subscript stuff
+
+    def visitSlice(self, node):
+        self.visit(node.expr)
+        slice = 0
+        if node.lower:
+            self.visit(node.lower)
+            slice = slice | 1
+        if node.upper:
+            self.visit(node.upper)
+            slice = slice | 2
+        if node.flags == 'OP_APPLY':
+            self.emit('SLICE+%d' % slice)
+        elif node.flags == 'OP_ASSIGN':
+            self.emit('STORE_SLICE+%d' % slice)
+        elif node.flags == 'OP_DELETE':
+            self.emit('DELETE_SLICE+%d' % slice)
+        else:
+            print "weird slice", node.flags
+            raise
+
+    def visitSubscript(self, node):
+        self.visit(node.expr)
+        for sub in node.subs:
+            self.visit(sub)
+        if len(node.subs) > 1:
+            self.emit('BUILD_TUPLE', len(node.subs))
+        if node.flags == 'OP_APPLY':
+            self.emit('BINARY_SUBSCR')
+        elif node.flags == 'OP_ASSIGN':
+            self.emit('STORE_SUBSCR')
+        elif node.flags == 'OP_DELETE':
+            self.emit('DELETE_SUBSCR')
+
     # binary ops
 
     def binaryOp(self, node, op):
         self.visit(node.left)
         self.visit(node.right)
         self.emit(op)
-        return 1
 
     def visitAdd(self, node):
         return self.binaryOp(node, 'BINARY_ADD')
@@ -560,7 +565,6 @@ class CodeGenerator:
     def unaryOp(self, node, op):
         self.visit(node.expr)
         self.emit(op)
-        return 1
 
     def visitInvert(self, node):
         return self.unaryOp(node, 'UNARY_INVERT')
@@ -587,7 +591,6 @@ class CodeGenerator:
         for node in nodes[1:]:
             self.visit(node)
             self.emit(op)
-        return 1
 
     def visitBitand(self, node):
         return self.bitOp(node.nodes, 'BINARY_AND')
@@ -598,121 +601,137 @@ class CodeGenerator:
     def visitBitxor(self, node):
         return self.bitOp(node.nodes, 'BINARY_XOR')
 
-    def visitAssert(self, node):
-        # XXX __debug__ and AssertionError appear to be special cases
-        # -- they are always loaded as globals even if there are local
-        # names.  I guess this is a sort of renaming op.
-        skip = StackRef()
-        self.emit('SET_LINENO', node.lineno)
-        self.emit('LOAD_GLOBAL', '__debug__')
-        self.emit('JUMP_IF_FALSE', skip)
-        self.emit('POP_TOP')
-        self.visit(node.test)
-        self.emit('JUMP_IF_TRUE', skip)
-        self.emit('LOAD_GLOBAL', 'AssertionError')
-        self.visit(node.fail)
-        self.emit('RAISE_VARARGS', 2)
-        skip.bind(self.code.getCurInst())
-        self.emit('POP_TOP')
-        return 1
-
-    def visitTest(self, node, jump):
-        end = StackRef()
-        for child in node.nodes[:-1]:
-            self.visit(child)
-            self.emit(jump, end)
-            self.emit('POP_TOP')
-        self.visit(node.nodes[-1])
-        end.bind(self.code.getCurInst())
-        return 1
-
-    def visitAnd(self, node):
-        return self.visitTest(node, 'JUMP_IF_FALSE')
-
-    def visitOr(self, node):
-        return self.visitTest(node, 'JUMP_IF_TRUE')
-
-    def visitName(self, node):
-        self.loadName(node.name)
-
-    def visitConst(self, node):
-        self.emit('LOAD_CONST', node.value)
-        return 1
+    # object constructors
 
     def visitEllipsis(self, node):
         self.emit('LOAD_CONST', Ellipsis)
-        return 1
 
     def visitTuple(self, node):
         for elt in node.nodes:
             self.visit(elt)
         self.emit('BUILD_TUPLE', len(node.nodes))
-        return 1
 
     def visitList(self, node):
         for elt in node.nodes:
             self.visit(elt)
         self.emit('BUILD_LIST', len(node.nodes))
-        return 1
+
+    def visitSliceobj(self, node):
+        for child in node.nodes:
+            self.visit(child)
+        self.emit('BUILD_SLICE', len(node.nodes))
 
     def visitDict(self, node):
+	# XXX is this a good general strategy?  could it be done 
+	# separately from the general visitor
+	lineno = getattr(node, 'lineno', None)
+	if lineno:
+	    self.emit('SET_LINENO', lineno)
         self.emit('BUILD_MAP', 0)
         for k, v in node.items:
-            # XXX need to add set lineno when there aren't constants
+	    lineno2 = getattr(node, 'lineno', None)
+	    if lineno != lineno2:
+		self.emit('SET_LINENO', lineno2)
+		lineno = lineno2
             self.emit('DUP_TOP')
             self.visit(v)
             self.emit('ROT_TWO')
             self.visit(k)
             self.emit('STORE_SUBSCR')
-        return 1
 
-    def visitReturn(self, node):
-        self.emit('SET_LINENO', node.lineno)
-        self.visit(node.value)
-        self.emit('RETURN_VALUE')
-        return 1
+class ModuleCodeGenerator(CodeGenerator):
+    super_init = CodeGenerator.__init__
+    
+    def __init__(self, filename):
+	# XXX <module> is ? in compile.c
+	self.graph = pyassem.PyFlowGraph("<module>", filename)
+	self.super_init(filename)
+
+class FunctionCodeGenerator(CodeGenerator):
+    super_init = CodeGenerator.__init__
+
+    optimized = 1
+    lambdaCount = 0
+
+    def __init__(self, func, filename, isLambda=0):
+        if isLambda:
+            klass = FunctionCodeGenerator
+            name = "<lambda.%d>" % klass.lambdaCount
+            klass.lambdaCount = klass.lambdaCount + 1
+        else:
+            name = func.name
+	args, hasTupleArg = generateArgList(func.argnames)
+	self.graph = pyassem.PyFlowGraph(name, filename, args, 
+					   optimized=1) 
+	self.isLambda = isLambda
+	self.super_init(filename)
 
-    def visitRaise(self, node):
-        self.emit('SET_LINENO', node.lineno)
-        n = 0
-        if node.expr1:
-            self.visit(node.expr1)
-            n = n + 1
-        if node.expr2:
-            self.visit(node.expr2)
-            n = n + 1
-        if node.expr3:
-            self.visit(node.expr3)
-            n = n + 1
-        self.emit('RAISE_VARARGS', n)
-        return 1
+        lnf = walk(func.code, LocalNameFinder(args), 0)
+        self.locals.push(lnf.getLocals())
+	if func.varargs:
+	    self.graph.setFlag(CO_VARARGS)
+	if func.kwargs:
+	    self.graph.setFlag(CO_VARKEYWORDS)
+        self.emit('SET_LINENO', func.lineno)
+        if hasTupleArg:
+            self.generateArgUnpack(func.argnames)
 
-    def visitPrint(self, node):
-        self.emit('SET_LINENO', node.lineno)
-        for child in node.nodes:
-            self.visit(child)
-            self.emit('PRINT_ITEM')
-        return 1
+    def finish(self):
+	self.graph.startExitBlock()
+	if not self.isLambda:
+	    self.emit('LOAD_CONST', None)
+	self.emit('RETURN_VALUE')
 
-    def visitPrintnl(self, node):
-        self.visitPrint(node)
-        self.emit('PRINT_NEWLINE')
-        return 1
+    def generateArgUnpack(self, args):
+        count = 0
+        for arg in args:
+            if type(arg) == types.TupleType:
+                self.emit('LOAD_FAST', '.nested%d' % count)
+                count = count + 1
+                self.unpackTuple(arg)
+                        
+    def unpackTuple(self, tup):
+        self.emit('UNPACK_TUPLE', len(tup))
+        for elt in tup:
+            if type(elt) == types.TupleType:
+                self.unpackTuple(elt)
+            else:
+                self.emit('STORE_FAST', elt)
 
-    def visitExec(self, node):
-        self.visit(node.expr)
-        if node.locals is None:
-            self.emit('LOAD_CONST', None)
-        else:
-            self.visit(node.locals)
-        if node.globals is None:
-            self.emit('DUP_TOP')
-        else:
-            self.visit(node.globals)
-        self.emit('EXEC_STMT')
-        return 1
+class ClassCodeGenerator(CodeGenerator):
+    super_init = CodeGenerator.__init__
+
+    def __init__(self, klass, filename):
+	self.graph = pyassem.PyFlowGraph(klass.name, filename,
+					   optimized=0)
+        self.super_init(filename)
+        lnf = walk(klass.code, LocalNameFinder(), 0)
+        self.locals.push(lnf.getLocals())
+
+    def finish(self):
+	self.graph.startExitBlock()
+        self.emit('LOAD_LOCALS')
+	self.emit('RETURN_VALUE')
+
+
+def generateArgList(arglist):
+    """Generate an arg list marking TupleArgs"""
+    args = []
+    extra = []
+    count = 0
+    for elt in arglist:
+	if type(elt) == types.StringType:
+	    args.append(elt)
+	elif type(elt) == types.TupleType:
+	    args.append(TupleArg(count, elt))
+	    count = count + 1
+	    extra.extend(misc.flatten(elt))
+	else:
+	    raise ValueError, "unexpect argument type:", elt
+    return args + extra, count
 
 class LocalNameFinder:
+    """Find local names in scope"""
     def __init__(self, names=()):
         self.names = misc.Set()
         self.globals = misc.Set()
@@ -720,25 +739,23 @@ class LocalNameFinder:
             self.names.add(name)
 
     def getLocals(self):
-        for elt in self.globals.items():
+        for elt in self.globals.elements():
             if self.names.has_elt(elt):
                 self.names.remove(elt)
         return self.names
 
     def visitDict(self, node):
-        return 1
+	pass
 
     def visitGlobal(self, node):
         for name in node.names:
             self.globals.add(name)
-        return 1
 
     def visitFunction(self, node):
         self.names.add(node.name)
-        return 1
 
     def visitLambda(self, node):
-        return 1
+	pass
 
     def visitImport(self, node):
         for name in node.names:
@@ -750,11 +767,16 @@ class LocalNameFinder:
 
     def visitClass(self, node):
         self.names.add(node.name)
-        return 1
 
     def visitAssName(self, node):
         self.names.add(node.name)
 
+def findOp(node):
+    """Find the op (DELETE, LOAD, STORE) in an AssTuple tree"""
+    v = OpFinder()
+    walk(node, v, 0)
+    return v.op
+
 class OpFinder:
     def __init__(self):
         self.op = None
@@ -764,57 +786,8 @@ class OpFinder:
         elif self.op != node.flags:
             raise ValueError, "mixed ops in stmt"
 
-def findOp(node):
-    v = OpFinder()
-    walk(node, v)
-    return v.op
-
-class Loop:
-    def __init__(self):
-        self.startAnchor = StackRef()
-        self.breakAnchor = StackRef()
-        self.extentAnchor = StackRef()
-
-class CompiledModule:
-    """Store the code object for a compiled module
-
-    XXX Not clear how the code objects will be stored.  Seems possible
-    that a single code attribute is sufficient, because it will
-    contains references to all the need code objects.  That might be
-    messy, though.
-    """
-    MAGIC = (20121 | (ord('\r')<<16) | (ord('\n')<<24))
-
-    def __init__(self, source, filename):
-        self.source = source
-        self.filename = filename
+if __name__ == "__main__":
+    import sys
 
-    def compile(self):
-	self.ast = parse(self.source)
-        cg = CodeGenerator(self.filename)
-        walk(self.ast, cg)
-        self.code = cg.asConst()
-
-    def dump(self, path):
-        """create a .pyc file"""
-        f = open(path, 'wb')
-        f.write(self._pyc_header())
-        marshal.dump(self.code, f)
-        f.close()
-        
-    def _pyc_header(self):
-        # compile.c uses marshal to write a long directly, with
-        # calling the interface that would also generate a 1-byte code
-        # to indicate the type of the value.  simplest way to get the
-        # same effect is to call marshal and then skip the code.
-        magic = marshal.dumps(self.MAGIC)[1:]
-        mtime = os.stat(self.filename)[stat.ST_MTIME]
-        mtime = struct.pack('i', mtime)
-        return magic + mtime
-
-def compile(filename):
-    buf = open(filename).read()
-    mod = CompiledModule(buf, filename)
-    mod.compile()
-    mod.dump(filename + 'c')
-        
+    for file in sys.argv[1:]:
+	compile(file)