PEP 3107 - Function Annotations thanks to Tony Lownds

14 files changed, 460 insertions, 182 deletions

diff --git a/Lib/compiler/ast.py b/Lib/compiler/ast.py
index d21905f..ac48710 100644
--- a/Lib/compiler/ast.py
+++ b/Lib/compiler/ast.py
@@ -33,7 +33,10 @@ class Node:
         pass # implemented by subclasses
 
 class EmptyNode(Node):
-    pass
+    def getChildNodes(self):
+        return ()
+    def getChildren(self):
+        return ()
 
 class Expression(Node):
     # Expression is an artificial node class to support "eval"
@@ -487,12 +490,13 @@ class From(Node):
         return "From(%s, %s, %s)" % (repr(self.modname), repr(self.names), repr(self.level))
 
 class Function(Node):
-    def __init__(self, decorators, name, argnames, defaults, kwonlyargs, flags, doc, code, lineno=None):
+    def __init__(self, decorators, name, arguments, defaults, kwonlyargs, returns, flags, doc, code, lineno=None):
         self.decorators = decorators
         self.name = name
-        self.argnames = argnames
+        self.arguments = arguments
         self.defaults = defaults
         self.kwonlyargs = kwonlyargs
+        self.returns = returns
         self.flags = flags
         self.doc = doc
         self.code = code
@@ -508,9 +512,10 @@ class Function(Node):
         children = []
         children.append(self.decorators)
         children.append(self.name)
-        children.append(self.argnames)
+        children.extend(flatten(self.arguments))
         children.extend(flatten(self.defaults))
-        children.append(self.kwonlyargs)
+        children.extend(flatten(self.kwonlyargs))
+        children.append(self.returns)
         children.append(self.flags)
         children.append(self.doc)
         children.append(self.code)
@@ -520,18 +525,22 @@ class Function(Node):
         nodelist = []
         if self.decorators is not None:
             nodelist.append(self.decorators)
+        nodelist.extend(flatten_nodes(self.arguments))
         nodelist.extend(flatten_nodes(self.defaults))
+        nodelist.extend(flatten_nodes(self.kwonlyargs))
+        if self.returns is not None:
+            nodelist.append(self.returns)
         nodelist.append(self.code)
         return tuple(nodelist)
 
     def __repr__(self):
-        return "Function(%s, %s, %s, %s, %s, %s, %s, %s)" % (repr(self.decorators), repr(self.name), repr(self.argnames), repr(self.defaults), repr(self.kwonlyargs), repr(self.flags), repr(self.doc), repr(self.code))
+        return "Function(%s, %s, %s, %s, %s, %s, %s, %s, %s)" % (repr(self.decorators), repr(self.name), repr(self.arguments), repr(self.defaults), repr(self.kwonlyargs), repr(self.returns), repr(self.flags), repr(self.doc), repr(self.code))
 
 class GenExpr(Node):
     def __init__(self, code, lineno=None):
         self.code = code
         self.lineno = lineno
-        self.argnames = ['.0']
+        self.arguments = [SimpleArg('.0', None)]
         self.varargs = self.kwargs = None
         self.kwonlyargs = ()
 
@@ -715,9 +724,24 @@ class Keyword(Node):
     def __repr__(self):
         return "Keyword(%s, %s)" % (repr(self.name), repr(self.expr))
 
+class Kwarg(Node):
+    def __init__(self, arg, expr, lineno=None):
+        self.arg = arg
+        self.expr = expr
+        self.lineno = lineno
+
+    def getChildren(self):
+        return self.arg, self.expr
+
+    def getChildNodes(self):
+        return self.arg, self.expr
+
+    def __repr__(self):
+        return "Kwarg(%s, %s)" % (repr(self.arg), repr(self.expr))
+
 class Lambda(Node):
-    def __init__(self, argnames, defaults, kwonlyargs, flags, code, lineno=None):
-        self.argnames = argnames
+    def __init__(self, arguments, defaults, kwonlyargs, flags, code, lineno=None):
+        self.arguments = arguments
         self.defaults = defaults
         self.kwonlyargs = kwonlyargs
         self.flags = flags
@@ -728,25 +752,28 @@ class Lambda(Node):
             self.varargs = 1
         if flags & CO_VARKEYWORDS:
             self.kwargs = 1
+        self.returns = None
 
 
     def getChildren(self):
         children = []
-        children.append(self.argnames)
+        children.extend(flatten(self.arguments))
         children.extend(flatten(self.defaults))
-        children.append(self.kwonlyargs)
+        children.extend(flatten(self.kwonlyargs))
         children.append(self.flags)
         children.append(self.code)
         return tuple(children)
 
     def getChildNodes(self):
         nodelist = []
+        nodelist.extend(flatten_nodes(self.arguments))
         nodelist.extend(flatten_nodes(self.defaults))
+        nodelist.extend(flatten_nodes(self.kwonlyargs))
         nodelist.append(self.code)
         return tuple(nodelist)
 
     def __repr__(self):
-        return "Lambda(%s, %s, %s, %s, %s)" % (repr(self.argnames), repr(self.defaults), repr(self.kwonlyargs), repr(self.flags), repr(self.code))
+        return "Lambda(%s, %s, %s, %s, %s)" % (repr(self.arguments), repr(self.defaults), repr(self.kwonlyargs), repr(self.flags), repr(self.code))
 
 class LeftShift(Node):
     def __init__(self, (left, right), lineno=None):
@@ -897,6 +924,22 @@ class Name(Node):
     def __repr__(self):
         return "Name(%s)" % (repr(self.name),)
 
+class NestedArgs(Node):
+    def __init__(self, args, lineno=None):
+        self.args = args
+        self.lineno = lineno
+
+    def getChildren(self):
+        return tuple(flatten(self.args))
+
+    def getChildNodes(self):
+        nodelist = []
+        nodelist.extend(flatten_nodes(self.args))
+        return tuple(nodelist)
+
+    def __repr__(self):
+        return "NestedArgs(%s)" % (repr(self.args),)
+
 class Not(Node):
     def __init__(self, expr, lineno=None):
         self.expr = expr
@@ -1071,6 +1114,27 @@ class Set(Node):
     def __repr__(self):
         return "Set(%s)" % (repr(self.items),)
 
+class SimpleArg(Node):
+    def __init__(self, name, annotation, lineno=None):
+        self.name = name
+        self.annotation = annotation
+        self.lineno = lineno
+
+    def getChildren(self):
+        children = []
+        children.append(self.name)
+        children.append(self.annotation)
+        return tuple(children)
+
+    def getChildNodes(self):
+        nodelist = []
+        if self.annotation is not None:
+            nodelist.append(self.annotation)
+        return tuple(nodelist)
+
+    def __repr__(self):
+        return "SimpleArg(%s, %s)" % (repr(self.name), repr(self.annotation))
+
 class Slice(Node):
     def __init__(self, expr, flags, lower, upper, lineno=None):
         self.expr = expr
diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py
index 7f44d46..86e2c49 100644
--- a/Lib/compiler/pyassem.py
+++ b/Lib/compiler/pyassem.py
@@ -314,7 +314,7 @@ class PyFlowGraph(FlowGraph):
     super_init = FlowGraph.__init__
 
     def __init__(self, name, filename,
-                 args=(), kwonlyargs={}, optimized=0, klass=None):
+                 args=(), kwonlyargs=(), optimized=0, klass=None):
         self.super_init()
         self.name = name
         self.filename = filename
@@ -338,24 +338,40 @@ class PyFlowGraph(FlowGraph):
         # The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both
         # kinds of variables.
         self.closure = []
-        self.varnames = list(args) or []
-        for i in range(len(self.varnames)):
+        # The varnames list needs to be computed after flags have been set
+        self.varnames = []
+        self.stage = RAW
+
+    def computeVarnames(self):
+        # self.args is positional, vararg, kwarg, kwonly, unpacked. This
+        # order is due to the visit order in symbol module and could change.
+        # argcount is # len(self.args) - len(unpacked). We want
+        # self.varnames to be positional, kwonly, vararg, kwarg, unpacked
+        # and argcount to be len(positional).
+
+        # determine starting index of unpacked, kwonly, vararg
+        u = self.argcount    # starting index of unpacked
+        k = u - len(self.kwonlyargs)  # starting index of kwonly
+        v = k - self.checkFlag(CO_VARARGS) - self.checkFlag(CO_VARKEYWORDS)
+
+        vars = list(self.args)
+        self.varnames = vars[:v] + vars[k:u] + vars[v:k] + vars[u:]
+        self.argcount = v
+
+        # replace TupleArgs with calculated var name
+        for i in range(self.argcount):
             var = self.varnames[i]
             if isinstance(var, TupleArg):
                 self.varnames[i] = var.getName()
-        self.stage = RAW
 
     def setDocstring(self, doc):
         self.docstring = doc
 
     def setFlag(self, flag):
         self.flags = self.flags | flag
-        if flag == CO_VARARGS:
-            self.argcount = self.argcount - 1
 
     def checkFlag(self, flag):
-        if self.flags & flag:
-            return 1
+        return (self.flags & flag) == flag
 
     def setFreeVars(self, names):
         self.freevars = list(names)
@@ -366,6 +382,7 @@ class PyFlowGraph(FlowGraph):
     def getCode(self):
         """Get a Python code object"""
         assert self.stage == RAW
+        self.computeVarnames()
         self.computeStackDepth()
         self.flattenGraph()
         assert self.stage == FLAT
@@ -575,6 +592,12 @@ class PyFlowGraph(FlowGraph):
                     lnotab.nextLine(oparg)
                     continue
                 hi, lo = twobyte(oparg)
+
+                extended, hi = twobyte(hi)
+                if extended:
+                  ehi, elo = twobyte(extended)
+                  lnotab.addCode(self.opnum['EXTENDED_ARG'], elo, ehi)
+
                 try:
                     lnotab.addCode(self.opnum[opname], lo, hi)
                 except ValueError:
@@ -595,8 +618,6 @@ class PyFlowGraph(FlowGraph):
         else:
             nlocals = len(self.varnames)
         argcount = self.argcount
-        if self.flags & CO_VARKEYWORDS:
-            argcount = argcount - 1
         kwonlyargcount = len(self.kwonlyargs)
         return new.code(argcount, kwonlyargcount,
                         nlocals, self.stacksize, self.flags,
@@ -809,7 +830,8 @@ class StackDepthTracker:
         return self.CALL_FUNCTION(argc)-2
     def MAKE_FUNCTION(self, argc):
         hi, lo = divmod(argc, 256)
-        return -(lo + hi * 2)
+        ehi, hi = divmod(hi, 256)
+        return -(lo + hi * 2 + ehi)
     def MAKE_CLOSURE(self, argc):
         # XXX need to account for free variables too!
         return -argc
diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py
index 353c2c9..325ca06 100644
--- a/Lib/compiler/pycodegen.py
+++ b/Lib/compiler/pycodegen.py
@@ -378,18 +378,57 @@ class CodeGenerator:
         walk(node.code, gen)
         gen.finish()
         self.set_lineno(node)
+        num_kwargs = 0
         for keyword in node.kwonlyargs:
             default = keyword.expr
             if isinstance(default, ast.EmptyNode):
                 continue
-            self.emit('LOAD_CONST', keyword.name)
+            self.emit('LOAD_CONST', keyword.arg.name)
             self.visit(default)
+            num_kwargs += 1
         for default in node.defaults:
             self.visit(default)
-        self._makeClosure(gen, len(node.defaults))
+
+        num_annotations = self._visit_annotations(node)
+
+        oparg = len(node.defaults)
+        oparg |= num_kwargs << 8
+        oparg |= num_annotations << 16
+
+        self._makeClosure(gen, oparg)
         for i in range(ndecorators):
             self.emit('CALL_FUNCTION', 1)
 
+    def _visit_annotations(self, node):
+        # emit code, return num_annotations
+        annotations = []
+        annotations.extend(self._visit_argument_annotations(node.arguments))
+        annotations.extend(self._visit_kwarg_annotations(node.kwonlyargs))
+        if node.returns:
+            self.visit(node.returns)
+            annotations.append('return')
+        if not annotations:
+            return 0
+        self.emit('LOAD_CONST', tuple(annotations))
+        return len(annotations) + 1
+
+    def _visit_argument_annotations(self, arguments):
+        for arg in arguments:
+            if isinstance(arg, ast.SimpleArg):
+                if arg.annotation:
+                    self.visit(arg.annotation)
+                    yield arg.name
+            else:
+                for name in self._visit_argument_annotations(arg.args):
+                    yield name
+
+    def _visit_kwarg_annotations(self, kwargs):
+        for kwarg in kwargs:
+            arg = kwarg.arg
+            if arg.annotation:
+                self.visit(arg.annotation)
+                yield arg.name
+
     def visitClass(self, node):
         gen = self.ClassGen(node, self.scopes,
                             self.get_module())
@@ -1323,7 +1362,7 @@ class AbstractFunctionCode:
         else:
             name = func.name
 
-        args, hasTupleArg = generateArgList(func.argnames)
+        args, hasTupleArg = generateArgList(func.arguments)
         kwonlyargs = generateKwonlyArgList(func.kwonlyargs)
         self.graph = pyassem.PyFlowGraph(name, func.filename, args,
                                          kwonlyargs=kwonlyargs,
@@ -1334,7 +1373,7 @@ class AbstractFunctionCode:
         if not isLambda and func.doc:
             self.setDocstring(func.doc)
 
-        lnf = walk(func.code, self.NameFinder(args), verbose=0)
+        lnf = walk(func.code, self.NameFinder(args+kwonlyargs), verbose=0)
         self.locals.push(lnf.getLocals())
         if func.varargs:
             self.graph.setFlag(CO_VARARGS)
@@ -1342,7 +1381,7 @@ class AbstractFunctionCode:
             self.graph.setFlag(CO_VARKEYWORDS)
         self.set_lineno(func)
         if hasTupleArg:
-            self.generateArgUnpack(func.argnames)
+            self.generateArgUnpack(func.arguments)
 
     def get_module(self):
         return self.module
@@ -1356,9 +1395,9 @@ class AbstractFunctionCode:
     def generateArgUnpack(self, args):
         for i in range(len(args)):
             arg = args[i]
-            if isinstance(arg, tuple):
+            if isinstance(arg, ast.NestedArgs):
                 self.emit('LOAD_FAST', '.%d' % (i * 2))
-                self.unpackSequence(arg)
+                self.unpackSequence(tuple(_nested_names(arg)))
 
     def unpackSequence(self, tup):
         if VERSION > 1:
@@ -1452,21 +1491,29 @@ def generateArgList(arglist):
     count = 0
     for i in range(len(arglist)):
         elt = arglist[i]
-        if isinstance(elt, str):
-            args.append(elt)
-        elif isinstance(elt, tuple):
-            args.append(TupleArg(i * 2, elt))
-            extra.extend(misc.flatten(elt))
+        if isinstance(elt, ast.SimpleArg):
+            args.append(elt.name)
+        elif isinstance(elt, ast.NestedArgs):
+            t = tuple(_nested_names(elt))
+            args.append(TupleArg(i * 2, t))
+            extra.extend(misc.flatten(t))
             count = count + 1
         else:
             raise ValueError, "unexpect argument type:", elt
     return args + extra, count
 
+def _nested_names(elt):
+  for arg in elt.args:
+    if isinstance(arg, ast.SimpleArg):
+      yield arg.name
+    elif isinstance(arg, ast.NestedArgs):
+      yield tuple(_nested_names(arg))
+
 def generateKwonlyArgList(keywordOnlyArgs):
-    kwonlyargs = {}
+    kwonlyargs = []
     for elt in keywordOnlyArgs:
-        assert isinstance(elt, ast.Keyword)
-        kwonlyargs[elt.name] = elt.expr
+        assert isinstance(elt, ast.Kwarg)
+        kwonlyargs.append(elt.arg.name)
     return kwonlyargs
     
 def findOp(node):
diff --git a/Lib/compiler/symbols.py b/Lib/compiler/symbols.py
index 7ddf42c..3585efc 100644
--- a/Lib/compiler/symbols.py
+++ b/Lib/compiler/symbols.py
@@ -233,7 +233,12 @@ class SymbolVisitor:
         if parent.nested or isinstance(parent, FunctionScope):
             scope.nested = 1
         self.scopes[node] = scope
-        self._do_args(scope, node.argnames)
+
+        args = node.arguments
+        for kwonly in node.kwonlyargs:
+            args.append(kwonly.arg)
+        self._do_arguments(scope, args)
+
         self.visit(node.code, scope)
         self.handle_free_vars(scope, parent)
 
@@ -275,16 +280,18 @@ class SymbolVisitor:
         if parent.nested or isinstance(parent, FunctionScope):
             scope.nested = 1
         self.scopes[node] = scope
-        self._do_args(scope, node.argnames)
+        self._do_arguments(scope, node.arguments)
         self.visit(node.code, scope)
         self.handle_free_vars(scope, parent)
 
-    def _do_args(self, scope, args):
-        for name in args:
-            if type(name) == types.TupleType:
-                self._do_args(scope, name)
+    def _do_arguments(self, scope, arguments):
+        for node in arguments:
+            if isinstance(node, ast.SimpleArg):
+                scope.add_param(node.name)
+                if node.annotation:
+                    self.visit(node.annotation, scope)
             else:
-                scope.add_param(name)
+                self._do_arguments(scope, node.args)
 
     def handle_free_vars(self, scope, parent):
         parent.add_child(scope)
diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py
index dc88222..4a8e623 100644
--- a/Lib/compiler/transformer.py
+++ b/Lib/compiler/transformer.py
@@ -234,25 +234,24 @@ class Transformer:
         return Decorators(items)
 
     def funcdef(self, nodelist):
-        #                    -6   -5    -4         -3  -2    -1
-        # funcdef: [decorators] 'def' NAME parameters ':' suite
-        # parameters: '(' [varargslist] ')'
-
-        if len(nodelist) == 6:
-            assert nodelist[0][0] == symbol.decorators
+        #                         0    1    2                4         -1
+        # funcdef: [decorators] 'def' NAME parameters ['->' test] ':' suite
+        # parameters: '(' [typedargslist] ')'
+        if nodelist[0][0] == symbol.decorators:
             decorators = self.decorators(nodelist[0][1:])
+            nodelist = nodelist[1:]
         else:
-            assert len(nodelist) == 5
             decorators = None
+        assert len(nodelist) in (5, 7)
 
-        lineno = nodelist[-4][2]
-        name = nodelist[-4][1]
-        args = nodelist[-3][2]
+        lineno = nodelist[0][2]
+        name = nodelist[1][1]
+        args = nodelist[2][2]
 
-        if args[0] == symbol.varargslist:
-            names, defaults, kwonlyargs, flags = self.com_arglist(args[1:])
+        if args[0] == symbol.varargslist or args[0] == symbol.typedargslist:
+            arguments, defaults, kwonly, flags = self.com_arglist(args[1:])
         else:
-            names = defaults = kwonlyargs = ()
+            arguments = defaults = kwonly = ()
             flags = 0
         doc = self.get_docstring(nodelist[-1])
 
@@ -263,22 +262,28 @@ class Transformer:
             assert isinstance(code, Stmt)
             assert isinstance(code.nodes[0], Discard)
             del code.nodes[0]
-        return Function(decorators, name, names, defaults,
-                        kwonlyargs, flags, doc, code, lineno=lineno)
+
+        if len(nodelist) == 7:
+            returns = self.com_node(nodelist[4])
+        else:
+            returns = None
+
+        return Function(decorators, name, arguments, defaults,
+                        kwonly, returns, flags, doc, code, lineno=lineno)
 
     def lambdef(self, nodelist):
         # lambdef: 'lambda' [varargslist] ':' test
         if nodelist[2][0] == symbol.varargslist:
-            names, defaults, kwonlyargs, flags = \
+            arguments, defaults, kwonlyargs, flags = \
                                 self.com_arglist(nodelist[2][1:])
         else:
-            names = defaults = kwonlyargs = ()
+            arguments = defaults = kwonlyargs = ()
             flags = 0
 
         # code for lambda
         code = self.com_node(nodelist[-1])
 
-        return Lambda(names, defaults, kwonlyargs,
+        return Lambda(arguments, defaults, kwonlyargs,
                       flags, code, lineno=nodelist[1][2])
     old_lambdef = lambdef
 
@@ -324,10 +329,25 @@ class Transformer:
     def varargslist(self, nodelist):
         raise WalkerError
 
-    def fpdef(self, nodelist):
+    def vfpdef(self, nodelist):
         raise WalkerError
 
-    def fplist(self, nodelist):
+    def vfplist(self, nodelist):
+        raise WalkerError
+
+    def vname(self, nodelist):
+        raise WalkerError
+
+    def typedargslist(self, nodelist):
+        raise WalkerError
+
+    def tfpdef(self, nodelist):
+        raise WalkerError
+
+    def tfplist(self, nodelist):
+        raise WalkerError
+
+    def tname(self, nodelist):
         raise WalkerError
 
     def dotted_name(self, nodelist):
@@ -786,9 +806,10 @@ class Transformer:
         return Discard(Const(None))
 
     def keywordonlyargs(self, nodelist):
-        # (',' NAME ['=' test])*
+        # (',' tname ['=' test])*
         #      ^^^
         # ------+
+        # tname and vname are handled.
         kwonlyargs = []
         i = 0
         while i < len(nodelist):
@@ -802,10 +823,25 @@ class Transformer:
                 i += 2
             if node[0] == token.DOUBLESTAR:
                 return kwonlyargs, i
-            elif node[0] == token.NAME:
-                kwonlyargs.append(Keyword(node[1], default, lineno=node[2]))
+            elif node[0] in (symbol.vname, symbol.tname):
+                lineno = extractLineNo(node)
+                kwarg = Kwarg(self._simplearg(node), default, lineno=lineno)
+                kwonlyargs.append(kwarg)
                 i += 2
         return kwonlyargs, i
+
+    def _simplearg(self, node):
+        # tname: NAME [':' test]
+        # vname: NAME
+        assert node[0] == symbol.vname or node[0] == symbol.tname
+        name = node[1][1]
+        lineno = node[1][2]
+        assert isinstance(name, str)
+        if len(node) > 2:
+            annotation = self.com_node(node[3])
+        else:
+            annotation = None
+        return SimpleArg(name, annotation, lineno)
         
     def com_arglist(self, nodelist):
         # varargslist:
@@ -814,7 +850,7 @@ class Transformer:
         #      | fpdef ['=' test] (',' fpdef ['=' test])* [',']
         # fpdef: NAME | '(' fplist ')'
         # fplist: fpdef (',' fpdef)* [',']
-        names = []
+        arguments = []
         kwonlyargs = []
         defaults = []
         flags = 0
@@ -825,14 +861,15 @@ class Transformer:
             if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
                 if node[0] == token.STAR:
                     node = nodelist[i+1]
-                    if node[0] == token.NAME: # vararg
-                        names.append(node[1])
+                    if node[0] in (symbol.tname, symbol.vname): # vararg
+                        arguments.append(self._simplearg(node))
                         flags = flags | CO_VARARGS
                         i = i + 3
                     else: # no vararg
                         assert node[0] == token.COMMA
                         i += 2
-                    if i < len(nodelist) and nodelist[i][0] == token.NAME:
+                    if i < len(nodelist) and \
+                       nodelist[i][0] in (symbol.tname, symbol.vname):
                         kwonlyargs, skip = self.keywordonlyargs(nodelist[i:])
                         i += skip
 
@@ -843,13 +880,13 @@ class Transformer:
                         node = nodelist[i+1]
                     else:
                         raise ValueError, "unexpected token: %s" % t
-                    names.append(node[1])
+                    arguments.append(self._simplearg(node))
                     flags = flags | CO_VARKEYWORDS
 
                 break
 
-            # fpdef: NAME | '(' fplist ')'
-            names.append(self.com_fpdef(node))
+            # tfpdef: tname | '(' tfplist ')'
+            arguments.append(self.com_tfpdef(node))
 
             i = i + 1
             if i < len(nodelist) and nodelist[i][0] == token.EQUAL:
@@ -863,21 +900,24 @@ class Transformer:
             # skip the comma
             i = i + 1
 
-        return names, defaults, kwonlyargs, flags
+        return arguments, defaults, kwonlyargs, flags
 
-    def com_fpdef(self, node):
-        # fpdef: NAME | '(' fplist ')'
+    def com_tfpdef(self, node):
+        # tfpdef: tname | '(' tfplist ')'
+        # def f((x)): -- x is not nested
+        while node[1][0] == token.LPAR and len(node[2]) == 2:
+            node = node[2][1]
         if node[1][0] == token.LPAR:
-            return self.com_fplist(node[2])
-        return node[1][1]
+            return NestedArgs(self.com_tfplist(node[2]))
+        return self._simplearg(node[1])
 
-    def com_fplist(self, node):
-        # fplist: fpdef (',' fpdef)* [',']
+    def com_tfplist(self, node):
+        # tfplist: tfpdef (',' tfpdef)* [',']
         if len(node) == 2:
-            return self.com_fpdef(node[1])
+            return self.com_tfpdef(node[1]),
         list = []
         for i in range(1, len(node), 2):
-            list.append(self.com_fpdef(node[i]))
+            list.append(self.com_tfpdef(node[i]))
         return tuple(list)
 
     def com_dotted_name(self, node):
diff --git a/Lib/symbol.py b/Lib/symbol.py
index a7f7a85..658974c 100755
--- a/Lib/symbol.py
+++ b/Lib/symbol.py
@@ -17,82 +17,87 @@ decorator = 259
 decorators = 260
 funcdef = 261
 parameters = 262
-varargslist = 263
-fpdef = 264
-fplist = 265
-stmt = 266
-simple_stmt = 267
-small_stmt = 268
-expr_stmt = 269
-augassign = 270
-print_stmt = 271
-del_stmt = 272
-pass_stmt = 273
-flow_stmt = 274
-break_stmt = 275
-continue_stmt = 276
-return_stmt = 277
-yield_stmt = 278
-raise_stmt = 279
-import_stmt = 280
-import_name = 281
-import_from = 282
-import_as_name = 283
-dotted_as_name = 284
-import_as_names = 285
-dotted_as_names = 286
-dotted_name = 287
-global_stmt = 288
-assert_stmt = 289
-compound_stmt = 290
-if_stmt = 291
-while_stmt = 292
-for_stmt = 293
-try_stmt = 294
-with_stmt = 295
-with_var = 296
-except_clause = 297
-suite = 298
-testlist_safe = 299
-old_test = 300
-old_lambdef = 301
-test = 302
-or_test = 303
-and_test = 304
-not_test = 305
-comparison = 306
-comp_op = 307
-expr = 308
-xor_expr = 309
-and_expr = 310
-shift_expr = 311
-arith_expr = 312
-term = 313
-factor = 314
-power = 315
-atom = 316
-listmaker = 317
-testlist_gexp = 318
-lambdef = 319
-trailer = 320
-subscriptlist = 321
-subscript = 322
-sliceop = 323
-exprlist = 324
-testlist = 325
-dictsetmaker = 326
-classdef = 327
-arglist = 328
-argument = 329
-list_iter = 330
-list_for = 331
-list_if = 332
-gen_iter = 333
-gen_for = 334
-gen_if = 335
-testlist1 = 336
-encoding_decl = 337
-yield_expr = 338
+typedargslist = 263
+tname = 264
+tfpdef = 265
+tfplist = 266
+varargslist = 267
+vname = 268
+vfpdef = 269
+vfplist = 270
+stmt = 271
+simple_stmt = 272
+small_stmt = 273
+expr_stmt = 274
+augassign = 275
+print_stmt = 276
+del_stmt = 277
+pass_stmt = 278
+flow_stmt = 279
+break_stmt = 280
+continue_stmt = 281
+return_stmt = 282
+yield_stmt = 283
+raise_stmt = 284
+import_stmt = 285
+import_name = 286
+import_from = 287
+import_as_name = 288
+dotted_as_name = 289
+import_as_names = 290
+dotted_as_names = 291
+dotted_name = 292
+global_stmt = 293
+assert_stmt = 294
+compound_stmt = 295
+if_stmt = 296
+while_stmt = 297
+for_stmt = 298
+try_stmt = 299
+with_stmt = 300
+with_var = 301
+except_clause = 302
+suite = 303
+testlist_safe = 304
+old_test = 305
+old_lambdef = 306
+test = 307
+or_test = 308
+and_test = 309
+not_test = 310
+comparison = 311
+comp_op = 312
+expr = 313
+xor_expr = 314
+and_expr = 315
+shift_expr = 316
+arith_expr = 317
+term = 318
+factor = 319
+power = 320
+atom = 321
+listmaker = 322
+testlist_gexp = 323
+lambdef = 324
+trailer = 325
+subscriptlist = 326
+subscript = 327
+sliceop = 328
+exprlist = 329
+testlist = 330
+dictsetmaker = 331
+classdef = 332
+arglist = 333
+argument = 334
+list_iter = 335
+list_for = 336
+list_if = 337
+gen_iter = 338
+gen_for = 339
+gen_if = 340
+testlist1 = 341
+encoding_decl = 342
+yield_expr = 343
 #--end constants--
 
 sym_name = {}
diff --git a/Lib/test/output/test_tokenize b/Lib/test/output/test_tokenize
index 4a3d58c..a46824d 100644
--- a/Lib/test/output/test_tokenize
+++ b/Lib/test/output/test_tokenize
@@ -682,4 +682,20 @@ test_tokenize
 177,11-177,15:	NAME	'pass'
 177,15-177,16:	NEWLINE	'\n'
 178,0-178,1:	NL	'\n'
-179,0-179,0:	ENDMARKER	''
+179,0-179,1:	OP	'@'
+179,1-179,13:	NAME	'staticmethod'
+179,13-179,14:	NEWLINE	'\n'
+180,0-180,3:	NAME	'def'
+180,4-180,7:	NAME	'foo'
+180,7-180,8:	OP	'('
+180,8-180,9:	NAME	'x'
+180,9-180,10:	OP	':'
+180,10-180,11:	NUMBER	'1'
+180,11-180,12:	OP	')'
+180,12-180,14:	OP	'->'
+180,14-180,15:	NUMBER	'1'
+180,15-180,16:	OP	':'
+180,17-180,21:	NAME	'pass'
+180,21-180,22:	NEWLINE	'\n'
+181,0-181,1:	NL	'\n'
+182,0-182,0:	ENDMARKER	''
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 11623ec..914f1d9 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -151,9 +151,9 @@ def run_tests():
 
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], None, [], []), [('Pass', (1, 9))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]),
 ('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
 ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
 ('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
 ('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]),
@@ -180,13 +180,13 @@ eval_results = [
 ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
 ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
 ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
-('Expression', ('Lambda', (1, 0), ('arguments', [], None, [], None, [], []), ('Name', (1, 7), 'None', ('Load',)))),
+('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Name', (1, 7), 'None', ('Load',)))),
 ('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
 ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
 ('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
-('Expression', ('Num', (1, 0), 10L)),
+('Expression', ('Num', (1, 0), 10)),
 ('Expression', ('Str', (1, 0), 'string')),
 ('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
 ('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
diff --git a/Lib/test/test_compiler.py b/Lib/test/test_compiler.py
index 783a34c..2ecd093 100644
--- a/Lib/test/test_compiler.py
+++ b/Lib/test/test_compiler.py
@@ -115,6 +115,24 @@ class CompilerTest(unittest.TestCase):
         dct = {}
         exec(c, dct)
         self.assertEquals(dct.get('result'), 3)
+        c = compiler.compile('def g(a):\n'
+                             '    def f(): return a + 2\n'
+                             '    return f()\n'
+                             'result = g(1)',
+                             '<string>',
+                             'exec')
+        dct = {}
+        exec(c, dct)
+        self.assertEquals(dct.get('result'), 3)
+        c = compiler.compile('def g((a, b)):\n'
+                             '    def f(): return a + b\n'
+                             '    return f()\n'
+                             'result = g((1, 2))',
+                             '<string>',
+                             'exec')
+        dct = {}
+        exec(c, dct)
+        self.assertEquals(dct.get('result'), 3)
 
     def testGenExp(self):
         c = compiler.compile('list((i,j) for i in range(3) if i < 3'
@@ -123,6 +141,22 @@ class CompilerTest(unittest.TestCase):
                              'eval')
         self.assertEquals(eval(c), [(0, 3), (1, 3), (2, 3)])
 
+    def testFuncAnnotations(self):
+        testdata = [
+            ('def f(a: 1): pass', {'a': 1}),
+            ('''def f(a, (b:1, c:2, d), e:3=4, f=5,
+                    *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass
+             ''', {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
+                   'k': 11, 'return': 12}),
+        ]
+        for sourcecode, expected in testdata:
+            # avoid IndentationError: unexpected indent from trailing lines
+            sourcecode = sourcecode.rstrip()+'\n'
+            c = compiler.compile(sourcecode, '<string>', 'exec')
+            dct = {}
+            exec(c, dct)
+            self.assertEquals(dct['f'].func_annotations, expected)
+
 
 NOLINENO = (compiler.ast.Module, compiler.ast.Stmt, compiler.ast.Discard)
 
@@ -167,10 +201,11 @@ from math import *
 
 ###############################################################################
 
-def test_main():
+def test_main(all=False):
     global TEST_ALL
-    TEST_ALL = test.test_support.is_resource_enabled("compiler")
+    TEST_ALL = all or test.test_support.is_resource_enabled("compiler")
     test.test_support.run_unittest(CompilerTest)
 
 if __name__ == "__main__":
-    test_main()
+    import sys
+    test_main('all' in sys.argv)
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index f4a0478..34c550e 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -138,16 +138,22 @@ class GrammarTests(unittest.TestCase):
         x = eval('1, 0 or 1')
 
     def testFuncdef(self):
-        ### 'def' NAME parameters ':' suite
-        ### parameters: '(' [varargslist] ')'
-        ### varargslist: (fpdef ['=' test] ',')*
-        ###           ('*' (NAME|',' fpdef ['=' test]) [',' ('**'|'*' '*') NAME]
-        ###            | ('**'|'*' '*') NAME)
-        ###            | fpdef ['=' test] (',' fpdef ['=' test])* [',']
-        ### fpdef: NAME | '(' fplist ')'
-        ### fplist: fpdef (',' fpdef)* [',']
-        ### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
-        ### argument: [test '='] test   # Really [keyword '='] test
+        ### [decorators] 'def' NAME parameters ['->' test] ':' suite
+        ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        ### decorators: decorator+
+        ### parameters: '(' [typedargslist] ')'
+        ### typedargslist: ((tfpdef ['=' test] ',')*
+        ###                ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname)
+        ###                | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+        ### tname: NAME [':' test]
+        ### tfpdef: tname | '(' tfplist ')'
+        ### tfplist: tfpdef (',' tfpdef)* [',']
+        ### varargslist: ((vfpdef ['=' test] ',')*
+        ###              ('*' [vname] (',' vname ['=' test])*  [',' '**' vname] | '**' vname)
+        ###              | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+        ### vname: NAME
+        ### vfpdef: vname | '(' vfplist ')'
+        ### vfplist: vfpdef (',' vfpdef)* [',']
         def f1(): pass
         f1()
         f1(*())
@@ -294,6 +300,28 @@ class GrammarTests(unittest.TestCase):
         pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
         pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
 
+        # argument annotation tests
+        def f(x) -> list: pass
+        self.assertEquals(f.func_annotations, {'return': list})
+        def f(x:int): pass
+        self.assertEquals(f.func_annotations, {'x': int})
+        def f(*x:str): pass
+        self.assertEquals(f.func_annotations, {'x': str})
+        def f(**x:float): pass
+        self.assertEquals(f.func_annotations, {'x': float})
+        def f(x, y:1+2): pass
+        self.assertEquals(f.func_annotations, {'y': 3})
+        def f(a, (b:1, c:2, d)): pass
+        self.assertEquals(f.func_annotations, {'b': 1, 'c': 2})
+        def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass
+        self.assertEquals(f.func_annotations,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6})
+        def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
+              **k:11) -> 12: pass
+        self.assertEquals(f.func_annotations,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
+                           'k': 11, 'return': 12})
+
     def testLambdef(self):
         ### lambdef: 'lambda' [varargslist] ':' test
         l1 = lambda : 0
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index be6c18d..de6e888 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -219,5 +219,15 @@ def foo():
     if verbose:
         print 'finished'
 
+def test_rarrow():
+    """
+    This function exists solely to test the tokenization of the RARROW
+    operator.
+
+    >>> tokenize(iter(['->']).next)   #doctest: +NORMALIZE_WHITESPACE
+    1,0-1,2:\tOP\t'->'
+    2,0-2,0:\tENDMARKER\t''
+    """
+
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/tokenize_tests.txt b/Lib/test/tokenize_tests.txt
index 1facfc1..b1aa020 100644
--- a/Lib/test/tokenize_tests.txt
+++ b/Lib/test/tokenize_tests.txt
@@ -176,3 +176,6 @@ x = sys.modules['time'].time()
 @staticmethod
 def foo(): pass
 
+@staticmethod
+def foo(x:1)->1: pass
+
diff --git a/Lib/token.py b/Lib/token.py
index 5f8d53a..2770cfd 100755
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -60,9 +60,10 @@ DOUBLESTAREQUAL = 47
 DOUBLESLASH = 48
 DOUBLESLASHEQUAL = 49
 AT = 50
-OP = 51
-ERRORTOKEN = 52
-N_TOKENS = 53
+RARROW = 51
+OP = 52
+ERRORTOKEN = 53
+N_TOKENS = 54
 NT_OFFSET = 256
 #--end constants--
 
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1253822..152bfdb 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -78,7 +78,7 @@ String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
 # longest operators first (e.g., if = came before ==, == would get
 # recognized as two instances of =).
 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
-                 r"//=?",
+                 r"//=?", r"->",
                  r"[+\-*/%&|^=<>]=?",
                  r"~")