diff options
54 files changed, 13207 insertions, 6342 deletions
diff --git a/Include/Python-ast.h b/Include/Python-ast.h new file mode 100644 index 0000000..0859acf --- /dev/null +++ b/Include/Python-ast.h @@ -0,0 +1,418 @@ +/* File automatically generated by ../Parser/asdl_c.py */ + +#include "asdl.h" + +typedef struct _mod *mod_ty; + +typedef struct _stmt *stmt_ty; + +typedef struct _expr *expr_ty; + +typedef enum _expr_context { Load=1, Store=2, Del=3, AugLoad=4, AugStore=5, + Param=6 } expr_context_ty; + +typedef struct _slice *slice_ty; + +typedef enum _boolop { And=1, Or=2 } boolop_ty; + +typedef enum _operator { Add=1, Sub=2, Mult=3, Div=4, Mod=5, Pow=6, LShift=7, + RShift=8, BitOr=9, BitXor=10, BitAnd=11, FloorDiv=12 } + operator_ty; + +typedef enum _unaryop { Invert=1, Not=2, UAdd=3, USub=4 } unaryop_ty; + +typedef enum _cmpop { Eq=1, NotEq=2, Lt=3, LtE=4, Gt=5, GtE=6, Is=7, IsNot=8, + In=9, NotIn=10 } cmpop_ty; + +typedef struct _comprehension *comprehension_ty; + +typedef struct _excepthandler *excepthandler_ty; + +typedef struct _arguments *arguments_ty; + +typedef struct _keyword *keyword_ty; + +typedef struct _alias *alias_ty; + +struct _mod { + enum { Module_kind=1, Interactive_kind=2, Expression_kind=3, + Suite_kind=4 } kind; + union { + struct { + asdl_seq *body; + } Module; + + struct { + asdl_seq *body; + } Interactive; + + struct { + expr_ty body; + } Expression; + + struct { + asdl_seq *body; + } Suite; + + } v; +}; + +struct _stmt { + enum { FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3, + Delete_kind=4, Assign_kind=5, AugAssign_kind=6, Print_kind=7, + For_kind=8, While_kind=9, If_kind=10, Raise_kind=11, + TryExcept_kind=12, TryFinally_kind=13, Assert_kind=14, + Import_kind=15, ImportFrom_kind=16, Exec_kind=17, + Global_kind=18, Expr_kind=19, Pass_kind=20, Break_kind=21, + Continue_kind=22 } kind; + union { + struct { + identifier name; + arguments_ty args; + asdl_seq *body; + asdl_seq *decorators; + } FunctionDef; + + struct { + identifier name; + asdl_seq *bases; + asdl_seq *body; + } ClassDef; + + struct { + expr_ty value; + } Return; + + struct { + asdl_seq *targets; + } Delete; + + struct { + asdl_seq *targets; + expr_ty value; + } Assign; + + struct { + expr_ty target; + operator_ty op; + expr_ty value; + } AugAssign; + + struct { + expr_ty dest; + asdl_seq *values; + bool nl; + } Print; + + struct { + expr_ty target; + expr_ty iter; + asdl_seq *body; + asdl_seq *orelse; + } For; + + struct { + expr_ty test; + asdl_seq *body; + asdl_seq *orelse; + } While; + + struct { + expr_ty test; + asdl_seq *body; + asdl_seq *orelse; + } If; + + struct { + expr_ty type; + expr_ty inst; + expr_ty tback; + } Raise; + + struct { + asdl_seq *body; + asdl_seq *handlers; + asdl_seq *orelse; + } TryExcept; + + struct { + asdl_seq *body; + asdl_seq *finalbody; + } TryFinally; + + struct { + expr_ty test; + expr_ty msg; + } Assert; + + struct { + asdl_seq *names; + } Import; + + struct { + identifier module; + asdl_seq *names; + } ImportFrom; + + struct { + expr_ty body; + expr_ty globals; + expr_ty locals; + } Exec; + + struct { + asdl_seq *names; + } Global; + + struct { + expr_ty value; + } Expr; + + } v; + int lineno; +}; + +struct _expr { + enum { BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, + Dict_kind=5, ListComp_kind=6, GeneratorExp_kind=7, Yield_kind=8, + Compare_kind=9, Call_kind=10, Repr_kind=11, Num_kind=12, + Str_kind=13, Attribute_kind=14, Subscript_kind=15, Name_kind=16, + List_kind=17, Tuple_kind=18 } kind; + union { + struct { + boolop_ty op; + asdl_seq *values; + } BoolOp; + + struct { + expr_ty left; + operator_ty op; + expr_ty right; + } BinOp; + + struct { + unaryop_ty op; + expr_ty operand; + } UnaryOp; + + struct { + arguments_ty args; + expr_ty body; + } Lambda; + + struct { + asdl_seq *keys; + asdl_seq *values; + } Dict; + + struct { + expr_ty elt; + asdl_seq *generators; + } ListComp; + + struct { + expr_ty elt; + asdl_seq *generators; + } GeneratorExp; + + struct { + expr_ty value; + } Yield; + + struct { + expr_ty left; + asdl_seq *ops; + asdl_seq *comparators; + } Compare; + + struct { + expr_ty func; + asdl_seq *args; + asdl_seq *keywords; + expr_ty starargs; + expr_ty kwargs; + } Call; + + struct { + expr_ty value; + } Repr; + + struct { + object n; + } Num; + + struct { + string s; + } Str; + + struct { + expr_ty value; + identifier attr; + expr_context_ty ctx; + } Attribute; + + struct { + expr_ty value; + slice_ty slice; + expr_context_ty ctx; + } Subscript; + + struct { + identifier id; + expr_context_ty ctx; + } Name; + + struct { + asdl_seq *elts; + expr_context_ty ctx; + } List; + + struct { + asdl_seq *elts; + expr_context_ty ctx; + } Tuple; + + } v; + int lineno; +}; + +struct _slice { + enum { Ellipsis_kind=1, Slice_kind=2, ExtSlice_kind=3, Index_kind=4 } + kind; + union { + struct { + expr_ty lower; + expr_ty upper; + expr_ty step; + } Slice; + + struct { + asdl_seq *dims; + } ExtSlice; + + struct { + expr_ty value; + } Index; + + } v; +}; + +struct _comprehension { + expr_ty target; + expr_ty iter; + asdl_seq *ifs; +}; + +struct _excepthandler { + expr_ty type; + expr_ty name; + asdl_seq *body; +}; + +struct _arguments { + asdl_seq *args; + identifier vararg; + identifier kwarg; + asdl_seq *defaults; +}; + +struct _keyword { + identifier arg; + expr_ty value; +}; + +struct _alias { + identifier name; + identifier asname; +}; + +mod_ty Module(asdl_seq * body); +mod_ty Interactive(asdl_seq * body); +mod_ty Expression(expr_ty body); +mod_ty Suite(asdl_seq * body); +stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body, + asdl_seq * decorators, int lineno); +stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int + lineno); +stmt_ty Return(expr_ty value, int lineno); +stmt_ty Delete(asdl_seq * targets, int lineno); +stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno); +stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno); +stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno); +stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, + int lineno); +stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno); +stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno); +stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno); +stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int + lineno); +stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno); +stmt_ty Assert(expr_ty test, expr_ty msg, int lineno); +stmt_ty Import(asdl_seq * names, int lineno); +stmt_ty ImportFrom(identifier module, asdl_seq * names, int lineno); +stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno); +stmt_ty Global(asdl_seq * names, int lineno); +stmt_ty Expr(expr_ty value, int lineno); +stmt_ty Pass(int lineno); +stmt_ty Break(int lineno); +stmt_ty Continue(int lineno); +expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno); +expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno); +expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno); +expr_ty Lambda(arguments_ty args, expr_ty body, int lineno); +expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno); +expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno); +expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno); +expr_ty Yield(expr_ty value, int lineno); +expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int + lineno); +expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty + starargs, expr_ty kwargs, int lineno); +expr_ty Repr(expr_ty value, int lineno); +expr_ty Num(object n, int lineno); +expr_ty Str(string s, int lineno); +expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int + lineno); +expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int + lineno); +expr_ty Name(identifier id, expr_context_ty ctx, int lineno); +expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno); +expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno); +slice_ty Ellipsis(void); +slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step); +slice_ty ExtSlice(asdl_seq * dims); +slice_ty Index(expr_ty value); +comprehension_ty comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs); +excepthandler_ty excepthandler(expr_ty type, expr_ty name, asdl_seq * body); +arguments_ty arguments(asdl_seq * args, identifier vararg, identifier kwarg, + asdl_seq * defaults); +keyword_ty keyword(identifier arg, expr_ty value); +alias_ty alias(identifier name, identifier asname); +void free_mod(mod_ty); +void free_stmt(stmt_ty); +void free_expr(expr_ty); +void free_expr_context(expr_context_ty); +void free_slice(slice_ty); +void free_boolop(boolop_ty); +void free_operator(operator_ty); +void free_unaryop(unaryop_ty); +void free_cmpop(cmpop_ty); +void free_comprehension(comprehension_ty); +void free_excepthandler(excepthandler_ty); +void free_arguments(arguments_ty); +void free_keyword(keyword_ty); +void free_alias(alias_ty); +int marshal_write_mod(PyObject **, int *, mod_ty); +int marshal_write_stmt(PyObject **, int *, stmt_ty); +int marshal_write_expr(PyObject **, int *, expr_ty); +int marshal_write_expr_context(PyObject **, int *, expr_context_ty); +int marshal_write_slice(PyObject **, int *, slice_ty); +int marshal_write_boolop(PyObject **, int *, boolop_ty); +int marshal_write_operator(PyObject **, int *, operator_ty); +int marshal_write_unaryop(PyObject **, int *, unaryop_ty); +int marshal_write_cmpop(PyObject **, int *, cmpop_ty); +int marshal_write_comprehension(PyObject **, int *, comprehension_ty); +int marshal_write_excepthandler(PyObject **, int *, excepthandler_ty); +int marshal_write_arguments(PyObject **, int *, arguments_ty); +int marshal_write_keyword(PyObject **, int *, keyword_ty); +int marshal_write_alias(PyObject **, int *, alias_ty); diff --git a/Include/Python.h b/Include/Python.h index 2d48d2e..f941cba 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -128,8 +128,7 @@ #include "pystrtod.h" /* _Py_Mangle is defined in compile.c */ -PyAPI_FUNC(int) _Py_Mangle(char *p, char *name, \ - char *buffer, size_t maxlen); +PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name); /* PyArg_GetInt is deprecated and should not be used, use PyArg_Parse(). */ #define PyArg_GetInt(v, a) PyArg_Parse((v), "i", (a)) diff --git a/Include/asdl.h b/Include/asdl.h new file mode 100644 index 0000000..8ad46fa --- /dev/null +++ b/Include/asdl.h @@ -0,0 +1,54 @@ +#ifndef Py_ASDL_H +#define Py_ASDL_H + +typedef PyObject * identifier; +typedef PyObject * string; +typedef PyObject * object; + +typedef enum {false, true} bool; + +/* It would be nice if the code generated by asdl_c.py was completely + independent of Python, but it is a goal the requires too much work + at this stage. So, for example, I'll represent identifiers as + interned Python strings. +*/ + +/* XXX A sequence should be typed so that its use can be typechecked. */ + +/* XXX We shouldn't pay for offset when we don't need APPEND. */ + +typedef struct { + int size; + int offset; + void *elements[1]; +} asdl_seq; + +asdl_seq *asdl_seq_new(int size); +void asdl_seq_free(asdl_seq *); + +#ifdef Py_DEBUG +#define asdl_seq_GET(S, I) (S)->elements[(I)] +#define asdl_seq_SET(S, I, V) { \ + int _asdl_i = (I); \ + assert((S) && _asdl_i < (S)->size); \ + (S)->elements[_asdl_i] = (V); \ +} +#define asdl_seq_APPEND(S, V) { \ + assert((S) && (S)->offset < (S)->size); \ + (S)->elements[(S)->offset++] = (V); \ +} +#else +#define asdl_seq_GET(S, I) (S)->elements[(I)] +#define asdl_seq_SET(S, I, V) (S)->elements[I] = (V) +#define asdl_seq_APPEND(S, V) (S)->elements[(S)->offset++] = (V) +#endif +#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size) + +/* Routines to marshal the basic types. */ +int marshal_write_int(PyObject **, int *, int); +int marshal_write_bool(PyObject **, int *, bool); +int marshal_write_identifier(PyObject **, int *, identifier); +int marshal_write_string(PyObject **, int *, string); +int marshal_write_object(PyObject **, int *, object); + +#endif /* !Py_ASDL_H */ diff --git a/Include/ast.h b/Include/ast.h new file mode 100644 index 0000000..8912f38 --- /dev/null +++ b/Include/ast.h @@ -0,0 +1,13 @@ +#ifndef Py_AST_H +#define Py_AST_H +#ifdef __cplusplus +extern "C" { +#endif + +extern DL_IMPORT(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags, + const char *); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_AST_H */ diff --git a/Include/code.h b/Include/code.h new file mode 100644 index 0000000..e6478b0 --- /dev/null +++ b/Include/code.h @@ -0,0 +1,73 @@ +/* Definitions for bytecode */ + +#ifndef Py_CODE_H +#define Py_CODE_H +#ifdef __cplusplus +extern "C" { +#endif + +/* Bytecode object */ +typedef struct { + PyObject_HEAD + int co_argcount; /* #arguments, except *args */ + int co_nlocals; /* #local variables */ + int co_stacksize; /* #entries needed for evaluation stack */ + int co_flags; /* CO_..., see below */ + PyObject *co_code; /* instruction opcodes */ + PyObject *co_consts; /* list (constants used) */ + PyObject *co_names; /* list of strings (names used) */ + PyObject *co_varnames; /* tuple of strings (local variable names) */ + PyObject *co_freevars; /* tuple of strings (free variable names) */ + PyObject *co_cellvars; /* tuple of strings (cell variable names) */ + /* The rest doesn't count for hash/cmp */ + PyObject *co_filename; /* string (where it was loaded from) */ + PyObject *co_name; /* string (name, for reference) */ + int co_firstlineno; /* first source line number */ + PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */ +} PyCodeObject; + +/* Masks for co_flags above */ +#define CO_OPTIMIZED 0x0001 +#define CO_NEWLOCALS 0x0002 +#define CO_VARARGS 0x0004 +#define CO_VARKEYWORDS 0x0008 +#define CO_NESTED 0x0010 +#define CO_GENERATOR 0x0020 +/* The CO_NOFREE flag is set if there are no free or cell variables. + This information is redundant, but it allows a single flag test + to determine whether there is any extra work to be done when the + call frame it setup. +*/ +#define CO_NOFREE 0x0040 +/* XXX Temporary hack. Until generators are a permanent part of the + language, we need a way for a code object to record that generators + were *possible* when it was compiled. This is so code dynamically + compiled *by* a code object knows whether to allow yield stmts. In + effect, this passes on the "from __future__ import generators" state + in effect when the code block was compiled. */ +#define CO_GENERATOR_ALLOWED 0x1000 /* no longer used in an essential way */ +#define CO_FUTURE_DIVISION 0x2000 + +#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */ + +extern DL_IMPORT(PyTypeObject) PyCode_Type; + +#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type) +#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars)) + +/* Public interface */ +DL_IMPORT(PyCodeObject *) PyCode_New( + int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *, + PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *); + /* same as struct above */ +DL_IMPORT(int) PyCode_Addr2Line(PyCodeObject *, int); + +/* for internal use only */ +#define _PyCode_GETCODEPTR(co, pp) \ + ((*(co)->co_code->ob_type->tp_as_buffer->bf_getreadbuffer) \ + ((co)->co_code, 0, (void **)(pp))) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CODE_H */ diff --git a/Include/compile.h b/Include/compile.h index 82bf708..27a3f76 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -1,5 +1,6 @@ - -/* Definitions for bytecode */ +#ifndef Py_CODE_H +#include "code.h" +#endif #ifndef Py_COMPILE_H #define Py_COMPILE_H @@ -7,55 +8,6 @@ extern "C" { #endif -/* Bytecode object */ -typedef struct { - PyObject_HEAD - int co_argcount; /* #arguments, except *args */ - int co_nlocals; /* #local variables */ - int co_stacksize; /* #entries needed for evaluation stack */ - int co_flags; /* CO_..., see below */ - PyObject *co_code; /* instruction opcodes */ - PyObject *co_consts; /* list (constants used) */ - PyObject *co_names; /* list of strings (names used) */ - PyObject *co_varnames; /* tuple of strings (local variable names) */ - PyObject *co_freevars; /* tuple of strings (free variable names) */ - PyObject *co_cellvars; /* tuple of strings (cell variable names) */ - /* The rest doesn't count for hash/cmp */ - PyObject *co_filename; /* string (where it was loaded from) */ - PyObject *co_name; /* string (name, for reference) */ - int co_firstlineno; /* first source line number */ - PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */ -} PyCodeObject; - -/* Masks for co_flags above */ -#define CO_OPTIMIZED 0x0001 -#define CO_NEWLOCALS 0x0002 -#define CO_VARARGS 0x0004 -#define CO_VARKEYWORDS 0x0008 -#define CO_NESTED 0x0010 -#define CO_GENERATOR 0x0020 -/* The CO_NOFREE flag is set if there are no free or cell variables. - This information is redundant, but it allows a single flag test - to determine whether there is any extra work to be done when the - call frame it setup. -*/ -#define CO_NOFREE 0x0040 -/* XXX Temporary hack. Until generators are a permanent part of the - language, we need a way for a code object to record that generators - were *possible* when it was compiled. This is so code dynamically - compiled *by* a code object knows whether to allow yield stmts. In - effect, this passes on the "from __future__ import generators" state - in effect when the code block was compiled. */ -#define CO_GENERATOR_ALLOWED 0x1000 /* no longer used in an essential way */ -#define CO_FUTURE_DIVISION 0x2000 - -PyAPI_DATA(PyTypeObject) PyCode_Type; - -#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type) -#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars)) - -#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */ - /* Public interface */ struct _node; /* Declare the existence of this type */ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); @@ -68,19 +20,22 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int); /* Future feature support */ typedef struct { - int ff_found_docstring; - int ff_last_lineno; - int ff_features; + int ff_features; /* flags set by future statements */ + int ff_lineno; /* line number of last future statement */ } PyFutureFeatures; -PyAPI_FUNC(PyFutureFeatures *) PyNode_Future(struct _node *, const char *); -PyAPI_FUNC(PyCodeObject *) PyNode_CompileFlags(struct _node *, const char *, - PyCompilerFlags *); - #define FUTURE_NESTED_SCOPES "nested_scopes" #define FUTURE_GENERATORS "generators" #define FUTURE_DIVISION "division" +struct _mod; /* Declare the existence of this type */ +DL_IMPORT(PyCodeObject *) PyAST_Compile(struct _mod *, const char *, + PyCompilerFlags *); +DL_IMPORT(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *); + +#define ERR_LATE_FUTURE \ +"from __future__ imports must occur at the beginning of the file" + #ifdef __cplusplus } #endif diff --git a/Include/pyport.h b/Include/pyport.h index 2440c55..ea2091b 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -583,6 +583,7 @@ typedef struct fd_set { #ifndef INT_MAX #define INT_MAX 2147483647 +#define INT_MIN (-INT_MAX - 1) #endif #ifndef LONG_MAX diff --git a/Include/pythonrun.h b/Include/pythonrun.h index f461d13..490613e 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -29,46 +29,37 @@ PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); -PyAPI_FUNC(int) PyRun_AnyFile(FILE *, const char *); -PyAPI_FUNC(int) PyRun_AnyFileEx(FILE *, const char *, int); - -PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *); - -PyAPI_FUNC(int) PyRun_SimpleString(const char *); +PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, char *, PyCompilerFlags *); +PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, char *, int, PyCompilerFlags *); PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_SimpleFile(FILE *, const char *); -PyAPI_FUNC(int) PyRun_SimpleFileEx(FILE *, const char *, int); PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_InteractiveOne(FILE *, const char *); PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_InteractiveLoop(FILE *, const char *); PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *); -PyAPI_FUNC(struct _node *) PyParser_SimpleParseString(const char *, int); -PyAPI_FUNC(struct _node *) PyParser_SimpleParseFile(FILE *, const char *, int); -PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, int); -PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlagsFilename(const char *, - const char *, - int, - int); +PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *, + int, PyCompilerFlags *flags); +PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int, + char *, char *, + PyCompilerFlags *, int *); +#define PyParser_SimpleParseString(S, B) \ + PyParser_SimpleParseStringFlags(S, B, 0) +#define PyParser_SimpleParseFile(FP, S, B) \ + PyParser_SimpleParseFileFlags(FP, S, B, 0) +PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, + int); PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *, int, int); -PyAPI_FUNC(PyObject *) PyRun_String(const char *, int, PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyRun_File(FILE *, const char *, int, PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyRun_FileEx(FILE *, const char *, int, - PyObject *, PyObject *, int); -PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, PyObject *, - PyCompilerFlags *); -PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *, const char *, int, PyObject *, - PyObject *, PyCompilerFlags *); -PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, PyObject *, - PyObject *, int, PyCompilerFlags *); - -PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int); +PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, + PyObject *, PyCompilerFlags *); + +PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, + PyObject *, PyObject *, int, + PyCompilerFlags *); + +#define Py_CompileString(str, p, s) Py_CompileStringFlags(str, p, s, NULL) PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *, const char *, int, - PyCompilerFlags *); + PyCompilerFlags *); PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int); PyAPI_FUNC(void) PyErr_Print(void); @@ -84,6 +75,25 @@ PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *); /* Bootstrap */ PyAPI_FUNC(int) Py_Main(int argc, char **argv); +/* Use macros for a bunch of old variants */ +#define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL) +#define PyRun_AnyFile(fp, name) PyRun_AnyFileExFlags(fp, name, 0, NULL) +#define PyRun_AnyFileEx(fp, name, closeit) \ + PyRun_AnyFileExFlags(fp, name, closeit, NULL) +#define PyRun_AnyFileFlags(fp, name, flags) \ + PyRun_AnyFileExFlags(fp, name, 0, flags) +#define PyRun_SimpleString(s, f) PyRunSimpleStringFlags(s, f, NULL) +#define PyRun_SimpleFile(f, p) PyRun_SimpleFileExFlags(f, p, 0, NULL) +#define PyRun_SimpleFileEx(f, p, c) PyRun_SimpleFileExFlags(f, p, c, NULL) +#define PyRun_InteractiveOne(f, p) PyRun_InteractiveOneFlags(f, p, NULL) +#define PyRun_InteractiveLoop(f, p) PyRun_InteractiveLoopFlags(f, p, NULL) +#define PyRun_File(fp, p, s, g, l) \ + PyRun_FileExFlags(fp, p, s, g, l, 0, NULL) +#define PyRun_FileEx(fp, p, s, g, l, c) \ + PyRun_FileExFlags(fp, p, s, g, l, c, NULL) +#define PyRun_FileFlags(fp, p, s, g, l, flags) \ + PyRun_FileExFlags(fp, p, s, g, l, 0, flags) + /* In getpath.c */ PyAPI_FUNC(char *) Py_GetProgramFullPath(void); PyAPI_FUNC(char *) Py_GetPrefix(void); diff --git a/Include/symtable.h b/Include/symtable.h index 628c3e6..646602c 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -4,64 +4,59 @@ extern "C" { #endif -/* A symbol table is constructed each time PyNode_Compile() is - called. The table walks the entire parse tree and identifies each - use or definition of a variable. - - The symbol table contains a dictionary for each code block in a - module: The symbol dictionary for the block. They keys of these - dictionaries are the name of all variables used or defined in the - block; the integer values are used to store several flags, - e.g. DEF_PARAM indicates that a variable is a parameter to a - function. -*/ +typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock } + block_ty; struct _symtable_entry; struct symtable { - int st_pass; /* pass == 1 or 2 */ const char *st_filename; /* name of file being compiled */ struct _symtable_entry *st_cur; /* current symbol table entry */ + struct _symtable_entry *st_top; /* module entry */ PyObject *st_symbols; /* dictionary of symbol table entries */ PyObject *st_stack; /* stack of namespace info */ PyObject *st_global; /* borrowed ref to MODULE in st_symbols */ - int st_nscopes; /* number of scopes */ - int st_errors; /* number of errors */ + int st_nblocks; /* number of blocks */ char *st_private; /* name of current class or NULL */ + int st_tmpname; /* temporary name counter */ PyFutureFeatures *st_future; /* module's future features */ }; typedef struct _symtable_entry { PyObject_HEAD - PyObject *ste_id; /* int: key in st_symbols) */ - PyObject *ste_symbols; /* dict: name to flags) */ - PyObject *ste_name; /* string: name of scope */ + PyObject *ste_id; /* int: key in st_symbols */ + PyObject *ste_symbols; /* dict: name to flags */ + PyObject *ste_name; /* string: name of block */ PyObject *ste_varnames; /* list of variable names */ PyObject *ste_children; /* list of child ids */ - int ste_type; /* module, class, or function */ - int ste_lineno; /* first line of scope */ - int ste_optimized; /* true if namespace can't be optimized */ - int ste_nested; /* true if scope is nested */ - int ste_child_free; /* true if a child scope has free variables, + block_ty ste_type; /* module, class, or function */ + int ste_unoptimized; /* false if namespace is optimized */ + int ste_nested : 1; /* true if block is nested */ + int ste_free : 1; /* true if block has free variables */ + int ste_child_free : 1; /* true if a child block has free variables, including free refs to globals */ - int ste_generator; /* true if namespace is a generator */ + int ste_generator : 1; /* true if namespace is a generator */ + int ste_varargs : 1; /* true if block has varargs */ + int ste_varkeywords : 1; /* true if block has varkeywords */ + int ste_lineno; /* first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ - int ste_tmpname; /* temporary name counter */ + int ste_tmpname; /* counter for listcomp temp vars */ struct symtable *ste_table; -} PySymtableEntryObject; - -PyAPI_DATA(PyTypeObject) PySymtableEntry_Type; +} PySTEntryObject; -#define PySymtableEntry_Check(op) ((op)->ob_type == &PySymtableEntry_Type) +PyAPI_DATA(PyTypeObject) PySTEntry_Type; -PyAPI_FUNC(PyObject *) PySymtableEntry_New(struct symtable *, - char *, int, int); +#define PySTEntry_Check(op) ((op)->ob_type == &PySTEntry_Type) -PyAPI_FUNC(struct symtable *) PyNode_CompileSymtable(struct _node *, const char *); -PyAPI_FUNC(void) PySymtable_Free(struct symtable *); +PyAPI_FUNC(PySTEntryObject *) \ + PySTEntry_New(struct symtable *, identifier, block_ty, void *, int); +PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *); +PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *, + PyFutureFeatures *); +PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *); -#define TOP "global" +PyAPI_FUNC(void) PySymtable_Free(struct symtable *); /* Flags for def-use information */ @@ -72,16 +67,19 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *); #define DEF_STAR 2<<3 /* parameter is star arg */ #define DEF_DOUBLESTAR 2<<4 /* parameter is star-star arg */ #define DEF_INTUPLE 2<<5 /* name defined in tuple in parameters */ -#define DEF_FREE 2<<6 /* name used but not defined in nested scope */ +#define DEF_FREE 2<<6 /* name used but not defined in nested block */ #define DEF_FREE_GLOBAL 2<<7 /* free variable is actually implicit global */ #define DEF_FREE_CLASS 2<<8 /* free variable from class's method */ #define DEF_IMPORT 2<<9 /* assignment occurred via import */ #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) -#define TYPE_FUNCTION 1 -#define TYPE_CLASS 2 -#define TYPE_MODULE 3 +/* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol + table. GLOBAL is returned from PyST_GetScope() for either of them. + It is stored in ste_symbols at bits 12-14. +*/ +#define SCOPE_OFF 11 +#define SCOPE_MASK 7 #define LOCAL 1 #define GLOBAL_EXPLICIT 2 @@ -89,9 +87,14 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *); #define FREE 4 #define CELL 5 +/* The following three names are used for the ste_unoptimized bit field */ #define OPT_IMPORT_STAR 1 #define OPT_EXEC 2 #define OPT_BARE_EXEC 4 +#define OPT_TOPLEVEL 8 /* top-level names, including eval and exec */ + +#define GENERATOR 1 +#define GENERATOR_EXPRESSION 2 #define GENERATOR 1 #define GENERATOR_EXPRESSION 2 diff --git a/Lib/cgitb.py b/Lib/cgitb.py index 8d979b8..ae25cf1 100644 --- a/Lib/cgitb.py +++ b/Lib/cgitb.py @@ -22,6 +22,7 @@ The default handler displays output as HTML. """ __author__ = 'Ka-Ping Yee' + __version__ = '$Revision$' import sys diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py index 0547eeb..e1fb063 100644 --- a/Lib/compiler/pyassem.py +++ b/Lib/compiler/pyassem.py @@ -364,16 +364,15 @@ class PyFlowGraph(FlowGraph): def getCode(self): """Get a Python code object""" - if self.stage == RAW: - self.computeStackDepth() - self.flattenGraph() - if self.stage == FLAT: - self.convertArgs() - if self.stage == CONV: - self.makeByteCode() - if self.stage == DONE: - return self.newCodeObject() - raise RuntimeError, "inconsistent PyFlowGraph state" + assert self.stage == RAW + self.computeStackDepth() + self.flattenGraph() + assert self.stage == FLAT + self.convertArgs() + assert self.stage == CONV + self.makeByteCode() + assert self.stage == DONE + return self.newCodeObject() def dump(self, io=None): if io: diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 4084b7e..5d16fa5 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -36,6 +36,7 @@ Reference Manual pages. __author__ = "Ka-Ping Yee <ping@lfw.org>" __date__ = "26 February 2001" + __version__ = "$Revision$" __credits__ = """Guido van Rossum, for an excellent programming language. Tommy Burnette, the original creator of manpy. diff --git a/Lib/test/output/test_grammar b/Lib/test/output/test_grammar index 6174e7a..fed4197 100644 --- a/Lib/test/output/test_grammar +++ b/Lib/test/output/test_grammar @@ -34,6 +34,7 @@ continue + try/except ok continue + try/finally ok testing continue and break in try/except in loop return_stmt +yield_stmt raise_stmt import_name import_from diff --git a/Lib/test/output/test_profile b/Lib/test/output/test_profile index b46bb6a..e745075 100644 --- a/Lib/test/output/test_profile +++ b/Lib/test/output/test_profile @@ -7,7 +7,7 @@ test_profile 12 0.000 0.000 0.012 0.001 :0(hasattr) 8 0.000 0.000 0.000 0.000 :0(range) 1 0.000 0.000 0.000 0.000 :0(setprofile) - 1 0.000 0.000 1.000 1.000 <string>:1(?) + 1 0.000 0.000 1.000 1.000 <string>:1(<module>) 0 0.000 0.000 profile:0(profiler) 1 0.000 0.000 1.000 1.000 profile:0(testfunc()) 1 0.400 0.400 1.000 1.000 test_profile.py:23(testfunc) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py new file mode 100644 index 0000000..ff95c6a --- /dev/null +++ b/Lib/test/test_code.py @@ -0,0 +1,85 @@ +"""This module includes tests of the code object representation. + +>>> def f(x): +... def g(y): +... return x + y +... return g +... + +>>> dump(f.func_code) +name: f +argcount: 1 +names: () +varnames: ('x', 'g') +cellvars: ('x',) +freevars: () +nlocals: 2 +flags: 3 +consts: ('None', '<code object g>') + +>>> dump(f(4).func_code) +name: g +argcount: 1 +names: () +varnames: ('y',) +cellvars: () +freevars: ('x',) +nlocals: 1 +flags: 19 +consts: ('None',) + +>>> def h(x, y): +... a = x + y +... b = x - y +... c = a * b +... return c +... +>>> dump(h.func_code) +name: h +argcount: 2 +names: () +varnames: ('x', 'y', 'a', 'b', 'c') +cellvars: () +freevars: () +nlocals: 5 +flags: 67 +consts: ('None',) + +>>> def attrs(obj): +... print obj.attr1 +... print obj.attr2 +... print obj.attr3 + +>>> dump(attrs.func_code) +name: attrs +argcount: 1 +names: ('attr1', 'attr2', 'attr3') +varnames: ('obj',) +cellvars: () +freevars: () +nlocals: 1 +flags: 67 +consts: ('None',) + +""" + +def consts(t): + """Yield a doctest-safe sequence of object reprs.""" + for elt in t: + r = repr(elt) + if r.startswith("<code object"): + yield "<code object %s>" % elt.co_name + else: + yield r + +def dump(co): + """Print out a text representation of a code object.""" + for attr in ["name", "argcount", "names", "varnames", "cellvars", + "freevars", "nlocals", "flags"]: + print "%s: %s" % (attr, getattr(co, "co_" + attr)) + print "consts:", tuple(consts(co.co_consts)) + +def test_main(verbose=None): + from test.test_support import run_doctest + from test import test_code + run_doctest(test_code, verbose) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index fc4841a..9c39ee8 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -1559,11 +1559,11 @@ Run the debugger on the docstring, and then restore sys.stdin. >>> try: doctest.debug_src(s) ... finally: sys.stdin = real_stdin - > <string>(1)?() + > <string>(1)<module>() (Pdb) next 12 --Return-- - > <string>(1)?()->None + > <string>(1)<module>()->None (Pdb) print x 12 (Pdb) continue @@ -1601,7 +1601,7 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin --Return-- - > <doctest foo[1]>(1)?()->None + > <doctest foo[1]>(1)<module>()->None -> import pdb; pdb.set_trace() (Pdb) print x 42 @@ -1637,7 +1637,7 @@ def test_pdb_set_trace(): (Pdb) print y 2 (Pdb) up - > <doctest foo[1]>(1)?() + > <doctest foo[1]>(1)<module>() -> calls_set_trace() (Pdb) print x 1 @@ -1686,7 +1686,7 @@ def test_pdb_set_trace(): [EOF] (Pdb) next --Return-- - > <doctest foo[2]>(1)?()->None + > <doctest foo[2]>(1)<module>()->None -> f(3) (Pdb) list 1 -> f(3) @@ -1779,7 +1779,7 @@ def test_pdb_set_trace_nested(): (Pdb) print y 1 (Pdb) up - > <doctest foo[1]>(1)?() + > <doctest foo[1]>(1)<module>() -> calls_set_trace() (Pdb) print foo *** NameError: name 'foo' is not defined diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index 683649d..aae3518 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -7,21 +7,21 @@ from test import test_support class EOFTestCase(unittest.TestCase): def test_EOFC(self): + expect = "EOL while scanning single-quoted string (<string>, line 1)" try: eval("""'this is a test\ """) except SyntaxError, msg: - self.assertEqual(str(msg), - "EOL while scanning single-quoted string (line 1)") + self.assertEqual(str(msg), expect) else: raise test_support.TestFailed def test_EOFS(self): + expect = "EOF while scanning triple-quoted string (<string>, line 1)" try: eval("""'''this is a test""") except SyntaxError, msg: - self.assertEqual(str(msg), - "EOF while scanning triple-quoted string (line 1)") + self.assertEqual(str(msg), expect) else: raise test_support.TestFailed diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index d226043..cb7e992 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -774,7 +774,7 @@ These are fine: ... try: ... 1//0 ... except ZeroDivisionError: -... yield 666 # bad because *outer* try has finally +... yield 666 ... except: ... pass ... finally: diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index 7c6fe4a..894ce6a 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -125,13 +125,12 @@ Verify that syntax error's are raised for genexps used as lvalues >>> (y for y in (1,2)) = 10 Traceback (most recent call last): ... - SyntaxError: assign to generator expression not possible + SyntaxError: assignment to generator expression not possible (<doctest test.test_genexps.__test__.doctests[38]>, line 1) >>> (y for y in (1,2)) += 10 Traceback (most recent call last): ... - SyntaxError: augmented assign to tuple literal, yield, or generator expression not possible - + SyntaxError: augmented assignment to generator expression not possible (<doctest test.test_genexps.__test__.doctests[39]>, line 1) ########### Tests borrowed from or inspired by test_generators.py ############ diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 1b4a506..820fab5 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -8,7 +8,7 @@ # regression test, the filterwarnings() call has been added to # regrtest.py. -from test.test_support import TestFailed, verify, check_syntax +from test.test_support import TestFailed, verify, vereq, check_syntax import sys print '1. Parser' @@ -157,28 +157,31 @@ def f2(one_argument): pass def f3(two, arguments): pass def f4(two, (compound, (argument, list))): pass def f5((compound, first), two): pass -verify(f2.func_code.co_varnames == ('one_argument',)) -verify(f3.func_code.co_varnames == ('two', 'arguments')) +vereq(f2.func_code.co_varnames, ('one_argument',)) +vereq(f3.func_code.co_varnames, ('two', 'arguments')) if sys.platform.startswith('java'): - verify(f4.func_code.co_varnames == + vereq(f4.func_code.co_varnames, ('two', '(compound, (argument, list))', 'compound', 'argument', 'list',)) - verify(f5.func_code.co_varnames == + vereq(f5.func_code.co_varnames, ('(compound, first)', 'two', 'compound', 'first')) else: - verify(f4.func_code.co_varnames == ('two', '.2', 'compound', - 'argument', 'list')) - verify(f5.func_code.co_varnames == ('.0', 'two', 'compound', 'first')) + vereq(f4.func_code.co_varnames, + ('two', '.1', 'compound', 'argument', 'list')) + vereq(f5.func_code.co_varnames, + ('.0', 'two', 'compound', 'first')) def a1(one_arg,): pass def a2(two, args,): pass def v0(*rest): pass def v1(a, *rest): pass def v2(a, b, *rest): pass def v3(a, (b, c), *rest): return a, b, c, rest +# ceval unpacks the formal arguments into the first argcount names; +# thus, the names nested inside tuples must appear after these names. if sys.platform.startswith('java'): verify(v3.func_code.co_varnames == ('a', '(b, c)', 'rest', 'b', 'c')) else: - verify(v3.func_code.co_varnames == ('a', '.2', 'rest', 'b', 'c')) + vereq(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c')) verify(v3(1, (2, 3), 4) == (1, 2, 3, (4,))) def d01(a=1): pass d01() @@ -410,6 +413,10 @@ def g1(): return def g2(): return 1 g1() x = g2() +check_syntax("class foo:return 1") + +print 'yield_stmt' +check_syntax("class foo:yield 1") print 'raise_stmt' # 'raise' test [',' test] try: raise RuntimeError, 'just testing' diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py index b89f09b..72f27fa 100644 --- a/Lib/test/test_import.py +++ b/Lib/test/test_import.py @@ -192,3 +192,16 @@ def test_failing_reload(): del sys.modules[TESTFN] test_failing_reload() + +def test_import_name_binding(): + # import x.y.z binds x in the current namespace + import test as x + import test.test_support + assert x is test, x.__name__ + assert hasattr(test.test_support, "__file__") + + # import x.y.z as w binds z as w + import test.test_support as y + assert y is test.test_support, y.__name__ + +test_import_name_binding() diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index d1ace67..771fe9d 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -411,10 +411,32 @@ class IllegalSyntaxTestCase(unittest.TestCase): (0, '')) self.check_bad_tree(tree, "malformed global ast") + +class CompileTestCase(unittest.TestCase): + + # These tests are very minimal. :-( + + def test_compile_expr(self): + st = parser.expr('2 + 3') + code = parser.compilest(st) + self.assertEquals(eval(code), 5) + + def test_compile_suite(self): + st = parser.suite('x = 2; y = x + 3') + code = parser.compilest(st) + globs = {} + exec code in globs + self.assertEquals(globs['y'], 5) + + def test_compile_error(self): + st = parser.suite('1 = 3 + 4') + self.assertRaises(SyntaxError, parser.compilest, st) + def test_main(): test_support.run_unittest( RoundtripLegalSyntaxTestCase, - IllegalSyntaxTestCase + IllegalSyntaxTestCase, + CompileTestCase, ) diff --git a/Lib/test/test_repr.py b/Lib/test/test_repr.py index 4ded484..a139473 100644 --- a/Lib/test/test_repr.py +++ b/Lib/test/test_repr.py @@ -123,7 +123,7 @@ class ReprTests(unittest.TestCase): def test_lambda(self): self.failUnless(repr(lambda x: x).startswith( - "<function <lambda")) + "<function lambda")) # XXX anonymous functions? see func_repr def test_builtin_function(self): diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index bf9a658..34801bd 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -1,4 +1,4 @@ -from test.test_support import verify, TestFailed, check_syntax +from test.test_support import verify, TestFailed, check_syntax, vereq import warnings warnings.filterwarnings("ignore", r"import \*", SyntaxWarning, "<string>") @@ -13,8 +13,8 @@ def make_adder(x): inc = make_adder(1) plus10 = make_adder(10) -verify(inc(1) == 2) -verify(plus10(-2) == 8) +vereq(inc(1), 2) +vereq(plus10(-2), 8) print "2. extra nesting" @@ -28,8 +28,8 @@ def make_adder2(x): inc = make_adder2(1) plus10 = make_adder2(10) -verify(inc(1) == 2) -verify(plus10(-2) == 8) +vereq(inc(1), 2) +vereq(plus10(-2), 8) print "3. simple nesting + rebinding" @@ -42,8 +42,8 @@ def make_adder3(x): inc = make_adder3(0) plus10 = make_adder3(9) -verify(inc(1) == 2) -verify(plus10(-2) == 8) +vereq(inc(1), 2) +vereq(plus10(-2), 8) print "4. nesting with global but no free" @@ -58,10 +58,10 @@ def make_adder4(): # XXX add exta level of indirection global_x = 1 adder = make_adder4() -verify(adder(1) == 2) +vereq(adder(1), 2) global_x = 10 -verify(adder(-2) == 8) +vereq(adder(-2), 8) print "5. nesting through class" @@ -74,8 +74,8 @@ def make_adder5(x): inc = make_adder5(1) plus10 = make_adder5(10) -verify(inc(1) == 2) -verify(plus10(-2) == 8) +vereq(inc(1), 2) +vereq(plus10(-2), 8) print "6. nesting plus free ref to global" @@ -89,8 +89,8 @@ def make_adder6(x): inc = make_adder6(1) plus10 = make_adder6(10) -verify(inc(1) == 11) # there's only one global -verify(plus10(-2) == 8) +vereq(inc(1), 11) # there's only one global +vereq(plus10(-2), 8) print "7. nearest enclosing scope" @@ -103,7 +103,7 @@ def f(x): return g(2) test_func = f(10) -verify(test_func(5) == 47) +vereq(test_func(5), 47) print "8. mixed freevars and cellvars" @@ -123,7 +123,7 @@ def f(x, y, z): g = f(1, 2, 3) h = g(2, 4, 6) -verify(h() == 39) +vereq(h(), 39) print "9. free variable in method" @@ -141,9 +141,9 @@ def test(): return Test() t = test() -verify(t.test() == "var") -verify(t.method_and_var() == "method") -verify(t.actual_global() == "global") +vereq(t.test(), "var") +vereq(t.method_and_var(), "method") +vereq(t.actual_global(), "global") method_and_var = "var" class Test: @@ -158,9 +158,9 @@ class Test: return str(self) t = Test() -verify(t.test() == "var") -verify(t.method_and_var() == "method") -verify(t.actual_global() == "global") +vereq(t.test(), "var") +vereq(t.method_and_var(), "method") +vereq(t.actual_global(), "global") print "10. recursion" @@ -175,7 +175,7 @@ def f(x): else: raise ValueError, "x must be >= 0" -verify(f(6) == 720) +vereq(f(6), 720) print "11. unoptimized namespaces" @@ -252,24 +252,24 @@ print "12. lambdas" f1 = lambda x: lambda y: x + y inc = f1(1) plus10 = f1(10) -verify(inc(1) == 2) -verify(plus10(5) == 15) +vereq(inc(1), 2) +vereq(plus10(5), 15) f2 = lambda x: (lambda : lambda y: x + y)() inc = f2(1) plus10 = f2(10) -verify(inc(1) == 2) -verify(plus10(5) == 15) +vereq(inc(1), 2) +vereq(plus10(5), 15) f3 = lambda x: lambda y: global_x + y global_x = 1 inc = f3(None) -verify(inc(2) == 3) +vereq(inc(2), 3) f8 = lambda x, y, z: lambda a, b, c: lambda : z * (b + y) g = f8(1, 2, 3) h = g(2, 4, 6) -verify(h() == 18) +vereq(h(), 18) print "13. UnboundLocal" @@ -306,21 +306,21 @@ def makeReturner(*lst): return lst return returner -verify(makeReturner(1,2,3)() == (1,2,3)) +vereq(makeReturner(1,2,3)(), (1,2,3)) def makeReturner2(**kwargs): def returner(): return kwargs return returner -verify(makeReturner2(a=11)()['a'] == 11) +vereq(makeReturner2(a=11)()['a'], 11) def makeAddPair((a, b)): def addPair((c, d)): return (a + c, b + d) return addPair -verify(makeAddPair((1, 2))((100, 200)) == (101,202)) +vereq(makeAddPair((1, 2))((100, 200)), (101,202)) print "15. scope of global statements" # Examples posted by Samuele Pedroni to python-dev on 3/1/2001 @@ -337,8 +337,8 @@ def f(): return h() return i() return g() -verify(f() == 7) -verify(x == 7) +vereq(f(), 7) +vereq(x, 7) # II x = 7 @@ -352,8 +352,8 @@ def f(): return h() return i() return g() -verify(f() == 2) -verify(x == 7) +vereq(f(), 2) +vereq(x, 7) # III x = 7 @@ -368,8 +368,8 @@ def f(): return h() return i() return g() -verify(f() == 2) -verify(x == 2) +vereq(f(), 2) +vereq(x, 2) # IV x = 7 @@ -384,8 +384,25 @@ def f(): return h() return i() return g() -verify(f() == 2) -verify(x == 2) +vereq(f(), 2) +vereq(x, 2) + +# XXX what about global statements in class blocks? +# do they affect methods? + +x = 12 +class Global: + global x + x = 13 + def set(self, val): + x = val + def get(self): + return x + +g = Global() +vereq(g.get(), 13) +g.set(15) +vereq(g.get(), 13) print "16. check leaks" @@ -407,7 +424,7 @@ def f1(): for i in range(100): f1() -verify(Foo.count == 0) +vereq(Foo.count, 0) print "17. class and global" @@ -419,9 +436,9 @@ def test(x): return Foo() x = 0 -verify(test(6)(2) == 8) +vereq(test(6)(2), 8) x = -1 -verify(test(3)(2) == 5) +vereq(test(3)(2), 5) print "18. verify that locals() works" @@ -437,7 +454,7 @@ def f(x): d = f(2)(4) verify(d.has_key('h')) del d['h'] -verify(d == {'x': 2, 'y': 7, 'w': 6}) +vereq(d, {'x': 2, 'y': 7, 'w': 6}) print "19. var is bound and free in class" @@ -449,7 +466,7 @@ def f(x): return C inst = f(3)() -verify(inst.a == inst.m()) +vereq(inst.a, inst.m()) print "20. interaction with trace function" diff --git a/Makefile.pre.in b/Makefile.pre.in index 66bc88a..7a69a6b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -216,10 +216,22 @@ PGOBJS= \ PGENOBJS= $(PGENMAIN) $(POBJS) $(PGOBJS) +########################################################################## +# AST +AST_H= $(srcdir)/Include/Python-ast.h +AST_C= $(srcdir)/Python/Python-ast.c +AST_ASDL= $(srcdir)/Parser/Python.asdl + +ASDLGEN_FILES= $(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py +# XXX Note that a build now requires Python exist before the build starts +ASDLGEN= $(srcdir)/Parser/asdl_c.py -h $(srcdir)/Include -c $(srcdir)/Python ########################################################################## # Python PYTHON_OBJS= \ + Python/Python-ast.o \ + Python/asdl.o \ + Python/ast.o \ Python/bltinmodule.o \ Python/exceptions.o \ Python/ceval.o \ @@ -265,6 +277,7 @@ OBJECT_OBJS= \ Objects/cellobject.o \ Objects/classobject.o \ Objects/cobject.o \ + Objects/codeobject.o \ Objects/complexobject.o \ Objects/descrobject.o \ Objects/enumobject.o \ @@ -457,8 +470,10 @@ Parser/metagrammar.o: $(srcdir)/Parser/metagrammar.c Parser/tokenizer_pgen.o: $(srcdir)/Parser/tokenizer.c +$(AST_H) $(AST_C): $(AST_ASDL) $(ASDLGEN_FILES) + $(ASDLGEN) $(AST_ASDL) -Python/compile.o Python/symtable.o: $(GRAMMAR_H) +Python/compile.o Python/symtable.o: $(GRAMMAR_H) $(AST_H) Python/getplatform.o: $(srcdir)/Python/getplatform.c $(CC) -c $(PY_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c @@ -474,12 +489,15 @@ Objects/unicodectype.o: $(srcdir)/Objects/unicodectype.c \ PYTHON_HEADERS= \ Include/Python.h \ + Include/Python-ast.h \ + Include/asdl.h \ Include/abstract.h \ Include/boolobject.h \ Include/bufferobject.h \ Include/ceval.h \ Include/classobject.h \ Include/cobject.h \ + Include/code.h \ Include/codecs.h \ Include/compile.h \ Include/complexobject.h \ @@ -165,6 +165,7 @@ Eugene Dvurechenski Maxim Dzumanenko Hans Eckardt Grant Edwards +John Ehresman Andrew Eland Lance Ellinghaus David Ely diff --git a/Modules/_hotshot.c b/Modules/_hotshot.c index b75b1a32..0842728 100644 --- a/Modules/_hotshot.c +++ b/Modules/_hotshot.c @@ -3,6 +3,7 @@ */ #include "Python.h" +#include "code.h" #include "compile.h" #include "eval.h" #include "frameobject.h" diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 909a404..7a52aae 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -1,6 +1,8 @@ #include "Python.h" +#include "code.h" #include "compile.h" +#include "Python-ast.h" #include "symtable.h" static PyObject * @@ -64,9 +66,9 @@ init_symtable(void) PyModule_AddIntConstant(m, "DEF_IMPORT", DEF_IMPORT); PyModule_AddIntConstant(m, "DEF_BOUND", DEF_BOUND); - PyModule_AddIntConstant(m, "TYPE_FUNCTION", TYPE_FUNCTION); - PyModule_AddIntConstant(m, "TYPE_CLASS", TYPE_CLASS); - PyModule_AddIntConstant(m, "TYPE_MODULE", TYPE_MODULE); + PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock); + PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock); + PyModule_AddIntConstant(m, "TYPE_MODULE", ModuleBlock); PyModule_AddIntConstant(m, "OPT_IMPORT_STAR", OPT_IMPORT_STAR); PyModule_AddIntConstant(m, "OPT_EXEC", OPT_EXEC); diff --git a/Objects/codeobject.c b/Objects/codeobject.c new file mode 100644 index 0000000..c5ddfd5 --- /dev/null +++ b/Objects/codeobject.c @@ -0,0 +1,453 @@ +#include "Python.h" +#include "code.h" +#include "structmember.h" + +#define NAME_CHARS \ + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" + +/* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */ + +static int +all_name_chars(unsigned char *s) +{ + static char ok_name_char[256]; + static unsigned char *name_chars = (unsigned char *)NAME_CHARS; + + if (ok_name_char[*name_chars] == 0) { + unsigned char *p; + for (p = name_chars; *p; p++) + ok_name_char[*p] = 1; + } + while (*s) { + if (ok_name_char[*s++] == 0) + return 0; + } + return 1; +} + +static void +intern_strings(PyObject *tuple) +{ + int i; + + for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { + PyObject *v = PyTuple_GET_ITEM(tuple, i); + if (v == NULL || !PyString_CheckExact(v)) { + Py_FatalError("non-string found in code slot"); + } + PyString_InternInPlace(&PyTuple_GET_ITEM(tuple, i)); + } +} + + +PyCodeObject * +PyCode_New(int argcount, int nlocals, int stacksize, int flags, + PyObject *code, PyObject *consts, PyObject *names, + PyObject *varnames, PyObject *freevars, PyObject *cellvars, + PyObject *filename, PyObject *name, int firstlineno, + PyObject *lnotab) +{ + PyCodeObject *co; + int i; + /* Check argument types */ + if (argcount < 0 || nlocals < 0 || + code == NULL || + consts == NULL || !PyTuple_Check(consts) || + names == NULL || !PyTuple_Check(names) || + varnames == NULL || !PyTuple_Check(varnames) || + freevars == NULL || !PyTuple_Check(freevars) || + cellvars == NULL || !PyTuple_Check(cellvars) || + name == NULL || !PyString_Check(name) || + filename == NULL || !PyString_Check(filename) || + lnotab == NULL || !PyString_Check(lnotab) || + !PyObject_CheckReadBuffer(code)) { + PyErr_BadInternalCall(); + return NULL; + } + intern_strings(names); + intern_strings(varnames); + intern_strings(freevars); + intern_strings(cellvars); + /* Intern selected string constants */ + for (i = PyTuple_Size(consts); --i >= 0; ) { + PyObject *v = PyTuple_GetItem(consts, i); + if (!PyString_Check(v)) + continue; + if (!all_name_chars((unsigned char *)PyString_AS_STRING(v))) + continue; + PyString_InternInPlace(&PyTuple_GET_ITEM(consts, i)); + } + co = PyObject_NEW(PyCodeObject, &PyCode_Type); + if (co != NULL) { + co->co_argcount = argcount; + co->co_nlocals = nlocals; + co->co_stacksize = stacksize; + co->co_flags = flags; + Py_INCREF(code); + co->co_code = code; + Py_INCREF(consts); + co->co_consts = consts; + Py_INCREF(names); + co->co_names = names; + Py_INCREF(varnames); + co->co_varnames = varnames; + Py_INCREF(freevars); + co->co_freevars = freevars; + Py_INCREF(cellvars); + co->co_cellvars = cellvars; + Py_INCREF(filename); + co->co_filename = filename; + Py_INCREF(name); + co->co_name = name; + co->co_firstlineno = firstlineno; + Py_INCREF(lnotab); + co->co_lnotab = lnotab; + } + return co; +} + + +#define OFF(x) offsetof(PyCodeObject, x) + +static PyMemberDef code_memberlist[] = { + {"co_argcount", T_INT, OFF(co_argcount), READONLY}, + {"co_nlocals", T_INT, OFF(co_nlocals), READONLY}, + {"co_stacksize",T_INT, OFF(co_stacksize), READONLY}, + {"co_flags", T_INT, OFF(co_flags), READONLY}, + {"co_code", T_OBJECT, OFF(co_code), READONLY}, + {"co_consts", T_OBJECT, OFF(co_consts), READONLY}, + {"co_names", T_OBJECT, OFF(co_names), READONLY}, + {"co_varnames", T_OBJECT, OFF(co_varnames), READONLY}, + {"co_freevars", T_OBJECT, OFF(co_freevars), READONLY}, + {"co_cellvars", T_OBJECT, OFF(co_cellvars), READONLY}, + {"co_filename", T_OBJECT, OFF(co_filename), READONLY}, + {"co_name", T_OBJECT, OFF(co_name), READONLY}, + {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY}, + {"co_lnotab", T_OBJECT, OFF(co_lnotab), READONLY}, + {NULL} /* Sentinel */ +}; + +/* Helper for code_new: return a shallow copy of a tuple that is + guaranteed to contain exact strings, by converting string subclasses + to exact strings and complaining if a non-string is found. */ +static PyObject* +validate_and_copy_tuple(PyObject *tup) +{ + PyObject *newtuple; + PyObject *item; + int i, len; + + len = PyTuple_GET_SIZE(tup); + newtuple = PyTuple_New(len); + if (newtuple == NULL) + return NULL; + + for (i = 0; i < len; i++) { + item = PyTuple_GET_ITEM(tup, i); + if (PyString_CheckExact(item)) { + Py_INCREF(item); + } + else if (!PyString_Check(item)) { + PyErr_Format( + PyExc_TypeError, + "name tuples must contain only " + "strings, not '%.500s'", + item->ob_type->tp_name); + Py_DECREF(newtuple); + return NULL; + } + else { + item = PyString_FromStringAndSize( + PyString_AS_STRING(item), + PyString_GET_SIZE(item)); + if (item == NULL) { + Py_DECREF(newtuple); + return NULL; + } + } + PyTuple_SET_ITEM(newtuple, i, item); + } + + return newtuple; +} + +PyDoc_STRVAR(code_doc, +"code(argcount, nlocals, stacksize, flags, codestring, constants, names,\n\ + varnames, filename, name, firstlineno, lnotab[, freevars[, cellvars]])\n\ +\n\ +Create a code object. Not for the faint of heart."); + +static PyObject * +code_new(PyTypeObject *type, PyObject *args, PyObject *kw) +{ + int argcount; + int nlocals; + int stacksize; + int flags; + PyObject *co = NULL; + PyObject *code; + PyObject *consts; + PyObject *names, *ournames = NULL; + PyObject *varnames, *ourvarnames = NULL; + PyObject *freevars = NULL, *ourfreevars = NULL; + PyObject *cellvars = NULL, *ourcellvars = NULL; + PyObject *filename; + PyObject *name; + int firstlineno; + PyObject *lnotab; + + if (!PyArg_ParseTuple(args, "iiiiSO!O!O!SSiS|O!O!:code", + &argcount, &nlocals, &stacksize, &flags, + &code, + &PyTuple_Type, &consts, + &PyTuple_Type, &names, + &PyTuple_Type, &varnames, + &filename, &name, + &firstlineno, &lnotab, + &PyTuple_Type, &freevars, + &PyTuple_Type, &cellvars)) + return NULL; + + if (argcount < 0) { + PyErr_SetString( + PyExc_ValueError, + "code: argcount must not be negative"); + goto cleanup; + } + + if (nlocals < 0) { + PyErr_SetString( + PyExc_ValueError, + "code: nlocals must not be negative"); + goto cleanup; + } + + ournames = validate_and_copy_tuple(names); + if (ournames == NULL) + goto cleanup; + ourvarnames = validate_and_copy_tuple(varnames); + if (ourvarnames == NULL) + goto cleanup; + if (freevars) + ourfreevars = validate_and_copy_tuple(freevars); + else + ourfreevars = PyTuple_New(0); + if (ourfreevars == NULL) + goto cleanup; + if (cellvars) + ourcellvars = validate_and_copy_tuple(cellvars); + else + ourcellvars = PyTuple_New(0); + if (ourcellvars == NULL) + goto cleanup; + + co = (PyObject *)PyCode_New(argcount, nlocals, stacksize, flags, + code, consts, ournames, ourvarnames, + ourfreevars, ourcellvars, filename, + name, firstlineno, lnotab); + cleanup: + Py_XDECREF(ournames); + Py_XDECREF(ourvarnames); + Py_XDECREF(ourfreevars); + Py_XDECREF(ourcellvars); + return co; +} + +static void +code_dealloc(PyCodeObject *co) +{ + Py_XDECREF(co->co_code); + Py_XDECREF(co->co_consts); + Py_XDECREF(co->co_names); + Py_XDECREF(co->co_varnames); + Py_XDECREF(co->co_freevars); + Py_XDECREF(co->co_cellvars); + Py_XDECREF(co->co_filename); + Py_XDECREF(co->co_name); + Py_XDECREF(co->co_lnotab); + PyObject_DEL(co); +} + +static PyObject * +code_repr(PyCodeObject *co) +{ + char buf[500]; + int lineno = -1; + char *filename = "???"; + char *name = "???"; + + if (co->co_firstlineno != 0) + lineno = co->co_firstlineno; + if (co->co_filename && PyString_Check(co->co_filename)) + filename = PyString_AS_STRING(co->co_filename); + if (co->co_name && PyString_Check(co->co_name)) + name = PyString_AS_STRING(co->co_name); + PyOS_snprintf(buf, sizeof(buf), + "<code object %.100s at %p, file \"%.300s\", line %d>", + name, co, filename, lineno); + return PyString_FromString(buf); +} + +static int +code_compare(PyCodeObject *co, PyCodeObject *cp) +{ + int cmp; + cmp = PyObject_Compare(co->co_name, cp->co_name); + if (cmp) return cmp; + cmp = co->co_argcount - cp->co_argcount; + if (cmp) goto normalize; + cmp = co->co_nlocals - cp->co_nlocals; + if (cmp) goto normalize; + cmp = co->co_flags - cp->co_flags; + if (cmp) goto normalize; + cmp = co->co_firstlineno - cp->co_firstlineno; + if (cmp) goto normalize; + cmp = PyObject_Compare(co->co_code, cp->co_code); + if (cmp) return cmp; + cmp = PyObject_Compare(co->co_consts, cp->co_consts); + if (cmp) return cmp; + cmp = PyObject_Compare(co->co_names, cp->co_names); + if (cmp) return cmp; + cmp = PyObject_Compare(co->co_varnames, cp->co_varnames); + if (cmp) return cmp; + cmp = PyObject_Compare(co->co_freevars, cp->co_freevars); + if (cmp) return cmp; + cmp = PyObject_Compare(co->co_cellvars, cp->co_cellvars); + return cmp; + + normalize: + if (cmp > 0) + return 1; + else if (cmp < 0) + return -1; + else + return 0; +} + +static long +code_hash(PyCodeObject *co) +{ + long h, h0, h1, h2, h3, h4, h5, h6; + h0 = PyObject_Hash(co->co_name); + if (h0 == -1) return -1; + h1 = PyObject_Hash(co->co_code); + if (h1 == -1) return -1; + h2 = PyObject_Hash(co->co_consts); + if (h2 == -1) return -1; + h3 = PyObject_Hash(co->co_names); + if (h3 == -1) return -1; + h4 = PyObject_Hash(co->co_varnames); + if (h4 == -1) return -1; + h5 = PyObject_Hash(co->co_freevars); + if (h5 == -1) return -1; + h6 = PyObject_Hash(co->co_cellvars); + if (h6 == -1) return -1; + h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ h6 ^ + co->co_argcount ^ co->co_nlocals ^ co->co_flags; + if (h == -1) h = -2; + return h; +} + +/* XXX code objects need to participate in GC? */ + +PyTypeObject PyCode_Type = { + PyObject_HEAD_INIT(&PyType_Type) + 0, + "code", + sizeof(PyCodeObject), + 0, + (destructor)code_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + (cmpfunc)code_compare, /* tp_compare */ + (reprfunc)code_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + (hashfunc)code_hash, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + code_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + code_memberlist, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + code_new, /* tp_new */ +}; + +/* All about c_lnotab. + +c_lnotab is an array of unsigned bytes disguised as a Python string. In -O +mode, SET_LINENO opcodes aren't generated, and bytecode offsets are mapped +to source code line #s (when needed for tracebacks) via c_lnotab instead. +The array is conceptually a list of + (bytecode offset increment, line number increment) +pairs. The details are important and delicate, best illustrated by example: + + byte code offset source code line number + 0 1 + 6 2 + 50 7 + 350 307 + 361 308 + +The first trick is that these numbers aren't stored, only the increments +from one row to the next (this doesn't really work, but it's a start): + + 0, 1, 6, 1, 44, 5, 300, 300, 11, 1 + +The second trick is that an unsigned byte can't hold negative values, or +values larger than 255, so (a) there's a deep assumption that byte code +offsets and their corresponding line #s both increase monotonically, and (b) +if at least one column jumps by more than 255 from one row to the next, more +than one pair is written to the table. In case #b, there's no way to know +from looking at the table later how many were written. That's the delicate +part. A user of c_lnotab desiring to find the source line number +corresponding to a bytecode address A should do something like this + + lineno = addr = 0 + for addr_incr, line_incr in c_lnotab: + addr += addr_incr + if addr > A: + return lineno + lineno += line_incr + +In order for this to work, when the addr field increments by more than 255, +the line # increment in each pair generated must be 0 until the remaining addr +increment is < 256. So, in the example above, com_set_lineno should not (as +was actually done until 2.2) expand 300, 300 to 255, 255, 45, 45, but to +255, 0, 45, 255, 0, 45. +*/ + +int +PyCode_Addr2Line(PyCodeObject *co, int addrq) +{ + int size = PyString_Size(co->co_lnotab) / 2; + unsigned char *p = (unsigned char*)PyString_AsString(co->co_lnotab); + int line = co->co_firstlineno; + int addr = 0; + while (--size >= 0) { + addr += *p++; + if (addr > addrq) + break; + line += *p++; + } + return line; +} diff --git a/Objects/frameobject.c b/Objects/frameobject.c index bc8cae9..8ebf500 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -3,6 +3,7 @@ #include "Python.h" +#include "code.h" #include "compile.h" #include "frameobject.h" #include "opcode.h" diff --git a/Objects/funcobject.c b/Objects/funcobject.c index c0c91c9..85cef6f 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -2,7 +2,7 @@ /* Function object implementation */ #include "Python.h" -#include "compile.h" +#include "code.h" #include "eval.h" #include "structmember.h" @@ -144,7 +144,9 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure) Py_XINCREF(closure); } else { - PyErr_SetString(PyExc_SystemError, "non-tuple closure"); + PyErr_Format(PyExc_SystemError, + "expected tuple for closure, got '%.100s'", + closure->ob_type->tp_name); return -1; } Py_XDECREF(((PyFunctionObject *) op) -> func_closure); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 88c4a15..591c62b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1737,20 +1737,14 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) goto bad_slots; for (i = j = 0; i < nslots; i++) { char *s; - char buffer[256]; tmp = PyTuple_GET_ITEM(slots, i); s = PyString_AS_STRING(tmp); if ((add_dict && strcmp(s, "__dict__") == 0) || (add_weak && strcmp(s, "__weakref__") == 0)) continue; - if (_Py_Mangle(PyString_AS_STRING(name), - PyString_AS_STRING(tmp), - buffer, sizeof(buffer))) - { - tmp = PyString_FromString(buffer); - } else { - Py_INCREF(tmp); - } + tmp =_Py_Mangle(name, tmp); + if (!tmp) + goto bad_slots; PyTuple_SET_ITEM(newslots, j, tmp); j++; } diff --git a/PCbuild/pythoncore.vcproj b/PCbuild/pythoncore.vcproj index 945fd5c..ff9e48b 100644 --- a/PCbuild/pythoncore.vcproj +++ b/PCbuild/pythoncore.vcproj @@ -3,6 +3,7 @@ ProjectType="Visual C++" Version="7.10" Name="pythoncore" + ProjectGUID="{7AFA1F0B-A8A1-455A-A832-BF263404BBEF}" SccProjectName="pythoncore" SccLocalPath=".."> <Platforms> @@ -478,6 +479,12 @@ </FileConfiguration> </File> <File + RelativePath="..\Python\asdl.c"> + </File> + <File + RelativePath="..\Python\ast.c"> + </File> + <File RelativePath="..\Modules\audioop.c"> <FileConfiguration Name="Release|Win32"> @@ -802,10 +809,13 @@ </FileConfiguration> </File> <File + RelativePath="..\Objects\codeobject.c"> + </File> + <File RelativePath="..\Modules\collectionsmodule.c"> </File> <File - RelativePath="..\Python\compile.c"> + RelativePath="..\Objects\complexobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -832,7 +842,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\complexobject.c"> + RelativePath="..\PC\config.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -859,7 +869,7 @@ </FileConfiguration> </File> <File - RelativePath="..\PC\config.c"> + RelativePath="..\Modules\cPickle.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -886,7 +896,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\cPickle.c"> + RelativePath="..\Modules\cStringIO.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -913,7 +923,10 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\cStringIO.c"> + RelativePath="..\Modules\datetimemodule.c"> + </File> + <File + RelativePath="..\Objects\descrobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -940,10 +953,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\datetimemodule.c"> - </File> - <File - RelativePath="..\Objects\descrobject.c"> + RelativePath="..\Objects\dictobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -970,7 +980,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\dictobject.c"> + RelativePath="..\PC\dl_nt.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -997,7 +1007,7 @@ </FileConfiguration> </File> <File - RelativePath="..\PC\dl_nt.c"> + RelativePath="..\Python\dynload_win.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1024,7 +1034,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Python\dynload_win.c"> + RelativePath="..\Objects\enumobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1051,7 +1061,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\enumobject.c"> + RelativePath="..\Modules\errnomodule.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1078,7 +1088,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\errnomodule.c"> + RelativePath="..\Python\errors.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1105,7 +1115,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Python\errors.c"> + RelativePath="..\Python\exceptions.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1132,7 +1142,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Python\exceptions.c"> + RelativePath="..\Objects\fileobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1159,7 +1169,10 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\fileobject.c"> + RelativePath="..\Parser\firstsets.c"> + </File> + <File + RelativePath="..\Objects\floatobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1186,7 +1199,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\floatobject.c"> + RelativePath="..\Objects\frameobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1213,7 +1226,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\frameobject.c"> + RelativePath="..\Python\frozen.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1240,7 +1253,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Python\frozen.c"> + RelativePath="..\Objects\funcobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1267,7 +1280,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\funcobject.c"> + RelativePath="..\Modules\functionalmodule.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -1648,6 +1661,9 @@ </FileConfiguration> </File> <File + RelativePath="..\Parser\grammar.c"> + </File> + <File RelativePath="..\Parser\grammar1.c"> <FileConfiguration Name="Release|Win32"> @@ -2302,6 +2318,9 @@ </FileConfiguration> </File> <File + RelativePath="..\Python\compile.c"> + </File> + <File RelativePath="..\Parser\node.c"> <FileConfiguration Name="Release|Win32"> @@ -2437,9 +2456,6 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\parsermodule.c"> - </File> - <File RelativePath="..\Parser\parsetok.c"> <FileConfiguration Name="Release|Win32"> @@ -2467,6 +2483,9 @@ </FileConfiguration> </File> <File + RelativePath="..\Parser\pgen.c"> + </File> + <File RelativePath="..\Modules\posixmodule.c"> <FileConfiguration Name="Release|Win32"> @@ -2551,6 +2570,9 @@ RelativePath="..\Python\pystrtod.c"> </File> <File + RelativePath="..\Python\Python-ast.c"> + </File> + <File RelativePath="..\PC\python_nt.rc"> <FileConfiguration Name="Release|Win32"> @@ -2764,6 +2786,7 @@ </FileConfiguration> </File> <File +<<<<<<< pythoncore.vcproj RelativePath="..\Modules\sha256module.c"> <FileConfiguration Name="Release|Win32"> @@ -2819,6 +2842,9 @@ </File> <File RelativePath="..\Modules\signalmodule.c"> +======= + RelativePath="..\Modules\sha256module.c"> +>>>>>>> 1.26.2.3 <FileConfiguration Name="Release|Win32"> <Tool @@ -2845,7 +2871,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\sliceobject.c"> + RelativePath="..\Modules\sha512module.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -2872,7 +2898,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\stringobject.c"> + RelativePath="..\Modules\signalmodule.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -2899,7 +2925,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\stropmodule.c"> + RelativePath="..\Objects\sliceobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -2926,7 +2952,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Python\structmember.c"> + RelativePath="..\Objects\stringobject.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -2953,7 +2979,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\structmodule.c"> + RelativePath="..\Modules\stropmodule.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -2980,7 +3006,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Objects\structseq.c"> + RelativePath="..\Python\structmember.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -3007,7 +3033,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Python\symtable.c"> + RelativePath="..\Modules\structmodule.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -3034,10 +3060,34 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\symtablemodule.c"> + RelativePath="..\Objects\structseq.c"> + <FileConfiguration + Name="Release|Win32"> + <Tool + Name="VCCLCompilerTool" + Optimization="2" + AdditionalIncludeDirectories="" + PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/> + </FileConfiguration> + <FileConfiguration + Name="Debug|Win32"> + <Tool + Name="VCCLCompilerTool" + Optimization="0" + AdditionalIncludeDirectories="" + PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/> + </FileConfiguration> + <FileConfiguration + Name="ReleaseItanium|Win32"> + <Tool + Name="VCCLCompilerTool" + Optimization="2" + AdditionalIncludeDirectories="" + PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/> + </FileConfiguration> </File> <File - RelativePath="..\Python\sysmodule.c"> + RelativePath="..\Python\symtable.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -3064,7 +3114,10 @@ </FileConfiguration> </File> <File - RelativePath="..\Python\thread.c"> + RelativePath="..\Modules\symtablemodule.c"> + </File> + <File + RelativePath="..\Python\sysmodule.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -3091,7 +3144,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\threadmodule.c"> + RelativePath="..\Python\thread.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -3118,7 +3171,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Modules\timemodule.c"> + RelativePath="..\Modules\threadmodule.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -3145,7 +3198,7 @@ </FileConfiguration> </File> <File - RelativePath="..\Parser\tokenizer.c"> + RelativePath="..\Modules\timemodule.c"> <FileConfiguration Name="Release|Win32"> <Tool @@ -3172,6 +3225,9 @@ </FileConfiguration> </File> <File + RelativePath="..\Parser\tokenizer_pgen.c"> + </File> + <File RelativePath="..\Python\traceback.c"> <FileConfiguration Name="Release|Win32"> diff --git a/Parser/.cvsignore b/Parser/.cvsignore index a7dd8e6..f300915 100644 --- a/Parser/.cvsignore +++ b/Parser/.cvsignore @@ -1,3 +1,6 @@ Makefile pgen add2lib +asdl.pyc +asdl_c.pyc +spark.pyc diff --git a/Parser/Python.asdl b/Parser/Python.asdl new file mode 100644 index 0000000..0883d91 --- /dev/null +++ b/Parser/Python.asdl @@ -0,0 +1,107 @@ +-- ASDL's three builtin types are identifier, int, string + +module Python +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorators) + | ClassDef(identifier name, expr* bases, stmt* body) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- not sure if bool is allowed, can always use int + | Print(expr? dest, expr* values, bool nl) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + + -- 'type' is a bad name + | Raise(expr? type, expr? inst, expr? tback) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier module, alias* names) + + -- Doesn't capture requirement that locals must be + -- defined if globals is + -- still supports use as a function! + | Exec(expr body, expr? globals, expr? locals) + + | Global(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + attributes (int lineno) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | Dict(expr* keys, expr* values) + | ListComp(expr elt, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Repr(expr value) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr *elts, expr_context ctx) + + attributes (int lineno) + + expr_context = Load | Store | Del | AugLoad | AugStore | Param + + slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And | Or + + operator = Add | Sub | Mult | Div | Mod | Pow | LShift + | RShift | BitOr | BitXor | BitAnd | FloorDiv + + unaryop = Invert | Not | UAdd | USub + + cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + + excepthandler = (expr? type, expr? name, stmt* body) + + arguments = (expr* args, identifier? vararg, + identifier? kwarg, expr* defaults) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) +} diff --git a/Parser/asdl.py b/Parser/asdl.py new file mode 100644 index 0000000..0db4e3b --- /dev/null +++ b/Parser/asdl.py @@ -0,0 +1,393 @@ +"""An implementation of the Zephyr Abstract Syntax Definition Language. + +See http://asdl.sourceforge.net/ and +http://www.cs.princeton.edu/~danwang/Papers/dsl97/dsl97-abstract.html. + +Only supports top level module decl, not view. I'm guessing that view +is intended to support the browser and I'm not interested in the +browser. +""" + +#__metaclass__ = type + +import os +import traceback + +import spark + +class Token: + # spark seems to dispatch in the parser based on a token's + # type attribute + def __init__(self, type, lineno): + self.type = type + self.lineno = lineno + + def __str__(self): + return self.type + + def __repr__(self): + return str(self) + +class Id(Token): + def __init__(self, value, lineno): + self.type = 'Id' + self.value = value + self.lineno = lineno + + def __str__(self): + return self.value + +class ASDLSyntaxError: + + def __init__(self, lineno, token=None, msg=None): + self.lineno = lineno + self.token = token + self.msg = msg + + def __str__(self): + if self.msg is None: + return "Error at '%s', line %d" % (self.token, self.lineno) + else: + return "%s, line %d" % (self.msg, self.lineno) + +class ASDLScanner(spark.GenericScanner, object): + + def tokenize(self, input): + self.rv = [] + self.lineno = 1 + super(ASDLScanner, self).tokenize(input) + return self.rv + + def t_id(self, s): + r"[\w\.]+" + # XXX doesn't distinguish upper vs. lower, which is + # significant for ASDL. + self.rv.append(Id(s, self.lineno)) + + def t_xxx(self, s): # not sure what this production means + r"<=" + self.rv.append(Token(s, self.lineno)) + + def t_punctuation(self, s): + r"[\{\}\*\=\|\(\)\,\?\:]" + self.rv.append(Token(s, self.lineno)) + + def t_comment(self, s): + r"\-\-[^\n]*" + pass + + def t_newline(self, s): + r"\n" + self.lineno += 1 + + def t_whitespace(self, s): + r"[ \t]+" + pass + + def t_default(self, s): + r" . +" + raise ValueError, "unmatched input: %s" % `s` + +class ASDLParser(spark.GenericParser, object): + def __init__(self): + super(ASDLParser, self).__init__("module") + + def typestring(self, tok): + return tok.type + + def error(self, tok): + raise ASDLSyntaxError(tok.lineno, tok) + + def p_module_0(self, (module, name, _0, _1)): + " module ::= Id Id { } " + if module.value != "module": + raise ASDLSyntaxError(module.lineno, + msg="expected 'module', found %s" % module) + return Module(name, None) + + def p_module(self, (module, name, _0, definitions, _1)): + " module ::= Id Id { definitions } " + if module.value != "module": + raise ASDLSyntaxError(module.lineno, + msg="expected 'module', found %s" % module) + return Module(name, definitions) + + def p_definition_0(self, (definition,)): + " definitions ::= definition " + return definition + + def p_definition_1(self, (definitions, definition)): + " definitions ::= definition definitions " + return definitions + definition + + def p_definition(self, (id, _, type)): + " definition ::= Id = type " + return [Type(id, type)] + + def p_type_0(self, (product,)): + " type ::= product " + return product + + def p_type_1(self, (sum,)): + " type ::= sum " + return Sum(sum) + + def p_type_2(self, (sum, id, _0, attributes, _1)): + " type ::= sum Id ( fields ) " + if id.value != "attributes": + raise ASDLSyntaxError(id.lineno, + msg="expected attributes, found %s" % id) + return Sum(sum, attributes) + + def p_product(self, (_0, fields, _1)): + " product ::= ( fields ) " + # XXX can't I just construct things in the right order? + fields.reverse() + return Product(fields) + + def p_sum_0(self, (constructor,)): + " sum ::= constructor """ + return [constructor] + + def p_sum_1(self, (constructor, _, sum)): + " sum ::= constructor | sum " + return [constructor] + sum + + def p_sum_2(self, (constructor, _, sum)): + " sum ::= constructor | sum " + return [constructor] + sum + + def p_constructor_0(self, (id,)): + " constructor ::= Id " + return Constructor(id) + + def p_constructor_1(self, (id, _0, fields, _1)): + " constructor ::= Id ( fields ) " + # XXX can't I just construct things in the right order? + fields.reverse() + return Constructor(id, fields) + + def p_fields_0(self, (field,)): + " fields ::= field " + return [field] + + def p_fields_1(self, (field, _, fields)): + " fields ::= field , fields " + return fields + [field] + + def p_field_0(self, (type,)): + " field ::= Id " + return Field(type) + + def p_field_1(self, (type, name)): + " field ::= Id Id " + return Field(type, name) + + def p_field_2(self, (type, _, name)): + " field ::= Id * Id " + return Field(type, name, seq=1) + + def p_field_3(self, (type, _, name)): + " field ::= Id ? Id " + return Field(type, name, opt=1) + + def p_field_4(self, (type, _)): + " field ::= Id * " + return Field(type, seq=1) + + def p_field_5(self, (type, _)): + " field ::= Id ? " + return Field(type, opt=1) + +builtin_types = ("identifier", "string", "int", "bool", "object") + +# below is a collection of classes to capture the AST of an AST :-) +# not sure if any of the methods are useful yet, but I'm adding them +# piecemeal as they seem helpful + +class AST: + pass # a marker class + +class Module(AST): + def __init__(self, name, dfns): + self.name = name + self.dfns = dfns + self.types = {} # maps type name to value (from dfns) + for type in dfns: + self.types[type.name.value] = type.value + + def __repr__(self): + return "Module(%s, %s)" % (self.name, self.dfns) + +class Type(AST): + def __init__(self, name, value): + self.name = name + self.value = value + + def __repr__(self): + return "Type(%s, %s)" % (self.name, self.value) + +class Constructor(AST): + def __init__(self, name, fields=None): + self.name = name + self.fields = fields or [] + + def __repr__(self): + return "Constructor(%s, %s)" % (self.name, self.fields) + +class Field(AST): + def __init__(self, type, name=None, seq=0, opt=0): + self.type = type + self.name = name + self.seq = seq + self.opt = opt + + def __repr__(self): + if self.seq: + extra = ", seq=1" + elif self.opt: + extra = ", opt=1" + else: + extra = "" + if self.name is None: + return "Field(%s%s)" % (self.type, extra) + else: + return "Field(%s, %s%s)" % (self.type, self.name, extra) + +class Sum(AST): + def __init__(self, types, attributes=None): + self.types = types + self.attributes = attributes or [] + + def __repr__(self): + if self.attributes is None: + return "Sum(%s)" % self.types + else: + return "Sum(%s, %s)" % (self.types, self.attributes) + +class Product(AST): + def __init__(self, fields): + self.fields = fields + + def __repr__(self): + return "Product(%s)" % self.fields + +class VisitorBase(object): + + def __init__(self, skip=0): + self.cache = {} + self.skip = skip + + def visit(self, object, *args): + meth = self._dispatch(object) + if meth is None: + return + try: + meth(object, *args) + except Exception, err: + print "Error visiting", repr(object) + print err + traceback.print_exc() + # XXX hack + if hasattr(self, 'file'): + self.file.flush() + os._exit(1) + + def _dispatch(self, object): + assert isinstance(object, AST), repr(object) + klass = object.__class__ + meth = self.cache.get(klass) + if meth is None: + methname = "visit" + klass.__name__ + if self.skip: + meth = getattr(self, methname, None) + else: + meth = getattr(self, methname) + self.cache[klass] = meth + return meth + +class Check(VisitorBase): + + def __init__(self): + super(Check, self).__init__(skip=1) + self.cons = {} + self.errors = 0 + self.types = {} + + def visitModule(self, mod): + for dfn in mod.dfns: + self.visit(dfn) + + def visitType(self, type): + self.visit(type.value, str(type.name)) + + def visitSum(self, sum, name): + for t in sum.types: + self.visit(t, name) + + def visitConstructor(self, cons, name): + key = str(cons.name) + conflict = self.cons.get(key) + if conflict is None: + self.cons[key] = name + else: + print "Redefinition of constructor %s" % key + print "Defined in %s and %s" % (conflict, name) + self.errors += 1 + for f in cons.fields: + self.visit(f, key) + + def visitField(self, field, name): + key = str(field.type) + l = self.types.setdefault(key, []) + l.append(name) + + def visitProduct(self, prod, name): + for f in prod.fields: + self.visit(f, name) + +def check(mod): + v = Check() + v.visit(mod) + + for t in v.types: + if not mod.types.has_key(t) and not t in builtin_types: + v.errors += 1 + uses = ", ".join(v.types[t]) + print "Undefined type %s, used in %s" % (t, uses) + + return not v.errors + +def parse(file): + scanner = ASDLScanner() + parser = ASDLParser() + + buf = open(file).read() + tokens = scanner.tokenize(buf) + try: + return parser.parse(tokens) + except ASDLSyntaxError, err: + print err + lines = buf.split("\n") + print lines[err.lineno - 1] # lines starts at 0, files at 1 + +if __name__ == "__main__": + import glob + import sys + + if len(sys.argv) > 1: + files = sys.argv[1:] + else: + testdir = "tests" + files = glob.glob(testdir + "/*.asdl") + + for file in files: + print file + mod = parse(file) + print "module", mod.name + print len(mod.dfns), "definitions" + if not check(mod): + print "Check failed" + else: + for dfn in mod.dfns: + print dfn.type diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py new file mode 100755 index 0000000..7c6df4e --- /dev/null +++ b/Parser/asdl_c.py @@ -0,0 +1,621 @@ +#! /usr/bin/env python +"""Generate C code from an ASDL description.""" + +# TO DO +# handle fields that have a type but no name + +import os, sys, traceback + +import asdl + +TABSIZE = 8 +MAX_COL = 80 + +def get_c_type(name): + """Return a string for the C name of the type. + + This function special cases the default types provided by asdl: + identifier, string, int, bool. + """ + # XXX ack! need to figure out where Id is useful and where string + if isinstance(name, asdl.Id): + name = name.value + if name in asdl.builtin_types: + return name + else: + return "%s_ty" % name + +def reflow_lines(s, depth): + """Reflow the line s indented depth tabs. + + Return a sequence of lines where no line extends beyond MAX_COL + when properly indented. The first line is properly indented based + exclusively on depth * TABSIZE. All following lines -- these are + the reflowed lines generated by this function -- start at the same + column as the first character beyond the opening { in the first + line. + """ + size = MAX_COL - depth * TABSIZE + if len(s) < size: + return [s] + + lines = [] + cur = s + padding = "" + while len(cur) > size: + i = cur.rfind(' ', 0, size) + # XXX this should be fixed for real + if i == -1 and 'GeneratorExp' in cur: + i = size + 3 + assert i != -1, "Impossible line %d to reflow: %s" % (size, `s`) + lines.append(padding + cur[:i]) + if len(lines) == 1: + # find new size based on brace + j = cur.find('{', 0, i) + if j >= 0: + j += 2 # account for the brace and the space after it + size -= j + padding = " " * j + else: + j = cur.find('(', 0, i) + if j >= 0: + j += 1 # account for the paren (no space after it) + size -= j + padding = " " * j + cur = cur[i+1:] + else: + lines.append(padding + cur) + return lines + +def is_simple(sum): + """Return True if a sum is a simple. + + A sum is simple if its types have no fields, e.g. + unaryop = Invert | Not | UAdd | USub + """ + + for t in sum.types: + if t.fields: + return False + return True + +class EmitVisitor(asdl.VisitorBase): + """Visit that emits lines""" + + def __init__(self, file): + self.file = file + super(EmitVisitor, self).__init__() + + def emit(self, s, depth, reflow=1): + # XXX reflow long lines? + if reflow: + lines = reflow_lines(s, depth) + else: + lines = [s] + for line in lines: + line = (" " * TABSIZE * depth) + line + "\n" + self.file.write(line) + +class TypeDefVisitor(EmitVisitor): + def visitModule(self, mod): + for dfn in mod.dfns: + self.visit(dfn) + + def visitType(self, type, depth=0): + self.visit(type.value, type.name, depth) + + def visitSum(self, sum, name, depth): + if is_simple(sum): + self.simple_sum(sum, name, depth) + else: + self.sum_with_constructors(sum, name, depth) + + def simple_sum(self, sum, name, depth): + enum = [] + for i in range(len(sum.types)): + type = sum.types[i] + enum.append("%s=%d" % (type.name, i + 1)) + enums = ", ".join(enum) + ctype = get_c_type(name) + s = "typedef enum _%s { %s } %s;" % (name, enums, ctype) + self.emit(s, depth) + self.emit("", depth) + + def sum_with_constructors(self, sum, name, depth): + ctype = get_c_type(name) + s = "typedef struct _%(name)s *%(ctype)s;" % locals() + self.emit(s, depth) + self.emit("", depth) + + def visitProduct(self, product, name, depth): + ctype = get_c_type(name) + s = "typedef struct _%(name)s *%(ctype)s;" % locals() + self.emit(s, depth) + self.emit("", depth) + +class StructVisitor(EmitVisitor): + """Visitor to generate typdefs for AST.""" + + def visitModule(self, mod): + for dfn in mod.dfns: + self.visit(dfn) + + def visitType(self, type, depth=0): + self.visit(type.value, type.name, depth) + + def visitSum(self, sum, name, depth): + if not is_simple(sum): + self.sum_with_constructors(sum, name, depth) + + def sum_with_constructors(self, sum, name, depth): + def emit(s, depth=depth): + self.emit(s % sys._getframe(1).f_locals, depth) + enum = [] + for i in range(len(sum.types)): + type = sum.types[i] + enum.append("%s_kind=%d" % (type.name, i + 1)) + + emit("struct _%(name)s {") + emit("enum { " + ", ".join(enum) + " } kind;", depth + 1) + emit("union {", depth + 1) + for t in sum.types: + self.visit(t, depth + 2) + emit("} v;", depth + 1) + for field in sum.attributes: + # rudimentary attribute handling + type = str(field.type) + assert type in asdl.builtin_types, type + emit("%s %s;" % (type, field.name), depth + 1); + emit("};") + emit("") + + def visitConstructor(self, cons, depth): + if cons.fields: + self.emit("struct {", depth) + for f in cons.fields: + self.visit(f, depth + 1) + self.emit("} %s;" % cons.name, depth) + self.emit("", depth) + else: + # XXX not sure what I want here, nothing is probably fine + pass + + def visitField(self, field, depth): + # XXX need to lookup field.type, because it might be something + # like a builtin... + ctype = get_c_type(field.type) + name = field.name + if field.seq: + self.emit("asdl_seq *%(name)s;" % locals(), depth) + else: + self.emit("%(ctype)s %(name)s;" % locals(), depth) + + def visitProduct(self, product, name, depth): + self.emit("struct _%(name)s {" % locals(), depth) + for f in product.fields: + self.visit(f, depth + 1) + self.emit("};", depth) + self.emit("", depth) + +class PrototypeVisitor(EmitVisitor): + """Generate function prototypes for the .h file""" + + def visitModule(self, mod): + for dfn in mod.dfns: + self.visit(dfn) + + def visitType(self, type): + self.visit(type.value, type.name) + + def visitSum(self, sum, name): + if is_simple(sum): + pass # XXX + else: + for t in sum.types: + self.visit(t, name, sum.attributes) + + def get_args(self, fields): + """Return list of C argument into, one for each field. + + Argument info is 3-tuple of a C type, variable name, and flag + that is true if type can be NULL. + """ + args = [] + unnamed = {} + for f in fields: + if f.name is None: + name = f.type + c = unnamed[name] = unnamed.get(name, 0) + 1 + if c > 1: + name = "name%d" % (c - 1) + else: + name = f.name + # XXX should extend get_c_type() to handle this + if f.seq: + ctype = "asdl_seq *" + else: + ctype = get_c_type(f.type) + args.append((ctype, name, f.opt or f.seq)) + return args + + def visitConstructor(self, cons, type, attrs): + args = self.get_args(cons.fields) + attrs = self.get_args(attrs) + ctype = get_c_type(type) + self.emit_function(cons.name, ctype, args, attrs) + + def emit_function(self, name, ctype, args, attrs, union=1): + args = args + attrs + if args: + argstr = ", ".join(["%s %s" % (atype, aname) + for atype, aname, opt in args]) + else: + argstr = "void" + self.emit("%s %s(%s);" % (ctype, name, argstr), 0) + + def visitProduct(self, prod, name): + self.emit_function(name, get_c_type(name), + self.get_args(prod.fields), [], union=0) + +class FunctionVisitor(PrototypeVisitor): + """Visitor to generate constructor functions for AST.""" + + def emit_function(self, name, ctype, args, attrs, union=1): + def emit(s, depth=0, reflow=1): + self.emit(s, depth, reflow) + argstr = ", ".join(["%s %s" % (atype, aname) + for atype, aname, opt in args + attrs]) + self.emit("%s" % ctype, 0) + emit("%s(%s)" % (name, argstr)) + emit("{") + emit("%s p;" % ctype, 1) + for argtype, argname, opt in args: + # XXX hack alert: false is allowed for a bool + if not opt and not argtype == "bool": + emit("if (!%s) {" % argname, 1) + emit("PyErr_SetString(PyExc_ValueError,", 2) + msg = "field %s is required for %s" % (argname, name) + emit(' "%s");' % msg, + 2, reflow=0) + emit('return NULL;', 2) + emit('}', 1) + + emit("p = (%s)malloc(sizeof(*p));" % ctype, 1) + emit("if (!p) {", 1) + emit("PyErr_SetString(PyExc_MemoryError, \"no memory\");", 2) + emit("return NULL;", 2) + emit("}", 1) + if union: + self.emit_body_union(name, args, attrs) + else: + self.emit_body_struct(name, args, attrs) + emit("return p;", 1) + emit("}") + emit("") + + def emit_body_union(self, name, args, attrs): + def emit(s, depth=0, reflow=1): + self.emit(s, depth, reflow) + emit("p->kind = %s_kind;" % name, 1) + for argtype, argname, opt in args: + emit("p->v.%s.%s = %s;" % (name, argname, argname), 1) + for argtype, argname, opt in attrs: + emit("p->%s = %s;" % (argname, argname), 1) + + def emit_body_struct(self, name, args, attrs): + def emit(s, depth=0, reflow=1): + self.emit(s, depth, reflow) + for argtype, argname, opt in args: + emit("p->%s = %s;" % (argname, argname), 1) + assert not attrs + +class PickleVisitor(EmitVisitor): + + def visitModule(self, mod): + for dfn in mod.dfns: + self.visit(dfn) + + def visitType(self, type): + self.visit(type.value, type.name) + + def visitSum(self, sum, name): + pass + + def visitProduct(self, sum, name): + pass + + def visitConstructor(self, cons, name): + pass + + def visitField(self, sum): + pass + +class MarshalPrototypeVisitor(PickleVisitor): + + def prototype(self, sum, name): + ctype = get_c_type(name) + self.emit("int marshal_write_%s(PyObject **, int *, %s);" + % (name, ctype), 0) + + visitProduct = visitSum = prototype + +class FreePrototypeVisitor(PickleVisitor): + + def prototype(self, sum, name): + ctype = get_c_type(name) + self.emit("void free_%s(%s);" % (name, ctype), 0) + + visitProduct = visitSum = prototype + +_SPECIALIZED_SEQUENCES = ('stmt', 'expr') + +def find_sequence(fields, doing_specialization): + """Return True if any field uses a sequence.""" + for f in fields: + if f.seq: + if not doing_specialization: + return True + if str(f.type) not in _SPECIALIZED_SEQUENCES: + return True + return False + +def has_sequence(types, doing_specialization): + for t in types: + if find_sequence(t.fields, doing_specialization): + return True + return False + + +class StaticVisitor(PickleVisitor): + '''Very simple, always emit this static code''' + + CODE = '''static void +free_seq_exprs(asdl_seq *seq) +{ + int i, n; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_expr((expr_ty)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); +} + +static void +free_seq_stmts(asdl_seq *seq) +{ + int i, n; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_stmt((stmt_ty)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); +} +''' + + def visit(self, object): + self.emit(self.CODE, 0, reflow=False) + + +class FreeVisitor(PickleVisitor): + + def func_begin(self, name, has_seq): + ctype = get_c_type(name) + self.emit("void", 0) + self.emit("free_%s(%s o)" % (name, ctype), 0) + self.emit("{", 0) + if has_seq: + self.emit("int i, n;", 1) + self.emit("asdl_seq *seq;", 1) + self.emit('', 0) + self.emit('if (!o)', 1) + self.emit('return;', 2) + self.emit('', 0) + + def func_end(self): + self.emit("}", 0) + self.emit("", 0) + + def visitSum(self, sum, name): + has_seq = has_sequence(sum.types, True) + self.func_begin(name, has_seq) + if not is_simple(sum): + self.emit("switch (o->kind) {", 1) + for i in range(len(sum.types)): + t = sum.types[i] + self.visitConstructor(t, i + 1, name) + self.emit("}", 1) + self.emit("", 0) + self.emit("free(o);", 1) + self.func_end() + + def visitProduct(self, prod, name): + self.func_begin(name, find_sequence(prod.fields, True)) + for field in prod.fields: + self.visitField(field, name, 1, True) + self.emit("", 0) + self.emit("free(o);", 1) + self.func_end() + + def visitConstructor(self, cons, enum, name): + self.emit("case %s_kind:" % cons.name, 1) + for f in cons.fields: + self.visitField(f, cons.name, 2, False) + self.emit("break;", 2) + + def visitField(self, field, name, depth, product): + def emit(s, d): + self.emit(s, depth + d) + if product: + value = "o->%s" % field.name + else: + value = "o->v.%s.%s" % (name, field.name) + if field.seq: + self.emitSeq(field, value, depth, emit) + + # XXX need to know the simple types in advance, so that we + # don't call free_TYPE() for them. + + elif field.opt: + emit("if (%s) {" % value, 0) + self.free(field, value, depth + 1) + emit("}", 0) + else: + self.free(field, value, depth) + + def emitSeq(self, field, value, depth, emit): + # specialize for freeing sequences of statements and expressions + if str(field.type) in _SPECIALIZED_SEQUENCES: + c_code = "free_seq_%ss(%s);" % (field.type, value) + emit(c_code, 0) + else: + emit("seq = %s;" % value, 0) + emit("n = asdl_seq_LEN(seq);", 0) + emit("for (i = 0; i < n; i++)", 0) + self.free(field, "asdl_seq_GET(seq, i)", depth + 1) + emit("asdl_seq_free(seq);", 0) + + def free(self, field, value, depth): + if str(field.type) in ("identifier", "string", "object"): + ctype = get_c_type(field.type) + self.emit("Py_DECREF((%s)%s);" % (ctype, value), depth) + elif str(field.type) == "bool": + return + else: + ctype = get_c_type(field.type) + self.emit("free_%s((%s)%s);" % (field.type, ctype, value), depth) + + +class MarshalFunctionVisitor(PickleVisitor): + + def func_begin(self, name, has_seq): + ctype = get_c_type(name) + self.emit("int", 0) + self.emit("marshal_write_%s(PyObject **buf, int *off, %s o)" % + (name, ctype), 0) + self.emit("{", 0) + # XXX: add declaration of "int i;" properly + if has_seq or True: + self.emit("int i;", 1) # XXX only need it for sequences + + def func_end(self): + self.emit("return 1;", 1) + self.emit("}", 0) + self.emit("", 0) + + def visitSum(self, sum, name): + has_seq = has_sequence(sum.types, False) + self.func_begin(name, has_seq) + simple = is_simple(sum) + if simple: + self.emit("switch (o) {", 1) + else: + self.emit("switch (o->kind) {", 1) + for i in range(len(sum.types)): + t = sum.types[i] + self.visitConstructor(t, i + 1, name, simple) + self.emit("}", 1) + self.func_end() + + def visitProduct(self, prod, name): + self.func_begin(name, find_sequence(prod.fields, True)) + for field in prod.fields: + self.visitField(field, name, 1, 1) + self.func_end() + + def visitConstructor(self, cons, enum, name, simple): + if simple: + self.emit("case %s:" % cons.name, 1) + self.emit("marshal_write_int(buf, off, %d);" % enum, 2); + self.emit("break;", 2) + else: + self.emit("case %s_kind:" % cons.name, 1) + self.emit("marshal_write_int(buf, off, %d);" % enum, 2) + for f in cons.fields: + self.visitField(f, cons.name, 2, 0) + self.emit("break;", 2) + + def visitField(self, field, name, depth, product): + def emit(s, d): + self.emit(s, depth + d) + if product: + value = "o->%s" % field.name + else: + value = "o->v.%s.%s" % (name, field.name) + if field.seq: + emit("marshal_write_int(buf, off, asdl_seq_LEN(%s));" % value, 0) + emit("for (i = 0; i < asdl_seq_LEN(%s); i++) {" % value, 0) + emit("void *elt = asdl_seq_GET(%s, i);" % value, 1); + ctype = get_c_type(field.type); + emit("marshal_write_%s(buf, off, (%s)elt);" % (field.type, + ctype), 1) + emit("}", 0) + elif field.opt: + emit("if (%s) {" % value, 0) + emit("marshal_write_int(buf, off, 1);", 1) + emit("marshal_write_%s(buf, off, %s);" % (field.type, value), 1) + emit("}", 0) + emit("else {", 0) + emit("marshal_write_int(buf, off, 0);", 1) + emit("}", 0) + else: + emit("marshal_write_%s(buf, off, %s);" % (field.type, value), 0) + +class ChainOfVisitors: + def __init__(self, *visitors): + self.visitors = visitors + + def visit(self, object): + for v in self.visitors: + v.visit(object) + +def main(srcfile): + auto_gen_msg = '/* File automatically generated by %s */\n' % sys.argv[0] + mod = asdl.parse(srcfile) + if not asdl.check(mod): + sys.exit(1) + if INC_DIR: + p = "%s/%s-ast.h" % (INC_DIR, mod.name) + else: + p = "%s-ast.h" % mod.name + f = open(p, "wb") + print >> f, auto_gen_msg + print >> f, '#include "asdl.h"\n' + c = ChainOfVisitors(TypeDefVisitor(f), + StructVisitor(f), + PrototypeVisitor(f), + FreePrototypeVisitor(f), + MarshalPrototypeVisitor(f), + ) + c.visit(mod) + f.close() + + if SRC_DIR: + p = "%s/%s-ast.c" % (SRC_DIR, mod.name) + else: + p = "%s-ast.c" % mod.name + f = open(p, "wb") + print >> f, auto_gen_msg + print >> f, '#include "Python.h"' + print >> f, '#include "%s-ast.h"' % mod.name + print >> f + v = ChainOfVisitors(FunctionVisitor(f), + StaticVisitor(f), + FreeVisitor(f), + MarshalFunctionVisitor(f), + ) + v.visit(mod) + f.close() + +if __name__ == "__main__": + import sys + import getopt + + INC_DIR = '' + SRC_DIR = '' + opts, args = getopt.getopt(sys.argv[1:], "h:c:") + for o, v in opts: + if o == '-h': + INC_DIR = v + if o == '-c': + SRC_DIR = v + if len(args) != 1: + print "Must specify single input file" + main(args[0]) diff --git a/Parser/grammar.mak b/Parser/grammar.mak index a6f1abe..55f028f 100644 --- a/Parser/grammar.mak +++ b/Parser/grammar.mak @@ -15,7 +15,7 @@ # particular case --pragma in PC\pyconfig.h, which demands that # python23.lib get linked in). -LIBS= ..\PCbuild\python23.lib +LIBS= ..\PCbuild\python25.lib CFLAGS= /I ..\Include /I ..\PC /D MS_NO_COREDLL /D PGEN /MD diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 1d25437..11d2232 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -21,7 +21,7 @@ static void initerr(perrdetail *err_ret, const char* filename); node * PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret) { - return PyParser_ParseStringFlags(s, g, start, err_ret, 0); + return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0); } node * @@ -56,7 +56,6 @@ PyParser_ParseStringFlagsFilename(const char *s, const char *filename, return parsetok(tok, g, start, err_ret, flags); } - /* Parse input coming from a file. Return error code, print some errors. */ node * @@ -210,7 +209,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, } static void -initerr(perrdetail *err_ret, const char* filename) +initerr(perrdetail *err_ret, const char *filename) { err_ret->error = E_OK; err_ret->filename = filename; diff --git a/Parser/spark.py b/Parser/spark.py new file mode 100644 index 0000000..00d9733 --- /dev/null +++ b/Parser/spark.py @@ -0,0 +1,840 @@ +# Copyright (c) 1998-2002 John Aycock +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__version__ = 'SPARK-0.7 (pre-alpha-5)' + +import re +import sys +import string + +def _namelist(instance): + namelist, namedict, classlist = [], {}, [instance.__class__] + for c in classlist: + for b in c.__bases__: + classlist.append(b) + for name in c.__dict__.keys(): + if not namedict.has_key(name): + namelist.append(name) + namedict[name] = 1 + return namelist + +class GenericScanner: + def __init__(self, flags=0): + pattern = self.reflect() + self.re = re.compile(pattern, re.VERBOSE|flags) + + self.index2func = {} + for name, number in self.re.groupindex.items(): + self.index2func[number-1] = getattr(self, 't_' + name) + + def makeRE(self, name): + doc = getattr(self, name).__doc__ + rv = '(?P<%s>%s)' % (name[2:], doc) + return rv + + def reflect(self): + rv = [] + for name in _namelist(self): + if name[:2] == 't_' and name != 't_default': + rv.append(self.makeRE(name)) + + rv.append(self.makeRE('t_default')) + return string.join(rv, '|') + + def error(self, s, pos): + print "Lexical error at position %s" % pos + raise SystemExit + + def tokenize(self, s): + pos = 0 + n = len(s) + while pos < n: + m = self.re.match(s, pos) + if m is None: + self.error(s, pos) + + groups = m.groups() + for i in range(len(groups)): + if groups[i] and self.index2func.has_key(i): + self.index2func[i](groups[i]) + pos = m.end() + + def t_default(self, s): + r'( . | \n )+' + print "Specification error: unmatched input" + raise SystemExit + +# +# Extracted from GenericParser and made global so that [un]picking works. +# +class _State: + def __init__(self, stateno, items): + self.T, self.complete, self.items = [], [], items + self.stateno = stateno + +class GenericParser: + # + # An Earley parser, as per J. Earley, "An Efficient Context-Free + # Parsing Algorithm", CACM 13(2), pp. 94-102. Also J. C. Earley, + # "An Efficient Context-Free Parsing Algorithm", Ph.D. thesis, + # Carnegie-Mellon University, August 1968. New formulation of + # the parser according to J. Aycock, "Practical Earley Parsing + # and the SPARK Toolkit", Ph.D. thesis, University of Victoria, + # 2001, and J. Aycock and R. N. Horspool, "Practical Earley + # Parsing", unpublished paper, 2001. + # + + def __init__(self, start): + self.rules = {} + self.rule2func = {} + self.rule2name = {} + self.collectRules() + self.augment(start) + self.ruleschanged = 1 + + _NULLABLE = '\e_' + _START = 'START' + _BOF = '|-' + + # + # When pickling, take the time to generate the full state machine; + # some information is then extraneous, too. Unfortunately we + # can't save the rule2func map. + # + def __getstate__(self): + if self.ruleschanged: + # + # XXX - duplicated from parse() + # + self.computeNull() + self.newrules = {} + self.new2old = {} + self.makeNewRules() + self.ruleschanged = 0 + self.edges, self.cores = {}, {} + self.states = { 0: self.makeState0() } + self.makeState(0, self._BOF) + # + # XXX - should find a better way to do this.. + # + changes = 1 + while changes: + changes = 0 + for k, v in self.edges.items(): + if v is None: + state, sym = k + if self.states.has_key(state): + self.goto(state, sym) + changes = 1 + rv = self.__dict__.copy() + for s in self.states.values(): + del s.items + del rv['rule2func'] + del rv['nullable'] + del rv['cores'] + return rv + + def __setstate__(self, D): + self.rules = {} + self.rule2func = {} + self.rule2name = {} + self.collectRules() + start = D['rules'][self._START][0][1][1] # Blech. + self.augment(start) + D['rule2func'] = self.rule2func + D['makeSet'] = self.makeSet_fast + self.__dict__ = D + + # + # A hook for GenericASTBuilder and GenericASTMatcher. Mess + # thee not with this; nor shall thee toucheth the _preprocess + # argument to addRule. + # + def preprocess(self, rule, func): return rule, func + + def addRule(self, doc, func, _preprocess=1): + fn = func + rules = string.split(doc) + + index = [] + for i in range(len(rules)): + if rules[i] == '::=': + index.append(i-1) + index.append(len(rules)) + + for i in range(len(index)-1): + lhs = rules[index[i]] + rhs = rules[index[i]+2:index[i+1]] + rule = (lhs, tuple(rhs)) + + if _preprocess: + rule, fn = self.preprocess(rule, func) + + if self.rules.has_key(lhs): + self.rules[lhs].append(rule) + else: + self.rules[lhs] = [ rule ] + self.rule2func[rule] = fn + self.rule2name[rule] = func.__name__[2:] + self.ruleschanged = 1 + + def collectRules(self): + for name in _namelist(self): + if name[:2] == 'p_': + func = getattr(self, name) + doc = func.__doc__ + self.addRule(doc, func) + + def augment(self, start): + rule = '%s ::= %s %s' % (self._START, self._BOF, start) + self.addRule(rule, lambda args: args[1], 0) + + def computeNull(self): + self.nullable = {} + tbd = [] + + for rulelist in self.rules.values(): + lhs = rulelist[0][0] + self.nullable[lhs] = 0 + for rule in rulelist: + rhs = rule[1] + if len(rhs) == 0: + self.nullable[lhs] = 1 + continue + # + # We only need to consider rules which + # consist entirely of nonterminal symbols. + # This should be a savings on typical + # grammars. + # + for sym in rhs: + if not self.rules.has_key(sym): + break + else: + tbd.append(rule) + changes = 1 + while changes: + changes = 0 + for lhs, rhs in tbd: + if self.nullable[lhs]: + continue + for sym in rhs: + if not self.nullable[sym]: + break + else: + self.nullable[lhs] = 1 + changes = 1 + + def makeState0(self): + s0 = _State(0, []) + for rule in self.newrules[self._START]: + s0.items.append((rule, 0)) + return s0 + + def finalState(self, tokens): + # + # Yuck. + # + if len(self.newrules[self._START]) == 2 and len(tokens) == 0: + return 1 + start = self.rules[self._START][0][1][1] + return self.goto(1, start) + + def makeNewRules(self): + worklist = [] + for rulelist in self.rules.values(): + for rule in rulelist: + worklist.append((rule, 0, 1, rule)) + + for rule, i, candidate, oldrule in worklist: + lhs, rhs = rule + n = len(rhs) + while i < n: + sym = rhs[i] + if not self.rules.has_key(sym) or \ + not self.nullable[sym]: + candidate = 0 + i = i + 1 + continue + + newrhs = list(rhs) + newrhs[i] = self._NULLABLE+sym + newrule = (lhs, tuple(newrhs)) + worklist.append((newrule, i+1, + candidate, oldrule)) + candidate = 0 + i = i + 1 + else: + if candidate: + lhs = self._NULLABLE+lhs + rule = (lhs, rhs) + if self.newrules.has_key(lhs): + self.newrules[lhs].append(rule) + else: + self.newrules[lhs] = [ rule ] + self.new2old[rule] = oldrule + + def typestring(self, token): + return None + + def error(self, token): + print "Syntax error at or near `%s' token" % token + raise SystemExit + + def parse(self, tokens): + sets = [ [(1,0), (2,0)] ] + self.links = {} + + if self.ruleschanged: + self.computeNull() + self.newrules = {} + self.new2old = {} + self.makeNewRules() + self.ruleschanged = 0 + self.edges, self.cores = {}, {} + self.states = { 0: self.makeState0() } + self.makeState(0, self._BOF) + + for i in xrange(len(tokens)): + sets.append([]) + + if sets[i] == []: + break + self.makeSet(tokens[i], sets, i) + else: + sets.append([]) + self.makeSet(None, sets, len(tokens)) + + #_dump(tokens, sets, self.states) + + finalitem = (self.finalState(tokens), 0) + if finalitem not in sets[-2]: + if len(tokens) > 0: + self.error(tokens[i-1]) + else: + self.error(None) + + return self.buildTree(self._START, finalitem, + tokens, len(sets)-2) + + def isnullable(self, sym): + # + # For symbols in G_e only. If we weren't supporting 1.5, + # could just use sym.startswith(). + # + return self._NULLABLE == sym[0:len(self._NULLABLE)] + + def skip(self, (lhs, rhs), pos=0): + n = len(rhs) + while pos < n: + if not self.isnullable(rhs[pos]): + break + pos = pos + 1 + return pos + + def makeState(self, state, sym): + assert sym is not None + # + # Compute \epsilon-kernel state's core and see if + # it exists already. + # + kitems = [] + for rule, pos in self.states[state].items: + lhs, rhs = rule + if rhs[pos:pos+1] == (sym,): + kitems.append((rule, self.skip(rule, pos+1))) + core = kitems + + core.sort() + tcore = tuple(core) + if self.cores.has_key(tcore): + return self.cores[tcore] + # + # Nope, doesn't exist. Compute it and the associated + # \epsilon-nonkernel state together; we'll need it right away. + # + k = self.cores[tcore] = len(self.states) + K, NK = _State(k, kitems), _State(k+1, []) + self.states[k] = K + predicted = {} + + edges = self.edges + rules = self.newrules + for X in K, NK: + worklist = X.items + for item in worklist: + rule, pos = item + lhs, rhs = rule + if pos == len(rhs): + X.complete.append(rule) + continue + + nextSym = rhs[pos] + key = (X.stateno, nextSym) + if not rules.has_key(nextSym): + if not edges.has_key(key): + edges[key] = None + X.T.append(nextSym) + else: + edges[key] = None + if not predicted.has_key(nextSym): + predicted[nextSym] = 1 + for prule in rules[nextSym]: + ppos = self.skip(prule) + new = (prule, ppos) + NK.items.append(new) + # + # Problem: we know K needs generating, but we + # don't yet know about NK. Can't commit anything + # regarding NK to self.edges until we're sure. Should + # we delay committing on both K and NK to avoid this + # hacky code? This creates other problems.. + # + if X is K: + edges = {} + + if NK.items == []: + return k + + # + # Check for \epsilon-nonkernel's core. Unfortunately we + # need to know the entire set of predicted nonterminals + # to do this without accidentally duplicating states. + # + core = predicted.keys() + core.sort() + tcore = tuple(core) + if self.cores.has_key(tcore): + self.edges[(k, None)] = self.cores[tcore] + return k + + nk = self.cores[tcore] = self.edges[(k, None)] = NK.stateno + self.edges.update(edges) + self.states[nk] = NK + return k + + def goto(self, state, sym): + key = (state, sym) + if not self.edges.has_key(key): + # + # No transitions from state on sym. + # + return None + + rv = self.edges[key] + if rv is None: + # + # Target state isn't generated yet. Remedy this. + # + rv = self.makeState(state, sym) + self.edges[key] = rv + return rv + + def gotoT(self, state, t): + return [self.goto(state, t)] + + def gotoST(self, state, st): + rv = [] + for t in self.states[state].T: + if st == t: + rv.append(self.goto(state, t)) + return rv + + def add(self, set, item, i=None, predecessor=None, causal=None): + if predecessor is None: + if item not in set: + set.append(item) + else: + key = (item, i) + if item not in set: + self.links[key] = [] + set.append(item) + self.links[key].append((predecessor, causal)) + + def makeSet(self, token, sets, i): + cur, next = sets[i], sets[i+1] + + ttype = token is not None and self.typestring(token) or None + if ttype is not None: + fn, arg = self.gotoT, ttype + else: + fn, arg = self.gotoST, token + + for item in cur: + ptr = (item, i) + state, parent = item + add = fn(state, arg) + for k in add: + if k is not None: + self.add(next, (k, parent), i+1, ptr) + nk = self.goto(k, None) + if nk is not None: + self.add(next, (nk, i+1)) + + if parent == i: + continue + + for rule in self.states[state].complete: + lhs, rhs = rule + for pitem in sets[parent]: + pstate, pparent = pitem + k = self.goto(pstate, lhs) + if k is not None: + why = (item, i, rule) + pptr = (pitem, parent) + self.add(cur, (k, pparent), + i, pptr, why) + nk = self.goto(k, None) + if nk is not None: + self.add(cur, (nk, i)) + + def makeSet_fast(self, token, sets, i): + # + # Call *only* when the entire state machine has been built! + # It relies on self.edges being filled in completely, and + # then duplicates and inlines code to boost speed at the + # cost of extreme ugliness. + # + cur, next = sets[i], sets[i+1] + ttype = token is not None and self.typestring(token) or None + + for item in cur: + ptr = (item, i) + state, parent = item + if ttype is not None: + k = self.edges.get((state, ttype), None) + if k is not None: + #self.add(next, (k, parent), i+1, ptr) + #INLINED --v + new = (k, parent) + key = (new, i+1) + if new not in next: + self.links[key] = [] + next.append(new) + self.links[key].append((ptr, None)) + #INLINED --^ + #nk = self.goto(k, None) + nk = self.edges.get((k, None), None) + if nk is not None: + #self.add(next, (nk, i+1)) + #INLINED --v + new = (nk, i+1) + if new not in next: + next.append(new) + #INLINED --^ + else: + add = self.gotoST(state, token) + for k in add: + if k is not None: + self.add(next, (k, parent), i+1, ptr) + #nk = self.goto(k, None) + nk = self.edges.get((k, None), None) + if nk is not None: + self.add(next, (nk, i+1)) + + if parent == i: + continue + + for rule in self.states[state].complete: + lhs, rhs = rule + for pitem in sets[parent]: + pstate, pparent = pitem + #k = self.goto(pstate, lhs) + k = self.edges.get((pstate, lhs), None) + if k is not None: + why = (item, i, rule) + pptr = (pitem, parent) + #self.add(cur, (k, pparent), + # i, pptr, why) + #INLINED --v + new = (k, pparent) + key = (new, i) + if new not in cur: + self.links[key] = [] + cur.append(new) + self.links[key].append((pptr, why)) + #INLINED --^ + #nk = self.goto(k, None) + nk = self.edges.get((k, None), None) + if nk is not None: + #self.add(cur, (nk, i)) + #INLINED --v + new = (nk, i) + if new not in cur: + cur.append(new) + #INLINED --^ + + def predecessor(self, key, causal): + for p, c in self.links[key]: + if c == causal: + return p + assert 0 + + def causal(self, key): + links = self.links[key] + if len(links) == 1: + return links[0][1] + choices = [] + rule2cause = {} + for p, c in links: + rule = c[2] + choices.append(rule) + rule2cause[rule] = c + return rule2cause[self.ambiguity(choices)] + + def deriveEpsilon(self, nt): + if len(self.newrules[nt]) > 1: + rule = self.ambiguity(self.newrules[nt]) + else: + rule = self.newrules[nt][0] + #print rule + + rhs = rule[1] + attr = [None] * len(rhs) + + for i in range(len(rhs)-1, -1, -1): + attr[i] = self.deriveEpsilon(rhs[i]) + return self.rule2func[self.new2old[rule]](attr) + + def buildTree(self, nt, item, tokens, k): + state, parent = item + + choices = [] + for rule in self.states[state].complete: + if rule[0] == nt: + choices.append(rule) + rule = choices[0] + if len(choices) > 1: + rule = self.ambiguity(choices) + #print rule + + rhs = rule[1] + attr = [None] * len(rhs) + + for i in range(len(rhs)-1, -1, -1): + sym = rhs[i] + if not self.newrules.has_key(sym): + if sym != self._BOF: + attr[i] = tokens[k-1] + key = (item, k) + item, k = self.predecessor(key, None) + #elif self.isnullable(sym): + elif self._NULLABLE == sym[0:len(self._NULLABLE)]: + attr[i] = self.deriveEpsilon(sym) + else: + key = (item, k) + why = self.causal(key) + attr[i] = self.buildTree(sym, why[0], + tokens, why[1]) + item, k = self.predecessor(key, why) + return self.rule2func[self.new2old[rule]](attr) + + def ambiguity(self, rules): + # + # XXX - problem here and in collectRules() if the same rule + # appears in >1 method. Also undefined results if rules + # causing the ambiguity appear in the same method. + # + sortlist = [] + name2index = {} + for i in range(len(rules)): + lhs, rhs = rule = rules[i] + name = self.rule2name[self.new2old[rule]] + sortlist.append((len(rhs), name)) + name2index[name] = i + sortlist.sort() + list = map(lambda (a,b): b, sortlist) + return rules[name2index[self.resolve(list)]] + + def resolve(self, list): + # + # Resolve ambiguity in favor of the shortest RHS. + # Since we walk the tree from the top down, this + # should effectively resolve in favor of a "shift". + # + return list[0] + +# +# GenericASTBuilder automagically constructs a concrete/abstract syntax tree +# for a given input. The extra argument is a class (not an instance!) +# which supports the "__setslice__" and "__len__" methods. +# +# XXX - silently overrides any user code in methods. +# + +class GenericASTBuilder(GenericParser): + def __init__(self, AST, start): + GenericParser.__init__(self, start) + self.AST = AST + + def preprocess(self, rule, func): + rebind = lambda lhs, self=self: \ + lambda args, lhs=lhs, self=self: \ + self.buildASTNode(args, lhs) + lhs, rhs = rule + return rule, rebind(lhs) + + def buildASTNode(self, args, lhs): + children = [] + for arg in args: + if isinstance(arg, self.AST): + children.append(arg) + else: + children.append(self.terminal(arg)) + return self.nonterminal(lhs, children) + + def terminal(self, token): return token + + def nonterminal(self, type, args): + rv = self.AST(type) + rv[:len(args)] = args + return rv + +# +# GenericASTTraversal is a Visitor pattern according to Design Patterns. For +# each node it attempts to invoke the method n_<node type>, falling +# back onto the default() method if the n_* can't be found. The preorder +# traversal also looks for an exit hook named n_<node type>_exit (no default +# routine is called if it's not found). To prematurely halt traversal +# of a subtree, call the prune() method -- this only makes sense for a +# preorder traversal. Node type is determined via the typestring() method. +# + +class GenericASTTraversalPruningException: + pass + +class GenericASTTraversal: + def __init__(self, ast): + self.ast = ast + + def typestring(self, node): + return node.type + + def prune(self): + raise GenericASTTraversalPruningException + + def preorder(self, node=None): + if node is None: + node = self.ast + + try: + name = 'n_' + self.typestring(node) + if hasattr(self, name): + func = getattr(self, name) + func(node) + else: + self.default(node) + except GenericASTTraversalPruningException: + return + + for kid in node: + self.preorder(kid) + + name = name + '_exit' + if hasattr(self, name): + func = getattr(self, name) + func(node) + + def postorder(self, node=None): + if node is None: + node = self.ast + + for kid in node: + self.postorder(kid) + + name = 'n_' + self.typestring(node) + if hasattr(self, name): + func = getattr(self, name) + func(node) + else: + self.default(node) + + + def default(self, node): + pass + +# +# GenericASTMatcher. AST nodes must have "__getitem__" and "__cmp__" +# implemented. +# +# XXX - makes assumptions about how GenericParser walks the parse tree. +# + +class GenericASTMatcher(GenericParser): + def __init__(self, start, ast): + GenericParser.__init__(self, start) + self.ast = ast + + def preprocess(self, rule, func): + rebind = lambda func, self=self: \ + lambda args, func=func, self=self: \ + self.foundMatch(args, func) + lhs, rhs = rule + rhslist = list(rhs) + rhslist.reverse() + + return (lhs, tuple(rhslist)), rebind(func) + + def foundMatch(self, args, func): + func(args[-1]) + return args[-1] + + def match_r(self, node): + self.input.insert(0, node) + children = 0 + + for child in node: + if children == 0: + self.input.insert(0, '(') + children = children + 1 + self.match_r(child) + + if children > 0: + self.input.insert(0, ')') + + def match(self, ast=None): + if ast is None: + ast = self.ast + self.input = [] + + self.match_r(ast) + self.parse(self.input) + + def resolve(self, list): + # + # Resolve ambiguity in favor of the longest RHS. + # + return list[-1] + +def _dump(tokens, sets, states): + for i in range(len(sets)): + print 'set', i + for item in sets[i]: + print '\t', item + for (lhs, rhs), pos in states[item[0]].items: + print '\t\t', lhs, '::=', + print string.join(rhs[:pos]), + print '.', + print string.join(rhs[pos:]) + if i < len(tokens): + print + print 'token', str(tokens[i]) + print diff --git a/Python/Python-ast.c b/Python/Python-ast.c new file mode 100644 index 0000000..e82b61c --- /dev/null +++ b/Python/Python-ast.c @@ -0,0 +1,2281 @@ +/* File automatically generated by ../Parser/asdl_c.py */ + +#include "Python.h" +#include "Python-ast.h" + +mod_ty +Module(asdl_seq * body) +{ + mod_ty p; + p = (mod_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Module_kind; + p->v.Module.body = body; + return p; +} + +mod_ty +Interactive(asdl_seq * body) +{ + mod_ty p; + p = (mod_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Interactive_kind; + p->v.Interactive.body = body; + return p; +} + +mod_ty +Expression(expr_ty body) +{ + mod_ty p; + if (!body) { + PyErr_SetString(PyExc_ValueError, + "field body is required for Expression"); + return NULL; + } + p = (mod_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Expression_kind; + p->v.Expression.body = body; + return p; +} + +mod_ty +Suite(asdl_seq * body) +{ + mod_ty p; + p = (mod_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Suite_kind; + p->v.Suite.body = body; + return p; +} + +stmt_ty +FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * + decorators, int lineno) +{ + stmt_ty p; + if (!name) { + PyErr_SetString(PyExc_ValueError, + "field name is required for FunctionDef"); + return NULL; + } + if (!args) { + PyErr_SetString(PyExc_ValueError, + "field args is required for FunctionDef"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = FunctionDef_kind; + p->v.FunctionDef.name = name; + p->v.FunctionDef.args = args; + p->v.FunctionDef.body = body; + p->v.FunctionDef.decorators = decorators; + p->lineno = lineno; + return p; +} + +stmt_ty +ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno) +{ + stmt_ty p; + if (!name) { + PyErr_SetString(PyExc_ValueError, + "field name is required for ClassDef"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = ClassDef_kind; + p->v.ClassDef.name = name; + p->v.ClassDef.bases = bases; + p->v.ClassDef.body = body; + p->lineno = lineno; + return p; +} + +stmt_ty +Return(expr_ty value, int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Return_kind; + p->v.Return.value = value; + p->lineno = lineno; + return p; +} + +stmt_ty +Delete(asdl_seq * targets, int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Delete_kind; + p->v.Delete.targets = targets; + p->lineno = lineno; + return p; +} + +stmt_ty +Assign(asdl_seq * targets, expr_ty value, int lineno) +{ + stmt_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Assign"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Assign_kind; + p->v.Assign.targets = targets; + p->v.Assign.value = value; + p->lineno = lineno; + return p; +} + +stmt_ty +AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno) +{ + stmt_ty p; + if (!target) { + PyErr_SetString(PyExc_ValueError, + "field target is required for AugAssign"); + return NULL; + } + if (!op) { + PyErr_SetString(PyExc_ValueError, + "field op is required for AugAssign"); + return NULL; + } + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for AugAssign"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = AugAssign_kind; + p->v.AugAssign.target = target; + p->v.AugAssign.op = op; + p->v.AugAssign.value = value; + p->lineno = lineno; + return p; +} + +stmt_ty +Print(expr_ty dest, asdl_seq * values, bool nl, int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Print_kind; + p->v.Print.dest = dest; + p->v.Print.values = values; + p->v.Print.nl = nl; + p->lineno = lineno; + return p; +} + +stmt_ty +For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int + lineno) +{ + stmt_ty p; + if (!target) { + PyErr_SetString(PyExc_ValueError, + "field target is required for For"); + return NULL; + } + if (!iter) { + PyErr_SetString(PyExc_ValueError, + "field iter is required for For"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = For_kind; + p->v.For.target = target; + p->v.For.iter = iter; + p->v.For.body = body; + p->v.For.orelse = orelse; + p->lineno = lineno; + return p; +} + +stmt_ty +While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno) +{ + stmt_ty p; + if (!test) { + PyErr_SetString(PyExc_ValueError, + "field test is required for While"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = While_kind; + p->v.While.test = test; + p->v.While.body = body; + p->v.While.orelse = orelse; + p->lineno = lineno; + return p; +} + +stmt_ty +If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno) +{ + stmt_ty p; + if (!test) { + PyErr_SetString(PyExc_ValueError, + "field test is required for If"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = If_kind; + p->v.If.test = test; + p->v.If.body = body; + p->v.If.orelse = orelse; + p->lineno = lineno; + return p; +} + +stmt_ty +Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Raise_kind; + p->v.Raise.type = type; + p->v.Raise.inst = inst; + p->v.Raise.tback = tback; + p->lineno = lineno; + return p; +} + +stmt_ty +TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = TryExcept_kind; + p->v.TryExcept.body = body; + p->v.TryExcept.handlers = handlers; + p->v.TryExcept.orelse = orelse; + p->lineno = lineno; + return p; +} + +stmt_ty +TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = TryFinally_kind; + p->v.TryFinally.body = body; + p->v.TryFinally.finalbody = finalbody; + p->lineno = lineno; + return p; +} + +stmt_ty +Assert(expr_ty test, expr_ty msg, int lineno) +{ + stmt_ty p; + if (!test) { + PyErr_SetString(PyExc_ValueError, + "field test is required for Assert"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Assert_kind; + p->v.Assert.test = test; + p->v.Assert.msg = msg; + p->lineno = lineno; + return p; +} + +stmt_ty +Import(asdl_seq * names, int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Import_kind; + p->v.Import.names = names; + p->lineno = lineno; + return p; +} + +stmt_ty +ImportFrom(identifier module, asdl_seq * names, int lineno) +{ + stmt_ty p; + if (!module) { + PyErr_SetString(PyExc_ValueError, + "field module is required for ImportFrom"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = ImportFrom_kind; + p->v.ImportFrom.module = module; + p->v.ImportFrom.names = names; + p->lineno = lineno; + return p; +} + +stmt_ty +Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno) +{ + stmt_ty p; + if (!body) { + PyErr_SetString(PyExc_ValueError, + "field body is required for Exec"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Exec_kind; + p->v.Exec.body = body; + p->v.Exec.globals = globals; + p->v.Exec.locals = locals; + p->lineno = lineno; + return p; +} + +stmt_ty +Global(asdl_seq * names, int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Global_kind; + p->v.Global.names = names; + p->lineno = lineno; + return p; +} + +stmt_ty +Expr(expr_ty value, int lineno) +{ + stmt_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Expr"); + return NULL; + } + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Expr_kind; + p->v.Expr.value = value; + p->lineno = lineno; + return p; +} + +stmt_ty +Pass(int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Pass_kind; + p->lineno = lineno; + return p; +} + +stmt_ty +Break(int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Break_kind; + p->lineno = lineno; + return p; +} + +stmt_ty +Continue(int lineno) +{ + stmt_ty p; + p = (stmt_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Continue_kind; + p->lineno = lineno; + return p; +} + +expr_ty +BoolOp(boolop_ty op, asdl_seq * values, int lineno) +{ + expr_ty p; + if (!op) { + PyErr_SetString(PyExc_ValueError, + "field op is required for BoolOp"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = BoolOp_kind; + p->v.BoolOp.op = op; + p->v.BoolOp.values = values; + p->lineno = lineno; + return p; +} + +expr_ty +BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno) +{ + expr_ty p; + if (!left) { + PyErr_SetString(PyExc_ValueError, + "field left is required for BinOp"); + return NULL; + } + if (!op) { + PyErr_SetString(PyExc_ValueError, + "field op is required for BinOp"); + return NULL; + } + if (!right) { + PyErr_SetString(PyExc_ValueError, + "field right is required for BinOp"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = BinOp_kind; + p->v.BinOp.left = left; + p->v.BinOp.op = op; + p->v.BinOp.right = right; + p->lineno = lineno; + return p; +} + +expr_ty +UnaryOp(unaryop_ty op, expr_ty operand, int lineno) +{ + expr_ty p; + if (!op) { + PyErr_SetString(PyExc_ValueError, + "field op is required for UnaryOp"); + return NULL; + } + if (!operand) { + PyErr_SetString(PyExc_ValueError, + "field operand is required for UnaryOp"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = UnaryOp_kind; + p->v.UnaryOp.op = op; + p->v.UnaryOp.operand = operand; + p->lineno = lineno; + return p; +} + +expr_ty +Lambda(arguments_ty args, expr_ty body, int lineno) +{ + expr_ty p; + if (!args) { + PyErr_SetString(PyExc_ValueError, + "field args is required for Lambda"); + return NULL; + } + if (!body) { + PyErr_SetString(PyExc_ValueError, + "field body is required for Lambda"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Lambda_kind; + p->v.Lambda.args = args; + p->v.Lambda.body = body; + p->lineno = lineno; + return p; +} + +expr_ty +Dict(asdl_seq * keys, asdl_seq * values, int lineno) +{ + expr_ty p; + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Dict_kind; + p->v.Dict.keys = keys; + p->v.Dict.values = values; + p->lineno = lineno; + return p; +} + +expr_ty +ListComp(expr_ty elt, asdl_seq * generators, int lineno) +{ + expr_ty p; + if (!elt) { + PyErr_SetString(PyExc_ValueError, + "field elt is required for ListComp"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = ListComp_kind; + p->v.ListComp.elt = elt; + p->v.ListComp.generators = generators; + p->lineno = lineno; + return p; +} + +expr_ty +GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno) +{ + expr_ty p; + if (!elt) { + PyErr_SetString(PyExc_ValueError, + "field elt is required for GeneratorExp"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = GeneratorExp_kind; + p->v.GeneratorExp.elt = elt; + p->v.GeneratorExp.generators = generators; + p->lineno = lineno; + return p; +} + +expr_ty +Yield(expr_ty value, int lineno) +{ + expr_ty p; + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Yield_kind; + p->v.Yield.value = value; + p->lineno = lineno; + return p; +} + +expr_ty +Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno) +{ + expr_ty p; + if (!left) { + PyErr_SetString(PyExc_ValueError, + "field left is required for Compare"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Compare_kind; + p->v.Compare.left = left; + p->v.Compare.ops = ops; + p->v.Compare.comparators = comparators; + p->lineno = lineno; + return p; +} + +expr_ty +Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs, + expr_ty kwargs, int lineno) +{ + expr_ty p; + if (!func) { + PyErr_SetString(PyExc_ValueError, + "field func is required for Call"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Call_kind; + p->v.Call.func = func; + p->v.Call.args = args; + p->v.Call.keywords = keywords; + p->v.Call.starargs = starargs; + p->v.Call.kwargs = kwargs; + p->lineno = lineno; + return p; +} + +expr_ty +Repr(expr_ty value, int lineno) +{ + expr_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Repr"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Repr_kind; + p->v.Repr.value = value; + p->lineno = lineno; + return p; +} + +expr_ty +Num(object n, int lineno) +{ + expr_ty p; + if (!n) { + PyErr_SetString(PyExc_ValueError, + "field n is required for Num"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Num_kind; + p->v.Num.n = n; + p->lineno = lineno; + return p; +} + +expr_ty +Str(string s, int lineno) +{ + expr_ty p; + if (!s) { + PyErr_SetString(PyExc_ValueError, + "field s is required for Str"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Str_kind; + p->v.Str.s = s; + p->lineno = lineno; + return p; +} + +expr_ty +Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno) +{ + expr_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Attribute"); + return NULL; + } + if (!attr) { + PyErr_SetString(PyExc_ValueError, + "field attr is required for Attribute"); + return NULL; + } + if (!ctx) { + PyErr_SetString(PyExc_ValueError, + "field ctx is required for Attribute"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Attribute_kind; + p->v.Attribute.value = value; + p->v.Attribute.attr = attr; + p->v.Attribute.ctx = ctx; + p->lineno = lineno; + return p; +} + +expr_ty +Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno) +{ + expr_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Subscript"); + return NULL; + } + if (!slice) { + PyErr_SetString(PyExc_ValueError, + "field slice is required for Subscript"); + return NULL; + } + if (!ctx) { + PyErr_SetString(PyExc_ValueError, + "field ctx is required for Subscript"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Subscript_kind; + p->v.Subscript.value = value; + p->v.Subscript.slice = slice; + p->v.Subscript.ctx = ctx; + p->lineno = lineno; + return p; +} + +expr_ty +Name(identifier id, expr_context_ty ctx, int lineno) +{ + expr_ty p; + if (!id) { + PyErr_SetString(PyExc_ValueError, + "field id is required for Name"); + return NULL; + } + if (!ctx) { + PyErr_SetString(PyExc_ValueError, + "field ctx is required for Name"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Name_kind; + p->v.Name.id = id; + p->v.Name.ctx = ctx; + p->lineno = lineno; + return p; +} + +expr_ty +List(asdl_seq * elts, expr_context_ty ctx, int lineno) +{ + expr_ty p; + if (!ctx) { + PyErr_SetString(PyExc_ValueError, + "field ctx is required for List"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = List_kind; + p->v.List.elts = elts; + p->v.List.ctx = ctx; + p->lineno = lineno; + return p; +} + +expr_ty +Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno) +{ + expr_ty p; + if (!ctx) { + PyErr_SetString(PyExc_ValueError, + "field ctx is required for Tuple"); + return NULL; + } + p = (expr_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Tuple_kind; + p->v.Tuple.elts = elts; + p->v.Tuple.ctx = ctx; + p->lineno = lineno; + return p; +} + +slice_ty +Ellipsis() +{ + slice_ty p; + p = (slice_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Ellipsis_kind; + return p; +} + +slice_ty +Slice(expr_ty lower, expr_ty upper, expr_ty step) +{ + slice_ty p; + p = (slice_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Slice_kind; + p->v.Slice.lower = lower; + p->v.Slice.upper = upper; + p->v.Slice.step = step; + return p; +} + +slice_ty +ExtSlice(asdl_seq * dims) +{ + slice_ty p; + p = (slice_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = ExtSlice_kind; + p->v.ExtSlice.dims = dims; + return p; +} + +slice_ty +Index(expr_ty value) +{ + slice_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Index"); + return NULL; + } + p = (slice_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->kind = Index_kind; + p->v.Index.value = value; + return p; +} + +comprehension_ty +comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs) +{ + comprehension_ty p; + if (!target) { + PyErr_SetString(PyExc_ValueError, + "field target is required for comprehension"); + return NULL; + } + if (!iter) { + PyErr_SetString(PyExc_ValueError, + "field iter is required for comprehension"); + return NULL; + } + p = (comprehension_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->target = target; + p->iter = iter; + p->ifs = ifs; + return p; +} + +excepthandler_ty +excepthandler(expr_ty type, expr_ty name, asdl_seq * body) +{ + excepthandler_ty p; + p = (excepthandler_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->type = type; + p->name = name; + p->body = body; + return p; +} + +arguments_ty +arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * + defaults) +{ + arguments_ty p; + p = (arguments_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->args = args; + p->vararg = vararg; + p->kwarg = kwarg; + p->defaults = defaults; + return p; +} + +keyword_ty +keyword(identifier arg, expr_ty value) +{ + keyword_ty p; + if (!arg) { + PyErr_SetString(PyExc_ValueError, + "field arg is required for keyword"); + return NULL; + } + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for keyword"); + return NULL; + } + p = (keyword_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->arg = arg; + p->value = value; + return p; +} + +alias_ty +alias(identifier name, identifier asname) +{ + alias_ty p; + if (!name) { + PyErr_SetString(PyExc_ValueError, + "field name is required for alias"); + return NULL; + } + p = (alias_ty)malloc(sizeof(*p)); + if (!p) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + p->name = name; + p->asname = asname; + return p; +} + +static void +free_seq_exprs(asdl_seq *seq) +{ + int i, n; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_expr((expr_ty)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); +} + +static void +free_seq_stmts(asdl_seq *seq) +{ + int i, n; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_stmt((stmt_ty)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); +} + +void +free_mod(mod_ty o) +{ + if (!o) + return; + + switch (o->kind) { + case Module_kind: + free_seq_stmts(o->v.Module.body); + break; + case Interactive_kind: + free_seq_stmts(o->v.Interactive.body); + break; + case Expression_kind: + free_expr((expr_ty)o->v.Expression.body); + break; + case Suite_kind: + free_seq_stmts(o->v.Suite.body); + break; + } + + free(o); +} + +void +free_stmt(stmt_ty o) +{ + int i, n; + asdl_seq *seq; + + if (!o) + return; + + switch (o->kind) { + case FunctionDef_kind: + Py_DECREF((identifier)o->v.FunctionDef.name); + free_arguments((arguments_ty)o->v.FunctionDef.args); + free_seq_stmts(o->v.FunctionDef.body); + free_seq_exprs(o->v.FunctionDef.decorators); + break; + case ClassDef_kind: + Py_DECREF((identifier)o->v.ClassDef.name); + free_seq_exprs(o->v.ClassDef.bases); + free_seq_stmts(o->v.ClassDef.body); + break; + case Return_kind: + if (o->v.Return.value) { + free_expr((expr_ty)o->v.Return.value); + } + break; + case Delete_kind: + free_seq_exprs(o->v.Delete.targets); + break; + case Assign_kind: + free_seq_exprs(o->v.Assign.targets); + free_expr((expr_ty)o->v.Assign.value); + break; + case AugAssign_kind: + free_expr((expr_ty)o->v.AugAssign.target); + free_operator((operator_ty)o->v.AugAssign.op); + free_expr((expr_ty)o->v.AugAssign.value); + break; + case Print_kind: + if (o->v.Print.dest) { + free_expr((expr_ty)o->v.Print.dest); + } + free_seq_exprs(o->v.Print.values); + break; + case For_kind: + free_expr((expr_ty)o->v.For.target); + free_expr((expr_ty)o->v.For.iter); + free_seq_stmts(o->v.For.body); + free_seq_stmts(o->v.For.orelse); + break; + case While_kind: + free_expr((expr_ty)o->v.While.test); + free_seq_stmts(o->v.While.body); + free_seq_stmts(o->v.While.orelse); + break; + case If_kind: + free_expr((expr_ty)o->v.If.test); + free_seq_stmts(o->v.If.body); + free_seq_stmts(o->v.If.orelse); + break; + case Raise_kind: + if (o->v.Raise.type) { + free_expr((expr_ty)o->v.Raise.type); + } + if (o->v.Raise.inst) { + free_expr((expr_ty)o->v.Raise.inst); + } + if (o->v.Raise.tback) { + free_expr((expr_ty)o->v.Raise.tback); + } + break; + case TryExcept_kind: + free_seq_stmts(o->v.TryExcept.body); + seq = o->v.TryExcept.handlers; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_excepthandler((excepthandler_ty)asdl_seq_GET(seq, + i)); + asdl_seq_free(seq); + free_seq_stmts(o->v.TryExcept.orelse); + break; + case TryFinally_kind: + free_seq_stmts(o->v.TryFinally.body); + free_seq_stmts(o->v.TryFinally.finalbody); + break; + case Assert_kind: + free_expr((expr_ty)o->v.Assert.test); + if (o->v.Assert.msg) { + free_expr((expr_ty)o->v.Assert.msg); + } + break; + case Import_kind: + seq = o->v.Import.names; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_alias((alias_ty)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); + break; + case ImportFrom_kind: + Py_DECREF((identifier)o->v.ImportFrom.module); + seq = o->v.ImportFrom.names; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_alias((alias_ty)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); + break; + case Exec_kind: + free_expr((expr_ty)o->v.Exec.body); + if (o->v.Exec.globals) { + free_expr((expr_ty)o->v.Exec.globals); + } + if (o->v.Exec.locals) { + free_expr((expr_ty)o->v.Exec.locals); + } + break; + case Global_kind: + seq = o->v.Global.names; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + Py_DECREF((identifier)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); + break; + case Expr_kind: + free_expr((expr_ty)o->v.Expr.value); + break; + case Pass_kind: + break; + case Break_kind: + break; + case Continue_kind: + break; + } + + free(o); +} + +void +free_expr(expr_ty o) +{ + int i, n; + asdl_seq *seq; + + if (!o) + return; + + switch (o->kind) { + case BoolOp_kind: + free_boolop((boolop_ty)o->v.BoolOp.op); + free_seq_exprs(o->v.BoolOp.values); + break; + case BinOp_kind: + free_expr((expr_ty)o->v.BinOp.left); + free_operator((operator_ty)o->v.BinOp.op); + free_expr((expr_ty)o->v.BinOp.right); + break; + case UnaryOp_kind: + free_unaryop((unaryop_ty)o->v.UnaryOp.op); + free_expr((expr_ty)o->v.UnaryOp.operand); + break; + case Lambda_kind: + free_arguments((arguments_ty)o->v.Lambda.args); + free_expr((expr_ty)o->v.Lambda.body); + break; + case Dict_kind: + free_seq_exprs(o->v.Dict.keys); + free_seq_exprs(o->v.Dict.values); + break; + case ListComp_kind: + free_expr((expr_ty)o->v.ListComp.elt); + seq = o->v.ListComp.generators; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_comprehension((comprehension_ty)asdl_seq_GET(seq, + i)); + asdl_seq_free(seq); + break; + case GeneratorExp_kind: + free_expr((expr_ty)o->v.GeneratorExp.elt); + seq = o->v.GeneratorExp.generators; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_comprehension((comprehension_ty)asdl_seq_GET(seq, + i)); + asdl_seq_free(seq); + break; + case Yield_kind: + if (o->v.Yield.value) { + free_expr((expr_ty)o->v.Yield.value); + } + break; + case Compare_kind: + free_expr((expr_ty)o->v.Compare.left); + seq = o->v.Compare.ops; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_cmpop((cmpop_ty)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); + free_seq_exprs(o->v.Compare.comparators); + break; + case Call_kind: + free_expr((expr_ty)o->v.Call.func); + free_seq_exprs(o->v.Call.args); + seq = o->v.Call.keywords; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_keyword((keyword_ty)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); + if (o->v.Call.starargs) { + free_expr((expr_ty)o->v.Call.starargs); + } + if (o->v.Call.kwargs) { + free_expr((expr_ty)o->v.Call.kwargs); + } + break; + case Repr_kind: + free_expr((expr_ty)o->v.Repr.value); + break; + case Num_kind: + Py_DECREF((object)o->v.Num.n); + break; + case Str_kind: + Py_DECREF((string)o->v.Str.s); + break; + case Attribute_kind: + free_expr((expr_ty)o->v.Attribute.value); + Py_DECREF((identifier)o->v.Attribute.attr); + free_expr_context((expr_context_ty)o->v.Attribute.ctx); + break; + case Subscript_kind: + free_expr((expr_ty)o->v.Subscript.value); + free_slice((slice_ty)o->v.Subscript.slice); + free_expr_context((expr_context_ty)o->v.Subscript.ctx); + break; + case Name_kind: + Py_DECREF((identifier)o->v.Name.id); + free_expr_context((expr_context_ty)o->v.Name.ctx); + break; + case List_kind: + free_seq_exprs(o->v.List.elts); + free_expr_context((expr_context_ty)o->v.List.ctx); + break; + case Tuple_kind: + free_seq_exprs(o->v.Tuple.elts); + free_expr_context((expr_context_ty)o->v.Tuple.ctx); + break; + } + + free(o); +} + +void +free_expr_context(expr_context_ty o) +{ + if (!o) + return; + +} + +void +free_slice(slice_ty o) +{ + int i, n; + asdl_seq *seq; + + if (!o) + return; + + switch (o->kind) { + case Ellipsis_kind: + break; + case Slice_kind: + if (o->v.Slice.lower) { + free_expr((expr_ty)o->v.Slice.lower); + } + if (o->v.Slice.upper) { + free_expr((expr_ty)o->v.Slice.upper); + } + if (o->v.Slice.step) { + free_expr((expr_ty)o->v.Slice.step); + } + break; + case ExtSlice_kind: + seq = o->v.ExtSlice.dims; + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_slice((slice_ty)asdl_seq_GET(seq, i)); + asdl_seq_free(seq); + break; + case Index_kind: + free_expr((expr_ty)o->v.Index.value); + break; + } + + free(o); +} + +void +free_boolop(boolop_ty o) +{ + if (!o) + return; + +} + +void +free_operator(operator_ty o) +{ + if (!o) + return; + +} + +void +free_unaryop(unaryop_ty o) +{ + if (!o) + return; + +} + +void +free_cmpop(cmpop_ty o) +{ + if (!o) + return; + +} + +void +free_comprehension(comprehension_ty o) +{ + if (!o) + return; + + free_expr((expr_ty)o->target); + free_expr((expr_ty)o->iter); + free_seq_exprs(o->ifs); + + free(o); +} + +void +free_excepthandler(excepthandler_ty o) +{ + if (!o) + return; + + if (o->type) { + free_expr((expr_ty)o->type); + } + if (o->name) { + free_expr((expr_ty)o->name); + } + free_seq_stmts(o->body); + + free(o); +} + +void +free_arguments(arguments_ty o) +{ + if (!o) + return; + + free_seq_exprs(o->args); + if (o->vararg) { + Py_DECREF((identifier)o->vararg); + } + if (o->kwarg) { + Py_DECREF((identifier)o->kwarg); + } + free_seq_exprs(o->defaults); + + free(o); +} + +void +free_keyword(keyword_ty o) +{ + if (!o) + return; + + Py_DECREF((identifier)o->arg); + free_expr((expr_ty)o->value); + + free(o); +} + +void +free_alias(alias_ty o) +{ + if (!o) + return; + + Py_DECREF((identifier)o->name); + if (o->asname) { + Py_DECREF((identifier)o->asname); + } + + free(o); +} + +int +marshal_write_mod(PyObject **buf, int *off, mod_ty o) +{ + int i; + switch (o->kind) { + case Module_kind: + marshal_write_int(buf, off, 1); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Module.body)); + for (i = 0; i < asdl_seq_LEN(o->v.Module.body); i++) { + void *elt = asdl_seq_GET(o->v.Module.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + break; + case Interactive_kind: + marshal_write_int(buf, off, 2); + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.Interactive.body)); + for (i = 0; i < asdl_seq_LEN(o->v.Interactive.body); i++) { + void *elt = asdl_seq_GET(o->v.Interactive.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + break; + case Expression_kind: + marshal_write_int(buf, off, 3); + marshal_write_expr(buf, off, o->v.Expression.body); + break; + case Suite_kind: + marshal_write_int(buf, off, 4); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Suite.body)); + for (i = 0; i < asdl_seq_LEN(o->v.Suite.body); i++) { + void *elt = asdl_seq_GET(o->v.Suite.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + break; + } + return 1; +} + +int +marshal_write_stmt(PyObject **buf, int *off, stmt_ty o) +{ + int i; + switch (o->kind) { + case FunctionDef_kind: + marshal_write_int(buf, off, 1); + marshal_write_identifier(buf, off, o->v.FunctionDef.name); + marshal_write_arguments(buf, off, o->v.FunctionDef.args); + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.FunctionDef.body)); + for (i = 0; i < asdl_seq_LEN(o->v.FunctionDef.body); i++) { + void *elt = asdl_seq_GET(o->v.FunctionDef.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.FunctionDef.decorators)); + for (i = 0; i < asdl_seq_LEN(o->v.FunctionDef.decorators); i++) + { + void *elt = asdl_seq_GET(o->v.FunctionDef.decorators, + i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + break; + case ClassDef_kind: + marshal_write_int(buf, off, 2); + marshal_write_identifier(buf, off, o->v.ClassDef.name); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.ClassDef.bases)); + for (i = 0; i < asdl_seq_LEN(o->v.ClassDef.bases); i++) { + void *elt = asdl_seq_GET(o->v.ClassDef.bases, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + marshal_write_int(buf, off, asdl_seq_LEN(o->v.ClassDef.body)); + for (i = 0; i < asdl_seq_LEN(o->v.ClassDef.body); i++) { + void *elt = asdl_seq_GET(o->v.ClassDef.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + break; + case Return_kind: + marshal_write_int(buf, off, 3); + if (o->v.Return.value) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Return.value); + } + else { + marshal_write_int(buf, off, 0); + } + break; + case Delete_kind: + marshal_write_int(buf, off, 4); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Delete.targets)); + for (i = 0; i < asdl_seq_LEN(o->v.Delete.targets); i++) { + void *elt = asdl_seq_GET(o->v.Delete.targets, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + break; + case Assign_kind: + marshal_write_int(buf, off, 5); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Assign.targets)); + for (i = 0; i < asdl_seq_LEN(o->v.Assign.targets); i++) { + void *elt = asdl_seq_GET(o->v.Assign.targets, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + marshal_write_expr(buf, off, o->v.Assign.value); + break; + case AugAssign_kind: + marshal_write_int(buf, off, 6); + marshal_write_expr(buf, off, o->v.AugAssign.target); + marshal_write_operator(buf, off, o->v.AugAssign.op); + marshal_write_expr(buf, off, o->v.AugAssign.value); + break; + case Print_kind: + marshal_write_int(buf, off, 7); + if (o->v.Print.dest) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Print.dest); + } + else { + marshal_write_int(buf, off, 0); + } + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Print.values)); + for (i = 0; i < asdl_seq_LEN(o->v.Print.values); i++) { + void *elt = asdl_seq_GET(o->v.Print.values, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + marshal_write_bool(buf, off, o->v.Print.nl); + break; + case For_kind: + marshal_write_int(buf, off, 8); + marshal_write_expr(buf, off, o->v.For.target); + marshal_write_expr(buf, off, o->v.For.iter); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.For.body)); + for (i = 0; i < asdl_seq_LEN(o->v.For.body); i++) { + void *elt = asdl_seq_GET(o->v.For.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + marshal_write_int(buf, off, asdl_seq_LEN(o->v.For.orelse)); + for (i = 0; i < asdl_seq_LEN(o->v.For.orelse); i++) { + void *elt = asdl_seq_GET(o->v.For.orelse, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + break; + case While_kind: + marshal_write_int(buf, off, 9); + marshal_write_expr(buf, off, o->v.While.test); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.While.body)); + for (i = 0; i < asdl_seq_LEN(o->v.While.body); i++) { + void *elt = asdl_seq_GET(o->v.While.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + marshal_write_int(buf, off, asdl_seq_LEN(o->v.While.orelse)); + for (i = 0; i < asdl_seq_LEN(o->v.While.orelse); i++) { + void *elt = asdl_seq_GET(o->v.While.orelse, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + break; + case If_kind: + marshal_write_int(buf, off, 10); + marshal_write_expr(buf, off, o->v.If.test); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.If.body)); + for (i = 0; i < asdl_seq_LEN(o->v.If.body); i++) { + void *elt = asdl_seq_GET(o->v.If.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + marshal_write_int(buf, off, asdl_seq_LEN(o->v.If.orelse)); + for (i = 0; i < asdl_seq_LEN(o->v.If.orelse); i++) { + void *elt = asdl_seq_GET(o->v.If.orelse, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + break; + case Raise_kind: + marshal_write_int(buf, off, 11); + if (o->v.Raise.type) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Raise.type); + } + else { + marshal_write_int(buf, off, 0); + } + if (o->v.Raise.inst) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Raise.inst); + } + else { + marshal_write_int(buf, off, 0); + } + if (o->v.Raise.tback) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Raise.tback); + } + else { + marshal_write_int(buf, off, 0); + } + break; + case TryExcept_kind: + marshal_write_int(buf, off, 12); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.TryExcept.body)); + for (i = 0; i < asdl_seq_LEN(o->v.TryExcept.body); i++) { + void *elt = asdl_seq_GET(o->v.TryExcept.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.TryExcept.handlers)); + for (i = 0; i < asdl_seq_LEN(o->v.TryExcept.handlers); i++) { + void *elt = asdl_seq_GET(o->v.TryExcept.handlers, i); + marshal_write_excepthandler(buf, off, + (excepthandler_ty)elt); + } + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.TryExcept.orelse)); + for (i = 0; i < asdl_seq_LEN(o->v.TryExcept.orelse); i++) { + void *elt = asdl_seq_GET(o->v.TryExcept.orelse, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + break; + case TryFinally_kind: + marshal_write_int(buf, off, 13); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.TryFinally.body)); + for (i = 0; i < asdl_seq_LEN(o->v.TryFinally.body); i++) { + void *elt = asdl_seq_GET(o->v.TryFinally.body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.TryFinally.finalbody)); + for (i = 0; i < asdl_seq_LEN(o->v.TryFinally.finalbody); i++) { + void *elt = asdl_seq_GET(o->v.TryFinally.finalbody, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + break; + case Assert_kind: + marshal_write_int(buf, off, 14); + marshal_write_expr(buf, off, o->v.Assert.test); + if (o->v.Assert.msg) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Assert.msg); + } + else { + marshal_write_int(buf, off, 0); + } + break; + case Import_kind: + marshal_write_int(buf, off, 15); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Import.names)); + for (i = 0; i < asdl_seq_LEN(o->v.Import.names); i++) { + void *elt = asdl_seq_GET(o->v.Import.names, i); + marshal_write_alias(buf, off, (alias_ty)elt); + } + break; + case ImportFrom_kind: + marshal_write_int(buf, off, 16); + marshal_write_identifier(buf, off, o->v.ImportFrom.module); + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.ImportFrom.names)); + for (i = 0; i < asdl_seq_LEN(o->v.ImportFrom.names); i++) { + void *elt = asdl_seq_GET(o->v.ImportFrom.names, i); + marshal_write_alias(buf, off, (alias_ty)elt); + } + break; + case Exec_kind: + marshal_write_int(buf, off, 17); + marshal_write_expr(buf, off, o->v.Exec.body); + if (o->v.Exec.globals) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Exec.globals); + } + else { + marshal_write_int(buf, off, 0); + } + if (o->v.Exec.locals) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Exec.locals); + } + else { + marshal_write_int(buf, off, 0); + } + break; + case Global_kind: + marshal_write_int(buf, off, 18); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Global.names)); + for (i = 0; i < asdl_seq_LEN(o->v.Global.names); i++) { + void *elt = asdl_seq_GET(o->v.Global.names, i); + marshal_write_identifier(buf, off, (identifier)elt); + } + break; + case Expr_kind: + marshal_write_int(buf, off, 19); + marshal_write_expr(buf, off, o->v.Expr.value); + break; + case Pass_kind: + marshal_write_int(buf, off, 20); + break; + case Break_kind: + marshal_write_int(buf, off, 21); + break; + case Continue_kind: + marshal_write_int(buf, off, 22); + break; + } + return 1; +} + +int +marshal_write_expr(PyObject **buf, int *off, expr_ty o) +{ + int i; + switch (o->kind) { + case BoolOp_kind: + marshal_write_int(buf, off, 1); + marshal_write_boolop(buf, off, o->v.BoolOp.op); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.BoolOp.values)); + for (i = 0; i < asdl_seq_LEN(o->v.BoolOp.values); i++) { + void *elt = asdl_seq_GET(o->v.BoolOp.values, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + break; + case BinOp_kind: + marshal_write_int(buf, off, 2); + marshal_write_expr(buf, off, o->v.BinOp.left); + marshal_write_operator(buf, off, o->v.BinOp.op); + marshal_write_expr(buf, off, o->v.BinOp.right); + break; + case UnaryOp_kind: + marshal_write_int(buf, off, 3); + marshal_write_unaryop(buf, off, o->v.UnaryOp.op); + marshal_write_expr(buf, off, o->v.UnaryOp.operand); + break; + case Lambda_kind: + marshal_write_int(buf, off, 4); + marshal_write_arguments(buf, off, o->v.Lambda.args); + marshal_write_expr(buf, off, o->v.Lambda.body); + break; + case Dict_kind: + marshal_write_int(buf, off, 5); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Dict.keys)); + for (i = 0; i < asdl_seq_LEN(o->v.Dict.keys); i++) { + void *elt = asdl_seq_GET(o->v.Dict.keys, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Dict.values)); + for (i = 0; i < asdl_seq_LEN(o->v.Dict.values); i++) { + void *elt = asdl_seq_GET(o->v.Dict.values, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + break; + case ListComp_kind: + marshal_write_int(buf, off, 6); + marshal_write_expr(buf, off, o->v.ListComp.elt); + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.ListComp.generators)); + for (i = 0; i < asdl_seq_LEN(o->v.ListComp.generators); i++) { + void *elt = asdl_seq_GET(o->v.ListComp.generators, i); + marshal_write_comprehension(buf, off, + (comprehension_ty)elt); + } + break; + case GeneratorExp_kind: + marshal_write_int(buf, off, 7); + marshal_write_expr(buf, off, o->v.GeneratorExp.elt); + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.GeneratorExp.generators)); + for (i = 0; i < asdl_seq_LEN(o->v.GeneratorExp.generators); + i++) { + void *elt = asdl_seq_GET(o->v.GeneratorExp.generators, + i); + marshal_write_comprehension(buf, off, + (comprehension_ty)elt); + } + break; + case Yield_kind: + marshal_write_int(buf, off, 8); + if (o->v.Yield.value) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Yield.value); + } + else { + marshal_write_int(buf, off, 0); + } + break; + case Compare_kind: + marshal_write_int(buf, off, 9); + marshal_write_expr(buf, off, o->v.Compare.left); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Compare.ops)); + for (i = 0; i < asdl_seq_LEN(o->v.Compare.ops); i++) { + void *elt = asdl_seq_GET(o->v.Compare.ops, i); + marshal_write_cmpop(buf, off, (cmpop_ty)elt); + } + marshal_write_int(buf, off, + asdl_seq_LEN(o->v.Compare.comparators)); + for (i = 0; i < asdl_seq_LEN(o->v.Compare.comparators); i++) { + void *elt = asdl_seq_GET(o->v.Compare.comparators, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + break; + case Call_kind: + marshal_write_int(buf, off, 10); + marshal_write_expr(buf, off, o->v.Call.func); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Call.args)); + for (i = 0; i < asdl_seq_LEN(o->v.Call.args); i++) { + void *elt = asdl_seq_GET(o->v.Call.args, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Call.keywords)); + for (i = 0; i < asdl_seq_LEN(o->v.Call.keywords); i++) { + void *elt = asdl_seq_GET(o->v.Call.keywords, i); + marshal_write_keyword(buf, off, (keyword_ty)elt); + } + if (o->v.Call.starargs) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Call.starargs); + } + else { + marshal_write_int(buf, off, 0); + } + if (o->v.Call.kwargs) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Call.kwargs); + } + else { + marshal_write_int(buf, off, 0); + } + break; + case Repr_kind: + marshal_write_int(buf, off, 11); + marshal_write_expr(buf, off, o->v.Repr.value); + break; + case Num_kind: + marshal_write_int(buf, off, 12); + marshal_write_object(buf, off, o->v.Num.n); + break; + case Str_kind: + marshal_write_int(buf, off, 13); + marshal_write_string(buf, off, o->v.Str.s); + break; + case Attribute_kind: + marshal_write_int(buf, off, 14); + marshal_write_expr(buf, off, o->v.Attribute.value); + marshal_write_identifier(buf, off, o->v.Attribute.attr); + marshal_write_expr_context(buf, off, o->v.Attribute.ctx); + break; + case Subscript_kind: + marshal_write_int(buf, off, 15); + marshal_write_expr(buf, off, o->v.Subscript.value); + marshal_write_slice(buf, off, o->v.Subscript.slice); + marshal_write_expr_context(buf, off, o->v.Subscript.ctx); + break; + case Name_kind: + marshal_write_int(buf, off, 16); + marshal_write_identifier(buf, off, o->v.Name.id); + marshal_write_expr_context(buf, off, o->v.Name.ctx); + break; + case List_kind: + marshal_write_int(buf, off, 17); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.List.elts)); + for (i = 0; i < asdl_seq_LEN(o->v.List.elts); i++) { + void *elt = asdl_seq_GET(o->v.List.elts, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + marshal_write_expr_context(buf, off, o->v.List.ctx); + break; + case Tuple_kind: + marshal_write_int(buf, off, 18); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.Tuple.elts)); + for (i = 0; i < asdl_seq_LEN(o->v.Tuple.elts); i++) { + void *elt = asdl_seq_GET(o->v.Tuple.elts, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + marshal_write_expr_context(buf, off, o->v.Tuple.ctx); + break; + } + return 1; +} + +int +marshal_write_expr_context(PyObject **buf, int *off, expr_context_ty o) +{ + int i; + switch (o) { + case Load: + marshal_write_int(buf, off, 1); + break; + case Store: + marshal_write_int(buf, off, 2); + break; + case Del: + marshal_write_int(buf, off, 3); + break; + case AugLoad: + marshal_write_int(buf, off, 4); + break; + case AugStore: + marshal_write_int(buf, off, 5); + break; + case Param: + marshal_write_int(buf, off, 6); + break; + } + return 1; +} + +int +marshal_write_slice(PyObject **buf, int *off, slice_ty o) +{ + int i; + switch (o->kind) { + case Ellipsis_kind: + marshal_write_int(buf, off, 1); + break; + case Slice_kind: + marshal_write_int(buf, off, 2); + if (o->v.Slice.lower) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Slice.lower); + } + else { + marshal_write_int(buf, off, 0); + } + if (o->v.Slice.upper) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Slice.upper); + } + else { + marshal_write_int(buf, off, 0); + } + if (o->v.Slice.step) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->v.Slice.step); + } + else { + marshal_write_int(buf, off, 0); + } + break; + case ExtSlice_kind: + marshal_write_int(buf, off, 3); + marshal_write_int(buf, off, asdl_seq_LEN(o->v.ExtSlice.dims)); + for (i = 0; i < asdl_seq_LEN(o->v.ExtSlice.dims); i++) { + void *elt = asdl_seq_GET(o->v.ExtSlice.dims, i); + marshal_write_slice(buf, off, (slice_ty)elt); + } + break; + case Index_kind: + marshal_write_int(buf, off, 4); + marshal_write_expr(buf, off, o->v.Index.value); + break; + } + return 1; +} + +int +marshal_write_boolop(PyObject **buf, int *off, boolop_ty o) +{ + int i; + switch (o) { + case And: + marshal_write_int(buf, off, 1); + break; + case Or: + marshal_write_int(buf, off, 2); + break; + } + return 1; +} + +int +marshal_write_operator(PyObject **buf, int *off, operator_ty o) +{ + int i; + switch (o) { + case Add: + marshal_write_int(buf, off, 1); + break; + case Sub: + marshal_write_int(buf, off, 2); + break; + case Mult: + marshal_write_int(buf, off, 3); + break; + case Div: + marshal_write_int(buf, off, 4); + break; + case Mod: + marshal_write_int(buf, off, 5); + break; + case Pow: + marshal_write_int(buf, off, 6); + break; + case LShift: + marshal_write_int(buf, off, 7); + break; + case RShift: + marshal_write_int(buf, off, 8); + break; + case BitOr: + marshal_write_int(buf, off, 9); + break; + case BitXor: + marshal_write_int(buf, off, 10); + break; + case BitAnd: + marshal_write_int(buf, off, 11); + break; + case FloorDiv: + marshal_write_int(buf, off, 12); + break; + } + return 1; +} + +int +marshal_write_unaryop(PyObject **buf, int *off, unaryop_ty o) +{ + int i; + switch (o) { + case Invert: + marshal_write_int(buf, off, 1); + break; + case Not: + marshal_write_int(buf, off, 2); + break; + case UAdd: + marshal_write_int(buf, off, 3); + break; + case USub: + marshal_write_int(buf, off, 4); + break; + } + return 1; +} + +int +marshal_write_cmpop(PyObject **buf, int *off, cmpop_ty o) +{ + int i; + switch (o) { + case Eq: + marshal_write_int(buf, off, 1); + break; + case NotEq: + marshal_write_int(buf, off, 2); + break; + case Lt: + marshal_write_int(buf, off, 3); + break; + case LtE: + marshal_write_int(buf, off, 4); + break; + case Gt: + marshal_write_int(buf, off, 5); + break; + case GtE: + marshal_write_int(buf, off, 6); + break; + case Is: + marshal_write_int(buf, off, 7); + break; + case IsNot: + marshal_write_int(buf, off, 8); + break; + case In: + marshal_write_int(buf, off, 9); + break; + case NotIn: + marshal_write_int(buf, off, 10); + break; + } + return 1; +} + +int +marshal_write_comprehension(PyObject **buf, int *off, comprehension_ty o) +{ + int i; + marshal_write_expr(buf, off, o->target); + marshal_write_expr(buf, off, o->iter); + marshal_write_int(buf, off, asdl_seq_LEN(o->ifs)); + for (i = 0; i < asdl_seq_LEN(o->ifs); i++) { + void *elt = asdl_seq_GET(o->ifs, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + return 1; +} + +int +marshal_write_excepthandler(PyObject **buf, int *off, excepthandler_ty o) +{ + int i; + if (o->type) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->type); + } + else { + marshal_write_int(buf, off, 0); + } + if (o->name) { + marshal_write_int(buf, off, 1); + marshal_write_expr(buf, off, o->name); + } + else { + marshal_write_int(buf, off, 0); + } + marshal_write_int(buf, off, asdl_seq_LEN(o->body)); + for (i = 0; i < asdl_seq_LEN(o->body); i++) { + void *elt = asdl_seq_GET(o->body, i); + marshal_write_stmt(buf, off, (stmt_ty)elt); + } + return 1; +} + +int +marshal_write_arguments(PyObject **buf, int *off, arguments_ty o) +{ + int i; + marshal_write_int(buf, off, asdl_seq_LEN(o->args)); + for (i = 0; i < asdl_seq_LEN(o->args); i++) { + void *elt = asdl_seq_GET(o->args, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + if (o->vararg) { + marshal_write_int(buf, off, 1); + marshal_write_identifier(buf, off, o->vararg); + } + else { + marshal_write_int(buf, off, 0); + } + if (o->kwarg) { + marshal_write_int(buf, off, 1); + marshal_write_identifier(buf, off, o->kwarg); + } + else { + marshal_write_int(buf, off, 0); + } + marshal_write_int(buf, off, asdl_seq_LEN(o->defaults)); + for (i = 0; i < asdl_seq_LEN(o->defaults); i++) { + void *elt = asdl_seq_GET(o->defaults, i); + marshal_write_expr(buf, off, (expr_ty)elt); + } + return 1; +} + +int +marshal_write_keyword(PyObject **buf, int *off, keyword_ty o) +{ + int i; + marshal_write_identifier(buf, off, o->arg); + marshal_write_expr(buf, off, o->value); + return 1; +} + +int +marshal_write_alias(PyObject **buf, int *off, alias_ty o) +{ + int i; + marshal_write_identifier(buf, off, o->name); + if (o->asname) { + marshal_write_int(buf, off, 1); + marshal_write_identifier(buf, off, o->asname); + } + else { + marshal_write_int(buf, off, 0); + } + return 1; +} + diff --git a/Python/asdl.c b/Python/asdl.c new file mode 100644 index 0000000..bb29857 --- /dev/null +++ b/Python/asdl.c @@ -0,0 +1,92 @@ +#include "Python.h" +#include "asdl.h" + +asdl_seq * +asdl_seq_new(int size) +{ + asdl_seq *seq = NULL; + size_t n = sizeof(asdl_seq) + + (size ? (sizeof(void *) * (size - 1)) : 0); + + seq = (asdl_seq *)PyObject_Malloc(n); + if (!seq) { + PyErr_SetString(PyExc_MemoryError, "no memory"); + return NULL; + } + memset(seq, 0, n); + seq->size = size; + return seq; +} + +void +asdl_seq_free(asdl_seq *seq) +{ + PyObject_Free(seq); +} + +#define CHECKSIZE(BUF, OFF, MIN) { \ + int need = *(OFF) + MIN; \ + if (need >= PyString_GET_SIZE(*(BUF))) { \ + int newsize = PyString_GET_SIZE(*(BUF)) * 2; \ + if (newsize < need) \ + newsize = need; \ + if (_PyString_Resize((BUF), newsize) < 0) \ + return 0; \ + } \ +} + +int +marshal_write_int(PyObject **buf, int *offset, int x) +{ + char *s; + + CHECKSIZE(buf, offset, 4) + s = PyString_AS_STRING(*buf) + (*offset); + s[0] = (x & 0xff); + s[1] = (x >> 8) & 0xff; + s[2] = (x >> 16) & 0xff; + s[3] = (x >> 24) & 0xff; + *offset += 4; + return 1; +} + +int +marshal_write_bool(PyObject **buf, int *offset, bool b) +{ + if (b) + marshal_write_int(buf, offset, 1); + else + marshal_write_int(buf, offset, 0); + return 1; +} + +int +marshal_write_identifier(PyObject **buf, int *offset, identifier id) +{ + int l = PyString_GET_SIZE(id); + marshal_write_int(buf, offset, l); + CHECKSIZE(buf, offset, l); + memcpy(PyString_AS_STRING(*buf) + *offset, + PyString_AS_STRING(id), l); + *offset += l; + return 1; +} + +int +marshal_write_string(PyObject **buf, int *offset, string s) +{ + int len = PyString_GET_SIZE(s); + marshal_write_int(buf, offset, len); + CHECKSIZE(buf, offset, len); + memcpy(PyString_AS_STRING(*buf) + *offset, + PyString_AS_STRING(s), len); + *offset += len; + return 1; +} + +int +marshal_write_object(PyObject **buf, int *offset, object s) +{ + /* XXX */ + return 0; +} diff --git a/Python/ast.c b/Python/ast.c new file mode 100644 index 0000000..475382c --- /dev/null +++ b/Python/ast.c @@ -0,0 +1,3114 @@ +/* + * This file includes functions to transform a concrete syntax tree (CST) to + * an abstract syntax tree (AST). The main function is PyAST_FromNode(). + * + */ +#include "Python.h" +#include "Python-ast.h" +#include "grammar.h" +#include "node.h" +#include "ast.h" +#include "token.h" +#include "parsetok.h" +#include "graminit.h" + +#include <assert.h> + +#if 0 +#define fprintf if (0) fprintf +#endif + +/* XXX TO DO + - re-indent this file (should be done) + - internal error checking (freeing memory, etc.) + - syntax errors +*/ + + +/* Data structure used internally */ +struct compiling { + char *c_encoding; /* source encoding */ +}; + +static asdl_seq *seq_for_testlist(struct compiling *, const node *); +static expr_ty ast_for_expr(struct compiling *, const node *); +static stmt_ty ast_for_stmt(struct compiling *, const node *); +static asdl_seq *ast_for_suite(struct compiling *, const node *); +static asdl_seq *ast_for_exprlist(struct compiling *, const node *, int); +static expr_ty ast_for_testlist(struct compiling *, const node *, int); + +/* Note different signature for ast_for_call */ +static expr_ty ast_for_call(struct compiling *, const node *, expr_ty); + +static PyObject *parsenumber(const char *); +static PyObject *parsestr(const char *s, const char *encoding); +static PyObject *parsestrplus(struct compiling *, const node *n); + +extern grammar _PyParser_Grammar; /* From graminit.c */ + +#ifndef LINENO +#define LINENO(n) ((n)->n_lineno) +#endif + +#define NEW_IDENTIFIER(n) PyString_InternFromString(STR(n)) + +static void +asdl_stmt_seq_free(asdl_seq* seq) +{ + int n, i; + + if (!seq) + return; + + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_stmt(asdl_seq_GET(seq, i)); + asdl_seq_free(seq); +} + +static void +asdl_expr_seq_free(asdl_seq* seq) +{ + int n, i; + + if (!seq) + return; + + n = asdl_seq_LEN(seq); + for (i = 0; i < n; i++) + free_expr(asdl_seq_GET(seq, i)); + asdl_seq_free(seq); +} + +/* This routine provides an invalid object for the syntax error. + The outermost routine must unpack this error and create the + proper object. We do this so that we don't have to pass + the filename to everything function. + + XXX Maybe we should just pass the filename... +*/ + +static int +ast_error(const node *n, const char *errstr) +{ + PyObject *u = Py_BuildValue("zi", errstr, LINENO(n)); + if (!u) + return 0; + PyErr_SetObject(PyExc_SyntaxError, u); + Py_DECREF(u); + return 0; +} + +static void +ast_error_finish(const char *filename) +{ + PyObject *type, *value, *tback, *errstr, *loc, *tmp; + int lineno; + + assert(PyErr_Occurred()); + if (!PyErr_ExceptionMatches(PyExc_SyntaxError)) + return; + + PyErr_Fetch(&type, &value, &tback); + errstr = PyTuple_GetItem(value, 0); + if (!errstr) + return; + Py_INCREF(errstr); + lineno = PyInt_AsLong(PyTuple_GetItem(value, 1)); + if (lineno == -1) + return; + Py_DECREF(value); + + loc = PyErr_ProgramText(filename, lineno); + if (!loc) { + Py_INCREF(Py_None); + loc = Py_None; + } + tmp = Py_BuildValue("(ziOO)", filename, lineno, Py_None, loc); + Py_DECREF(loc); + if (!tmp) + return; + value = Py_BuildValue("(OO)", errstr, tmp); + Py_DECREF(errstr); + Py_DECREF(tmp); + if (!value) + return; + PyErr_Restore(type, value, tback); +} + +/* num_stmts() returns number of contained statements. + + Use this routine to determine how big a sequence is needed for + the statements in a parse tree. Its raison d'etre is this bit of + grammar: + + stmt: simple_stmt | compound_stmt + simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE + + A simple_stmt can contain multiple small_stmt elements joined + by semicolons. If the arg is a simple_stmt, the number of + small_stmt elements is returned. +*/ + +static int +num_stmts(const node *n) +{ + int i, l; + node *ch; + + switch (TYPE(n)) { + case single_input: + if (TYPE(CHILD(n, 0)) == NEWLINE) + return 0; + else + return num_stmts(CHILD(n, 0)); + case file_input: + l = 0; + for (i = 0; i < NCH(n); i++) { + ch = CHILD(n, i); + if (TYPE(ch) == stmt) + l += num_stmts(ch); + } + return l; + case stmt: + return num_stmts(CHILD(n, 0)); + case compound_stmt: + return 1; + case simple_stmt: + return NCH(n) / 2; /* Divide by 2 to remove count of semi-colons */ + case suite: + if (NCH(n) == 1) + return num_stmts(CHILD(n, 0)); + else { + l = 0; + for (i = 2; i < (NCH(n) - 1); i++) + l += num_stmts(CHILD(n, i)); + return l; + } + default: { + char buf[128]; + + sprintf(buf, "Non-statement found: %d %d\n", + TYPE(n), NCH(n)); + Py_FatalError(buf); + } + } + assert(0); + return 0; +} + +/* Transform the CST rooted at node * to the appropriate AST +*/ + +mod_ty +PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename) +{ + int i, j, num; + asdl_seq *stmts = NULL; + stmt_ty s; + node *ch; + struct compiling c; + + if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) { + c.c_encoding = "utf-8"; + } else if (TYPE(n) == encoding_decl) { + c.c_encoding = STR(n); + n = CHILD(n, 0); + } else { + c.c_encoding = NULL; + } + + switch (TYPE(n)) { + case file_input: + stmts = asdl_seq_new(num_stmts(n)); + if (!stmts) + return NULL; + for (i = 0; i < NCH(n) - 1; i++) { + ch = CHILD(n, i); + if (TYPE(ch) == NEWLINE) + continue; + REQ(ch, stmt); + num = num_stmts(ch); + if (num == 1) { + s = ast_for_stmt(&c, ch); + if (!s) + goto error; + asdl_seq_APPEND(stmts, s); + } + else { + ch = CHILD(ch, 0); + REQ(ch, simple_stmt); + for (j = 0; j < num; j++) { + s = ast_for_stmt(&c, CHILD(ch, j * 2)); + if (!s) + goto error; + asdl_seq_APPEND(stmts, s); + } + } + } + return Module(stmts); + case eval_input: { + expr_ty testlist_ast; + + /* XXX Why not gen_for here? */ + testlist_ast = ast_for_testlist(&c, CHILD(n, 0), 0); + if (!testlist_ast) + goto error; + return Expression(testlist_ast); + } + case single_input: + if (TYPE(CHILD(n, 0)) == NEWLINE) { + stmts = asdl_seq_new(1); + if (!stmts) + goto error; + asdl_seq_SET(stmts, 0, Pass(n->n_lineno)); + return Interactive(stmts); + } + else { + n = CHILD(n, 0); + num = num_stmts(n); + stmts = asdl_seq_new(num); + if (!stmts) + goto error; + if (num == 1) { + stmt_ty s = ast_for_stmt(&c, n); + if (!s) + goto error; + asdl_seq_SET(stmts, 0, s); + } + else { + /* Only a simple_stmt can contain multiple statements. */ + REQ(n, simple_stmt); + for (i = 0; i < NCH(n); i += 2) { + stmt_ty s; + if (TYPE(CHILD(n, i)) == NEWLINE) + break; + s = ast_for_stmt(&c, CHILD(n, i)); + if (!s) + goto error; + asdl_seq_SET(stmts, i / 2, s); + } + } + + return Interactive(stmts); + } + default: + goto error; + } + error: + if (stmts) + asdl_stmt_seq_free(stmts); + ast_error_finish(filename); + return NULL; +} + +/* Return the AST repr. of the operator represented as syntax (|, ^, etc.) +*/ + +static operator_ty +get_operator(const node *n) +{ + switch (TYPE(n)) { + case VBAR: + return BitOr; + case CIRCUMFLEX: + return BitXor; + case AMPER: + return BitAnd; + case LEFTSHIFT: + return LShift; + case RIGHTSHIFT: + return RShift; + case PLUS: + return Add; + case MINUS: + return Sub; + case STAR: + return Mult; + case SLASH: + return Div; + case DOUBLESLASH: + return FloorDiv; + case PERCENT: + return Mod; + default: + return 0; + } +} + +/* Set the context ctx for expr_ty e returning 0 on success, -1 on error. + + Only sets context for expr kinds that "can appear in assignment context" + (according to ../Parser/Python.asdl). For other expr kinds, it sets + an appropriate syntax error and returns false. + + If e is a sequential type, items in sequence will also have their context + set. + +*/ + +static int +set_context(expr_ty e, expr_context_ty ctx, const node *n) +{ + asdl_seq *s = NULL; + + switch (e->kind) { + case Attribute_kind: + if (ctx == Store && + !strcmp(PyString_AS_STRING(e->v.Attribute.attr), "None")) { + return ast_error(n, "assignment to None"); + } + e->v.Attribute.ctx = ctx; + break; + case Subscript_kind: + e->v.Subscript.ctx = ctx; + break; + case Name_kind: + if (ctx == Store && + !strcmp(PyString_AS_STRING(e->v.Name.id), "None")) { + return ast_error(n, "assignment to None"); + } + e->v.Name.ctx = ctx; + break; + case List_kind: + e->v.List.ctx = ctx; + s = e->v.List.elts; + break; + case Tuple_kind: + if (asdl_seq_LEN(e->v.Tuple.elts) == 0) + return ast_error(n, "can't assign to ()"); + e->v.Tuple.ctx = ctx; + s = e->v.Tuple.elts; + break; + case Call_kind: + if (ctx == Store) + return ast_error(n, "can't assign to function call"); + else if (ctx == Del) + return ast_error(n, "can't delete function call"); + else + return ast_error(n, "unexpected operation on function call"); + break; + case BinOp_kind: + return ast_error(n, "can't assign to operator"); + case GeneratorExp_kind: + return ast_error(n, "assignment to generator expression " + "not possible"); + case Num_kind: + case Str_kind: + return ast_error(n, "can't assign to literal"); + default: { + char buf[300]; + PyOS_snprintf(buf, sizeof(buf), + "unexpected expression in assignment %d (line %d)", + e->kind, e->lineno); + return ast_error(n, buf); + } + } + /* If the LHS is a list or tuple, we need to set the assignment + context for all the tuple elements. + */ + if (s) { + int i; + + for (i = 0; i < asdl_seq_LEN(s); i++) { + if (!set_context(asdl_seq_GET(s, i), ctx, n)) + return 0; + } + } + return 1; +} + +static operator_ty +ast_for_augassign(const node *n) +{ + REQ(n, augassign); + n = CHILD(n, 0); + switch (STR(n)[0]) { + case '+': + return Add; + case '-': + return Sub; + case '/': + if (STR(n)[1] == '/') + return FloorDiv; + else + return Div; + case '%': + return Mod; + case '<': + return LShift; + case '>': + return RShift; + case '&': + return BitAnd; + case '^': + return BitXor; + case '|': + return BitOr; + case '*': + if (STR(n)[1] == '*') + return Pow; + else + return Mult; + default: + PyErr_Format(PyExc_Exception, "invalid augassign: %s", STR(n)); + return 0; + } +} + +static cmpop_ty +ast_for_comp_op(const node *n) +{ + /* comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is' + |'is' 'not' + */ + REQ(n, comp_op); + if (NCH(n) == 1) { + n = CHILD(n, 0); + switch (TYPE(n)) { + case LESS: + return Lt; + case GREATER: + return Gt; + case EQEQUAL: /* == */ + case EQUAL: + return Eq; + case LESSEQUAL: + return LtE; + case GREATEREQUAL: + return GtE; + case NOTEQUAL: + return NotEq; + case NAME: + if (strcmp(STR(n), "in") == 0) + return In; + if (strcmp(STR(n), "is") == 0) + return Is; + default: + PyErr_Format(PyExc_Exception, "invalid comp_op: %s", + STR(n)); + return 0; + } + } + else if (NCH(n) == 2) { + /* handle "not in" and "is not" */ + switch (TYPE(CHILD(n, 0))) { + case NAME: + if (strcmp(STR(CHILD(n, 1)), "in") == 0) + return NotIn; + if (strcmp(STR(CHILD(n, 0)), "is") == 0) + return IsNot; + default: + PyErr_Format(PyExc_Exception, "invalid comp_op: %s %s", + STR(CHILD(n, 0)), STR(CHILD(n, 1))); + return 0; + } + } + PyErr_Format(PyExc_Exception, "invalid comp_op: has %d children", + NCH(n)); + return 0; +} + +static asdl_seq * +seq_for_testlist(struct compiling *c, const node *n) +{ + /* testlist: test (',' test)* [','] */ + assert(TYPE(n) == testlist + || TYPE(n) == listmaker + || TYPE(n) == testlist_gexp + || TYPE(n) == testlist_safe + ); + asdl_seq *seq; + expr_ty expression; + int i; + + seq = asdl_seq_new((NCH(n) + 1) / 2); + if (!seq) + return NULL; + + for (i = 0; i < NCH(n); i += 2) { + REQ(CHILD(n, i), test); + + expression = ast_for_expr(c, CHILD(n, i)); + if (!expression) { + asdl_seq_free(seq); + return NULL; + } + + assert(i / 2 < seq->size); + asdl_seq_SET(seq, i / 2, expression); + } + return seq; +} + +static expr_ty +compiler_complex_args(const node *n) +{ + int i, len = (NCH(n) + 1) / 2; + expr_ty result; + asdl_seq *args = asdl_seq_new(len); + if (!args) + return NULL; + + REQ(n, fplist); + + for (i = 0; i < len; i++) { + const node *child = CHILD(CHILD(n, 2*i), 0); + expr_ty arg; + if (TYPE(child) == NAME) { + if (!strcmp(STR(child), "None")) { + ast_error(child, "assignment to None"); + return NULL; + } + arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child)); + } + else + arg = compiler_complex_args(CHILD(CHILD(n, 2*i), 1)); + set_context(arg, Store, n); + asdl_seq_SET(args, i, arg); + } + + result = Tuple(args, Store, LINENO(n)); + set_context(result, Store, n); + return result; +} + +/* Create AST for argument list. + + XXX TO DO: + - check for invalid argument lists like normal after default +*/ + +static arguments_ty +ast_for_arguments(struct compiling *c, const node *n) +{ + /* parameters: '(' [varargslist] ')' + varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] + | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [','] + */ + int i, n_args = 0, n_defaults = 0, found_default = 0; + asdl_seq *args, *defaults; + identifier vararg = NULL, kwarg = NULL; + node *ch; + + if (TYPE(n) == parameters) { + if (NCH(n) == 2) /* () as argument list */ + return arguments(NULL, NULL, NULL, NULL); + n = CHILD(n, 1); + } + REQ(n, varargslist); + + /* first count the number of normal args & defaults */ + for (i = 0; i < NCH(n); i++) { + ch = CHILD(n, i); + if (TYPE(ch) == fpdef) { + n_args++; + } + if (TYPE(ch) == EQUAL) + n_defaults++; + } + args = (n_args ? asdl_seq_new(n_args) : NULL); + if (!args && n_args) + return NULL; /* Don't need to go to NULL; nothing allocated */ + defaults = (n_defaults ? asdl_seq_new(n_defaults) : NULL); + if (!defaults && n_defaults) + goto error; + + /* fpdef: NAME | '(' fplist ')' + fplist: fpdef (',' fpdef)* [','] + */ + i = 0; + while (i < NCH(n)) { + ch = CHILD(n, i); + switch (TYPE(ch)) { + case fpdef: + /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is + anything other than EQUAL or a comma? */ + /* XXX Should NCH(n) check be made a separate check? */ + if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) { + asdl_seq_APPEND(defaults, + ast_for_expr(c, CHILD(n, i + 2))); + i += 2; + found_default = 1; + } + else if (found_default) { + ast_error(n, + "non-default argument follows default argument"); + goto error; + } + + if (NCH(ch) == 3) { + asdl_seq_APPEND(args, + compiler_complex_args(CHILD(ch, 1))); + } + else if (TYPE(CHILD(ch, 0)) == NAME) { + if (!strcmp(STR(CHILD(ch, 0)), "None")) { + ast_error(CHILD(ch, 0), "assignment to None"); + goto error; + } + expr_ty name = Name(NEW_IDENTIFIER(CHILD(ch, 0)), + Param, LINENO(ch)); + if (!name) + goto error; + asdl_seq_APPEND(args, name); + + } + i += 2; /* the name and the comma */ + break; + case STAR: + if (!strcmp(STR(CHILD(n, i+1)), "None")) { + ast_error(CHILD(n, i+1), "assignment to None"); + goto error; + } + vararg = NEW_IDENTIFIER(CHILD(n, i+1)); + i += 3; + break; + case DOUBLESTAR: + if (!strcmp(STR(CHILD(n, i+1)), "None")) { + ast_error(CHILD(n, i+1), "assignment to None"); + goto error; + } + kwarg = NEW_IDENTIFIER(CHILD(n, i+1)); + i += 3; + break; + default: + PyErr_Format(PyExc_Exception, + "unexpected node in varargslist: %d @ %d", + TYPE(ch), i); + goto error; + } + } + + return arguments(args, vararg, kwarg, defaults); + + error: + if (args) + asdl_seq_free(args); + if (defaults) + asdl_seq_free(defaults); + return NULL; +} + +static expr_ty +ast_for_dotted_name(struct compiling *c, const node *n) +{ + expr_ty e = NULL; + expr_ty attrib = NULL; + identifier id = NULL; + int i; + + REQ(n, dotted_name); + + id = NEW_IDENTIFIER(CHILD(n, 0)); + if (!id) + goto error; + e = Name(id, Load, LINENO(n)); + if (!e) + goto error; + id = NULL; + + for (i = 2; i < NCH(n); i+=2) { + id = NEW_IDENTIFIER(CHILD(n, i)); + if (!id) + goto error; + attrib = Attribute(e, id, Load, LINENO(CHILD(n, i))); + if (!attrib) + goto error; + e = attrib; + attrib = NULL; + } + + return e; + + error: + Py_XDECREF(id); + free_expr(e); + return NULL; +} + +static expr_ty +ast_for_decorator(struct compiling *c, const node *n) +{ + /* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */ + expr_ty d = NULL; + expr_ty name_expr = NULL; + + REQ(n, decorator); + + if ((NCH(n) < 3 && NCH(n) != 5 && NCH(n) != 6) + || TYPE(CHILD(n, 0)) != AT || TYPE(RCHILD(n, -1)) != NEWLINE) { + ast_error(n, "Invalid decorator node"); + goto error; + } + + name_expr = ast_for_dotted_name(c, CHILD(n, 1)); + if (!name_expr) + goto error; + + if (NCH(n) == 3) { /* No arguments */ + d = name_expr; + name_expr = NULL; + } + else if (NCH(n) == 5) { /* Call with no arguments */ + d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n)); + if (!d) + goto error; + name_expr = NULL; + } + else { + d = ast_for_call(c, CHILD(n, 3), name_expr); + if (!d) + goto error; + name_expr = NULL; + } + + return d; + + error: + free_expr(name_expr); + free_expr(d); + return NULL; +} + +static asdl_seq* +ast_for_decorators(struct compiling *c, const node *n) +{ + asdl_seq* decorator_seq = NULL; + expr_ty d = NULL; + int i; + + REQ(n, decorators); + + decorator_seq = asdl_seq_new(NCH(n)); + if (!decorator_seq) + return NULL; + + for (i = 0; i < NCH(n); i++) { + d = ast_for_decorator(c, CHILD(n, i)); + if (!d) + goto error; + asdl_seq_APPEND(decorator_seq, d); + d = NULL; + } + return decorator_seq; + error: + asdl_expr_seq_free(decorator_seq); + free_expr(d); + return NULL; +} + +static stmt_ty +ast_for_funcdef(struct compiling *c, const node *n) +{ + /* funcdef: 'def' [decorators] NAME parameters ':' suite */ + identifier name = NULL; + arguments_ty args = NULL; + asdl_seq *body = NULL; + asdl_seq *decorator_seq = NULL; + int name_i; + + REQ(n, funcdef); + + if (NCH(n) == 6) { /* decorators are present */ + decorator_seq = ast_for_decorators(c, CHILD(n, 0)); + if (!decorator_seq) + goto error; + name_i = 2; + } + else { + name_i = 1; + } + + name = NEW_IDENTIFIER(CHILD(n, name_i)); + if (!name) + goto error; + else if (!strcmp(STR(CHILD(n, name_i)), "None")) { + ast_error(CHILD(n, name_i), "assignment to None"); + goto error; + } + args = ast_for_arguments(c, CHILD(n, name_i + 1)); + if (!args) + goto error; + body = ast_for_suite(c, CHILD(n, name_i + 3)); + if (!body) + goto error; + + return FunctionDef(name, args, body, decorator_seq, LINENO(n)); + +error: + asdl_stmt_seq_free(body); + asdl_expr_seq_free(decorator_seq); + free_arguments(args); + Py_XDECREF(name); + return NULL; +} + +static expr_ty +ast_for_lambdef(struct compiling *c, const node *n) +{ + /* lambdef: 'lambda' [varargslist] ':' test */ + arguments_ty args; + expr_ty expression; + + if (NCH(n) == 3) { + args = arguments(NULL, NULL, NULL, NULL); + if (!args) + return NULL; + expression = ast_for_expr(c, CHILD(n, 2)); + if (!expression) { + free_arguments(args); + return NULL; + } + } + else { + args = ast_for_arguments(c, CHILD(n, 1)); + if (!args) + return NULL; + expression = ast_for_expr(c, CHILD(n, 3)); + if (!expression) { + free_arguments(args); + return NULL; + } + } + + return Lambda(args, expression, LINENO(n)); +} + +/* Count the number of 'for' loop in a list comprehension. + + Helper for ast_for_listcomp(). +*/ + +static int +count_list_fors(const node *n) +{ + int n_fors = 0; + node *ch = CHILD(n, 1); + + count_list_for: + n_fors++; + REQ(ch, list_for); + if (NCH(ch) == 5) + ch = CHILD(ch, 4); + else + return n_fors; + count_list_iter: + REQ(ch, list_iter); + ch = CHILD(ch, 0); + if (TYPE(ch) == list_for) + goto count_list_for; + else if (TYPE(ch) == list_if) { + if (NCH(ch) == 3) { + ch = CHILD(ch, 2); + goto count_list_iter; + } + else + return n_fors; + } + else { + /* Should never be reached */ + PyErr_SetString(PyExc_Exception, "logic error in count_list_fors"); + return -1; + } +} + +/* Count the number of 'if' statements in a list comprehension. + + Helper for ast_for_listcomp(). +*/ + +static int +count_list_ifs(const node *n) +{ + int n_ifs = 0; + + count_list_iter: + REQ(n, list_iter); + if (TYPE(CHILD(n, 0)) == list_for) + return n_ifs; + n = CHILD(n, 0); + REQ(n, list_if); + n_ifs++; + if (NCH(n) == 2) + return n_ifs; + n = CHILD(n, 2); + goto count_list_iter; +} + +static expr_ty +ast_for_listcomp(struct compiling *c, const node *n) +{ + /* listmaker: test ( list_for | (',' test)* [','] ) + list_for: 'for' exprlist 'in' testlist_safe [list_iter] + list_iter: list_for | list_if + list_if: 'if' test [list_iter] + testlist_safe: test [(',' test)+ [',']] + */ + expr_ty elt; + asdl_seq *listcomps; + int i, n_fors; + node *ch; + + REQ(n, listmaker); + assert(NCH(n) > 1); + + elt = ast_for_expr(c, CHILD(n, 0)); + if (!elt) + return NULL; + + n_fors = count_list_fors(n); + if (n_fors == -1) + return NULL; + + listcomps = asdl_seq_new(n_fors); + if (!listcomps) { + free_expr(elt); + return NULL; + } + + ch = CHILD(n, 1); + for (i = 0; i < n_fors; i++) { + comprehension_ty lc; + asdl_seq *t; + expr_ty expression; + + REQ(ch, list_for); + + t = ast_for_exprlist(c, CHILD(ch, 1), Store); + if (!t) { + asdl_seq_free(listcomps); + free_expr(elt); + return NULL; + } + expression = ast_for_testlist(c, CHILD(ch, 3), 0); + if (!expression) { + asdl_seq_free(t); + asdl_seq_free(listcomps); + free_expr(elt); + return NULL; + } + + if (asdl_seq_LEN(t) == 1) + lc = comprehension(asdl_seq_GET(t, 0), expression, NULL); + else + lc = comprehension(Tuple(t, Store, LINENO(ch)), expression, NULL); + + if (!lc) { + asdl_seq_free(t); + asdl_seq_free(listcomps); + free_expr(expression); + free_expr(elt); + return NULL; + } + + if (NCH(ch) == 5) { + int j, n_ifs; + asdl_seq *ifs; + + ch = CHILD(ch, 4); + n_ifs = count_list_ifs(ch); + if (n_ifs == -1) { + asdl_seq_free(listcomps); + free_expr(elt); + return NULL; + } + + ifs = asdl_seq_new(n_ifs); + if (!ifs) { + asdl_seq_free(listcomps); + free_expr(elt); + return NULL; + } + + for (j = 0; j < n_ifs; j++) { + REQ(ch, list_iter); + + ch = CHILD(ch, 0); + REQ(ch, list_if); + + asdl_seq_APPEND(ifs, ast_for_expr(c, CHILD(ch, 1))); + if (NCH(ch) == 3) + ch = CHILD(ch, 2); + } + /* on exit, must guarantee that ch is a list_for */ + if (TYPE(ch) == list_iter) + ch = CHILD(ch, 0); + lc->ifs = ifs; + } + asdl_seq_APPEND(listcomps, lc); + } + + return ListComp(elt, listcomps, LINENO(n)); +} + +/* + Count the number of 'for' loops in a generator expression. + + Helper for ast_for_genexp(). +*/ + +static int +count_gen_fors(const node *n) +{ + int n_fors = 0; + node *ch = CHILD(n, 1); + + count_gen_for: + n_fors++; + REQ(ch, gen_for); + if (NCH(ch) == 5) + ch = CHILD(ch, 4); + else + return n_fors; + count_gen_iter: + REQ(ch, gen_iter); + ch = CHILD(ch, 0); + if (TYPE(ch) == gen_for) + goto count_gen_for; + else if (TYPE(ch) == gen_if) { + if (NCH(ch) == 3) { + ch = CHILD(ch, 2); + goto count_gen_iter; + } + else + return n_fors; + } + else { + /* Should never be reached */ + PyErr_SetString(PyExc_Exception, "logic error in count_gen_fors"); + return -1; + } +} + +/* Count the number of 'if' statements in a generator expression. + + Helper for ast_for_genexp(). +*/ + +static int +count_gen_ifs(const node *n) +{ + int n_ifs = 0; + + while (1) { + REQ(n, gen_iter); + if (TYPE(CHILD(n, 0)) == gen_for) + return n_ifs; + n = CHILD(n, 0); + REQ(n, gen_if); + n_ifs++; + if (NCH(n) == 2) + return n_ifs; + n = CHILD(n, 2); + } +} + +static expr_ty +ast_for_genexp(struct compiling *c, const node *n) +{ + /* testlist_gexp: test ( gen_for | (',' test)* [','] ) + argument: [test '='] test [gen_for] # Really [keyword '='] test */ + expr_ty elt; + asdl_seq *genexps; + int i, n_fors; + node *ch; + + assert(TYPE(n) == (testlist_gexp) || TYPE(n) == (argument)); + assert(NCH(n) > 1); + + elt = ast_for_expr(c, CHILD(n, 0)); + if (!elt) + return NULL; + + n_fors = count_gen_fors(n); + if (n_fors == -1) + return NULL; + + genexps = asdl_seq_new(n_fors); + if (!genexps) { + free_expr(elt); + return NULL; + } + + ch = CHILD(n, 1); + for (i = 0; i < n_fors; i++) { + comprehension_ty ge; + asdl_seq *t; + expr_ty expression; + + REQ(ch, gen_for); + + t = ast_for_exprlist(c, CHILD(ch, 1), Store); + if (!t) { + asdl_seq_free(genexps); + free_expr(elt); + return NULL; + } + expression = ast_for_testlist(c, CHILD(ch, 3), 1); + if (!expression) { + asdl_seq_free(genexps); + free_expr(elt); + return NULL; + } + + if (asdl_seq_LEN(t) == 1) + ge = comprehension(asdl_seq_GET(t, 0), expression, + NULL); + else + ge = comprehension(Tuple(t, Store, LINENO(ch)), + expression, NULL); + + if (!ge) { + asdl_seq_free(genexps); + free_expr(elt); + return NULL; + } + + if (NCH(ch) == 5) { + int j, n_ifs; + asdl_seq *ifs; + + ch = CHILD(ch, 4); + n_ifs = count_gen_ifs(ch); + if (n_ifs == -1) { + asdl_seq_free(genexps); + free_expr(elt); + return NULL; + } + + ifs = asdl_seq_new(n_ifs); + if (!ifs) { + asdl_seq_free(genexps); + free_expr(elt); + return NULL; + } + + for (j = 0; j < n_ifs; j++) { + REQ(ch, gen_iter); + ch = CHILD(ch, 0); + REQ(ch, gen_if); + + asdl_seq_APPEND(ifs, ast_for_expr(c, CHILD(ch, 1))); + if (NCH(ch) == 3) + ch = CHILD(ch, 2); + } + /* on exit, must guarantee that ch is a gen_for */ + if (TYPE(ch) == gen_iter) + ch = CHILD(ch, 0); + ge->ifs = ifs; + } + asdl_seq_APPEND(genexps, ge); + } + + return GeneratorExp(elt, genexps, LINENO(n)); +} + +static expr_ty +ast_for_atom(struct compiling *c, const node *n) +{ + /* atom: '(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' + | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+ + */ + node *ch = CHILD(n, 0); + + switch (TYPE(ch)) { + case NAME: + /* All names start in Load context, but may later be + changed. */ + return Name(NEW_IDENTIFIER(ch), Load, LINENO(n)); + case STRING: { + PyObject *str = parsestrplus(c, n); + + if (!str) + return NULL; + + return Str(str, LINENO(n)); + } + case NUMBER: { + PyObject *pynum = parsenumber(STR(ch)); + + if (!pynum) + return NULL; + + return Num(pynum, LINENO(n)); + } + case LPAR: /* some parenthesized expressions */ + ch = CHILD(n, 1); + + if (TYPE(ch) == RPAR) + return Tuple(NULL, Load, LINENO(n)); + + if (TYPE(ch) == yield_expr) + return ast_for_expr(c, ch); + + if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == gen_for)) + return ast_for_genexp(c, ch); + + return ast_for_testlist(c, ch, 1); + case LSQB: /* list (or list comprehension) */ + ch = CHILD(n, 1); + + if (TYPE(ch) == RSQB) + return List(NULL, Load, LINENO(n)); + + REQ(ch, listmaker); + if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { + asdl_seq *elts = seq_for_testlist(c, ch); + + if (!elts) + return NULL; + + return List(elts, Load, LINENO(n)); + } + else + return ast_for_listcomp(c, ch); + case LBRACE: { + /* dictmaker: test ':' test (',' test ':' test)* [','] */ + int i, size; + asdl_seq *keys, *values; + + ch = CHILD(n, 1); + size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */ + keys = asdl_seq_new(size); + if (!keys) + return NULL; + + values = asdl_seq_new(size); + if (!values) { + asdl_seq_free(keys); + return NULL; + } + + for (i = 0; i < NCH(ch); i += 4) { + expr_ty expression; + + expression = ast_for_expr(c, CHILD(ch, i)); + if (!expression) + return NULL; + + asdl_seq_SET(keys, i / 4, expression); + + expression = ast_for_expr(c, CHILD(ch, i + 2)); + if (!expression) + return NULL; + + asdl_seq_SET(values, i / 4, expression); + } + return Dict(keys, values, LINENO(n)); + } + case BACKQUOTE: { /* repr */ + expr_ty expression = ast_for_testlist(c, CHILD(n, 1), 0); + + if (!expression) + return NULL; + + return Repr(expression, LINENO(n)); + } + default: + PyErr_Format(PyExc_Exception, "unhandled atom %d", + TYPE(ch)); + return NULL; + } +} + +static slice_ty +ast_for_slice(struct compiling *c, const node *n) +{ + node *ch; + expr_ty lower = NULL, upper = NULL, step = NULL; + + REQ(n, subscript); + + /* + subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] + sliceop: ':' [test] + */ + ch = CHILD(n, 0); + if (TYPE(ch) == DOT) + return Ellipsis(); + + if (NCH(n) == 1 && TYPE(ch) == test) { + /* 'step' variable hold no significance in terms of being used over + other vars */ + step = ast_for_expr(c, ch); + if (!step) + return NULL; + + return Index(step); + } + + if (TYPE(ch) == test) { + lower = ast_for_expr(c, ch); + if (!lower) + return NULL; + } + + /* If there's an upper bound it's in the second or third position. */ + if (TYPE(ch) == COLON) { + if (NCH(n) > 1) { + node *n2 = CHILD(n, 1); + + if (TYPE(n2) == test) { + upper = ast_for_expr(c, n2); + if (!upper) + return NULL; + } + } + } else if (NCH(n) > 2) { + node *n2 = CHILD(n, 2); + + if (TYPE(n2) == test) { + upper = ast_for_expr(c, n2); + if (!upper) + return NULL; + } + } + + ch = CHILD(n, NCH(n) - 1); + if (TYPE(ch) == sliceop) { + if (NCH(ch) == 1) + /* XXX: If only 1 child, then should just be a colon. Should we + just skip assigning and just get to the return? */ + ch = CHILD(ch, 0); + else + ch = CHILD(ch, 1); + if (TYPE(ch) == test) { + step = ast_for_expr(c, ch); + if (!step) + return NULL; + } + } + + return Slice(lower, upper, step); +} + +static expr_ty +ast_for_binop(struct compiling *c, const node *n) +{ + /* Must account for a sequence of expressions. + How should A op B op C by represented? + BinOp(BinOp(A, op, B), op, C). + */ + + int i, nops; + expr_ty expr1, expr2, result; + operator_ty operator; + + expr1 = ast_for_expr(c, CHILD(n, 0)); + if (!expr1) + return NULL; + + expr2 = ast_for_expr(c, CHILD(n, 2)); + if (!expr2) + return NULL; + + operator = get_operator(CHILD(n, 1)); + if (!operator) + return NULL; + + result = BinOp(expr1, operator, expr2, LINENO(n)); + if (!result) + return NULL; + + nops = (NCH(n) - 1) / 2; + for (i = 1; i < nops; i++) { + expr_ty tmp_result, tmp; + const node* next_oper = CHILD(n, i * 2 + 1); + + operator = get_operator(next_oper); + if (!operator) + return NULL; + + tmp = ast_for_expr(c, CHILD(n, i * 2 + 2)); + if (!tmp) + return NULL; + + tmp_result = BinOp(result, operator, tmp, + LINENO(next_oper)); + if (!tmp) + return NULL; + result = tmp_result; + } + return result; +} + +/* Do not name a variable 'expr'! Will cause a compile error. +*/ + +static expr_ty +ast_for_expr(struct compiling *c, const node *n) +{ + /* handle the full range of simple expressions + test: and_test ('or' and_test)* | lambdef + and_test: not_test ('and' not_test)* + not_test: 'not' not_test | comparison + comparison: expr (comp_op expr)* + expr: xor_expr ('|' xor_expr)* + xor_expr: and_expr ('^' and_expr)* + and_expr: shift_expr ('&' shift_expr)* + shift_expr: arith_expr (('<<'|'>>') arith_expr)* + arith_expr: term (('+'|'-') term)* + term: factor (('*'|'/'|'%'|'//') factor)* + factor: ('+'|'-'|'~') factor | power + power: atom trailer* ('**' factor)* + */ + + asdl_seq *seq; + int i; + + loop: + switch (TYPE(n)) { + case test: + if (TYPE(CHILD(n, 0)) == lambdef) + return ast_for_lambdef(c, CHILD(n, 0)); + /* Fall through to and_test */ + case and_test: + if (NCH(n) == 1) { + n = CHILD(n, 0); + goto loop; + } + seq = asdl_seq_new((NCH(n) + 1) / 2); + if (!seq) + return NULL; + for (i = 0; i < NCH(n); i += 2) { + expr_ty e = ast_for_expr(c, CHILD(n, i)); + if (!e) + return NULL; + asdl_seq_SET(seq, i / 2, e); + } + if (!strcmp(STR(CHILD(n, 1)), "and")) + return BoolOp(And, seq, LINENO(n)); + else { + assert(!strcmp(STR(CHILD(n, 1)), "or")); + return BoolOp(Or, seq, LINENO(n)); + } + break; + case not_test: + if (NCH(n) == 1) { + n = CHILD(n, 0); + goto loop; + } + else { + expr_ty expression = ast_for_expr(c, CHILD(n, 1)); + if (!expression) + return NULL; + + return UnaryOp(Not, expression, LINENO(n)); + } + case comparison: + if (NCH(n) == 1) { + n = CHILD(n, 0); + goto loop; + } + else { + expr_ty expression; + asdl_seq *ops, *cmps; + ops = asdl_seq_new(NCH(n) / 2); + if (!ops) + return NULL; + cmps = asdl_seq_new(NCH(n) / 2); + if (!cmps) { + asdl_seq_free(ops); + return NULL; + } + for (i = 1; i < NCH(n); i += 2) { + /* XXX cmpop_ty is just an enum */ + cmpop_ty operator; + + operator = ast_for_comp_op(CHILD(n, i)); + if (!operator) + return NULL; + + expression = ast_for_expr(c, CHILD(n, i + 1)); + if (!expression) + return NULL; + + asdl_seq_SET(ops, i / 2, (void *)operator); + asdl_seq_SET(cmps, i / 2, expression); + } + expression = ast_for_expr(c, CHILD(n, 0)); + if (!expression) + return NULL; + + return Compare(expression, ops, cmps, LINENO(n)); + } + break; + + /* The next five cases all handle BinOps. The main body of code + is the same in each case, but the switch turned inside out to + reuse the code for each type of operator. + */ + case expr: + case xor_expr: + case and_expr: + case shift_expr: + case arith_expr: + case term: + if (NCH(n) == 1) { + n = CHILD(n, 0); + goto loop; + } + return ast_for_binop(c, n); + case yield_expr: { + expr_ty exp = NULL; + if (NCH(n) == 2) { + exp = ast_for_testlist(c, CHILD(n, 1), 0); + if (!exp) + return NULL; + } + return Yield(exp, LINENO(n)); + } + case factor: { + expr_ty expression; + + if (NCH(n) == 1) { + n = CHILD(n, 0); + goto loop; + } + + expression = ast_for_expr(c, CHILD(n, 1)); + if (!expression) + return NULL; + + switch (TYPE(CHILD(n, 0))) { + case PLUS: + return UnaryOp(UAdd, expression, LINENO(n)); + case MINUS: + return UnaryOp(USub, expression, LINENO(n)); + case TILDE: + return UnaryOp(Invert, expression, LINENO(n)); + } + break; + } + case power: { + expr_ty e = ast_for_atom(c, CHILD(n, 0)); + if (!e) + return NULL; + if (NCH(n) == 1) + return e; + /* power: atom trailer* ('**' factor)* + trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME + + XXX What about atom trailer trailer ** factor? + */ + for (i = 1; i < NCH(n); i++) { + expr_ty new = e; + node *ch = CHILD(n, i); + if (ch->n_str && strcmp(ch->n_str, "**") == 0) + break; + if (TYPE(CHILD(ch, 0)) == LPAR) { + if (NCH(ch) == 2) + new = Call(new, NULL, NULL, NULL, NULL, LINENO(ch)); + else + new = ast_for_call(c, CHILD(ch, 1), new); + + if (!new) { + free_expr(e); + return NULL; + } + } + else if (TYPE(CHILD(ch, 0)) == LSQB) { + REQ(CHILD(ch, 2), RSQB); + ch = CHILD(ch, 1); + if (NCH(ch) <= 2) { + slice_ty slc = ast_for_slice(c, CHILD(ch, 0)); + if (!slc) { + free_expr(e); + return NULL; + } + + new = Subscript(e, slc, Load, LINENO(ch)); + if (!new) { + free_expr(e); + free_slice(slc); + return NULL; + } + } + else { + int j; + slice_ty slc; + asdl_seq *slices = asdl_seq_new((NCH(ch) + 1) / 2); + if (!slices) { + free_expr(e); + return NULL; + } + + for (j = 0; j < NCH(ch); j += 2) { + slc = ast_for_slice(c, CHILD(ch, j)); + if (!slc) { + free_expr(e); + asdl_seq_free(slices); + return NULL; + } + asdl_seq_SET(slices, j / 2, slc); + } + new = Subscript(e, ExtSlice(slices), Load, LINENO(ch)); + if (!new) { + free_expr(e); + asdl_seq_free(slices); + return NULL; + } + } + } + else { + assert(TYPE(CHILD(ch, 0)) == DOT); + new = Attribute(e, NEW_IDENTIFIER(CHILD(ch, 1)), Load, + LINENO(ch)); + if (!new) { + free_expr(e); + return NULL; + } + } + e = new; + } + if (TYPE(CHILD(n, NCH(n) - 1)) == factor) { + expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1)); + if (!f) { + free_expr(e); + return NULL; + } + return BinOp(e, Pow, f, LINENO(n)); + } + return e; + } + default: + abort(); + PyErr_Format(PyExc_Exception, "unhandled expr: %d", TYPE(n)); + return NULL; + } + /* should never get here */ + return NULL; +} + +static expr_ty +ast_for_call(struct compiling *c, const node *n, expr_ty func) +{ + /* + arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] + | '**' test) + argument: [test '='] test [gen_for] # Really [keyword '='] test + */ + + int i, nargs, nkeywords, ngens; + asdl_seq *args = NULL; + asdl_seq *keywords = NULL; + expr_ty vararg = NULL, kwarg = NULL; + + REQ(n, arglist); + + nargs = 0; + nkeywords = 0; + ngens = 0; + for (i = 0; i < NCH(n); i++) { + node *ch = CHILD(n, i); + if (TYPE(ch) == argument) { + if (NCH(ch) == 1) + nargs++; + else if (TYPE(CHILD(ch, 1)) == gen_for) + ngens++; + else + nkeywords++; + } + } + if (ngens > 1 || (ngens && (nargs || nkeywords))) { + ast_error(n, "Generator expression must be parenthesised " + "if not sole argument"); + return NULL; + } + + if (nargs + nkeywords + ngens > 255) { + ast_error(n, "more than 255 arguments"); + return NULL; + } + + args = asdl_seq_new(nargs + ngens); + if (!args) + goto error; + keywords = asdl_seq_new(nkeywords); + if (!keywords) + goto error; + nargs = 0; + nkeywords = 0; + for (i = 0; i < NCH(n); i++) { + node *ch = CHILD(n, i); + if (TYPE(ch) == argument) { + expr_ty e; + if (NCH(ch) == 1) { + e = ast_for_expr(c, CHILD(ch, 0)); + if (!e) + goto error; + asdl_seq_SET(args, nargs++, e); + } + else if (TYPE(CHILD(ch, 1)) == gen_for) { + e = ast_for_genexp(c, ch); + if (!e) + goto error; + asdl_seq_SET(args, nargs++, e); + } + else { + keyword_ty kw; + identifier key; + + /* CHILD(ch, 0) is test, but must be an identifier? */ + e = ast_for_expr(c, CHILD(ch, 0)); + if (!e) + goto error; + /* f(lambda x: x[0] = 3) ends up getting parsed with + * LHS test = lambda x: x[0], and RHS test = 3. + * SF bug 132313 points out that complaining about a keyword + * then is very confusing. + */ + if (e->kind == Lambda_kind) { + ast_error(CHILD(ch, 0), "lambda cannot contain assignment"); + goto error; + } else if (e->kind != Name_kind) { + ast_error(CHILD(ch, 0), "keyword can't be an expression"); + goto error; + } + key = e->v.Name.id; + free(e); + e = ast_for_expr(c, CHILD(ch, 2)); + if (!e) + goto error; + kw = keyword(key, e); + if (!kw) + goto error; + asdl_seq_SET(keywords, nkeywords++, kw); + } + } + else if (TYPE(ch) == STAR) { + vararg = ast_for_expr(c, CHILD(n, i+1)); + i++; + } + else if (TYPE(ch) == DOUBLESTAR) { + kwarg = ast_for_expr(c, CHILD(n, i+1)); + i++; + } + } + + return Call(func, args, keywords, vararg, kwarg, LINENO(n)); + + error: + if (args) + asdl_seq_free(args); + if (keywords) + asdl_seq_free(keywords); + return NULL; +} + +/* Unlike other ast_for_XXX() functions, this takes a flag that + indicates whether generator expressions are allowed. If gexp is + non-zero, check for testlist_gexp instead of plain testlist. +*/ + +static expr_ty +ast_for_testlist(struct compiling *c, const node* n, int gexp) +{ + /* testlist_gexp: test ( gen_for | (',' test)* [','] ) + testlist: test (',' test)* [','] + */ + + assert(NCH(n) > 0); + if (NCH(n) == 1) + return ast_for_expr(c, CHILD(n, 0)); + if (TYPE(CHILD(n, 1)) == gen_for) { + if (!gexp) { + ast_error(n, "illegal generator expression"); + return NULL; + } + return ast_for_genexp(c, n); + } + else { + asdl_seq *tmp = seq_for_testlist(c, n); + if (!tmp) + return NULL; + + return Tuple(tmp, Load, LINENO(n)); + } + return NULL; /* unreachable */ +} + +static stmt_ty +ast_for_expr_stmt(struct compiling *c, const node *n) +{ + REQ(n, expr_stmt); + /* expr_stmt: testlist (augassign (yield_expr|testlist) + | ('=' (yield_expr|testlist))*) + testlist: test (',' test)* [','] + augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' + | '<<=' | '>>=' | '**=' | '//=' + test: ... here starts the operator precendence dance + */ + + if (NCH(n) == 1) { + expr_ty e = ast_for_testlist(c, CHILD(n, 0), 0); + if (!e) + return NULL; + + return Expr(e, LINENO(n)); + } + else if (TYPE(CHILD(n, 1)) == augassign) { + expr_ty expr1, expr2; + operator_ty operator; + node *ch = CHILD(n, 0); + + if (TYPE(ch) == testlist) + expr1 = ast_for_testlist(c, ch, 0); + else + expr1 = Yield(ast_for_expr(c, CHILD(ch, 0)), LINENO(ch)); + + if (!expr1) + return NULL; + if (expr1->kind == GeneratorExp_kind) { + ast_error(ch, "augmented assignment to generator " + "expression not possible"); + return NULL; + } + if (expr1->kind == Name_kind) { + char *var_name = PyString_AS_STRING(expr1->v.Name.id); + if (var_name[0] == 'N' && !strcmp(var_name, "None")) { + ast_error(ch, "assignment to None"); + return NULL; + } + } + + ch = CHILD(n, 2); + if (TYPE(ch) == testlist) + expr2 = ast_for_testlist(c, ch, 0); + else + expr2 = Yield(ast_for_expr(c, ch), LINENO(ch)); + if (!expr2) + return NULL; + + operator = ast_for_augassign(CHILD(n, 1)); + if (!operator) + return NULL; + + return AugAssign(expr1, operator, expr2, LINENO(n)); + } + else { + int i; + asdl_seq *targets; + node *value; + expr_ty expression; + + /* a normal assignment */ + REQ(CHILD(n, 1), EQUAL); + targets = asdl_seq_new(NCH(n) / 2); + if (!targets) + return NULL; + for (i = 0; i < NCH(n) - 2; i += 2) { + node *ch = CHILD(n, i); + if (TYPE(ch) == yield_expr) { + ast_error(ch, "assignment to yield expression not possible"); + goto error; + } + expr_ty e = ast_for_testlist(c, ch, 0); + + /* set context to assign */ + if (!e) + goto error; + + if (!set_context(e, Store, CHILD(n, i))) { + free_expr(e); + goto error; + } + + asdl_seq_SET(targets, i / 2, e); + } + value = CHILD(n, NCH(n) - 1); + if (TYPE(value) == testlist) + expression = ast_for_testlist(c, value, 0); + else + expression = ast_for_expr(c, value); + if (!expression) + return NULL; + return Assign(targets, expression, LINENO(n)); + error: + for (i = i / 2; i >= 0; i--) + free_expr((expr_ty)asdl_seq_GET(targets, i)); + asdl_seq_free(targets); + return NULL; + } + return NULL; +} + +static stmt_ty +ast_for_print_stmt(struct compiling *c, const node *n) +{ + /* print_stmt: 'print' ( [ test (',' test)* [','] ] + | '>>' test [ (',' test)+ [','] ] ) + */ + expr_ty dest = NULL, expression; + asdl_seq *seq; + bool nl; + int i, start = 1; + + REQ(n, print_stmt); + if (NCH(n) >= 2 && TYPE(CHILD(n, 1)) == RIGHTSHIFT) { + dest = ast_for_expr(c, CHILD(n, 2)); + if (!dest) + return NULL; + start = 4; + } + seq = asdl_seq_new((NCH(n) + 1 - start) / 2); + if (!seq) + return NULL; + for (i = start; i < NCH(n); i += 2) { + expression = ast_for_expr(c, CHILD(n, i)); + if (!expression) { + asdl_seq_free(seq); + return NULL; + } + + asdl_seq_APPEND(seq, expression); + } + nl = (TYPE(CHILD(n, NCH(n) - 1)) == COMMA) ? false : true; + return Print(dest, seq, nl, LINENO(n)); +} + +static asdl_seq * +ast_for_exprlist(struct compiling *c, const node *n, int context) +{ + asdl_seq *seq; + int i; + expr_ty e; + + REQ(n, exprlist); + + seq = asdl_seq_new((NCH(n) + 1) / 2); + if (!seq) + return NULL; + for (i = 0; i < NCH(n); i += 2) { + e = ast_for_expr(c, CHILD(n, i)); + if (!e) { + asdl_seq_free(seq); + return NULL; + } + if (context) { + if (!set_context(e, context, CHILD(n, i))) + return NULL; + } + asdl_seq_SET(seq, i / 2, e); + } + return seq; +} + +static stmt_ty +ast_for_del_stmt(struct compiling *c, const node *n) +{ + asdl_seq *expr_list; + + /* del_stmt: 'del' exprlist */ + REQ(n, del_stmt); + + expr_list = ast_for_exprlist(c, CHILD(n, 1), Del); + if (!expr_list) + return NULL; + return Delete(expr_list, LINENO(n)); +} + +static stmt_ty +ast_for_flow_stmt(struct compiling *c, const node *n) +{ + /* + flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt + | yield_stmt + break_stmt: 'break' + continue_stmt: 'continue' + return_stmt: 'return' [testlist] + yield_stmt: yield_expr + yield_expr: 'yield' testlist + raise_stmt: 'raise' [test [',' test [',' test]]] + */ + node *ch; + + REQ(n, flow_stmt); + ch = CHILD(n, 0); + switch (TYPE(ch)) { + case break_stmt: + return Break(LINENO(n)); + case continue_stmt: + return Continue(LINENO(n)); + case yield_stmt: { /* will reduce to yield_expr */ + expr_ty exp = ast_for_expr(c, CHILD(ch, 0)); + if (!exp) + return NULL; + return Expr(exp, LINENO(n)); + } + case return_stmt: + if (NCH(ch) == 1) + return Return(NULL, LINENO(n)); + else { + expr_ty expression = ast_for_testlist(c, CHILD(ch, 1), 0); + if (!expression) + return NULL; + return Return(expression, LINENO(n)); + } + case raise_stmt: + if (NCH(ch) == 1) + return Raise(NULL, NULL, NULL, LINENO(n)); + else if (NCH(ch) == 2) { + expr_ty expression = ast_for_expr(c, CHILD(ch, 1)); + if (!expression) + return NULL; + return Raise(expression, NULL, NULL, LINENO(n)); + } + else if (NCH(ch) == 4) { + expr_ty expr1, expr2; + + expr1 = ast_for_expr(c, CHILD(ch, 1)); + if (!expr1) + return NULL; + expr2 = ast_for_expr(c, CHILD(ch, 3)); + if (!expr2) + return NULL; + + return Raise(expr1, expr2, NULL, LINENO(n)); + } + else if (NCH(ch) == 6) { + expr_ty expr1, expr2, expr3; + + expr1 = ast_for_expr(c, CHILD(ch, 1)); + if (!expr1) + return NULL; + expr2 = ast_for_expr(c, CHILD(ch, 3)); + if (!expr2) + return NULL; + expr3 = ast_for_expr(c, CHILD(ch, 5)); + if (!expr3) + return NULL; + + return Raise(expr1, expr2, expr3, LINENO(n)); + } + default: + PyErr_Format(PyExc_Exception, + "unexpected flow_stmt: %d", TYPE(ch)); + return NULL; + } +} + +static alias_ty +alias_for_import_name(const node *n) +{ + /* + import_as_name: NAME [NAME NAME] + dotted_as_name: dotted_name [NAME NAME] + dotted_name: NAME ('.' NAME)* + */ + loop: + switch (TYPE(n)) { + case import_as_name: + if (NCH(n) == 3) + return alias(NEW_IDENTIFIER(CHILD(n, 0)), + NEW_IDENTIFIER(CHILD(n, 2))); + else + return alias(NEW_IDENTIFIER(CHILD(n, 0)), + NULL); + break; + case dotted_as_name: + if (NCH(n) == 1) { + n = CHILD(n, 0); + goto loop; + } + else { + alias_ty a = alias_for_import_name(CHILD(n, 0)); + assert(!a->asname); + a->asname = NEW_IDENTIFIER(CHILD(n, 2)); + return a; + } + break; + case dotted_name: + if (NCH(n) == 1) + return alias(NEW_IDENTIFIER(CHILD(n, 0)), NULL); + else { + /* Create a string of the form "a.b.c" */ + int i, len; + PyObject *str; + char *s; + + len = 0; + for (i = 0; i < NCH(n); i += 2) + /* length of string plus one for the dot */ + len += strlen(STR(CHILD(n, i))) + 1; + len--; /* the last name doesn't have a dot */ + str = PyString_FromStringAndSize(NULL, len); + if (!str) + return NULL; + s = PyString_AS_STRING(str); + if (!s) + return NULL; + for (i = 0; i < NCH(n); i += 2) { + char *sch = STR(CHILD(n, i)); + strcpy(s, STR(CHILD(n, i))); + s += strlen(sch); + *s++ = '.'; + } + --s; + *s = '\0'; + PyString_InternInPlace(&str); + return alias(str, NULL); + } + break; + case STAR: + return alias(PyString_InternFromString("*"), NULL); + default: + PyErr_Format(PyExc_Exception, + "unexpected import name: %d", TYPE(n)); + return NULL; + } + return NULL; +} + +static stmt_ty +ast_for_import_stmt(struct compiling *c, const node *n) +{ + /* + import_stmt: import_name | import_from + import_name: 'import' dotted_as_names + import_from: 'from' dotted_name 'import' ('*' | + '(' import_as_names ')' | + import_as_names) + */ + int i; + asdl_seq *aliases; + + REQ(n, import_stmt); + n = CHILD(n, 0); + if (STR(CHILD(n, 0))[0] == 'i') { /* import */ + n = CHILD(n, 1); + aliases = asdl_seq_new((NCH(n) + 1) / 2); + if (!aliases) + return NULL; + for (i = 0; i < NCH(n); i += 2) { + alias_ty import_alias = alias_for_import_name(CHILD(n, i)); + if (!import_alias) { + asdl_seq_free(aliases); + return NULL; + } + asdl_seq_SET(aliases, i / 2, import_alias); + } + return Import(aliases, LINENO(n)); + } + else if (STR(CHILD(n, 0))[0] == 'f') { /* from */ + stmt_ty import; + int n_children; + const char *from_modules; + int lineno = LINENO(n); + alias_ty mod = alias_for_import_name(CHILD(n, 1)); + if (!mod) + return NULL; + + /* XXX this needs to be cleaned up */ + + from_modules = STR(CHILD(n, 3)); + if (!from_modules) { + n = CHILD(n, 3); /* from ... import x, y, z */ + if (NCH(n) % 2 == 0) { + /* it ends with a comma, not valid but the parser allows it */ + ast_error(n, "trailing comma not allowed without" + " surrounding parentheses"); + return NULL; + } + } + else if (from_modules[0] == '*') { + n = CHILD(n, 3); /* from ... import * */ + } + else if (from_modules[0] == '(') + n = CHILD(n, 4); /* from ... import (x, y, z) */ + else + return NULL; + + n_children = NCH(n); + if (from_modules && from_modules[0] == '*') + n_children = 1; + + aliases = asdl_seq_new((n_children + 1) / 2); + if (!aliases) { + free_alias(mod); + return NULL; + } + + /* handle "from ... import *" special b/c there's no children */ + if (from_modules && from_modules[0] == '*') { + alias_ty import_alias = alias_for_import_name(n); + if (!import_alias) { + asdl_seq_free(aliases); + free_alias(mod); + return NULL; + } + asdl_seq_APPEND(aliases, import_alias); + } + + for (i = 0; i < NCH(n); i += 2) { + alias_ty import_alias = alias_for_import_name(CHILD(n, i)); + if (!import_alias) { + asdl_seq_free(aliases); + free_alias(mod); + return NULL; + } + asdl_seq_APPEND(aliases, import_alias); + } + Py_INCREF(mod->name); + import = ImportFrom(mod->name, aliases, lineno); + free_alias(mod); + return import; + } + PyErr_Format(PyExc_Exception, + "unknown import statement: starts with command '%s'", + STR(CHILD(n, 0))); + return NULL; +} + +static stmt_ty +ast_for_global_stmt(struct compiling *c, const node *n) +{ + /* global_stmt: 'global' NAME (',' NAME)* */ + identifier name; + asdl_seq *s; + int i; + + REQ(n, global_stmt); + s = asdl_seq_new(NCH(n) / 2); + if (!s) + return NULL; + for (i = 1; i < NCH(n); i += 2) { + name = NEW_IDENTIFIER(CHILD(n, i)); + if (!name) { + asdl_seq_free(s); + return NULL; + } + asdl_seq_SET(s, i / 2, name); + } + return Global(s, LINENO(n)); +} + +static stmt_ty +ast_for_exec_stmt(struct compiling *c, const node *n) +{ + expr_ty expr1, globals = NULL, locals = NULL; + int n_children = NCH(n); + if (n_children != 2 && n_children != 4 && n_children != 6) { + PyErr_Format(PyExc_Exception, + "poorly formed 'exec' statement: %d parts to statement", + n_children); + return NULL; + } + + /* exec_stmt: 'exec' expr ['in' test [',' test]] */ + REQ(n, exec_stmt); + expr1 = ast_for_expr(c, CHILD(n, 1)); + if (!expr1) + return NULL; + if (n_children >= 4) { + globals = ast_for_expr(c, CHILD(n, 3)); + if (!globals) + return NULL; + } + if (n_children == 6) { + locals = ast_for_expr(c, CHILD(n, 5)); + if (!locals) + return NULL; + } + + return Exec(expr1, globals, locals, LINENO(n)); +} + +static stmt_ty +ast_for_assert_stmt(struct compiling *c, const node *n) +{ + /* assert_stmt: 'assert' test [',' test] */ + REQ(n, assert_stmt); + if (NCH(n) == 2) { + expr_ty expression = ast_for_expr(c, CHILD(n, 1)); + if (!expression) + return NULL; + return Assert(expression, NULL, LINENO(n)); + } + else if (NCH(n) == 4) { + expr_ty expr1, expr2; + + expr1 = ast_for_expr(c, CHILD(n, 1)); + if (!expr1) + return NULL; + expr2 = ast_for_expr(c, CHILD(n, 3)); + if (!expr2) + return NULL; + + return Assert(expr1, expr2, LINENO(n)); + } + PyErr_Format(PyExc_Exception, + "improper number of parts to 'assert' statement: %d", + NCH(n)); + return NULL; +} + +static asdl_seq * +ast_for_suite(struct compiling *c, const node *n) +{ + /* suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT */ + asdl_seq *seq = NULL; + stmt_ty s; + int i, total, num, end, pos = 0; + node *ch; + + REQ(n, suite); + + total = num_stmts(n); + seq = asdl_seq_new(total); + if (!seq) + return NULL; + if (TYPE(CHILD(n, 0)) == simple_stmt) { + n = CHILD(n, 0); + /* simple_stmt always ends with a NEWLINE, + and may have a trailing SEMI + */ + end = NCH(n) - 1; + if (TYPE(CHILD(n, end - 1)) == SEMI) + end--; + /* loop by 2 to skip semi-colons */ + for (i = 0; i < end; i += 2) { + ch = CHILD(n, i); + s = ast_for_stmt(c, ch); + if (!s) + goto error; + asdl_seq_SET(seq, pos++, s); + } + } + else { + for (i = 2; i < (NCH(n) - 1); i++) { + ch = CHILD(n, i); + REQ(ch, stmt); + num = num_stmts(ch); + if (num == 1) { + /* small_stmt or compound_stmt with only one child */ + s = ast_for_stmt(c, ch); + if (!s) + goto error; + asdl_seq_SET(seq, pos++, s); + } + else { + int j; + ch = CHILD(ch, 0); + REQ(ch, simple_stmt); + for (j = 0; j < NCH(ch); j += 2) { + s = ast_for_stmt(c, CHILD(ch, j)); + if (!s) + goto error; + asdl_seq_SET(seq, pos++, s); + } + } + } + } + assert(pos == seq->size); + return seq; + error: + if (seq) + asdl_seq_free(seq); + return NULL; +} + +static stmt_ty +ast_for_if_stmt(struct compiling *c, const node *n) +{ + /* if_stmt: 'if' test ':' suite ('elif' test ':' suite)* + ['else' ':' suite] + */ + char *s; + + REQ(n, if_stmt); + + if (NCH(n) == 4) { + expr_ty expression; + asdl_seq *suite_seq; + + expression = ast_for_expr(c, CHILD(n, 1)); + if (!expression) + return NULL; + suite_seq = ast_for_suite(c, CHILD(n, 3)); + if (!suite_seq) + return NULL; + + return If(expression, suite_seq, NULL, LINENO(n)); + } + s = STR(CHILD(n, 4)); + /* s[2], the third character in the string, will be + 's' for el_s_e, or + 'i' for el_i_f + */ + if (s[2] == 's') { + expr_ty expression; + asdl_seq *seq1, *seq2; + + expression = ast_for_expr(c, CHILD(n, 1)); + if (!expression) + return NULL; + seq1 = ast_for_suite(c, CHILD(n, 3)); + if (!seq1) + return NULL; + seq2 = ast_for_suite(c, CHILD(n, 6)); + if (!seq2) + return NULL; + + return If(expression, seq1, seq2, LINENO(n)); + } + else if (s[2] == 'i') { + int i, n_elif, has_else = 0; + asdl_seq *orelse = NULL; + n_elif = NCH(n) - 4; + /* must reference the child n_elif+1 since 'else' token is third, + not fourth, child from the end. */ + if (TYPE(CHILD(n, (n_elif + 1))) == NAME + && STR(CHILD(n, (n_elif + 1)))[2] == 's') { + has_else = 1; + n_elif -= 3; + } + n_elif /= 4; + + if (has_else) { + expr_ty expression; + asdl_seq *seq1, *seq2; + + orelse = asdl_seq_new(1); + if (!orelse) + return NULL; + expression = ast_for_expr(c, CHILD(n, NCH(n) - 6)); + if (!expression) { + asdl_seq_free(orelse); + return NULL; + } + seq1 = ast_for_suite(c, CHILD(n, NCH(n) - 4)); + if (!seq1) { + asdl_seq_free(orelse); + return NULL; + } + seq2 = ast_for_suite(c, CHILD(n, NCH(n) - 1)); + if (!seq2) { + asdl_seq_free(orelse); + return NULL; + } + + asdl_seq_SET(orelse, 0, If(expression, seq1, seq2, + LINENO(CHILD(n, NCH(n) - 6)))); + /* the just-created orelse handled the last elif */ + n_elif--; + } + else + orelse = NULL; + + for (i = 0; i < n_elif; i++) { + int off = 5 + (n_elif - i - 1) * 4; + expr_ty expression; + asdl_seq *suite_seq; + asdl_seq *new = asdl_seq_new(1); + if (!new) + return NULL; + expression = ast_for_expr(c, CHILD(n, off)); + if (!expression) { + asdl_seq_free(new); + return NULL; + } + suite_seq = ast_for_suite(c, CHILD(n, off + 2)); + if (!suite_seq) { + asdl_seq_free(new); + return NULL; + } + + asdl_seq_SET(new, 0, + If(expression, suite_seq, orelse, + LINENO(CHILD(n, off)))); + orelse = new; + } + return If(ast_for_expr(c, CHILD(n, 1)), + ast_for_suite(c, CHILD(n, 3)), + orelse, LINENO(n)); + } + else { + PyErr_Format(PyExc_Exception, + "unexpected token in 'if' statement: %s", s); + return NULL; + } +} + +static stmt_ty +ast_for_while_stmt(struct compiling *c, const node *n) +{ + /* while_stmt: 'while' test ':' suite ['else' ':' suite] */ + REQ(n, while_stmt); + + if (NCH(n) == 4) { + expr_ty expression; + asdl_seq *suite_seq; + + expression = ast_for_expr(c, CHILD(n, 1)); + if (!expression) + return NULL; + suite_seq = ast_for_suite(c, CHILD(n, 3)); + if (!suite_seq) + return NULL; + return While(expression, suite_seq, NULL, LINENO(n)); + } + else if (NCH(n) == 7) { + expr_ty expression; + asdl_seq *seq1, *seq2; + + expression = ast_for_expr(c, CHILD(n, 1)); + if (!expression) + return NULL; + seq1 = ast_for_suite(c, CHILD(n, 3)); + if (!seq1) + return NULL; + seq2 = ast_for_suite(c, CHILD(n, 6)); + if (!seq2) + return NULL; + + return While(expression, seq1, seq2, LINENO(n)); + } + else { + PyErr_Format(PyExc_Exception, + "wrong number of tokens for 'while' statement: %d", + NCH(n)); + return NULL; + } +} + +static stmt_ty +ast_for_for_stmt(struct compiling *c, const node *n) +{ + asdl_seq *_target = NULL, *seq = NULL, *suite_seq = NULL; + expr_ty expression; + expr_ty target; + /* for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] */ + REQ(n, for_stmt); + + if (NCH(n) == 9) { + seq = ast_for_suite(c, CHILD(n, 8)); + if (!seq) + return NULL; + } + + _target = ast_for_exprlist(c, CHILD(n, 1), Store); + if (!_target) + return NULL; + if (asdl_seq_LEN(_target) == 1) { + target = asdl_seq_GET(_target, 0); + asdl_seq_free(_target); + } + else + target = Tuple(_target, Store, LINENO(n)); + + expression = ast_for_testlist(c, CHILD(n, 3), 0); + if (!expression) + return NULL; + suite_seq = ast_for_suite(c, CHILD(n, 5)); + if (!suite_seq) + return NULL; + + return For(target, expression, suite_seq, seq, LINENO(n)); +} + +static excepthandler_ty +ast_for_except_clause(struct compiling *c, const node *exc, node *body) +{ + /* except_clause: 'except' [test [',' test]] */ + REQ(exc, except_clause); + REQ(body, suite); + + if (NCH(exc) == 1) { + asdl_seq *suite_seq = ast_for_suite(c, body); + if (!suite_seq) + return NULL; + + return excepthandler(NULL, NULL, suite_seq); + } + else if (NCH(exc) == 2) { + expr_ty expression; + asdl_seq *suite_seq; + + expression = ast_for_expr(c, CHILD(exc, 1)); + if (!expression) + return NULL; + suite_seq = ast_for_suite(c, body); + if (!suite_seq) + return NULL; + + return excepthandler(expression, NULL, suite_seq); + } + else if (NCH(exc) == 4) { + asdl_seq *suite_seq; + expr_ty expression; + expr_ty e = ast_for_expr(c, CHILD(exc, 3)); + if (!e) + return NULL; + if (!set_context(e, Store, CHILD(exc, 3))) + return NULL; + expression = ast_for_expr(c, CHILD(exc, 1)); + if (!expression) + return NULL; + suite_seq = ast_for_suite(c, body); + if (!suite_seq) + return NULL; + + return excepthandler(expression, e, suite_seq); + } + else { + PyErr_Format(PyExc_Exception, + "wrong number of children for 'except' clause: %d", + NCH(exc)); + return NULL; + } +} + +static stmt_ty +ast_for_try_stmt(struct compiling *c, const node *n) +{ + REQ(n, try_stmt); + + if (TYPE(CHILD(n, 3)) == NAME) {/* must be 'finally' */ + /* try_stmt: 'try' ':' suite 'finally' ':' suite) */ + asdl_seq *s1, *s2; + s1 = ast_for_suite(c, CHILD(n, 2)); + if (!s1) + return NULL; + s2 = ast_for_suite(c, CHILD(n, 5)); + if (!s2) + return NULL; + + return TryFinally(s1, s2, LINENO(n)); + } + else if (TYPE(CHILD(n, 3)) == except_clause) { + /* try_stmt: ('try' ':' suite (except_clause ':' suite)+ + ['else' ':' suite] + */ + asdl_seq *suite_seq1, *suite_seq2; + asdl_seq *handlers; + int i, has_else = 0, n_except = NCH(n) - 3; + if (TYPE(CHILD(n, NCH(n) - 3)) == NAME) { + has_else = 1; + n_except -= 3; + } + n_except /= 3; + handlers = asdl_seq_new(n_except); + if (!handlers) + return NULL; + for (i = 0; i < n_except; i++) { + excepthandler_ty e = ast_for_except_clause(c, + CHILD(n, 3 + i * 3), + CHILD(n, 5 + i * 3)); + if (!e) + return NULL; + asdl_seq_SET(handlers, i, e); + } + + suite_seq1 = ast_for_suite(c, CHILD(n, 2)); + if (!suite_seq1) + return NULL; + if (has_else) { + suite_seq2 = ast_for_suite(c, CHILD(n, NCH(n) - 1)); + if (!suite_seq2) + return NULL; + } + else + suite_seq2 = NULL; + + return TryExcept(suite_seq1, handlers, suite_seq2, LINENO(n)); + } + else { + PyErr_SetString(PyExc_Exception, "malformed 'try' statement"); + return NULL; + } +} + +static stmt_ty +ast_for_classdef(struct compiling *c, const node *n) +{ + /* classdef: 'class' NAME ['(' testlist ')'] ':' suite */ + expr_ty _bases; + asdl_seq *bases, *s; + + REQ(n, classdef); + + if (!strcmp(STR(CHILD(n, 1)), "None")) { + ast_error(n, "assignment to None"); + return NULL; + } + + if (NCH(n) == 4) { + s = ast_for_suite(c, CHILD(n, 3)); + if (!s) + return NULL; + return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n)); + } + /* check for empty base list */ + if (TYPE(CHILD(n,3)) == RPAR) { + s = ast_for_suite(c, CHILD(n,5)); + if (!s) + return NULL; + return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n)); + } + + /* else handle the base class list */ + _bases = ast_for_testlist(c, CHILD(n, 3), 0); + if (!_bases) + return NULL; + /* XXX: I don't think we can set to diff types here, how to free??? + + Here's the allocation chain: + Tuple (Python-ast.c:907) + ast_for_testlist (ast.c:1782) + ast_for_classdef (ast.c:2677) + */ + if (_bases->kind == Tuple_kind) + bases = _bases->v.Tuple.elts; + else { + bases = asdl_seq_new(1); + if (!bases) { + free_expr(_bases); + /* XXX: free _bases */ + return NULL; + } + asdl_seq_SET(bases, 0, _bases); + } + + s = ast_for_suite(c, CHILD(n, 6)); + if (!s) { + /* XXX: I think this free is correct, but needs to change see above */ + if (_bases->kind == Tuple_kind) + free_expr(_bases); + else { + free_expr(_bases); + asdl_seq_free(bases); + } + return NULL; + } + return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases, s, LINENO(n)); +} + +static stmt_ty +ast_for_stmt(struct compiling *c, const node *n) +{ + if (TYPE(n) == stmt) { + assert(NCH(n) == 1); + n = CHILD(n, 0); + } + if (TYPE(n) == simple_stmt) { + assert(num_stmts(n) == 1); + n = CHILD(n, 0); + } + if (TYPE(n) == small_stmt) { + REQ(n, small_stmt); + n = CHILD(n, 0); + /* small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt + | flow_stmt | import_stmt | global_stmt | exec_stmt + | assert_stmt + */ + switch (TYPE(n)) { + case expr_stmt: + return ast_for_expr_stmt(c, n); + case print_stmt: + return ast_for_print_stmt(c, n); + case del_stmt: + return ast_for_del_stmt(c, n); + case pass_stmt: + return Pass(LINENO(n)); + case flow_stmt: + return ast_for_flow_stmt(c, n); + case import_stmt: + return ast_for_import_stmt(c, n); + case global_stmt: + return ast_for_global_stmt(c, n); + case exec_stmt: + return ast_for_exec_stmt(c, n); + case assert_stmt: + return ast_for_assert_stmt(c, n); + default: + PyErr_Format(PyExc_Exception, + "unhandled small_stmt: TYPE=%d NCH=%d\n", + TYPE(n), NCH(n)); + return NULL; + } + } + else { + /* compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt + | funcdef | classdef + */ + node *ch = CHILD(n, 0); + REQ(n, compound_stmt); + switch (TYPE(ch)) { + case if_stmt: + return ast_for_if_stmt(c, ch); + case while_stmt: + return ast_for_while_stmt(c, ch); + case for_stmt: + return ast_for_for_stmt(c, ch); + case try_stmt: + return ast_for_try_stmt(c, ch); + case funcdef: + return ast_for_funcdef(c, ch); + case classdef: + return ast_for_classdef(c, ch); + default: + PyErr_Format(PyExc_Exception, + "unhandled small_stmt: TYPE=%d NCH=%d\n", + TYPE(n), NCH(n)); + return NULL; + } + } +} + +static PyObject * +parsenumber(const char *s) +{ + const char *end; + long x; + double dx; +#ifndef WITHOUT_COMPLEX + Py_complex c; + int imflag; +#endif + + errno = 0; + end = s + strlen(s) - 1; +#ifndef WITHOUT_COMPLEX + imflag = *end == 'j' || *end == 'J'; +#endif + if (*end == 'l' || *end == 'L') + return PyLong_FromString((char *)s, (char **)0, 0); + if (s[0] == '0') { + x = (long) PyOS_strtoul((char *)s, (char **)&end, 0); + if (x < 0 && errno == 0) { + return PyLong_FromString((char *)s, + (char **)0, + 0); + } + } + else + x = PyOS_strtol((char *)s, (char **)&end, 0); + if (*end == '\0') { + if (errno != 0) + return PyLong_FromString((char *)s, (char **)0, 0); + return PyInt_FromLong(x); + } + /* XXX Huge floats may silently fail */ +#ifndef WITHOUT_COMPLEX + if (imflag) { + c.real = 0.; + PyFPE_START_PROTECT("atof", return 0) + c.imag = atof(s); + PyFPE_END_PROTECT(c) + return PyComplex_FromCComplex(c); + } + else +#endif + { + PyFPE_START_PROTECT("atof", return 0) + dx = atof(s); + PyFPE_END_PROTECT(dx) + return PyFloat_FromDouble(dx); + } +} + +static PyObject * +decode_utf8(const char **sPtr, const char *end, char* encoding) +{ +#ifndef Py_USING_UNICODE + Py_FatalError("decode_utf8 should not be called in this build."); + return NULL; +#else + PyObject *u, *v; + char *s, *t; + t = s = (char *)*sPtr; + /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */ + while (s < end && (*s & 0x80)) s++; + *sPtr = s; + u = PyUnicode_DecodeUTF8(t, s - t, NULL); + if (u == NULL) + return NULL; + v = PyUnicode_AsEncodedString(u, encoding, NULL); + Py_DECREF(u); + return v; +#endif +} + +static PyObject * +decode_unicode(const char *s, size_t len, int rawmode, const char *encoding) +{ + PyObject *v, *u; + char *buf; + char *p; + const char *end; + if (encoding == NULL) { + buf = (char *)s; + u = NULL; + } else if (strcmp(encoding, "iso-8859-1") == 0) { + buf = (char *)s; + u = NULL; + } else { + /* "\XX" may become "\u005c\uHHLL" (12 bytes) */ + u = PyString_FromStringAndSize((char *)NULL, len * 4); + if (u == NULL) + return NULL; + p = buf = PyString_AsString(u); + end = s + len; + while (s < end) { + if (*s == '\\') { + *p++ = *s++; + if (*s & 0x80) { + strcpy(p, "u005c"); + p += 5; + } + } + if (*s & 0x80) { /* XXX inefficient */ + PyObject *w; + char *r; + int rn, i; + w = decode_utf8(&s, end, "utf-16-be"); + if (w == NULL) { + Py_DECREF(u); + return NULL; + } + r = PyString_AsString(w); + rn = PyString_Size(w); + assert(rn % 2 == 0); + for (i = 0; i < rn; i += 2) { + sprintf(p, "\\u%02x%02x", + r[i + 0] & 0xFF, + r[i + 1] & 0xFF); + p += 6; + } + Py_DECREF(w); + } else { + *p++ = *s++; + } + } + len = p - buf; + s = buf; + } + if (rawmode) + v = PyUnicode_DecodeRawUnicodeEscape(s, len, NULL); + else + v = PyUnicode_DecodeUnicodeEscape(s, len, NULL); + Py_XDECREF(u); + return v; +} + +/* s is a Python string literal, including the bracketing quote characters, + * and r &/or u prefixes (if any), and embedded escape sequences (if any). + * parsestr parses it, and returns the decoded Python string object. + */ +static PyObject * +parsestr(const char *s, const char *encoding) +{ + PyObject *v; + size_t len; + int quote = *s; + int rawmode = 0; + int need_encoding; + int unicode = 0; + + if (isalpha(quote) || quote == '_') { + if (quote == 'u' || quote == 'U') { + quote = *++s; + unicode = 1; + } + if (quote == 'r' || quote == 'R') { + quote = *++s; + rawmode = 1; + } + } + if (quote != '\'' && quote != '\"') { + PyErr_BadInternalCall(); + return NULL; + } + s++; + len = strlen(s); + if (len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "string to parse is too long"); + return NULL; + } + if (s[--len] != quote) { + PyErr_BadInternalCall(); + return NULL; + } + if (len >= 4 && s[0] == quote && s[1] == quote) { + s += 2; + len -= 2; + if (s[--len] != quote || s[--len] != quote) { + PyErr_BadInternalCall(); + return NULL; + } + } +#ifdef Py_USING_UNICODE + if (unicode || Py_UnicodeFlag) { + return decode_unicode(s, len, rawmode, encoding); + } +#endif + need_encoding = (encoding != NULL && + strcmp(encoding, "utf-8") != 0 && + strcmp(encoding, "iso-8859-1") != 0); + if (rawmode || strchr(s, '\\') == NULL) { + if (need_encoding) { +#ifndef Py_USING_UNICODE + /* This should not happen - we never see any other + encoding. */ + Py_FatalError("cannot deal with encodings in this build."); +#else + PyObject* u = PyUnicode_DecodeUTF8(s, len, NULL); + if (u == NULL) + return NULL; + v = PyUnicode_AsEncodedString(u, encoding, NULL); + Py_DECREF(u); + return v; +#endif + } else { + return PyString_FromStringAndSize(s, len); + } + } + + v = PyString_DecodeEscape(s, len, NULL, unicode, + need_encoding ? encoding : NULL); + return v; +} + +/* Build a Python string object out of a STRING atom. This takes care of + * compile-time literal catenation, calling parsestr() on each piece, and + * pasting the intermediate results together. + */ +static PyObject * +parsestrplus(struct compiling *c, const node *n) +{ + PyObject *v; + int i; + REQ(CHILD(n, 0), STRING); + if ((v = parsestr(STR(CHILD(n, 0)), c->c_encoding)) != NULL) { + /* String literal concatenation */ + for (i = 1; i < NCH(n); i++) { + PyObject *s; + s = parsestr(STR(CHILD(n, i)), c->c_encoding); + if (s == NULL) + goto onError; + if (PyString_Check(v) && PyString_Check(s)) { + PyString_ConcatAndDel(&v, s); + if (v == NULL) + goto onError; + } +#ifdef Py_USING_UNICODE + else { + PyObject *temp; + temp = PyUnicode_Concat(v, s); + Py_DECREF(s); + if (temp == NULL) + goto onError; + Py_DECREF(v); + v = temp; + } +#endif + } + } + return v; + + onError: + Py_XDECREF(v); + return NULL; +} diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 3e5f4eb..2d51531 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1,9 +1,9 @@ - /* Built-in functions */ #include "Python.h" #include "node.h" +#include "code.h" #include "compile.h" #include "eval.h" diff --git a/Python/ceval.c b/Python/ceval.c index 38b1328..fdfe83e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -8,7 +8,7 @@ #include "Python.h" -#include "compile.h" +#include "code.h" #include "frameobject.h" #include "eval.h" #include "opcode.h" @@ -543,7 +543,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throw) #ifdef LLTRACE int lltrace; #endif -#if defined(Py_DEBUG) || defined(LLTRACE) +#if defined(Py_DEBUG) /* Make it easier to find out where we are with a debugger */ char *filename; #endif @@ -743,9 +743,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throw) f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */ #ifdef LLTRACE - lltrace = PyDict_GetItemString(f->f_globals,"__lltrace__") != NULL; + lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL; #endif -#if defined(Py_DEBUG) || defined(LLTRACE) +#if defined(Py_DEBUG) filename = PyString_AsString(co->co_filename); #endif @@ -2257,23 +2257,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throw) case MAKE_CLOSURE: { - int nfree; v = POP(); /* code object */ x = PyFunction_New(v, f->f_globals); - nfree = PyCode_GetNumFree((PyCodeObject *)v); Py_DECREF(v); - /* XXX Maybe this should be a separate opcode? */ - if (x != NULL && nfree > 0) { - v = PyTuple_New(nfree); - if (v == NULL) { - Py_DECREF(x); - x = NULL; - break; - } - while (--nfree >= 0) { - w = POP(); - PyTuple_SET_ITEM(v, nfree, w); - } + if (x != NULL) { + v = POP(); err = PyFunction_SetClosure(x, v); Py_DECREF(v); } @@ -2695,12 +2683,18 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, if (co->co_flags & CO_VARKEYWORDS) nargs++; - /* Check for cells that shadow args */ - for (i = 0; i < f->f_ncells && j < nargs; ++i) { + /* Initialize each cell var, taking into account + cell vars that are initialized from arguments. + + Should arrange for the compiler to put cellvars + that are arguments at the beginning of the cellvars + list so that we can march over it more efficiently? + */ + for (i = 0; i < f->f_ncells; ++i) { cellname = PyString_AS_STRING( PyTuple_GET_ITEM(co->co_cellvars, i)); found = 0; - while (j < nargs) { + for (j = 0; j < nargs; j++) { argname = PyString_AS_STRING( PyTuple_GET_ITEM(co->co_varnames, j)); if (strcmp(cellname, argname) == 0) { @@ -2711,7 +2705,6 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, found = 1; break; } - j++; } if (found == 0) { c = PyCell_New(NULL); @@ -2720,14 +2713,6 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, SETLOCAL(f->f_nlocals + i, c); } } - /* Initialize any that are left */ - while (i < f->f_ncells) { - c = PyCell_New(NULL); - if (c == NULL) - goto fail; - SETLOCAL(f->f_nlocals + i, c); - i++; - } } if (f->f_nfreevars) { int i; diff --git a/Python/compile.c b/Python/compile.c index 99ccf29..10c94e7 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1,385 +1,379 @@ -/* Compile an expression node to intermediate code */ - -/* XXX TO DO: - XXX add __doc__ attribute == co_doc to code object attributes? - XXX (it's currently the first item of the co_const tuple) - XXX Generate simple jump for break/return outside 'try...finally' - XXX Allow 'continue' inside finally clause of try-finally - XXX New opcode for loading the initial index for a for loop - XXX other JAR tricks? -*/ +/* + * This file compiles an abstract syntax tree (AST) into Python bytecode. + * + * The primary entry point is PyAST_Compile(), which returns a + * PyCodeObject. The compiler makes several passes to build the code + * object: + * 1. Checks for future statements. See future.c + * 2. Builds a symbol table. See symtable.c. + * 3. Generate code for basic blocks. See compiler_mod() in this file. + * 4. Assemble the basic blocks into final code. See assemble() in + * this file. + * + * Note that compiler_mod() suggests module, but the module ast type + * (mod_ty) has cases for expressions and interactive statements. + */ #include "Python.h" +#include "Python-ast.h" #include "node.h" -#include "token.h" -#include "graminit.h" +#include "ast.h" +#include "code.h" #include "compile.h" #include "symtable.h" #include "opcode.h" -#include "structmember.h" - -#include <ctype.h> - -/* Three symbols from graminit.h are also defined in Python.h, with - Py_ prefixes to their names. Python.h can't include graminit.h - (which defines too many confusing symbols), but we can check here - that they haven't changed (which is very unlikely, but possible). */ -#if Py_single_input != single_input - #error "single_input has changed -- update Py_single_input in Python.h" -#endif -#if Py_file_input != file_input - #error "file_input has changed -- update Py_file_input in Python.h" -#endif -#if Py_eval_input != eval_input - #error "eval_input has changed -- update Py_eval_input in Python.h" -#endif int Py_OptimizeFlag = 0; -#define OP_DELETE 0 -#define OP_ASSIGN 1 -#define OP_APPLY 2 +/* + ISSUES: -#define VAR_LOAD 0 -#define VAR_STORE 1 -#define VAR_DELETE 2 + character encodings aren't handled -#define DEL_CLOSURE_ERROR \ -"can not delete variable '%.400s' referenced in nested scope" + ref leaks in interpreter when press return on empty line -#define DUPLICATE_ARGUMENT \ -"duplicate argument '%s' in function definition" + opcode_stack_effect() function should be reviewed since stack depth bugs + could be really hard to find later. -#define GLOBAL_AFTER_ASSIGN \ -"name '%.400s' is assigned to before global declaration" - -#define GLOBAL_AFTER_USE \ -"name '%.400s' is used prior to global declaration" + Dead code is being generated (i.e. after unconditional jumps). +*/ -#define PARAM_GLOBAL \ -"name '%.400s' is a function parameter and declared global" +#define DEFAULT_BLOCK_SIZE 16 +#define DEFAULT_BLOCKS 8 +#define DEFAULT_CODE_SIZE 128 +#define DEFAULT_LNOTAB_SIZE 16 + +struct instr { + int i_jabs : 1; + int i_jrel : 1; + int i_hasarg : 1; + unsigned char i_opcode; + int i_oparg; + struct basicblock_ *i_target; /* target block (if jump instruction) */ + int i_lineno; +}; -#define LATE_FUTURE \ -"from __future__ imports must occur at the beginning of the file" +typedef struct basicblock_ { + /* next block in the list of blocks for a unit (don't confuse with + * b_next) */ + struct basicblock_ *b_list; + /* number of instructions used */ + int b_iused; + /* length of instruction array (b_instr) */ + int b_ialloc; + /* pointer to an array of instructions, initially NULL */ + struct instr *b_instr; + /* If b_next is non-NULL, it is a pointer to the next + block reached by normal control flow. */ + struct basicblock_ *b_next; + /* b_seen is used to perform a DFS of basicblocks. */ + int b_seen : 1; + /* b_return is true if a RETURN_VALUE opcode is inserted. */ + int b_return : 1; + /* depth of stack upon entry of block, computed by stackdepth() */ + int b_startdepth; + /* instruction offset for block, computed by assemble_jump_offsets() */ + int b_offset; +} basicblock; + +/* fblockinfo tracks the current frame block. + + A frame block is used to handle loops, try/except, and try/finally. + It's called a frame block to distinguish it from a basic block in the + compiler IR. +*/ -#define ASSIGN_DEBUG \ -"can not assign to __debug__" +enum fblocktype { LOOP, EXCEPT, FINALLY_TRY, FINALLY_END }; -#define MANGLE_LEN 256 +struct fblockinfo { + enum fblocktype fb_type; + basicblock *fb_block; +}; -#define OFF(x) offsetof(PyCodeObject, x) +/* The following items change on entry and exit of code blocks. + They must be saved and restored when returning to a block. +*/ +struct compiler_unit { + PySTEntryObject *u_ste; -static PyMemberDef code_memberlist[] = { - {"co_argcount", T_INT, OFF(co_argcount), READONLY}, - {"co_nlocals", T_INT, OFF(co_nlocals), READONLY}, - {"co_stacksize",T_INT, OFF(co_stacksize), READONLY}, - {"co_flags", T_INT, OFF(co_flags), READONLY}, - {"co_code", T_OBJECT, OFF(co_code), READONLY}, - {"co_consts", T_OBJECT, OFF(co_consts), READONLY}, - {"co_names", T_OBJECT, OFF(co_names), READONLY}, - {"co_varnames", T_OBJECT, OFF(co_varnames), READONLY}, - {"co_freevars", T_OBJECT, OFF(co_freevars), READONLY}, - {"co_cellvars", T_OBJECT, OFF(co_cellvars), READONLY}, - {"co_filename", T_OBJECT, OFF(co_filename), READONLY}, - {"co_name", T_OBJECT, OFF(co_name), READONLY}, - {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY}, - {"co_lnotab", T_OBJECT, OFF(co_lnotab), READONLY}, - {NULL} /* Sentinel */ + PyObject *u_name; + /* The following fields are dicts that map objects to + the index of them in co_XXX. The index is used as + the argument for opcodes that refer to those collections. + */ + PyObject *u_consts; /* all constants */ + PyObject *u_names; /* all names */ + PyObject *u_varnames; /* local variables */ + PyObject *u_cellvars; /* cell variables */ + PyObject *u_freevars; /* free variables */ + + PyObject *u_private; /* for private name mangling */ + + int u_argcount; /* number of arguments for block */ + basicblock *u_blocks; /* pointer to list of blocks */ + basicblock *u_curblock; /* pointer to current block */ + int u_tmpname; /* temporary variables for list comps */ + + int u_nfblocks; + struct fblockinfo u_fblock[CO_MAXBLOCKS]; + + int u_firstlineno; /* the first lineno of the block */ + int u_lineno; /* the lineno for the current stmt */ + bool u_lineno_set; /* boolean to indicate whether instr + has been generated with current lineno */ }; -/* Helper for code_new: return a shallow copy of a tuple that is - guaranteed to contain exact strings, by converting string subclasses - to exact strings and complaining if a non-string is found. */ -static PyObject* -validate_and_copy_tuple(PyObject *tup) -{ - PyObject *newtuple; - PyObject *item; - int i, len; +/* This struct captures the global state of a compilation. - len = PyTuple_GET_SIZE(tup); - newtuple = PyTuple_New(len); - if (newtuple == NULL) - return NULL; + The u pointer points to the current compilation unit, while units + for enclosing blocks are stored in c_stack. The u and c_stack are + managed by compiler_enter_scope() and compiler_exit_scope(). +*/ - for (i = 0; i < len; i++) { - item = PyTuple_GET_ITEM(tup, i); - if (PyString_CheckExact(item)) { - Py_INCREF(item); - } - else if (!PyString_Check(item)) { - PyErr_Format( - PyExc_TypeError, - "name tuples must contain only " - "strings, not '%.500s'", - item->ob_type->tp_name); - Py_DECREF(newtuple); - return NULL; - } - else { - item = PyString_FromStringAndSize( - PyString_AS_STRING(item), - PyString_GET_SIZE(item)); - if (item == NULL) { - Py_DECREF(newtuple); - return NULL; - } - } - PyTuple_SET_ITEM(newtuple, i, item); - } +struct compiler { + const char *c_filename; + struct symtable *c_st; + PyFutureFeatures *c_future; /* pointer to module's __future__ */ + PyCompilerFlags *c_flags; - return newtuple; -} + int c_interactive; + int c_nestlevel; -PyDoc_STRVAR(code_doc, -"code(argcount, nlocals, stacksize, flags, codestring, constants, names,\n\ - varnames, filename, name, firstlineno, lnotab[, freevars[, cellvars]])\n\ -\n\ -Create a code object. Not for the faint of heart."); + struct compiler_unit *u; /* compiler state for current block */ + PyObject *c_stack; /* Python list holding compiler_unit ptrs */ + char *c_encoding; /* source encoding (a borrowed reference) */ +}; -static PyObject * -code_new(PyTypeObject *type, PyObject *args, PyObject *kw) +struct assembler { + PyObject *a_bytecode; /* string containing bytecode */ + int a_offset; /* offset into bytecode */ + int a_nblocks; /* number of reachable blocks */ + basicblock **a_postorder; /* list of blocks in dfs postorder */ + PyObject *a_lnotab; /* string containing lnotab */ + int a_lnotab_off; /* offset into lnotab */ + int a_lineno; /* last lineno of emitted instruction */ + int a_lineno_off; /* bytecode offset of last lineno */ +}; + +static int compiler_enter_scope(struct compiler *, identifier, void *, int); +static void compiler_free(struct compiler *); +static basicblock *compiler_new_block(struct compiler *); +static int compiler_next_instr(struct compiler *, basicblock *); +static int compiler_addop(struct compiler *, int); +static int compiler_addop_o(struct compiler *, int, PyObject *, PyObject *); +static int compiler_addop_i(struct compiler *, int, int); +static int compiler_addop_j(struct compiler *, int, basicblock *, int); +static void compiler_use_block(struct compiler *, basicblock *); +static basicblock *compiler_use_new_block(struct compiler *); +static int compiler_error(struct compiler *, const char *); +static int compiler_nameop(struct compiler *, identifier, expr_context_ty); + +static PyCodeObject *compiler_mod(struct compiler *, mod_ty); +static int compiler_visit_stmt(struct compiler *, stmt_ty); +static int compiler_visit_keyword(struct compiler *, keyword_ty); +static int compiler_visit_expr(struct compiler *, expr_ty); +static int compiler_augassign(struct compiler *, stmt_ty); +static int compiler_visit_slice(struct compiler *, slice_ty, + expr_context_ty); + +static int compiler_push_fblock(struct compiler *, enum fblocktype, + basicblock *); +static void compiler_pop_fblock(struct compiler *, enum fblocktype, + basicblock *); + +static int inplace_binop(struct compiler *, operator_ty); +static int expr_constant(expr_ty e); + +static PyCodeObject *assemble(struct compiler *, int addNone); +static PyObject *__doc__; + +PyObject * +_Py_Mangle(PyObject *private, PyObject *ident) { - int argcount; - int nlocals; - int stacksize; - int flags; - PyObject *co = NULL; - PyObject *code; - PyObject *consts; - PyObject *names, *ournames = NULL; - PyObject *varnames, *ourvarnames = NULL; - PyObject *freevars = NULL, *ourfreevars = NULL; - PyObject *cellvars = NULL, *ourcellvars = NULL; - PyObject *filename; - PyObject *name; - int firstlineno; - PyObject *lnotab; - - if (!PyArg_ParseTuple(args, "iiiiSO!O!O!SSiS|O!O!:code", - &argcount, &nlocals, &stacksize, &flags, - &code, - &PyTuple_Type, &consts, - &PyTuple_Type, &names, - &PyTuple_Type, &varnames, - &filename, &name, - &firstlineno, &lnotab, - &PyTuple_Type, &freevars, - &PyTuple_Type, &cellvars)) - return NULL; + /* Name mangling: __private becomes _classname__private. + This is independent from how the name is used. */ + const char *p, *name = PyString_AsString(ident); + char *buffer; + size_t nlen, plen; + if (private == NULL || name == NULL || name[0] != '_' || name[1] != '_') { + Py_INCREF(ident); + return ident; + } + p = PyString_AsString(private); + nlen = strlen(name); + if (name[nlen-1] == '_' && name[nlen-2] == '_') { + Py_INCREF(ident); + return ident; /* Don't mangle __whatever__ */ + } + /* Strip leading underscores from class name */ + while (*p == '_') + p++; + if (*p == '\0') { + Py_INCREF(ident); + return ident; /* Don't mangle if class is just underscores */ + } + plen = strlen(p); + ident = PyString_FromStringAndSize(NULL, 1 + nlen + plen); + if (!ident) + return 0; + /* ident = "_" + p[:plen] + name # i.e. 1+plen+nlen bytes */ + buffer = PyString_AS_STRING(ident); + buffer[0] = '_'; + strncpy(buffer+1, p, plen); + strcpy(buffer+1+plen, name); + return ident; +} - if (argcount < 0) { - PyErr_SetString( - PyExc_ValueError, - "code: argcount must not be negative"); - goto cleanup; - } +static int +compiler_init(struct compiler *c) +{ + memset(c, 0, sizeof(struct compiler)); + + c->c_stack = PyList_New(0); + if (!c->c_stack) + return 0; + + return 1; +} - if (nlocals < 0) { - PyErr_SetString( - PyExc_ValueError, - "code: nlocals must not be negative"); - goto cleanup; +PyCodeObject * +PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags) +{ + struct compiler c; + PyCodeObject *co = NULL; + PyCompilerFlags local_flags; + int merged; + + if (!__doc__) { + __doc__ = PyString_InternFromString("__doc__"); + if (!__doc__) + goto error; + } + + if (!compiler_init(&c)) + goto error; + c.c_filename = filename; + c.c_future = PyFuture_FromAST(mod, filename); + if (c.c_future == NULL) + goto error; + if (!flags) { + local_flags.cf_flags = 0; + flags = &local_flags; + } + merged = c.c_future->ff_features | flags->cf_flags; + c.c_future->ff_features = merged; + flags->cf_flags = merged; + c.c_flags = flags; + c.c_nestlevel = 0; + + c.c_st = PySymtable_Build(mod, filename, c.c_future); + if (c.c_st == NULL) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_SystemError, "no symtable"); + goto error; } - ournames = validate_and_copy_tuple(names); - if (ournames == NULL) - goto cleanup; - ourvarnames = validate_and_copy_tuple(varnames); - if (ourvarnames == NULL) - goto cleanup; - if (freevars) - ourfreevars = validate_and_copy_tuple(freevars); - else - ourfreevars = PyTuple_New(0); - if (ourfreevars == NULL) - goto cleanup; - if (cellvars) - ourcellvars = validate_and_copy_tuple(cellvars); - else - ourcellvars = PyTuple_New(0); - if (ourcellvars == NULL) - goto cleanup; - - co = (PyObject *) PyCode_New(argcount, nlocals, stacksize, flags, - code, consts, ournames, ourvarnames, - ourfreevars, ourcellvars, filename, - name, firstlineno, lnotab); - cleanup: - Py_XDECREF(ournames); - Py_XDECREF(ourvarnames); - Py_XDECREF(ourfreevars); - Py_XDECREF(ourcellvars); + /* XXX initialize to NULL for now, need to handle */ + c.c_encoding = NULL; + + co = compiler_mod(&c, mod); + + error: + compiler_free(&c); return co; } -static void -code_dealloc(PyCodeObject *co) +PyCodeObject * +PyNode_Compile(struct _node *n, const char *filename) { - Py_XDECREF(co->co_code); - Py_XDECREF(co->co_consts); - Py_XDECREF(co->co_names); - Py_XDECREF(co->co_varnames); - Py_XDECREF(co->co_freevars); - Py_XDECREF(co->co_cellvars); - Py_XDECREF(co->co_filename); - Py_XDECREF(co->co_name); - Py_XDECREF(co->co_lnotab); - PyObject_DEL(co); + PyCodeObject *co; + mod_ty mod = PyAST_FromNode(n, NULL, filename); + if (!mod) + return NULL; + co = PyAST_Compile(mod, filename, NULL); + free_mod(mod); + return co; } -static PyObject * -code_repr(PyCodeObject *co) +static void +compiler_free(struct compiler *c) { - char buf[500]; - int lineno = -1; - char *filename = "???"; - char *name = "???"; - - if (co->co_firstlineno != 0) - lineno = co->co_firstlineno; - if (co->co_filename && PyString_Check(co->co_filename)) - filename = PyString_AS_STRING(co->co_filename); - if (co->co_name && PyString_Check(co->co_name)) - name = PyString_AS_STRING(co->co_name); - PyOS_snprintf(buf, sizeof(buf), - "<code object %.100s at %p, file \"%.300s\", line %d>", - name, co, filename, lineno); - return PyString_FromString(buf); + if (c->c_st) + PySymtable_Free(c->c_st); + if (c->c_future) + PyObject_Free((void *)c->c_future); + Py_DECREF(c->c_stack); } -static int -code_compare(PyCodeObject *co, PyCodeObject *cp) +static PyObject * +list2dict(PyObject *list) { - int cmp; - cmp = PyObject_Compare(co->co_name, cp->co_name); - if (cmp) return cmp; - cmp = co->co_argcount - cp->co_argcount; - if (cmp) return (cmp<0)?-1:1; - cmp = co->co_nlocals - cp->co_nlocals; - if (cmp) return (cmp<0)?-1:1; - cmp = co->co_flags - cp->co_flags; - if (cmp) return (cmp<0)?-1:1; - cmp = co->co_firstlineno - cp->co_firstlineno; - if (cmp) return (cmp<0)?-1:1; - cmp = PyObject_Compare(co->co_code, cp->co_code); - if (cmp) return cmp; - cmp = PyObject_Compare(co->co_consts, cp->co_consts); - if (cmp) return cmp; - cmp = PyObject_Compare(co->co_names, cp->co_names); - if (cmp) return cmp; - cmp = PyObject_Compare(co->co_varnames, cp->co_varnames); - if (cmp) return cmp; - cmp = PyObject_Compare(co->co_freevars, cp->co_freevars); - if (cmp) return cmp; - cmp = PyObject_Compare(co->co_cellvars, cp->co_cellvars); - return cmp; -} + int i, n; + PyObject *v, *k, *dict = PyDict_New(); -static long -code_hash(PyCodeObject *co) -{ - long h, h0, h1, h2, h3, h4, h5, h6; - h0 = PyObject_Hash(co->co_name); - if (h0 == -1) return -1; - h1 = PyObject_Hash(co->co_code); - if (h1 == -1) return -1; - h2 = PyObject_Hash(co->co_consts); - if (h2 == -1) return -1; - h3 = PyObject_Hash(co->co_names); - if (h3 == -1) return -1; - h4 = PyObject_Hash(co->co_varnames); - if (h4 == -1) return -1; - h5 = PyObject_Hash(co->co_freevars); - if (h5 == -1) return -1; - h6 = PyObject_Hash(co->co_cellvars); - if (h6 == -1) return -1; - h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ h6 ^ - co->co_argcount ^ co->co_nlocals ^ co->co_flags; - if (h == -1) h = -2; - return h; + n = PyList_Size(list); + for (i = 0; i < n; i++) { + v = PyInt_FromLong(i); + if (!v) { + Py_DECREF(dict); + return NULL; + } + k = PyList_GET_ITEM(list, i); + k = Py_BuildValue("(OO)", k, k->ob_type); + if (k == NULL || PyDict_SetItem(dict, k, v) < 0) { + Py_XDECREF(k); + Py_DECREF(v); + Py_DECREF(dict); + return NULL; + } + Py_DECREF(v); + } + return dict; } -/* XXX code objects need to participate in GC? */ - -PyTypeObject PyCode_Type = { - PyObject_HEAD_INIT(&PyType_Type) - 0, - "code", - sizeof(PyCodeObject), - 0, - (destructor)code_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - (cmpfunc)code_compare, /* tp_compare */ - (reprfunc)code_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - (hashfunc)code_hash, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - code_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - code_memberlist, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - code_new, /* tp_new */ -}; +/* Return new dict containing names from src that match scope(s). -#define NAME_CHARS \ - "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" - -/* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */ + src is a symbol table dictionary. If the scope of a name matches + either scope_type or flag is set, insert it into the new dict. The + values are integers, starting at offset and increasing by one for + each key. +*/ -static int -all_name_chars(unsigned char *s) +static PyObject * +dictbytype(PyObject *src, int scope_type, int flag, int offset) { - static char ok_name_char[256]; - static unsigned char *name_chars = (unsigned char *)NAME_CHARS; + int pos = 0, i = offset, scope; + PyObject *k, *v, *dest = PyDict_New(); - if (ok_name_char[*name_chars] == 0) { - unsigned char *p; - for (p = name_chars; *p; p++) - ok_name_char[*p] = 1; - } - while (*s) { - if (ok_name_char[*s++] == 0) - return 0; - } - return 1; -} + assert(offset >= 0); + if (dest == NULL) + return NULL; -static void -intern_strings(PyObject *tuple) -{ - int i; + while (PyDict_Next(src, &pos, &k, &v)) { + /* XXX this should probably be a macro in symtable.h */ + assert(PyInt_Check(v)); + scope = (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; - for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { - PyObject *v = PyTuple_GET_ITEM(tuple, i); - if (v == NULL || !PyString_CheckExact(v)) { - Py_FatalError("non-string found in code slot"); + if (scope == scope_type || PyInt_AS_LONG(v) & flag) { + PyObject *tuple, *item = PyInt_FromLong(i); + if (item == NULL) { + Py_DECREF(dest); + return NULL; + } + i++; + tuple = Py_BuildValue("(OO)", k, k->ob_type); + if (!tuple || PyDict_SetItem(dest, tuple, item) < 0) { + Py_DECREF(item); + Py_DECREF(dest); + Py_XDECREF(tuple); + return NULL; } - PyString_InternInPlace(&PyTuple_GET_ITEM(tuple, i)); + Py_DECREF(item); + Py_DECREF(tuple); + } } + return dest; } /* Begin: Peephole optimizations ----------------------------------------- */ @@ -974,2262 +968,813 @@ exitUnchanged: /* End: Peephole optimizations ----------------------------------------- */ -PyCodeObject * -PyCode_New(int argcount, int nlocals, int stacksize, int flags, - PyObject *code, PyObject *consts, PyObject *names, - PyObject *varnames, PyObject *freevars, PyObject *cellvars, - PyObject *filename, PyObject *name, int firstlineno, - PyObject *lnotab) -{ - PyCodeObject *co; - int i; - /* Check argument types */ - if (argcount < 0 || nlocals < 0 || - code == NULL || - consts == NULL || !PyTuple_Check(consts) || - names == NULL || !PyTuple_Check(names) || - varnames == NULL || !PyTuple_Check(varnames) || - freevars == NULL || !PyTuple_Check(freevars) || - cellvars == NULL || !PyTuple_Check(cellvars) || - name == NULL || !PyString_Check(name) || - filename == NULL || !PyString_Check(filename) || - lnotab == NULL || !PyString_Check(lnotab) || - !PyObject_CheckReadBuffer(code)) { - PyErr_BadInternalCall(); - return NULL; - } - intern_strings(names); - intern_strings(varnames); - intern_strings(freevars); - intern_strings(cellvars); - /* Intern selected string constants */ - for (i = PyTuple_Size(consts); --i >= 0; ) { - PyObject *v = PyTuple_GetItem(consts, i); - if (!PyString_Check(v)) - continue; - if (!all_name_chars((unsigned char *)PyString_AS_STRING(v))) - continue; - PyString_InternInPlace(&PyTuple_GET_ITEM(consts, i)); - } - co = PyObject_NEW(PyCodeObject, &PyCode_Type); - if (co != NULL) { - co->co_argcount = argcount; - co->co_nlocals = nlocals; - co->co_stacksize = stacksize; - co->co_flags = flags; - Py_INCREF(code); - co->co_code = code; - Py_INCREF(consts); - co->co_consts = consts; - Py_INCREF(names); - co->co_names = names; - Py_INCREF(varnames); - co->co_varnames = varnames; - Py_INCREF(freevars); - co->co_freevars = freevars; - Py_INCREF(cellvars); - co->co_cellvars = cellvars; - Py_INCREF(filename); - co->co_filename = filename; - Py_INCREF(name); - co->co_name = name; - co->co_firstlineno = firstlineno; - Py_INCREF(lnotab); - co->co_lnotab = lnotab; - if (PyTuple_GET_SIZE(freevars) == 0 && - PyTuple_GET_SIZE(cellvars) == 0) - co->co_flags |= CO_NOFREE; +/* + +Leave this debugging code for just a little longer. + +static void +compiler_display_symbols(PyObject *name, PyObject *symbols) +{ + PyObject *key, *value; + int flags, pos = 0; + + fprintf(stderr, "block %s\n", PyString_AS_STRING(name)); + while (PyDict_Next(symbols, &pos, &key, &value)) { + flags = PyInt_AsLong(value); + fprintf(stderr, "var %s:", PyString_AS_STRING(key)); + if (flags & DEF_GLOBAL) + fprintf(stderr, " declared_global"); + if (flags & DEF_LOCAL) + fprintf(stderr, " local"); + if (flags & DEF_PARAM) + fprintf(stderr, " param"); + if (flags & DEF_STAR) + fprintf(stderr, " stararg"); + if (flags & DEF_DOUBLESTAR) + fprintf(stderr, " starstar"); + if (flags & DEF_INTUPLE) + fprintf(stderr, " tuple"); + if (flags & DEF_FREE) + fprintf(stderr, " free"); + if (flags & DEF_FREE_GLOBAL) + fprintf(stderr, " global"); + if (flags & DEF_FREE_CLASS) + fprintf(stderr, " free/class"); + if (flags & DEF_IMPORT) + fprintf(stderr, " import"); + fprintf(stderr, "\n"); } - return co; + fprintf(stderr, "\n"); } - - -/* Data structure used internally */ - -/* The compiler uses two passes to generate bytecodes. The first pass - builds the symbol table. The second pass generates the bytecode. - - The first pass uses a single symtable struct. The second pass uses - a compiling struct for each code block. The compiling structs - share a reference to the symtable. - - The two passes communicate via symtable_load_symbols() and via - is_local() and is_global(). The former initializes several slots - in the compiling struct: c_varnames, c_locals, c_nlocals, - c_argcount, c_globals, and c_flags. */ -/* All about c_lnotab. - -c_lnotab is an array of unsigned bytes disguised as a Python string. Since -version 2.3, SET_LINENO opcodes are never generated and bytecode offsets are -mapped to source code line #s via c_lnotab instead. - -The array is conceptually a list of - (bytecode offset increment, line number increment) -pairs. The details are important and delicate, best illustrated by example: - - byte code offset source code line number - 0 1 - 6 2 - 50 7 - 350 307 - 361 308 - -The first trick is that these numbers aren't stored, only the increments -from one row to the next (this doesn't really work, but it's a start): - - 0, 1, 6, 1, 44, 5, 300, 300, 11, 1 - -The second trick is that an unsigned byte can't hold negative values, or -values larger than 255, so (a) there's a deep assumption that byte code -offsets and their corresponding line #s both increase monotonically, and (b) -if at least one column jumps by more than 255 from one row to the next, more -than one pair is written to the table. In case #b, there's no way to know -from looking at the table later how many were written. That's the delicate -part. A user of c_lnotab desiring to find the source line number -corresponding to a bytecode address A should do something like this - - lineno = addr = 0 - for addr_incr, line_incr in c_lnotab: - addr += addr_incr - if addr > A: - return lineno - lineno += line_incr - -In order for this to work, when the addr field increments by more than 255, -the line # increment in each pair generated must be 0 until the remaining addr -increment is < 256. So, in the example above, com_set_lineno should not (as -was actually done until 2.2) expand 300, 300 to 255, 255, 45, 45, but to -255, 0, 45, 255, 0, 45. -*/ - -struct compiling { - PyObject *c_code; /* string */ - PyObject *c_consts; /* list of objects */ - PyObject *c_const_dict; /* inverse of c_consts */ - PyObject *c_names; /* list of strings (names) */ - PyObject *c_name_dict; /* inverse of c_names */ - PyObject *c_globals; /* dictionary (value=None or True) */ - PyObject *c_locals; /* dictionary (value=localID) */ - PyObject *c_varnames; /* list (inverse of c_locals) */ - PyObject *c_freevars; /* dictionary (value=None) */ - PyObject *c_cellvars; /* dictionary */ - int c_nlocals; /* index of next local */ - int c_argcount; /* number of top-level arguments */ - int c_flags; /* same as co_flags */ - int c_nexti; /* index into c_code */ - int c_errors; /* counts errors occurred */ - int c_infunction; /* set when compiling a function */ - int c_interactive; /* generating code for interactive command */ - int c_loops; /* counts nested loops */ - int c_begin; /* begin of current loop, for 'continue' */ - int c_block[CO_MAXBLOCKS]; /* stack of block types */ - int c_nblocks; /* current block stack level */ - const char *c_filename; /* filename of current node */ - char *c_name; /* name of object (e.g. function) */ - int c_lineno; /* Current line number */ - int c_stacklevel; /* Current stack level */ - int c_maxstacklevel; /* Maximum stack level */ - int c_firstlineno; - PyObject *c_lnotab; /* Table mapping address to line number */ - int c_last_addr; /* last op addr seen and recorded in lnotab */ - int c_last_line; /* last line seen and recorded in lnotab */ - int c_lnotab_next; /* current length of lnotab */ - int c_lnotab_last; /* start of last lnotab record added */ - char *c_private; /* for private name mangling */ - int c_tmpname; /* temporary local name counter */ - int c_nested; /* Is block nested funcdef or lamdef? */ - int c_closure; /* Is nested w/freevars? */ - struct symtable *c_symtable; /* pointer to module symbol table */ - PyFutureFeatures *c_future; /* pointer to module's __future__ */ - char *c_encoding; /* source encoding (a borrowed reference) */ -}; - -static int -is_free(int v) -{ - if ((v & (USE | DEF_FREE)) - && !(v & (DEF_LOCAL | DEF_PARAM | DEF_GLOBAL))) - return 1; - if (v & DEF_FREE_CLASS) - return 1; - return 0; -} - -static void -com_error(struct compiling *c, PyObject *exc, char *msg) -{ - PyObject *t = NULL, *v = NULL, *w = NULL, *line = NULL; - - if (c == NULL) { - /* Error occurred via symtable call to - is_constant_false */ - PyErr_SetString(exc, msg); - return; - } - c->c_errors++; - if (c->c_lineno < 1 || c->c_interactive) { - /* Unknown line number or interactive input */ - PyErr_SetString(exc, msg); - return; - } - v = PyString_FromString(msg); - if (v == NULL) - return; /* MemoryError, too bad */ - - line = PyErr_ProgramText(c->c_filename, c->c_lineno); - if (line == NULL) { - Py_INCREF(Py_None); - line = Py_None; - } - if (exc == PyExc_SyntaxError) { - t = Py_BuildValue("(ziOO)", c->c_filename, c->c_lineno, - Py_None, line); - if (t == NULL) - goto exit; - w = PyTuple_Pack(2, v, t); - if (w == NULL) - goto exit; - PyErr_SetObject(exc, w); - } else { - /* Make sure additional exceptions are printed with - file and line, also. */ - PyErr_SetObject(exc, v); - PyErr_SyntaxLocation(c->c_filename, c->c_lineno); - } - exit: - Py_XDECREF(t); - Py_XDECREF(v); - Py_XDECREF(w); - Py_XDECREF(line); -} - -/* Interface to the block stack */ - static void -block_push(struct compiling *c, int type) +compiler_unit_check(struct compiler_unit *u) { - if (c->c_nblocks >= CO_MAXBLOCKS) { - com_error(c, PyExc_SystemError, - "too many statically nested blocks"); - } - else { - c->c_block[c->c_nblocks++] = type; - } -} - -static void -block_pop(struct compiling *c, int type) -{ - if (c->c_nblocks > 0) - c->c_nblocks--; - if (c->c_block[c->c_nblocks] != type && c->c_errors == 0) { - com_error(c, PyExc_SystemError, "bad block pop"); + basicblock *block; + for (block = u->u_blocks; block != NULL; block = block->b_list) { + assert(block != (void *)0xcbcbcbcb); + assert(block != (void *)0xfbfbfbfb); + assert(block != (void *)0xdbdbdbdb); + if (block->b_instr != NULL) { + assert(block->b_ialloc > 0); + assert(block->b_iused > 0); + assert(block->b_ialloc >= block->b_iused); + } + else { + assert (block->b_iused == 0); + assert (block->b_ialloc == 0); + } } } -/* Prototype forward declarations */ - -static int issue_warning(const char *, const char *, int); -static int com_init(struct compiling *, const char *); -static void com_free(struct compiling *); -static void com_push(struct compiling *, int); -static void com_pop(struct compiling *, int); -static void com_done(struct compiling *); -static void com_node(struct compiling *, node *); -static void com_factor(struct compiling *, node *); -static void com_addbyte(struct compiling *, int); -static void com_addint(struct compiling *, int); -static void com_addoparg(struct compiling *, int, int); -static void com_addfwref(struct compiling *, int, int *); -static void com_backpatch(struct compiling *, int); -static int com_add(struct compiling *, PyObject *, PyObject *, PyObject *); -static int com_addconst(struct compiling *, PyObject *); -static int com_addname(struct compiling *, PyObject *); -static void com_addopname(struct compiling *, int, node *); -static void com_test(struct compiling *c, node *n); -static void com_list(struct compiling *, node *, int); -static void com_list_iter(struct compiling *, node *, node *, char *); -static void com_gen_iter(struct compiling *, node *, node *); -static int com_argdefs(struct compiling *, node *); -static void com_assign(struct compiling *, node *, int, node *); -static void com_assign_name(struct compiling *, node *, int); -static int com_make_closure(struct compiling *c, PyCodeObject *co); - -static PyCodeObject *icompile(node *, struct compiling *); -static PyCodeObject *jcompile(node *, const char *, struct compiling *, - PyCompilerFlags *); -static PyObject *parsestrplus(struct compiling*, node *); -static PyObject *parsestr(struct compiling *, char *); -static node *get_rawdocstring(node *); - -static int get_ref_type(struct compiling *, char *); - -/* symtable operations */ -static int symtable_lookup(struct symtable *st, char *name); -static struct symtable *symtable_build(node *, PyFutureFeatures *, - const char *filename); -static int symtable_load_symbols(struct compiling *); -static struct symtable *symtable_init(void); -static void symtable_enter_scope(struct symtable *, char *, int, int); -static int symtable_exit_scope(struct symtable *); -static int symtable_add_def(struct symtable *, char *, int); -static int symtable_add_def_o(struct symtable *, PyObject *, PyObject *, int); - -static void symtable_node(struct symtable *, node *); -static void symtable_funcdef(struct symtable *, node *); -static void symtable_default_args(struct symtable *, node *); -static void symtable_params(struct symtable *, node *); -static void symtable_params_fplist(struct symtable *, node *n); -static void symtable_global(struct symtable *, node *); -static void symtable_import(struct symtable *, node *); -static void symtable_assign(struct symtable *, node *, int); -static void symtable_list_comprehension(struct symtable *, node *); -static void symtable_generator_expression(struct symtable *, node *); -static void symtable_list_for(struct symtable *, node *); -static void symtable_gen_for(struct symtable *, node *, int); -static void symtable_gen_iter(struct symtable *, node *); - -static int symtable_update_free_vars(struct symtable *); -static int symtable_undo_free(struct symtable *, PyObject *, PyObject *); -static int symtable_check_global(struct symtable *, PyObject *, PyObject *); - -/* helper */ -static void -do_pad(int pad) -{ - int i; - for (i = 0; i < pad; ++i) - fprintf(stderr, " "); -} - -static void -dump(node *n, int pad, int depth) -{ - int i; - if (depth == 0) - return; - do_pad(pad); - fprintf(stderr, "%d: %s\n", TYPE(n), STR(n)); - if (depth > 0) - depth--; - for (i = 0; i < NCH(n); ++i) - dump(CHILD(n, i), pad + 1, depth); -} - -static int -com_init(struct compiling *c, const char *filename) -{ - memset((void *)c, '\0', sizeof(struct compiling)); - if ((c->c_code = PyString_FromStringAndSize((char *)NULL, - 1000)) == NULL) - goto fail; - if ((c->c_consts = PyList_New(0)) == NULL) - goto fail; - if ((c->c_const_dict = PyDict_New()) == NULL) - goto fail; - if ((c->c_names = PyList_New(0)) == NULL) - goto fail; - if ((c->c_name_dict = PyDict_New()) == NULL) - goto fail; - if ((c->c_locals = PyDict_New()) == NULL) - goto fail; - if ((c->c_lnotab = PyString_FromStringAndSize((char *)NULL, - 1000)) == NULL) - goto fail; - c->c_globals = NULL; - c->c_varnames = NULL; - c->c_freevars = NULL; - c->c_cellvars = NULL; - c->c_nlocals = 0; - c->c_argcount = 0; - c->c_flags = 0; - c->c_nexti = 0; - c->c_errors = 0; - c->c_infunction = 0; - c->c_interactive = 0; - c->c_loops = 0; - c->c_begin = 0; - c->c_nblocks = 0; - c->c_filename = filename; - c->c_name = "?"; - c->c_lineno = 0; - c->c_stacklevel = 0; - c->c_maxstacklevel = 0; - c->c_firstlineno = 0; - c->c_last_addr = 0; - c->c_last_line = 0; - c->c_lnotab_next = 0; - c->c_lnotab_last = 0; - c->c_tmpname = 0; - c->c_nested = 0; - c->c_closure = 0; - c->c_symtable = NULL; - return 1; - - fail: - com_free(c); - return 0; -} - static void -com_free(struct compiling *c) +compiler_unit_free(struct compiler_unit *u) { - Py_XDECREF(c->c_code); - Py_XDECREF(c->c_consts); - Py_XDECREF(c->c_const_dict); - Py_XDECREF(c->c_names); - Py_XDECREF(c->c_name_dict); - Py_XDECREF(c->c_globals); - Py_XDECREF(c->c_locals); - Py_XDECREF(c->c_varnames); - Py_XDECREF(c->c_freevars); - Py_XDECREF(c->c_cellvars); - Py_XDECREF(c->c_lnotab); - if (c->c_future) - PyObject_FREE((void *)c->c_future); -} + basicblock *b, *next; -static void -com_push(struct compiling *c, int n) -{ - c->c_stacklevel += n; - if (c->c_stacklevel > c->c_maxstacklevel) { - c->c_maxstacklevel = c->c_stacklevel; - /* - fprintf(stderr, "%s:%s:%d max stack nexti=%d level=%d n=%d\n", - c->c_filename, c->c_name, c->c_lineno, - c->c_nexti, c->c_stacklevel, n); - */ + compiler_unit_check(u); + b = u->u_blocks; + while (b != NULL) { + if (b->b_instr) + PyObject_Free((void *)b->b_instr); + next = b->b_list; + PyObject_Free((void *)b); + b = next; } -} - -static void -com_pop(struct compiling *c, int n) -{ - if (c->c_stacklevel < n) - c->c_stacklevel = 0; - else - c->c_stacklevel -= n; -} - -static void -com_done(struct compiling *c) -{ - if (c->c_code != NULL) - _PyString_Resize(&c->c_code, c->c_nexti); - if (c->c_lnotab != NULL) - _PyString_Resize(&c->c_lnotab, c->c_lnotab_next); + Py_XDECREF(u->u_ste); + Py_XDECREF(u->u_name); + Py_XDECREF(u->u_consts); + Py_XDECREF(u->u_names); + Py_XDECREF(u->u_varnames); + Py_XDECREF(u->u_freevars); + Py_XDECREF(u->u_cellvars); + Py_XDECREF(u->u_private); + PyObject_Free(u); } static int -com_check_size(PyObject **s, int offset) -{ - int len = PyString_GET_SIZE(*s); - if (offset >= len) - return _PyString_Resize(s, len * 2); - return 0; -} - -static void -com_addbyte(struct compiling *c, int byte) -{ - /*fprintf(stderr, "%3d: %3d\n", c->c_nexti, byte);*/ - assert(byte >= 0 && byte <= 255); - assert(c->c_code != 0); - if (com_check_size(&c->c_code, c->c_nexti)) { - c->c_errors++; - return; - } - PyString_AS_STRING(c->c_code)[c->c_nexti++] = byte; -} - -static void -com_addint(struct compiling *c, int x) +compiler_enter_scope(struct compiler *c, identifier name, void *key, + int lineno) { - com_addbyte(c, x & 0xff); - com_addbyte(c, x >> 8); /* XXX x should be positive */ -} + struct compiler_unit *u; -static void -com_add_lnotab(struct compiling *c, int addr, int line) -{ - char *p; - if (c->c_lnotab == NULL) - return; - if (com_check_size(&c->c_lnotab, c->c_lnotab_next + 2)) { - c->c_errors++; - return; + u = PyObject_Malloc(sizeof(struct compiler_unit)); + memset(u, 0, sizeof(struct compiler_unit)); + u->u_argcount = 0; + u->u_ste = PySymtable_Lookup(c->c_st, key); + if (!u->u_ste) { + compiler_unit_free(u); + return 0; } - p = PyString_AS_STRING(c->c_lnotab) + c->c_lnotab_next; - *p++ = addr; - *p++ = line; - c->c_lnotab_next += 2; -} - -static void -com_set_lineno(struct compiling *c, int lineno) -{ - c->c_lineno = lineno; - if (c->c_firstlineno == 0) { - c->c_firstlineno = c->c_last_line = lineno; + Py_INCREF(name); + u->u_name = name; + u->u_varnames = list2dict(u->u_ste->ste_varnames); + u->u_cellvars = dictbytype(u->u_ste->ste_symbols, CELL, 0, 0); + u->u_freevars = dictbytype(u->u_ste->ste_symbols, FREE, DEF_FREE_CLASS, + PyDict_Size(u->u_cellvars)); + + u->u_blocks = NULL; + u->u_tmpname = 0; + u->u_nfblocks = 0; + u->u_firstlineno = lineno; + u->u_lineno = 0; + u->u_lineno_set = false; + u->u_consts = PyDict_New(); + if (!u->u_consts) { + compiler_unit_free(u); + return 0; } - else { - int incr_addr = c->c_nexti - c->c_last_addr; - int incr_line = lineno - c->c_last_line; - c->c_lnotab_last = c->c_lnotab_next; - while (incr_addr > 255) { - com_add_lnotab(c, 255, 0); - incr_addr -= 255; - } - while (incr_line > 255) { - com_add_lnotab(c, incr_addr, 255); - incr_line -=255; - incr_addr = 0; - } - if (incr_addr > 0 || incr_line > 0) - com_add_lnotab(c, incr_addr, incr_line); - c->c_last_addr = c->c_nexti; - c->c_last_line = lineno; + u->u_names = PyDict_New(); + if (!u->u_names) { + compiler_unit_free(u); + return 0; } -} -static void -com_strip_lnotab(struct compiling *c) -{ - /* strip the last lnotab entry if no opcode were emitted. - * This prevents a line number to be generated on a final - * pass, like in the following example: - * - * if a: - * print 5 - * else: - * pass - * - * Without the fix, a line trace event would be generated - * on the pass even if a is true (because of the implicit - * return). - */ - if (c->c_nexti == c->c_last_addr && c->c_lnotab_last > 0) { - c->c_lnotab_next = c->c_lnotab_last; - } -} + u->u_private = NULL; -static void -com_addoparg(struct compiling *c, int op, int arg) -{ - int extended_arg = arg >> 16; - if (extended_arg){ - com_addbyte(c, EXTENDED_ARG); - com_addint(c, extended_arg); - arg &= 0xffff; + /* Push the old compiler_unit on the stack. */ + if (c->u) { + PyObject *wrapper = PyCObject_FromVoidPtr(c->u, NULL); + if (PyList_Append(c->c_stack, wrapper) < 0) { + compiler_unit_free(u); + return 0; + } + Py_DECREF(wrapper); + u->u_private = c->u->u_private; + Py_XINCREF(u->u_private); } - com_addbyte(c, op); - com_addint(c, arg); -} + c->u = u; -static void -com_addfwref(struct compiling *c, int op, int *p_anchor) -{ - /* Compile a forward reference for backpatching */ - int here; - int anchor; - com_addbyte(c, op); - here = c->c_nexti; - anchor = *p_anchor; - *p_anchor = here; - com_addint(c, anchor == 0 ? 0 : here - anchor); -} + c->c_nestlevel++; + if (compiler_use_new_block(c) < 0) + return 0; -static void -com_backpatch(struct compiling *c, int anchor) -{ - unsigned char *code = (unsigned char *) PyString_AS_STRING(c->c_code); - int target = c->c_nexti; - int dist; - int prev; - for (;;) { - /* Make the JUMP instruction at anchor point to target */ - prev = code[anchor] + (code[anchor+1] << 8); - dist = target - (anchor+2); - code[anchor] = dist & 0xff; - dist >>= 8; - code[anchor+1] = dist; - dist >>= 8; - if (dist) { - com_error(c, PyExc_SystemError, - "com_backpatch: offset too large"); - break; - } - if (!prev) - break; - anchor -= prev; - } + return 1; } -/* Handle literals and names uniformly */ - static int -com_add(struct compiling *c, PyObject *list, PyObject *dict, PyObject *v) -{ - PyObject *w, *t, *np=NULL; - long n; - - t = PyTuple_Pack(2, v, v->ob_type); - if (t == NULL) - goto fail; - w = PyDict_GetItem(dict, t); - if (w != NULL) { - n = PyInt_AsLong(w); - } else { - n = PyList_Size(list); - np = PyInt_FromLong(n); - if (np == NULL) - goto fail; - if (PyList_Append(list, v) != 0) - goto fail; - if (PyDict_SetItem(dict, t, np) != 0) - goto fail; - Py_DECREF(np); +compiler_exit_scope(struct compiler *c) +{ + int n; + PyObject *wrapper; + + c->c_nestlevel--; + compiler_unit_free(c->u); + /* Restore c->u to the parent unit. */ + n = PyList_GET_SIZE(c->c_stack) - 1; + if (n >= 0) { + wrapper = PyList_GET_ITEM(c->c_stack, n); + c->u = (struct compiler_unit *)PyCObject_AsVoidPtr(wrapper); + if (PySequence_DelItem(c->c_stack, n) < 0) + return 0; + compiler_unit_check(c->u); } - Py_DECREF(t); - return n; - fail: - Py_XDECREF(np); - Py_XDECREF(t); - c->c_errors++; - return 0; -} + else + c->u = NULL; -static int -com_addconst(struct compiling *c, PyObject *v) -{ - return com_add(c, c->c_consts, c->c_const_dict, v); + return 1; /* XXX void? */ } -static int -com_addname(struct compiling *c, PyObject *v) -{ - return com_add(c, c->c_names, c->c_name_dict, v); -} +/* Allocate a new block and return a pointer to it. + Returns NULL on error. +*/ -int -_Py_Mangle(char *p, char *name, char *buffer, size_t maxlen) +static basicblock * +compiler_new_block(struct compiler *c) { - /* Name mangling: __private becomes _classname__private. - This is independent from how the name is used. */ - size_t nlen, plen; - if (p == NULL || name == NULL || name[0] != '_' || name[1] != '_') - return 0; - nlen = strlen(name); - if (nlen+2 >= maxlen) - return 0; /* Don't mangle __extremely_long_names */ - if (name[nlen-1] == '_' && name[nlen-2] == '_') - return 0; /* Don't mangle __whatever__ */ - /* Strip leading underscores from class name */ - while (*p == '_') - p++; - if (*p == '\0') - return 0; /* Don't mangle if class is just underscores */ - plen = strlen(p); - if (plen + nlen >= maxlen) - plen = maxlen-nlen-2; /* Truncate class name if too long */ - /* buffer = "_" + p[:plen] + name # i.e. 1+plen+nlen bytes */ - buffer[0] = '_'; - strncpy(buffer+1, p, plen); - strcpy(buffer+1+plen, name); - return 1; + basicblock *b; + struct compiler_unit *u; + + u = c->u; + b = (basicblock *)PyObject_Malloc(sizeof(basicblock)); + if (b == NULL) + return NULL; + memset((void *)b, 0, sizeof(basicblock)); + assert (b->b_next == NULL); + b->b_list = u->u_blocks; + u->u_blocks = b; + return b; } static void -com_addop_name(struct compiling *c, int op, char *name) +compiler_use_block(struct compiler *c, basicblock *block) { - PyObject *v; - int i; - char buffer[MANGLE_LEN]; - - if (_Py_Mangle(c->c_private, name, buffer, sizeof(buffer))) - name = buffer; - if (name == NULL || (v = PyString_InternFromString(name)) == NULL) { - c->c_errors++; - i = 255; - } - else { - i = com_addname(c, v); - Py_DECREF(v); - } - com_addoparg(c, op, i); + assert (block != NULL); + c->u->u_curblock = block; } -#define NAME_LOCAL 0 -#define NAME_GLOBAL 1 -#define NAME_DEFAULT 2 -#define NAME_CLOSURE 3 - -static int -com_lookup_arg(PyObject *dict, PyObject *name) +static basicblock * +compiler_use_new_block(struct compiler *c) { - PyObject *v = PyDict_GetItem(dict, name); - if (v == NULL) - return -1; - else - return PyInt_AS_LONG(v); + basicblock *block = compiler_new_block(c); + if (block == NULL) + return NULL; + c->u->u_curblock = block; + return block; } -static int -none_assignment_check(struct compiling *c, char *name, int assigning) +static basicblock * +compiler_next_block(struct compiler *c) { - if (name[0] == 'N' && strcmp(name, "None") == 0) { - char *msg; - if (assigning) - msg = "assignment to None"; - else - msg = "deleting None"; - com_error(c, PyExc_SyntaxError, msg); - return -1; - } - return 0; + basicblock *block = compiler_new_block(c); + if (block == NULL) + return NULL; + c->u->u_curblock->b_next = block; + c->u->u_curblock = block; + return block; } -static void -com_addop_varname(struct compiling *c, int kind, char *name) +static basicblock * +compiler_use_next_block(struct compiler *c, basicblock *block) { - PyObject *v; - int i, reftype; - int scope = NAME_DEFAULT; - int op = STOP_CODE; - char buffer[MANGLE_LEN]; - - if (kind != VAR_LOAD && - none_assignment_check(c, name, kind == VAR_STORE)) - { - i = 255; - goto done; - } - if (_Py_Mangle(c->c_private, name, buffer, sizeof(buffer))) - name = buffer; - if (name == NULL || (v = PyString_InternFromString(name)) == NULL) { - c->c_errors++; - i = 255; - goto done; - } - - reftype = get_ref_type(c, name); - switch (reftype) { - case LOCAL: - if (c->c_symtable->st_cur->ste_type == TYPE_FUNCTION) - scope = NAME_LOCAL; - break; - case GLOBAL_EXPLICIT: - scope = NAME_GLOBAL; - break; - case GLOBAL_IMPLICIT: - if (c->c_flags & CO_OPTIMIZED) - scope = NAME_GLOBAL; - break; - case FREE: - case CELL: - scope = NAME_CLOSURE; - break; - } + assert(block != NULL); + c->u->u_curblock->b_next = block; + c->u->u_curblock = block; + return block; +} - i = com_addname(c, v); - if (scope == NAME_LOCAL) - i = com_lookup_arg(c->c_locals, v); - else if (reftype == FREE) - i = com_lookup_arg(c->c_freevars, v); - else if (reftype == CELL) - i = com_lookup_arg(c->c_cellvars, v); - if (i == -1) { - c->c_errors++; /* XXX no exception set */ - i = 255; - goto done; - } - Py_DECREF(v); +/* Returns the offset of the next instruction in the current block's + b_instr array. Resizes the b_instr as necessary. + Returns -1 on failure. + */ - switch (kind) { - case VAR_LOAD: - switch (scope) { - case NAME_LOCAL: - op = LOAD_FAST; - break; - case NAME_GLOBAL: - op = LOAD_GLOBAL; - break; - case NAME_DEFAULT: - op = LOAD_NAME; - break; - case NAME_CLOSURE: - op = LOAD_DEREF; - break; - } - break; - case VAR_STORE: - switch (scope) { - case NAME_LOCAL: - op = STORE_FAST; - break; - case NAME_GLOBAL: - op = STORE_GLOBAL; - break; - case NAME_DEFAULT: - op = STORE_NAME; - break; - case NAME_CLOSURE: - op = STORE_DEREF; - break; - } - break; - case VAR_DELETE: - switch (scope) { - case NAME_LOCAL: - op = DELETE_FAST; - break; - case NAME_GLOBAL: - op = DELETE_GLOBAL; - break; - case NAME_DEFAULT: - op = DELETE_NAME; - break; - case NAME_CLOSURE: { - char buf[500]; - PyOS_snprintf(buf, sizeof(buf), - DEL_CLOSURE_ERROR, name); - com_error(c, PyExc_SyntaxError, buf); - i = 255; - break; +static int +compiler_next_instr(struct compiler *c, basicblock *b) +{ + assert(b != NULL); + if (b->b_instr == NULL) { + b->b_instr = PyObject_Malloc(sizeof(struct instr) * + DEFAULT_BLOCK_SIZE); + if (b->b_instr == NULL) { + PyErr_NoMemory(); + return -1; } + b->b_ialloc = DEFAULT_BLOCK_SIZE; + memset((char *)b->b_instr, 0, + sizeof(struct instr) * DEFAULT_BLOCK_SIZE); + } + else if (b->b_iused == b->b_ialloc) { + size_t oldsize, newsize; + oldsize = b->b_ialloc * sizeof(struct instr); + newsize = oldsize << 1; + if (newsize == 0) { + PyErr_NoMemory(); + return -1; } - break; + b->b_ialloc <<= 1; + b->b_instr = PyObject_Realloc((void *)b->b_instr, newsize); + if (b->b_instr == NULL) + return -1; + memset((char *)b->b_instr + oldsize, 0, newsize - oldsize); } -done: - com_addoparg(c, op, i); + return b->b_iused++; } static void -com_addopname(struct compiling *c, int op, node *n) -{ - char *name; - char buffer[1000]; - /* XXX it is possible to write this code without the 1000 - chars on the total length of dotted names, I just can't be - bothered right now */ - if (TYPE(n) == STAR) - name = "*"; - else if (TYPE(n) == dotted_name) { - char *p = buffer; - int i; - name = buffer; - for (i = 0; i < NCH(n); i += 2) { - char *s = STR(CHILD(n, i)); - if (p + strlen(s) > buffer + (sizeof buffer) - 2) { - com_error(c, PyExc_MemoryError, - "dotted_name too long"); - name = NULL; - break; - } - if (p != buffer) - *p++ = '.'; - strcpy(p, s); - p = strchr(p, '\0'); - } - } - else { - REQ(n, NAME); - name = STR(n); - } - com_addop_name(c, op, name); -} - -static PyObject * -parsenumber(struct compiling *c, char *s) +compiler_set_lineno(struct compiler *c, int off) { - char *end; - long x; - double dx; -#ifndef WITHOUT_COMPLEX - int imflag; -#endif - - errno = 0; - end = s + strlen(s) - 1; -#ifndef WITHOUT_COMPLEX - imflag = *end == 'j' || *end == 'J'; -#endif - if (*end == 'l' || *end == 'L') - return PyLong_FromString(s, (char **)0, 0); - if (s[0] == '0') { - x = (long) PyOS_strtoul(s, &end, 0); - if (x < 0 && errno == 0) { - return PyLong_FromString(s, (char **)0, 0); - } - } - else - x = PyOS_strtol(s, &end, 0); - if (*end == '\0') { - if (errno != 0) - return PyLong_FromString(s, (char **)0, 0); - return PyInt_FromLong(x); - } - /* XXX Huge floats may silently fail */ -#ifndef WITHOUT_COMPLEX - if (imflag) { - Py_complex z; - z.real = 0.; - PyFPE_START_PROTECT("atof", return 0) - z.imag = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(z) - return PyComplex_FromCComplex(z); - } - else -#endif - { - PyFPE_START_PROTECT("atof", return 0) - dx = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(dx) - return PyFloat_FromDouble(dx); - } -} - -static PyObject * -decode_utf8(char **sPtr, char *end, char* encoding) -{ -#ifndef Py_USING_UNICODE - Py_FatalError("decode_utf8 should not be called in this build."); - return NULL; -#else - PyObject *u, *v; - char *s, *t; - t = s = *sPtr; - /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */ - while (s < end && (*s & 0x80)) s++; - *sPtr = s; - u = PyUnicode_DecodeUTF8(t, s - t, NULL); - if (u == NULL) - return NULL; - v = PyUnicode_AsEncodedString(u, encoding, NULL); - Py_DECREF(u); - return v; -#endif + basicblock *b; + if (c->u->u_lineno_set) + return; + c->u->u_lineno_set = true; + b = c->u->u_curblock; + b->b_instr[off].i_lineno = c->u->u_lineno; } -/* compiler.transformer.Transformer.decode_literal depends on what - might seem like minor details of this function -- changes here - must be reflected there. */ -static PyObject * -parsestr(struct compiling *c, char *s) +static int +opcode_stack_effect(int opcode, int oparg) { - PyObject *v; - size_t len; - int quote = *s; - int rawmode = 0; - char* encoding = ((c == NULL) ? NULL : c->c_encoding); - int need_encoding; - int unicode = 0; - - if (isalpha(quote) || quote == '_') { - if (quote == 'u' || quote == 'U') { - quote = *++s; - unicode = 1; - } - if (quote == 'r' || quote == 'R') { - quote = *++s; - rawmode = 1; - } - } - if (quote != '\'' && quote != '\"') { - PyErr_BadInternalCall(); - return NULL; - } - s++; - len = strlen(s); - if (len > INT_MAX) { - com_error(c, PyExc_OverflowError, - "string to parse is too long"); - return NULL; - } - if (s[--len] != quote) { - PyErr_BadInternalCall(); - return NULL; - } - if (len >= 4 && s[0] == quote && s[1] == quote) { - s += 2; - len -= 2; - if (s[--len] != quote || s[--len] != quote) { - PyErr_BadInternalCall(); - return NULL; - } - } -#ifdef Py_USING_UNICODE - if (unicode || Py_UnicodeFlag) { - PyObject *u, *w; - char *buf; - char *p; - char *end; - if (encoding == NULL) { - buf = s; - u = NULL; - } else if (strcmp(encoding, "iso-8859-1") == 0) { - buf = s; - u = NULL; - } else { - /* "\XX" may become "\u005c\uHHLL" (12 bytes) */ - u = PyString_FromStringAndSize((char *)NULL, len * 4); - if (u == NULL) - return NULL; - p = buf = PyString_AsString(u); - end = s + len; - while (s < end) { - if (*s == '\\') { - *p++ = *s++; - if (*s & 0x80) { - strcpy(p, "u005c"); - p += 5; - } - } - if (*s & 0x80) { /* XXX inefficient */ - char *r; - int rn, i; - w = decode_utf8(&s, end, "utf-16-be"); - if (w == NULL) { - Py_DECREF(u); - return NULL; - } - r = PyString_AsString(w); - rn = PyString_Size(w); - assert(rn % 2 == 0); - for (i = 0; i < rn; i += 2) { - sprintf(p, "\\u%02x%02x", - r[i + 0] & 0xFF, - r[i + 1] & 0xFF); - p += 6; - } - Py_DECREF(w); - } else { - *p++ = *s++; - } - } - len = p - buf; - } - if (rawmode) - v = PyUnicode_DecodeRawUnicodeEscape(buf, len, NULL); - else - v = PyUnicode_DecodeUnicodeEscape(buf, len, NULL); - Py_XDECREF(u); - if (v == NULL) - PyErr_SyntaxLocation(c->c_filename, c->c_lineno); - return v; - - } -#endif - need_encoding = (encoding != NULL && - strcmp(encoding, "utf-8") != 0 && - strcmp(encoding, "iso-8859-1") != 0); - if (rawmode || strchr(s, '\\') == NULL) { - if (need_encoding) { -#ifndef Py_USING_UNICODE - /* This should not happen - we never see any other - encoding. */ - Py_FatalError("cannot deal with encodings in this build."); -#else - PyObject* u = PyUnicode_DecodeUTF8(s, len, NULL); - if (u == NULL) - return NULL; - v = PyUnicode_AsEncodedString(u, encoding, NULL); - Py_DECREF(u); - return v; -#endif - } else { - return PyString_FromStringAndSize(s, len); - } - } + switch (opcode) { + case POP_TOP: + return -1; + case ROT_TWO: + case ROT_THREE: + return 0; + case DUP_TOP: + return 1; + case ROT_FOUR: + return 0; - v = PyString_DecodeEscape(s, len, NULL, unicode, - need_encoding ? encoding : NULL); - if (v == NULL) - PyErr_SyntaxLocation(c->c_filename, c->c_lineno); - return v; -} + case UNARY_POSITIVE: + case UNARY_NEGATIVE: + case UNARY_NOT: + case UNARY_CONVERT: + case UNARY_INVERT: + return 0; -static PyObject * -parsestrplus(struct compiling* c, node *n) -{ - PyObject *v; - int i; - REQ(CHILD(n, 0), STRING); - if ((v = parsestr(c, STR(CHILD(n, 0)))) != NULL) { - /* String literal concatenation */ - for (i = 1; i < NCH(n); i++) { - PyObject *s; - s = parsestr(c, STR(CHILD(n, i))); - if (s == NULL) - goto onError; - if (PyString_Check(v) && PyString_Check(s)) { - PyString_ConcatAndDel(&v, s); - if (v == NULL) - goto onError; - } -#ifdef Py_USING_UNICODE - else { - PyObject *temp; - temp = PyUnicode_Concat(v, s); - Py_DECREF(s); - if (temp == NULL) - goto onError; - Py_DECREF(v); - v = temp; - } -#endif - } - } - return v; + case BINARY_POWER: + case BINARY_MULTIPLY: + case BINARY_DIVIDE: + case BINARY_MODULO: + case BINARY_ADD: + case BINARY_SUBTRACT: + case BINARY_SUBSCR: + case BINARY_FLOOR_DIVIDE: + case BINARY_TRUE_DIVIDE: + return -1; + case INPLACE_FLOOR_DIVIDE: + case INPLACE_TRUE_DIVIDE: + return -1; - onError: - Py_XDECREF(v); - return NULL; -} + case SLICE+0: + return 1; + case SLICE+1: + return 0; + case SLICE+2: + return 0; + case SLICE+3: + return -1; -static void -com_list_for(struct compiling *c, node *n, node *e, char *t) -{ - int anchor = 0; - int save_begin = c->c_begin; - - /* list_for: for v in expr [list_iter] */ - com_node(c, CHILD(n, 3)); /* expr */ - com_addbyte(c, GET_ITER); - c->c_begin = c->c_nexti; - com_addfwref(c, FOR_ITER, &anchor); - com_push(c, 1); - com_assign(c, CHILD(n, 1), OP_ASSIGN, NULL); - c->c_loops++; - com_list_iter(c, n, e, t); - c->c_loops--; - com_addoparg(c, JUMP_ABSOLUTE, c->c_begin); - c->c_begin = save_begin; - com_backpatch(c, anchor); - com_pop(c, 1); /* FOR_ITER has popped this */ -} + case STORE_SLICE+0: + return -2; + case STORE_SLICE+1: + return -3; + case STORE_SLICE+2: + return -3; + case STORE_SLICE+3: + return -4; -static void -com_gen_for(struct compiling *c, node *n, node *t, int is_outmost) -{ - int break_anchor = 0; - int anchor = 0; - int save_begin = c->c_begin; + case DELETE_SLICE+0: + return -1; + case DELETE_SLICE+1: + return -2; + case DELETE_SLICE+2: + return -2; + case DELETE_SLICE+3: + return -3; + + case INPLACE_ADD: + case INPLACE_SUBTRACT: + case INPLACE_MULTIPLY: + case INPLACE_DIVIDE: + case INPLACE_MODULO: + return -1; + case STORE_SUBSCR: + return -3; + case DELETE_SUBSCR: + return -2; - REQ(n, gen_for); - /* gen_for: for v in test [gen_iter] */ + case BINARY_LSHIFT: + case BINARY_RSHIFT: + case BINARY_AND: + case BINARY_XOR: + case BINARY_OR: + return -1; + case INPLACE_POWER: + return -1; + case GET_ITER: + return 0; - com_addfwref(c, SETUP_LOOP, &break_anchor); - block_push(c, SETUP_LOOP); + case PRINT_EXPR: + return -1; + case PRINT_ITEM: + return -1; + case PRINT_NEWLINE: + return 0; + case PRINT_ITEM_TO: + return -2; + case PRINT_NEWLINE_TO: + return -1; + case INPLACE_LSHIFT: + case INPLACE_RSHIFT: + case INPLACE_AND: + case INPLACE_XOR: + case INPLACE_OR: + return -1; + case BREAK_LOOP: + return 0; - if (is_outmost) { - com_addop_varname(c, VAR_LOAD, "[outmost-iterable]"); - com_push(c, 1); - } - else { - com_node(c, CHILD(n, 3)); - com_addbyte(c, GET_ITER); - } + case LOAD_LOCALS: + return 1; + case RETURN_VALUE: + return -1; + case IMPORT_STAR: + return -1; + case EXEC_STMT: + return -3; + case YIELD_VALUE: + return 0; - c->c_begin = c->c_nexti; - com_set_lineno(c, c->c_last_line); - com_addfwref(c, FOR_ITER, &anchor); - com_push(c, 1); - com_assign(c, CHILD(n, 1), OP_ASSIGN, NULL); + case POP_BLOCK: + return 0; + case END_FINALLY: + return -1; /* or -2 or -3 if exception occurred */ + case BUILD_CLASS: + return -2; - if (NCH(n) == 5) - com_gen_iter(c, CHILD(n, 4), t); - else { - com_test(c, t); - com_addbyte(c, YIELD_VALUE); - com_addbyte(c, POP_TOP); - com_pop(c, 1); - } + case STORE_NAME: + return -1; + case DELETE_NAME: + return 0; + case UNPACK_SEQUENCE: + return oparg-1; + case FOR_ITER: + return 1; - com_addoparg(c, JUMP_ABSOLUTE, c->c_begin); - c->c_begin = save_begin; + case STORE_ATTR: + return -2; + case DELETE_ATTR: + return -1; + case STORE_GLOBAL: + return -1; + case DELETE_GLOBAL: + return 0; + case DUP_TOPX: + return oparg; + case LOAD_CONST: + return 1; + case LOAD_NAME: + return 1; + case BUILD_TUPLE: + case BUILD_LIST: + return 1-oparg; + case BUILD_MAP: + return 1; + case LOAD_ATTR: + return 0; + case COMPARE_OP: + return -1; + case IMPORT_NAME: + return 0; + case IMPORT_FROM: + return 1; - com_backpatch(c, anchor); - com_pop(c, 1); /* FOR_ITER has popped this */ - com_addbyte(c, POP_BLOCK); - block_pop(c, SETUP_LOOP); - com_backpatch(c, break_anchor); -} + case JUMP_FORWARD: + case JUMP_IF_FALSE: + case JUMP_IF_TRUE: + case JUMP_ABSOLUTE: + return 0; -static void -com_list_if(struct compiling *c, node *n, node *e, char *t) -{ - int anchor = 0; - int a = 0; - /* list_iter: 'if' test [list_iter] */ - com_node(c, CHILD(n, 1)); - com_addfwref(c, JUMP_IF_FALSE, &a); - com_addbyte(c, POP_TOP); - com_pop(c, 1); - com_list_iter(c, n, e, t); - com_addfwref(c, JUMP_FORWARD, &anchor); - com_backpatch(c, a); - /* We jump here with an extra entry which we now pop */ - com_addbyte(c, POP_TOP); - com_backpatch(c, anchor); -} + case LOAD_GLOBAL: + return 1; -static void -com_gen_if(struct compiling *c, node *n, node *t) -{ - /* gen_if: 'if' test [gen_iter] */ - int anchor = 0; - int a=0; + case CONTINUE_LOOP: + return 0; + case SETUP_LOOP: + return 0; + case SETUP_EXCEPT: + case SETUP_FINALLY: + return 3; /* actually pushed by an exception */ - com_node(c, CHILD(n, 1)); - com_addfwref(c, JUMP_IF_FALSE, &a); - com_addbyte(c, POP_TOP); - com_pop(c, 1); + case LOAD_FAST: + return 1; + case STORE_FAST: + return -1; + case DELETE_FAST: + return 0; - if (NCH(n) == 3) - com_gen_iter(c, CHILD(n, 2), t); - else { - com_test(c, t); - com_addbyte(c, YIELD_VALUE); - com_addbyte(c, POP_TOP); - com_pop(c, 1); - } - com_addfwref(c, JUMP_FORWARD, &anchor); - com_backpatch(c, a); - /* We jump here with an extra entry which we now pop */ - com_addbyte(c, POP_TOP); - com_backpatch(c, anchor); -} + case RAISE_VARARGS: + return -oparg; +#define NARGS(o) (((o) % 256) + 2*((o) / 256)) + case CALL_FUNCTION: + return -NARGS(oparg); + case CALL_FUNCTION_VAR: + case CALL_FUNCTION_KW: + return -NARGS(oparg)-1; + case CALL_FUNCTION_VAR_KW: + return -NARGS(oparg)-2; +#undef NARGS + case MAKE_FUNCTION: + return -oparg; + case BUILD_SLICE: + if (oparg == 3) + return -2; + else + return -1; -static void -com_list_iter(struct compiling *c, - node *p, /* parent of list_iter node */ - node *e, /* element expression node */ - char *t /* name of result list temp local */) -{ - /* list_iter is the last child in a listmaker, list_for, or list_if */ - node *n = CHILD(p, NCH(p)-1); - if (TYPE(n) == list_iter) { - n = CHILD(n, 0); - switch (TYPE(n)) { - case list_for: - com_list_for(c, n, e, t); - break; - case list_if: - com_list_if(c, n, e, t); - break; + case MAKE_CLOSURE: + return -oparg; + case LOAD_CLOSURE: + return 1; + case LOAD_DEREF: + return 1; + case STORE_DEREF: + return -1; default: - com_error(c, PyExc_SystemError, - "invalid list_iter node type"); - } - } - else { - com_addop_varname(c, VAR_LOAD, t); - com_push(c, 1); - com_node(c, e); - com_addbyte(c, LIST_APPEND); - com_pop(c, 2); - } -} - -static void -com_gen_iter(struct compiling *c, node *n, node *t) -{ - /* gen_iter: gen_for | gen_if */ - node *ch; - REQ(n, gen_iter); - - ch = CHILD(n, 0); + fprintf(stderr, "opcode = %d\n", opcode); + Py_FatalError("opcode_stack_effect()"); - switch (TYPE(ch)) { - case gen_for: - com_gen_for(c, ch, t, 0); - break; - case gen_if: - com_gen_if(c, ch, t); - break; - default: - com_error(c, PyExc_SystemError, - "invalid gen_iter node type"); } + return 0; /* not reachable */ } -static void -com_list_comprehension(struct compiling *c, node *n) -{ - /* listmaker: test list_for */ - char tmpname[30]; - - REQ(n, listmaker); - PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", ++c->c_tmpname); - com_addoparg(c, BUILD_LIST, 0); - com_addbyte(c, DUP_TOP); /* leave the result on the stack */ - com_push(c, 2); - com_addop_varname(c, VAR_STORE, tmpname); - com_pop(c, 1); - com_list_for(c, CHILD(n, 1), CHILD(n, 0), tmpname); - com_addop_varname(c, VAR_DELETE, tmpname); - --c->c_tmpname; -} +/* Add an opcode with no argument. + Returns 0 on failure, 1 on success. +*/ -static void -com_listmaker(struct compiling *c, node *n) +static int +compiler_addop(struct compiler *c, int opcode) { - /* listmaker: test ( list_for | (',' test)* [','] ) */ - if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == list_for) - com_list_comprehension(c, n); - else { - int len = 0; - int i; - for (i = 0; i < NCH(n); i += 2, len++) - com_node(c, CHILD(n, i)); - com_addoparg(c, BUILD_LIST, len); - com_pop(c, len-1); - } + basicblock *b; + struct instr *i; + int off; + off = compiler_next_instr(c, c->u->u_curblock); + if (off < 0) + return 0; + b = c->u->u_curblock; + i = &b->b_instr[off]; + i->i_opcode = opcode; + i->i_hasarg = 0; + if (opcode == RETURN_VALUE) + b->b_return = 1; + compiler_set_lineno(c, off); + return 1; } -static void -com_generator_expression(struct compiling *c, node *n) -{ - /* testlist_gexp: test gen_for */ - /* argument: test gen_for */ - PyCodeObject *co; - - REQ(CHILD(n, 0), test); - REQ(CHILD(n, 1), gen_for); - - symtable_enter_scope(c->c_symtable, "<genexpr>", TYPE(n), - n->n_lineno); - co = icompile(n, c); - symtable_exit_scope(c->c_symtable); - - if (co == NULL) - c->c_errors++; - else { - int closure = com_make_closure(c, co); - int i = com_addconst(c, (PyObject *)co); - - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - if (closure) - com_addoparg(c, MAKE_CLOSURE, 0); - else - com_addoparg(c, MAKE_FUNCTION, 0); - - com_test(c, CHILD(CHILD(n, 1), 3)); - com_addbyte(c, GET_ITER); - com_addoparg(c, CALL_FUNCTION, 1); - com_pop(c, 1); - - Py_DECREF(co); +static int +compiler_add_o(struct compiler *c, PyObject *dict, PyObject *o) +{ + PyObject *t, *v; + int arg; + + /* necessary to make sure types aren't coerced (e.g., int and long) */ + /* XXX should use: t = PyTuple_Pack(2, o, o->ob_type); */ + t = Py_BuildValue("(OO)", o, o->ob_type); + if (t == NULL) + return -1; + + v = PyDict_GetItem(dict, t); + if (!v) { + arg = PyDict_Size(dict); + v = PyInt_FromLong(arg); + if (!v) { + Py_DECREF(t); + return -1; + } + if (PyDict_SetItem(dict, t, v) < 0) { + Py_DECREF(t); + Py_DECREF(v); + return -1; + } + Py_DECREF(v); } + else + arg = PyInt_AsLong(v); + Py_DECREF(t); + return arg; } -static void -com_testlist_gexp(struct compiling *c, node *n) +static int +compiler_addop_o(struct compiler *c, int opcode, PyObject *dict, + PyObject *o) { - /* testlist_gexp: test ( gen_for | (',' test)* [','] ) */ - if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for) - com_generator_expression(c, n); - else com_list(c, n, 0); + int arg = compiler_add_o(c, dict, o); + if (arg < 0) + return 0; + return compiler_addop_i(c, opcode, arg); } +static int +compiler_addop_name(struct compiler *c, int opcode, PyObject *dict, + PyObject *o) +{ + int arg; + PyObject *mangled = _Py_Mangle(c->u->u_private, o); + if (!mangled) + return 0; + arg = compiler_add_o(c, dict, mangled); + Py_DECREF(mangled); + if (arg < 0) + return 0; + return compiler_addop_i(c, opcode, arg); +} + +/* Add an opcode with an integer argument. + Returns 0 on failure, 1 on success. +*/ -static void -com_dictmaker(struct compiling *c, node *n) +static int +compiler_addop_i(struct compiler *c, int opcode, int oparg) { - int i; - /* dictmaker: test ':' test (',' test ':' value)* [','] */ - for (i = 0; i+2 < NCH(n); i += 4) { - /* We must arrange things just right for STORE_SUBSCR. - It wants the stack to look like (value) (dict) (key) */ - com_addbyte(c, DUP_TOP); - com_push(c, 1); - com_node(c, CHILD(n, i)); /* key */ - com_node(c, CHILD(n, i+2)); /* value */ - com_addbyte(c, ROT_THREE); - com_addbyte(c, STORE_SUBSCR); - com_pop(c, 3); - } + struct instr *i; + int off; + off = compiler_next_instr(c, c->u->u_curblock); + if (off < 0) + return 0; + i = &c->u->u_curblock->b_instr[off]; + i->i_opcode = opcode; + i->i_oparg = oparg; + i->i_hasarg = 1; + compiler_set_lineno(c, off); + return 1; } - -/* forward reference */ -static void com_yield_expr(struct compiling *c, node *n); - -static void -com_atom(struct compiling *c, node *n) +static int +compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) { - node *ch; - PyObject *v; - int i; - REQ(n, atom); - ch = CHILD(n, 0); - switch (TYPE(ch)) { - case LPAR: - if (TYPE(CHILD(n, 1)) == RPAR) { - com_addoparg(c, BUILD_TUPLE, 0); - com_push(c, 1); - } - else - if (TYPE(CHILD(n, 1)) == yield_expr) - com_yield_expr(c, CHILD(n, 1)); - else - com_testlist_gexp(c, CHILD(n, 1)); - break; - case LSQB: /* '[' [listmaker] ']' */ - if (TYPE(CHILD(n, 1)) == RSQB) { - com_addoparg(c, BUILD_LIST, 0); - com_push(c, 1); - } - else - com_listmaker(c, CHILD(n, 1)); - break; - case LBRACE: /* '{' [dictmaker] '}' */ - com_addoparg(c, BUILD_MAP, 0); - com_push(c, 1); - if (TYPE(CHILD(n, 1)) == dictmaker) - com_dictmaker(c, CHILD(n, 1)); - break; - case BACKQUOTE: - com_node(c, CHILD(n, 1)); - com_addbyte(c, UNARY_CONVERT); - break; - case NUMBER: - if ((v = parsenumber(c, STR(ch))) == NULL) { - i = 255; - } - else { - i = com_addconst(c, v); - Py_DECREF(v); - } - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - break; - case STRING: - v = parsestrplus(c, n); - if (v == NULL) { - c->c_errors++; - i = 255; - } - else { - i = com_addconst(c, v); - Py_DECREF(v); - } - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - break; - case NAME: - com_addop_varname(c, VAR_LOAD, STR(ch)); - com_push(c, 1); - break; - default: - com_error(c, PyExc_SystemError, - "com_atom: unexpected node type"); - } -} + struct instr *i; + int off; -static void -com_slice(struct compiling *c, node *n, int op) -{ - if (NCH(n) == 1) { - com_addbyte(c, op); - } - else if (NCH(n) == 2) { - if (TYPE(CHILD(n, 0)) != COLON) { - com_node(c, CHILD(n, 0)); - com_addbyte(c, op+1); - } - else { - com_node(c, CHILD(n, 1)); - com_addbyte(c, op+2); - } - com_pop(c, 1); - } - else { - com_node(c, CHILD(n, 0)); - com_node(c, CHILD(n, 2)); - com_addbyte(c, op+3); - com_pop(c, 2); - } + assert(b != NULL); + off = compiler_next_instr(c, c->u->u_curblock); + if (off < 0) + return 0; + compiler_set_lineno(c, off); + i = &c->u->u_curblock->b_instr[off]; + i->i_opcode = opcode; + i->i_target = b; + i->i_hasarg = 1; + if (absolute) + i->i_jabs = 1; + else + i->i_jrel = 1; + return 1; } -static void -com_augassign_slice(struct compiling *c, node *n, int opcode, node *augn) -{ - if (NCH(n) == 1) { - com_addbyte(c, DUP_TOP); - com_push(c, 1); - com_addbyte(c, SLICE); - com_node(c, augn); - com_addbyte(c, opcode); - com_pop(c, 1); - com_addbyte(c, ROT_TWO); - com_addbyte(c, STORE_SLICE); - com_pop(c, 2); - } else if (NCH(n) == 2 && TYPE(CHILD(n, 0)) != COLON) { - com_node(c, CHILD(n, 0)); - com_addoparg(c, DUP_TOPX, 2); - com_push(c, 2); - com_addbyte(c, SLICE+1); - com_pop(c, 1); - com_node(c, augn); - com_addbyte(c, opcode); - com_pop(c, 1); - com_addbyte(c, ROT_THREE); - com_addbyte(c, STORE_SLICE+1); - com_pop(c, 3); - } else if (NCH(n) == 2) { - com_node(c, CHILD(n, 1)); - com_addoparg(c, DUP_TOPX, 2); - com_push(c, 2); - com_addbyte(c, SLICE+2); - com_pop(c, 1); - com_node(c, augn); - com_addbyte(c, opcode); - com_pop(c, 1); - com_addbyte(c, ROT_THREE); - com_addbyte(c, STORE_SLICE+2); - com_pop(c, 3); - } else { - com_node(c, CHILD(n, 0)); - com_node(c, CHILD(n, 2)); - com_addoparg(c, DUP_TOPX, 3); - com_push(c, 3); - com_addbyte(c, SLICE+3); - com_pop(c, 2); - com_node(c, augn); - com_addbyte(c, opcode); - com_pop(c, 1); - com_addbyte(c, ROT_FOUR); - com_addbyte(c, STORE_SLICE+3); - com_pop(c, 4); - } -} +/* The distinction between NEW_BLOCK and NEXT_BLOCK is subtle. (I'd + like to find better names.) NEW_BLOCK() creates a new block and sets + it as the current block. NEXT_BLOCK() also creates an implicit jump + from the current block to the new block. +*/ -static void -com_argument(struct compiling *c, node *n, PyObject **pkeywords) -{ - node *m; - REQ(n, argument); /* [test '='] test [gen_for]; really [keyword '='] test */ - if (NCH(n) == 1) { - if (*pkeywords != NULL) { - com_error(c, PyExc_SyntaxError, - "non-keyword arg after keyword arg"); - } - else { - com_node(c, CHILD(n, 0)); - } - return; - } - if (NCH(n) == 2) { - com_generator_expression(c, n); - return; - } +/* XXX The returns inside these macros make it impossible to decref + objects created in the local function. +*/ - m = n; - do { - m = CHILD(m, 0); - } while (NCH(m) == 1); - if (TYPE(m) != NAME) { - /* f(lambda x: x[0] = 3) ends up getting parsed with - * LHS test = lambda x: x[0], and RHS test = 3. - * SF bug 132313 points out that complaining about a keyword - * then is very confusing. - */ - com_error(c, PyExc_SyntaxError, - TYPE(m) == lambdef ? - "lambda cannot contain assignment" : - "keyword can't be an expression"); - } - else { - PyObject *v = PyString_InternFromString(STR(m)); - (void) none_assignment_check(c, STR(m), 1); - if (v != NULL && *pkeywords == NULL) - *pkeywords = PyDict_New(); - if (v == NULL) - c->c_errors++; - else if (*pkeywords == NULL) { - c->c_errors++; - Py_DECREF(v); - } else { - if (PyDict_GetItem(*pkeywords, v) != NULL) - com_error(c, PyExc_SyntaxError, - "duplicate keyword argument"); - else - if (PyDict_SetItem(*pkeywords, v, v) != 0) - c->c_errors++; - com_addoparg(c, LOAD_CONST, com_addconst(c, v)); - com_push(c, 1); - Py_DECREF(v); - } - } - com_node(c, CHILD(n, 2)); -} -static void -com_call_function(struct compiling *c, node *n) -{ - if (TYPE(n) == RPAR) { - com_addoparg(c, CALL_FUNCTION, 0); - } - else { - PyObject *keywords = NULL; - int i, na, nk; - int lineno = n->n_lineno; - int star_flag = 0; - int starstar_flag = 0; - int opcode; - REQ(n, arglist); - na = 0; - nk = 0; - for (i = 0; i < NCH(n); i += 2) { - node *ch = CHILD(n, i); - if (TYPE(ch) == STAR || - TYPE(ch) == DOUBLESTAR) - break; - if (ch->n_lineno != lineno) { - lineno = ch->n_lineno; - com_set_lineno(c, lineno); - } - com_argument(c, ch, &keywords); - if (keywords == NULL) - na++; - else - nk++; - } - Py_XDECREF(keywords); - while (i < NCH(n)) { - node *tok = CHILD(n, i); - node *ch = CHILD(n, i+1); - i += 3; - switch (TYPE(tok)) { - case STAR: star_flag = 1; break; - case DOUBLESTAR: starstar_flag = 1; break; - } - com_node(c, ch); - } - if (na > 255 || nk > 255) { - com_error(c, PyExc_SyntaxError, - "more than 255 arguments"); - } - if (star_flag || starstar_flag) - opcode = CALL_FUNCTION_VAR - 1 + - star_flag + (starstar_flag << 1); - else - opcode = CALL_FUNCTION; - com_addoparg(c, opcode, na | (nk << 8)); - com_pop(c, na + 2*nk + star_flag + starstar_flag); - } +#define NEW_BLOCK(C) { \ + if (compiler_use_new_block((C)) == NULL) \ + return 0; \ } -static void -com_select_member(struct compiling *c, node *n) -{ - com_addopname(c, LOAD_ATTR, n); +#define NEXT_BLOCK(C) { \ + if (compiler_next_block((C)) == NULL) \ + return 0; \ } -static void -com_sliceobj(struct compiling *c, node *n) -{ - int i=0; - int ns=2; /* number of slice arguments */ - node *ch; - - /* first argument */ - if (TYPE(CHILD(n,i)) == COLON) { - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - i++; - } - else { - com_node(c, CHILD(n,i)); - i++; - REQ(CHILD(n,i),COLON); - i++; - } - /* second argument */ - if (i < NCH(n) && TYPE(CHILD(n,i)) == test) { - com_node(c, CHILD(n,i)); - i++; - } - else { - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - } - /* remaining arguments */ - for (; i < NCH(n); i++) { - ns++; - ch=CHILD(n,i); - REQ(ch, sliceop); - if (NCH(ch) == 1) { - /* right argument of ':' missing */ - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - } - else - com_node(c, CHILD(ch,1)); - } - com_addoparg(c, BUILD_SLICE, ns); - com_pop(c, 1 + (ns == 3)); +#define ADDOP(C, OP) { \ + if (!compiler_addop((C), (OP))) \ + return 0; \ } -static void -com_subscript(struct compiling *c, node *n) -{ - node *ch; - REQ(n, subscript); - ch = CHILD(n,0); - /* check for rubber index */ - if (TYPE(ch) == DOT && TYPE(CHILD(n,1)) == DOT) { - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_Ellipsis)); - com_push(c, 1); - } - else { - /* check for slice */ - if ((TYPE(ch) == COLON || NCH(n) > 1)) - com_sliceobj(c, n); - else { - REQ(ch, test); - com_node(c, ch); - } - } +#define ADDOP_O(C, OP, O, TYPE) { \ + if (!compiler_addop_o((C), (OP), (C)->u->u_ ## TYPE, (O))) \ + return 0; \ } -static void -com_subscriptlist(struct compiling *c, node *n, int assigning, node *augn) -{ - int i, op; - REQ(n, subscriptlist); - /* Check to make backward compatible slice behavior for '[i:j]' */ - if (NCH(n) == 1) { - node *sub = CHILD(n, 0); /* subscript */ - /* 'Basic' slice, should have exactly one colon. */ - if ((TYPE(CHILD(sub, 0)) == COLON - || (NCH(sub) > 1 && TYPE(CHILD(sub, 1)) == COLON)) - && (TYPE(CHILD(sub,NCH(sub)-1)) != sliceop)) - { - switch (assigning) { - case OP_DELETE: - op = DELETE_SLICE; - break; - case OP_ASSIGN: - op = STORE_SLICE; - break; - case OP_APPLY: - op = SLICE; - break; - default: - com_augassign_slice(c, sub, assigning, augn); - return; - } - com_slice(c, sub, op); - if (op == STORE_SLICE) - com_pop(c, 2); - else if (op == DELETE_SLICE) - com_pop(c, 1); - return; - } - } - /* Else normal subscriptlist. Compile each subscript. */ - for (i = 0; i < NCH(n); i += 2) - com_subscript(c, CHILD(n, i)); - /* Put multiple subscripts into a tuple */ - if (NCH(n) > 1) { - i = (NCH(n)+1) / 2; - com_addoparg(c, BUILD_TUPLE, i); - com_pop(c, i-1); - } - switch (assigning) { - case OP_DELETE: - op = DELETE_SUBSCR; - i = 2; - break; - default: - case OP_ASSIGN: - op = STORE_SUBSCR; - i = 3; - break; - case OP_APPLY: - op = BINARY_SUBSCR; - i = 1; - break; - } - if (assigning > OP_APPLY) { - com_addoparg(c, DUP_TOPX, 2); - com_push(c, 2); - com_addbyte(c, BINARY_SUBSCR); - com_pop(c, 1); - com_node(c, augn); - com_addbyte(c, assigning); - com_pop(c, 1); - com_addbyte(c, ROT_THREE); - } - com_addbyte(c, op); - com_pop(c, i); +#define ADDOP_NAME(C, OP, O, TYPE) { \ + if (!compiler_addop_name((C), (OP), (C)->u->u_ ## TYPE, (O))) \ + return 0; \ } -static void -com_apply_trailer(struct compiling *c, node *n) -{ - REQ(n, trailer); - switch (TYPE(CHILD(n, 0))) { - case LPAR: - com_call_function(c, CHILD(n, 1)); - break; - case DOT: - com_select_member(c, CHILD(n, 1)); - break; - case LSQB: - com_subscriptlist(c, CHILD(n, 1), OP_APPLY, NULL); - break; - default: - com_error(c, PyExc_SystemError, - "com_apply_trailer: unknown trailer type"); - } +#define ADDOP_I(C, OP, O) { \ + if (!compiler_addop_i((C), (OP), (O))) \ + return 0; \ } -static void -com_power(struct compiling *c, node *n) -{ - int i; - REQ(n, power); - com_atom(c, CHILD(n, 0)); - for (i = 1; i < NCH(n); i++) { - if (TYPE(CHILD(n, i)) == DOUBLESTAR) { - com_factor(c, CHILD(n, i+1)); - com_addbyte(c, BINARY_POWER); - com_pop(c, 1); - break; - } - else - com_apply_trailer(c, CHILD(n, i)); - } +#define ADDOP_JABS(C, OP, O) { \ + if (!compiler_addop_j((C), (OP), (O), 1)) \ + return 0; \ } -static void -com_invert_constant(struct compiling *c, node *n) -{ - /* Compute the inverse of int and longs and use them directly, - but be prepared to generate code for all other - possibilities (invalid numbers, floats, complex). - */ - PyObject *num, *inv = NULL; - int i; - - REQ(n, NUMBER); - num = parsenumber(c, STR(n)); - if (num == NULL) - i = 255; - else { - inv = PyNumber_Invert(num); - if (inv == NULL) { - PyErr_Clear(); - i = com_addconst(c, num); - } else { - i = com_addconst(c, inv); - Py_DECREF(inv); - } - Py_DECREF(num); - } - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - if (num != NULL && inv == NULL) - com_addbyte(c, UNARY_INVERT); +#define ADDOP_JREL(C, OP, O) { \ + if (!compiler_addop_j((C), (OP), (O), 0)) \ + return 0; \ } -static int -is_float_zero(const char *p) -{ - int found_radix_point = 0; - int ch; - while ((ch = Py_CHARMASK(*p++)) != '\0') { - switch (ch) { - case '0': - /* no reason to believe it's not 0 -- continue */ - break; - - case 'e': case 'E': case 'j': case 'J': - /* If this was a hex constant, we already would have - returned 0 due to the 'x' or 'X', so 'e' or 'E' - must be an exponent marker, and we haven't yet - seen a non-zero digit, and it doesn't matter what - the exponent is then. For 'j' or 'J' similarly, - except that this is an imaginary 0 then. */ - return 1; - - case '.': - found_radix_point = 1; - break; +/* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use + the ASDL name to synthesize the name of the C type and the visit function. +*/ - default: - return 0; - } - } - return found_radix_point; +#define VISIT(C, TYPE, V) {\ + if (!compiler_visit_ ## TYPE((C), (V))) \ + return 0; \ } -static void -com_factor(struct compiling *c, node *n) -{ - int childtype = TYPE(CHILD(n, 0)); - node *pfactor, *ppower, *patom, *pnum; - REQ(n, factor); - /* If the unary +, -, or ~ operator is applied to a constant, - don't generate a UNARY_xxx opcode. Just store the - approriate value as a constant. If the value is negative, - extend the string containing the constant and insert a - negative in the 0th position -- unless we're doing unary minus - of a floating zero! In that case the sign is significant, but - the const dict can't distinguish +0.0 from -0.0. - */ - if ((childtype == PLUS || childtype == MINUS || childtype == TILDE) - && NCH(n) == 2 - && TYPE((pfactor = CHILD(n, 1))) == factor - && NCH(pfactor) == 1 - && TYPE((ppower = CHILD(pfactor, 0))) == power - && NCH(ppower) == 1 - && TYPE((patom = CHILD(ppower, 0))) == atom - && TYPE((pnum = CHILD(patom, 0))) == NUMBER - && !(childtype == MINUS && - (STR(pnum)[0] == '0' || is_float_zero(STR(pnum))))) { - if (childtype == TILDE) { - com_invert_constant(c, pnum); - return; - } - if (childtype == MINUS) { - char *s = PyObject_MALLOC(strlen(STR(pnum)) + 2); - if (s == NULL) { - com_error(c, PyExc_MemoryError, ""); - com_addbyte(c, 255); - return; - } - s[0] = '-'; - strcpy(s + 1, STR(pnum)); - PyObject_FREE(STR(pnum)); - STR(pnum) = s; - } - com_atom(c, patom); - } - else if (childtype == PLUS) { - com_factor(c, CHILD(n, 1)); - com_addbyte(c, UNARY_POSITIVE); - } - else if (childtype == MINUS) { - com_factor(c, CHILD(n, 1)); - com_addbyte(c, UNARY_NEGATIVE); - } - else if (childtype == TILDE) { - com_factor(c, CHILD(n, 1)); - com_addbyte(c, UNARY_INVERT); - } - else { - com_power(c, CHILD(n, 0)); - } +#define VISIT_SLICE(C, V, CTX) {\ + if (!compiler_visit_slice((C), (V), (CTX))) \ + return 0; \ } -static void -com_term(struct compiling *c, node *n) -{ - int i; - int op; - REQ(n, term); - com_factor(c, CHILD(n, 0)); - for (i = 2; i < NCH(n); i += 2) { - com_factor(c, CHILD(n, i)); - switch (TYPE(CHILD(n, i-1))) { - case STAR: - op = BINARY_MULTIPLY; - break; - case SLASH: - if (c->c_flags & CO_FUTURE_DIVISION) - op = BINARY_TRUE_DIVIDE; - else - op = BINARY_DIVIDE; - break; - case PERCENT: - op = BINARY_MODULO; - break; - case DOUBLESLASH: - op = BINARY_FLOOR_DIVIDE; - break; - default: - com_error(c, PyExc_SystemError, - "com_term: operator not *, /, // or %"); - op = 255; - } - com_addbyte(c, op); - com_pop(c, 1); - } -} - -static void -com_arith_expr(struct compiling *c, node *n) -{ - int i; - int op; - REQ(n, arith_expr); - com_term(c, CHILD(n, 0)); - for (i = 2; i < NCH(n); i += 2) { - com_term(c, CHILD(n, i)); - switch (TYPE(CHILD(n, i-1))) { - case PLUS: - op = BINARY_ADD; - break; - case MINUS: - op = BINARY_SUBTRACT; - break; - default: - com_error(c, PyExc_SystemError, - "com_arith_expr: operator not + or -"); - op = 255; - } - com_addbyte(c, op); - com_pop(c, 1); - } +#define VISIT_SEQ(C, TYPE, SEQ) { \ + int i; \ + asdl_seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = asdl_seq_GET(seq, i); \ + if (!compiler_visit_ ## TYPE((C), elt)) \ + return 0; \ + } \ } -static void -com_shift_expr(struct compiling *c, node *n) +static int +compiler_isdocstring(stmt_ty s) { - int i; - int op; - REQ(n, shift_expr); - com_arith_expr(c, CHILD(n, 0)); - for (i = 2; i < NCH(n); i += 2) { - com_arith_expr(c, CHILD(n, i)); - switch (TYPE(CHILD(n, i-1))) { - case LEFTSHIFT: - op = BINARY_LSHIFT; - break; - case RIGHTSHIFT: - op = BINARY_RSHIFT; - break; - default: - com_error(c, PyExc_SystemError, - "com_shift_expr: operator not << or >>"); - op = 255; - } - com_addbyte(c, op); - com_pop(c, 1); - } + if (s->kind != Expr_kind) + return 0; + return s->v.Expr.value->kind == Str_kind; } -static void -com_and_expr(struct compiling *c, node *n) -{ - int i; - int op; - REQ(n, and_expr); - com_shift_expr(c, CHILD(n, 0)); - for (i = 2; i < NCH(n); i += 2) { - com_shift_expr(c, CHILD(n, i)); - if (TYPE(CHILD(n, i-1)) == AMPER) { - op = BINARY_AND; - } - else { - com_error(c, PyExc_SystemError, - "com_and_expr: operator not &"); - op = 255; - } - com_addbyte(c, op); - com_pop(c, 1); - } -} +/* Compile a sequence of statements, checking for a docstring. */ -static void -com_xor_expr(struct compiling *c, node *n) +static int +compiler_body(struct compiler *c, asdl_seq *stmts) { - int i; - int op; - REQ(n, xor_expr); - com_and_expr(c, CHILD(n, 0)); - for (i = 2; i < NCH(n); i += 2) { - com_and_expr(c, CHILD(n, i)); - if (TYPE(CHILD(n, i-1)) == CIRCUMFLEX) { - op = BINARY_XOR; - } - else { - com_error(c, PyExc_SystemError, - "com_xor_expr: operator not ^"); - op = 255; - } - com_addbyte(c, op); - com_pop(c, 1); - } -} + int i = 0; + stmt_ty st; -static void -com_expr(struct compiling *c, node *n) -{ - int i; - int op; - REQ(n, expr); - com_xor_expr(c, CHILD(n, 0)); - for (i = 2; i < NCH(n); i += 2) { - com_xor_expr(c, CHILD(n, i)); - if (TYPE(CHILD(n, i-1)) == VBAR) { - op = BINARY_OR; - } - else { - com_error(c, PyExc_SystemError, - "com_expr: expr operator not |"); - op = 255; - } - com_addbyte(c, op); - com_pop(c, 1); + if (!asdl_seq_LEN(stmts)) + return 1; + st = asdl_seq_GET(stmts, 0); + if (compiler_isdocstring(st)) { + i = 1; + VISIT(c, expr, st->v.Expr.value); + if (!compiler_nameop(c, __doc__, Store)) + return 0; } + for (; i < asdl_seq_LEN(stmts); i++) + VISIT(c, stmt, asdl_seq_GET(stmts, i)); + return 1; } -static enum cmp_op -cmp_type(node *n) +static PyCodeObject * +compiler_mod(struct compiler *c, mod_ty mod) { - REQ(n, comp_op); - /* comp_op: '<' | '>' | '>=' | '<=' | '<>' | '!=' | '==' - | 'in' | 'not' 'in' | 'is' | 'is' not' */ - if (NCH(n) == 1) { - n = CHILD(n, 0); - switch (TYPE(n)) { - case LESS: return PyCmp_LT; - case GREATER: return PyCmp_GT; - case EQEQUAL: return PyCmp_EQ; - case LESSEQUAL: return PyCmp_LE; - case GREATEREQUAL: return PyCmp_GE; - case NOTEQUAL: return PyCmp_NE; /* <> or != */ - case NAME: if (strcmp(STR(n), "in") == 0) return PyCmp_IN; - if (strcmp(STR(n), "is") == 0) return PyCmp_IS; - } + PyCodeObject *co; + int addNone = 1; + static PyObject *module; + if (!module) { + module = PyString_FromString("<module>"); + if (!module) + return NULL; } - else if (NCH(n) == 2) { - switch (TYPE(CHILD(n, 0))) { - case NAME: if (strcmp(STR(CHILD(n, 1)), "in") == 0) - return PyCmp_NOT_IN; - if (strcmp(STR(CHILD(n, 0)), "is") == 0) - return PyCmp_IS_NOT; - } + if (!compiler_enter_scope(c, module, mod, 1)) + return NULL; + switch (mod->kind) { + case Module_kind: + if (!compiler_body(c, mod->v.Module.body)) + return 0; + break; + case Interactive_kind: + c->c_interactive = 1; + VISIT_SEQ(c, stmt, mod->v.Interactive.body); + break; + case Expression_kind: + VISIT(c, expr, mod->v.Expression.body); + addNone = 0; + break; + case Suite_kind: + assert(0); /* XXX: what should we do here? */ + VISIT_SEQ(c, stmt, mod->v.Suite.body); + break; + default: + assert(0); } - return PyCmp_BAD; + co = assemble(c, addNone); + compiler_exit_scope(c); + return co; } -static void -com_comparison(struct compiling *c, node *n) -{ - int i; - enum cmp_op op; - int anchor; - REQ(n, comparison); /* comparison: expr (comp_op expr)* */ - com_expr(c, CHILD(n, 0)); - if (NCH(n) == 1) - return; - - /**************************************************************** - The following code is generated for all but the last - comparison in a chain: - - label: on stack: opcode: jump to: - - a <code to load b> - a, b DUP_TOP - a, b, b ROT_THREE - b, a, b COMPARE_OP - b, 0-or-1 JUMP_IF_FALSE L1 - b, 1 POP_TOP - b - - We are now ready to repeat this sequence for the next - comparison in the chain. - - For the last we generate: - - b <code to load c> - b, c COMPARE_OP - 0-or-1 - - If there were any jumps to L1 (i.e., there was more than one - comparison), we generate: - - 0-or-1 JUMP_FORWARD L2 - L1: b, 0 ROT_TWO - 0, b POP_TOP - 0 - L2: 0-or-1 - ****************************************************************/ - - anchor = 0; - - for (i = 2; i < NCH(n); i += 2) { - com_expr(c, CHILD(n, i)); - if (i+2 < NCH(n)) { - com_addbyte(c, DUP_TOP); - com_push(c, 1); - com_addbyte(c, ROT_THREE); - } - op = cmp_type(CHILD(n, i-1)); - if (op == PyCmp_BAD) { - com_error(c, PyExc_SystemError, - "com_comparison: unknown comparison op"); - } - com_addoparg(c, COMPARE_OP, op); - com_pop(c, 1); - if (i+2 < NCH(n)) { - com_addfwref(c, JUMP_IF_FALSE, &anchor); - com_addbyte(c, POP_TOP); - com_pop(c, 1); - } - } - - if (anchor) { - int anchor2 = 0; - com_addfwref(c, JUMP_FORWARD, &anchor2); - com_backpatch(c, anchor); - com_addbyte(c, ROT_TWO); - com_addbyte(c, POP_TOP); - com_backpatch(c, anchor2); - } -} +/* The test for LOCAL must come before the test for FREE in order to + handle classes where name is both local and free. The local var is + a method and the free var is a free var referenced within a method. +*/ -static void -com_not_test(struct compiling *c, node *n) -{ - REQ(n, not_test); /* 'not' not_test | comparison */ - if (NCH(n) == 1) { - com_comparison(c, CHILD(n, 0)); - } - else { - com_not_test(c, CHILD(n, 1)); - com_addbyte(c, UNARY_NOT); - } +static int +get_ref_type(struct compiler *c, PyObject *name) +{ + int scope = PyST_GetScope(c->u->u_ste, name); + if (scope == 0) { + char buf[350]; + PyOS_snprintf(buf, sizeof(buf), + "unknown scope for %.100s in %.100s(%s) in %s\n" + "symbols: %s\nlocals: %s\nglobals: %s\n", + PyString_AS_STRING(name), + PyString_AS_STRING(c->u->u_name), + PyObject_REPR(c->u->u_ste->ste_id), + c->c_filename, + PyObject_REPR(c->u->u_ste->ste_symbols), + PyObject_REPR(c->u->u_varnames), + PyObject_REPR(c->u->u_names) + ); + Py_FatalError(buf); + } + + return scope; } -static void -com_and_test(struct compiling *c, node *n) +static int +compiler_lookup_arg(PyObject *dict, PyObject *name) { - int i; - int anchor; - REQ(n, and_test); /* not_test ('and' not_test)* */ - anchor = 0; - i = 0; - for (;;) { - com_not_test(c, CHILD(n, i)); - if ((i += 2) >= NCH(n)) - break; - com_addfwref(c, JUMP_IF_FALSE, &anchor); - com_addbyte(c, POP_TOP); - com_pop(c, 1); - } - if (anchor) - com_backpatch(c, anchor); + PyObject *k, *v; + k = Py_BuildValue("(OO)", name, name->ob_type); + if (k == NULL) + return -1; + v = PyDict_GetItem(dict, k); + if (v == NULL) + return -1; + return PyInt_AS_LONG(v); } static int -com_make_closure(struct compiling *c, PyCodeObject *co) +compiler_make_closure(struct compiler *c, PyCodeObject *co, int args) { int i, free = PyCode_GetNumFree(co); - if (free == 0) - return 0; + if (free == 0) { + ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); + ADDOP_I(c, MAKE_FUNCTION, args); + return 1; + } for (i = 0; i < free; ++i) { /* Bypass com_addop_varname because it will generate LOAD_DEREF but LOAD_CLOSURE is needed. @@ -3243,919 +1788,385 @@ com_make_closure(struct compiling *c, PyCodeObject *co) class. It should be handled by the closure, as well as by the normal name loookup logic. */ - reftype = get_ref_type(c, PyString_AS_STRING(name)); + reftype = get_ref_type(c, name); if (reftype == CELL) - arg = com_lookup_arg(c->c_cellvars, name); + arg = compiler_lookup_arg(c->u->u_cellvars, name); else /* (reftype == FREE) */ - arg = com_lookup_arg(c->c_freevars, name); + arg = compiler_lookup_arg(c->u->u_freevars, name); if (arg == -1) { - fprintf(stderr, "lookup %s in %s %d %d\n" + printf("lookup %s in %s %d %d\n" "freevars of %s: %s\n", PyObject_REPR(name), - c->c_name, + PyString_AS_STRING(c->u->u_name), reftype, arg, PyString_AS_STRING(co->co_name), PyObject_REPR(co->co_freevars)); - Py_FatalError("com_make_closure()"); + Py_FatalError("compiler_make_closure()"); } - com_addoparg(c, LOAD_CLOSURE, arg); - + ADDOP_I(c, LOAD_CLOSURE, arg); } - com_push(c, free); - return 1; + ADDOP_I(c, BUILD_TUPLE, free); + ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); + ADDOP_I(c, MAKE_CLOSURE, args); + return 1; } -static void -com_test(struct compiling *c, node *n) +static int +compiler_decorators(struct compiler *c, asdl_seq* decos) { - REQ(n, test); /* and_test ('or' and_test)* | lambdef */ - if (NCH(n) == 1 && TYPE(CHILD(n, 0)) == lambdef) { - PyCodeObject *co; - int i, closure; - int ndefs = com_argdefs(c, CHILD(n, 0)); - symtable_enter_scope(c->c_symtable, "lambda", lambdef, - n->n_lineno); - co = icompile(CHILD(n, 0), c); - if (co == NULL) { - c->c_errors++; - return; - } - symtable_exit_scope(c->c_symtable); - i = com_addconst(c, (PyObject *)co); - closure = com_make_closure(c, co); - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - if (closure) { - com_addoparg(c, MAKE_CLOSURE, ndefs); - com_pop(c, PyCode_GetNumFree(co)); - } else - com_addoparg(c, MAKE_FUNCTION, ndefs); - Py_DECREF(co); - com_pop(c, ndefs); - } - else { - int anchor = 0; - int i = 0; - for (;;) { - com_and_test(c, CHILD(n, i)); - if ((i += 2) >= NCH(n)) - break; - com_addfwref(c, JUMP_IF_TRUE, &anchor); - com_addbyte(c, POP_TOP); - com_pop(c, 1); - } - if (anchor) - com_backpatch(c, anchor); + int i; + + if (!decos) + return 1; + + for (i = 0; i < asdl_seq_LEN(decos); i++) { + VISIT(c, expr, asdl_seq_GET(decos, i)); } + return 1; } -static void -com_list(struct compiling *c, node *n, int toplevel) +static int +compiler_arguments(struct compiler *c, arguments_ty args) { - /* exprlist: expr (',' expr)* [',']; likewise for testlist */ - if (NCH(n) == 1 && !toplevel) { - com_node(c, CHILD(n, 0)); - } - else { - int i; - int len; - len = (NCH(n) + 1) / 2; - for (i = 0; i < NCH(n); i += 2) - com_node(c, CHILD(n, i)); - com_addoparg(c, BUILD_TUPLE, len); - com_pop(c, len-1); + int i; + int n = asdl_seq_LEN(args->args); + /* Correctly handle nested argument lists */ + for (i = 0; i < n; i++) { + expr_ty arg = asdl_seq_GET(args->args, i); + if (arg->kind == Tuple_kind) { + PyObject *id = PyString_FromFormat(".%d", i); + if (id == NULL) { + return 0; + } + if (!compiler_nameop(c, id, Load)) { + Py_DECREF(id); + return 0; + } + Py_DECREF(id); + VISIT(c, expr, arg); + } } + return 1; } +static int +compiler_function(struct compiler *c, stmt_ty s) +{ + PyCodeObject *co; + PyObject *first_const = Py_None; + arguments_ty args = s->v.FunctionDef.args; + asdl_seq* decos = s->v.FunctionDef.decorators; + stmt_ty st; + int i, n, docstring; -/* Begin of assignment compilation */ + assert(s->kind == FunctionDef_kind); + if (!compiler_decorators(c, decos)) + return 0; + if (args->defaults) + VISIT_SEQ(c, expr, args->defaults); + if (!compiler_enter_scope(c, s->v.FunctionDef.name, (void *)s, + s->lineno)) + return 0; -static void -com_augassign_attr(struct compiling *c, node *n, int opcode, node *augn) -{ - com_addbyte(c, DUP_TOP); - com_push(c, 1); - com_addopname(c, LOAD_ATTR, n); - com_node(c, augn); - com_addbyte(c, opcode); - com_pop(c, 1); - com_addbyte(c, ROT_TWO); - com_addopname(c, STORE_ATTR, n); - com_pop(c, 2); -} + st = asdl_seq_GET(s->v.FunctionDef.body, 0); + docstring = compiler_isdocstring(st); + if (docstring) + first_const = st->v.Expr.value->v.Str.s; + if (compiler_add_o(c, c->u->u_consts, first_const) < 0) + return 0; + + /* unpack nested arguments */ + compiler_arguments(c, args); + + c->u->u_argcount = asdl_seq_LEN(args->args); + n = asdl_seq_LEN(s->v.FunctionDef.body); + /* if there was a docstring, we need to skip the first statement */ + for (i = docstring; i < n; i++) { + stmt_ty s2 = asdl_seq_GET(s->v.FunctionDef.body, i); + if (i == 0 && s2->kind == Expr_kind && + s2->v.Expr.value->kind == Str_kind) + continue; + VISIT(c, stmt, s2); + } + co = assemble(c, 1); + if (co == NULL) + return 0; + compiler_exit_scope(c); -static void -com_assign_attr(struct compiling *c, node *n, int assigning) -{ - if (none_assignment_check(c, STR(n), assigning)) - return; - com_addopname(c, assigning ? STORE_ATTR : DELETE_ATTR, n); - com_pop(c, assigning ? 2 : 1); -} + compiler_make_closure(c, co, asdl_seq_LEN(args->defaults)); -static void -com_assign_trailer(struct compiling *c, node *n, int assigning, node *augn) -{ - REQ(n, trailer); - switch (TYPE(CHILD(n, 0))) { - case LPAR: /* '(' [exprlist] ')' */ - if (assigning == OP_DELETE) - com_error(c, PyExc_SyntaxError, - "can't delete function call"); - else - com_error(c, PyExc_SyntaxError, - "can't assign to function call"); - break; - case DOT: /* '.' NAME */ - if (assigning > OP_APPLY) - com_augassign_attr(c, CHILD(n, 1), assigning, augn); - else - com_assign_attr(c, CHILD(n, 1), assigning); - break; - case LSQB: /* '[' subscriptlist ']' */ - com_subscriptlist(c, CHILD(n, 1), assigning, augn); - break; - default: - com_error(c, PyExc_SystemError, "unknown trailer type"); + for (i = 0; i < asdl_seq_LEN(decos); i++) { + ADDOP_I(c, CALL_FUNCTION, 1); } -} -static void -com_assign_sequence(struct compiling *c, node *n, int assigning) -{ - int i; - if (TYPE(n) != testlist && TYPE(n) != testlist_gexp && - TYPE(n) != listmaker) - REQ(n, exprlist); - if (assigning) { - i = (NCH(n)+1)/2; - com_addoparg(c, UNPACK_SEQUENCE, i); - com_push(c, i-1); - } - for (i = 0; i < NCH(n); i += 2) - com_assign(c, CHILD(n, i), assigning, NULL); + return compiler_nameop(c, s->v.FunctionDef.name, Store); } -static void -com_augassign_name(struct compiling *c, node *n, int opcode, node *augn) +static int +compiler_class(struct compiler *c, stmt_ty s) { - REQ(n, NAME); - com_addop_varname(c, VAR_LOAD, STR(n)); - com_push(c, 1); - com_node(c, augn); - com_addbyte(c, opcode); - com_pop(c, 1); - com_assign_name(c, n, OP_ASSIGN); -} + int n; + PyCodeObject *co; + PyObject *str; + /* push class name on stack, needed by BUILD_CLASS */ + ADDOP_O(c, LOAD_CONST, s->v.ClassDef.name, consts); + /* push the tuple of base classes on the stack */ + n = asdl_seq_LEN(s->v.ClassDef.bases); + if (n > 0) + VISIT_SEQ(c, expr, s->v.ClassDef.bases); + ADDOP_I(c, BUILD_TUPLE, n); + if (!compiler_enter_scope(c, s->v.ClassDef.name, (void *)s, + s->lineno)) + return 0; + c->u->u_private = s->v.ClassDef.name; + Py_INCREF(c->u->u_private); + str = PyString_InternFromString("__name__"); + if (!str || !compiler_nameop(c, str, Load)) { + Py_XDECREF(str); + return 0; + } + + Py_DECREF(str); + str = PyString_InternFromString("__module__"); + if (!str || !compiler_nameop(c, str, Store)) { + Py_XDECREF(str); + return 0; + } + Py_DECREF(str); -static void -com_assign_name(struct compiling *c, node *n, int assigning) -{ - REQ(n, NAME); - com_addop_varname(c, assigning ? VAR_STORE : VAR_DELETE, STR(n)); - if (assigning) - com_pop(c, 1); -} + if (!compiler_body(c, s->v.ClassDef.body)) + return 0; -static void -com_assign(struct compiling *c, node *n, int assigning, node *augn) -{ - /* Loop to avoid trivial recursion */ - for (;;) { - switch (TYPE(n)) { - - case exprlist: - case testlist: - case testlist1: - case testlist_gexp: - if (NCH(n) > 1) { - if (TYPE(CHILD(n, 1)) == gen_for) { - com_error(c, PyExc_SyntaxError, - "assign to generator expression not possible"); - return; - } - if (assigning > OP_APPLY) { - com_error(c, PyExc_SyntaxError, - "augmented assign to generator expression not possible"); - return; - } - com_assign_sequence(c, n, assigning); - return; - } - n = CHILD(n, 0); - break; - case yield_expr: - com_error(c, PyExc_SyntaxError, - "assignment to yield expression not possible"); - return; - - case test: - case and_test: - case not_test: - case comparison: - case expr: - case xor_expr: - case and_expr: - case shift_expr: - case arith_expr: - case term: - case factor: - if (NCH(n) > 1) { - com_error(c, PyExc_SyntaxError, - "can't assign to operator"); - return; - } - n = CHILD(n, 0); - break; - - case power: /* atom trailer* ('**' power)* - ('+'|'-'|'~') factor | atom trailer* */ - if (TYPE(CHILD(n, 0)) != atom) { - com_error(c, PyExc_SyntaxError, - "can't assign to operator"); - return; - } - if (NCH(n) > 1) { /* trailer or exponent present */ - int i; - com_node(c, CHILD(n, 0)); - for (i = 1; i+1 < NCH(n); i++) { - if (TYPE(CHILD(n, i)) == DOUBLESTAR) { - com_error(c, PyExc_SyntaxError, - "can't assign to operator"); - return; - } - com_apply_trailer(c, CHILD(n, i)); - } /* NB i is still alive */ - com_assign_trailer(c, - CHILD(n, i), assigning, augn); - return; - } - n = CHILD(n, 0); - break; - - case atom: - switch (TYPE(CHILD(n, 0))) { - case LPAR: - n = CHILD(n, 1); - if (TYPE(n) == RPAR) { - /* XXX Should allow () = () ??? */ - com_error(c, PyExc_SyntaxError, - "can't assign to ()"); - return; - } - if (assigning > OP_APPLY) { - com_error(c, PyExc_SyntaxError, - "augmented assign to tuple literal, yield, or generator expression not possible"); - return; - } - break; - case LSQB: - n = CHILD(n, 1); - if (TYPE(n) == RSQB) { - com_error(c, PyExc_SyntaxError, - "can't assign to []"); - return; - } - if (assigning > OP_APPLY) { - com_error(c, PyExc_SyntaxError, - "augmented assign to list literal or comprehension not possible"); - return; - } - if (NCH(n) > 1 - && TYPE(CHILD(n, 1)) == list_for) { - com_error(c, PyExc_SyntaxError, - "can't assign to list comprehension"); - return; - } - com_assign_sequence(c, n, assigning); - return; - case NAME: - if (assigning > OP_APPLY) - com_augassign_name(c, CHILD(n, 0), - assigning, augn); - else - com_assign_name(c, CHILD(n, 0), - assigning); - return; - default: - com_error(c, PyExc_SyntaxError, - "can't assign to literal"); - return; - } - break; + ADDOP(c, LOAD_LOCALS); + ADDOP(c, RETURN_VALUE); + co = assemble(c, 1); + if (co == NULL) + return 0; + compiler_exit_scope(c); - case lambdef: - com_error(c, PyExc_SyntaxError, - "can't assign to lambda"); - return; - - default: - com_error(c, PyExc_SystemError, - "com_assign: bad node"); - return; - - } - } + compiler_make_closure(c, co, 0); + ADDOP_I(c, CALL_FUNCTION, 0); + ADDOP(c, BUILD_CLASS); + if (!compiler_nameop(c, s->v.ClassDef.name, Store)) + return 0; + return 1; } -static void -com_augassign(struct compiling *c, node *n) +static int +compiler_lambda(struct compiler *c, expr_ty e) { - int opcode; - - switch (STR(CHILD(CHILD(n, 1), 0))[0]) { - case '+': opcode = INPLACE_ADD; break; - case '-': opcode = INPLACE_SUBTRACT; break; - case '/': - if (STR(CHILD(CHILD(n, 1), 0))[1] == '/') - opcode = INPLACE_FLOOR_DIVIDE; - else if (c->c_flags & CO_FUTURE_DIVISION) - opcode = INPLACE_TRUE_DIVIDE; - else - opcode = INPLACE_DIVIDE; - break; - case '%': opcode = INPLACE_MODULO; break; - case '<': opcode = INPLACE_LSHIFT; break; - case '>': opcode = INPLACE_RSHIFT; break; - case '&': opcode = INPLACE_AND; break; - case '^': opcode = INPLACE_XOR; break; - case '|': opcode = INPLACE_OR; break; - case '*': - if (STR(CHILD(CHILD(n, 1), 0))[1] == '*') - opcode = INPLACE_POWER; - else - opcode = INPLACE_MULTIPLY; - break; - default: - com_error(c, PyExc_SystemError, "com_augassign: bad operator"); - return; - } - com_assign(c, CHILD(n, 0), opcode, CHILD(n, 2)); -} + PyCodeObject *co; + identifier name; + arguments_ty args = e->v.Lambda.args; + assert(e->kind == Lambda_kind); -static void -com_expr_stmt(struct compiling *c, node *n) -{ - REQ(n, expr_stmt); - /* testlist (('=' testlist)* | augassign testlist) */ - /* Forget it if we have just a doc string here */ - if (!c->c_interactive && NCH(n) == 1 && get_rawdocstring(n) != NULL) - return; - if (NCH(n) == 1) { - com_node(c, CHILD(n, NCH(n)-1)); - if (c->c_interactive) - com_addbyte(c, PRINT_EXPR); - else - com_addbyte(c, POP_TOP); - com_pop(c, 1); - } - else if (TYPE(CHILD(n,1)) == augassign) - com_augassign(c, n); - else { - int i; - com_node(c, CHILD(n, NCH(n)-1)); - for (i = 0; i < NCH(n)-2; i+=2) { - if (i+2 < NCH(n)-2) { - com_addbyte(c, DUP_TOP); - com_push(c, 1); - } - com_assign(c, CHILD(n, i), OP_ASSIGN, NULL); - } - } -} + name = PyString_InternFromString("lambda"); + if (!name) + return 0; -static void -com_assert_stmt(struct compiling *c, node *n) -{ - int a = 0; - int i; - REQ(n, assert_stmt); /* 'assert' test [',' test] */ - if (Py_OptimizeFlag) - return; - /* Generate code like - - if not <test>: - raise AssertionError [, <message>] + if (args->defaults) + VISIT_SEQ(c, expr, args->defaults); + if (!compiler_enter_scope(c, name, (void *)e, e->lineno)) + return 0; + + /* unpack nested arguments */ + compiler_arguments(c, args); + + c->u->u_argcount = asdl_seq_LEN(args->args); + VISIT(c, expr, e->v.Lambda.body); + ADDOP(c, RETURN_VALUE); + co = assemble(c, 1); + if (co == NULL) + return 0; + compiler_exit_scope(c); - where <message> is the second test, if present. - */ - com_node(c, CHILD(n, 1)); - com_addfwref(c, JUMP_IF_TRUE, &a); - com_addbyte(c, POP_TOP); - com_pop(c, 1); - /* Raise that exception! */ - com_addop_name(c, LOAD_GLOBAL, "AssertionError"); - com_push(c, 1); - i = NCH(n)/2; /* Either 2 or 4 */ - if (i > 1) - com_node(c, CHILD(n, 3)); - com_addoparg(c, RAISE_VARARGS, i); - com_pop(c, i); - /* The interpreter does not fall through */ - /* Jump ends up here */ - com_backpatch(c, a); - com_addbyte(c, POP_TOP); + compiler_make_closure(c, co, asdl_seq_LEN(args->defaults)); + Py_DECREF(name); + + return 1; } -static void -com_print_stmt(struct compiling *c, node *n) -{ - int i = 1; - node* stream = NULL; - - REQ(n, print_stmt); /* 'print' (test ',')* [test] */ - - /* are we using the extended print form? */ - if (NCH(n) >= 2 && TYPE(CHILD(n, 1)) == RIGHTSHIFT) { - stream = CHILD(n, 2); - com_node(c, stream); - /* stack: [...] => [... stream] */ - com_push(c, 1); - if (NCH(n) > 3 && TYPE(CHILD(n, 3)) == COMMA) - i = 4; - else - i = 3; - } - for (; i < NCH(n); i += 2) { - if (stream != NULL) { - com_addbyte(c, DUP_TOP); - /* stack: [stream] => [stream stream] */ - com_push(c, 1); - com_node(c, CHILD(n, i)); - /* stack: [stream stream] => [stream stream obj] */ - com_addbyte(c, ROT_TWO); - /* stack: [stream stream obj] => [stream obj stream] */ - com_addbyte(c, PRINT_ITEM_TO); - /* stack: [stream obj stream] => [stream] */ - com_pop(c, 2); +static int +compiler_print(struct compiler *c, stmt_ty s) +{ + int i, n; + bool dest; + + assert(s->kind == Print_kind); + n = asdl_seq_LEN(s->v.Print.values); + dest = false; + if (s->v.Print.dest) { + VISIT(c, expr, s->v.Print.dest); + dest = true; + } + for (i = 0; i < n; i++) { + expr_ty e = (expr_ty)asdl_seq_GET(s->v.Print.values, i); + if (dest) { + ADDOP(c, DUP_TOP); + VISIT(c, expr, e); + ADDOP(c, ROT_TWO); + ADDOP(c, PRINT_ITEM_TO); } else { - com_node(c, CHILD(n, i)); - /* stack: [...] => [... obj] */ - com_addbyte(c, PRINT_ITEM); - com_pop(c, 1); + VISIT(c, expr, e); + ADDOP(c, PRINT_ITEM); } } - /* XXX Alternatively, LOAD_CONST '\n' and then PRINT_ITEM */ - if (TYPE(CHILD(n, NCH(n)-1)) == COMMA) { - if (stream != NULL) { - /* must pop the extra stream object off the stack */ - com_addbyte(c, POP_TOP); - /* stack: [... stream] => [...] */ - com_pop(c, 1); - } - } - else { - if (stream != NULL) { - /* this consumes the last stream object on stack */ - com_addbyte(c, PRINT_NEWLINE_TO); - /* stack: [... stream] => [...] */ - com_pop(c, 1); - } + if (s->v.Print.nl) { + if (dest) + ADDOP(c, PRINT_NEWLINE_TO) else - com_addbyte(c, PRINT_NEWLINE); + ADDOP(c, PRINT_NEWLINE) } + else if (dest) + ADDOP(c, POP_TOP); + return 1; } -static void -com_return_stmt(struct compiling *c, node *n) -{ - REQ(n, return_stmt); /* 'return' [testlist] */ - if (!c->c_infunction) { - com_error(c, PyExc_SyntaxError, "'return' outside function"); - } - if (c->c_flags & CO_GENERATOR) { - if (NCH(n) > 1) { - com_error(c, PyExc_SyntaxError, - "'return' with argument inside generator"); - } - } - if (NCH(n) < 2) { - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - } - else - com_node(c, CHILD(n, 1)); - com_addbyte(c, RETURN_VALUE); - com_pop(c, 1); -} - -static void -com_yield_expr(struct compiling *c, node *n) +static int +compiler_if(struct compiler *c, stmt_ty s) { - REQ(n, yield_expr); /* 'yield' testlist */ - if (!c->c_infunction) { - com_error(c, PyExc_SyntaxError, "'yield' outside function"); - } - - /* for (i = 0; i < c->c_nblocks; ++i) { - if (c->c_block[i] == SETUP_FINALLY) { - com_error(c, PyExc_SyntaxError, - "'yield' not allowed in a 'try' block " - "with a 'finally' clause"); - return; - } - } */ + basicblock *end, *next; - if (NCH(n) < 2) { - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - } - else - com_node(c, CHILD(n, 1)); - com_addbyte(c, YIELD_VALUE); + assert(s->kind == If_kind); + end = compiler_new_block(c); + if (end == NULL) + return 0; + next = compiler_new_block(c); + if (next == NULL) + return 0; + VISIT(c, expr, s->v.If.test); + ADDOP_JREL(c, JUMP_IF_FALSE, next); + ADDOP(c, POP_TOP); + VISIT_SEQ(c, stmt, s->v.If.body); + ADDOP_JREL(c, JUMP_FORWARD, end); + compiler_use_next_block(c, next); + ADDOP(c, POP_TOP); + if (s->v.If.orelse) + VISIT_SEQ(c, stmt, s->v.If.orelse); + compiler_use_next_block(c, end); + return 1; } -static void -com_yield_stmt(struct compiling *c, node *n) +static int +compiler_for(struct compiler *c, stmt_ty s) { - REQ(n, yield_stmt); /* yield_expr */ - com_node(c, CHILD(n, 0)); - com_addbyte(c, POP_TOP); - com_pop(c, 1); -} + basicblock *start, *cleanup, *end; - -static void -com_raise_stmt(struct compiling *c, node *n) -{ - int i; - REQ(n, raise_stmt); /* 'raise' [test [',' test [',' test]]] */ - if (NCH(n) > 1) { - com_node(c, CHILD(n, 1)); - if (NCH(n) > 3) { - com_node(c, CHILD(n, 3)); - if (NCH(n) > 5) - com_node(c, CHILD(n, 5)); - } - } - i = NCH(n)/2; - com_addoparg(c, RAISE_VARARGS, i); - com_pop(c, i); + start = compiler_new_block(c); + cleanup = compiler_new_block(c); + end = compiler_new_block(c); + if (start == NULL || end == NULL || cleanup == NULL) + return 0; + ADDOP_JREL(c, SETUP_LOOP, end); + if (!compiler_push_fblock(c, LOOP, start)) + return 0; + VISIT(c, expr, s->v.For.iter); + ADDOP(c, GET_ITER); + compiler_use_next_block(c, start); + ADDOP_JREL(c, FOR_ITER, cleanup); + VISIT(c, expr, s->v.For.target); + VISIT_SEQ(c, stmt, s->v.For.body); + ADDOP_JABS(c, JUMP_ABSOLUTE, start); + compiler_use_next_block(c, cleanup); + ADDOP(c, POP_BLOCK); + compiler_pop_fblock(c, LOOP, start); + VISIT_SEQ(c, stmt, s->v.For.orelse); + compiler_use_next_block(c, end); + return 1; } -static void -com_from_import(struct compiling *c, node *n) +static int +compiler_while(struct compiler *c, stmt_ty s) { - com_addopname(c, IMPORT_FROM, CHILD(n, 0)); - com_push(c, 1); - if (NCH(n) > 1) { - if (strcmp(STR(CHILD(n, 1)), "as") != 0) { - com_error(c, PyExc_SyntaxError, "invalid syntax"); - return; - } - com_addop_varname(c, VAR_STORE, STR(CHILD(n, 2))); - } else - com_addop_varname(c, VAR_STORE, STR(CHILD(n, 0))); - com_pop(c, 1); -} + basicblock *loop, *orelse, *end, *anchor = NULL; + int constant = expr_constant(s->v.While.test); -static void -com_import_stmt(struct compiling *c, node *n) -{ - node *nn; - int i; - REQ(n, import_stmt); - n = CHILD(n, 0); - /* import_stmt: import_name | import_from */ - if (TYPE(n) == import_from) { - /* 'from' dotted_name 'import' ('*' | - '(' import_as_names ')' | import_as_names) */ - PyObject *tup; - REQ(CHILD(n, 1), dotted_name); - nn = CHILD(n, 3 + (TYPE(CHILD(n, 3)) == LPAR)); - if (TYPE(nn) == STAR) - tup = Py_BuildValue("(s)", "*"); - else { - if (TYPE(CHILD(nn, NCH(nn) - 1)) == COMMA && - TYPE(CHILD(n, 3)) != LPAR) { - com_error(c, PyExc_SyntaxError, - "trailing comma not allowed " - "without surrounding parentheses"); - return; - } - REQ(nn, import_as_names); - tup = PyTuple_New((NCH(nn) + 1) / 2); - for (i = 0; i < NCH(nn); i += 2) { - PyObject *s = PyString_FromString( - STR(CHILD(CHILD(nn, i), 0))); - if (s == NULL) { - Py_CLEAR(tup); - break; - } else - PyTuple_SET_ITEM(tup, i / 2, s); - } - if (tup == NULL) { - /* Assume that failue above was MemoryError */ - com_error(c, PyExc_MemoryError, ""); - return; - } - } - com_addoparg(c, LOAD_CONST, com_addconst(c, tup)); - Py_DECREF(tup); - com_push(c, 1); - com_addopname(c, IMPORT_NAME, CHILD(n, 1)); - if (TYPE(nn) == STAR) - com_addbyte(c, IMPORT_STAR); - else { - for (i = 0; i < NCH(nn); i += 2) - com_from_import(c, CHILD(nn, i)); - com_addbyte(c, POP_TOP); - } - com_pop(c, 1); + if (constant == 0) + return 1; + loop = compiler_new_block(c); + end = compiler_new_block(c); + if (constant == -1) { + anchor = compiler_new_block(c); + if (anchor == NULL) + return 0; } - else { - /* 'import' dotted_as_names */ - nn = CHILD(n, 1); - REQ(nn, dotted_as_names); - for (i = 0; i < NCH(nn); i += 2) { - node *subn = CHILD(nn, i); - REQ(subn, dotted_as_name); - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - com_addopname(c, IMPORT_NAME, CHILD(subn, 0)); - if (NCH(subn) > 1) { - int j; - if (strcmp(STR(CHILD(subn, 1)), "as") != 0) { - com_error(c, PyExc_SyntaxError, - "invalid syntax"); - return; - } - for (j=2 ; j < NCH(CHILD(subn, 0)); j += 2) - com_addopname(c, LOAD_ATTR, - CHILD(CHILD(subn, 0), - j)); - com_addop_varname(c, VAR_STORE, - STR(CHILD(subn, 2))); - } else - com_addop_varname(c, VAR_STORE, - STR(CHILD(CHILD(subn, 0), - 0))); - com_pop(c, 1); - } + if (loop == NULL || end == NULL) + return 0; + if (s->v.While.orelse) { + orelse = compiler_new_block(c); + if (orelse == NULL) + return 0; } -} + else + orelse = NULL; -static void -com_exec_stmt(struct compiling *c, node *n) -{ - REQ(n, exec_stmt); - /* exec_stmt: 'exec' expr ['in' expr [',' expr]] */ - com_node(c, CHILD(n, 1)); - if (NCH(n) >= 4) - com_node(c, CHILD(n, 3)); - else { - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); + ADDOP_JREL(c, SETUP_LOOP, end); + compiler_use_next_block(c, loop); + if (!compiler_push_fblock(c, LOOP, loop)) + return 0; + if (constant == -1) { + VISIT(c, expr, s->v.While.test); + ADDOP_JREL(c, JUMP_IF_FALSE, anchor); + ADDOP(c, POP_TOP); } - if (NCH(n) >= 6) - com_node(c, CHILD(n, 5)); - else { - com_addbyte(c, DUP_TOP); - com_push(c, 1); + VISIT_SEQ(c, stmt, s->v.While.body); + ADDOP_JABS(c, JUMP_ABSOLUTE, loop); + + /* XXX should the two POP instructions be in a separate block + if there is no else clause ? + */ + + if (constant == -1) { + compiler_use_next_block(c, anchor); + ADDOP(c, POP_TOP); + ADDOP(c, POP_BLOCK); } - com_addbyte(c, EXEC_STMT); - com_pop(c, 3); + compiler_pop_fblock(c, LOOP, loop); + if (orelse != NULL) + VISIT_SEQ(c, stmt, s->v.While.orelse); + compiler_use_next_block(c, end); + + return 1; } static int -is_constant_false(struct compiling *c, node *n) +compiler_continue(struct compiler *c) { - PyObject *v; + static const char LOOP_ERROR_MSG[] = "'continue' not properly in loop"; int i; - /* argument c will be NULL when called from symtable_node() */ - - /* Label to avoid tail recursion */ - next: - switch (TYPE(n)) { - case suite: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto next; - } - /* Fall through */ - case file_input: - for (i = 0; i < NCH(n); i++) { - node *ch = CHILD(n, i); - if (TYPE(ch) == stmt) { - n = ch; - goto next; - } - } + if (!c->u->u_nfblocks) + return compiler_error(c, LOOP_ERROR_MSG); + i = c->u->u_nfblocks - 1; + switch (c->u->u_fblock[i].fb_type) { + case LOOP: + ADDOP_JABS(c, JUMP_ABSOLUTE, c->u->u_fblock[i].fb_block); break; - - case stmt: - case simple_stmt: - case small_stmt: - n = CHILD(n, 0); - goto next; - - case expr_stmt: - case testlist: - case testlist1: - case test: - case and_test: - case not_test: - case comparison: - case expr: - case xor_expr: - case and_expr: - case shift_expr: - case arith_expr: - case term: - case factor: - case power: - case atom: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto next; - } - break; - - case NAME: - if (Py_OptimizeFlag && strcmp(STR(n), "__debug__") == 0) - return 1; + case EXCEPT: + case FINALLY_TRY: + while (--i >= 0 && c->u->u_fblock[i].fb_type != LOOP) + ; + if (i == -1) + return compiler_error(c, LOOP_ERROR_MSG); + ADDOP_JABS(c, CONTINUE_LOOP, c->u->u_fblock[i].fb_block); break; - - case NUMBER: - v = parsenumber(c, STR(n)); - if (v == NULL) { - PyErr_Clear(); - break; - } - i = PyObject_IsTrue(v); - Py_DECREF(v); - return i == 0; - - case STRING: - v = parsestr(c, STR(n)); - if (v == NULL) { - PyErr_Clear(); - break; - } - i = PyObject_IsTrue(v); - Py_DECREF(v); - return i == 0; - - } - return 0; -} - - -/* Look under n for a return stmt with an expression. - * This hack is used to find illegal returns under "if 0:" blocks in - * functions already known to be generators (as determined by the symtable - * pass). - * Return the offending return node if found, else NULL. - */ -static node * -look_for_offending_return(node *n) -{ - int i; - - for (i = 0; i < NCH(n); ++i) { - node *kid = CHILD(n, i); - - switch (TYPE(kid)) { - case classdef: - case funcdef: - case lambdef: - /* Stuff in nested functions & classes doesn't - affect the code block we started in. */ - return NULL; - - case return_stmt: - if (NCH(kid) > 1) - return kid; - break; - - default: { - node *bad = look_for_offending_return(kid); - if (bad != NULL) - return bad; - } - } - } - - return NULL; -} - -static void -com_if_stmt(struct compiling *c, node *n) -{ - int i; - int anchor = 0; - REQ(n, if_stmt); - /*'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] */ - for (i = 0; i+3 < NCH(n); i+=4) { - int a = 0; - node *ch = CHILD(n, i+1); - if (is_constant_false(c, ch)) { - /* We're going to skip this block. However, if this - is a generator, we have to check the dead code - anyway to make sure there aren't any return stmts - with expressions, in the same scope. */ - if (c->c_flags & CO_GENERATOR) { - node *p = look_for_offending_return(n); - if (p != NULL) { - int savelineno = c->c_lineno; - c->c_lineno = p->n_lineno; - com_error(c, PyExc_SyntaxError, - "'return' with argument " - "inside generator"); - c->c_lineno = savelineno; - } - } - continue; - } - if (i > 0) - com_set_lineno(c, ch->n_lineno); - com_node(c, ch); - com_addfwref(c, JUMP_IF_FALSE, &a); - com_addbyte(c, POP_TOP); - com_pop(c, 1); - com_node(c, CHILD(n, i+3)); - com_addfwref(c, JUMP_FORWARD, &anchor); - com_backpatch(c, a); - /* We jump here with an extra entry which we now pop */ - com_addbyte(c, POP_TOP); + case FINALLY_END: + return compiler_error(c, + "'continue' not supported inside 'finally' clause"); } - if (i+2 < NCH(n)) - com_node(c, CHILD(n, i+2)); - if (anchor) - com_backpatch(c, anchor); -} - -static void -com_while_stmt(struct compiling *c, node *n) -{ - int break_anchor = 0; - int anchor = 0; - int save_begin = c->c_begin; - REQ(n, while_stmt); /* 'while' test ':' suite ['else' ':' suite] */ - com_addfwref(c, SETUP_LOOP, &break_anchor); - block_push(c, SETUP_LOOP); - c->c_begin = c->c_nexti; - com_set_lineno(c, n->n_lineno); - com_node(c, CHILD(n, 1)); - com_addfwref(c, JUMP_IF_FALSE, &anchor); - com_addbyte(c, POP_TOP); - com_pop(c, 1); - c->c_loops++; - com_node(c, CHILD(n, 3)); - c->c_loops--; - com_addoparg(c, JUMP_ABSOLUTE, c->c_begin); - c->c_begin = save_begin; - com_backpatch(c, anchor); - /* We jump here with one entry more on the stack */ - com_addbyte(c, POP_TOP); - com_addbyte(c, POP_BLOCK); - block_pop(c, SETUP_LOOP); - if (NCH(n) > 4) - com_node(c, CHILD(n, 6)); - com_backpatch(c, break_anchor); -} -static void -com_for_stmt(struct compiling *c, node *n) -{ - int break_anchor = 0; - int anchor = 0; - int save_begin = c->c_begin; - REQ(n, for_stmt); - /* 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] */ - com_addfwref(c, SETUP_LOOP, &break_anchor); - block_push(c, SETUP_LOOP); - com_node(c, CHILD(n, 3)); - com_addbyte(c, GET_ITER); - c->c_begin = c->c_nexti; - com_set_lineno(c, c->c_last_line); - com_addfwref(c, FOR_ITER, &anchor); - com_push(c, 1); - com_assign(c, CHILD(n, 1), OP_ASSIGN, NULL); - c->c_loops++; - com_node(c, CHILD(n, 5)); - c->c_loops--; - com_addoparg(c, JUMP_ABSOLUTE, c->c_begin); - c->c_begin = save_begin; - com_backpatch(c, anchor); - com_pop(c, 1); /* FOR_ITER has popped this */ - com_addbyte(c, POP_BLOCK); - block_pop(c, SETUP_LOOP); - if (NCH(n) > 8) - com_node(c, CHILD(n, 8)); - com_backpatch(c, break_anchor); + return 1; } -/* Code generated for "try: S finally: Sf" is as follows: +/* Code generated for "try: <body> finally: <finalbody>" is as follows: SETUP_FINALLY L - <code for S> + <code for body> POP_BLOCK - LOAD_CONST <nil> - L: <code for Sf> + LOAD_CONST <None> + L: <code for finalbody> END_FINALLY The special instructions use the block stack. Each block @@ -4180,7 +2191,37 @@ com_for_stmt(struct compiling *c, node *n) onto the value stack (and the exception condition is cleared), and the interpreter jumps to the label gotten from the block stack. - +*/ + +static int +compiler_try_finally(struct compiler *c, stmt_ty s) +{ + basicblock *body, *end; + body = compiler_new_block(c); + end = compiler_new_block(c); + if (body == NULL || end == NULL) + return 0; + + ADDOP_JREL(c, SETUP_FINALLY, end); + compiler_use_next_block(c, body); + if (!compiler_push_fblock(c, FINALLY_TRY, body)) + return 0; + VISIT_SEQ(c, stmt, s->v.TryFinally.body); + ADDOP(c, POP_BLOCK); + compiler_pop_fblock(c, FINALLY_TRY, body); + + ADDOP_O(c, LOAD_CONST, Py_None, consts); + compiler_use_next_block(c, end); + if (!compiler_push_fblock(c, FINALLY_END, end)) + return 0; + VISIT_SEQ(c, stmt, s->v.TryFinally.finalbody); + ADDOP(c, END_FINALLY); + compiler_pop_fblock(c, FINALLY_END, end); + + return 1; +} + +/* Code generated for "try: S except E1, V1: S1 except E2, V2: S2 ...": (The contents of the value stack is shown in [], with the top at the right; 'tb' is trace-back info, 'val' the exception's @@ -4214,2609 +2255,1835 @@ com_for_stmt(struct compiling *c, node *n) Of course, parts are not generated if Vi or Ei is not present. */ - -static void -com_try_except(struct compiling *c, node *n) +static int +compiler_try_except(struct compiler *c, stmt_ty s) { - int except_anchor = 0; - int end_anchor = 0; - int else_anchor = 0; - int i; - node *ch; - - com_addfwref(c, SETUP_EXCEPT, &except_anchor); - block_push(c, SETUP_EXCEPT); - com_node(c, CHILD(n, 2)); - com_addbyte(c, POP_BLOCK); - block_pop(c, SETUP_EXCEPT); - com_addfwref(c, JUMP_FORWARD, &else_anchor); - com_backpatch(c, except_anchor); - for (i = 3; - i < NCH(n) && TYPE(ch = CHILD(n, i)) == except_clause; - i += 3) { - /* except_clause: 'except' [expr [',' var]] */ - if (except_anchor == 0) { - com_error(c, PyExc_SyntaxError, - "default 'except:' must be last"); - break; + basicblock *body, *orelse, *except, *end; + int i, n; + + body = compiler_new_block(c); + except = compiler_new_block(c); + orelse = compiler_new_block(c); + end = compiler_new_block(c); + if (body == NULL || except == NULL || orelse == NULL || end == NULL) + return 0; + ADDOP_JREL(c, SETUP_EXCEPT, except); + compiler_use_next_block(c, body); + if (!compiler_push_fblock(c, EXCEPT, body)) + return 0; + VISIT_SEQ(c, stmt, s->v.TryExcept.body); + ADDOP(c, POP_BLOCK); + compiler_pop_fblock(c, EXCEPT, body); + ADDOP_JREL(c, JUMP_FORWARD, orelse); + n = asdl_seq_LEN(s->v.TryExcept.handlers); + compiler_use_next_block(c, except); + for (i = 0; i < n; i++) { + excepthandler_ty handler = asdl_seq_GET( + s->v.TryExcept.handlers, i); + if (!handler->type && i < n-1) + return compiler_error(c, "default 'except:' must be last"); + except = compiler_new_block(c); + if (except == NULL) + return 0; + if (handler->type) { + ADDOP(c, DUP_TOP); + VISIT(c, expr, handler->type); + ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); + ADDOP_JREL(c, JUMP_IF_FALSE, except); + ADDOP(c, POP_TOP); } - except_anchor = 0; - com_push(c, 3); /* tb, val, exc pushed by exception */ - com_set_lineno(c, ch->n_lineno); - if (NCH(ch) > 1) { - com_addbyte(c, DUP_TOP); - com_push(c, 1); - com_node(c, CHILD(ch, 1)); - com_addoparg(c, COMPARE_OP, PyCmp_EXC_MATCH); - com_pop(c, 1); - com_addfwref(c, JUMP_IF_FALSE, &except_anchor); - com_addbyte(c, POP_TOP); - com_pop(c, 1); + ADDOP(c, POP_TOP); + if (handler->name) { + VISIT(c, expr, handler->name); } - com_addbyte(c, POP_TOP); - com_pop(c, 1); - if (NCH(ch) > 3) - com_assign(c, CHILD(ch, 3), OP_ASSIGN, NULL); else { - com_addbyte(c, POP_TOP); - com_pop(c, 1); - } - com_addbyte(c, POP_TOP); - com_pop(c, 1); - com_node(c, CHILD(n, i+2)); - com_addfwref(c, JUMP_FORWARD, &end_anchor); - if (except_anchor) { - com_backpatch(c, except_anchor); - /* We come in with [tb, val, exc, 0] on the - stack; one pop and it's the same as - expected at the start of the loop */ - com_addbyte(c, POP_TOP); - } - } - /* We actually come in here with [tb, val, exc] but the - END_FINALLY will zap those and jump around. - The c_stacklevel does not reflect them so we need not pop - anything. */ - com_addbyte(c, END_FINALLY); - com_backpatch(c, else_anchor); - if (i < NCH(n)) - com_node(c, CHILD(n, i+2)); - com_backpatch(c, end_anchor); + ADDOP(c, POP_TOP); + } + ADDOP(c, POP_TOP); + VISIT_SEQ(c, stmt, handler->body); + ADDOP_JREL(c, JUMP_FORWARD, end); + compiler_use_next_block(c, except); + if (handler->type) + ADDOP(c, POP_TOP); + } + ADDOP(c, END_FINALLY); + compiler_use_next_block(c, orelse); + VISIT_SEQ(c, stmt, s->v.TryExcept.orelse); + compiler_use_next_block(c, end); + return 1; } -static void -com_try_finally(struct compiling *c, node *n) +static int +compiler_import_as(struct compiler *c, identifier name, identifier asname) { - int finally_anchor = 0; - node *ch; - - com_addfwref(c, SETUP_FINALLY, &finally_anchor); - block_push(c, SETUP_FINALLY); - com_node(c, CHILD(n, 2)); - com_addbyte(c, POP_BLOCK); - block_pop(c, SETUP_FINALLY); - block_push(c, END_FINALLY); - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - /* While the generated code pushes only one item, - the try-finally handling can enter here with - up to three items. OK, here are the details: - 3 for an exception, 2 for RETURN, 1 for BREAK. */ - com_push(c, 3); - com_backpatch(c, finally_anchor); - ch = CHILD(n, NCH(n)-1); - com_set_lineno(c, ch->n_lineno); - com_node(c, ch); - com_addbyte(c, END_FINALLY); - block_pop(c, END_FINALLY); - com_pop(c, 3); /* Matches the com_push above */ -} + /* The IMPORT_NAME opcode was already generated. This function + merely needs to bind the result to a name. -static void -com_try_stmt(struct compiling *c, node *n) -{ - REQ(n, try_stmt); - /* 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] - | 'try' ':' suite 'finally' ':' suite */ - if (TYPE(CHILD(n, 3)) != except_clause) - com_try_finally(c, n); - else - com_try_except(c, n); + If there is a dot in name, we need to split it and emit a + LOAD_ATTR for each name. + */ + const char *src = PyString_AS_STRING(name); + const char *dot = strchr(src, '.'); + if (dot) { + /* Consume the base module name to get the first attribute */ + src = dot + 1; + while (dot) { + /* NB src is only defined when dot != NULL */ + dot = strchr(src, '.'); + PyObject *attr = PyString_FromStringAndSize(src, + dot ? dot - src : strlen(src)); + ADDOP_O(c, LOAD_ATTR, attr, names); + src = dot + 1; + } + } + return compiler_nameop(c, asname, Store); } -static node * -get_rawdocstring(node *n) -{ - int i; - - /* Label to avoid tail recursion */ - next: - switch (TYPE(n)) { - - case suite: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto next; - } - /* Fall through */ - case file_input: - for (i = 0; i < NCH(n); i++) { - node *ch = CHILD(n, i); - if (TYPE(ch) == stmt) { - n = ch; - goto next; +static int +compiler_import(struct compiler *c, stmt_ty s) +{ + /* The Import node stores a module name like a.b.c as a single + string. This is convenient for all cases except + import a.b.c as d + where we need to parse that string to extract the individual + module names. + XXX Perhaps change the representation to make this case simpler? + */ + int i, n = asdl_seq_LEN(s->v.Import.names); + for (i = 0; i < n; i++) { + alias_ty alias = asdl_seq_GET(s->v.Import.names, i); + int r; + + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP_NAME(c, IMPORT_NAME, alias->name, names); + + if (alias->asname) { + return compiler_import_as(c, + alias->name, alias->asname); + } + else { + identifier tmp = alias->name; + const char *base = PyString_AS_STRING(alias->name); + char *dot = strchr(base, '.'); + if (dot) + tmp = PyString_FromStringAndSize(base, + dot - base); + r = compiler_nameop(c, tmp, Store); + if (dot) { + Py_DECREF(tmp); } + if (!r) + return r; } - break; - - case stmt: - case simple_stmt: - case small_stmt: - n = CHILD(n, 0); - goto next; - - case expr_stmt: - case testlist: - case testlist1: - case test: - case and_test: - case not_test: - case comparison: - case expr: - case xor_expr: - case and_expr: - case shift_expr: - case arith_expr: - case term: - case factor: - case power: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto next; - } - break; - - case atom: - if (TYPE(CHILD(n, 0)) == STRING) - return n; - break; - } - return NULL; + return 1; } -static PyObject * -get_docstring(struct compiling *c, node *n) +static int +compiler_from_import(struct compiler *c, stmt_ty s) { - /* Don't generate doc-strings if run with -OO */ - if (Py_OptimizeFlag > 1) - return NULL; - n = get_rawdocstring(n); - if (n == NULL) - return NULL; - return parsestrplus(c, n); -} + int i, n = asdl_seq_LEN(s->v.ImportFrom.names); + int star = 0; -static void -com_suite(struct compiling *c, node *n) -{ - REQ(n, suite); - /* simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT */ - if (NCH(n) == 1) { - com_node(c, CHILD(n, 0)); + PyObject *names = PyTuple_New(n); + if (!names) + return 0; + + /* build up the names */ + for (i = 0; i < n; i++) { + alias_ty alias = asdl_seq_GET(s->v.ImportFrom.names, i); + Py_INCREF(alias->name); + PyTuple_SET_ITEM(names, i, alias->name); } - else { - int i; - for (i = 0; i < NCH(n) && c->c_errors == 0; i++) { - node *ch = CHILD(n, i); - if (TYPE(ch) == stmt) - com_node(c, ch); + + if (s->lineno > c->c_future->ff_lineno) { + if (!strcmp(PyString_AS_STRING(s->v.ImportFrom.module), + "__future__")) { + Py_DECREF(names); + return compiler_error(c, + "from __future__ imports must occur " + "at the beginning of the file"); + } } -} -/* ARGSUSED */ -static void -com_continue_stmt(struct compiling *c, node *n) -{ - int i = c->c_nblocks; - if (i-- > 0 && c->c_block[i] == SETUP_LOOP) { - com_addoparg(c, JUMP_ABSOLUTE, c->c_begin); - } - else if (i <= 0) { - /* at the outer level */ - com_error(c, PyExc_SyntaxError, - "'continue' not properly in loop"); - } - else { - int j; - for (j = i-1; j >= 0; --j) { - if (c->c_block[j] == SETUP_LOOP) - break; + ADDOP_O(c, LOAD_CONST, names, consts); + ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names); + for (i = 0; i < n; i++) { + alias_ty alias = asdl_seq_GET(s->v.ImportFrom.names, i); + identifier store_name; + + if (i == 0 && *PyString_AS_STRING(alias->name) == '*') { + assert(n == 1); + ADDOP(c, IMPORT_STAR); + star = 1; + break; } - if (j >= 0) { - /* there is a loop, but something interferes */ - for (; i > j; --i) { - if (c->c_block[i] == SETUP_EXCEPT || - c->c_block[i] == SETUP_FINALLY) { - com_addoparg(c, CONTINUE_LOOP, - c->c_begin); - return; - } - if (c->c_block[i] == END_FINALLY) { - com_error(c, PyExc_SyntaxError, - "'continue' not supported inside 'finally' clause"); - return; - } - } + + ADDOP_NAME(c, IMPORT_FROM, alias->name, names); + store_name = alias->name; + if (alias->asname) + store_name = alias->asname; + + if (!compiler_nameop(c, store_name, Store)) { + Py_DECREF(names); + return 0; } - com_error(c, PyExc_SyntaxError, - "'continue' not properly in loop"); } - /* XXX Could allow it inside a 'finally' clause - XXX if we could pop the exception still on the stack */ + if (!star) + /* remove imported module */ + ADDOP(c, POP_TOP); + return 1; } -/* Return the number of default values in the argument list. - - If a non-default argument follows a default argument, set an - exception and return -1. -*/ - static int -com_argdefs(struct compiling *c, node *n) +compiler_assert(struct compiler *c, stmt_ty s) { - int i, nch, ndefs; - if (TYPE(n) == lambdef) { - /* lambdef: 'lambda' [varargslist] ':' test */ - n = CHILD(n, 1); + static PyObject *assertion_error = NULL; + basicblock *end; + + if (Py_OptimizeFlag) + return 1; + if (assertion_error == NULL) { + assertion_error = PyString_FromString("AssertionError"); + if (assertion_error == NULL) + return 0; + } + VISIT(c, expr, s->v.Assert.test); + end = compiler_new_block(c); + if (end == NULL) + return 0; + ADDOP_JREL(c, JUMP_IF_TRUE, end); + ADDOP(c, POP_TOP); + ADDOP_O(c, LOAD_GLOBAL, assertion_error, names); + if (s->v.Assert.msg) { + VISIT(c, expr, s->v.Assert.msg); + ADDOP_I(c, RAISE_VARARGS, 2); } else { - REQ(n, funcdef); - /* funcdef: [decorators] 'def' NAME parameters ':' suite */ - n = RCHILD(n, -3); - REQ(n, parameters); /* parameters: '(' [varargslist] ')' */ - n = CHILD(n, 1); + ADDOP_I(c, RAISE_VARARGS, 1); } - if (TYPE(n) != varargslist) - return 0; - /* varargslist: - (fpdef ['=' test] ',')* '*' ....... | - fpdef ['=' test] (',' fpdef ['=' test])* [','] */ - nch = NCH(n); - ndefs = 0; - for (i = 0; i < nch; i++) { - int t; - if (TYPE(CHILD(n, i)) == STAR || - TYPE(CHILD(n, i)) == DOUBLESTAR) - break; - i++; - if (i >= nch) - t = RPAR; /* Anything except EQUAL or COMMA */ + compiler_use_block(c, end); + ADDOP(c, POP_TOP); + return 1; +} + +static int +compiler_visit_stmt(struct compiler *c, stmt_ty s) +{ + int i, n; + + c->u->u_lineno = s->lineno; + c->u->u_lineno_set = false; + switch (s->kind) { + case FunctionDef_kind: + return compiler_function(c, s); + case ClassDef_kind: + return compiler_class(c, s); + case Return_kind: + if (c->u->u_ste->ste_type != FunctionBlock) + return compiler_error(c, "'return' outside function"); + if (s->v.Return.value) { + if (c->u->u_ste->ste_generator) { + return compiler_error(c, + "'return' with argument inside generator"); + } + VISIT(c, expr, s->v.Return.value); + } else - t = TYPE(CHILD(n, i)); - if (t == EQUAL) { - i++; - ndefs++; - com_node(c, CHILD(n, i)); - i++; - if (i >= nch) - break; - t = TYPE(CHILD(n, i)); + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, RETURN_VALUE); + break; + case Delete_kind: + VISIT_SEQ(c, expr, s->v.Delete.targets) + break; + case Assign_kind: + n = asdl_seq_LEN(s->v.Assign.targets); + VISIT(c, expr, s->v.Assign.value); + for (i = 0; i < n; i++) { + if (i < n - 1) + ADDOP(c, DUP_TOP); + VISIT(c, expr, + (expr_ty)asdl_seq_GET(s->v.Assign.targets, i)); + } + break; + case AugAssign_kind: + return compiler_augassign(c, s); + case Print_kind: + return compiler_print(c, s); + case For_kind: + return compiler_for(c, s); + case While_kind: + return compiler_while(c, s); + case If_kind: + return compiler_if(c, s); + case Raise_kind: + n = 0; + if (s->v.Raise.type) { + VISIT(c, expr, s->v.Raise.type); + n++; + if (s->v.Raise.inst) { + VISIT(c, expr, s->v.Raise.inst); + n++; + if (s->v.Raise.tback) { + VISIT(c, expr, s->v.Raise.tback); + n++; + } + } } - else { - /* Treat "(a=1, b)" as an error */ - if (ndefs) { - com_error(c, PyExc_SyntaxError, - "non-default argument follows default argument"); - return -1; + ADDOP_I(c, RAISE_VARARGS, n); + break; + case TryExcept_kind: + return compiler_try_except(c, s); + case TryFinally_kind: + return compiler_try_finally(c, s); + case Assert_kind: + return compiler_assert(c, s); + case Import_kind: + return compiler_import(c, s); + case ImportFrom_kind: + return compiler_from_import(c, s); + case Exec_kind: + VISIT(c, expr, s->v.Exec.body); + if (s->v.Exec.globals) { + VISIT(c, expr, s->v.Exec.globals); + if (s->v.Exec.locals) { + VISIT(c, expr, s->v.Exec.locals); + } else { + ADDOP(c, DUP_TOP); } + } else { + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, DUP_TOP); } - if (t != COMMA) - break; + ADDOP(c, EXEC_STMT); + break; + case Global_kind: + break; + case Expr_kind: + VISIT(c, expr, s->v.Expr.value); + if (c->c_interactive && c->c_nestlevel <= 1) { + ADDOP(c, PRINT_EXPR); + } + else { + ADDOP(c, POP_TOP); + } + break; + case Pass_kind: + break; + case Break_kind: + if (!c->u->u_nfblocks) + return compiler_error(c, "'break' outside loop"); + ADDOP(c, BREAK_LOOP); + break; + case Continue_kind: + return compiler_continue(c); } - return ndefs; + return 1; } -static void -com_decorator_name(struct compiling *c, node *n) -{ - /* dotted_name: NAME ('.' NAME)* */ - - int i, nch; - node *varname; - - REQ(n, dotted_name); - nch = NCH(n); - assert(nch >= 1 && nch % 2 == 1); - - varname = CHILD(n, 0); - REQ(varname, NAME); - com_addop_varname(c, VAR_LOAD, STR(varname)); - com_push(c, 1); - - for (i = 1; i < nch; i += 2) { - node *attrname; - - REQ(CHILD(n, i), DOT); - - attrname = CHILD(n, i + 1); - REQ(attrname, NAME); - com_addop_name(c, LOAD_ATTR, STR(attrname)); +static int +unaryop(unaryop_ty op) +{ + switch (op) { + case Invert: + return UNARY_INVERT; + case Not: + return UNARY_NOT; + case UAdd: + return UNARY_POSITIVE; + case USub: + return UNARY_NEGATIVE; } + return 0; } -static void -com_decorator(struct compiling *c, node *n) -{ - /* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */ - int nch = NCH(n); - assert(nch >= 3); - REQ(CHILD(n, 0), AT); - REQ(RCHILD(n, -1), NEWLINE); - com_decorator_name(c, CHILD(n, 1)); - - if (nch > 3) { - assert(nch == 5 || nch == 6); - REQ(CHILD(n, 2), LPAR); - REQ(RCHILD(n, -2), RPAR); - com_call_function(c, CHILD(n, 3)); +static int +binop(struct compiler *c, operator_ty op) +{ + switch (op) { + case Add: + return BINARY_ADD; + case Sub: + return BINARY_SUBTRACT; + case Mult: + return BINARY_MULTIPLY; + case Div: + if (c->c_flags && c->c_flags->cf_flags & CO_FUTURE_DIVISION) + return BINARY_TRUE_DIVIDE; + else + return BINARY_DIVIDE; + case Mod: + return BINARY_MODULO; + case Pow: + return BINARY_POWER; + case LShift: + return BINARY_LSHIFT; + case RShift: + return BINARY_RSHIFT; + case BitOr: + return BINARY_OR; + case BitXor: + return BINARY_XOR; + case BitAnd: + return BINARY_AND; + case FloorDiv: + return BINARY_FLOOR_DIVIDE; } + return 0; } static int -com_decorators(struct compiling *c, node *n) -{ - int i, nch; - - /* decorator+ */ - nch = NCH(n); - assert(nch >= 1); - - for (i = 0; i < nch; ++i) { - node *ch = CHILD(n, i); - REQ(ch, decorator); - - com_decorator(c, ch); +cmpop(cmpop_ty op) +{ + switch (op) { + case Eq: + return PyCmp_EQ; + case NotEq: + return PyCmp_NE; + case Lt: + return PyCmp_LT; + case LtE: + return PyCmp_LE; + case Gt: + return PyCmp_GT; + case GtE: + return PyCmp_GE; + case Is: + return PyCmp_IS; + case IsNot: + return PyCmp_IS_NOT; + case In: + return PyCmp_IN; + case NotIn: + return PyCmp_NOT_IN; } - - return nch; + return PyCmp_BAD; } -static void -com_funcdef(struct compiling *c, node *n) -{ - PyObject *co; - int ndefs, ndecorators; - - REQ(n, funcdef); - /* -6 -5 -4 -3 -2 -1 - funcdef: [decorators] 'def' NAME parameters ':' suite */ - - if (NCH(n) == 6) - ndecorators = com_decorators(c, CHILD(n, 0)); - else - ndecorators = 0; - - ndefs = com_argdefs(c, n); - if (ndefs < 0) - return; - symtable_enter_scope(c->c_symtable, STR(RCHILD(n, -4)), TYPE(n), - n->n_lineno); - co = (PyObject *)icompile(n, c); - symtable_exit_scope(c->c_symtable); - if (co == NULL) - c->c_errors++; - else { - int closure = com_make_closure(c, (PyCodeObject *)co); - int i = com_addconst(c, co); - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - if (closure) - com_addoparg(c, MAKE_CLOSURE, ndefs); +static int +inplace_binop(struct compiler *c, operator_ty op) +{ + switch (op) { + case Add: + return INPLACE_ADD; + case Sub: + return INPLACE_SUBTRACT; + case Mult: + return INPLACE_MULTIPLY; + case Div: + if (c->c_flags && c->c_flags->cf_flags & CO_FUTURE_DIVISION) + return INPLACE_TRUE_DIVIDE; else - com_addoparg(c, MAKE_FUNCTION, ndefs); - com_pop(c, ndefs); - - while (ndecorators > 0) { - com_addoparg(c, CALL_FUNCTION, 1); - com_pop(c, 1); - --ndecorators; - } - - com_addop_varname(c, VAR_STORE, STR(RCHILD(n, -4))); - com_pop(c, 1); - Py_DECREF(co); - } + return INPLACE_DIVIDE; + case Mod: + return INPLACE_MODULO; + case Pow: + return INPLACE_POWER; + case LShift: + return INPLACE_LSHIFT; + case RShift: + return INPLACE_RSHIFT; + case BitOr: + return INPLACE_OR; + case BitXor: + return INPLACE_XOR; + case BitAnd: + return INPLACE_AND; + case FloorDiv: + return INPLACE_FLOOR_DIVIDE; + } + assert(0); + return 0; } -static void -com_bases(struct compiling *c, node *n) +static int +compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) { - int i; - REQ(n, testlist); - /* testlist: test (',' test)* [','] */ - for (i = 0; i < NCH(n); i += 2) - com_node(c, CHILD(n, i)); - i = (NCH(n)+1) / 2; - com_addoparg(c, BUILD_TUPLE, i); - com_pop(c, i-1); -} + int op, scope; + enum { OP_FAST, OP_GLOBAL, OP_DEREF, OP_NAME } optype; -static void -com_classdef(struct compiling *c, node *n) -{ - int i; - PyObject *v; - PyCodeObject *co; - char *name; + PyObject *dict = c->u->u_names; + /* XXX AugStore isn't used anywhere! */ - REQ(n, classdef); - /* classdef: class NAME ['(' [testlist] ')'] ':' suite */ - if ((v = PyString_InternFromString(STR(CHILD(n, 1)))) == NULL) { - c->c_errors++; - return; - } - /* Push the class name on the stack */ - i = com_addconst(c, v); - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - Py_DECREF(v); - /* Push the tuple of base classes on the stack */ - if (TYPE(CHILD(n, 2)) != LPAR || - TYPE(CHILD(n, 3)) == RPAR) { - com_addoparg(c, BUILD_TUPLE, 0); - com_push(c, 1); - } - else - com_bases(c, CHILD(n, 3)); - name = STR(CHILD(n, 1)); - symtable_enter_scope(c->c_symtable, name, TYPE(n), n->n_lineno); - co = icompile(n, c); - symtable_exit_scope(c->c_symtable); - if (co == NULL) - c->c_errors++; - else { - int closure = com_make_closure(c, co); - i = com_addconst(c, (PyObject *)co); - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - if (closure) { - com_addoparg(c, MAKE_CLOSURE, 0); - com_pop(c, PyCode_GetNumFree(co)); - } else - com_addoparg(c, MAKE_FUNCTION, 0); - com_addoparg(c, CALL_FUNCTION, 0); - com_addbyte(c, BUILD_CLASS); - com_pop(c, 2); - com_addop_varname(c, VAR_STORE, STR(CHILD(n, 1))); - com_pop(c, 1); - Py_DECREF(co); + /* First check for assignment to __debug__. Param? */ + if ((ctx == Store || ctx == AugStore || ctx == Del) + && !strcmp(PyString_AS_STRING(name), "__debug__")) { + return compiler_error(c, "can not assign to __debug__"); } -} - -static void -com_node(struct compiling *c, node *n) -{ - loop: - if (c->c_errors) - return; - switch (TYPE(n)) { - - /* Definition nodes */ - - case funcdef: - com_funcdef(c, n); - break; - case classdef: - com_classdef(c, n); - break; - - /* Trivial parse tree nodes */ - - case stmt: - case small_stmt: - case flow_stmt: - n = CHILD(n, 0); - goto loop; - - case simple_stmt: - /* small_stmt (';' small_stmt)* [';'] NEWLINE */ - com_set_lineno(c, n->n_lineno); - { - int i; - for (i = 0; i < NCH(n)-1; i += 2) - com_node(c, CHILD(n, i)); - } - break; - - case compound_stmt: - com_set_lineno(c, n->n_lineno); - n = CHILD(n, 0); - goto loop; - /* Statement nodes */ - - case expr_stmt: - com_expr_stmt(c, n); - break; - case print_stmt: - com_print_stmt(c, n); - break; - case del_stmt: /* 'del' exprlist */ - com_assign(c, CHILD(n, 1), OP_DELETE, NULL); - break; - case pass_stmt: - break; - case break_stmt: - if (c->c_loops == 0) { - com_error(c, PyExc_SyntaxError, - "'break' outside loop"); - } - com_addbyte(c, BREAK_LOOP); - break; - case continue_stmt: - com_continue_stmt(c, n); - break; - case return_stmt: - com_return_stmt(c, n); - break; - case yield_stmt: - com_yield_stmt(c, n); - break; - case raise_stmt: - com_raise_stmt(c, n); - break; - case import_stmt: - com_import_stmt(c, n); - break; - case global_stmt: - break; - case exec_stmt: - com_exec_stmt(c, n); - break; - case assert_stmt: - com_assert_stmt(c, n); - break; - case if_stmt: - com_if_stmt(c, n); - break; - case while_stmt: - com_while_stmt(c, n); + op = 0; + optype = OP_NAME; + scope = PyST_GetScope(c->u->u_ste, name); + switch (scope) { + case FREE: + dict = c->u->u_freevars; + optype = OP_DEREF; break; - case for_stmt: - com_for_stmt(c, n); + case CELL: + dict = c->u->u_cellvars; + optype = OP_DEREF; break; - case try_stmt: - com_try_stmt(c, n); + case LOCAL: + if (c->u->u_ste->ste_type == FunctionBlock) + optype = OP_FAST; break; - case suite: - com_suite(c, n); + case GLOBAL_IMPLICIT: + if (!c->u->u_ste->ste_unoptimized) + optype = OP_GLOBAL; break; - - /* Expression nodes */ - - case yield_expr: - com_yield_expr(c, n); + case GLOBAL_EXPLICIT: + optype = OP_GLOBAL; break; + } - case testlist: - case testlist1: - case testlist_safe: - com_list(c, n, 0); - break; - case test: - com_test(c, n); - break; - case and_test: - com_and_test(c, n); - break; - case not_test: - com_not_test(c, n); - break; - case comparison: - com_comparison(c, n); - break; - case exprlist: - com_list(c, n, 0); - break; - case expr: - com_expr(c, n); - break; - case xor_expr: - com_xor_expr(c, n); - break; - case and_expr: - com_and_expr(c, n); - break; - case shift_expr: - com_shift_expr(c, n); - break; - case arith_expr: - com_arith_expr(c, n); - break; - case term: - com_term(c, n); - break; - case factor: - com_factor(c, n); + /* XXX Leave assert here, but handle __doc__ and the like better */ + assert(scope || PyString_AS_STRING(name)[0] == '_'); + + switch (optype) { + case OP_DEREF: + switch (ctx) { + case Load: op = LOAD_DEREF; break; + case Store: op = STORE_DEREF; break; + case AugLoad: + case AugStore: + break; + case Del: + PyErr_Format(PyExc_SyntaxError, + "can not delete variable '%s' referenced " + "in nested scope", + PyString_AS_STRING(name)); + return 0; + break; + case Param: + assert(0); /* impossible */ + } break; - case power: - com_power(c, n); + case OP_FAST: + switch (ctx) { + case Load: op = LOAD_FAST; break; + case Store: op = STORE_FAST; break; + case Del: op = DELETE_FAST; break; + case AugLoad: + case AugStore: + break; + case Param: + assert(0); /* impossible */ + } + ADDOP_O(c, op, name, varnames); + return 1; + case OP_GLOBAL: + switch (ctx) { + case Load: op = LOAD_GLOBAL; break; + case Store: op = STORE_GLOBAL; break; + case Del: op = DELETE_GLOBAL; break; + case AugLoad: + case AugStore: + break; + case Param: + assert(0); /* impossible */ + } break; - case atom: - com_atom(c, n); + case OP_NAME: + switch (ctx) { + case Load: op = LOAD_NAME; break; + case Store: op = STORE_NAME; break; + case Del: op = DELETE_NAME; break; + case AugLoad: + case AugStore: + break; + case Param: + assert(0); /* impossible */ + } break; - - default: - com_error(c, PyExc_SystemError, - "com_node: unexpected node type"); } -} - -static void com_fplist(struct compiling *, node *); -static void -com_fpdef(struct compiling *c, node *n) -{ - REQ(n, fpdef); /* fpdef: NAME | '(' fplist ')' */ - if (TYPE(CHILD(n, 0)) == LPAR) - com_fplist(c, CHILD(n, 1)); - else { - com_addop_varname(c, VAR_STORE, STR(CHILD(n, 0))); - com_pop(c, 1); - } + assert(op); + return compiler_addop_name(c, op, dict, name); } -static void -com_fplist(struct compiling *c, node *n) +static int +compiler_boolop(struct compiler *c, expr_ty e) { - REQ(n, fplist); /* fplist: fpdef (',' fpdef)* [','] */ - if (NCH(n) == 1) { - com_fpdef(c, CHILD(n, 0)); - } - else { - int i = (NCH(n)+1)/2; - com_addoparg(c, UNPACK_SEQUENCE, i); - com_push(c, i-1); - for (i = 0; i < NCH(n); i += 2) - com_fpdef(c, CHILD(n, i)); - } -} + basicblock *end; + int jumpi, i, n; + asdl_seq *s; -static void -com_arglist(struct compiling *c, node *n) -{ - int nch, i, narg; - int complex = 0; - char nbuf[30]; - REQ(n, varargslist); - /* varargslist: - (fpdef ['=' test] ',')* (fpdef ['=' test] | '*' .....) */ - nch = NCH(n); - /* Enter all arguments in table of locals */ - for (i = 0, narg = 0; i < nch; i++) { - node *ch = CHILD(n, i); - node *fp; - if (TYPE(ch) == STAR || TYPE(ch) == DOUBLESTAR) - break; - REQ(ch, fpdef); /* fpdef: NAME | '(' fplist ')' */ - fp = CHILD(ch, 0); - if (TYPE(fp) != NAME) { - PyOS_snprintf(nbuf, sizeof(nbuf), ".%d", i); - complex = 1; - } - narg++; - /* all name updates handled by symtable */ - if (++i >= nch) - break; - ch = CHILD(n, i); - if (TYPE(ch) == EQUAL) - i += 2; - else - REQ(ch, COMMA); - } - if (complex) { - /* Generate code for complex arguments only after - having counted the simple arguments */ - int ilocal = 0; - for (i = 0; i < nch; i++) { - node *ch = CHILD(n, i); - node *fp; - if (TYPE(ch) == STAR || TYPE(ch) == DOUBLESTAR) - break; - REQ(ch, fpdef); /* fpdef: NAME | '(' fplist ')' */ - fp = CHILD(ch, 0); - if (TYPE(fp) != NAME) { - com_addoparg(c, LOAD_FAST, ilocal); - com_push(c, 1); - com_fpdef(c, ch); - } - ilocal++; - if (++i >= nch) - break; - ch = CHILD(n, i); - if (TYPE(ch) == EQUAL) - i += 2; - else - REQ(ch, COMMA); - } - } + assert(e->kind == BoolOp_kind); + if (e->v.BoolOp.op == And) + jumpi = JUMP_IF_FALSE; + else + jumpi = JUMP_IF_TRUE; + end = compiler_new_block(c); + if (end < 0) + return 0; + s = e->v.BoolOp.values; + n = asdl_seq_LEN(s) - 1; + for (i = 0; i < n; ++i) { + VISIT(c, expr, asdl_seq_GET(s, i)); + ADDOP_JREL(c, jumpi, end); + ADDOP(c, POP_TOP) + } + VISIT(c, expr, asdl_seq_GET(s, n)); + compiler_use_next_block(c, end); + return 1; } -static void -com_file_input(struct compiling *c, node *n) +static int +compiler_list(struct compiler *c, expr_ty e) { - int i; - PyObject *doc; - REQ(n, file_input); /* (NEWLINE | stmt)* ENDMARKER */ - doc = get_docstring(c, n); - if (doc != NULL) { - int i = com_addconst(c, doc); - Py_DECREF(doc); - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - com_addop_name(c, STORE_NAME, "__doc__"); - com_pop(c, 1); + int n = asdl_seq_LEN(e->v.List.elts); + if (e->v.List.ctx == Store) { + ADDOP_I(c, UNPACK_SEQUENCE, n); } - for (i = 0; i < NCH(n); i++) { - node *ch = CHILD(n, i); - if (TYPE(ch) != ENDMARKER && TYPE(ch) != NEWLINE) - com_node(c, ch); + VISIT_SEQ(c, expr, e->v.List.elts); + if (e->v.List.ctx == Load) { + ADDOP_I(c, BUILD_LIST, n); } + return 1; } -/* Top-level compile-node interface */ - -static void -compile_funcdef(struct compiling *c, node *n) +static int +compiler_tuple(struct compiler *c, expr_ty e) { - PyObject *doc; - node *ch; - REQ(n, funcdef); - /* -6 -5 -4 -3 -2 -1 - funcdef: [decorators] 'def' NAME parameters ':' suite */ - c->c_name = STR(RCHILD(n, -4)); - doc = get_docstring(c, RCHILD(n, -1)); - if (doc != NULL) { - (void) com_addconst(c, doc); - Py_DECREF(doc); + int n = asdl_seq_LEN(e->v.Tuple.elts); + if (e->v.Tuple.ctx == Store) { + ADDOP_I(c, UNPACK_SEQUENCE, n); } - else - (void) com_addconst(c, Py_None); /* No docstring */ - ch = RCHILD(n, -3); /* parameters: '(' [varargslist] ')' */ - ch = CHILD(ch, 1); /* ')' | varargslist */ - if (TYPE(ch) == varargslist) - com_arglist(c, ch); - c->c_infunction = 1; - com_node(c, RCHILD(n, -1)); - c->c_infunction = 0; - com_strip_lnotab(c); - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - com_addbyte(c, RETURN_VALUE); - com_pop(c, 1); -} - -static void -compile_lambdef(struct compiling *c, node *n) -{ - node *ch; - REQ(n, lambdef); /* lambdef: 'lambda' [varargslist] ':' test */ - c->c_name = "<lambda>"; - - ch = CHILD(n, 1); - (void) com_addconst(c, Py_None); /* No docstring */ - if (TYPE(ch) == varargslist) { - com_arglist(c, ch); - ch = CHILD(n, 3); + VISIT_SEQ(c, expr, e->v.Tuple.elts); + if (e->v.Tuple.ctx == Load) { + ADDOP_I(c, BUILD_TUPLE, n); } - else - ch = CHILD(n, 2); - com_node(c, ch); - com_addbyte(c, RETURN_VALUE); - com_pop(c, 1); + return 1; } -static void -compile_classdef(struct compiling *c, node *n) -{ - node *ch; - PyObject *doc; - REQ(n, classdef); - /* classdef: 'class' NAME ['(' testlist ')'] ':' suite */ - c->c_name = STR(CHILD(n, 1)); - c->c_private = c->c_name; - /* Initialize local __module__ from global __name__ */ - com_addop_name(c, LOAD_GLOBAL, "__name__"); - com_addop_name(c, STORE_NAME, "__module__"); - ch = CHILD(n, NCH(n)-1); /* The suite */ - doc = get_docstring(c, ch); - if (doc != NULL) { - int i = com_addconst(c, doc); - Py_DECREF(doc); - com_addoparg(c, LOAD_CONST, i); - com_push(c, 1); - com_addop_name(c, STORE_NAME, "__doc__"); - com_pop(c, 1); +static int +compiler_compare(struct compiler *c, expr_ty e) +{ + int i, n; + basicblock *cleanup = NULL; + + /* XXX the logic can be cleaned up for 1 or multiple comparisons */ + VISIT(c, expr, e->v.Compare.left); + n = asdl_seq_LEN(e->v.Compare.ops); + assert(n > 0); + if (n > 1) { + cleanup = compiler_new_block(c); + if (cleanup == NULL) + return 0; + VISIT(c, expr, asdl_seq_GET(e->v.Compare.comparators, 0)); + } + for (i = 1; i < n; i++) { + ADDOP(c, DUP_TOP); + ADDOP(c, ROT_THREE); + /* XXX We're casting a void* to cmpop_ty in the next stmt. */ + ADDOP_I(c, COMPARE_OP, + cmpop((cmpop_ty)asdl_seq_GET(e->v.Compare.ops, i - 1))); + ADDOP_JREL(c, JUMP_IF_FALSE, cleanup); + NEXT_BLOCK(c); + ADDOP(c, POP_TOP); + if (i < (n - 1)) + VISIT(c, expr, asdl_seq_GET(e->v.Compare.comparators, i)); + } + VISIT(c, expr, asdl_seq_GET(e->v.Compare.comparators, n - 1)); + ADDOP_I(c, COMPARE_OP, + /* XXX We're casting a void* to cmpop_ty in the next stmt. */ + cmpop((cmpop_ty)asdl_seq_GET(e->v.Compare.ops, n - 1))); + if (n > 1) { + basicblock *end = compiler_new_block(c); + if (end == NULL) + return 0; + ADDOP_JREL(c, JUMP_FORWARD, end); + compiler_use_next_block(c, cleanup); + ADDOP(c, ROT_TWO); + ADDOP(c, POP_TOP); + compiler_use_next_block(c, end); } - else - (void) com_addconst(c, Py_None); - com_node(c, ch); - com_strip_lnotab(c); - com_addbyte(c, LOAD_LOCALS); - com_push(c, 1); - com_addbyte(c, RETURN_VALUE); - com_pop(c, 1); + return 1; } -static void -compile_generator_expression(struct compiling *c, node *n) +static int +compiler_call(struct compiler *c, expr_ty e) { - /* testlist_gexp: test gen_for */ - /* argument: test gen_for */ - REQ(CHILD(n, 0), test); - REQ(CHILD(n, 1), gen_for); - - c->c_name = "<generator expression>"; - c->c_infunction = 1; - com_gen_for(c, CHILD(n, 1), CHILD(n, 0), 1); - c->c_infunction = 0; - - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - com_addbyte(c, RETURN_VALUE); - com_pop(c, 1); -} + int n, code = 0; -static void -compile_node(struct compiling *c, node *n) -{ - com_set_lineno(c, n->n_lineno); - - switch (TYPE(n)) { - - case single_input: /* One interactive command */ - /* NEWLINE | simple_stmt | compound_stmt NEWLINE */ - c->c_interactive++; - n = CHILD(n, 0); - if (TYPE(n) != NEWLINE) - com_node(c, n); - com_strip_lnotab(c); - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - com_addbyte(c, RETURN_VALUE); - com_pop(c, 1); - c->c_interactive--; + VISIT(c, expr, e->v.Call.func); + n = asdl_seq_LEN(e->v.Call.args); + VISIT_SEQ(c, expr, e->v.Call.args); + if (e->v.Call.keywords) { + VISIT_SEQ(c, keyword, e->v.Call.keywords); + n |= asdl_seq_LEN(e->v.Call.keywords) << 8; + } + if (e->v.Call.starargs) { + VISIT(c, expr, e->v.Call.starargs); + code |= 1; + } + if (e->v.Call.kwargs) { + VISIT(c, expr, e->v.Call.kwargs); + code |= 2; + } + switch (code) { + case 0: + ADDOP_I(c, CALL_FUNCTION, n); break; - - case file_input: /* A whole file, or built-in function exec() */ - com_file_input(c, n); - com_strip_lnotab(c); - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - com_addbyte(c, RETURN_VALUE); - com_pop(c, 1); + case 1: + ADDOP_I(c, CALL_FUNCTION_VAR, n); break; - - case eval_input: /* Built-in function input() */ - com_node(c, CHILD(n, 0)); - com_addbyte(c, RETURN_VALUE); - com_pop(c, 1); + case 2: + ADDOP_I(c, CALL_FUNCTION_KW, n); break; - - case lambdef: /* anonymous function definition */ - compile_lambdef(c, n); + case 3: + ADDOP_I(c, CALL_FUNCTION_VAR_KW, n); break; + } + return 1; +} - case funcdef: /* A function definition */ - compile_funcdef(c, n); - break; - - case classdef: /* A class definition */ - compile_classdef(c, n); - break; +static int +compiler_listcomp_generator(struct compiler *c, PyObject *tmpname, + asdl_seq *generators, int gen_index, + expr_ty elt) +{ + /* generate code for the iterator, then each of the ifs, + and then write to the element */ + + comprehension_ty l; + basicblock *start, *anchor, *skip, *if_cleanup; + int i, n; + + start = compiler_new_block(c); + skip = compiler_new_block(c); + if_cleanup = compiler_new_block(c); + anchor = compiler_new_block(c); + + if (start == NULL || skip == NULL || if_cleanup == NULL || + anchor == NULL) + return 0; + + l = asdl_seq_GET(generators, gen_index); + VISIT(c, expr, l->iter); + ADDOP(c, GET_ITER); + compiler_use_next_block(c, start); + ADDOP_JREL(c, FOR_ITER, anchor); + NEXT_BLOCK(c); + VISIT(c, expr, l->target); + + /* XXX this needs to be cleaned up...a lot! */ + n = asdl_seq_LEN(l->ifs); + for (i = 0; i < n; i++) { + expr_ty e = asdl_seq_GET(l->ifs, i); + VISIT(c, expr, e); + ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup); + NEXT_BLOCK(c); + ADDOP(c, POP_TOP); + } + + if (++gen_index < asdl_seq_LEN(generators)) + if (!compiler_listcomp_generator(c, tmpname, + generators, gen_index, elt)) + return 0; + + /* only append after the last for generator */ + if (gen_index >= asdl_seq_LEN(generators)) { + if (!compiler_nameop(c, tmpname, Load)) + return 0; + VISIT(c, expr, elt); + ADDOP_I(c, CALL_FUNCTION, 1); + ADDOP(c, POP_TOP); + + compiler_use_next_block(c, skip); + } + for (i = 0; i < n; i++) { + ADDOP_I(c, JUMP_FORWARD, 1); + if (i == 0) + compiler_use_next_block(c, if_cleanup); + ADDOP(c, POP_TOP); + } + ADDOP_JABS(c, JUMP_ABSOLUTE, start); + compiler_use_next_block(c, anchor); + /* delete the append method added to locals */ + if (gen_index == 1) + if (!compiler_nameop(c, tmpname, Del)) + return 0; - case testlist_gexp: /* A generator expression */ - case argument: /* A generator expression */ - compile_generator_expression(c, n); - break; + return 1; +} - default: - com_error(c, PyExc_SystemError, - "compile_node: unexpected node type"); +static int +compiler_listcomp(struct compiler *c, expr_ty e) +{ + char tmpname[256]; + identifier tmp; + int rc = 0; + static identifier append; + asdl_seq *generators = e->v.ListComp.generators; + + assert(e->kind == ListComp_kind); + if (!append) { + append = PyString_InternFromString("append"); + if (!append) + return 0; } + PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", ++c->u->u_tmpname); + tmp = PyString_FromString(tmpname); + if (!tmp) + return 0; + ADDOP_I(c, BUILD_LIST, 0); + ADDOP(c, DUP_TOP); + ADDOP_O(c, LOAD_ATTR, append, names); + if (compiler_nameop(c, tmp, Store)) + rc = compiler_listcomp_generator(c, tmp, generators, 0, + e->v.ListComp.elt); + Py_DECREF(tmp); + return rc; } -static PyObject * -dict_keys_inorder(PyObject *dict, int offset) +static int +compiler_genexp_generator(struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt) { - PyObject *tuple, *k, *v; - int i, pos = 0, size = PyDict_Size(dict); + /* generate code for the iterator, then each of the ifs, + and then write to the element */ - tuple = PyTuple_New(size); - if (tuple == NULL) - return NULL; - while (PyDict_Next(dict, &pos, &k, &v)) { - i = PyInt_AS_LONG(v); - Py_INCREF(k); - assert((i - offset) < size); - PyTuple_SET_ITEM(tuple, i - offset, k); - } - return tuple; -} + comprehension_ty ge; + basicblock *start, *anchor, *skip, *if_cleanup, *end; + int i, n; -PyCodeObject * -PyNode_Compile(node *n, const char *filename) -{ - return PyNode_CompileFlags(n, filename, NULL); -} + start = compiler_new_block(c); + skip = compiler_new_block(c); + if_cleanup = compiler_new_block(c); + anchor = compiler_new_block(c); + end = compiler_new_block(c); -PyCodeObject * -PyNode_CompileFlags(node *n, const char *filename, PyCompilerFlags *flags) -{ - return jcompile(n, filename, NULL, flags); -} + if (start == NULL || skip == NULL || if_cleanup == NULL || + anchor == NULL || end == NULL) + return 0; -struct symtable * -PyNode_CompileSymtable(node *n, const char *filename) -{ - struct symtable *st; - PyFutureFeatures *ff; + ge = asdl_seq_GET(generators, gen_index); + ADDOP_JREL(c, SETUP_LOOP, end); + if (!compiler_push_fblock(c, LOOP, start)) + return 0; - ff = PyNode_Future(n, filename); - if (ff == NULL) - return NULL; - st = symtable_build(n, ff, filename); - if (st == NULL) { - PyObject_FREE((void *)ff); - return NULL; + if (gen_index == 0) { + /* Receive outermost iter as an implicit argument */ + c->u->u_argcount = 1; + ADDOP_I(c, LOAD_FAST, 0); } - return st; -} + else { + /* Sub-iter - calculate on the fly */ + VISIT(c, expr, ge->iter); + ADDOP(c, GET_ITER); + } + compiler_use_next_block(c, start); + ADDOP_JREL(c, FOR_ITER, anchor); + NEXT_BLOCK(c); + VISIT(c, expr, ge->target); + + /* XXX this needs to be cleaned up...a lot! */ + n = asdl_seq_LEN(ge->ifs); + for (i = 0; i < n; i++) { + expr_ty e = asdl_seq_GET(ge->ifs, i); + VISIT(c, expr, e); + ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup); + NEXT_BLOCK(c); + ADDOP(c, POP_TOP); + } + + if (++gen_index < asdl_seq_LEN(generators)) + if (!compiler_genexp_generator(c, generators, gen_index, elt)) + return 0; -static PyCodeObject * -icompile(node *n, struct compiling *base) -{ - return jcompile(n, base->c_filename, base, NULL); + /* only append after the last 'for' generator */ + if (gen_index >= asdl_seq_LEN(generators)) { + VISIT(c, expr, elt); + ADDOP(c, YIELD_VALUE); + ADDOP(c, POP_TOP); + + compiler_use_next_block(c, skip); + } + for (i = 0; i < n; i++) { + ADDOP_I(c, JUMP_FORWARD, 1); + if (i == 0) + compiler_use_next_block(c, if_cleanup); + + ADDOP(c, POP_TOP); + } + ADDOP_JABS(c, JUMP_ABSOLUTE, start); + compiler_use_next_block(c, anchor); + ADDOP(c, POP_BLOCK); + compiler_pop_fblock(c, LOOP, start); + compiler_use_next_block(c, end); + + return 1; } -static PyCodeObject * -jcompile(node *n, const char *filename, struct compiling *base, - PyCompilerFlags *flags) +static int +compiler_genexp(struct compiler *c, expr_ty e) { - struct compiling sc; + PyObject *name; PyCodeObject *co; - if (!com_init(&sc, filename)) - return NULL; - if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) { - sc.c_encoding = "utf-8"; - } else if (TYPE(n) == encoding_decl) { - sc.c_encoding = STR(n); - n = CHILD(n, 0); - } else { - sc.c_encoding = NULL; - } - if (base) { - sc.c_private = base->c_private; - sc.c_symtable = base->c_symtable; - /* c_symtable still points to parent's symbols */ - if (base->c_nested - || (sc.c_symtable->st_cur->ste_type == TYPE_FUNCTION)) - sc.c_nested = 1; - sc.c_flags |= base->c_flags & PyCF_MASK; - if (base->c_encoding != NULL) { - assert(sc.c_encoding == NULL); - sc.c_encoding = base->c_encoding; - } - } else { - sc.c_private = NULL; - sc.c_future = PyNode_Future(n, filename); - if (sc.c_future == NULL) { - com_free(&sc); - return NULL; - } - if (flags) { - int merged = sc.c_future->ff_features | - flags->cf_flags; - sc.c_future->ff_features = merged; - flags->cf_flags = merged; - } - sc.c_symtable = symtable_build(n, sc.c_future, sc.c_filename); - if (sc.c_symtable == NULL) { - com_free(&sc); - return NULL; - } - /* reset symbol table for second pass */ - sc.c_symtable->st_nscopes = 1; - sc.c_symtable->st_pass = 2; - } - co = NULL; - if (symtable_load_symbols(&sc) < 0) { - sc.c_errors++; - goto exit; - } - compile_node(&sc, n); - com_done(&sc); - if (sc.c_errors == 0) { - PyObject *consts, *names, *varnames, *filename, *name, - *freevars, *cellvars, *code; - names = PyList_AsTuple(sc.c_names); - varnames = PyList_AsTuple(sc.c_varnames); - cellvars = dict_keys_inorder(sc.c_cellvars, 0); - freevars = dict_keys_inorder(sc.c_freevars, - PyTuple_GET_SIZE(cellvars)); - filename = PyString_InternFromString(sc.c_filename); - name = PyString_InternFromString(sc.c_name); - code = optimize_code(sc.c_code, sc.c_consts, names, sc.c_lnotab); - consts = PyList_AsTuple(sc.c_consts); - if (!PyErr_Occurred()) - co = PyCode_New(sc.c_argcount, - sc.c_nlocals, - sc.c_maxstacklevel, - sc.c_flags, - code, - consts, - names, - varnames, - freevars, - cellvars, - filename, - name, - sc.c_firstlineno, - sc.c_lnotab); - Py_XDECREF(consts); - Py_XDECREF(names); - Py_XDECREF(varnames); - Py_XDECREF(freevars); - Py_XDECREF(cellvars); - Py_XDECREF(filename); - Py_XDECREF(name); - Py_XDECREF(code); - } - else if (!PyErr_Occurred()) { - /* This could happen if someone called PyErr_Clear() after an - error was reported above. That's not supposed to happen, - but I just plugged one case and I'm not sure there can't be - others. In that case, raise SystemError so that at least - it gets reported instead dumping core. */ - PyErr_SetString(PyExc_SystemError, "lost syntax error"); - } - exit: - if (base == NULL) { - PySymtable_Free(sc.c_symtable); - sc.c_symtable = NULL; - } - com_free(&sc); - return co; -} + expr_ty outermost_iter = ((comprehension_ty) + (asdl_seq_GET(e->v.GeneratorExp.generators, + 0)))->iter; -int -PyCode_Addr2Line(PyCodeObject *co, int addrq) -{ - int size = PyString_Size(co->co_lnotab) / 2; - unsigned char *p = (unsigned char*)PyString_AsString(co->co_lnotab); - int line = co->co_firstlineno; - int addr = 0; - while (--size >= 0) { - addr += *p++; - if (addr > addrq) - break; - line += *p++; - } - return line; -} + name = PyString_FromString("<generator expression>"); + if (!name) + return 0; -/* The test for LOCAL must come before the test for FREE in order to - handle classes where name is both local and free. The local var is - a method and the free var is a free var referenced within a method. -*/ + if (!compiler_enter_scope(c, name, (void *)e, e->lineno)) + return 0; + compiler_genexp_generator(c, e->v.GeneratorExp.generators, 0, + e->v.GeneratorExp.elt); + co = assemble(c, 1); + if (co == NULL) + return 0; + compiler_exit_scope(c); + + compiler_make_closure(c, co, 0); + VISIT(c, expr, outermost_iter); + ADDOP(c, GET_ITER); + ADDOP_I(c, CALL_FUNCTION, 1); + Py_DECREF(name); + Py_DECREF(co); + + return 1; +} static int -get_ref_type(struct compiling *c, char *name) +compiler_visit_keyword(struct compiler *c, keyword_ty k) { - char buf[350]; - PyObject *v; - - if (PyDict_GetItemString(c->c_cellvars, name) != NULL) - return CELL; - if (PyDict_GetItemString(c->c_locals, name) != NULL) - return LOCAL; - if (PyDict_GetItemString(c->c_freevars, name) != NULL) - return FREE; - v = PyDict_GetItemString(c->c_globals, name); - if (v) { - if (v == Py_None) - return GLOBAL_EXPLICIT; - else { - return GLOBAL_IMPLICIT; - } - } - PyOS_snprintf(buf, sizeof(buf), - "unknown scope for %.100s in %.100s(%s) " - "in %s\nsymbols: %s\nlocals: %s\nglobals: %s\n", - name, c->c_name, - PyObject_REPR(c->c_symtable->st_cur->ste_id), - c->c_filename, - PyObject_REPR(c->c_symtable->st_cur->ste_symbols), - PyObject_REPR(c->c_locals), - PyObject_REPR(c->c_globals) - ); - - Py_FatalError(buf); - return -1; + ADDOP_O(c, LOAD_CONST, k->arg, consts); + VISIT(c, expr, k->value); + return 1; } -/* Helper functions to issue warnings */ +/* Test whether expression is constant. For constants, report + whether they are true or false. + + Return values: 1 for true, 0 for false, -1 for non-constant. + */ static int -issue_warning(const char *msg, const char *filename, int lineno) +expr_constant(expr_ty e) { - if (PyErr_Occurred()) { - /* This can happen because symtable_node continues - processing even after raising a SyntaxError. - Calling PyErr_WarnExplicit now would clobber the - pending exception; instead we fail and let that - exception propagate. - */ - return -1; - } - if (PyErr_WarnExplicit(PyExc_SyntaxWarning, msg, filename, - lineno, NULL, NULL) < 0) { - if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { - PyErr_SetString(PyExc_SyntaxError, msg); - PyErr_SyntaxLocation(filename, lineno); - } + switch (e->kind) { + case Num_kind: + return PyObject_IsTrue(e->v.Num.n); + case Str_kind: + return PyObject_IsTrue(e->v.Str.s); + default: return -1; } - return 0; } static int -symtable_warn(struct symtable *st, char *msg) -{ - if (issue_warning(msg, st->st_filename, st->st_cur->ste_lineno) < 0) { - st->st_errors++; - return -1; +compiler_visit_expr(struct compiler *c, expr_ty e) +{ + int i, n; + + if (e->lineno > c->u->u_lineno) { + c->u->u_lineno = e->lineno; + c->u->u_lineno_set = false; + } + switch (e->kind) { + case BoolOp_kind: + return compiler_boolop(c, e); + case BinOp_kind: + VISIT(c, expr, e->v.BinOp.left); + VISIT(c, expr, e->v.BinOp.right); + ADDOP(c, binop(c, e->v.BinOp.op)); + break; + case UnaryOp_kind: + VISIT(c, expr, e->v.UnaryOp.operand); + ADDOP(c, unaryop(e->v.UnaryOp.op)); + break; + case Lambda_kind: + return compiler_lambda(c, e); + case Dict_kind: + /* XXX get rid of arg? */ + ADDOP_I(c, BUILD_MAP, 0); + n = asdl_seq_LEN(e->v.Dict.values); + /* We must arrange things just right for STORE_SUBSCR. + It wants the stack to look like (value) (dict) (key) */ + for (i = 0; i < n; i++) { + ADDOP(c, DUP_TOP); + VISIT(c, expr, asdl_seq_GET(e->v.Dict.values, i)); + ADDOP(c, ROT_TWO); + VISIT(c, expr, asdl_seq_GET(e->v.Dict.keys, i)); + ADDOP(c, STORE_SUBSCR); + } + break; + case ListComp_kind: + return compiler_listcomp(c, e); + case GeneratorExp_kind: + return compiler_genexp(c, e); + case Yield_kind: + if (c->u->u_ste->ste_type != FunctionBlock) + return compiler_error(c, "'yield' outside function"); + /* + for (i = 0; i < c->u->u_nfblocks; i++) { + if (c->u->u_fblock[i].fb_type == FINALLY_TRY) + return compiler_error( + c, "'yield' not allowed in a 'try' " + "block with a 'finally' clause"); + } + */ + if (e->v.Yield.value) { + VISIT(c, expr, e->v.Yield.value); + } + else { + ADDOP_O(c, LOAD_CONST, Py_None, consts); + } + ADDOP(c, YIELD_VALUE); + break; + case Compare_kind: + return compiler_compare(c, e); + case Call_kind: + return compiler_call(c, e); + case Repr_kind: + VISIT(c, expr, e->v.Repr.value); + ADDOP(c, UNARY_CONVERT); + break; + case Num_kind: + ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts); + break; + case Str_kind: + ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts); + break; + /* The following exprs can be assignment targets. */ + case Attribute_kind: + if (e->v.Attribute.ctx != AugStore) + VISIT(c, expr, e->v.Attribute.value); + switch (e->v.Attribute.ctx) { + case AugLoad: + ADDOP(c, DUP_TOP); + /* Fall through to load */ + case Load: + ADDOP_NAME(c, LOAD_ATTR, e->v.Attribute.attr, names); + break; + case AugStore: + ADDOP(c, ROT_TWO); + /* Fall through to save */ + case Store: + ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names); + break; + case Del: + ADDOP_NAME(c, DELETE_ATTR, e->v.Attribute.attr, names); + break; + case Param: + assert(0); + break; + } + break; + case Subscript_kind: + switch (e->v.Subscript.ctx) { + case AugLoad: + VISIT(c, expr, e->v.Subscript.value); + VISIT_SLICE(c, e->v.Subscript.slice, AugLoad); + break; + case Load: + VISIT(c, expr, e->v.Subscript.value); + VISIT_SLICE(c, e->v.Subscript.slice, Load); + break; + case AugStore: + VISIT_SLICE(c, e->v.Subscript.slice, AugStore); + break; + case Store: + VISIT(c, expr, e->v.Subscript.value); + VISIT_SLICE(c, e->v.Subscript.slice, Store); + break; + case Del: + VISIT(c, expr, e->v.Subscript.value); + VISIT_SLICE(c, e->v.Subscript.slice, Del); + break; + case Param: + assert(0); + break; + } + break; + case Name_kind: + return compiler_nameop(c, e->v.Name.id, e->v.Name.ctx); + /* child nodes of List and Tuple will have expr_context set */ + case List_kind: + return compiler_list(c, e); + case Tuple_kind: + return compiler_tuple(c, e); } - return 0; + return 1; } -/* Helper function for setting lineno and filename */ - -static struct symtable * -symtable_build(node *n, PyFutureFeatures *ff, const char *filename) -{ - struct symtable *st; - - st = symtable_init(); - if (st == NULL) - return NULL; - st->st_future = ff; - st->st_filename = filename; - symtable_enter_scope(st, TOP, TYPE(n), n->n_lineno); - if (st->st_errors > 0) - goto fail; - symtable_node(st, n); - if (st->st_errors > 0) - goto fail; - return st; - fail: - if (!PyErr_Occurred()) { - /* This could happen because after a syntax error is - detected, the symbol-table-building continues for - a while, and PyErr_Clear() might erroneously be - called during that process. One such case has been - fixed, but there might be more (now or later). - */ - PyErr_SetString(PyExc_SystemError, "lost exception"); +static int +compiler_augassign(struct compiler *c, stmt_ty s) +{ + expr_ty e = s->v.AugAssign.target; + expr_ty auge; + + assert(s->kind == AugAssign_kind); + + switch (e->kind) { + case Attribute_kind: + auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr, + AugLoad, e->lineno); + if (auge == NULL) + return 0; + VISIT(c, expr, auge); + VISIT(c, expr, s->v.AugAssign.value); + ADDOP(c, inplace_binop(c, s->v.AugAssign.op)); + auge->v.Attribute.ctx = AugStore; + VISIT(c, expr, auge); + free(auge); + break; + case Subscript_kind: + auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice, + AugLoad, e->lineno); + if (auge == NULL) + return 0; + VISIT(c, expr, auge); + VISIT(c, expr, s->v.AugAssign.value); + ADDOP(c, inplace_binop(c, s->v.AugAssign.op)); + auge->v.Subscript.ctx = AugStore; + VISIT(c, expr, auge); + free(auge); + break; + case Name_kind: + VISIT(c, expr, s->v.AugAssign.target); + VISIT(c, expr, s->v.AugAssign.value); + ADDOP(c, inplace_binop(c, s->v.AugAssign.op)); + return compiler_nameop(c, e->v.Name.id, Store); + default: + fprintf(stderr, + "invalid node type for augmented assignment\n"); + return 0; } - st->st_future = NULL; - st->st_filename = NULL; - PySymtable_Free(st); - return NULL; + return 1; } static int -symtable_init_compiling_symbols(struct compiling *c) +compiler_push_fblock(struct compiler *c, enum fblocktype t, basicblock *b) { - PyObject *varnames; - - varnames = c->c_symtable->st_cur->ste_varnames; - if (varnames == NULL) { - varnames = PyList_New(0); - if (varnames == NULL) - return -1; - c->c_symtable->st_cur->ste_varnames = varnames; - Py_INCREF(varnames); - } else - Py_INCREF(varnames); - c->c_varnames = varnames; - - c->c_globals = PyDict_New(); - if (c->c_globals == NULL) - return -1; - c->c_freevars = PyDict_New(); - if (c->c_freevars == NULL) - return -1; - c->c_cellvars = PyDict_New(); - if (c->c_cellvars == NULL) - return -1; - return 0; + struct fblockinfo *f; + if (c->u->u_nfblocks >= CO_MAXBLOCKS) + return 0; + f = &c->u->u_fblock[c->u->u_nfblocks++]; + f->fb_type = t; + f->fb_block = b; + return 1; } -struct symbol_info { - int si_nlocals; - int si_ncells; - int si_nfrees; - int si_nimplicit; -}; - static void -symtable_init_info(struct symbol_info *si) +compiler_pop_fblock(struct compiler *c, enum fblocktype t, basicblock *b) { - si->si_nlocals = 0; - si->si_ncells = 0; - si->si_nfrees = 0; - si->si_nimplicit = 0; + struct compiler_unit *u = c->u; + assert(u->u_nfblocks > 0); + u->u_nfblocks--; + assert(u->u_fblock[u->u_nfblocks].fb_type == t); + assert(u->u_fblock[u->u_nfblocks].fb_block == b); } +/* Raises a SyntaxError and returns 0. + If something goes wrong, a different exception may be raised. +*/ + static int -symtable_resolve_free(struct compiling *c, PyObject *name, int flags, - struct symbol_info *si) +compiler_error(struct compiler *c, const char *errstr) { - PyObject *dict, *v; + PyObject *loc; + PyObject *u = NULL, *v = NULL; - /* Seperate logic for DEF_FREE. If it occurs in a function, - it indicates a local that we must allocate storage for (a - cell var). If it occurs in a class, then the class has a - method and a free variable with the same name. - */ - if (c->c_symtable->st_cur->ste_type == TYPE_FUNCTION) { - /* If it isn't declared locally, it can't be a cell. */ - if (!(flags & (DEF_LOCAL | DEF_PARAM))) - return 0; - v = PyInt_FromLong(si->si_ncells++); - dict = c->c_cellvars; - } else { - /* If it is free anyway, then there is no need to do - anything here. - */ - if (is_free(flags ^ DEF_FREE_CLASS) - || (flags == DEF_FREE_CLASS)) - return 0; - v = PyInt_FromLong(si->si_nfrees++); - dict = c->c_freevars; - } - if (v == NULL) - return -1; - if (PyDict_SetItem(dict, name, v) < 0) { - Py_DECREF(v); - return -1; + loc = PyErr_ProgramText(c->c_filename, c->u->u_lineno); + if (!loc) { + Py_INCREF(Py_None); + loc = Py_None; } - Py_DECREF(v); + u = Py_BuildValue("(ziOO)", c->c_filename, c->u->u_lineno, + Py_None, loc); + if (!u) + goto exit; + v = Py_BuildValue("(zO)", errstr, u); + if (!v) + goto exit; + PyErr_SetObject(PyExc_SyntaxError, v); + exit: + Py_DECREF(loc); + Py_XDECREF(u); + Py_XDECREF(v); return 0; } -/* If a variable is a cell and an argument, make sure that appears in - co_cellvars before any variable to its right in varnames. -*/ - +static int +compiler_handle_subscr(struct compiler *c, const char *kind, + expr_context_ty ctx) +{ + int op = 0; + + /* XXX this code is duplicated */ + switch (ctx) { + case AugLoad: /* fall through to Load */ + case Load: op = BINARY_SUBSCR; break; + case AugStore:/* fall through to Store */ + case Store: op = STORE_SUBSCR; break; + case Del: op = DELETE_SUBSCR; break; + case Param: + fprintf(stderr, + "invalid %s kind %d in subscript\n", + kind, ctx); + return 0; + } + if (ctx == AugLoad) { + ADDOP_I(c, DUP_TOPX, 2); + } + else if (ctx == AugStore) { + ADDOP(c, ROT_THREE); + } + ADDOP(c, op); + return 1; +} static int -symtable_cellvar_offsets(PyObject **cellvars, int argcount, - PyObject *varnames, int flags) +compiler_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) { - PyObject *v = NULL; - PyObject *w, *d, *list = NULL; - int i, pos; - - if (flags & CO_VARARGS) - argcount++; - if (flags & CO_VARKEYWORDS) - argcount++; - for (i = argcount; --i >= 0; ) { - v = PyList_GET_ITEM(varnames, i); - if (PyDict_GetItem(*cellvars, v)) { - if (list == NULL) { - list = PyList_New(1); - if (list == NULL) - return -1; - PyList_SET_ITEM(list, 0, v); - Py_INCREF(v); - } else { - if (PyList_Insert(list, 0, v) < 0) { - Py_DECREF(list); - return -1; - } - } - } - } - if (list == NULL) - return 0; + int n = 2; + assert(s->kind == Slice_kind); - /* There are cellvars that are also arguments. Create a dict - to replace cellvars and put the args at the front. - */ - d = PyDict_New(); - if (d == NULL) - return -1; - for (i = PyList_GET_SIZE(list); --i >= 0; ) { - v = PyInt_FromLong(i); - if (v == NULL) - goto fail; - if (PyDict_SetItem(d, PyList_GET_ITEM(list, i), v) < 0) - goto fail; - if (PyDict_DelItem(*cellvars, PyList_GET_ITEM(list, i)) < 0) - goto fail; - Py_DECREF(v); + /* only handles the cases where BUILD_SLICE is emitted */ + if (s->v.Slice.lower) { + VISIT(c, expr, s->v.Slice.lower); } - pos = 0; - i = PyList_GET_SIZE(list); - Py_DECREF(list); - while (PyDict_Next(*cellvars, &pos, &v, &w)) { - w = PyInt_FromLong(i++); /* don't care about the old key */ - if (w == NULL) - goto fail; - if (PyDict_SetItem(d, v, w) < 0) { - Py_DECREF(w); - v = NULL; - goto fail; - } - Py_DECREF(w); + else { + ADDOP_O(c, LOAD_CONST, Py_None, consts); + } + + if (s->v.Slice.upper) { + VISIT(c, expr, s->v.Slice.upper); + } + else { + ADDOP_O(c, LOAD_CONST, Py_None, consts); } - Py_DECREF(*cellvars); - *cellvars = d; - return 1; - fail: - Py_DECREF(d); - Py_XDECREF(v); - return -1; -} -static int -symtable_freevar_offsets(PyObject *freevars, int offset) -{ - PyObject *name, *v; - int pos; - - /* The cell vars are the first elements of the closure, - followed by the free vars. Update the offsets in - c_freevars to account for number of cellvars. */ - pos = 0; - while (PyDict_Next(freevars, &pos, &name, &v)) { - int i = PyInt_AS_LONG(v) + offset; - PyObject *o = PyInt_FromLong(i); - if (o == NULL) - return -1; - if (PyDict_SetItem(freevars, name, o) < 0) { - Py_DECREF(o); - return -1; - } - Py_DECREF(o); + if (s->v.Slice.step) { + n++; + VISIT(c, expr, s->v.Slice.step); } - return 0; + ADDOP_I(c, BUILD_SLICE, n); + return 1; } static int -symtable_check_unoptimized(struct compiling *c, - PySymtableEntryObject *ste, - struct symbol_info *si) -{ - char buf[300]; - - if (!(si->si_ncells || si->si_nfrees || ste->ste_child_free - || (ste->ste_nested && si->si_nimplicit))) - return 0; - -#define ILLEGAL_CONTAINS "contains a nested function with free variables" - -#define ILLEGAL_IS "is a nested function" - -#define ILLEGAL_IMPORT_STAR \ -"import * is not allowed in function '%.100s' because it %s" - -#define ILLEGAL_BARE_EXEC \ -"unqualified exec is not allowed in function '%.100s' it %s" - -#define ILLEGAL_EXEC_AND_IMPORT_STAR \ -"function '%.100s' uses import * and bare exec, which are illegal " \ -"because it %s" - - /* XXX perhaps the linenos for these opt-breaking statements - should be stored so the exception can point to them. */ - - if (ste->ste_child_free) { - if (ste->ste_optimized == OPT_IMPORT_STAR) - PyOS_snprintf(buf, sizeof(buf), - ILLEGAL_IMPORT_STAR, - PyString_AS_STRING(ste->ste_name), - ILLEGAL_CONTAINS); - else if (ste->ste_optimized == (OPT_BARE_EXEC | OPT_EXEC)) - PyOS_snprintf(buf, sizeof(buf), - ILLEGAL_BARE_EXEC, - PyString_AS_STRING(ste->ste_name), - ILLEGAL_CONTAINS); - else { - PyOS_snprintf(buf, sizeof(buf), - ILLEGAL_EXEC_AND_IMPORT_STAR, - PyString_AS_STRING(ste->ste_name), - ILLEGAL_CONTAINS); - } - } else { - if (ste->ste_optimized == OPT_IMPORT_STAR) - PyOS_snprintf(buf, sizeof(buf), - ILLEGAL_IMPORT_STAR, - PyString_AS_STRING(ste->ste_name), - ILLEGAL_IS); - else if (ste->ste_optimized == (OPT_BARE_EXEC | OPT_EXEC)) - PyOS_snprintf(buf, sizeof(buf), - ILLEGAL_BARE_EXEC, - PyString_AS_STRING(ste->ste_name), - ILLEGAL_IS); - else { - PyOS_snprintf(buf, sizeof(buf), - ILLEGAL_EXEC_AND_IMPORT_STAR, - PyString_AS_STRING(ste->ste_name), - ILLEGAL_IS); - } +compiler_simple_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) +{ + int op = 0, slice_offset = 0, stack_count = 0; + + assert(s->v.Slice.step == NULL); + if (s->v.Slice.lower) { + slice_offset++; + stack_count++; + if (ctx != AugStore) + VISIT(c, expr, s->v.Slice.lower); + } + if (s->v.Slice.upper) { + slice_offset += 2; + stack_count++; + if (ctx != AugStore) + VISIT(c, expr, s->v.Slice.upper); + } + + if (ctx == AugLoad) { + switch (stack_count) { + case 0: ADDOP(c, DUP_TOP); break; + case 1: ADDOP_I(c, DUP_TOPX, 2); break; + case 2: ADDOP_I(c, DUP_TOPX, 3); break; + } + } + else if (ctx == AugStore) { + switch (stack_count) { + case 0: ADDOP(c, ROT_TWO); break; + case 1: ADDOP(c, ROT_THREE); break; + case 2: ADDOP(c, ROT_FOUR); break; + } + } + + switch (ctx) { + case AugLoad: /* fall through to Load */ + case Load: op = SLICE; break; + case AugStore:/* fall through to Store */ + case Store: op = STORE_SLICE; break; + case Del: op = DELETE_SLICE; break; + case Param: /* XXX impossible? */ + fprintf(stderr, "param invalid\n"); + assert(0); } - PyErr_SetString(PyExc_SyntaxError, buf); - PyErr_SyntaxLocation(c->c_symtable->st_filename, - ste->ste_opt_lineno); - return -1; + ADDOP(c, op + slice_offset); + return 1; } static int -symtable_update_flags(struct compiling *c, PySymtableEntryObject *ste, - struct symbol_info *si) +compiler_visit_nested_slice(struct compiler *c, slice_ty s, + expr_context_ty ctx) { - if (c->c_future) - c->c_flags |= c->c_future->ff_features; - if (ste->ste_generator) - c->c_flags |= CO_GENERATOR; - if (ste->ste_type != TYPE_MODULE) - c->c_flags |= CO_NEWLOCALS; - if (ste->ste_type == TYPE_FUNCTION) { - c->c_nlocals = si->si_nlocals; - if (ste->ste_optimized == 0) - c->c_flags |= CO_OPTIMIZED; - else if (ste->ste_optimized != OPT_EXEC) - return symtable_check_unoptimized(c, ste, si); + switch (s->kind) { + case Ellipsis_kind: + ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts); + break; + case Slice_kind: + return compiler_slice(c, s, ctx); + break; + case Index_kind: + VISIT(c, expr, s->v.Index.value); + break; + case ExtSlice_kind: + assert(0); + break; } - return 0; + return 1; } -static int -symtable_error(struct symtable *st, int lineno) -{ - if (lineno == 0) - lineno = st->st_cur->ste_lineno; - PyErr_SyntaxLocation(st->st_filename, lineno); - st->st_errors++; - return -1; -} static int -symtable_load_symbols(struct compiling *c) +compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) { - struct symtable *st = c->c_symtable; - PySymtableEntryObject *ste = st->st_cur; - PyObject *name, *varnames, *v; - int i, flags, pos; - struct symbol_info si; - - v = NULL; - - if (symtable_init_compiling_symbols(c) < 0) - goto fail; - symtable_init_info(&si); - varnames = st->st_cur->ste_varnames; - si.si_nlocals = PyList_GET_SIZE(varnames); - c->c_argcount = si.si_nlocals; - - for (i = 0; i < si.si_nlocals; ++i) { - v = PyInt_FromLong(i); - if (v == NULL) - goto fail; - if (PyDict_SetItem(c->c_locals, - PyList_GET_ITEM(varnames, i), v) < 0) - goto fail; - Py_DECREF(v); - } - - /* XXX The cases below define the rules for whether a name is - local or global. The logic could probably be clearer. */ - pos = 0; - while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { - flags = PyInt_AS_LONG(v); - - if (flags & DEF_FREE_GLOBAL) - /* undo the original DEF_FREE */ - flags &= ~(DEF_FREE | DEF_FREE_CLASS); - - /* Deal with names that need two actions: - 1. Cell variables that are also locals. - 2. Free variables in methods that are also class - variables or declared global. - */ - if (flags & (DEF_FREE | DEF_FREE_CLASS)) - symtable_resolve_free(c, name, flags, &si); - - if (flags & DEF_STAR) { - c->c_argcount--; - c->c_flags |= CO_VARARGS; - } else if (flags & DEF_DOUBLESTAR) { - c->c_argcount--; - c->c_flags |= CO_VARKEYWORDS; - } else if (flags & DEF_INTUPLE) - c->c_argcount--; - else if (flags & DEF_GLOBAL) { - if (flags & DEF_PARAM) { - PyErr_Format(PyExc_SyntaxError, PARAM_GLOBAL, - PyString_AS_STRING(name)); - symtable_error(st, 0); - goto fail; - } - if (PyDict_SetItem(c->c_globals, name, Py_None) < 0) - goto fail; - } else if (flags & DEF_FREE_GLOBAL) { - si.si_nimplicit++; - if (PyDict_SetItem(c->c_globals, name, Py_True) < 0) - goto fail; - } else if ((flags & DEF_LOCAL) && !(flags & DEF_PARAM)) { - v = PyInt_FromLong(si.si_nlocals++); - if (v == NULL) - goto fail; - if (PyDict_SetItem(c->c_locals, name, v) < 0) - goto fail; - Py_DECREF(v); - if (ste->ste_type != TYPE_CLASS) - if (PyList_Append(c->c_varnames, name) < 0) - goto fail; - } else if (is_free(flags)) { - if (ste->ste_nested) { - v = PyInt_FromLong(si.si_nfrees++); - if (v == NULL) - goto fail; - if (PyDict_SetItem(c->c_freevars, name, v) < 0) - goto fail; - Py_DECREF(v); - } else { - si.si_nimplicit++; - if (PyDict_SetItem(c->c_globals, name, - Py_True) < 0) - goto fail; - if (st->st_nscopes != 1) { - v = PyInt_FromLong(flags); - if (v == NULL) - goto fail; - if (PyDict_SetItem(st->st_global, - name, v)) - goto fail; - Py_DECREF(v); - } - } + switch (s->kind) { + case Ellipsis_kind: + ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts); + break; + case Slice_kind: + if (!s->v.Slice.step) + return compiler_simple_slice(c, s, ctx); + if (!compiler_slice(c, s, ctx)) + return 0; + if (ctx == AugLoad) { + ADDOP_I(c, DUP_TOPX, 2); } + else if (ctx == AugStore) { + ADDOP(c, ROT_THREE); + } + return compiler_handle_subscr(c, "slice", ctx); + break; + case ExtSlice_kind: { + int i, n = asdl_seq_LEN(s->v.ExtSlice.dims); + for (i = 0; i < n; i++) { + slice_ty sub = asdl_seq_GET(s->v.ExtSlice.dims, i); + if (!compiler_visit_nested_slice(c, sub, ctx)) + return 0; + } + ADDOP_I(c, BUILD_TUPLE, n); + return compiler_handle_subscr(c, "extended slice", ctx); + break; } - assert(PyDict_Size(c->c_freevars) == si.si_nfrees); - - if (si.si_ncells > 1) { /* one cell is always in order */ - if (symtable_cellvar_offsets(&c->c_cellvars, c->c_argcount, - c->c_varnames, c->c_flags) < 0) - return -1; + case Index_kind: + if (ctx != AugStore) + VISIT(c, expr, s->v.Index.value); + return compiler_handle_subscr(c, "index", ctx); } - if (symtable_freevar_offsets(c->c_freevars, si.si_ncells) < 0) - return -1; - return symtable_update_flags(c, ste, &si); - fail: - /* is this always the right thing to do? */ - Py_XDECREF(v); - return -1; -} - -static struct symtable * -symtable_init() -{ - struct symtable *st; - - st = (struct symtable *)PyObject_MALLOC(sizeof(struct symtable)); - if (st == NULL) - return NULL; - st->st_pass = 1; - - st->st_filename = NULL; - st->st_symbols = NULL; - if ((st->st_stack = PyList_New(0)) == NULL) - goto fail; - if ((st->st_symbols = PyDict_New()) == NULL) - goto fail; - st->st_cur = NULL; - st->st_nscopes = 0; - st->st_errors = 0; - st->st_private = NULL; - return st; - fail: - PySymtable_Free(st); - return NULL; -} - -void -PySymtable_Free(struct symtable *st) -{ - Py_XDECREF(st->st_symbols); - Py_XDECREF(st->st_stack); - Py_XDECREF(st->st_cur); - PyObject_FREE((void *)st); + return 1; } -/* When the compiler exits a scope, it must should update the scope's - free variable information with the list of free variables in its - children. - - Variables that are free in children and defined in the current - scope are cellvars. - - If the scope being exited is defined at the top-level (ste_nested is - false), free variables in children that are not defined here are - implicit globals. +/* do depth-first search of basic block graph, starting with block. + post records the block indices in post-order. + XXX must handle implicit jumps from one block to next */ -static int -symtable_update_free_vars(struct symtable *st) +static void +dfs(struct compiler *c, basicblock *b, struct assembler *a) { - int i, j, def; - PyObject *o, *name, *list = NULL; - PySymtableEntryObject *child, *ste = st->st_cur; + int i; + struct instr *instr = NULL; - if (ste->ste_type == TYPE_CLASS) - def = DEF_FREE_CLASS; - else - def = DEF_FREE; - for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) { - int pos = 0; + if (b->b_seen) + return; + b->b_seen = 1; + if (b->b_next != NULL) + dfs(c, b->b_next, a); + for (i = 0; i < b->b_iused; i++) { + instr = &b->b_instr[i]; + if (instr->i_jrel || instr->i_jabs) + dfs(c, instr->i_target, a); + } + a->a_postorder[a->a_nblocks++] = b; +} - if (list && PyList_SetSlice(list, 0, - PyList_GET_SIZE(list), 0) < 0) - return -1; - child = (PySymtableEntryObject *) - PyList_GET_ITEM(ste->ste_children, i); - while (PyDict_Next(child->ste_symbols, &pos, &name, &o)) { - int flags = PyInt_AS_LONG(o); - if (!(is_free(flags))) - continue; /* avoids indentation */ - if (list == NULL) { - list = PyList_New(0); - if (list == NULL) - return -1; - } - ste->ste_child_free = 1; - if (PyList_Append(list, name) < 0) { - Py_DECREF(list); - return -1; - } - } - for (j = 0; list && j < PyList_GET_SIZE(list); j++) { - PyObject *v; - name = PyList_GET_ITEM(list, j); - v = PyDict_GetItem(ste->ste_symbols, name); - /* If a name N is declared global in scope A and - referenced in scope B contained (perhaps - indirectly) in A and there are no scopes - with bindings for N between B and A, then N - is global in B. Unless A is a class scope, - because class scopes are not considered for - nested scopes. - */ - if (v && (ste->ste_type != TYPE_CLASS)) { - int flags = PyInt_AS_LONG(v); - if (flags & DEF_GLOBAL) { - symtable_undo_free(st, child->ste_id, - name); - continue; - } - } - if (ste->ste_nested) { - if (symtable_add_def_o(st, ste->ste_symbols, - name, def) < 0) { - Py_DECREF(list); - return -1; - } - } else { - if (symtable_check_global(st, child->ste_id, - name) < 0) { - Py_DECREF(list); - return -1; - } +int +stackdepth_walk(struct compiler *c, basicblock *b, int depth, int maxdepth) +{ + int i; + struct instr *instr; + if (b->b_seen || b->b_startdepth >= depth) + return maxdepth; + b->b_seen = 1; + b->b_startdepth = depth; + for (i = 0; i < b->b_iused; i++) { + instr = &b->b_instr[i]; + depth += opcode_stack_effect(instr->i_opcode, instr->i_oparg); + if (depth > maxdepth) + maxdepth = depth; + assert(depth >= 0); /* invalid code or bug in stackdepth() */ + if (instr->i_jrel || instr->i_jabs) { + maxdepth = stackdepth_walk(c, instr->i_target, + depth, maxdepth); + if (instr->i_opcode == JUMP_ABSOLUTE || + instr->i_opcode == JUMP_FORWARD) { + goto out; /* remaining code is dead */ } } } - - Py_XDECREF(list); - return 0; + if (b->b_next) + maxdepth = stackdepth_walk(c, b->b_next, depth, maxdepth); +out: + b->b_seen = 0; + return maxdepth; } -/* If the current scope is a non-nested class or if name is not - defined in the current, non-nested scope, then it is an implicit - global in all nested scopes. -*/ - +/* Find the flow path that needs the largest stack. We assume that + * cycles in the flow graph have no net effect on the stack depth. + */ static int -symtable_check_global(struct symtable *st, PyObject *child, PyObject *name) +stackdepth(struct compiler *c) { - PyObject *o; - int v; - PySymtableEntryObject *ste = st->st_cur; - - if (ste->ste_type == TYPE_CLASS) - return symtable_undo_free(st, child, name); - o = PyDict_GetItem(ste->ste_symbols, name); - if (o == NULL) - return symtable_undo_free(st, child, name); - v = PyInt_AS_LONG(o); - - if (is_free(v) || (v & DEF_GLOBAL)) - return symtable_undo_free(st, child, name); - else - return symtable_add_def_o(st, ste->ste_symbols, - name, DEF_FREE); + basicblock *b, *entryblock; + entryblock = NULL; + for (b = c->u->u_blocks; b != NULL; b = b->b_list) { + b->b_seen = 0; + b->b_startdepth = INT_MIN; + entryblock = b; + } + return stackdepth_walk(c, entryblock, 0, 0); } static int -symtable_undo_free(struct symtable *st, PyObject *id, - PyObject *name) +assemble_init(struct assembler *a, int nblocks, int firstlineno) { - int i, v, x; - PyObject *info; - PySymtableEntryObject *ste; - - ste = (PySymtableEntryObject *)PyDict_GetItem(st->st_symbols, id); - if (ste == NULL) - return -1; - - info = PyDict_GetItem(ste->ste_symbols, name); - if (info == NULL) + memset(a, 0, sizeof(struct assembler)); + a->a_lineno = firstlineno; + a->a_bytecode = PyString_FromStringAndSize(NULL, DEFAULT_CODE_SIZE); + if (!a->a_bytecode) return 0; - v = PyInt_AS_LONG(info); - if (is_free(v)) { - if (symtable_add_def_o(st, ste->ste_symbols, name, - DEF_FREE_GLOBAL) < 0) - return -1; - } else - /* If the name is defined here or declared global, - then the recursion stops. */ + a->a_lnotab = PyString_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE); + if (!a->a_lnotab) return 0; - - for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) { - PySymtableEntryObject *child; - child = (PySymtableEntryObject *) - PyList_GET_ITEM(ste->ste_children, i); - x = symtable_undo_free(st, child->ste_id, name); - if (x < 0) - return x; - } - return 0; -} - -/* symtable_enter_scope() gets a reference via PySymtableEntry_New(). - This reference is released when the scope is exited, via the DECREF - in symtable_exit_scope(). -*/ - -static int -symtable_exit_scope(struct symtable *st) -{ - int end; - - if (st->st_pass == 1) - symtable_update_free_vars(st); - Py_DECREF(st->st_cur); - end = PyList_GET_SIZE(st->st_stack) - 1; - st->st_cur = (PySymtableEntryObject *)PyList_GET_ITEM(st->st_stack, - end); - if (PySequence_DelItem(st->st_stack, end) < 0) - return -1; - return 0; + a->a_postorder = (basicblock **)PyObject_Malloc( + sizeof(basicblock *) * nblocks); + if (!a->a_postorder) + return 0; + return 1; } static void -symtable_enter_scope(struct symtable *st, char *name, int type, - int lineno) +assemble_free(struct assembler *a) { - PySymtableEntryObject *prev = NULL; - - if (st->st_cur) { - prev = st->st_cur; - if (PyList_Append(st->st_stack, (PyObject *)st->st_cur) < 0) { - st->st_errors++; - return; - } - } - st->st_cur = (PySymtableEntryObject *) - PySymtableEntry_New(st, name, type, lineno); - if (st->st_cur == NULL) { - st->st_errors++; - return; - } - if (strcmp(name, TOP) == 0) - st->st_global = st->st_cur->ste_symbols; - if (prev && st->st_pass == 1) { - if (PyList_Append(prev->ste_children, - (PyObject *)st->st_cur) < 0) - st->st_errors++; - } + Py_XDECREF(a->a_bytecode); + Py_XDECREF(a->a_lnotab); + if (a->a_postorder) + PyObject_Free(a->a_postorder); } -static int -symtable_lookup(struct symtable *st, char *name) -{ - char buffer[MANGLE_LEN]; - PyObject *v; - int flags; - - if (_Py_Mangle(st->st_private, name, buffer, sizeof(buffer))) - name = buffer; - v = PyDict_GetItemString(st->st_cur->ste_symbols, name); - if (v == NULL) { - if (PyErr_Occurred()) - return -1; - else - return 0; - } - - flags = PyInt_AS_LONG(v); - return flags; -} +/* Return the size of a basic block in bytes. */ static int -symtable_add_def(struct symtable *st, char *name, int flag) +instrsize(struct instr *instr) { - PyObject *s; - char buffer[MANGLE_LEN]; - int ret; - - /* Warn about None, except inside a tuple (where the assignment - code already issues a warning). */ - if ((flag & DEF_PARAM) && !(flag & DEF_INTUPLE) && - *name == 'N' && strcmp(name, "None") == 0) - { - PyErr_SetString(PyExc_SyntaxError, - "Invalid syntax. Assignment to None."); - symtable_error(st, 0); - return -1; + int size = 1; + if (instr->i_hasarg) { + size += 2; + if (instr->i_oparg >> 16) + size += 2; } - if (_Py_Mangle(st->st_private, name, buffer, sizeof(buffer))) - name = buffer; - if ((s = PyString_InternFromString(name)) == NULL) - return -1; - ret = symtable_add_def_o(st, st->st_cur->ste_symbols, s, flag); - Py_DECREF(s); - return ret; + return size; } -/* Must only be called with mangled names */ - static int -symtable_add_def_o(struct symtable *st, PyObject *dict, - PyObject *name, int flag) +blocksize(basicblock *b) { - PyObject *o; - int val; - - if ((o = PyDict_GetItem(dict, name))) { - val = PyInt_AS_LONG(o); - if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { - PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, - PyString_AsString(name)); - return symtable_error(st, 0); - } - val |= flag; - } else - val = flag; - o = PyInt_FromLong(val); - if (o == NULL) - return -1; - if (PyDict_SetItem(dict, name, o) < 0) { - Py_DECREF(o); - return -1; - } - Py_DECREF(o); + int i; + int size = 0; - if (flag & DEF_PARAM) { - if (PyList_Append(st->st_cur->ste_varnames, name) < 0) - return -1; - } else if (flag & DEF_GLOBAL) { - /* XXX need to update DEF_GLOBAL for other flags too; - perhaps only DEF_FREE_GLOBAL */ - if ((o = PyDict_GetItem(st->st_global, name))) { - val = PyInt_AS_LONG(o); - val |= flag; - } else - val = flag; - o = PyInt_FromLong(val); - if (o == NULL) - return -1; - if (PyDict_SetItem(st->st_global, name, o) < 0) { - Py_DECREF(o); - return -1; - } - Py_DECREF(o); - } - return 0; + for (i = 0; i < b->b_iused; i++) + size += instrsize(&b->b_instr[i]); + return size; } -#define symtable_add_use(ST, NAME) symtable_add_def((ST), (NAME), USE) - -/* Look for a yield stmt or expr under n. Return 1 if found, else 0. - This hack is used to look inside "if 0:" blocks (which are normally - ignored) in case those are the only places a yield occurs (so that this - function is a generator). */ -static int -look_for_yield(node *n) -{ - int i; - - for (i = 0; i < NCH(n); ++i) { - node *kid = CHILD(n, i); +/* All about a_lnotab. - switch (TYPE(kid)) { +c_lnotab is an array of unsigned bytes disguised as a Python string. +It is used to map bytecode offsets to source code line #s (when needed +for tracebacks). - case classdef: - case funcdef: - case lambdef: - /* Stuff in nested functions and classes can't make - the parent a generator. */ - return 0; +The array is conceptually a list of + (bytecode offset increment, line number increment) +pairs. The details are important and delicate, best illustrated by example: - case yield_stmt: - case yield_expr: - return GENERATOR; + byte code offset source code line number + 0 1 + 6 2 + 50 7 + 350 307 + 361 308 - default: - if (look_for_yield(kid)) - return GENERATOR; - } - } - return 0; -} +The first trick is that these numbers aren't stored, only the increments +from one row to the next (this doesn't really work, but it's a start): -static void -symtable_node(struct symtable *st, node *n) -{ - int i; + 0, 1, 6, 1, 44, 5, 300, 300, 11, 1 - loop: - switch (TYPE(n)) { - case funcdef: { - char *func_name; - if (NCH(n) == 6) - symtable_node(st, CHILD(n, 0)); - func_name = STR(RCHILD(n, -4)); - symtable_add_def(st, func_name, DEF_LOCAL); - symtable_default_args(st, RCHILD(n, -3)); - symtable_enter_scope(st, func_name, TYPE(n), n->n_lineno); - symtable_funcdef(st, n); - symtable_exit_scope(st); - break; - } - case lambdef: - if (NCH(n) == 4) - symtable_default_args(st, CHILD(n, 1)); - symtable_enter_scope(st, "lambda", TYPE(n), n->n_lineno); - symtable_funcdef(st, n); - symtable_exit_scope(st); - break; - case classdef: { - char *tmp, *class_name = STR(CHILD(n, 1)); - symtable_add_def(st, class_name, DEF_LOCAL); - if (TYPE(CHILD(n, 2)) == LPAR) { - node *bases = CHILD(n, 3); - int i; - for (i = 0; i < NCH(bases); i += 2) { - symtable_node(st, CHILD(bases, i)); - } - } - symtable_enter_scope(st, class_name, TYPE(n), n->n_lineno); - tmp = st->st_private; - st->st_private = class_name; - symtable_node(st, CHILD(n, NCH(n) - 1)); - st->st_private = tmp; - symtable_exit_scope(st); - break; - } - case if_stmt: - for (i = 0; i + 3 < NCH(n); i += 4) { - if (is_constant_false(NULL, (CHILD(n, i + 1)))) { - if (st->st_cur->ste_generator == 0) - st->st_cur->ste_generator = - look_for_yield(CHILD(n, i+3)); - continue; - } - symtable_node(st, CHILD(n, i + 1)); - symtable_node(st, CHILD(n, i + 3)); - } - if (i + 2 < NCH(n)) - symtable_node(st, CHILD(n, i + 2)); - break; - case global_stmt: - symtable_global(st, n); - break; - case import_stmt: - symtable_import(st, n); - break; - case exec_stmt: { - st->st_cur->ste_optimized |= OPT_EXEC; - symtable_node(st, CHILD(n, 1)); - if (NCH(n) > 2) - symtable_node(st, CHILD(n, 3)); - else { - st->st_cur->ste_optimized |= OPT_BARE_EXEC; - st->st_cur->ste_opt_lineno = n->n_lineno; - } - if (NCH(n) > 4) - symtable_node(st, CHILD(n, 5)); - break; +The second trick is that an unsigned byte can't hold negative values, or +values larger than 255, so (a) there's a deep assumption that byte code +offsets and their corresponding line #s both increase monotonically, and (b) +if at least one column jumps by more than 255 from one row to the next, more +than one pair is written to the table. In case #b, there's no way to know +from looking at the table later how many were written. That's the delicate +part. A user of c_lnotab desiring to find the source line number +corresponding to a bytecode address A should do something like this - } - case assert_stmt: - if (Py_OptimizeFlag) - return; - if (NCH(n) == 2) { - n = CHILD(n, 1); - goto loop; - } else { - symtable_node(st, CHILD(n, 1)); - n = CHILD(n, 3); - goto loop; - } - case except_clause: - if (NCH(n) == 4) - symtable_assign(st, CHILD(n, 3), 0); - if (NCH(n) > 1) { - n = CHILD(n, 1); - goto loop; - } - break; - case del_stmt: - symtable_assign(st, CHILD(n, 1), 0); - break; - case yield_expr: - st->st_cur->ste_generator = 1; - if (NCH(n)==1) - break; - n = CHILD(n, 1); - goto loop; - case expr_stmt: - if (NCH(n) == 1) - n = CHILD(n, 0); - else { - if (TYPE(CHILD(n, 1)) == augassign) { - symtable_assign(st, CHILD(n, 0), 0); - symtable_node(st, CHILD(n, 2)); - break; - } else { - int i; - for (i = 0; i < NCH(n) - 2; i += 2) - symtable_assign(st, CHILD(n, i), 0); - n = CHILD(n, NCH(n) - 1); - } - } - goto loop; - case list_iter: - /* only occurs when there are multiple for loops - in a list comprehension */ - n = CHILD(n, 0); - if (TYPE(n) == list_for) - symtable_list_for(st, n); - else { - REQ(n, list_if); - symtable_node(st, CHILD(n, 1)); - if (NCH(n) == 3) { - n = CHILD(n, 2); - goto loop; - } - } - break; - case for_stmt: - symtable_assign(st, CHILD(n, 1), 0); - for (i = 3; i < NCH(n); ++i) - if (TYPE(CHILD(n, i)) >= single_input) - symtable_node(st, CHILD(n, i)); - break; - case arglist: - if (NCH(n) > 1) - for (i = 0; i < NCH(n); ++i) { - node *ch = CHILD(n, i); - if (TYPE(ch) == argument && NCH(ch) == 2 && - TYPE(CHILD(ch, 1)) == gen_for) { - PyErr_SetString(PyExc_SyntaxError, - "invalid syntax"); - symtable_error(st, n->n_lineno); - return; - } - } - /* The remaining cases fall through to default except in - special circumstances. This requires the individual cases - to be coded with great care, even though they look like - rather innocuous. Each case must double-check TYPE(n). - */ - case decorator: - if (TYPE(n) == decorator) { - /* decorator: '@' dotted_name [ '(' [arglist] ')' ] */ - node *name, *varname; - name = CHILD(n, 1); - REQ(name, dotted_name); - varname = CHILD(name, 0); - REQ(varname, NAME); - symtable_add_use(st, STR(varname)); - } - /* fall through */ - case argument: - if (TYPE(n) == argument && NCH(n) == 3) { - n = CHILD(n, 2); - goto loop; - } - else if (TYPE(n) == argument && NCH(n) == 2 && - TYPE(CHILD(n, 1)) == gen_for) { - symtable_generator_expression(st, n); - break; - } - /* fall through */ - case listmaker: - if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == list_for) { - symtable_list_comprehension(st, n); - break; - } - /* fall through */ - case testlist_gexp: - if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for) { - symtable_generator_expression(st, n); - break; - } - /* fall through */ + lineno = addr = 0 + for addr_incr, line_incr in c_lnotab: + addr += addr_incr + if addr > A: + return lineno + lineno += line_incr - case atom: - if (TYPE(n) == atom) { - if (TYPE(CHILD(n, 0)) == NAME) { - symtable_add_use(st, STR(CHILD(n, 0))); - break; - } - else if (TYPE(CHILD(n,0)) == LPAR) { - n = CHILD(n,1); - goto loop; - } - } - /* fall through */ - default: - /* Walk over every non-token child with a special case - for one child. - */ - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; - } - for (i = 0; i < NCH(n); ++i) - if (TYPE(CHILD(n, i)) >= single_input) - symtable_node(st, CHILD(n, i)); - } -} +In order for this to work, when the addr field increments by more than 255, +the line # increment in each pair generated must be 0 until the remaining addr +increment is < 256. So, in the example above, com_set_lineno should not (as +was actually done until 2.2) expand 300, 300 to 255, 255, 45, 45, but to +255, 0, 45, 255, 0, 45. +*/ -static void -symtable_funcdef(struct symtable *st, node *n) +static int +assemble_lnotab(struct assembler *a, struct instr *i) { - node *body; - - if (TYPE(n) == lambdef) { - if (NCH(n) == 4) - symtable_params(st, CHILD(n, 1)); - } else - symtable_params(st, RCHILD(n, -3)); - body = CHILD(n, NCH(n) - 1); - symtable_node(st, body); -} + int d_bytecode, d_lineno; + int len; + char *lnotab; -/* The next two functions parse the argument tuple. - symtable_default_args() checks for names in the default arguments, - which are references in the defining scope. symtable_params() - parses the parameter names, which are defined in the function's - body. + d_bytecode = a->a_offset - a->a_lineno_off; + d_lineno = i->i_lineno - a->a_lineno; - varargslist: - (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) - | fpdef ['=' test] (',' fpdef ['=' test])* [','] -*/ + assert(d_bytecode >= 0); + assert(d_lineno >= 0); -static void -symtable_default_args(struct symtable *st, node *n) -{ - node *c; - int i; + if (d_lineno == 0) + return 1; - if (TYPE(n) == parameters) { - n = CHILD(n, 1); - if (TYPE(n) == RPAR) - return; - } - REQ(n, varargslist); - for (i = 0; i < NCH(n); i += 2) { - c = CHILD(n, i); - if (TYPE(c) == STAR || TYPE(c) == DOUBLESTAR) { - break; + if (d_bytecode > 255) { + int i, nbytes, ncodes = d_bytecode / 255; + nbytes = a->a_lnotab_off + 2 * ncodes; + len = PyString_GET_SIZE(a->a_lnotab); + if (nbytes >= len) { + if (len * 2 < nbytes) + len = nbytes; + else + len *= 2; + if (_PyString_Resize(&a->a_lnotab, len) < 0) + return 0; + } + lnotab = PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off; + for (i = 0; i < ncodes; i++) { + *lnotab++ = 255; + *lnotab++ = 0; + } + d_bytecode -= ncodes * 255; + a->a_lnotab_off += ncodes * 2; + } + assert(d_bytecode <= 255); + if (d_lineno > 255) { + int i, nbytes, ncodes = d_lineno / 255; + nbytes = a->a_lnotab_off + 2 * ncodes; + len = PyString_GET_SIZE(a->a_lnotab); + if (nbytes >= len) { + if (len * 2 < nbytes) + len = nbytes; + else + len *= 2; + if (_PyString_Resize(&a->a_lnotab, len) < 0) + return 0; } - if (i > 0 && (TYPE(CHILD(n, i - 1)) == EQUAL)) - symtable_node(st, CHILD(n, i)); + lnotab = PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off; + *lnotab++ = 255; + *lnotab++ = d_bytecode; + d_bytecode = 0; + for (i = 1; i < ncodes; i++) { + *lnotab++ = 255; + *lnotab++ = 0; + } + d_lineno -= ncodes * 255; + a->a_lnotab_off += ncodes * 2; } -} -static void -symtable_params(struct symtable *st, node *n) -{ - int i, complex = -1, ext = 0; - node *c = NULL; - - if (TYPE(n) == parameters) { - n = CHILD(n, 1); - if (TYPE(n) == RPAR) - return; - } - REQ(n, varargslist); - for (i = 0; i < NCH(n); i += 2) { - c = CHILD(n, i); - if (TYPE(c) == STAR || TYPE(c) == DOUBLESTAR) { - ext = 1; - break; - } - if (TYPE(c) == test) { - continue; - } - if (TYPE(CHILD(c, 0)) == NAME) - symtable_add_def(st, STR(CHILD(c, 0)), DEF_PARAM); - else { - char nbuf[30]; - PyOS_snprintf(nbuf, sizeof(nbuf), ".%d", i); - symtable_add_def(st, nbuf, DEF_PARAM); - complex = i; - } + len = PyString_GET_SIZE(a->a_lnotab); + if (a->a_lnotab_off + 2 >= len) { + if (_PyString_Resize(&a->a_lnotab, len * 2) < 0) + return 0; } - if (ext) { - c = CHILD(n, i); - if (TYPE(c) == STAR) { - i++; - symtable_add_def(st, STR(CHILD(n, i)), - DEF_PARAM | DEF_STAR); - i += 2; - if (i >= NCH(n)) - c = NULL; - else - c = CHILD(n, i); - } - if (c && TYPE(c) == DOUBLESTAR) { - i++; - symtable_add_def(st, STR(CHILD(n, i)), - DEF_PARAM | DEF_DOUBLESTAR); - } + lnotab = PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off; + + a->a_lnotab_off += 2; + if (d_bytecode) { + *lnotab++ = d_bytecode; + *lnotab++ = d_lineno; } - if (complex >= 0) { - int j; - for (j = 0; j <= complex; j++) { - c = CHILD(n, j); - if (TYPE(c) == COMMA) - c = CHILD(n, ++j); - else if (TYPE(c) == EQUAL) - c = CHILD(n, j += 3); - if (TYPE(CHILD(c, 0)) == LPAR) - symtable_params_fplist(st, CHILD(c, 1)); - } + else { /* First line of a block; def stmt, etc. */ + *lnotab++ = 0; + *lnotab++ = d_lineno; } + a->a_lineno = i->i_lineno; + a->a_lineno_off = a->a_offset; + return 1; } -static void -symtable_params_fplist(struct symtable *st, node *n) +/* assemble_emit() + Extend the bytecode with a new instruction. + Update lnotab if necessary. +*/ + +static int +assemble_emit(struct assembler *a, struct instr *i) { - int i; - node *c; - - REQ(n, fplist); - for (i = 0; i < NCH(n); i += 2) { - c = CHILD(n, i); - REQ(c, fpdef); - if (NCH(c) == 1) - symtable_add_def(st, STR(CHILD(c, 0)), - DEF_PARAM | DEF_INTUPLE); + int arg = 0, size = 0, ext = i->i_oparg >> 16; + int len = PyString_GET_SIZE(a->a_bytecode); + char *code; + + if (!i->i_hasarg) + size = 1; + else { + if (ext) + size = 6; else - symtable_params_fplist(st, CHILD(c, 1)); + size = 3; + arg = i->i_oparg; } - + if (i->i_lineno && !assemble_lnotab(a, i)) + return 0; + if (a->a_offset + size >= len) { + if (_PyString_Resize(&a->a_bytecode, len * 2) < 0) + return 0; + } + code = PyString_AS_STRING(a->a_bytecode) + a->a_offset; + a->a_offset += size; + if (ext > 0) { + *code++ = (char)EXTENDED_ARG; + *code++ = ext & 0xff; + *code++ = ext >> 8; + arg &= 0xffff; + } + *code++ = i->i_opcode; + if (size == 1) + return 1; + *code++ = arg & 0xff; + *code++ = arg >> 8; + return 1; } -static void -symtable_global(struct symtable *st, node *n) +static int +assemble_jump_offsets(struct assembler *a, struct compiler *c) { + basicblock *b; + int bsize, totsize = 0; int i; - /* XXX It might be helpful to warn about module-level global - statements, but it's hard to tell the difference between - module-level and a string passed to exec. - */ - - for (i = 1; i < NCH(n); i += 2) { - char *name = STR(CHILD(n, i)); - int flags; - - flags = symtable_lookup(st, name); - if (flags < 0) - continue; - if (flags && flags != DEF_GLOBAL) { - char buf[500]; - if (flags & DEF_PARAM) { - PyErr_Format(PyExc_SyntaxError, PARAM_GLOBAL, - name); - symtable_error(st, 0); - return; - } - else { - if (flags & DEF_LOCAL) - PyOS_snprintf(buf, sizeof(buf), - GLOBAL_AFTER_ASSIGN, - name); - else - PyOS_snprintf(buf, sizeof(buf), - GLOBAL_AFTER_USE, name); - symtable_warn(st, buf); + /* Compute the size of each block and fixup jump args. + Replace block pointer with position in bytecode. */ + for (i = a->a_nblocks - 1; i >= 0; i--) { + basicblock *b = a->a_postorder[i]; + bsize = blocksize(b); + b->b_offset = totsize; + totsize += bsize; + } + for (b = c->u->u_blocks; b != NULL; b = b->b_list) { + bsize = b->b_offset; + for (i = 0; i < b->b_iused; i++) { + struct instr *instr = &b->b_instr[i]; + /* Relative jumps are computed relative to + the instruction pointer after fetching + the jump instruction. + */ + bsize += instrsize(instr); + if (instr->i_jabs) + instr->i_oparg = instr->i_target->b_offset; + else if (instr->i_jrel) { + int delta = instr->i_target->b_offset - bsize; + instr->i_oparg = delta; } } - symtable_add_def(st, name, DEF_GLOBAL); } + return 1; } -static void -symtable_list_comprehension(struct symtable *st, node *n) -{ - /* listmaker: test list_for */ - char tmpname[30]; - - REQ(n, listmaker); - PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", - ++st->st_cur->ste_tmpname); - symtable_add_def(st, tmpname, DEF_LOCAL); - symtable_list_for(st, CHILD(n, 1)); - symtable_node(st, CHILD(n, 0)); - --st->st_cur->ste_tmpname; -} - -static void -symtable_generator_expression(struct symtable *st, node *n) +static PyObject * +dict_keys_inorder(PyObject *dict, int offset) { - /* testlist_gexp: test gen_for */ - REQ(CHILD(n, 0), test); - REQ(CHILD(n, 1), gen_for); - - symtable_enter_scope(st, "<genexpr>", TYPE(n), n->n_lineno); - st->st_cur->ste_generator = GENERATOR_EXPRESSION; - - symtable_add_def(st, "[outmost-iterable]", DEF_PARAM); - - symtable_gen_for(st, CHILD(n, 1), 1); - symtable_node(st, CHILD(n, 0)); - symtable_exit_scope(st); - - /* for outmost iterable precomputation */ - symtable_node(st, CHILD(CHILD(n, 1), 3)); -} + PyObject *tuple, *k, *v; + int i, pos = 0, size = PyDict_Size(dict); -static void -symtable_list_for(struct symtable *st, node *n) -{ - REQ(n, list_for); - /* list_for: for v in expr [list_iter] */ - symtable_assign(st, CHILD(n, 1), 0); - symtable_node(st, CHILD(n, 3)); - if (NCH(n) == 5) - symtable_node(st, CHILD(n, 4)); + tuple = PyTuple_New(size); + if (tuple == NULL) + return NULL; + while (PyDict_Next(dict, &pos, &k, &v)) { + i = PyInt_AS_LONG(v); + k = PyTuple_GET_ITEM(k, 0); + Py_INCREF(k); + assert((i - offset) < size); + assert((i - offset) >= 0); + PyTuple_SET_ITEM(tuple, i - offset, k); + } + return tuple; } -static void -symtable_gen_for(struct symtable *st, node *n, int is_outmost) -{ - REQ(n, gen_for); +static int +compute_code_flags(struct compiler *c) +{ + PySTEntryObject *ste = c->u->u_ste; + int flags = 0, n; + if (ste->ste_type != ModuleBlock) + flags |= CO_NEWLOCALS; + if (ste->ste_type == FunctionBlock) { + if (!ste->ste_unoptimized) + flags |= CO_OPTIMIZED; + if (ste->ste_nested) + flags |= CO_NESTED; + if (ste->ste_generator) + flags |= CO_GENERATOR; + } + if (ste->ste_varargs) + flags |= CO_VARARGS; + if (ste->ste_varkeywords) + flags |= CO_VARKEYWORDS; + if (ste->ste_generator) + flags |= CO_GENERATOR; + if (c->c_flags->cf_flags & CO_FUTURE_DIVISION) + flags |= CO_FUTURE_DIVISION; + n = PyDict_Size(c->u->u_freevars); + if (n < 0) + return -1; + if (n == 0) { + n = PyDict_Size(c->u->u_cellvars); + if (n < 0) + return -1; + if (n == 0) { + flags |= CO_NOFREE; + } + } - /* gen_for: for v in test [gen_iter] */ - symtable_assign(st, CHILD(n, 1), 0); - if (is_outmost) - symtable_add_use(st, "[outmost-iterable]"); - else - symtable_node(st, CHILD(n, 3)); + return flags; +} - if (NCH(n) == 5) - symtable_gen_iter(st, CHILD(n, 4)); +static PyCodeObject * +makecode(struct compiler *c, struct assembler *a) +{ + PyObject *tmp; + PyCodeObject *co = NULL; + PyObject *consts = NULL; + PyObject *names = NULL; + PyObject *varnames = NULL; + PyObject *filename = NULL; + PyObject *name = NULL; + PyObject *freevars = NULL; + PyObject *cellvars = NULL; + PyObject *bytecode = NULL; + int nlocals, flags; + + tmp = dict_keys_inorder(c->u->u_consts, 0); + if (!tmp) + goto error; + consts = PySequence_List(tmp); /* optimize_code requires a list */ + Py_DECREF(tmp); + + names = dict_keys_inorder(c->u->u_names, 0); + varnames = dict_keys_inorder(c->u->u_varnames, 0); + if (!consts || !names || !varnames) + goto error; + + cellvars = dict_keys_inorder(c->u->u_cellvars, 0); + if (!cellvars) + goto error; + freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars)); + if (!freevars) + goto error; + filename = PyString_FromString(c->c_filename); + if (!filename) + goto error; + + nlocals = PyDict_Size(c->u->u_varnames); + flags = compute_code_flags(c); + if (flags < 0) + goto error; + + bytecode = optimize_code(a->a_bytecode, consts, names, a->a_lnotab); + if (!bytecode) + goto error; + + tmp = PyList_AsTuple(consts); /* PyCode_New requires a tuple */ + if (!tmp) + goto error; + Py_DECREF(consts); + consts = tmp; + + co = PyCode_New(c->u->u_argcount, nlocals, stackdepth(c), flags, + bytecode, consts, names, varnames, + freevars, cellvars, + filename, c->u->u_name, + c->u->u_firstlineno, + a->a_lnotab); + error: + Py_XDECREF(consts); + Py_XDECREF(names); + Py_XDECREF(varnames); + Py_XDECREF(filename); + Py_XDECREF(name); + Py_XDECREF(freevars); + Py_XDECREF(cellvars); + Py_XDECREF(bytecode); + return co; } -static void -symtable_gen_iter(struct symtable *st, node *n) +static PyCodeObject * +assemble(struct compiler *c, int addNone) { - REQ(n, gen_iter); - - n = CHILD(n, 0); - if (TYPE(n) == gen_for) - symtable_gen_for(st, n, 0); - else { - REQ(n, gen_if); - symtable_node(st, CHILD(n, 1)); + basicblock *b, *entryblock; + struct assembler a; + int i, j, nblocks; + PyCodeObject *co = NULL; - if (NCH(n) == 3) - symtable_gen_iter(st, CHILD(n, 2)); + /* Make sure every block that falls off the end returns None. + XXX NEXT_BLOCK() isn't quite right, because if the last + block ends with a jump or return b_next shouldn't set. + */ + if (!c->u->u_curblock->b_return) { + NEXT_BLOCK(c); + if (addNone) + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, RETURN_VALUE); } -} -static void -symtable_import(struct symtable *st, node *n) -{ - node *nn; - int i; - /* import_stmt: import_name | import_from */ - n = CHILD(n, 0); - if (TYPE(n) == import_from) { - /* import_from: 'from' dotted_name 'import' ('*' | - | '(' import_as_names ')' | import_as_names) */ - node *dotname = CHILD(n, 1); - REQ(dotname, dotted_name); - if (strcmp(STR(CHILD(dotname, 0)), "__future__") == 0) { - /* check for bogus imports */ - if (n->n_lineno >= st->st_future->ff_last_lineno) { - PyErr_SetString(PyExc_SyntaxError, - LATE_FUTURE); - symtable_error(st, n->n_lineno); - return; - } - } - nn = CHILD(n, 3 + (TYPE(CHILD(n, 3)) == LPAR)); - if (TYPE(nn) == STAR) { - if (st->st_cur->ste_type != TYPE_MODULE) { - if (symtable_warn(st, - "import * only allowed at module level") < 0) - return; - } - st->st_cur->ste_optimized |= OPT_IMPORT_STAR; - st->st_cur->ste_opt_lineno = n->n_lineno; - } else { - REQ(nn, import_as_names); - for (i = 0; i < NCH(nn); i += 2) { - node *c = CHILD(nn, i); - if (NCH(c) > 1) /* import as */ - symtable_assign(st, CHILD(c, 2), - DEF_IMPORT); - else - symtable_assign(st, CHILD(c, 0), - DEF_IMPORT); - } - } - } else { - /* 'import' dotted_as_names */ - nn = CHILD(n, 1); - REQ(nn, dotted_as_names); - for (i = 0; i < NCH(nn); i += 2) - symtable_assign(st, CHILD(nn, i), DEF_IMPORT); + nblocks = 0; + entryblock = NULL; + for (b = c->u->u_blocks; b != NULL; b = b->b_list) { + nblocks++; + entryblock = b; } -} - -/* The third argument to symatble_assign() is a flag to be passed to - symtable_add_def() if it is eventually called. The flag is useful - to specify the particular type of assignment that should be - recorded, e.g. an assignment caused by import. - */ -static void -symtable_assign(struct symtable *st, node *n, int def_flag) -{ - node *tmp; - int i; + if (!assemble_init(&a, nblocks, c->u->u_firstlineno)) + goto error; + dfs(c, entryblock, &a); - loop: - switch (TYPE(n)) { - case lambdef: - /* invalid assignment, e.g. lambda x:x=2. The next - pass will catch this error. */ - return; - case power: - if (NCH(n) > 2) { - for (i = 2; i < NCH(n); ++i) - if (TYPE(CHILD(n, i)) != DOUBLESTAR) - symtable_node(st, CHILD(n, i)); - } - if (NCH(n) > 1) { - symtable_node(st, CHILD(n, 0)); - symtable_node(st, CHILD(n, 1)); - } else { - n = CHILD(n, 0); - goto loop; - } - return; - case listmaker: - if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == list_for) { - /* XXX This is an error, but the next pass - will catch it. */ - return; - } else { - for (i = 0; i < NCH(n); i += 2) - symtable_assign(st, CHILD(n, i), def_flag); - } - return; - case testlist_gexp: - if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for) { - /* XXX This is an error, but the next pass - will catch it. */ - return; - } else { - for (i = 0; i < NCH(n); i += 2) - symtable_assign(st, CHILD(n, i), def_flag); - } - return; + /* Can't modify the bytecode after computing jump offsets. */ + if (!assemble_jump_offsets(&a, c)) + goto error; - case exprlist: - case testlist: - case testlist1: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; - } - else { - int i; - for (i = 0; i < NCH(n); i += 2) - symtable_assign(st, CHILD(n, i), def_flag); - return; - } - case atom: - tmp = CHILD(n, 0); - if (TYPE(tmp) == LPAR || TYPE(tmp) == LSQB) { - n = CHILD(n, 1); - goto loop; - } else if (TYPE(tmp) == NAME) { - if (strcmp(STR(tmp), "__debug__") == 0) { - PyErr_SetString(PyExc_SyntaxError, - ASSIGN_DEBUG); - symtable_error(st, n->n_lineno); - return; - } - symtable_add_def(st, STR(tmp), DEF_LOCAL | def_flag); - } - return; + /* Emit code in reverse postorder from dfs. */ + for (i = a.a_nblocks - 1; i >= 0; i--) { + basicblock *b = a.a_postorder[i]; + for (j = 0; j < b->b_iused; j++) + if (!assemble_emit(&a, &b->b_instr[j])) + goto error; + } - case yield_expr: - st->st_cur->ste_generator = 1; - if (NCH(n)==2) { - n = CHILD(n, 1); - goto loop; - } - return; + if (_PyString_Resize(&a.a_lnotab, a.a_lnotab_off) < 0) + goto error; + if (_PyString_Resize(&a.a_bytecode, a.a_offset) < 0) + goto error; - case dotted_as_name: - if (NCH(n) == 3) - symtable_add_def(st, STR(CHILD(n, 2)), - DEF_LOCAL | def_flag); - else - symtable_add_def(st, - STR(CHILD(CHILD(n, - 0), 0)), - DEF_LOCAL | def_flag); - return; - case dotted_name: - symtable_add_def(st, STR(CHILD(n, 0)), DEF_LOCAL | def_flag); - return; - case NAME: - symtable_add_def(st, STR(n), DEF_LOCAL | def_flag); - return; - default: - if (NCH(n) == 0) - return; - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; - } - /* Should only occur for errors like x + 1 = 1, - which will be caught in the next pass. */ - for (i = 0; i < NCH(n); ++i) - if (TYPE(CHILD(n, i)) >= single_input) - symtable_assign(st, CHILD(n, i), def_flag); - } + co = makecode(c, &a); + error: + assemble_free(&a); + return co; } diff --git a/Python/future.c b/Python/future.c index 95d6a5c..a0cfeac 100644 --- a/Python/future.c +++ b/Python/future.c @@ -1,37 +1,30 @@ #include "Python.h" +#include "Python-ast.h" #include "node.h" #include "token.h" #include "graminit.h" +#include "code.h" #include "compile.h" #include "symtable.h" #define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined" #define FUTURE_IMPORT_STAR "future statement does not support import *" -/* FUTURE_POSSIBLE() is provided to accomodate doc strings, which is - the only statement that can occur before a future statement. -*/ -#define FUTURE_POSSIBLE(FF) ((FF)->ff_last_lineno == -1) - static int -future_check_features(PyFutureFeatures *ff, node *n, const char *filename) +future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename) { int i; - char *feature; - node *ch, *nn; + const char *feature; + asdl_seq *names; - REQ(n, import_from); - nn = CHILD(n, 3 + (TYPE(CHILD(n, 3)) == LPAR)); - if (TYPE(nn) == STAR) { - PyErr_SetString(PyExc_SyntaxError, FUTURE_IMPORT_STAR); - PyErr_SyntaxLocation(filename, nn->n_lineno); - return -1; - } - REQ(nn, import_as_names); - for (i = 0; i < NCH(nn); i += 2) { - ch = CHILD(nn, i); - REQ(ch, import_as_name); - feature = STR(CHILD(ch, 0)); + assert(s->kind == ImportFrom_kind); + + names = s->v.ImportFrom.names; + for (i = 0; i < asdl_seq_LEN(names); i++) { + alias_ty name = asdl_seq_GET(names, i); + feature = PyString_AsString(name->name); + if (!feature) + return 0; if (strcmp(feature, FUTURE_NESTED_SCOPES) == 0) { continue; } else if (strcmp(feature, FUTURE_GENERATORS) == 0) { @@ -41,218 +34,97 @@ future_check_features(PyFutureFeatures *ff, node *n, const char *filename) } else if (strcmp(feature, "braces") == 0) { PyErr_SetString(PyExc_SyntaxError, "not a chance"); - PyErr_SyntaxLocation(filename, CHILD(ch, 0)->n_lineno); - return -1; + PyErr_SyntaxLocation(filename, s->lineno); + return 0; } else { PyErr_Format(PyExc_SyntaxError, UNDEFINED_FUTURE_FEATURE, feature); - PyErr_SyntaxLocation(filename, CHILD(ch, 0)->n_lineno); - return -1; + PyErr_SyntaxLocation(filename, s->lineno); + return 0; } } - return 0; + return 1; } -static void -future_error(node *n, const char *filename) +int +future_parse(PyFutureFeatures *ff, mod_ty mod, const char *filename) { - PyErr_SetString(PyExc_SyntaxError, - "from __future__ imports must occur at the " - "beginning of the file"); - PyErr_SyntaxLocation(filename, n->n_lineno); -} - -/* Relevant portions of the grammar: - -single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE -file_input: (NEWLINE | stmt)* ENDMARKER -stmt: simple_stmt | compound_stmt -simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt - | import_stmt | global_stmt | exec_stmt | assert_stmt -import_stmt: 'import' dotted_as_name (',' dotted_as_name)* - | 'from' dotted_name 'import' ('*' | import_as_name (',' import_as_name)*) -import_as_name: NAME [NAME NAME] -dotted_as_name: dotted_name [NAME NAME] -dotted_name: NAME ('.' NAME)* -*/ - -/* future_parse() finds future statements at the beginnning of a - module. The function calls itself recursively, rather than - factoring out logic for different kinds of statements into - different routines. - - Return values: - -1 indicates an error occurred, e.g. unknown feature name - 0 indicates no feature was found - 1 indicates a feature was found -*/ + int i, found_docstring = 0, done = 0, prev_line = 0; -static int -future_parse(PyFutureFeatures *ff, node *n, const char *filename) -{ - int i, r; - loop: - switch (TYPE(n)) { + static PyObject *future; + if (!future) { + future = PyString_InternFromString("__future__"); + if (!future) + return 0; + } - case single_input: - if (TYPE(CHILD(n, 0)) == simple_stmt) { - n = CHILD(n, 0); - goto loop; - } - return 0; + if (!(mod->kind == Module_kind || mod->kind == Interactive_kind)) + return 1; - case file_input: - /* Check each statement in the file, starting with the - first, and continuing until the first statement - that isn't a future statement. + /* A subsequent pass will detect future imports that don't + appear at the beginning of the file. There's one case, + however, that is easier to handl here: A series of imports + joined by semi-colons, where the first import is a future + statement but some subsequent import has the future form + but is preceded by a regular import. + */ + + + for (i = 0; i < asdl_seq_LEN(mod->v.Module.body); i++) { + stmt_ty s = asdl_seq_GET(mod->v.Module.body, i); + + if (done && s->lineno > prev_line) + return 1; + prev_line = s->lineno; + + /* The tests below will return from this function unless it is + still possible to find a future statement. The only things + that can precede a future statement are another future + statement and a doc string. */ - for (i = 0; i < NCH(n); i++) { - node *ch = CHILD(n, i); - if (TYPE(ch) == stmt) { - r = future_parse(ff, ch, filename); - /* Need to check both conditions below - to accomodate doc strings, which - causes r < 0. - */ - if (r < 1 && !FUTURE_POSSIBLE(ff)) - return r; - } - } - return 0; - - case simple_stmt: - if (NCH(n) == 2) { - REQ(CHILD(n, 0), small_stmt); - n = CHILD(n, 0); - goto loop; - } else { - /* Deal with the special case of a series of - small statements on a single line. If a - future statement follows some other - statement, the SyntaxError is raised here. - In all other cases, the symtable pass - raises the exception. - */ - int found = 0, end_of_future = 0; - for (i = 0; i < NCH(n); i += 2) { - if (TYPE(CHILD(n, i)) == small_stmt) { - r = future_parse(ff, CHILD(n, i), - filename); - if (r < 1) - end_of_future = 1; - else { - found = 1; - if (end_of_future) { - future_error(n, - filename); - return -1; - } - } + if (s->kind == ImportFrom_kind) { + if (s->v.ImportFrom.module == future) { + if (done) { + PyErr_SetString(PyExc_SyntaxError, + ERR_LATE_FUTURE); + PyErr_SyntaxLocation(filename, + s->lineno); + return 0; } + if (!future_check_features(ff, s, filename)) + return 0; + ff->ff_lineno = s->lineno; } - - /* If we found one and only one, then the - current lineno is legal. - */ - if (found) - ff->ff_last_lineno = n->n_lineno + 1; else - ff->ff_last_lineno = n->n_lineno; - - if (end_of_future && found) - return 1; - else - return 0; - } - - case stmt: - if (TYPE(CHILD(n, 0)) == simple_stmt) { - n = CHILD(n, 0); - goto loop; - } else if (TYPE(CHILD(n, 0)) == expr_stmt) { - n = CHILD(n, 0); - goto loop; - } else { - REQ(CHILD(n, 0), compound_stmt); - ff->ff_last_lineno = n->n_lineno; - return 0; - } - - case small_stmt: - n = CHILD(n, 0); - goto loop; - - case import_stmt: { - node *name; - - n = CHILD(n, 0); - if (TYPE(n) != import_from) { - ff->ff_last_lineno = n->n_lineno; - return 0; + done = 1; } - name = CHILD(n, 1); - if (strcmp(STR(CHILD(name, 0)), "__future__") != 0) - return 0; - if (future_check_features(ff, n, filename) < 0) - return -1; - ff->ff_last_lineno = n->n_lineno + 1; - return 1; - } - - /* The cases below -- all of them! -- are necessary to find - and skip doc strings. */ - case expr_stmt: - case testlist: - case test: - case and_test: - case not_test: - case comparison: - case expr: - case xor_expr: - case and_expr: - case shift_expr: - case arith_expr: - case term: - case factor: - case power: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; - } - ff->ff_last_lineno = n->n_lineno; - break; - - case atom: - if (TYPE(CHILD(n, 0)) == STRING - && ff->ff_found_docstring == 0) { - ff->ff_found_docstring = 1; - return 0; + else if (s->kind == Expr_kind && !found_docstring) { + expr_ty e = s->v.Expr.value; + if (e->kind != Str_kind) + done = 1; + else + found_docstring = 1; } - ff->ff_last_lineno = n->n_lineno; - return 0; - - default: - ff->ff_last_lineno = n->n_lineno; - return 0; + else + done = 1; } - return 0; + return 1; } + PyFutureFeatures * -PyNode_Future(node *n, const char *filename) +PyFuture_FromAST(mod_ty mod, const char *filename) { PyFutureFeatures *ff; ff = (PyFutureFeatures *)PyMem_Malloc(sizeof(PyFutureFeatures)); if (ff == NULL) return NULL; - ff->ff_found_docstring = 0; - ff->ff_last_lineno = -1; ff->ff_features = 0; + ff->ff_lineno = -1; - if (future_parse(ff, n, filename) < 0) { + if (!future_parse(ff, mod, filename)) { PyMem_Free((void *)ff); return NULL; } diff --git a/Python/import.c b/Python/import.c index 35de13e..dcbca38 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3,10 +3,11 @@ #include "Python.h" -#include "node.h" -#include "token.h" +#include "Python-ast.h" +#include "pythonrun.h" #include "errcode.h" #include "marshal.h" +#include "code.h" #include "compile.h" #include "eval.h" #include "osdefs.h" @@ -766,17 +767,17 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) /* Parse a source file and return the corresponding code object */ static PyCodeObject * -parse_source_module(char *pathname, FILE *fp) +parse_source_module(const char *pathname, FILE *fp) { - PyCodeObject *co; - node *n; - - n = PyParser_SimpleParseFile(fp, pathname, Py_file_input); - if (n == NULL) - return NULL; - co = PyNode_Compile(n, pathname); - PyNode_Free(n); + PyCodeObject *co = NULL; + mod_ty mod; + mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0, + NULL); + if (mod) { + co = PyAST_Compile(mod, pathname, NULL); + free_mod(mod); + } return co; } diff --git a/Python/marshal.c b/Python/marshal.c index 20d637d..4114c8e 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -6,6 +6,7 @@ #include "Python.h" #include "longintrepr.h" +#include "code.h" #include "compile.h" #include "marshal.h" diff --git a/Python/pythonrun.c b/Python/pythonrun.c index e007f98..ad837d2 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -3,13 +3,16 @@ #include "Python.h" +#include "Python-ast.h" #include "grammar.h" #include "node.h" #include "token.h" #include "parsetok.h" #include "errcode.h" +#include "code.h" #include "compile.h" #include "symtable.h" +#include "ast.h" #include "eval.h" #include "marshal.h" @@ -32,9 +35,9 @@ extern grammar _PyParser_Grammar; /* From graminit.c */ /* Forward */ static void initmain(void); static void initsite(void); -static PyObject *run_err_node(node *, const char *, PyObject *, PyObject *, +static PyObject *run_err_mod(mod_ty, const char *, PyObject *, PyObject *, PyCompilerFlags *); -static PyObject *run_node(node *, const char *, PyObject *, PyObject *, +static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *, PyCompilerFlags *); static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *, PyCompilerFlags *); @@ -634,25 +637,7 @@ initsite(void) /* Parse input from a file and execute it */ int -PyRun_AnyFile(FILE *fp, const char *filename) -{ - return PyRun_AnyFileExFlags(fp, filename, 0, NULL); -} - -int -PyRun_AnyFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) -{ - return PyRun_AnyFileExFlags(fp, filename, 0, flags); -} - -int -PyRun_AnyFileEx(FILE *fp, const char *filename, int closeit) -{ - return PyRun_AnyFileExFlags(fp, filename, closeit, NULL); -} - -int -PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, +PyRun_AnyFileExFlags(FILE *fp, char *filename, int closeit, PyCompilerFlags *flags) { if (filename == NULL) @@ -668,12 +653,6 @@ PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, } int -PyRun_InteractiveLoop(FILE *fp, const char *filename) -{ - return PyRun_InteractiveLoopFlags(fp, filename, NULL); -} - -int PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) { PyObject *v; @@ -708,12 +687,6 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag } } -int -PyRun_InteractiveOne(FILE *fp, const char *filename) -{ - return PyRun_InteractiveOneFlags(fp, filename, NULL); -} - /* compute parser flags based on compiler flags */ #define PARSER_FLAGS(flags) \ (((flags) && (flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \ @@ -723,9 +696,9 @@ int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) { PyObject *m, *d, *v, *w; - node *n; - perrdetail err; + mod_ty mod; char *ps1 = "", *ps2 = ""; + int errcode = 0; v = PySys_GetObject("ps1"); if (v != NULL) { @@ -743,26 +716,25 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags else if (PyString_Check(w)) ps2 = PyString_AsString(w); } - n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, - Py_single_input, ps1, ps2, &err, - PARSER_FLAGS(flags)); + mod = PyParser_ASTFromFile(fp, filename, + Py_single_input, ps1, ps2, + flags, &errcode); Py_XDECREF(v); Py_XDECREF(w); - if (n == NULL) { - if (err.error == E_EOF) { - if (err.text) - PyMem_DEL(err.text); + if (mod == NULL) { + if (errcode == E_EOF) { + PyErr_Clear(); return E_EOF; } - err_input(&err); PyErr_Print(); - return err.error; + return -1; } m = PyImport_AddModule("__main__"); if (m == NULL) return -1; d = PyModule_GetDict(m); - v = run_node(n, filename, d, d, flags); + v = run_mod(mod, filename, d, d, flags); + free_mod(mod); if (v == NULL) { PyErr_Print(); return -1; @@ -773,12 +745,6 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags return 0; } -int -PyRun_SimpleFile(FILE *fp, const char *filename) -{ - return PyRun_SimpleFileEx(fp, filename, 0); -} - /* Check whether a file maybe a pyc file: Look at the extension, the file type, and, if we may close it, at the first few bytes. */ @@ -820,12 +786,6 @@ maybe_pyc_file(FILE *fp, const char* filename, const char* ext, int closeit) } int -PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit) -{ - return PyRun_SimpleFileExFlags(fp, filename, closeit, NULL); -} - -int PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) { @@ -874,12 +834,6 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, } int -PyRun_SimpleString(const char *command) -{ - return PyRun_SimpleStringFlags(command, NULL); -} - -int PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) { PyObject *m, *d, *v; @@ -1054,6 +1008,8 @@ PyErr_PrintEx(int set_sys_last_vars) handle_system_exit(); } PyErr_Fetch(&exception, &v, &tb); + if (exception == NULL) + return; PyErr_NormalizeException(&exception, &v, &tb); if (exception == NULL) return; @@ -1195,74 +1151,48 @@ void PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) } PyObject * -PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) -{ - return run_err_node(PyParser_SimpleParseString(str, start), - "<string>", globals, locals, NULL); -} - -PyObject * -PyRun_File(FILE *fp, const char *filename, int start, PyObject *globals, - PyObject *locals) +PyRun_StringFlags(const char *str, int start, PyObject *globals, + PyObject *locals, PyCompilerFlags *flags) { - return PyRun_FileEx(fp, filename, start, globals, locals, 0); -} - -PyObject * -PyRun_FileEx(FILE *fp, const char *filename, int start, PyObject *globals, - PyObject *locals, int closeit) -{ - node *n = PyParser_SimpleParseFile(fp, filename, start); - if (closeit) - fclose(fp); - return run_err_node(n, filename, globals, locals, NULL); -} - -PyObject * -PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals, - PyCompilerFlags *flags) -{ - return run_err_node(PyParser_SimpleParseStringFlags( - str, start, PARSER_FLAGS(flags)), - "<string>", globals, locals, flags); -} - -PyObject * -PyRun_FileFlags(FILE *fp, const char *filename, int start, PyObject *globals, - PyObject *locals, PyCompilerFlags *flags) -{ - return PyRun_FileExFlags(fp, filename, start, globals, locals, 0, - flags); + PyObject *ret; + mod_ty mod = PyParser_ASTFromString(str, "<string>", start, flags); + ret = run_err_mod(mod, "<string>", globals, locals, flags); + free_mod(mod); + return ret; } PyObject * PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) { - node *n = PyParser_SimpleParseFileFlags(fp, filename, start, - PARSER_FLAGS(flags)); + PyObject *ret; + mod_ty mod = PyParser_ASTFromFile(fp, filename, start, 0, 0, + flags, NULL); + if (mod == NULL) + return NULL; if (closeit) fclose(fp); - return run_err_node(n, filename, globals, locals, flags); + ret = run_err_mod(mod, filename, globals, locals, flags); + free_mod(mod); + return ret; } static PyObject * -run_err_node(node *n, const char *filename, PyObject *globals, PyObject *locals, - PyCompilerFlags *flags) +run_err_mod(mod_ty mod, const char *filename, PyObject *globals, + PyObject *locals, PyCompilerFlags *flags) { - if (n == NULL) + if (mod == NULL) return NULL; - return run_node(n, filename, globals, locals, flags); + return run_mod(mod, filename, globals, locals, flags); } static PyObject * -run_node(node *n, const char *filename, PyObject *globals, PyObject *locals, +run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) { PyCodeObject *co; PyObject *v; - co = PyNode_CompileFlags(n, filename, flags); - PyNode_Free(n); + co = PyAST_Compile(mod, filename, flags); if (co == NULL) return NULL; v = PyEval_EvalCode(co, globals, locals); @@ -1271,8 +1201,8 @@ run_node(node *n, const char *filename, PyObject *globals, PyObject *locals, } static PyObject * -run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals, - PyCompilerFlags *flags) +run_pyc_file(FILE *fp, const char *filename, PyObject *globals, + PyObject *locals, PyCompilerFlags *flags) { PyCodeObject *co; PyObject *v; @@ -1303,41 +1233,77 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals } PyObject * -Py_CompileString(const char *str, const char *filename, int start) -{ - return Py_CompileStringFlags(str, filename, start, NULL); -} - -PyObject * Py_CompileStringFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags) { - node *n; + mod_ty mod; PyCodeObject *co; - - n = PyParser_SimpleParseStringFlagsFilename(str, filename, start, - PARSER_FLAGS(flags)); - if (n == NULL) + mod = PyParser_ASTFromString(str, filename, start, flags); + if (mod == NULL) return NULL; - co = PyNode_CompileFlags(n, filename, flags); - PyNode_Free(n); + co = PyAST_Compile(mod, filename, flags); + free_mod(mod); return (PyObject *)co; } struct symtable * Py_SymtableString(const char *str, const char *filename, int start) { - node *n; + mod_ty mod; struct symtable *st; - n = PyParser_SimpleParseStringFlagsFilename(str, filename, - start, 0); - if (n == NULL) + + mod = PyParser_ASTFromString(str, filename, start, NULL); + if (mod == NULL) return NULL; - st = PyNode_CompileSymtable(n, filename); - PyNode_Free(n); + st = PySymtable_Build(mod, filename, 0); + free_mod(mod); return st; } +/* Preferred access to parser is through AST. */ +mod_ty +PyParser_ASTFromString(const char *s, const char *filename, int start, + PyCompilerFlags *flags) +{ + node *n; + mod_ty mod; + perrdetail err; + n = PyParser_ParseStringFlagsFilename(s, filename, &_PyParser_Grammar, + start, &err, + PARSER_FLAGS(flags)); + if (n) { + mod = PyAST_FromNode(n, flags, filename); + PyNode_Free(n); + return mod; + } + else { + err_input(&err); + return NULL; + } +} + +mod_ty +PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, + char *ps2, PyCompilerFlags *flags, int *errcode) +{ + node *n; + mod_ty mod; + perrdetail err; + n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start, + ps1, ps2, &err, PARSER_FLAGS(flags)); + if (n) { + mod = PyAST_FromNode(n, flags, filename); + PyNode_Free(n); + return mod; + } + else { + err_input(&err); + if (errcode) + *errcode = err.error; + return NULL; + } +} + /* Simplified interface to parsefile -- return node or set exception */ node * @@ -1349,15 +1315,10 @@ PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int fla (char *)0, (char *)0, &err, flags); if (n == NULL) err_input(&err); + return n; } -node * -PyParser_SimpleParseFile(FILE *fp, const char *filename, int start) -{ - return PyParser_SimpleParseFileFlags(fp, filename, start, 0); -} - /* Simplified interface to parsestring -- return node or set exception */ node * @@ -1373,12 +1334,6 @@ PyParser_SimpleParseStringFlags(const char *str, int start, int flags) } node * -PyParser_SimpleParseString(const char *str, int start) -{ - return PyParser_SimpleParseStringFlags(str, start, 0); -} - -node * PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename, int start, int flags) { @@ -1418,12 +1373,6 @@ err_input(perrdetail *err) PyObject* u = NULL; char *msg = NULL; errtype = PyExc_SyntaxError; - v = Py_BuildValue("(ziiz)", err->filename, - err->lineno, err->offset, err->text); - if (err->text != NULL) { - PyMem_DEL(err->text); - err->text = NULL; - } switch (err->error) { case E_SYNTAX: errtype = PyExc_IndentationError; @@ -1450,11 +1399,9 @@ err_input(perrdetail *err) case E_INTR: if (!PyErr_Occurred()) PyErr_SetNone(PyExc_KeyboardInterrupt); - Py_XDECREF(v); return; case E_NOMEM: PyErr_NoMemory(); - Py_XDECREF(v); return; case E_EOF: msg = "unexpected EOF while parsing"; @@ -1498,7 +1445,15 @@ err_input(perrdetail *err) msg = "unknown parsing error"; break; } - w = Py_BuildValue("(sO)", msg, v); + v = Py_BuildValue("(ziiz)", err->filename, + err->lineno, err->offset, err->text); + if (err->text != NULL) { + PyMem_DEL(err->text); + err->text = NULL; + } + w = NULL; + if (v != NULL) + w = Py_BuildValue("(sO)", msg, v); Py_XDECREF(u); Py_XDECREF(v); PyErr_SetObject(errtype, w); @@ -1687,3 +1642,20 @@ PyOS_setsig(int sig, PyOS_sighandler_t handler) return oldhandler; #endif } + +/* Deprecated C API functions still provided for binary compatiblity */ + +#undef PyParser_SimpleParseFile +#undef PyParser_SimpleParseString + +node * +PyParser_SimpleParseFile(FILE *fp, const char *filename, int start) +{ + return PyParser_SimpleParseFileFlags(fp, filename, start, 0); +} + +node * +PyParser_SimpleParseString(const char *str, int start) +{ + return PyParser_SimpleParseStringFlags(str, start, 0); +} diff --git a/Python/symtable.c b/Python/symtable.c index 5ca2041..bd41202 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1,48 +1,35 @@ #include "Python.h" +#include "Python-ast.h" +#include "code.h" #include "compile.h" #include "symtable.h" -#include "graminit.h" #include "structmember.h" -/* The compiler uses this function to load a PySymtableEntry object - for a code block. Each block is loaded twice, once during the - symbol table pass and once during the code gen pass. Entries - created during the first pass are cached for the second pass, using - the st_symbols dictionary. - - The cache is keyed by st_nscopes. Each code block node in a - module's parse tree can be assigned a unique id based on the order - in which the nodes are visited by the compiler. This strategy - works so long as the symbol table and codegen passes visit the same - nodes in the same order. -*/ +/* two error strings used for warnings */ +#define GLOBAL_AFTER_ASSIGN \ +"name '%.400s' is assigned to before global declaration" +#define GLOBAL_AFTER_USE \ +"name '%.400s' is used prior to global declaration" -PyObject * -PySymtableEntry_New(struct symtable *st, char *name, int type, int lineno) +PySTEntryObject * +PySTEntry_New(struct symtable *st, identifier name, block_ty block, + void *key, int lineno) { - PySymtableEntryObject *ste = NULL; + PySTEntryObject *ste = NULL; PyObject *k, *v; - k = PyInt_FromLong(st->st_nscopes++); + k = PyLong_FromVoidPtr(key); if (k == NULL) goto fail; - v = PyDict_GetItem(st->st_symbols, k); - if (v) { - Py_DECREF(k); - Py_INCREF(v); - return v; - } - - ste = (PySymtableEntryObject *)PyObject_New(PySymtableEntryObject, - &PySymtableEntry_Type); + ste = (PySTEntryObject *)PyObject_New(PySTEntryObject, + &PySTEntry_Type); ste->ste_table = st; ste->ste_id = k; + ste->ste_tmpname = 0; - v = PyString_FromString(name); - if (v == NULL) - goto fail; - ste->ste_name = v; + ste->ste_name = name; + Py_INCREF(name); v = PyDict_New(); if (v == NULL) @@ -59,61 +46,46 @@ PySymtableEntry_New(struct symtable *st, char *name, int type, int lineno) goto fail; ste->ste_children = v; - ste->ste_optimized = 0; + ste->ste_type = block; + ste->ste_unoptimized = 0; + ste->ste_nested = 0; + ste->ste_free = 0; + ste->ste_varargs = 0; + ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; ste->ste_tmpname = 0; ste->ste_lineno = lineno; - switch (type) { - case funcdef: - case lambdef: - case testlist_gexp: /* generator expression */ - case argument: /* generator expression */ - ste->ste_type = TYPE_FUNCTION; - break; - case classdef: - ste->ste_type = TYPE_CLASS; - break; - case single_input: - case eval_input: - case file_input: - ste->ste_type = TYPE_MODULE; - break; - } - if (st->st_cur == NULL) - ste->ste_nested = 0; - else if (st->st_cur->ste_nested - || st->st_cur->ste_type == TYPE_FUNCTION) + if (st->st_cur != NULL && + (st->st_cur->ste_nested || + st->st_cur->ste_type == FunctionBlock)) ste->ste_nested = 1; - else - ste->ste_nested = 0; ste->ste_child_free = 0; ste->ste_generator = 0; if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0) goto fail; - return (PyObject *)ste; + return ste; fail: Py_XDECREF(ste); return NULL; } static PyObject * -ste_repr(PySymtableEntryObject *ste) +ste_repr(PySTEntryObject *ste) { char buf[256]; PyOS_snprintf(buf, sizeof(buf), "<symtable entry %.100s(%ld), line %d>", PyString_AS_STRING(ste->ste_name), - PyInt_AS_LONG(ste->ste_id), - ste->ste_lineno); + PyInt_AS_LONG(ste->ste_id), ste->ste_lineno); return PyString_FromString(buf); } static void -ste_dealloc(PySymtableEntryObject *ste) +ste_dealloc(PySTEntryObject *ste) { ste->ste_table = NULL; Py_XDECREF(ste->ste_id); @@ -124,7 +96,7 @@ ste_dealloc(PySymtableEntryObject *ste) PyObject_Del(ste); } -#define OFF(x) offsetof(PySymtableEntryObject, x) +#define OFF(x) offsetof(PySTEntryObject, x) static PyMemberDef ste_memberlist[] = { {"id", T_OBJECT, OFF(ste_id), READONLY}, @@ -134,16 +106,14 @@ static PyMemberDef ste_memberlist[] = { {"children", T_OBJECT, OFF(ste_children), READONLY}, {"type", T_INT, OFF(ste_type), READONLY}, {"lineno", T_INT, OFF(ste_lineno), READONLY}, - {"optimized",T_INT, OFF(ste_optimized), READONLY}, - {"nested", T_INT, OFF(ste_nested), READONLY}, {NULL} }; -PyTypeObject PySymtableEntry_Type = { +PyTypeObject PySTEntry_Type = { PyObject_HEAD_INIT(&PyType_Type) 0, "symtable entry", - sizeof(PySymtableEntryObject), + sizeof(PySTEntryObject), 0, (destructor)ste_dealloc, /* tp_dealloc */ 0, /* tp_print */ @@ -180,3 +150,1148 @@ PyTypeObject PySymtableEntry_Type = { 0, /* tp_alloc */ 0, /* tp_new */ }; + +static int symtable_analyze(struct symtable *st); +static int symtable_warn(struct symtable *st, char *msg); +static int symtable_enter_block(struct symtable *st, identifier name, + block_ty block, void *ast, int lineno); +static int symtable_exit_block(struct symtable *st, void *ast); +static int symtable_visit_stmt(struct symtable *st, stmt_ty s); +static int symtable_visit_expr(struct symtable *st, expr_ty s); +static int symtable_visit_genexp(struct symtable *st, expr_ty s); +static int symtable_visit_arguments(struct symtable *st, arguments_ty); +static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); +static int symtable_visit_alias(struct symtable *st, alias_ty); +static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); +static int symtable_visit_keyword(struct symtable *st, keyword_ty); +static int symtable_visit_slice(struct symtable *st, slice_ty); +static int symtable_visit_params(struct symtable *st, asdl_seq *args, int top); +static int symtable_visit_params_nested(struct symtable *st, asdl_seq *args); +static int symtable_implicit_arg(struct symtable *st, int pos); + + +static identifier top = NULL, lambda = NULL; + +#define GET_IDENTIFIER(VAR) \ + ((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR))) + +#define DUPLICATE_ARGUMENT \ +"duplicate argument '%s' in function definition" + +static struct symtable * +symtable_new(void) +{ + struct symtable *st; + + st = (struct symtable *)PyMem_Malloc(sizeof(struct symtable)); + if (st == NULL) + return NULL; + + st->st_filename = NULL; + if ((st->st_stack = PyList_New(0)) == NULL) + goto fail; + if ((st->st_symbols = PyDict_New()) == NULL) + goto fail; + st->st_cur = NULL; + st->st_tmpname = 0; + st->st_private = NULL; + return st; + fail: + PySymtable_Free(st); + return NULL; +} + +struct symtable * +PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) +{ + struct symtable *st = symtable_new(); + asdl_seq *seq; + int i; + + if (st == NULL) + return st; + st->st_filename = filename; + st->st_future = future; + symtable_enter_block(st, GET_IDENTIFIER(top), ModuleBlock, + (void *)mod, 0); + st->st_top = st->st_cur; + st->st_cur->ste_unoptimized = OPT_TOPLEVEL; + /* Any other top-level initialization? */ + switch (mod->kind) { + case Module_kind: + seq = mod->v.Module.body; + for (i = 0; i < asdl_seq_LEN(seq); i++) + if (!symtable_visit_stmt(st, asdl_seq_GET(seq, i))) + goto error; + break; + case Expression_kind: + if (!symtable_visit_expr(st, mod->v.Expression.body)) + goto error; + break; + case Interactive_kind: + seq = mod->v.Interactive.body; + for (i = 0; i < asdl_seq_LEN(seq); i++) + if (!symtable_visit_stmt(st, asdl_seq_GET(seq, i))) + goto error; + break; + case Suite_kind: + PyErr_SetString(PyExc_RuntimeError, + "this compiler does not handle Suites"); + return NULL; + } + if (!symtable_exit_block(st, (void *)mod)) + return NULL; + if (symtable_analyze(st)) + return st; + error: + PySymtable_Free(st); + return NULL; +} + +void +PySymtable_Free(struct symtable *st) +{ + Py_XDECREF(st->st_symbols); + Py_XDECREF(st->st_stack); + PyMem_Free((void *)st); +} + +PySTEntryObject * +PySymtable_Lookup(struct symtable *st, void *key) +{ + PyObject *k, *v; + + k = PyLong_FromVoidPtr(key); + if (k == NULL) + return NULL; + v = PyDict_GetItem(st->st_symbols, k); + if (v) { + assert(PySTEntry_Check(v)); + Py_DECREF(k); + Py_INCREF(v); + return (PySTEntryObject *)v; + } + else { + PyErr_SetString(PyExc_KeyError, + "unknown symbol table entry"); + return NULL; + } +} + +int +PyST_GetScope(PySTEntryObject *ste, PyObject *name) +{ + PyObject *v = PyDict_GetItem(ste->ste_symbols, name); + if (!v) + return 0; + assert(PyInt_Check(v)); + return (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; +} + + +/* Analyze raw symbol information to determine scope of each name. + + The next several functions are helpers for PySymtable_Analyze(), + which determines whether a name is local, global, or free. In addition, + it determines which local variables are cell variables; they provide + bindings that are used for free variables in enclosed blocks. + + There are also two kinds of free variables, implicit and explicit. An + explicit global is declared with the global statement. An implicit + global is a free variable for which the compiler has found no binding + in an enclosing function scope. The implicit global is either a global + or a builtin. Python's module and class blocks use the xxx_NAME opcodes + to handle these names to implement slightly odd semantics. In such a + block, the name is treated as global until it is assigned to; then it + is treated as a local. + + The symbol table requires two passes to determine the scope of each name. + The first pass collects raw facts from the AST: the name is a parameter + here, the name is used by not defined here, etc. The second pass analyzes + these facts during a pass over the PySTEntryObjects created during pass 1. + + When a function is entered during the second pass, the parent passes + the set of all name bindings visible to its children. These bindings + are used to determine if the variable is free or an implicit global. + After doing the local analysis, it analyzes each of its child blocks + using an updated set of name bindings. + + The children update the free variable set. If a local variable is free + in a child, the variable is marked as a cell. The current function must + provide runtime storage for the variable that may outlive the function's + frame. Cell variables are removed from the free set before the analyze + function returns to its parent. + + The sets of bound and free variables are implemented as dictionaries + mapping strings to None. +*/ + +#define SET_SCOPE(DICT, NAME, I) { \ + PyObject *o = PyInt_FromLong(I); \ + if (!o) \ + return 0; \ + if (PyDict_SetItem((DICT), (NAME), o) < 0) \ + return 0; \ +} + +/* Decide on scope of name, given flags. + + The dicts passed in as arguments are modified as necessary. + ste is passed so that flags can be updated. +*/ + +static int +analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, int flags, + PyObject *bound, PyObject *local, PyObject *free, + PyObject *global) +{ + if (flags & DEF_GLOBAL) { + if (flags & DEF_PARAM) { + PyErr_Format(PyExc_SyntaxError, + "name '%s' is local and global", + PyString_AS_STRING(name)); + return 0; + } + SET_SCOPE(dict, name, GLOBAL_EXPLICIT); + if (PyDict_SetItem(global, name, Py_None) < 0) + return 0; + if (bound && PyDict_GetItem(bound, name)) { + if (PyDict_DelItem(bound, name) < 0) + return 0; + } + return 1; + } + if (flags & DEF_BOUND) { + SET_SCOPE(dict, name, LOCAL); + if (PyDict_SetItem(local, name, Py_None) < 0) + return 0; + if (PyDict_GetItem(global, name)) { + if (PyDict_DelItem(global, name) < 0) + return 0; + } + return 1; + } + /* If an enclosing block has a binding for this name, it + is a free variable rather than a global variable. + Note that having a non-NULL bound implies that the block + is nested. + */ + if (bound && PyDict_GetItem(bound, name)) { + SET_SCOPE(dict, name, FREE); + ste->ste_free = 1; + if (PyDict_SetItem(free, name, Py_None) < 0) + return 0; + return 1; + } + /* If a parent has a global statement, then call it global + explicit? It could also be global implicit. + */ + else if (global && PyDict_GetItem(global, name)) { + SET_SCOPE(dict, name, GLOBAL_EXPLICIT); + return 1; + } + else { + if (ste->ste_nested) + ste->ste_free = 1; + SET_SCOPE(dict, name, GLOBAL_IMPLICIT); + return 1; + } + return 0; /* Can't get here */ +} + +#undef SET_SCOPE + +/* If a name is defined in free and also in locals, then this block + provides the binding for the free variable. The name should be + marked CELL in this block and removed from the free list. + + Note that the current block's free variables are included in free. + That's safe because no name can be free and local in the same scope. +*/ + +static int +analyze_cells(PyObject *scope, PyObject *free) +{ + PyObject *name, *v, *w; + int flags, pos = 0, success = 0; + + w = PyInt_FromLong(CELL); + if (!w) + return 0; + while (PyDict_Next(scope, &pos, &name, &v)) { + assert(PyInt_Check(v)); + flags = PyInt_AS_LONG(v); + if (flags != LOCAL) + continue; + if (!PyDict_GetItem(free, name)) + continue; + /* Replace LOCAL with CELL for this name, and remove + from free. It is safe to replace the value of name + in the dict, because it will not cause a resize. + */ + if (PyDict_SetItem(scope, name, w) < 0) + goto error; + if (!PyDict_DelItem(free, name) < 0) + goto error; + } + success = 1; + error: + Py_DECREF(w); + return success; +} + +/* Check for illegal statements in unoptimized namespaces */ +static int +check_unoptimized(const PySTEntryObject* ste) { + char buf[300]; + + if (ste->ste_type == ModuleBlock || !ste->ste_unoptimized + || !(ste->ste_free || ste->ste_child_free)) + return 1; + + const char* trailer = (ste->ste_child_free ? + "contains a nested function with free variables" : + "is a nested function"); + + switch (ste->ste_unoptimized) { + case OPT_TOPLEVEL: /* exec / import * at top-level is fine */ + case OPT_EXEC: /* qualified exec is fine */ + return 1; + case OPT_IMPORT_STAR: + PyOS_snprintf(buf, sizeof(buf), + "import * is not allowed in function '%.100s' " + "because it is %s", + PyString_AS_STRING(ste->ste_name), trailer); + break; + case OPT_BARE_EXEC: + PyOS_snprintf(buf, sizeof(buf), + "unqualified exec is not allowed in function " + "'%.100s' it %s", + PyString_AS_STRING(ste->ste_name), trailer); + break; + default: + PyOS_snprintf(buf, sizeof(buf), + "function '%.100s' uses import * and bare exec, " + "which are illegal because it %s", + PyString_AS_STRING(ste->ste_name), trailer); + break; + } + + PyErr_SetString(PyExc_SyntaxError, buf); + PyErr_SyntaxLocation(ste->ste_table->st_filename, + ste->ste_opt_lineno); + return 0; +} + +/* Enter the final scope information into the st_symbols dict. + * + * All arguments are dicts. Modifies symbols, others are read-only. +*/ +static int +update_symbols(PyObject *symbols, PyObject *scope, + PyObject *bound, PyObject *free, int class) +{ + PyObject *name, *v, *u, *w, *free_value = NULL; + int i, flags, pos = 0; + + while (PyDict_Next(symbols, &pos, &name, &v)) { + assert(PyInt_Check(v)); + flags = PyInt_AS_LONG(v); + w = PyDict_GetItem(scope, name); + assert(w && PyInt_Check(w)); + i = PyInt_AS_LONG(w); + flags |= (i << SCOPE_OFF); + u = PyInt_FromLong(flags); + if (PyDict_SetItem(symbols, name, u) < 0) { + Py_DECREF(u); + return 0; + } + Py_DECREF(u); + } + + free_value = PyInt_FromLong(FREE << SCOPE_OFF); + if (!free_value) + return 0; + + /* add a free variable when it's only use is for creating a closure */ + pos = 0; + while (PyDict_Next(free, &pos, &name, &v)) { + PyObject *o = PyDict_GetItem(symbols, name); + + if (o) { + /* It could be a free variable in a method of + the class that has the same name as a local + or global in the class scope. + */ + if (class && + PyInt_AS_LONG(o) & (DEF_BOUND | DEF_GLOBAL)) { + int i = PyInt_AS_LONG(o) | DEF_FREE_CLASS; + o = PyInt_FromLong(i); + if (!o) { + Py_DECREF(free_value); + return 0; + } + if (PyDict_SetItem(symbols, name, o) < 0) { + Py_DECREF(o); + Py_DECREF(free_value); + return 0; + } + } + /* else it's not free, probably a cell */ + continue; + } + if (!PyDict_GetItem(bound, name)) + continue; /* it's a global */ + + if (PyDict_SetItem(symbols, name, free_value) < 0) { + Py_DECREF(free_value); + return 0; + } + } + Py_DECREF(free_value); + return 1; +} + +/* Make final symbol table decisions for block of ste. + Arguments: + ste -- current symtable entry (input/output) + bound -- set of variables bound in enclosing scopes (input) + free -- set of free variables in enclosed scopes (output) + globals -- set of declared global variables in enclosing scopes (input) +*/ + +static int +analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, + PyObject *global) +{ + PyObject *name, *v, *local = NULL, *scope = NULL, *newbound = NULL; + PyObject *newglobal = NULL, *newfree = NULL; + int i, flags, pos = 0, success = 0; + + local = PyDict_New(); + if (!local) + goto error; + scope = PyDict_New(); + if (!scope) + goto error; + newglobal = PyDict_New(); + if (!newglobal) + goto error; + newfree = PyDict_New(); + if (!newfree) + goto error; + newbound = PyDict_New(); + if (!newbound) + goto error; + + if (ste->ste_type == ClassBlock) { + /* make a copy of globals before calling analyze_name(), + because global statements in the class have no effect + on nested functions. + */ + if (PyDict_Update(newglobal, global) < 0) + goto error; + if (bound) + if (PyDict_Update(newbound, bound) < 0) + goto error; + } + + assert(PySTEntry_Check(ste)); + assert(PyDict_Check(ste->ste_symbols)); + while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { + flags = PyInt_AS_LONG(v); + if (!analyze_name(ste, scope, name, flags, bound, local, free, + global)) + goto error; + } + + if (ste->ste_type != ClassBlock) { + if (ste->ste_type == FunctionBlock) { + if (PyDict_Update(newbound, local) < 0) + goto error; + } + if (bound) { + if (PyDict_Update(newbound, bound) < 0) + goto error; + } + if (PyDict_Update(newglobal, global) < 0) + goto error; + } + + /* Recursively call analyze_block() on each child block */ + for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) { + PyObject *c = PyList_GET_ITEM(ste->ste_children, i); + assert(c && PySTEntry_Check(c)); + PySTEntryObject* entry = (PySTEntryObject*)c; + if (!analyze_block(entry, newbound, newfree, newglobal)) + goto error; + if (entry->ste_free || entry->ste_child_free) + ste->ste_child_free = 1; + } + + if (ste->ste_type == FunctionBlock && !analyze_cells(scope, newfree)) + goto error; + if (!update_symbols(ste->ste_symbols, scope, bound, newfree, + ste->ste_type == ClassBlock)) + goto error; + if (!check_unoptimized(ste)) + goto error; + + if (PyDict_Update(free, newfree) < 0) + goto error; + success = 1; + error: + Py_XDECREF(local); + Py_XDECREF(scope); + Py_XDECREF(newbound); + Py_XDECREF(newglobal); + Py_XDECREF(newfree); + if (!success) + assert(PyErr_Occurred()); + return success; +} + +static int +symtable_analyze(struct symtable *st) +{ + PyObject *free, *global; + int r; + + free = PyDict_New(); + if (!free) + return 0; + global = PyDict_New(); + if (!global) { + Py_DECREF(global); + return 0; + } + r = analyze_block(st->st_top, NULL, free, global); + Py_DECREF(free); + Py_DECREF(global); + return r; +} + + +static int +symtable_warn(struct symtable *st, char *msg) +{ + if (PyErr_WarnExplicit(PyExc_SyntaxWarning, msg, st->st_filename, + st->st_cur->ste_lineno, NULL, NULL) < 0) { + if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { + PyErr_SetString(PyExc_SyntaxError, msg); + PyErr_SyntaxLocation(st->st_filename, + st->st_cur->ste_lineno); + } + return 0; + } + return 1; +} + +/* symtable_enter_block() gets a reference via PySTEntry_New(). + This reference is released when the block is exited, via the DECREF + in symtable_exit_block(). +*/ + +static int +symtable_exit_block(struct symtable *st, void *ast) +{ + int end; + + Py_DECREF(st->st_cur); + end = PyList_GET_SIZE(st->st_stack) - 1; + if (end >= 0) { + st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, + end); + Py_INCREF(st->st_cur); + if (PySequence_DelItem(st->st_stack, end) < 0) + return 0; + } + return 1; +} + +static int +symtable_enter_block(struct symtable *st, identifier name, block_ty block, + void *ast, int lineno) +{ + PySTEntryObject *prev = NULL; + + if (st->st_cur) { + prev = st->st_cur; + if (PyList_Append(st->st_stack, (PyObject *)st->st_cur) < 0) { + Py_DECREF(st->st_cur); + return 0; + } + Py_DECREF(st->st_cur); + } + st->st_cur = PySTEntry_New(st, name, block, ast, lineno); + if (name == GET_IDENTIFIER(top)) + st->st_global = st->st_cur->ste_symbols; + if (prev) { + if (PyList_Append(prev->ste_children, + (PyObject *)st->st_cur) < 0) { + return 0; + } + } + return 1; +} + +static int +symtable_lookup(struct symtable *st, PyObject *name) +{ + PyObject *o; + + o = PyDict_GetItem(st->st_cur->ste_symbols, name); + if (!o) + return 0; + return PyInt_AsLong(o); +} + +static int +symtable_add_def(struct symtable *st, PyObject *name, int flag) +{ + PyObject *o; + PyObject *dict; + int val; + + dict = st->st_cur->ste_symbols; + if ((o = PyDict_GetItem(dict, name))) { + val = PyInt_AS_LONG(o); + if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { + PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, + PyString_AsString(name)); + PyErr_SyntaxLocation(st->st_filename, + st->st_cur->ste_lineno); + return 0; + } + val |= flag; + } else + val = flag; + o = PyInt_FromLong(val); + if (o == NULL) + return 0; + if (PyDict_SetItem(dict, name, o) < 0) { + Py_DECREF(o); + return 0; + } + Py_DECREF(o); + + if (flag & DEF_PARAM) { + if (PyList_Append(st->st_cur->ste_varnames, name) < 0) + return 0; + } else if (flag & DEF_GLOBAL) { + /* XXX need to update DEF_GLOBAL for other flags too; + perhaps only DEF_FREE_GLOBAL */ + val = flag; + if ((o = PyDict_GetItem(st->st_global, name))) { + val |= PyInt_AS_LONG(o); + } + o = PyInt_FromLong(val); + if (o == NULL) + return 0; + if (PyDict_SetItem(st->st_global, name, o) < 0) { + Py_DECREF(o); + return 0; + } + Py_DECREF(o); + } + return 1; +} + +/* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument. + They use the ASDL name to synthesize the name of the C type and the visit + function. + + VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is + useful if the first node in the sequence requires special treatment. +*/ + +#define VISIT(ST, TYPE, V) \ + if (!symtable_visit_ ## TYPE((ST), (V))) \ + return 0; + +#define VISIT_SEQ(ST, TYPE, SEQ) { \ + int i; \ + asdl_seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + return 0; \ + } \ +} + +#define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \ + int i; \ + asdl_seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = (START); i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + return 0; \ + } \ +} + +static int +symtable_visit_stmt(struct symtable *st, stmt_ty s) +{ + switch (s->kind) { + case FunctionDef_kind: + if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL)) + return 0; + if (s->v.FunctionDef.args->defaults) + VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults); + if (s->v.FunctionDef.decorators) + VISIT_SEQ(st, expr, s->v.FunctionDef.decorators); + if (!symtable_enter_block(st, s->v.FunctionDef.name, + FunctionBlock, (void *)s, s->lineno)) + return 0; + VISIT(st, arguments, s->v.FunctionDef.args); + VISIT_SEQ(st, stmt, s->v.FunctionDef.body); + if (!symtable_exit_block(st, s)) + return 0; + break; + case ClassDef_kind: + if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL)) + return 0; + VISIT_SEQ(st, expr, s->v.ClassDef.bases); + if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, + (void *)s, s->lineno)) + return 0; + VISIT_SEQ(st, stmt, s->v.ClassDef.body); + if (!symtable_exit_block(st, s)) + return 0; + break; + case Return_kind: + if (s->v.Return.value) + VISIT(st, expr, s->v.Return.value); + break; + case Delete_kind: + VISIT_SEQ(st, expr, s->v.Delete.targets); + break; + case Assign_kind: + VISIT_SEQ(st, expr, s->v.Assign.targets); + VISIT(st, expr, s->v.Assign.value); + break; + case AugAssign_kind: + VISIT(st, expr, s->v.AugAssign.target); + VISIT(st, expr, s->v.AugAssign.value); + break; + case Print_kind: + if (s->v.Print.dest) + VISIT(st, expr, s->v.Print.dest); + VISIT_SEQ(st, expr, s->v.Print.values); + break; + case For_kind: + VISIT(st, expr, s->v.For.target); + VISIT(st, expr, s->v.For.iter); + VISIT_SEQ(st, stmt, s->v.For.body); + if (s->v.For.orelse) + VISIT_SEQ(st, stmt, s->v.For.orelse); + break; + case While_kind: + VISIT(st, expr, s->v.While.test); + VISIT_SEQ(st, stmt, s->v.While.body); + if (s->v.While.orelse) + VISIT_SEQ(st, stmt, s->v.While.orelse); + break; + case If_kind: + /* XXX if 0: and lookup_yield() hacks */ + VISIT(st, expr, s->v.If.test); + VISIT_SEQ(st, stmt, s->v.If.body); + if (s->v.If.orelse) + VISIT_SEQ(st, stmt, s->v.If.orelse); + break; + case Raise_kind: + if (s->v.Raise.type) { + VISIT(st, expr, s->v.Raise.type); + if (s->v.Raise.inst) { + VISIT(st, expr, s->v.Raise.inst); + if (s->v.Raise.tback) + VISIT(st, expr, s->v.Raise.tback); + } + } + break; + case TryExcept_kind: + VISIT_SEQ(st, stmt, s->v.TryExcept.body); + VISIT_SEQ(st, stmt, s->v.TryExcept.orelse); + VISIT_SEQ(st, excepthandler, s->v.TryExcept.handlers); + break; + case TryFinally_kind: + VISIT_SEQ(st, stmt, s->v.TryFinally.body); + VISIT_SEQ(st, stmt, s->v.TryFinally.finalbody); + break; + case Assert_kind: + VISIT(st, expr, s->v.Assert.test); + if (s->v.Assert.msg) + VISIT(st, expr, s->v.Assert.msg); + break; + case Import_kind: + VISIT_SEQ(st, alias, s->v.Import.names); + /* XXX Don't have the lineno available inside + visit_alias */ + if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) + st->st_cur->ste_opt_lineno = s->lineno; + break; + case ImportFrom_kind: + VISIT_SEQ(st, alias, s->v.ImportFrom.names); + /* XXX Don't have the lineno available inside + visit_alias */ + if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) + st->st_cur->ste_opt_lineno = s->lineno; + break; + case Exec_kind: + VISIT(st, expr, s->v.Exec.body); + if (!st->st_cur->ste_opt_lineno) + st->st_cur->ste_opt_lineno = s->lineno; + if (s->v.Exec.globals) { + st->st_cur->ste_unoptimized |= OPT_EXEC; + VISIT(st, expr, s->v.Exec.globals); + if (s->v.Exec.locals) + VISIT(st, expr, s->v.Exec.locals); + } else { + st->st_cur->ste_unoptimized |= OPT_BARE_EXEC; + } + break; + case Global_kind: { + int i; + asdl_seq *seq = s->v.Global.names; + for (i = 0; i < asdl_seq_LEN(seq); i++) { + identifier name = asdl_seq_GET(seq, i); + char *c_name = PyString_AS_STRING(name); + int cur = symtable_lookup(st, name); + if (cur < 0) + return 0; + if (cur & (DEF_LOCAL | USE)) { + char buf[1000]; + if (cur & DEF_LOCAL) + PyOS_snprintf(buf, sizeof(buf), + GLOBAL_AFTER_ASSIGN, + c_name); + else + PyOS_snprintf(buf, sizeof(buf), + GLOBAL_AFTER_USE, + c_name); + if (!symtable_warn(st, buf)) + return 0; + } + if (!symtable_add_def(st, name, DEF_GLOBAL)) + return 0; + + } + + break; + } + case Expr_kind: + VISIT(st, expr, s->v.Expr.value); + break; + case Pass_kind: + case Break_kind: + case Continue_kind: + /* nothing to do here */ + break; + } + return 1; +} + +static int +symtable_visit_expr(struct symtable *st, expr_ty e) +{ + switch (e->kind) { + case BoolOp_kind: + VISIT_SEQ(st, expr, e->v.BoolOp.values); + break; + case BinOp_kind: + VISIT(st, expr, e->v.BinOp.left); + VISIT(st, expr, e->v.BinOp.right); + break; + case UnaryOp_kind: + VISIT(st, expr, e->v.UnaryOp.operand); + break; + case Lambda_kind: { + if (!symtable_add_def(st, GET_IDENTIFIER(lambda), DEF_LOCAL)) + return 0; + if (e->v.Lambda.args->defaults) + VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); + /* XXX how to get line numbers for expressions */ + if (!symtable_enter_block(st, GET_IDENTIFIER(lambda), + FunctionBlock, (void *)e, 0)) + return 0; + VISIT(st, arguments, e->v.Lambda.args); + VISIT(st, expr, e->v.Lambda.body); + if (!symtable_exit_block(st, (void *)e)) + return 0; + break; + } + case Dict_kind: + VISIT_SEQ(st, expr, e->v.Dict.keys); + VISIT_SEQ(st, expr, e->v.Dict.values); + break; + case ListComp_kind: { + char tmpname[256]; + identifier tmp; + + PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", + ++st->st_cur->ste_tmpname); + tmp = PyString_FromString(tmpname); + if (!symtable_add_def(st, tmp, DEF_LOCAL)) + return 0; + VISIT(st, expr, e->v.ListComp.elt); + VISIT_SEQ(st, comprehension, e->v.ListComp.generators); + break; + } + case GeneratorExp_kind: { + if (!symtable_visit_genexp(st, e)) { + return 0; + } + break; + } + case Yield_kind: + if (e->v.Yield.value) + VISIT(st, expr, e->v.Yield.value); + st->st_cur->ste_generator = 1; + break; + case Compare_kind: + VISIT(st, expr, e->v.Compare.left); + VISIT_SEQ(st, expr, e->v.Compare.comparators); + break; + case Call_kind: + VISIT(st, expr, e->v.Call.func); + VISIT_SEQ(st, expr, e->v.Call.args); + VISIT_SEQ(st, keyword, e->v.Call.keywords); + if (e->v.Call.starargs) + VISIT(st, expr, e->v.Call.starargs); + if (e->v.Call.kwargs) + VISIT(st, expr, e->v.Call.kwargs); + break; + case Repr_kind: + VISIT(st, expr, e->v.Repr.value); + break; + case Num_kind: + case Str_kind: + /* Nothing to do here. */ + break; + /* The following exprs can be assignment targets. */ + case Attribute_kind: + VISIT(st, expr, e->v.Attribute.value); + break; + case Subscript_kind: + VISIT(st, expr, e->v.Subscript.value); + VISIT(st, slice, e->v.Subscript.slice); + break; + case Name_kind: + if (!symtable_add_def(st, e->v.Name.id, + e->v.Name.ctx == Load ? USE : DEF_LOCAL)) + return 0; + break; + /* child nodes of List and Tuple will have expr_context set */ + case List_kind: + VISIT_SEQ(st, expr, e->v.List.elts); + break; + case Tuple_kind: + VISIT_SEQ(st, expr, e->v.Tuple.elts); + break; + } + return 1; +} + +static int +symtable_implicit_arg(struct symtable *st, int pos) +{ + PyObject *id = PyString_FromFormat(".%d", pos); + if (id == NULL) + return 0; + if (!symtable_add_def(st, id, DEF_PARAM)) { + Py_DECREF(id); + return 0; + } + Py_DECREF(id); + return 1; +} + +static int +symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel) +{ + int i, complex = 0; + + /* go through all the toplevel arguments first */ + for (i = 0; i < asdl_seq_LEN(args); i++) { + expr_ty arg = asdl_seq_GET(args, i); + if (arg->kind == Name_kind) { + assert(arg->v.Name.ctx == Param || + (arg->v.Name.ctx == Store && !toplevel)); + if (!symtable_add_def(st, arg->v.Name.id, DEF_PARAM)) + return 0; + } + else if (arg->kind == Tuple_kind) { + assert(arg->v.Tuple.ctx == Store); + complex = 1; + if (toplevel) { + if (!symtable_implicit_arg(st, i)) + return 0; + } + } + else { + /* syntax error */ + fprintf(stderr, "unexpected expr in parameter list\n"); + return 0; + } + } + + if (!toplevel) { + if (!symtable_visit_params_nested(st, args)) + return 0; + } + + return 1; +} + +static int +symtable_visit_params_nested(struct symtable *st, asdl_seq *args) +{ + int i; + for (i = 0; i < asdl_seq_LEN(args); i++) { + expr_ty arg = asdl_seq_GET(args, i); + if (arg->kind == Tuple_kind && + !symtable_visit_params(st, arg->v.Tuple.elts, 0)) + return 0; + } + + return 1; +} + +static int +symtable_visit_arguments(struct symtable *st, arguments_ty a) +{ + /* skip default arguments inside function block + XXX should ast be different? + */ + if (a->args && !symtable_visit_params(st, a->args, 1)) + return 0; + if (a->vararg) { + if (!symtable_add_def(st, a->vararg, DEF_PARAM)) + return 0; + st->st_cur->ste_varargs = 1; + } + if (a->kwarg) { + if (!symtable_add_def(st, a->kwarg, DEF_PARAM)) + return 0; + st->st_cur->ste_varkeywords = 1; + } + if (a->args && !symtable_visit_params_nested(st, a->args)) + return 0; + return 1; +} + + +static int +symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh) +{ + if (eh->type) + VISIT(st, expr, eh->type); + if (eh->name) + VISIT(st, expr, eh->name); + VISIT_SEQ(st, stmt, eh->body); + return 1; +} + + +static int +symtable_visit_alias(struct symtable *st, alias_ty a) +{ + /* Compute store_name, the name actually bound by the import + operation. It is diferent than a->name when a->name is a + dotted package name (e.g. spam.eggs) + */ + PyObject *store_name; + PyObject *name = (a->asname == NULL) ? a->name : a->asname; + const char *base = PyString_AS_STRING(name); + char *dot = strchr(base, '.'); + if (dot) + store_name = PyString_FromStringAndSize(base, dot - base); + else { + store_name = name; + Py_INCREF(store_name); + } + if (strcmp(PyString_AS_STRING(name), "*")) { + int r = symtable_add_def(st, store_name, DEF_IMPORT); + Py_DECREF(store_name); + return r; + } + else { + if (st->st_cur->ste_type != ModuleBlock) { + if (!symtable_warn(st, + "import * only allowed at module level")) + return 0; + } + st->st_cur->ste_unoptimized |= OPT_IMPORT_STAR; + return 1; + } +} + + +static int +symtable_visit_comprehension(struct symtable *st, comprehension_ty lc) +{ + VISIT(st, expr, lc->target); + VISIT(st, expr, lc->iter); + VISIT_SEQ(st, expr, lc->ifs); + return 1; +} + + +static int +symtable_visit_keyword(struct symtable *st, keyword_ty k) +{ + VISIT(st, expr, k->value); + return 1; +} + + +static int +symtable_visit_slice(struct symtable *st, slice_ty s) +{ + switch (s->kind) { + case Slice_kind: + if (s->v.Slice.lower) + VISIT(st, expr, s->v.Slice.lower) + if (s->v.Slice.upper) + VISIT(st, expr, s->v.Slice.upper) + if (s->v.Slice.step) + VISIT(st, expr, s->v.Slice.step) + break; + case ExtSlice_kind: + VISIT_SEQ(st, slice, s->v.ExtSlice.dims) + break; + case Index_kind: + VISIT(st, expr, s->v.Index.value) + break; + case Ellipsis_kind: + break; + } + return 1; +} + +static int +symtable_visit_genexp(struct symtable *st, expr_ty e) +{ + identifier tmp; + comprehension_ty outermost = ((comprehension_ty) + (asdl_seq_GET(e->v.GeneratorExp.generators, 0))); + /* Outermost iterator is evaluated in current scope */ + VISIT(st, expr, outermost->iter); + /* Create generator scope for the rest */ + tmp = PyString_FromString("<genexpr>"); + if (!symtable_enter_block(st, tmp, FunctionBlock, (void *)e, 0)) { + return 0; + } + st->st_cur->ste_generator = 1; + /* Outermost iter is received as an argument */ + if (!symtable_implicit_arg(st, 0)) { + return 0; + } + VISIT(st, expr, outermost->target); + VISIT_SEQ(st, expr, outermost->ifs); + VISIT_SEQ_TAIL(st, comprehension, e->v.GeneratorExp.generators, 1); + VISIT(st, expr, e->v.GeneratorExp.elt); + if (!symtable_exit_block(st, (void *)e)) + return 0; + return 1; +} diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 0775bb8..d9f1337 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -15,7 +15,7 @@ Data members: */ #include "Python.h" -#include "compile.h" +#include "code.h" #include "frameobject.h" #include "eval.h" diff --git a/Python/traceback.c b/Python/traceback.c index f40cfb4..b58e8ad 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -3,7 +3,7 @@ #include "Python.h" -#include "compile.h" +#include "code.h" #include "frameobject.h" #include "structmember.h" #include "osdefs.h" diff --git a/Tools/compiler/dumppyc.py b/Tools/compiler/dumppyc.py index dd460c9..8cfe3b1 100755 --- a/Tools/compiler/dumppyc.py +++ b/Tools/compiler/dumppyc.py @@ -28,7 +28,7 @@ def walk(co, match=None): if type(obj) == types.CodeType: walk(obj, match) -def main(filename, codename=None): +def load(filename, codename=None): co = loadCode(filename) walk(co, codename) @@ -39,6 +39,9 @@ if __name__ == "__main__": else: filename = sys.argv[1] codename = None - if filename.endswith('.py') and os.path.exists(filename+"c"): - filename += "c" - main(filename, codename) + if filename.endswith('.py'): + buf = open(filename).read() + co = compile(buf, filename, "exec") + walk(co) + else: + load(filename, codename) |