diff options
Diffstat (limited to 'Parser/asdl_c.py')
| -rwxr-xr-x | Parser/asdl_c.py | 773 |
1 files changed, 303 insertions, 470 deletions
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 52495e9..ac61c78 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1,18 +1,25 @@ #! /usr/bin/env python """Generate C code from an ASDL description.""" +# TO DO +# handle fields that have a type but no name + import os, sys import asdl -TABSIZE = 4 +TABSIZE = 8 MAX_COL = 80 def get_c_type(name): """Return a string for the C name of the type. - This function special cases the default types provided by asdl. + This function special cases the default types provided by asdl: + identifier, string, int, bool. """ + # XXX ack! need to figure out where Id is useful and where string + if isinstance(name, asdl.Id): + name = name.value if name in asdl.builtin_types: return name else: @@ -77,20 +84,8 @@ class EmitVisitor(asdl.VisitorBase): def __init__(self, file): self.file = file - self.identifiers = set() - self.singletons = set() - self.types = set() super(EmitVisitor, self).__init__() - def emit_identifier(self, name): - self.identifiers.add(str(name)) - - def emit_singleton(self, name): - self.singletons.add(str(name)) - - def emit_type(self, name): - self.types.add(str(name)) - def emit(self, s, depth, reflow=True): # XXX reflow long lines? if reflow: @@ -98,9 +93,8 @@ class EmitVisitor(asdl.VisitorBase): else: lines = [s] for line in lines: - if line: - line = (" " * TABSIZE * depth) + line - self.file.write(line + "\n") + line = (" " * TABSIZE * depth) + line + "\n" + self.file.write(line) class TypeDefVisitor(EmitVisitor): @@ -186,6 +180,9 @@ class StructVisitor(EmitVisitor): self.visit(f, depth + 1) self.emit("} %s;" % cons.name, depth) self.emit("", depth) + else: + # XXX not sure what I want here, nothing is probably fine + pass def visitField(self, field, depth): # XXX need to lookup field.type, because it might be something @@ -193,7 +190,7 @@ class StructVisitor(EmitVisitor): ctype = get_c_type(field.type) name = field.name if field.seq: - if field.type == 'cmpop': + if field.type.value in ('cmpop',): self.emit("asdl_int_seq *%(name)s;" % locals(), depth) else: self.emit("asdl_seq *%(name)s;" % locals(), depth) @@ -204,11 +201,6 @@ class StructVisitor(EmitVisitor): self.emit("struct _%(name)s {" % locals(), depth) for f in product.fields: self.visit(f, depth + 1) - for field in product.attributes: - # rudimentary attribute handling - type = str(field.type) - assert type in asdl.builtin_types, type - self.emit("%s %s;" % (type, field.name), depth + 1); self.emit("};", depth) self.emit("", depth) @@ -248,7 +240,7 @@ class PrototypeVisitor(EmitVisitor): name = f.name # XXX should extend get_c_type() to handle this if f.seq: - if f.type == 'cmpop': + if f.type.value in ('cmpop',): ctype = "asdl_int_seq *" else: ctype = "asdl_seq *" @@ -280,9 +272,7 @@ class PrototypeVisitor(EmitVisitor): def visitProduct(self, prod, name): self.emit_function(name, get_c_type(name), - self.get_args(prod.fields), - self.get_args(prod.attributes), - union=False) + self.get_args(prod.fields), [], union=False) class FunctionVisitor(PrototypeVisitor): @@ -302,7 +292,8 @@ class FunctionVisitor(PrototypeVisitor): emit("{") emit("%s p;" % ctype, 1) for argtype, argname, opt in args: - if not opt and argtype != "int": + # XXX hack alert: false is allowed for a bool + if not opt and not (argtype == "bool" or argtype == "int"): emit("if (!%s) {" % argname, 1) emit("PyErr_SetString(PyExc_ValueError,", 2) msg = "field %s is required for %s" % (argname, name) @@ -336,8 +327,7 @@ class FunctionVisitor(PrototypeVisitor): self.emit(s, depth, reflow) for argtype, argname, opt in args: emit("p->%s = %s;" % (argname, argname), 1) - for argtype, argname, opt in attrs: - emit("p->%s = %s;" % (argname, argname), 1) + assert not attrs class PickleVisitor(EmitVisitor): @@ -376,18 +366,20 @@ class Obj2ModVisitor(PickleVisitor): self.emit("int", 0) self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("{", 0) + self.emit("PyObject* tmp = NULL;", 1) self.emit("int isinstance;", 1) self.emit("", 0) - def sumTrailer(self, name, add_label=False): + def sumTrailer(self, name): self.emit("", 0) + self.emit("tmp = PyObject_Repr(obj);", 1) # there's really nothing more we can do if this fails ... - error = "expected some sort of %s, but got %%R" % name - format = "PyErr_Format(PyExc_TypeError, \"%s\", obj);" + self.emit("if (tmp == NULL) goto failed;", 1) + error = "expected some sort of %s, but got %%.400s" % name + format = "PyErr_Format(PyExc_TypeError, \"%s\", PyString_AS_STRING(tmp));" self.emit(format % error, 1, reflow=False) - if add_label: - self.emit("failed:", 1) - self.emit("Py_XDECREF(tmp);", 1) + self.emit("failed:", 0) + self.emit("Py_XDECREF(tmp);", 1) self.emit("return 1;", 1) self.emit("}", 0) self.emit("", 0) @@ -396,7 +388,7 @@ class Obj2ModVisitor(PickleVisitor): self.funcHeader(name) for t in sum.types: line = ("isinstance = PyObject_IsInstance(obj, " - "astmodulestate_global->%s_type);") + "(PyObject *)%s_type);") self.emit(line % (t.name,), 1) self.emit("if (isinstance == -1) {", 1) self.emit("return 1;", 2) @@ -412,8 +404,6 @@ class Obj2ModVisitor(PickleVisitor): def complexSum(self, sum, name): self.funcHeader(name) - self.emit("PyObject *tmp = NULL;", 1) - self.emit("PyObject *tp;", 1) for a in sum.attributes: self.visitAttributeDeclaration(a, name, sum=sum) self.emit("", 0) @@ -425,8 +415,8 @@ class Obj2ModVisitor(PickleVisitor): for a in sum.attributes: self.visitField(a, name, sum=sum, depth=1) for t in sum.types: - self.emit("tp = astmodulestate_global->%s_type;" % (t.name,), 1) - self.emit("isinstance = PyObject_IsInstance(obj, tp);", 1) + line = "isinstance = PyObject_IsInstance(obj, (PyObject*)%s_type);" + self.emit(line % (t.name,), 1) self.emit("if (isinstance == -1) {", 1) self.emit("return 1;", 2) self.emit("}", 1) @@ -436,12 +426,12 @@ class Obj2ModVisitor(PickleVisitor): self.emit("", 0) for f in t.fields: self.visitField(f, t.name, sum=sum, depth=2) - args = [f.name for f in t.fields] + [a.name for a in sum.attributes] + args = [f.name.value for f in t.fields] + [a.name.value for a in sum.attributes] self.emit("*out = %s(%s);" % (t.name, self.buildArgs(args)), 2) self.emit("if (*out == NULL) goto failed;", 2) self.emit("return 0;", 2) self.emit("}", 1) - self.sumTrailer(name, True) + self.sumTrailer(name) def visitAttributeDeclaration(self, a, name, sum=sum): ctype = get_c_type(a.type) @@ -461,15 +451,10 @@ class Obj2ModVisitor(PickleVisitor): self.emit("PyObject* tmp = NULL;", 1) for f in prod.fields: self.visitFieldDeclaration(f, name, prod=prod, depth=1) - for a in prod.attributes: - self.visitFieldDeclaration(a, name, prod=prod, depth=1) self.emit("", 0) for f in prod.fields: self.visitField(f, name, prod=prod, depth=1) - for a in prod.attributes: - self.visitField(a, name, prod=prod, depth=1) - args = [f.name for f in prod.fields] - args.extend([a.name for a in prod.attributes]) + args = [f.name.value for f in prod.fields] self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1) self.emit("return 0;", 1) self.emit("failed:", 0) @@ -491,8 +476,8 @@ class Obj2ModVisitor(PickleVisitor): def isSimpleSum(self, field): # XXX can the members of this list be determined automatically? - return field.type in ('expr_context', 'boolop', 'operator', - 'unaryop', 'cmpop') + return field.type.value in ('expr_context', 'boolop', 'operator', + 'unaryop', 'cmpop') def isNumeric(self, field): return get_c_type(field.type) in ("int", "bool") @@ -502,52 +487,31 @@ class Obj2ModVisitor(PickleVisitor): def visitField(self, field, name, sum=None, prod=None, depth=0): ctype = get_c_type(field.type) - line = "if (_PyObject_LookupAttr(obj, astmodulestate_global->%s, &tmp) < 0) {" - self.emit(line % field.name, depth) - self.emit("return 1;", depth+1) - self.emit("}", depth) - if not field.opt: - self.emit("if (tmp == NULL) {", depth) - message = "required field \\\"%s\\\" missing from %s" % (field.name, name) - format = "PyErr_SetString(PyExc_TypeError, \"%s\");" - self.emit(format % message, depth+1, reflow=False) - self.emit("return 1;", depth+1) - else: - self.emit("if (tmp == NULL || tmp == Py_None) {", depth) - self.emit("Py_CLEAR(tmp);", depth+1) - if self.isNumeric(field): - self.emit("%s = 0;" % field.name, depth+1) - elif not self.isSimpleType(field): - self.emit("%s = NULL;" % field.name, depth+1) - else: - raise TypeError("could not determine the default value for %s" % field.name) - self.emit("}", depth) - self.emit("else {", depth) - + self.emit("if (PyObject_HasAttrString(obj, \"%s\")) {" % field.name, depth) self.emit("int res;", depth+1) if field.seq: self.emit("Py_ssize_t len;", depth+1) self.emit("Py_ssize_t i;", depth+1) + self.emit("tmp = PyObject_GetAttrString(obj, \"%s\");" % field.name, depth+1) + self.emit("if (tmp == NULL) goto failed;", depth+1) + if field.seq: self.emit("if (!PyList_Check(tmp)) {", depth+1) self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must " - "be a list, not a %%.200s\", _PyType_Name(Py_TYPE(tmp)));" % + "be a list, not a %%.200s\", tmp->ob_type->tp_name);" % (name, field.name), depth+2, reflow=False) self.emit("goto failed;", depth+2) self.emit("}", depth+1) self.emit("len = PyList_GET_SIZE(tmp);", depth+1) if self.isSimpleType(field): - self.emit("%s = _Py_asdl_int_seq_new(len, arena);" % field.name, depth+1) + self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1) else: - self.emit("%s = _Py_asdl_seq_new(len, arena);" % field.name, depth+1) + self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1) self.emit("if (%s == NULL) goto failed;" % field.name, depth+1) self.emit("for (i = 0; i < len; i++) {", depth+1) self.emit("%s val;" % ctype, depth+2) - self.emit("PyObject *tmp2 = PyList_GET_ITEM(tmp, i);", depth+2) - self.emit("Py_INCREF(tmp2);", depth+2) - self.emit("res = obj2ast_%s(tmp2, &val, arena);" % + self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &val, arena);" % field.type, depth+2, reflow=False) - self.emit("Py_DECREF(tmp2);", depth+2) self.emit("if (res != 0) goto failed;", depth+2) self.emit("if (len != PyList_GET_SIZE(tmp)) {", depth+2) self.emit("PyErr_SetString(PyExc_RuntimeError, \"%s field \\\"%s\\\" " @@ -563,7 +527,21 @@ class Obj2ModVisitor(PickleVisitor): (field.type, field.name), depth+1) self.emit("if (res != 0) goto failed;", depth+1) - self.emit("Py_CLEAR(tmp);", depth+1) + self.emit("Py_XDECREF(tmp);", depth+1) + self.emit("tmp = NULL;", depth+1) + self.emit("} else {", depth) + if not field.opt: + message = "required field \\\"%s\\\" missing from %s" % (field.name, name) + format = "PyErr_SetString(PyExc_TypeError, \"%s\");" + self.emit(format % message, depth+1, reflow=False) + self.emit("return 1;", depth+1) + else: + if self.isNumeric(field): + self.emit("%s = 0;" % field.name, depth+1) + elif not self.isSimpleType(field): + self.emit("%s = NULL;" % field.name, depth+1) + else: + raise TypeError("could not determine the default value for %s" % field.name) self.emit("}", depth) @@ -580,46 +558,37 @@ class MarshalPrototypeVisitor(PickleVisitor): class PyTypesDeclareVisitor(PickleVisitor): def visitProduct(self, prod, name): - self.emit_type("%s_type" % name) + self.emit("static PyTypeObject *%s_type;" % name, 0) self.emit("static PyObject* ast2obj_%s(void*);" % name, 0) - if prod.attributes: - for a in prod.attributes: - self.emit_identifier(a.name) - self.emit("static const char * const %s_attributes[] = {" % name, 0) - for a in prod.attributes: - self.emit('"%s",' % a.name, 1) - self.emit("};", 0) if prod.fields: - for f in prod.fields: - self.emit_identifier(f.name) - self.emit("static const char * const %s_fields[]={" % name,0) + self.emit("static char *%s_fields[]={" % name,0) for f in prod.fields: self.emit('"%s",' % f.name, 1) self.emit("};", 0) def visitSum(self, sum, name): - self.emit_type("%s_type" % name) + self.emit("static PyTypeObject *%s_type;" % name, 0) if sum.attributes: - for a in sum.attributes: - self.emit_identifier(a.name) - self.emit("static const char * const %s_attributes[] = {" % name, 0) + self.emit("static char *%s_attributes[] = {" % name, 0) for a in sum.attributes: self.emit('"%s",' % a.name, 1) self.emit("};", 0) ptype = "void*" if is_simple(sum): ptype = get_c_type(name) + tnames = [] for t in sum.types: - self.emit_singleton("%s_singleton" % t.name) + tnames.append(str(t.name)+"_singleton") + tnames = ", *".join(tnames) + self.emit("static PyObject *%s;" % tnames, 0) self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0) for t in sum.types: self.visitConstructor(t, name) def visitConstructor(self, cons, name): + self.emit("static PyTypeObject *%s_type;" % cons.name, 0) if cons.fields: - for t in cons.fields: - self.emit_identifier(t.name) - self.emit("static const char * const %s_fields[]={" % cons.name, 0) + self.emit("static char *%s_fields[]={" % cons.name, 0) for t in cons.fields: self.emit('"%s",' % t.name, 1) self.emit("};",0) @@ -628,74 +597,43 @@ class PyTypesVisitor(PickleVisitor): def visitModule(self, mod): self.emit(""" - -typedef struct { - PyObject_HEAD - PyObject *dict; -} AST_object; - -static void -ast_dealloc(AST_object *self) -{ - /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - Py_CLEAR(self->dict); - freefunc free_func = PyType_GetSlot(tp, Py_tp_free); - assert(free_func != NULL); - free_func(self); - Py_DECREF(tp); -} - -static int -ast_traverse(AST_object *self, visitproc visit, void *arg) -{ - Py_VISIT(self->dict); - return 0; -} - -static int -ast_clear(AST_object *self) -{ - Py_CLEAR(self->dict); - return 0; -} - static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields; - if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), astmodulestate_global->_fields, &fields) < 0) { - goto cleanup; - } + fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields"); + if (!fields) + PyErr_Clear(); if (fields) { numfields = PySequence_Size(fields); if (numfields == -1) goto cleanup; } - res = 0; /* if no error occurs, this stays 0 to the end */ - if (numfields < PyTuple_GET_SIZE(args)) { - PyErr_Format(PyExc_TypeError, "%.400s constructor takes at most " - "%zd positional argument%s", - _PyType_Name(Py_TYPE(self)), - numfields, numfields == 1 ? "" : "s"); - res = -1; - goto cleanup; - } - for (i = 0; i < PyTuple_GET_SIZE(args); i++) { - /* cannot be reached when fields is NULL */ - PyObject *name = PySequence_GetItem(fields, i); - if (!name) { + if (PyTuple_GET_SIZE(args) > 0) { + if (numfields != PyTuple_GET_SIZE(args)) { + PyErr_Format(PyExc_TypeError, "%.400s constructor takes %s" + "%zd positional argument%s", + Py_TYPE(self)->tp_name, + numfields == 0 ? "" : "either 0 or ", + numfields, numfields == 1 ? "" : "s"); res = -1; goto cleanup; } - res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); - Py_DECREF(name); - if (res < 0) - goto cleanup; + for (i = 0; i < PyTuple_GET_SIZE(args); i++) { + /* cannot be reached when fields is NULL */ + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + res = -1; + goto cleanup; + } + res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + Py_DECREF(name); + if (res < 0) + goto cleanup; + } } if (kw) { i = 0; /* needed by PyDict_Next */ @@ -714,95 +652,105 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { - PyObject *dict; - if (_PyObject_LookupAttr(self, astmodulestate_global->__dict__, &dict) < 0) { - return NULL; + PyObject *res; + PyObject *dict = PyObject_GetAttrString(self, "__dict__"); + if (dict == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) + PyErr_Clear(); + else + return NULL; } if (dict) { - return Py_BuildValue("O()N", Py_TYPE(self), dict); + res = Py_BuildValue("O()O", Py_TYPE(self), dict); + Py_DECREF(dict); + return res; } return Py_BuildValue("O()", Py_TYPE(self)); } -static PyMemberDef ast_type_members[] = { - {"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY}, - {NULL} /* Sentinel */ -}; - static PyMethodDef ast_type_methods[] = { {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, {NULL} }; -static PyGetSetDef ast_type_getsets[] = { - {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, - {NULL} -}; - -static PyType_Slot AST_type_slots[] = { - {Py_tp_dealloc, ast_dealloc}, - {Py_tp_getattro, PyObject_GenericGetAttr}, - {Py_tp_setattro, PyObject_GenericSetAttr}, - {Py_tp_traverse, ast_traverse}, - {Py_tp_clear, ast_clear}, - {Py_tp_members, ast_type_members}, - {Py_tp_methods, ast_type_methods}, - {Py_tp_getset, ast_type_getsets}, - {Py_tp_init, ast_type_init}, - {Py_tp_alloc, PyType_GenericAlloc}, - {Py_tp_new, PyType_GenericNew}, - {Py_tp_free, PyObject_GC_Del}, - {0, 0}, -}; - -static PyType_Spec AST_type_spec = { +static PyTypeObject AST_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) "_ast.AST", - sizeof(AST_object), + sizeof(PyObject), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - AST_type_slots + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + PyObject_GenericSetAttr, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ast_type_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)ast_type_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + PyObject_Del, /* tp_free */ }; -static PyObject * -make_type(const char *type, PyObject* base, const char* const* fields, int num_fields) + +static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields) { PyObject *fnames, *result; int i; fnames = PyTuple_New(num_fields); if (!fnames) return NULL; for (i = 0; i < num_fields; i++) { - PyObject *field = PyUnicode_InternFromString(fields[i]); + PyObject *field = PyString_FromString(fields[i]); if (!field) { Py_DECREF(fnames); return NULL; } PyTuple_SET_ITEM(fnames, i, field); } - result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOO}", - type, base, - astmodulestate_global->_fields, fnames, - astmodulestate_global->__module__, - astmodulestate_global->_ast); + result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}", + type, base, "_fields", fnames, "__module__", "_ast"); Py_DECREF(fnames); - return result; + return (PyTypeObject*)result; } -static int -add_attributes(PyObject *type, const char * const *attrs, int num_fields) +static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) { int i, result; PyObject *s, *l = PyTuple_New(num_fields); if (!l) return 0; for (i = 0; i < num_fields; i++) { - s = PyUnicode_InternFromString(attrs[i]); + s = PyString_FromString(attrs[i]); if (!s) { Py_DECREF(l); return 0; } PyTuple_SET_ITEM(l, i, s); } - result = PyObject_SetAttr(type, astmodulestate_global->_attributes, l) >= 0; + result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0; Py_DECREF(l); return result; } @@ -811,7 +759,7 @@ add_attributes(PyObject *type, const char * const *attrs, int num_fields) static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*)) { - Py_ssize_t i, n = asdl_seq_LEN(seq); + int i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); PyObject *value; if (!result) @@ -834,15 +782,16 @@ static PyObject* ast2obj_object(void *o) Py_INCREF((PyObject*)o); return (PyObject*)o; } -#define ast2obj_singleton ast2obj_object -#define ast2obj_constant ast2obj_object #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -#define ast2obj_bytes ast2obj_object +static PyObject* ast2obj_bool(bool b) +{ + return PyBool_FromLong(b); +} static PyObject* ast2obj_int(long b) { - return PyLong_FromLong(b); + return PyInt_FromLong(b); } /* Conversion Python -> AST */ @@ -851,32 +800,18 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) { if (obj == Py_None) obj = NULL; - if (obj) { - if (PyArena_AddPyObject(arena, obj) < 0) { - *out = NULL; - return -1; - } - Py_INCREF(obj); - } - *out = obj; - return 0; -} - -static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) -{ - if (PyArena_AddPyObject(arena, obj) < 0) { - *out = NULL; - return -1; - } - Py_INCREF(obj); + if (obj) + PyArena_AddPyObject(arena, obj); + Py_XINCREF(obj); *out = obj; return 0; } static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) { - if (!PyUnicode_CheckExact(obj) && obj != Py_None) { - PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); + if (!PyString_CheckExact(obj) && obj != Py_None) { + PyErr_Format(PyExc_TypeError, + "AST identifier must be of type str"); return 1; } return obj2ast_object(obj, out, arena); @@ -884,8 +819,9 @@ static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) { - if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { - PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); + if (!PyString_CheckExact(obj) && !PyUnicode_CheckExact(obj)) { + PyErr_SetString(PyExc_TypeError, + "AST string must be of type str or unicode"); return 1; } return obj2ast_object(obj, out, arena); @@ -894,25 +830,47 @@ static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) { int i; - if (!PyLong_Check(obj)) { - PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj); + if (!_PyAnyInt_Check(obj)) { + PyObject *s = PyObject_Repr(obj); + if (s == NULL) return 1; + PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s", + PyString_AS_STRING(s)); + Py_DECREF(s); return 1; } - i = _PyLong_AsInt(obj); + i = (int)PyLong_AsLong(obj); if (i == -1 && PyErr_Occurred()) return 1; *out = i; return 0; } +static int obj2ast_bool(PyObject* obj, bool* out, PyArena* arena) +{ + if (!PyBool_Check(obj)) { + PyObject *s = PyObject_Repr(obj); + if (s == NULL) return 1; + PyErr_Format(PyExc_ValueError, "invalid boolean value: %.400s", + PyString_AS_STRING(s)); + Py_DECREF(s); + return 1; + } + + *out = (obj == Py_True); + return 0; +} + static int add_ast_fields(void) { - PyObject *empty_tuple; + PyObject *empty_tuple, *d; + if (PyType_Ready(&AST_type) < 0) + return -1; + d = AST_type.tp_dict; empty_tuple = PyTuple_New(0); if (!empty_tuple || - PyObject_SetAttrString(astmodulestate_global->AST_type, "_fields", empty_tuple) < 0 || - PyObject_SetAttrString(astmodulestate_global->AST_type, "_attributes", empty_tuple) < 0) { + PyDict_SetItemString(d, "_fields", empty_tuple) < 0 || + PyDict_SetItemString(d, "_attributes", empty_tuple) < 0) { Py_XDECREF(empty_tuple); return -1; } @@ -924,91 +882,71 @@ static int add_ast_fields(void) self.emit("static int init_types(void)",0) self.emit("{", 0) - self.emit("PyObject *m;", 1) - self.emit("if (PyState_FindModule(&_astmodule) == NULL) {", 1) - self.emit("m = PyModule_Create(&_astmodule);", 2) - self.emit("if (!m) return 0;", 2) - self.emit("PyState_AddModule(m, &_astmodule);", 2) - self.emit("}", 1) - self.emit("astmodulestate *state = astmodulestate_global;", 1) - self.emit("if (state->initialized) return 1;", 1) - self.emit("if (init_identifiers() < 0) return 0;", 1) - self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1) - self.emit("if (!state->AST_type) return 0;", 1) + self.emit("static int initialized;", 1) + self.emit("if (initialized) return 1;", 1) self.emit("if (add_ast_fields() < 0) return 0;", 1) for dfn in mod.dfns: self.visit(dfn) - self.emit("state->initialized = 1;", 1) + self.emit("initialized = 1;", 1) self.emit("return 1;", 1); self.emit("}", 0) def visitProduct(self, prod, name): if prod.fields: - fields = name+"_fields" + fields = name.value+"_fields" else: fields = "NULL" - self.emit('state->%s_type = make_type("%s", state->AST_type, %s, %d);' % + self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' % (name, name, fields, len(prod.fields)), 1) - self.emit("if (!state->%s_type) return 0;" % name, 1) - self.emit_type("AST_type") - self.emit_type("%s_type" % name) - if prod.attributes: - self.emit("if (!add_attributes(state->%s_type, %s_attributes, %d)) return 0;" % - (name, name, len(prod.attributes)), 1) - else: - self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1) + self.emit("if (!%s_type) return 0;" % name, 1) def visitSum(self, sum, name): - self.emit('state->%s_type = make_type("%s", state->AST_type, NULL, 0);' % + self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' % (name, name), 1) - self.emit_type("%s_type" % name) - self.emit("if (!state->%s_type) return 0;" % name, 1) + self.emit("if (!%s_type) return 0;" % name, 1) if sum.attributes: - self.emit("if (!add_attributes(state->%s_type, %s_attributes, %d)) return 0;" % + self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" % (name, name, len(sum.attributes)), 1) else: - self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1) + self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1) simple = is_simple(sum) for t in sum.types: self.visitConstructor(t, name, simple) def visitConstructor(self, cons, name, simple): if cons.fields: - fields = cons.name+"_fields" + fields = cons.name.value+"_fields" else: fields = "NULL" - self.emit('state->%s_type = make_type("%s", state->%s_type, %s, %d);' % + self.emit('%s_type = make_type("%s", %s_type, %s, %d);' % (cons.name, cons.name, name, fields, len(cons.fields)), 1) - self.emit("if (!state->%s_type) return 0;" % cons.name, 1) - self.emit_type("%s_type" % cons.name) + self.emit("if (!%s_type) return 0;" % cons.name, 1) if simple: - self.emit("state->%s_singleton = PyType_GenericNew((PyTypeObject *)" - "state->%s_type, NULL, NULL);" % + self.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" % (cons.name, cons.name), 1) - self.emit("if (!state->%s_singleton) return 0;" % cons.name, 1) + self.emit("if (!%s_singleton) return 0;" % cons.name, 1) class ASTModuleVisitor(PickleVisitor): def visitModule(self, mod): self.emit("PyMODINIT_FUNC", 0) - self.emit("PyInit__ast(void)", 0) + self.emit("init_ast(void)", 0) self.emit("{", 0) - self.emit("PyObject *m;", 1) - self.emit("if (!init_types()) return NULL;", 1) - self.emit('m = PyState_FindModule(&_astmodule);', 1) - self.emit("if (!m) return NULL;", 1) - self.emit('Py_INCREF(astmodulestate(m)->AST_type);', 1) - self.emit('if (PyModule_AddObject(m, "AST", astmodulestate_global->AST_type) < 0) return NULL;', 1) - self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)', 1) - self.emit("return NULL;", 2) - self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1) - self.emit("return NULL;", 2) - self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)', 1) - self.emit("return NULL;", 2) + self.emit("PyObject *m, *d;", 1) + self.emit("if (!init_types()) return;", 1) + self.emit('m = Py_InitModule3("_ast", NULL, NULL);', 1) + self.emit("if (!m) return;", 1) + self.emit("d = PyModule_GetDict(m);", 1) + self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return;', 1) + self.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1) + self.emit("return;", 2) + # Value of version: "$Revision$" + self.emit('if (PyModule_AddStringConstant(m, "__version__", "%s") < 0)' + % mod.version, 1) + self.emit("return;", 2) for dfn in mod.dfns: self.visit(dfn) - self.emit("return m;", 1) self.emit("}", 0) def visitProduct(self, prod, name): @@ -1023,9 +961,7 @@ class ASTModuleVisitor(PickleVisitor): self.addObj(cons.name) def addObj(self, name): - self.emit("if (PyModule_AddObject(m, \"%s\", " - "astmodulestate_global->%s_type) < 0) return NULL;" % (name, name), 1) - self.emit("Py_INCREF(astmodulestate(m)->%s_type);" % name, 1) + self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1) _SPECIALIZED_SEQUENCES = ('stmt', 'expr') @@ -1063,9 +999,9 @@ class ObjVisitor(PickleVisitor): self.emit("{", 0) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("PyObject *result = NULL, *value = NULL;", 1) - self.emit("PyTypeObject *tp;", 1) self.emit('if (!o) {', 1) - self.emit("Py_RETURN_NONE;", 2) + self.emit("Py_INCREF(Py_None);", 2) + self.emit('return Py_None;', 2) self.emit("}", 1) self.emit('', 0) @@ -1091,7 +1027,7 @@ class ObjVisitor(PickleVisitor): for a in sum.attributes: self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1) self.emit("if (!value) goto failed;", 1) - self.emit('if (PyObject_SetAttr(result, astmodulestate_global->%s, value) < 0)' % a.name, 1) + self.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a.name, 1) self.emit('goto failed;', 2) self.emit('Py_DECREF(value);', 1) self.func_end() @@ -1102,8 +1038,8 @@ class ObjVisitor(PickleVisitor): self.emit("switch(o) {", 1) for t in sum.types: self.emit("case %s:" % t.name, 2) - self.emit("Py_INCREF(astmodulestate_global->%s_singleton);" % t.name, 3) - self.emit("return astmodulestate_global->%s_singleton;" % t.name, 3) + self.emit("Py_INCREF(%s_singleton);" % t.name, 3) + self.emit("return %s_singleton;" % t.name, 3) self.emit("default:", 2) self.emit('/* should never happen, but just in case ... */', 3) code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name @@ -1114,23 +1050,15 @@ class ObjVisitor(PickleVisitor): def visitProduct(self, prod, name): self.func_begin(name) - self.emit("tp = (PyTypeObject *)astmodulestate_global->%s_type;" % name, 1) - self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 1); + self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1); self.emit("if (!result) return NULL;", 1) for field in prod.fields: self.visitField(field, name, 1, True) - for a in prod.attributes: - self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1) - self.emit("if (!value) goto failed;", 1) - self.emit("if (PyObject_SetAttr(result, astmodulestate_global->%s, value) < 0)" % a.name, 1) - self.emit('goto failed;', 2) - self.emit('Py_DECREF(value);', 1) self.func_end() def visitConstructor(self, cons, enum, name): self.emit("case %s_kind:" % cons.name, 1) - self.emit("tp = (PyTypeObject *)astmodulestate_global->%s_type;" % cons.name, 2) - self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 2); + self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2); self.emit("if (!result) goto failed;", 2) for f in cons.fields: self.visitField(f, cons.name, 2, False) @@ -1145,7 +1073,7 @@ class ObjVisitor(PickleVisitor): value = "o->v.%s.%s" % (name, field.name) self.set(field, value, depth) emit("if (!value) goto failed;", 0) - emit("if (PyObject_SetAttr(result, astmodulestate_global->%s, value) == -1)" % field.name, 0) + emit('if (PyObject_SetAttrString(result, "%s", value) == -1)' % field.name, 0) emit("goto failed;", 1) emit("Py_DECREF(value);", 0) @@ -1164,11 +1092,11 @@ class ObjVisitor(PickleVisitor): def set(self, field, value, depth): if field.seq: # XXX should really check for is_simple, but that requires a symbol table - if field.type == "cmpop": + if field.type.value == "cmpop": # While the sequence elements are stored as void*, # ast2obj_cmpop expects an enum self.emit("{", depth) - self.emit("Py_ssize_t i, n = asdl_seq_LEN(%s);" % value, depth+1) + self.emit("int i, n = asdl_seq_LEN(%s);" % value, depth+1) self.emit("value = PyList_New(n);", depth+1) self.emit("if (!value) goto failed;", depth+1) self.emit("for(i = 0; i < n; i++)", depth+1) @@ -1188,41 +1116,38 @@ class PartingShots(StaticVisitor): CODE = """ PyObject* PyAST_mod2obj(mod_ty t) { - if (!init_types()) - return NULL; + init_types(); return ast2obj_mod(t); } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) { + mod_ty res; PyObject *req_type[3]; - const char * const req_name[] = {"Module", "Expression", "Interactive"}; + char *req_name[3]; int isinstance; - if (PySys_Audit("compile", "OO", ast, Py_None) < 0) { - return NULL; - } + req_type[0] = (PyObject*)Module_type; + req_type[1] = (PyObject*)Expression_type; + req_type[2] = (PyObject*)Interactive_type; - req_type[0] = astmodulestate_global->Module_type; - req_type[1] = astmodulestate_global->Expression_type; - req_type[2] = astmodulestate_global->Interactive_type; + req_name[0] = "Module"; + req_name[1] = "Expression"; + req_name[2] = "Interactive"; assert(0 <= mode && mode <= 2); - if (!init_types()) - return NULL; + init_types(); isinstance = PyObject_IsInstance(ast, req_type[mode]); if (isinstance == -1) return NULL; if (!isinstance) { PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s", - req_name[mode], _PyType_Name(Py_TYPE(ast))); + req_name[mode], Py_TYPE(ast)->tp_name); return NULL; } - - mod_ty res = NULL; if (obj2ast_mod(ast, &res, arena) != 0) return NULL; else @@ -1231,9 +1156,8 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { - if (!init_types()) - return -1; - return PyObject_IsInstance(obj, astmodulestate_global->AST_type); + init_types(); + return PyObject_IsInstance(obj, (PyObject*)&AST_type); } """ @@ -1246,172 +1170,81 @@ class ChainOfVisitors: v.visit(object) v.emit("", 0) +common_msg = "/* File automatically generated by %s. */\n\n" -def generate_module_def(f, mod): - # Gather all the data needed for ModuleSpec - visitor_list = set() - with open(os.devnull, "w") as devnull: - visitor = PyTypesDeclareVisitor(devnull) - visitor.visit(mod) - visitor_list.add(visitor) - visitor = PyTypesVisitor(devnull) - visitor.visit(mod) - visitor_list.add(visitor) - - state_strings = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"]) - module_state = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"]) - for visitor in visitor_list: - for identifier in visitor.identifiers: - module_state.add(identifier) - state_strings.add(identifier) - for singleton in visitor.singletons: - module_state.add(singleton) - for tp in visitor.types: - module_state.add(tp) - state_strings = sorted(state_strings) - module_state = sorted(module_state) - f.write('typedef struct {\n') - f.write(' int initialized;\n') - for s in module_state: - f.write(' PyObject *' + s + ';\n') - f.write('} astmodulestate;\n\n') - f.write(""" -#define astmodulestate(o) ((astmodulestate *)PyModule_GetState(o)) - -static int astmodule_clear(PyObject *module) -{ -""") - for s in module_state: - f.write(" Py_CLEAR(astmodulestate(module)->" + s + ');\n') - f.write(""" - return 0; -} - -static int astmodule_traverse(PyObject *module, visitproc visit, void* arg) -{ -""") - for s in module_state: - f.write(" Py_VISIT(astmodulestate(module)->" + s + ');\n') - f.write(""" - return 0; -} - -static void astmodule_free(void* module) { - astmodule_clear((PyObject*)module); -} - -static struct PyModuleDef _astmodule = { - PyModuleDef_HEAD_INIT, - "_ast", - NULL, - sizeof(astmodulestate), - NULL, - NULL, - astmodule_traverse, - astmodule_clear, - astmodule_free, -}; - -#define astmodulestate_global ((astmodulestate *)PyModule_GetState(PyState_FindModule(&_astmodule))) - -""") - f.write('static int init_identifiers(void)\n') - f.write('{\n') - f.write(' astmodulestate *state = astmodulestate_global;\n') - for identifier in state_strings: - f.write(' if ((state->' + identifier) - f.write(' = PyUnicode_InternFromString("') - f.write(identifier + '")) == NULL) return 0;\n') - f.write(' return 1;\n') - f.write('};\n\n') +c_file_msg = """ +/* + __version__ %s. + This module must be committed separately after each AST grammar change; + The __version__ number is set to the revision number of the commit + containing the grammar change. +*/ -common_msg = "/* File automatically generated by %s. */\n\n" +""" -def main(srcfile, dump_module=False): +def main(srcfile): argv0 = sys.argv[0] components = argv0.split(os.sep) argv0 = os.sep.join(components[-2:]) auto_gen_msg = common_msg % argv0 mod = asdl.parse(srcfile) - if dump_module: - print('Parsed Module:') - print(mod) + mod.version = "82160" if not asdl.check(mod): sys.exit(1) - if H_FILE: - with open(H_FILE, "w") as f: - f.write(auto_gen_msg) - f.write('#ifndef Py_PYTHON_AST_H\n') - f.write('#define Py_PYTHON_AST_H\n') - f.write('#ifdef __cplusplus\n') - f.write('extern "C" {\n') - f.write('#endif\n') - f.write('\n') - f.write('#ifndef Py_LIMITED_API\n') - f.write('#include "asdl.h"\n') - f.write('\n') - f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n') - f.write('\n') - c = ChainOfVisitors(TypeDefVisitor(f), - StructVisitor(f)) - - c.visit(mod) - f.write("// Note: these macros affect function definitions, not only call sites.\n") - PrototypeVisitor(f).visit(mod) - f.write("\n") - f.write("PyObject* PyAST_mod2obj(mod_ty t);\n") - f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n") - f.write("int PyAST_Check(PyObject* obj);\n") - f.write("#endif /* !Py_LIMITED_API */\n") - f.write('\n') - f.write('#ifdef __cplusplus\n') - f.write('}\n') - f.write('#endif\n') - f.write('#endif /* !Py_PYTHON_AST_H */\n') - - if C_FILE: - with open(C_FILE, "w") as f: - f.write(auto_gen_msg) - f.write('#include <stddef.h>\n') - f.write('\n') - f.write('#include "Python.h"\n') - f.write('#include "%s-ast.h"\n' % mod.name) - f.write('#include "structmember.h"\n') - f.write('\n') - - generate_module_def(f, mod) - - v = ChainOfVisitors( - PyTypesDeclareVisitor(f), - PyTypesVisitor(f), - Obj2ModPrototypeVisitor(f), - FunctionVisitor(f), - ObjVisitor(f), - Obj2ModVisitor(f), - ASTModuleVisitor(f), - PartingShots(f), - ) - v.visit(mod) + if INC_DIR: + p = "%s/%s-ast.h" % (INC_DIR, mod.name) + f = open(p, "wb") + f.write(auto_gen_msg) + f.write('#include "asdl.h"\n\n') + c = ChainOfVisitors(TypeDefVisitor(f), + StructVisitor(f), + PrototypeVisitor(f), + ) + c.visit(mod) + f.write("PyObject* PyAST_mod2obj(mod_ty t);\n") + f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n") + f.write("int PyAST_Check(PyObject* obj);\n") + f.close() + + if SRC_DIR: + p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c") + f = open(p, "wb") + f.write(auto_gen_msg) + f.write(c_file_msg % mod.version) + f.write('#include "Python.h"\n') + f.write('#include "%s-ast.h"\n' % mod.name) + f.write('\n') + f.write("static PyTypeObject AST_type;\n") + v = ChainOfVisitors( + PyTypesDeclareVisitor(f), + PyTypesVisitor(f), + Obj2ModPrototypeVisitor(f), + FunctionVisitor(f), + ObjVisitor(f), + Obj2ModVisitor(f), + ASTModuleVisitor(f), + PartingShots(f), + ) + v.visit(mod) + f.close() if __name__ == "__main__": + import sys import getopt - H_FILE = '' - C_FILE = '' - dump_module = False - opts, args = getopt.getopt(sys.argv[1:], "dh:c:") + INC_DIR = '' + SRC_DIR = '' + opts, args = getopt.getopt(sys.argv[1:], "h:c:") + if len(opts) != 1: + print "Must specify exactly one output file" + sys.exit(1) for o, v in opts: if o == '-h': - H_FILE = v - elif o == '-c': - C_FILE = v - elif o == '-d': - dump_module = True - if H_FILE and C_FILE: - print('Must specify exactly one output file') - sys.exit(1) - elif len(args) != 1: - print('Must specify single input file') + INC_DIR = v + if o == '-c': + SRC_DIR = v + if len(args) != 1: + print "Must specify single input file" sys.exit(1) - main(args[0], dump_module) + main(args[0]) |