diff options
Diffstat (limited to 'Parser')
| -rw-r--r-- | Parser/Python.asdl | 12 | ||||
| -rw-r--r-- | Parser/asdl.py | 20 | ||||
| -rwxr-xr-x | Parser/asdl_c.py | 65 | ||||
| -rw-r--r-- | Parser/grammar1.c | 2 | ||||
| -rw-r--r-- | Parser/myreadline.c | 2 | ||||
| -rw-r--r-- | Parser/parsetok.c | 2 | ||||
| -rw-r--r-- | Parser/pgenmain.c | 1 | ||||
| -rw-r--r-- | Parser/tokenizer.c | 5 |
8 files changed, 88 insertions, 21 deletions
diff --git a/Parser/Python.asdl b/Parser/Python.asdl index c24d840..debd89e 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -1,4 +1,4 @@ --- ASDL's five builtin types are identifier, int, string, bytes, object +-- ASDL's six builtin types are identifier, int, string, bytes, object, singleton module Python { @@ -69,8 +69,8 @@ module Python | Num(object n) -- a number as a PyObject. | Str(string s) -- need to specify raw, unicode, etc? | Bytes(bytes s) + | NameConstant(singleton value) | Ellipsis - -- other literals? bools? -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) @@ -103,11 +103,11 @@ module Python excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) attributes (int lineno, int col_offset) - arguments = (arg* args, identifier? vararg, expr? varargannotation, - arg* kwonlyargs, identifier? kwarg, - expr? kwargannotation, expr* defaults, - expr* kw_defaults) + arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, + arg? kwarg, expr* defaults) + arg = (identifier arg, expr? annotation) + attributes (int lineno, int col_offset) -- keyword arguments supplied to call keyword = (identifier arg, expr value) diff --git a/Parser/asdl.py b/Parser/asdl.py index 08aa05b..7df76c0 100644 --- a/Parser/asdl.py +++ b/Parser/asdl.py @@ -158,11 +158,19 @@ class ASDLParser(spark.GenericParser, object): msg="expected attributes, found %s" % id) return Sum(sum, attributes) - def p_product(self, info): + def p_product_0(self, info): " product ::= ( fields ) " _0, fields, _1 = info return Product(fields) + def p_product_1(self, info): + " product ::= ( fields ) Id ( fields ) " + _0, fields, _1, id, _2, attributes, _3 = info + if id.value != "attributes": + raise ASDLSyntaxError(id.lineno, + msg="expected attributes, found %s" % id) + return Product(fields, attributes) + def p_sum_0(self, constructor): " sum ::= constructor " return [constructor[0]] @@ -222,7 +230,7 @@ class ASDLParser(spark.GenericParser, object): " field ::= Id ? " return Field(type[0], opt=True) -builtin_types = ("identifier", "string", "bytes", "int", "object") +builtin_types = ("identifier", "string", "bytes", "int", "object", "singleton") # below is a collection of classes to capture the AST of an AST :-) # not sure if any of the methods are useful yet, but I'm adding them @@ -289,11 +297,15 @@ class Sum(AST): return "Sum(%s, %s)" % (self.types, self.attributes) class Product(AST): - def __init__(self, fields): + def __init__(self, fields, attributes=None): self.fields = fields + self.attributes = attributes or [] def __repr__(self): - return "Product(%s)" % self.fields + if self.attributes is None: + return "Product(%s)" % self.fields + else: + return "Product(%s, %s)" % (self.fields, self.attributes) class VisitorBase(object): diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 8f0da4a..498531d 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -209,6 +209,11 @@ class StructVisitor(EmitVisitor): self.emit("struct _%(name)s {" % locals(), depth) for f in product.fields: self.visit(f, depth + 1) + for field in product.attributes: + # rudimentary attribute handling + type = str(field.type) + assert type in asdl.builtin_types, type + self.emit("%s %s;" % (type, field.name), depth + 1); self.emit("};", depth) self.emit("", depth) @@ -493,7 +498,11 @@ class Obj2ModVisitor(PickleVisitor): def visitField(self, field, name, sum=None, prod=None, depth=0): ctype = get_c_type(field.type) - self.emit("if (_PyObject_HasAttrId(obj, &PyId_%s)) {" % field.name, depth) + if field.opt: + check = "exists_not_none(obj, &PyId_%s)" % (field.name,) + else: + check = "_PyObject_HasAttrId(obj, &PyId_%s)" % (field.name,) + self.emit("if (%s) {" % (check,), depth, reflow=False) self.emit("int res;", depth+1) if field.seq: self.emit("Py_ssize_t len;", depth+1) @@ -559,6 +568,13 @@ class PyTypesDeclareVisitor(PickleVisitor): def visitProduct(self, prod, name): self.emit("static PyTypeObject *%s_type;" % name, 0) self.emit("static PyObject* ast2obj_%s(void*);" % name, 0) + if prod.attributes: + for a in prod.attributes: + self.emit_identifier(a.name) + self.emit("static char *%s_attributes[] = {" % name, 0) + for a in prod.attributes: + self.emit('"%s",' % a.name, 1) + self.emit("};", 0) if prod.fields: for f in prod.fields: self.emit_identifier(f.name) @@ -820,6 +836,7 @@ static PyObject* ast2obj_object(void *o) Py_INCREF((PyObject*)o); return (PyObject*)o; } +#define ast2obj_singleton ast2obj_object #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object #define ast2obj_bytes ast2obj_object @@ -831,6 +848,17 @@ static PyObject* ast2obj_int(long b) /* Conversion Python -> AST */ +static int obj2ast_singleton(PyObject *obj, PyObject** out, PyArena* arena) +{ + if (obj != Py_None && obj != Py_True && obj != Py_False) { + PyErr_SetString(PyExc_ValueError, + "AST singleton must be True, False, or None"); + return 1; + } + *out = obj; + return 0; +} + static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) { if (obj == Py_None) @@ -901,6 +929,19 @@ static int add_ast_fields(void) return 0; } +static int exists_not_none(PyObject *obj, _Py_Identifier *id) +{ + int isnone; + PyObject *attr = _PyObject_GetAttrId(obj, id); + if (!attr) { + PyErr_Clear(); + return 0; + } + isnone = attr == Py_None; + Py_DECREF(attr); + return !isnone; +} + """, 0, reflow=False) self.emit("static int init_types(void)",0) @@ -922,6 +963,11 @@ static int add_ast_fields(void) self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' % (name, name, fields, len(prod.fields)), 1) self.emit("if (!%s_type) return 0;" % name, 1) + if prod.attributes: + self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" % + (name, name, len(prod.attributes)), 1) + else: + self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1) def visitSum(self, sum, name): self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' % @@ -965,7 +1011,7 @@ class ASTModuleVisitor(PickleVisitor): self.emit("if (!m) return NULL;", 1) self.emit("d = PyModule_GetDict(m);", 1) self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1) - self.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1) + self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1) self.emit("return NULL;", 2) for dfn in mod.dfns: self.visit(dfn) @@ -1077,6 +1123,12 @@ class ObjVisitor(PickleVisitor): self.emit("if (!result) return NULL;", 1) for field in prod.fields: self.visitField(field, name, 1, True) + for a in prod.attributes: + self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1) + self.emit("if (!value) goto failed;", 1) + self.emit('if (_PyObject_SetAttrId(result, &PyId_%s, value) < 0)' % a.name, 1) + self.emit('goto failed;', 2) + self.emit('Py_DECREF(value);', 1) self.func_end() def visitConstructor(self, cons, enum, name): @@ -1139,7 +1191,8 @@ class PartingShots(StaticVisitor): CODE = """ PyObject* PyAST_mod2obj(mod_ty t) { - init_types(); + if (!init_types()) + return NULL; return ast2obj_mod(t); } @@ -1153,7 +1206,8 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int isinstance; assert(0 <= mode && mode <= 2); - init_types(); + if (!init_types()) + return NULL; isinstance = PyObject_IsInstance(ast, req_type[mode]); if (isinstance == -1) @@ -1171,7 +1225,8 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { - init_types(); + if (!init_types()) + return -1; return PyObject_IsInstance(obj, (PyObject*)&AST_type); } """ diff --git a/Parser/grammar1.c b/Parser/grammar1.c index 440ba44..7136463 100644 --- a/Parser/grammar1.c +++ b/Parser/grammar1.c @@ -30,7 +30,7 @@ PyGrammar_FindDFA(grammar *g, register int type) #endif } -char * +const char * PyGrammar_LabelRepr(label *lb) { static char buf[100]; diff --git a/Parser/myreadline.c b/Parser/myreadline.c index d864623..8b27045 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -68,7 +68,7 @@ my_fgets(char *buf, int len, FILE *fp) */ if (GetLastError()==ERROR_OPERATION_ABORTED) { hInterruptEvent = _PyOS_SigintEvent(); - switch (WaitForSingleObject(hInterruptEvent, 10)) { + switch (WaitForSingleObjectEx(hInterruptEvent, 10, FALSE)) { case WAIT_OBJECT_0: ResetEvent(hInterruptEvent); return 1; /* Interrupt */ diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 7beb735..d61ba92 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -138,7 +138,6 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, int started = 0; if ((ps = PyParser_New(g, start)) == NULL) { - fprintf(stderr, "no mem for new parser\n"); err_ret->error = E_NOMEM; PyTokenizer_Free(tok); return NULL; @@ -178,7 +177,6 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, len = b - a; /* XXX this may compute NULL - NULL */ str = (char *) PyObject_MALLOC(len + 1); if (str == NULL) { - fprintf(stderr, "no mem for next token\n"); err_ret->error = E_NOMEM; break; } diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index 52b8380..a624dd2 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -113,6 +113,7 @@ getgrammar(char *filename) Py_Exit(1); } g = pgen(n); + PyNode_Free(n); if (g == NULL) { printf("Bad grammar.\n"); Py_Exit(1); diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5c0bd6e..c4160fc 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -47,7 +47,7 @@ static void tok_backup(struct tok_state *tok, int c); /* Token names */ -char *_PyParser_TokenNames[] = { +const char *_PyParser_TokenNames[] = { "ENDMARKER", "NAME", "NUMBER", @@ -671,7 +671,8 @@ translate_into_utf8(const char* str, const char* enc) { static char * translate_newlines(const char *s, int exec_input, struct tok_state *tok) { - int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length; + int skip_next_lf = 0; + size_t needed_length = strlen(s) + 2, final_length; char *buf, *current; char c = '\0'; buf = PyMem_MALLOC(needed_length); |