diff options
Diffstat (limited to 'Python')
42 files changed, 6299 insertions, 3871 deletions
diff --git a/Python/Python-ast.c b/Python/Python-ast.c index a276b6c..2603b81 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1,14 +1,5 @@ /* File automatically generated by Parser/asdl_c.py. */ - -/* - __version__ 82163. - - This module must be committed separately after each AST grammar change; - The __version__ number is set to the revision number of the commit - containing the grammar change. -*/ - #include "Python.h" #include "Python-ast.h" @@ -16,6 +7,7 @@ static PyTypeObject AST_type; static PyTypeObject *mod_type; static PyObject* ast2obj_mod(void*); static PyTypeObject *Module_type; +_Py_IDENTIFIER(body); static char *Module_fields[]={ "body", }; @@ -32,12 +24,18 @@ static char *Suite_fields[]={ "body", }; static PyTypeObject *stmt_type; +_Py_IDENTIFIER(lineno); +_Py_IDENTIFIER(col_offset); static char *stmt_attributes[] = { "lineno", "col_offset", }; static PyObject* ast2obj_stmt(void*); static PyTypeObject *FunctionDef_type; +_Py_IDENTIFIER(name); +_Py_IDENTIFIER(args); +_Py_IDENTIFIER(decorator_list); +_Py_IDENTIFIER(returns); static char *FunctionDef_fields[]={ "name", "args", @@ -46,6 +44,10 @@ static char *FunctionDef_fields[]={ "returns", }; static PyTypeObject *ClassDef_type; +_Py_IDENTIFIER(bases); +_Py_IDENTIFIER(keywords); +_Py_IDENTIFIER(starargs); +_Py_IDENTIFIER(kwargs); static char *ClassDef_fields[]={ "name", "bases", @@ -56,10 +58,12 @@ static char *ClassDef_fields[]={ "decorator_list", }; static PyTypeObject *Return_type; +_Py_IDENTIFIER(value); static char *Return_fields[]={ "value", }; static PyTypeObject *Delete_type; +_Py_IDENTIFIER(targets); static char *Delete_fields[]={ "targets", }; @@ -69,12 +73,16 @@ static char *Assign_fields[]={ "value", }; static PyTypeObject *AugAssign_type; +_Py_IDENTIFIER(target); +_Py_IDENTIFIER(op); static char *AugAssign_fields[]={ "target", "op", "value", }; static PyTypeObject *For_type; +_Py_IDENTIFIER(iter); +_Py_IDENTIFIER(orelse); static char *For_fields[]={ "target", "iter", @@ -82,6 +90,7 @@ static char *For_fields[]={ "orelse", }; static PyTypeObject *While_type; +_Py_IDENTIFIER(test); static char *While_fields[]={ "test", "body", @@ -94,37 +103,41 @@ static char *If_fields[]={ "orelse", }; static PyTypeObject *With_type; +_Py_IDENTIFIER(items); static char *With_fields[]={ - "context_expr", - "optional_vars", + "items", "body", }; static PyTypeObject *Raise_type; +_Py_IDENTIFIER(exc); +_Py_IDENTIFIER(cause); static char *Raise_fields[]={ "exc", "cause", }; -static PyTypeObject *TryExcept_type; -static char *TryExcept_fields[]={ +static PyTypeObject *Try_type; +_Py_IDENTIFIER(handlers); +_Py_IDENTIFIER(finalbody); +static char *Try_fields[]={ "body", "handlers", "orelse", -}; -static PyTypeObject *TryFinally_type; -static char *TryFinally_fields[]={ - "body", "finalbody", }; static PyTypeObject *Assert_type; +_Py_IDENTIFIER(msg); static char *Assert_fields[]={ "test", "msg", }; static PyTypeObject *Import_type; +_Py_IDENTIFIER(names); static char *Import_fields[]={ "names", }; static PyTypeObject *ImportFrom_type; +_Py_IDENTIFIER(module); +_Py_IDENTIFIER(level); static char *ImportFrom_fields[]={ "module", "names", @@ -152,17 +165,21 @@ static char *expr_attributes[] = { }; static PyObject* ast2obj_expr(void*); static PyTypeObject *BoolOp_type; +_Py_IDENTIFIER(values); static char *BoolOp_fields[]={ "op", "values", }; static PyTypeObject *BinOp_type; +_Py_IDENTIFIER(left); +_Py_IDENTIFIER(right); static char *BinOp_fields[]={ "left", "op", "right", }; static PyTypeObject *UnaryOp_type; +_Py_IDENTIFIER(operand); static char *UnaryOp_fields[]={ "op", "operand", @@ -179,15 +196,19 @@ static char *IfExp_fields[]={ "orelse", }; static PyTypeObject *Dict_type; +_Py_IDENTIFIER(keys); static char *Dict_fields[]={ "keys", "values", }; static PyTypeObject *Set_type; +_Py_IDENTIFIER(elts); static char *Set_fields[]={ "elts", }; static PyTypeObject *ListComp_type; +_Py_IDENTIFIER(elt); +_Py_IDENTIFIER(generators); static char *ListComp_fields[]={ "elt", "generators", @@ -198,6 +219,7 @@ static char *SetComp_fields[]={ "generators", }; static PyTypeObject *DictComp_type; +_Py_IDENTIFIER(key); static char *DictComp_fields[]={ "key", "value", @@ -212,13 +234,20 @@ static PyTypeObject *Yield_type; static char *Yield_fields[]={ "value", }; +static PyTypeObject *YieldFrom_type; +static char *YieldFrom_fields[]={ + "value", +}; static PyTypeObject *Compare_type; +_Py_IDENTIFIER(ops); +_Py_IDENTIFIER(comparators); static char *Compare_fields[]={ "left", "ops", "comparators", }; static PyTypeObject *Call_type; +_Py_IDENTIFIER(func); static char *Call_fields[]={ "func", "args", @@ -227,10 +256,12 @@ static char *Call_fields[]={ "kwargs", }; static PyTypeObject *Num_type; +_Py_IDENTIFIER(n); static char *Num_fields[]={ "n", }; static PyTypeObject *Str_type; +_Py_IDENTIFIER(s); static char *Str_fields[]={ "s", }; @@ -240,12 +271,15 @@ static char *Bytes_fields[]={ }; static PyTypeObject *Ellipsis_type; static PyTypeObject *Attribute_type; +_Py_IDENTIFIER(attr); +_Py_IDENTIFIER(ctx); static char *Attribute_fields[]={ "value", "attr", "ctx", }; static PyTypeObject *Subscript_type; +_Py_IDENTIFIER(slice); static char *Subscript_fields[]={ "value", "slice", @@ -257,6 +291,7 @@ static char *Starred_fields[]={ "ctx", }; static PyTypeObject *Name_type; +_Py_IDENTIFIER(id); static char *Name_fields[]={ "id", "ctx", @@ -284,12 +319,16 @@ static PyTypeObject *Param_type; static PyTypeObject *slice_type; static PyObject* ast2obj_slice(void*); static PyTypeObject *Slice_type; +_Py_IDENTIFIER(lower); +_Py_IDENTIFIER(upper); +_Py_IDENTIFIER(step); static char *Slice_fields[]={ "lower", "upper", "step", }; static PyTypeObject *ExtSlice_type; +_Py_IDENTIFIER(dims); static char *ExtSlice_fields[]={ "dims", }; @@ -345,6 +384,7 @@ static PyTypeObject *In_type; static PyTypeObject *NotIn_type; static PyTypeObject *comprehension_type; static PyObject* ast2obj_comprehension(void*); +_Py_IDENTIFIER(ifs); static char *comprehension_fields[]={ "target", "iter", @@ -357,6 +397,7 @@ static char *excepthandler_attributes[] = { }; static PyObject* ast2obj_excepthandler(void*); static PyTypeObject *ExceptHandler_type; +_Py_IDENTIFIER(type); static char *ExceptHandler_fields[]={ "type", "name", @@ -364,6 +405,13 @@ static char *ExceptHandler_fields[]={ }; static PyTypeObject *arguments_type; static PyObject* ast2obj_arguments(void*); +_Py_IDENTIFIER(vararg); +_Py_IDENTIFIER(varargannotation); +_Py_IDENTIFIER(kwonlyargs); +_Py_IDENTIFIER(kwarg); +_Py_IDENTIFIER(kwargannotation); +_Py_IDENTIFIER(defaults); +_Py_IDENTIFIER(kw_defaults); static char *arguments_fields[]={ "args", "vararg", @@ -376,6 +424,8 @@ static char *arguments_fields[]={ }; static PyTypeObject *arg_type; static PyObject* ast2obj_arg(void*); +_Py_IDENTIFIER(arg); +_Py_IDENTIFIER(annotation); static char *arg_fields[]={ "arg", "annotation", @@ -388,19 +438,29 @@ static char *keyword_fields[]={ }; static PyTypeObject *alias_type; static PyObject* ast2obj_alias(void*); +_Py_IDENTIFIER(asname); static char *alias_fields[]={ "name", "asname", }; +static PyTypeObject *withitem_type; +static PyObject* ast2obj_withitem(void*); +_Py_IDENTIFIER(context_expr); +_Py_IDENTIFIER(optional_vars); +static char *withitem_fields[]={ + "context_expr", + "optional_vars", +}; static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { + _Py_IDENTIFIER(_fields); Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields; - fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields"); + fields = _PyObject_GetAttrId((PyObject*)Py_TYPE(self), &PyId__fields); if (!fields) PyErr_Clear(); if (fields) { @@ -450,7 +510,8 @@ static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { PyObject *res; - PyObject *dict = PyObject_GetAttrString(self, "__dict__"); + _Py_IDENTIFIER(__dict__); + PyObject *dict = _PyObject_GetAttrId(self, &PyId___dict__); if (dict == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); @@ -536,6 +597,7 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) { int i, result; + _Py_IDENTIFIER(_attributes); PyObject *s, *l = PyTuple_New(num_fields); if (!l) return 0; @@ -547,7 +609,7 @@ static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) } PyTuple_SET_ITEM(l, i, s); } - result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0; + result = _PyObject_SetAttrId((PyObject*)type, &PyId__attributes, l) >= 0; Py_DECREF(l); return result; } @@ -581,6 +643,7 @@ static PyObject* ast2obj_object(void *o) } #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object +#define ast2obj_bytes ast2obj_object static PyObject* ast2obj_int(long b) { @@ -618,6 +681,15 @@ static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) return obj2ast_object(obj, out, arena); } +static int obj2ast_bytes(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (!PyBytes_CheckExact(obj)) { + PyErr_SetString(PyExc_TypeError, "AST bytes must be of type bytes"); + return 1; + } + return obj2ast_object(obj, out, arena); +} + static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) { int i; @@ -691,15 +763,12 @@ static int init_types(void) if (!While_type) return 0; If_type = make_type("If", stmt_type, If_fields, 3); if (!If_type) return 0; - With_type = make_type("With", stmt_type, With_fields, 3); + With_type = make_type("With", stmt_type, With_fields, 2); if (!With_type) return 0; Raise_type = make_type("Raise", stmt_type, Raise_fields, 2); if (!Raise_type) return 0; - TryExcept_type = make_type("TryExcept", stmt_type, TryExcept_fields, 3); - if (!TryExcept_type) return 0; - TryFinally_type = make_type("TryFinally", stmt_type, TryFinally_fields, - 2); - if (!TryFinally_type) return 0; + Try_type = make_type("Try", stmt_type, Try_fields, 4); + if (!Try_type) return 0; Assert_type = make_type("Assert", stmt_type, Assert_fields, 2); if (!Assert_type) return 0; Import_type = make_type("Import", stmt_type, Import_fields, 1); @@ -747,6 +816,8 @@ static int init_types(void) if (!GeneratorExp_type) return 0; Yield_type = make_type("Yield", expr_type, Yield_fields, 1); if (!Yield_type) return 0; + YieldFrom_type = make_type("YieldFrom", expr_type, YieldFrom_fields, 1); + if (!YieldFrom_type) return 0; Compare_type = make_type("Compare", expr_type, Compare_fields, 3); if (!Compare_type) return 0; Call_type = make_type("Call", expr_type, Call_fields, 5); @@ -949,6 +1020,8 @@ static int init_types(void) if (!keyword_type) return 0; alias_type = make_type("alias", &AST_type, alias_fields, 2); if (!alias_type) return 0; + withitem_type = make_type("withitem", &AST_type, withitem_fields, 2); + if (!withitem_type) return 0; initialized = 1; return 1; } @@ -971,6 +1044,7 @@ static int obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena); static int obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena); static int obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena); static int obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena); +static int obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena); mod_ty Module(asdl_seq * body, PyArena *arena) @@ -1236,21 +1310,15 @@ If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int } stmt_ty -With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno, - int col_offset, PyArena *arena) +With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena + *arena) { stmt_ty p; - if (!context_expr) { - PyErr_SetString(PyExc_ValueError, - "field context_expr is required for With"); - return NULL; - } p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = With_kind; - p->v.With.context_expr = context_expr; - p->v.With.optional_vars = optional_vars; + p->v.With.items = items; p->v.With.body = body; p->lineno = lineno; p->col_offset = col_offset; @@ -1273,33 +1341,18 @@ Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena) } stmt_ty -TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno, - int col_offset, PyArena *arena) +Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, asdl_seq * + finalbody, int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = TryExcept_kind; - p->v.TryExcept.body = body; - p->v.TryExcept.handlers = handlers; - p->v.TryExcept.orelse = orelse; - p->lineno = lineno; - p->col_offset = col_offset; - return p; -} - -stmt_ty -TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int col_offset, - PyArena *arena) -{ - stmt_ty p; - p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = TryFinally_kind; - p->v.TryFinally.body = body; - p->v.TryFinally.finalbody = finalbody; + p->kind = Try_kind; + p->v.Try.body = body; + p->v.Try.handlers = handlers; + p->v.Try.orelse = orelse; + p->v.Try.finalbody = finalbody; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -1714,6 +1767,20 @@ Yield(expr_ty value, int lineno, int col_offset, PyArena *arena) } expr_ty +YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena) +{ + expr_ty p; + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = YieldFrom_kind; + p->v.YieldFrom.value = value; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno, int col_offset, PyArena *arena) { @@ -1798,7 +1865,7 @@ Str(string s, int lineno, int col_offset, PyArena *arena) } expr_ty -Bytes(string s, int lineno, int col_offset, PyArena *arena) +Bytes(bytes s, int lineno, int col_offset, PyArena *arena) { expr_ty p; if (!s) { @@ -2146,6 +2213,23 @@ alias(identifier name, identifier asname, PyArena *arena) return p; } +withitem_ty +withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena) +{ + withitem_ty p; + if (!context_expr) { + PyErr_SetString(PyExc_ValueError, + "field context_expr is required for withitem"); + return NULL; + } + p = (withitem_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->context_expr = context_expr; + p->optional_vars = optional_vars; + return p; +} + PyObject* ast2obj_mod(void* _o) @@ -2163,7 +2247,7 @@ ast2obj_mod(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Module.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); break; @@ -2172,7 +2256,7 @@ ast2obj_mod(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Interactive.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); break; @@ -2181,7 +2265,7 @@ ast2obj_mod(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Expression.body); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); break; @@ -2190,7 +2274,7 @@ ast2obj_mod(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Suite.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); break; @@ -2218,29 +2302,29 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_identifier(o->v.FunctionDef.name); if (!value) goto failed; - if (PyObject_SetAttrString(result, "name", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_name, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_arguments(o->v.FunctionDef.args); if (!value) goto failed; - if (PyObject_SetAttrString(result, "args", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_args, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.FunctionDef.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.FunctionDef.decorator_list, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "decorator_list", value) == + if (_PyObject_SetAttrId(result, &PyId_decorator_list, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.FunctionDef.returns); if (!value) goto failed; - if (PyObject_SetAttrString(result, "returns", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1) goto failed; Py_DECREF(value); break; @@ -2249,38 +2333,38 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_identifier(o->v.ClassDef.name); if (!value) goto failed; - if (PyObject_SetAttrString(result, "name", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_name, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.ClassDef.bases, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "bases", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_bases, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.ClassDef.keywords, ast2obj_keyword); if (!value) goto failed; - if (PyObject_SetAttrString(result, "keywords", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_keywords, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.ClassDef.starargs); if (!value) goto failed; - if (PyObject_SetAttrString(result, "starargs", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_starargs, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.ClassDef.kwargs); if (!value) goto failed; - if (PyObject_SetAttrString(result, "kwargs", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_kwargs, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.ClassDef.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.ClassDef.decorator_list, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "decorator_list", value) == + if (_PyObject_SetAttrId(result, &PyId_decorator_list, value) == -1) goto failed; Py_DECREF(value); @@ -2290,7 +2374,7 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Return.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); break; @@ -2299,7 +2383,7 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Delete.targets, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "targets", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_targets, value) == -1) goto failed; Py_DECREF(value); break; @@ -2308,12 +2392,12 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Assign.targets, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "targets", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_targets, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.Assign.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); break; @@ -2322,17 +2406,17 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.AugAssign.target); if (!value) goto failed; - if (PyObject_SetAttrString(result, "target", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_target, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_operator(o->v.AugAssign.op); if (!value) goto failed; - if (PyObject_SetAttrString(result, "op", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_op, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.AugAssign.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); break; @@ -2341,22 +2425,22 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.For.target); if (!value) goto failed; - if (PyObject_SetAttrString(result, "target", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_target, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.For.iter); if (!value) goto failed; - if (PyObject_SetAttrString(result, "iter", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_iter, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.For.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.For.orelse, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "orelse", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1) goto failed; Py_DECREF(value); break; @@ -2365,17 +2449,17 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.While.test); if (!value) goto failed; - if (PyObject_SetAttrString(result, "test", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_test, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.While.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.While.orelse, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "orelse", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1) goto failed; Py_DECREF(value); break; @@ -2384,37 +2468,31 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.If.test); if (!value) goto failed; - if (PyObject_SetAttrString(result, "test", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_test, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.If.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.If.orelse, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "orelse", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1) goto failed; Py_DECREF(value); break; case With_kind: result = PyType_GenericNew(With_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.With.context_expr); + value = ast2obj_list(o->v.With.items, ast2obj_withitem); if (!value) goto failed; - if (PyObject_SetAttrString(result, "context_expr", value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_expr(o->v.With.optional_vars); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "optional_vars", value) == - -1) + if (_PyObject_SetAttrId(result, &PyId_items, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.With.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); break; @@ -2423,46 +2501,36 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Raise.exc); if (!value) goto failed; - if (PyObject_SetAttrString(result, "exc", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_exc, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.Raise.cause); if (!value) goto failed; - if (PyObject_SetAttrString(result, "cause", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_cause, value) == -1) goto failed; Py_DECREF(value); break; - case TryExcept_kind: - result = PyType_GenericNew(TryExcept_type, NULL, NULL); + case Try_kind: + result = PyType_GenericNew(Try_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.TryExcept.body, ast2obj_stmt); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_list(o->v.TryExcept.handlers, - ast2obj_excepthandler); + value = ast2obj_list(o->v.Try.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "handlers", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.TryExcept.orelse, ast2obj_stmt); + value = ast2obj_list(o->v.Try.handlers, ast2obj_excepthandler); if (!value) goto failed; - if (PyObject_SetAttrString(result, "orelse", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_handlers, value) == -1) goto failed; Py_DECREF(value); - break; - case TryFinally_kind: - result = PyType_GenericNew(TryFinally_type, NULL, NULL); - if (!result) goto failed; - value = ast2obj_list(o->v.TryFinally.body, ast2obj_stmt); + value = ast2obj_list(o->v.Try.orelse, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.TryFinally.finalbody, ast2obj_stmt); + value = ast2obj_list(o->v.Try.finalbody, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "finalbody", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_finalbody, value) == -1) goto failed; Py_DECREF(value); break; @@ -2471,12 +2539,12 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Assert.test); if (!value) goto failed; - if (PyObject_SetAttrString(result, "test", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_test, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.Assert.msg); if (!value) goto failed; - if (PyObject_SetAttrString(result, "msg", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_msg, value) == -1) goto failed; Py_DECREF(value); break; @@ -2485,7 +2553,7 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Import.names, ast2obj_alias); if (!value) goto failed; - if (PyObject_SetAttrString(result, "names", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_names, value) == -1) goto failed; Py_DECREF(value); break; @@ -2494,17 +2562,17 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_identifier(o->v.ImportFrom.module); if (!value) goto failed; - if (PyObject_SetAttrString(result, "module", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_module, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.ImportFrom.names, ast2obj_alias); if (!value) goto failed; - if (PyObject_SetAttrString(result, "names", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_names, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_int(o->v.ImportFrom.level); if (!value) goto failed; - if (PyObject_SetAttrString(result, "level", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_level, value) == -1) goto failed; Py_DECREF(value); break; @@ -2513,7 +2581,7 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Global.names, ast2obj_identifier); if (!value) goto failed; - if (PyObject_SetAttrString(result, "names", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_names, value) == -1) goto failed; Py_DECREF(value); break; @@ -2522,7 +2590,7 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Nonlocal.names, ast2obj_identifier); if (!value) goto failed; - if (PyObject_SetAttrString(result, "names", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_names, value) == -1) goto failed; Py_DECREF(value); break; @@ -2531,7 +2599,7 @@ ast2obj_stmt(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Expr.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); break; @@ -2550,12 +2618,12 @@ ast2obj_stmt(void* _o) } value = ast2obj_int(o->lineno); if (!value) goto failed; - if (PyObject_SetAttrString(result, "lineno", value) < 0) + if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0) goto failed; Py_DECREF(value); value = ast2obj_int(o->col_offset); if (!value) goto failed; - if (PyObject_SetAttrString(result, "col_offset", value) < 0) + if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0) goto failed; Py_DECREF(value); return result; @@ -2581,12 +2649,12 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_boolop(o->v.BoolOp.op); if (!value) goto failed; - if (PyObject_SetAttrString(result, "op", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_op, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.BoolOp.values, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "values", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_values, value) == -1) goto failed; Py_DECREF(value); break; @@ -2595,17 +2663,17 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.BinOp.left); if (!value) goto failed; - if (PyObject_SetAttrString(result, "left", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_left, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_operator(o->v.BinOp.op); if (!value) goto failed; - if (PyObject_SetAttrString(result, "op", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_op, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.BinOp.right); if (!value) goto failed; - if (PyObject_SetAttrString(result, "right", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_right, value) == -1) goto failed; Py_DECREF(value); break; @@ -2614,12 +2682,12 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_unaryop(o->v.UnaryOp.op); if (!value) goto failed; - if (PyObject_SetAttrString(result, "op", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_op, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.UnaryOp.operand); if (!value) goto failed; - if (PyObject_SetAttrString(result, "operand", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_operand, value) == -1) goto failed; Py_DECREF(value); break; @@ -2628,12 +2696,12 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_arguments(o->v.Lambda.args); if (!value) goto failed; - if (PyObject_SetAttrString(result, "args", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_args, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.Lambda.body); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); break; @@ -2642,17 +2710,17 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.IfExp.test); if (!value) goto failed; - if (PyObject_SetAttrString(result, "test", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_test, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.IfExp.body); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.IfExp.orelse); if (!value) goto failed; - if (PyObject_SetAttrString(result, "orelse", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1) goto failed; Py_DECREF(value); break; @@ -2661,12 +2729,12 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Dict.keys, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "keys", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_keys, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.Dict.values, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "values", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_values, value) == -1) goto failed; Py_DECREF(value); break; @@ -2675,7 +2743,7 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Set.elts, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "elts", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_elts, value) == -1) goto failed; Py_DECREF(value); break; @@ -2684,13 +2752,13 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.ListComp.elt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "elt", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_elt, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.ListComp.generators, ast2obj_comprehension); if (!value) goto failed; - if (PyObject_SetAttrString(result, "generators", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1) goto failed; Py_DECREF(value); break; @@ -2699,13 +2767,13 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.SetComp.elt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "elt", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_elt, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.SetComp.generators, ast2obj_comprehension); if (!value) goto failed; - if (PyObject_SetAttrString(result, "generators", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1) goto failed; Py_DECREF(value); break; @@ -2714,18 +2782,18 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.DictComp.key); if (!value) goto failed; - if (PyObject_SetAttrString(result, "key", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_key, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.DictComp.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.DictComp.generators, ast2obj_comprehension); if (!value) goto failed; - if (PyObject_SetAttrString(result, "generators", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1) goto failed; Py_DECREF(value); break; @@ -2734,13 +2802,13 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.GeneratorExp.elt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "elt", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_elt, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.GeneratorExp.generators, ast2obj_comprehension); if (!value) goto failed; - if (PyObject_SetAttrString(result, "generators", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1) goto failed; Py_DECREF(value); break; @@ -2749,7 +2817,16 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Yield.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) + goto failed; + Py_DECREF(value); + break; + case YieldFrom_kind: + result = PyType_GenericNew(YieldFrom_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.YieldFrom.value); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); break; @@ -2758,7 +2835,7 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Compare.left); if (!value) goto failed; - if (PyObject_SetAttrString(result, "left", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_left, value) == -1) goto failed; Py_DECREF(value); { @@ -2769,12 +2846,12 @@ ast2obj_expr(void* _o) PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i))); } if (!value) goto failed; - if (PyObject_SetAttrString(result, "ops", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_ops, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.Compare.comparators, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "comparators", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_comparators, value) == -1) goto failed; Py_DECREF(value); break; @@ -2783,27 +2860,27 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Call.func); if (!value) goto failed; - if (PyObject_SetAttrString(result, "func", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_func, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.Call.args, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "args", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_args, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.Call.keywords, ast2obj_keyword); if (!value) goto failed; - if (PyObject_SetAttrString(result, "keywords", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_keywords, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.Call.starargs); if (!value) goto failed; - if (PyObject_SetAttrString(result, "starargs", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_starargs, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.Call.kwargs); if (!value) goto failed; - if (PyObject_SetAttrString(result, "kwargs", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_kwargs, value) == -1) goto failed; Py_DECREF(value); break; @@ -2812,7 +2889,7 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_object(o->v.Num.n); if (!value) goto failed; - if (PyObject_SetAttrString(result, "n", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_n, value) == -1) goto failed; Py_DECREF(value); break; @@ -2821,16 +2898,16 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_string(o->v.Str.s); if (!value) goto failed; - if (PyObject_SetAttrString(result, "s", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_s, value) == -1) goto failed; Py_DECREF(value); break; case Bytes_kind: result = PyType_GenericNew(Bytes_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_string(o->v.Bytes.s); + value = ast2obj_bytes(o->v.Bytes.s); if (!value) goto failed; - if (PyObject_SetAttrString(result, "s", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_s, value) == -1) goto failed; Py_DECREF(value); break; @@ -2843,17 +2920,17 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Attribute.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_identifier(o->v.Attribute.attr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "attr", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_attr, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr_context(o->v.Attribute.ctx); if (!value) goto failed; - if (PyObject_SetAttrString(result, "ctx", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1) goto failed; Py_DECREF(value); break; @@ -2862,17 +2939,17 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Subscript.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_slice(o->v.Subscript.slice); if (!value) goto failed; - if (PyObject_SetAttrString(result, "slice", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_slice, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr_context(o->v.Subscript.ctx); if (!value) goto failed; - if (PyObject_SetAttrString(result, "ctx", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1) goto failed; Py_DECREF(value); break; @@ -2881,12 +2958,12 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Starred.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr_context(o->v.Starred.ctx); if (!value) goto failed; - if (PyObject_SetAttrString(result, "ctx", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1) goto failed; Py_DECREF(value); break; @@ -2895,12 +2972,12 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_identifier(o->v.Name.id); if (!value) goto failed; - if (PyObject_SetAttrString(result, "id", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_id, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr_context(o->v.Name.ctx); if (!value) goto failed; - if (PyObject_SetAttrString(result, "ctx", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1) goto failed; Py_DECREF(value); break; @@ -2909,12 +2986,12 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.List.elts, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "elts", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_elts, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr_context(o->v.List.ctx); if (!value) goto failed; - if (PyObject_SetAttrString(result, "ctx", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1) goto failed; Py_DECREF(value); break; @@ -2923,24 +3000,24 @@ ast2obj_expr(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.Tuple.elts, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "elts", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_elts, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr_context(o->v.Tuple.ctx); if (!value) goto failed; - if (PyObject_SetAttrString(result, "ctx", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1) goto failed; Py_DECREF(value); break; } value = ast2obj_int(o->lineno); if (!value) goto failed; - if (PyObject_SetAttrString(result, "lineno", value) < 0) + if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0) goto failed; Py_DECREF(value); value = ast2obj_int(o->col_offset); if (!value) goto failed; - if (PyObject_SetAttrString(result, "col_offset", value) < 0) + if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0) goto failed; Py_DECREF(value); return result; @@ -2993,17 +3070,17 @@ ast2obj_slice(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Slice.lower); if (!value) goto failed; - if (PyObject_SetAttrString(result, "lower", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_lower, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.Slice.upper); if (!value) goto failed; - if (PyObject_SetAttrString(result, "upper", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_upper, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->v.Slice.step); if (!value) goto failed; - if (PyObject_SetAttrString(result, "step", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_step, value) == -1) goto failed; Py_DECREF(value); break; @@ -3012,7 +3089,7 @@ ast2obj_slice(void* _o) if (!result) goto failed; value = ast2obj_list(o->v.ExtSlice.dims, ast2obj_slice); if (!value) goto failed; - if (PyObject_SetAttrString(result, "dims", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_dims, value) == -1) goto failed; Py_DECREF(value); break; @@ -3021,7 +3098,7 @@ ast2obj_slice(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.Index.value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); break; @@ -3167,17 +3244,17 @@ ast2obj_comprehension(void* _o) if (!result) return NULL; value = ast2obj_expr(o->target); if (!value) goto failed; - if (PyObject_SetAttrString(result, "target", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_target, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->iter); if (!value) goto failed; - if (PyObject_SetAttrString(result, "iter", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_iter, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->ifs, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "ifs", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_ifs, value) == -1) goto failed; Py_DECREF(value); return result; @@ -3203,29 +3280,29 @@ ast2obj_excepthandler(void* _o) if (!result) goto failed; value = ast2obj_expr(o->v.ExceptHandler.type); if (!value) goto failed; - if (PyObject_SetAttrString(result, "type", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_type, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_identifier(o->v.ExceptHandler.name); if (!value) goto failed; - if (PyObject_SetAttrString(result, "name", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_name, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.ExceptHandler.body, ast2obj_stmt); if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); break; } value = ast2obj_int(o->lineno); if (!value) goto failed; - if (PyObject_SetAttrString(result, "lineno", value) < 0) + if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0) goto failed; Py_DECREF(value); value = ast2obj_int(o->col_offset); if (!value) goto failed; - if (PyObject_SetAttrString(result, "col_offset", value) < 0) + if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0) goto failed; Py_DECREF(value); return result; @@ -3249,42 +3326,42 @@ ast2obj_arguments(void* _o) if (!result) return NULL; value = ast2obj_list(o->args, ast2obj_arg); if (!value) goto failed; - if (PyObject_SetAttrString(result, "args", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_args, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_identifier(o->vararg); if (!value) goto failed; - if (PyObject_SetAttrString(result, "vararg", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_vararg, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->varargannotation); if (!value) goto failed; - if (PyObject_SetAttrString(result, "varargannotation", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_varargannotation, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->kwonlyargs, ast2obj_arg); if (!value) goto failed; - if (PyObject_SetAttrString(result, "kwonlyargs", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_kwonlyargs, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_identifier(o->kwarg); if (!value) goto failed; - if (PyObject_SetAttrString(result, "kwarg", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_kwarg, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->kwargannotation); if (!value) goto failed; - if (PyObject_SetAttrString(result, "kwargannotation", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_kwargannotation, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->defaults, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "defaults", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_defaults, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->kw_defaults, ast2obj_expr); if (!value) goto failed; - if (PyObject_SetAttrString(result, "kw_defaults", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_kw_defaults, value) == -1) goto failed; Py_DECREF(value); return result; @@ -3308,12 +3385,12 @@ ast2obj_arg(void* _o) if (!result) return NULL; value = ast2obj_identifier(o->arg); if (!value) goto failed; - if (PyObject_SetAttrString(result, "arg", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_arg, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->annotation); if (!value) goto failed; - if (PyObject_SetAttrString(result, "annotation", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_annotation, value) == -1) goto failed; Py_DECREF(value); return result; @@ -3337,12 +3414,12 @@ ast2obj_keyword(void* _o) if (!result) return NULL; value = ast2obj_identifier(o->arg); if (!value) goto failed; - if (PyObject_SetAttrString(result, "arg", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_arg, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(o->value); if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); return result; @@ -3366,12 +3443,41 @@ ast2obj_alias(void* _o) if (!result) return NULL; value = ast2obj_identifier(o->name); if (!value) goto failed; - if (PyObject_SetAttrString(result, "name", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_name, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_identifier(o->asname); if (!value) goto failed; - if (PyObject_SetAttrString(result, "asname", value) == -1) + if (_PyObject_SetAttrId(result, &PyId_asname, value) == -1) + goto failed; + Py_DECREF(value); + return result; +failed: + Py_XDECREF(value); + Py_XDECREF(result); + return NULL; +} + +PyObject* +ast2obj_withitem(void* _o) +{ + withitem_ty o = (withitem_ty)_o; + PyObject *result = NULL, *value = NULL; + if (!o) { + Py_INCREF(Py_None); + return Py_None; + } + + result = PyType_GenericNew(withitem_type, NULL, NULL); + if (!result) return NULL; + value = ast2obj_expr(o->context_expr); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_context_expr, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->optional_vars); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_optional_vars, value) == -1) goto failed; Py_DECREF(value); return result; @@ -3400,11 +3506,11 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) if (isinstance) { asdl_seq* body; - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Module field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3436,11 +3542,11 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) if (isinstance) { asdl_seq* body; - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Interactive field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3472,9 +3578,9 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) if (isinstance) { expr_ty body; - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &body, arena); if (res != 0) goto failed; @@ -3495,11 +3601,11 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) if (isinstance) { asdl_seq* body; - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Suite field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3544,9 +3650,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) *out = NULL; return 0; } - if (PyObject_HasAttrString(obj, "lineno")) { + if (_PyObject_HasAttrId(obj, &PyId_lineno)) { int res; - tmp = PyObject_GetAttrString(obj, "lineno"); + tmp = _PyObject_GetAttrId(obj, &PyId_lineno); if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &lineno, arena); if (res != 0) goto failed; @@ -3556,9 +3662,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from stmt"); return 1; } - if (PyObject_HasAttrString(obj, "col_offset")) { + if (_PyObject_HasAttrId(obj, &PyId_col_offset)) { int res; - tmp = PyObject_GetAttrString(obj, "col_offset"); + tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &col_offset, arena); if (res != 0) goto failed; @@ -3579,9 +3685,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* decorator_list; expr_ty returns; - if (PyObject_HasAttrString(obj, "name")) { + if (_PyObject_HasAttrId(obj, &PyId_name)) { int res; - tmp = PyObject_GetAttrString(obj, "name"); + tmp = _PyObject_GetAttrId(obj, &PyId_name); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; @@ -3591,9 +3697,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from FunctionDef"); return 1; } - if (PyObject_HasAttrString(obj, "args")) { + if (_PyObject_HasAttrId(obj, &PyId_args)) { int res; - tmp = PyObject_GetAttrString(obj, "args"); + tmp = _PyObject_GetAttrId(obj, &PyId_args); if (tmp == NULL) goto failed; res = obj2ast_arguments(tmp, &args, arena); if (res != 0) goto failed; @@ -3603,11 +3709,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from FunctionDef"); return 1; } - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "FunctionDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3628,11 +3734,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from FunctionDef"); return 1; } - if (PyObject_HasAttrString(obj, "decorator_list")) { + if (_PyObject_HasAttrId(obj, &PyId_decorator_list)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "decorator_list"); + tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "FunctionDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3653,9 +3759,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from FunctionDef"); return 1; } - if (PyObject_HasAttrString(obj, "returns")) { + if (_PyObject_HasAttrId(obj, &PyId_returns)) { int res; - tmp = PyObject_GetAttrString(obj, "returns"); + tmp = _PyObject_GetAttrId(obj, &PyId_returns); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &returns, arena); if (res != 0) goto failed; @@ -3682,9 +3788,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* decorator_list; - if (PyObject_HasAttrString(obj, "name")) { + if (_PyObject_HasAttrId(obj, &PyId_name)) { int res; - tmp = PyObject_GetAttrString(obj, "name"); + tmp = _PyObject_GetAttrId(obj, &PyId_name); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; @@ -3694,11 +3800,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from ClassDef"); return 1; } - if (PyObject_HasAttrString(obj, "bases")) { + if (_PyObject_HasAttrId(obj, &PyId_bases)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "bases"); + tmp = _PyObject_GetAttrId(obj, &PyId_bases); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ClassDef field \"bases\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3719,11 +3825,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"bases\" missing from ClassDef"); return 1; } - if (PyObject_HasAttrString(obj, "keywords")) { + if (_PyObject_HasAttrId(obj, &PyId_keywords)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "keywords"); + tmp = _PyObject_GetAttrId(obj, &PyId_keywords); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ClassDef field \"keywords\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3744,9 +3850,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"keywords\" missing from ClassDef"); return 1; } - if (PyObject_HasAttrString(obj, "starargs")) { + if (_PyObject_HasAttrId(obj, &PyId_starargs)) { int res; - tmp = PyObject_GetAttrString(obj, "starargs"); + tmp = _PyObject_GetAttrId(obj, &PyId_starargs); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &starargs, arena); if (res != 0) goto failed; @@ -3755,9 +3861,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } else { starargs = NULL; } - if (PyObject_HasAttrString(obj, "kwargs")) { + if (_PyObject_HasAttrId(obj, &PyId_kwargs)) { int res; - tmp = PyObject_GetAttrString(obj, "kwargs"); + tmp = _PyObject_GetAttrId(obj, &PyId_kwargs); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &kwargs, arena); if (res != 0) goto failed; @@ -3766,11 +3872,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } else { kwargs = NULL; } - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ClassDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3791,11 +3897,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from ClassDef"); return 1; } - if (PyObject_HasAttrString(obj, "decorator_list")) { + if (_PyObject_HasAttrId(obj, &PyId_decorator_list)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "decorator_list"); + tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ClassDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3828,9 +3934,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -3850,11 +3956,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* targets; - if (PyObject_HasAttrString(obj, "targets")) { + if (_PyObject_HasAttrId(obj, &PyId_targets)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "targets"); + tmp = _PyObject_GetAttrId(obj, &PyId_targets); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Delete field \"targets\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3887,11 +3993,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* targets; expr_ty value; - if (PyObject_HasAttrString(obj, "targets")) { + if (_PyObject_HasAttrId(obj, &PyId_targets)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "targets"); + tmp = _PyObject_GetAttrId(obj, &PyId_targets); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Assign field \"targets\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -3912,9 +4018,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"targets\" missing from Assign"); return 1; } - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -3937,9 +4043,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) operator_ty op; expr_ty value; - if (PyObject_HasAttrString(obj, "target")) { + if (_PyObject_HasAttrId(obj, &PyId_target)) { int res; - tmp = PyObject_GetAttrString(obj, "target"); + tmp = _PyObject_GetAttrId(obj, &PyId_target); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &target, arena); if (res != 0) goto failed; @@ -3949,9 +4055,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AugAssign"); return 1; } - if (PyObject_HasAttrString(obj, "op")) { + if (_PyObject_HasAttrId(obj, &PyId_op)) { int res; - tmp = PyObject_GetAttrString(obj, "op"); + tmp = _PyObject_GetAttrId(obj, &PyId_op); if (tmp == NULL) goto failed; res = obj2ast_operator(tmp, &op, arena); if (res != 0) goto failed; @@ -3961,9 +4067,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from AugAssign"); return 1; } - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -3987,9 +4093,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* orelse; - if (PyObject_HasAttrString(obj, "target")) { + if (_PyObject_HasAttrId(obj, &PyId_target)) { int res; - tmp = PyObject_GetAttrString(obj, "target"); + tmp = _PyObject_GetAttrId(obj, &PyId_target); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &target, arena); if (res != 0) goto failed; @@ -3999,9 +4105,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from For"); return 1; } - if (PyObject_HasAttrString(obj, "iter")) { + if (_PyObject_HasAttrId(obj, &PyId_iter)) { int res; - tmp = PyObject_GetAttrString(obj, "iter"); + tmp = _PyObject_GetAttrId(obj, &PyId_iter); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &iter, arena); if (res != 0) goto failed; @@ -4011,11 +4117,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from For"); return 1; } - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "For field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4036,11 +4142,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from For"); return 1; } - if (PyObject_HasAttrString(obj, "orelse")) { + if (_PyObject_HasAttrId(obj, &PyId_orelse)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "orelse"); + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "For field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4075,9 +4181,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* orelse; - if (PyObject_HasAttrString(obj, "test")) { + if (_PyObject_HasAttrId(obj, &PyId_test)) { int res; - tmp = PyObject_GetAttrString(obj, "test"); + tmp = _PyObject_GetAttrId(obj, &PyId_test); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &test, arena); if (res != 0) goto failed; @@ -4087,11 +4193,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from While"); return 1; } - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "While field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4112,11 +4218,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from While"); return 1; } - if (PyObject_HasAttrString(obj, "orelse")) { + if (_PyObject_HasAttrId(obj, &PyId_orelse)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "orelse"); + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "While field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4150,9 +4256,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* orelse; - if (PyObject_HasAttrString(obj, "test")) { + if (_PyObject_HasAttrId(obj, &PyId_test)) { int res; - tmp = PyObject_GetAttrString(obj, "test"); + tmp = _PyObject_GetAttrId(obj, &PyId_test); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &test, arena); if (res != 0) goto failed; @@ -4162,11 +4268,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from If"); return 1; } - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "If field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4187,11 +4293,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from If"); return 1; } - if (PyObject_HasAttrString(obj, "orelse")) { + if (_PyObject_HasAttrId(obj, &PyId_orelse)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "orelse"); + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "If field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4221,38 +4327,39 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) return 1; } if (isinstance) { - expr_ty context_expr; - expr_ty optional_vars; + asdl_seq* items; asdl_seq* body; - if (PyObject_HasAttrString(obj, "context_expr")) { + if (_PyObject_HasAttrId(obj, &PyId_items)) { int res; - tmp = PyObject_GetAttrString(obj, "context_expr"); + Py_ssize_t len; + Py_ssize_t i; + tmp = _PyObject_GetAttrId(obj, &PyId_items); if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &context_expr, arena); - if (res != 0) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "With field \"items\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + items = asdl_seq_new(len, arena); + if (items == NULL) goto failed; + for (i = 0; i < len; i++) { + withitem_ty value; + res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(items, i, value); + } Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from With"); + PyErr_SetString(PyExc_TypeError, "required field \"items\" missing from With"); return 1; } - if (PyObject_HasAttrString(obj, "optional_vars")) { - int res; - tmp = PyObject_GetAttrString(obj, "optional_vars"); - if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &optional_vars, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - optional_vars = NULL; - } - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "With field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4273,8 +4380,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from With"); return 1; } - *out = With(context_expr, optional_vars, body, lineno, - col_offset, arena); + *out = With(items, body, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4286,9 +4392,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) expr_ty exc; expr_ty cause; - if (PyObject_HasAttrString(obj, "exc")) { + if (_PyObject_HasAttrId(obj, &PyId_exc)) { int res; - tmp = PyObject_GetAttrString(obj, "exc"); + tmp = _PyObject_GetAttrId(obj, &PyId_exc); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &exc, arena); if (res != 0) goto failed; @@ -4297,9 +4403,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } else { exc = NULL; } - if (PyObject_HasAttrString(obj, "cause")) { + if (_PyObject_HasAttrId(obj, &PyId_cause)) { int res; - tmp = PyObject_GetAttrString(obj, "cause"); + tmp = _PyObject_GetAttrId(obj, &PyId_cause); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &cause, arena); if (res != 0) goto failed; @@ -4312,7 +4418,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)TryExcept_type); + isinstance = PyObject_IsInstance(obj, (PyObject*)Try_type); if (isinstance == -1) { return 1; } @@ -4320,15 +4426,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* handlers; asdl_seq* orelse; + asdl_seq* finalbody; - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryExcept field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Try field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; } len = PyList_GET_SIZE(tmp); @@ -4343,17 +4450,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from TryExcept"); + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Try"); return 1; } - if (PyObject_HasAttrString(obj, "handlers")) { + if (_PyObject_HasAttrId(obj, &PyId_handlers)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "handlers"); + tmp = _PyObject_GetAttrId(obj, &PyId_handlers); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryExcept field \"handlers\" must be a list, not a %.200s", tmp->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Try field \"handlers\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; } len = PyList_GET_SIZE(tmp); @@ -4368,17 +4475,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"handlers\" missing from TryExcept"); + PyErr_SetString(PyExc_TypeError, "required field \"handlers\" missing from Try"); return 1; } - if (PyObject_HasAttrString(obj, "orelse")) { + if (_PyObject_HasAttrId(obj, &PyId_orelse)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "orelse"); + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryExcept field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Try field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; } len = PyList_GET_SIZE(tmp); @@ -4393,55 +4500,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from TryExcept"); - return 1; - } - *out = TryExcept(body, handlers, orelse, lineno, col_offset, - arena); - if (*out == NULL) goto failed; - return 0; - } - isinstance = PyObject_IsInstance(obj, (PyObject*)TryFinally_type); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - asdl_seq* body; - asdl_seq* finalbody; - - if (PyObject_HasAttrString(obj, "body")) { - int res; - Py_ssize_t len; - Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); - if (tmp == NULL) goto failed; - if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryFinally field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); - goto failed; - } - len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); - } - Py_XDECREF(tmp); - tmp = NULL; - } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from TryFinally"); + PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from Try"); return 1; } - if (PyObject_HasAttrString(obj, "finalbody")) { + if (_PyObject_HasAttrId(obj, &PyId_finalbody)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "finalbody"); + tmp = _PyObject_GetAttrId(obj, &PyId_finalbody); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryFinally field \"finalbody\" must be a list, not a %.200s", tmp->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Try field \"finalbody\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; } len = PyList_GET_SIZE(tmp); @@ -4456,10 +4525,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"finalbody\" missing from TryFinally"); + PyErr_SetString(PyExc_TypeError, "required field \"finalbody\" missing from Try"); return 1; } - *out = TryFinally(body, finalbody, lineno, col_offset, arena); + *out = Try(body, handlers, orelse, finalbody, lineno, + col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4471,9 +4541,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) expr_ty test; expr_ty msg; - if (PyObject_HasAttrString(obj, "test")) { + if (_PyObject_HasAttrId(obj, &PyId_test)) { int res; - tmp = PyObject_GetAttrString(obj, "test"); + tmp = _PyObject_GetAttrId(obj, &PyId_test); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &test, arena); if (res != 0) goto failed; @@ -4483,9 +4553,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from Assert"); return 1; } - if (PyObject_HasAttrString(obj, "msg")) { + if (_PyObject_HasAttrId(obj, &PyId_msg)) { int res; - tmp = PyObject_GetAttrString(obj, "msg"); + tmp = _PyObject_GetAttrId(obj, &PyId_msg); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &msg, arena); if (res != 0) goto failed; @@ -4505,11 +4575,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* names; - if (PyObject_HasAttrString(obj, "names")) { + if (_PyObject_HasAttrId(obj, &PyId_names)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "names"); + tmp = _PyObject_GetAttrId(obj, &PyId_names); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Import field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4543,9 +4613,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* names; int level; - if (PyObject_HasAttrString(obj, "module")) { + if (_PyObject_HasAttrId(obj, &PyId_module)) { int res; - tmp = PyObject_GetAttrString(obj, "module"); + tmp = _PyObject_GetAttrId(obj, &PyId_module); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &module, arena); if (res != 0) goto failed; @@ -4554,11 +4624,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } else { module = NULL; } - if (PyObject_HasAttrString(obj, "names")) { + if (_PyObject_HasAttrId(obj, &PyId_names)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "names"); + tmp = _PyObject_GetAttrId(obj, &PyId_names); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ImportFrom field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4579,9 +4649,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from ImportFrom"); return 1; } - if (PyObject_HasAttrString(obj, "level")) { + if (_PyObject_HasAttrId(obj, &PyId_level)) { int res; - tmp = PyObject_GetAttrString(obj, "level"); + tmp = _PyObject_GetAttrId(obj, &PyId_level); if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &level, arena); if (res != 0) goto failed; @@ -4602,11 +4672,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* names; - if (PyObject_HasAttrString(obj, "names")) { + if (_PyObject_HasAttrId(obj, &PyId_names)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "names"); + tmp = _PyObject_GetAttrId(obj, &PyId_names); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Global field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4638,11 +4708,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* names; - if (PyObject_HasAttrString(obj, "names")) { + if (_PyObject_HasAttrId(obj, &PyId_names)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "names"); + tmp = _PyObject_GetAttrId(obj, &PyId_names); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Nonlocal field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4674,9 +4744,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -4740,9 +4810,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) *out = NULL; return 0; } - if (PyObject_HasAttrString(obj, "lineno")) { + if (_PyObject_HasAttrId(obj, &PyId_lineno)) { int res; - tmp = PyObject_GetAttrString(obj, "lineno"); + tmp = _PyObject_GetAttrId(obj, &PyId_lineno); if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &lineno, arena); if (res != 0) goto failed; @@ -4752,9 +4822,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from expr"); return 1; } - if (PyObject_HasAttrString(obj, "col_offset")) { + if (_PyObject_HasAttrId(obj, &PyId_col_offset)) { int res; - tmp = PyObject_GetAttrString(obj, "col_offset"); + tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &col_offset, arena); if (res != 0) goto failed; @@ -4772,9 +4842,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) boolop_ty op; asdl_seq* values; - if (PyObject_HasAttrString(obj, "op")) { + if (_PyObject_HasAttrId(obj, &PyId_op)) { int res; - tmp = PyObject_GetAttrString(obj, "op"); + tmp = _PyObject_GetAttrId(obj, &PyId_op); if (tmp == NULL) goto failed; res = obj2ast_boolop(tmp, &op, arena); if (res != 0) goto failed; @@ -4784,11 +4854,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BoolOp"); return 1; } - if (PyObject_HasAttrString(obj, "values")) { + if (_PyObject_HasAttrId(obj, &PyId_values)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "values"); + tmp = _PyObject_GetAttrId(obj, &PyId_values); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "BoolOp field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -4822,9 +4892,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) operator_ty op; expr_ty right; - if (PyObject_HasAttrString(obj, "left")) { + if (_PyObject_HasAttrId(obj, &PyId_left)) { int res; - tmp = PyObject_GetAttrString(obj, "left"); + tmp = _PyObject_GetAttrId(obj, &PyId_left); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &left, arena); if (res != 0) goto failed; @@ -4834,9 +4904,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from BinOp"); return 1; } - if (PyObject_HasAttrString(obj, "op")) { + if (_PyObject_HasAttrId(obj, &PyId_op)) { int res; - tmp = PyObject_GetAttrString(obj, "op"); + tmp = _PyObject_GetAttrId(obj, &PyId_op); if (tmp == NULL) goto failed; res = obj2ast_operator(tmp, &op, arena); if (res != 0) goto failed; @@ -4846,9 +4916,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BinOp"); return 1; } - if (PyObject_HasAttrString(obj, "right")) { + if (_PyObject_HasAttrId(obj, &PyId_right)) { int res; - tmp = PyObject_GetAttrString(obj, "right"); + tmp = _PyObject_GetAttrId(obj, &PyId_right); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &right, arena); if (res != 0) goto failed; @@ -4870,9 +4940,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) unaryop_ty op; expr_ty operand; - if (PyObject_HasAttrString(obj, "op")) { + if (_PyObject_HasAttrId(obj, &PyId_op)) { int res; - tmp = PyObject_GetAttrString(obj, "op"); + tmp = _PyObject_GetAttrId(obj, &PyId_op); if (tmp == NULL) goto failed; res = obj2ast_unaryop(tmp, &op, arena); if (res != 0) goto failed; @@ -4882,9 +4952,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from UnaryOp"); return 1; } - if (PyObject_HasAttrString(obj, "operand")) { + if (_PyObject_HasAttrId(obj, &PyId_operand)) { int res; - tmp = PyObject_GetAttrString(obj, "operand"); + tmp = _PyObject_GetAttrId(obj, &PyId_operand); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &operand, arena); if (res != 0) goto failed; @@ -4906,9 +4976,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) arguments_ty args; expr_ty body; - if (PyObject_HasAttrString(obj, "args")) { + if (_PyObject_HasAttrId(obj, &PyId_args)) { int res; - tmp = PyObject_GetAttrString(obj, "args"); + tmp = _PyObject_GetAttrId(obj, &PyId_args); if (tmp == NULL) goto failed; res = obj2ast_arguments(tmp, &args, arena); if (res != 0) goto failed; @@ -4918,9 +4988,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from Lambda"); return 1; } - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &body, arena); if (res != 0) goto failed; @@ -4943,9 +5013,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty body; expr_ty orelse; - if (PyObject_HasAttrString(obj, "test")) { + if (_PyObject_HasAttrId(obj, &PyId_test)) { int res; - tmp = PyObject_GetAttrString(obj, "test"); + tmp = _PyObject_GetAttrId(obj, &PyId_test); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &test, arena); if (res != 0) goto failed; @@ -4955,9 +5025,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from IfExp"); return 1; } - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &body, arena); if (res != 0) goto failed; @@ -4967,9 +5037,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from IfExp"); return 1; } - if (PyObject_HasAttrString(obj, "orelse")) { + if (_PyObject_HasAttrId(obj, &PyId_orelse)) { int res; - tmp = PyObject_GetAttrString(obj, "orelse"); + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &orelse, arena); if (res != 0) goto failed; @@ -4991,11 +5061,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) asdl_seq* keys; asdl_seq* values; - if (PyObject_HasAttrString(obj, "keys")) { + if (_PyObject_HasAttrId(obj, &PyId_keys)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "keys"); + tmp = _PyObject_GetAttrId(obj, &PyId_keys); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Dict field \"keys\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5016,11 +5086,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"keys\" missing from Dict"); return 1; } - if (PyObject_HasAttrString(obj, "values")) { + if (_PyObject_HasAttrId(obj, &PyId_values)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "values"); + tmp = _PyObject_GetAttrId(obj, &PyId_values); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Dict field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5052,11 +5122,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { asdl_seq* elts; - if (PyObject_HasAttrString(obj, "elts")) { + if (_PyObject_HasAttrId(obj, &PyId_elts)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "elts"); + tmp = _PyObject_GetAttrId(obj, &PyId_elts); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Set field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5089,9 +5159,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty elt; asdl_seq* generators; - if (PyObject_HasAttrString(obj, "elt")) { + if (_PyObject_HasAttrId(obj, &PyId_elt)) { int res; - tmp = PyObject_GetAttrString(obj, "elt"); + tmp = _PyObject_GetAttrId(obj, &PyId_elt); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &elt, arena); if (res != 0) goto failed; @@ -5101,11 +5171,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from ListComp"); return 1; } - if (PyObject_HasAttrString(obj, "generators")) { + if (_PyObject_HasAttrId(obj, &PyId_generators)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "generators"); + tmp = _PyObject_GetAttrId(obj, &PyId_generators); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ListComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5138,9 +5208,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty elt; asdl_seq* generators; - if (PyObject_HasAttrString(obj, "elt")) { + if (_PyObject_HasAttrId(obj, &PyId_elt)) { int res; - tmp = PyObject_GetAttrString(obj, "elt"); + tmp = _PyObject_GetAttrId(obj, &PyId_elt); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &elt, arena); if (res != 0) goto failed; @@ -5150,11 +5220,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from SetComp"); return 1; } - if (PyObject_HasAttrString(obj, "generators")) { + if (_PyObject_HasAttrId(obj, &PyId_generators)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "generators"); + tmp = _PyObject_GetAttrId(obj, &PyId_generators); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "SetComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5188,9 +5258,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty value; asdl_seq* generators; - if (PyObject_HasAttrString(obj, "key")) { + if (_PyObject_HasAttrId(obj, &PyId_key)) { int res; - tmp = PyObject_GetAttrString(obj, "key"); + tmp = _PyObject_GetAttrId(obj, &PyId_key); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &key, arena); if (res != 0) goto failed; @@ -5200,9 +5270,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"key\" missing from DictComp"); return 1; } - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -5212,11 +5282,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from DictComp"); return 1; } - if (PyObject_HasAttrString(obj, "generators")) { + if (_PyObject_HasAttrId(obj, &PyId_generators)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "generators"); + tmp = _PyObject_GetAttrId(obj, &PyId_generators); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "DictComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5250,9 +5320,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty elt; asdl_seq* generators; - if (PyObject_HasAttrString(obj, "elt")) { + if (_PyObject_HasAttrId(obj, &PyId_elt)) { int res; - tmp = PyObject_GetAttrString(obj, "elt"); + tmp = _PyObject_GetAttrId(obj, &PyId_elt); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &elt, arena); if (res != 0) goto failed; @@ -5262,11 +5332,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from GeneratorExp"); return 1; } - if (PyObject_HasAttrString(obj, "generators")) { + if (_PyObject_HasAttrId(obj, &PyId_generators)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "generators"); + tmp = _PyObject_GetAttrId(obj, &PyId_generators); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "GeneratorExp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5298,9 +5368,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -5313,6 +5383,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)YieldFrom_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty value; + + if (_PyObject_HasAttrId(obj, &PyId_value)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &value, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + value = NULL; + } + *out = YieldFrom(value, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)Compare_type); if (isinstance == -1) { return 1; @@ -5322,9 +5414,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) asdl_int_seq* ops; asdl_seq* comparators; - if (PyObject_HasAttrString(obj, "left")) { + if (_PyObject_HasAttrId(obj, &PyId_left)) { int res; - tmp = PyObject_GetAttrString(obj, "left"); + tmp = _PyObject_GetAttrId(obj, &PyId_left); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &left, arena); if (res != 0) goto failed; @@ -5334,11 +5426,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from Compare"); return 1; } - if (PyObject_HasAttrString(obj, "ops")) { + if (_PyObject_HasAttrId(obj, &PyId_ops)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "ops"); + tmp = _PyObject_GetAttrId(obj, &PyId_ops); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Compare field \"ops\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5359,11 +5451,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"ops\" missing from Compare"); return 1; } - if (PyObject_HasAttrString(obj, "comparators")) { + if (_PyObject_HasAttrId(obj, &PyId_comparators)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "comparators"); + tmp = _PyObject_GetAttrId(obj, &PyId_comparators); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Compare field \"comparators\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5400,9 +5492,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty starargs; expr_ty kwargs; - if (PyObject_HasAttrString(obj, "func")) { + if (_PyObject_HasAttrId(obj, &PyId_func)) { int res; - tmp = PyObject_GetAttrString(obj, "func"); + tmp = _PyObject_GetAttrId(obj, &PyId_func); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &func, arena); if (res != 0) goto failed; @@ -5412,11 +5504,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"func\" missing from Call"); return 1; } - if (PyObject_HasAttrString(obj, "args")) { + if (_PyObject_HasAttrId(obj, &PyId_args)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "args"); + tmp = _PyObject_GetAttrId(obj, &PyId_args); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Call field \"args\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5437,11 +5529,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from Call"); return 1; } - if (PyObject_HasAttrString(obj, "keywords")) { + if (_PyObject_HasAttrId(obj, &PyId_keywords)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "keywords"); + tmp = _PyObject_GetAttrId(obj, &PyId_keywords); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Call field \"keywords\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5462,9 +5554,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"keywords\" missing from Call"); return 1; } - if (PyObject_HasAttrString(obj, "starargs")) { + if (_PyObject_HasAttrId(obj, &PyId_starargs)) { int res; - tmp = PyObject_GetAttrString(obj, "starargs"); + tmp = _PyObject_GetAttrId(obj, &PyId_starargs); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &starargs, arena); if (res != 0) goto failed; @@ -5473,9 +5565,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } else { starargs = NULL; } - if (PyObject_HasAttrString(obj, "kwargs")) { + if (_PyObject_HasAttrId(obj, &PyId_kwargs)) { int res; - tmp = PyObject_GetAttrString(obj, "kwargs"); + tmp = _PyObject_GetAttrId(obj, &PyId_kwargs); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &kwargs, arena); if (res != 0) goto failed; @@ -5496,9 +5588,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { object n; - if (PyObject_HasAttrString(obj, "n")) { + if (_PyObject_HasAttrId(obj, &PyId_n)) { int res; - tmp = PyObject_GetAttrString(obj, "n"); + tmp = _PyObject_GetAttrId(obj, &PyId_n); if (tmp == NULL) goto failed; res = obj2ast_object(tmp, &n, arena); if (res != 0) goto failed; @@ -5519,9 +5611,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { string s; - if (PyObject_HasAttrString(obj, "s")) { + if (_PyObject_HasAttrId(obj, &PyId_s)) { int res; - tmp = PyObject_GetAttrString(obj, "s"); + tmp = _PyObject_GetAttrId(obj, &PyId_s); if (tmp == NULL) goto failed; res = obj2ast_string(tmp, &s, arena); if (res != 0) goto failed; @@ -5540,13 +5632,13 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) return 1; } if (isinstance) { - string s; + bytes s; - if (PyObject_HasAttrString(obj, "s")) { + if (_PyObject_HasAttrId(obj, &PyId_s)) { int res; - tmp = PyObject_GetAttrString(obj, "s"); + tmp = _PyObject_GetAttrId(obj, &PyId_s); if (tmp == NULL) goto failed; - res = obj2ast_string(tmp, &s, arena); + res = obj2ast_bytes(tmp, &s, arena); if (res != 0) goto failed; Py_XDECREF(tmp); tmp = NULL; @@ -5577,9 +5669,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) identifier attr; expr_context_ty ctx; - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -5589,9 +5681,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Attribute"); return 1; } - if (PyObject_HasAttrString(obj, "attr")) { + if (_PyObject_HasAttrId(obj, &PyId_attr)) { int res; - tmp = PyObject_GetAttrString(obj, "attr"); + tmp = _PyObject_GetAttrId(obj, &PyId_attr); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &attr, arena); if (res != 0) goto failed; @@ -5601,9 +5693,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"attr\" missing from Attribute"); return 1; } - if (PyObject_HasAttrString(obj, "ctx")) { + if (_PyObject_HasAttrId(obj, &PyId_ctx)) { int res; - tmp = PyObject_GetAttrString(obj, "ctx"); + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; @@ -5626,9 +5718,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) slice_ty slice; expr_context_ty ctx; - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -5638,9 +5730,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Subscript"); return 1; } - if (PyObject_HasAttrString(obj, "slice")) { + if (_PyObject_HasAttrId(obj, &PyId_slice)) { int res; - tmp = PyObject_GetAttrString(obj, "slice"); + tmp = _PyObject_GetAttrId(obj, &PyId_slice); if (tmp == NULL) goto failed; res = obj2ast_slice(tmp, &slice, arena); if (res != 0) goto failed; @@ -5650,9 +5742,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"slice\" missing from Subscript"); return 1; } - if (PyObject_HasAttrString(obj, "ctx")) { + if (_PyObject_HasAttrId(obj, &PyId_ctx)) { int res; - tmp = PyObject_GetAttrString(obj, "ctx"); + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; @@ -5674,9 +5766,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty value; expr_context_ty ctx; - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -5686,9 +5778,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Starred"); return 1; } - if (PyObject_HasAttrString(obj, "ctx")) { + if (_PyObject_HasAttrId(obj, &PyId_ctx)) { int res; - tmp = PyObject_GetAttrString(obj, "ctx"); + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; @@ -5710,9 +5802,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) identifier id; expr_context_ty ctx; - if (PyObject_HasAttrString(obj, "id")) { + if (_PyObject_HasAttrId(obj, &PyId_id)) { int res; - tmp = PyObject_GetAttrString(obj, "id"); + tmp = _PyObject_GetAttrId(obj, &PyId_id); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &id, arena); if (res != 0) goto failed; @@ -5722,9 +5814,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"id\" missing from Name"); return 1; } - if (PyObject_HasAttrString(obj, "ctx")) { + if (_PyObject_HasAttrId(obj, &PyId_ctx)) { int res; - tmp = PyObject_GetAttrString(obj, "ctx"); + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; @@ -5746,11 +5838,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) asdl_seq* elts; expr_context_ty ctx; - if (PyObject_HasAttrString(obj, "elts")) { + if (_PyObject_HasAttrId(obj, &PyId_elts)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "elts"); + tmp = _PyObject_GetAttrId(obj, &PyId_elts); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "List field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5771,9 +5863,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"elts\" missing from List"); return 1; } - if (PyObject_HasAttrString(obj, "ctx")) { + if (_PyObject_HasAttrId(obj, &PyId_ctx)) { int res; - tmp = PyObject_GetAttrString(obj, "ctx"); + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; @@ -5795,11 +5887,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) asdl_seq* elts; expr_context_ty ctx; - if (PyObject_HasAttrString(obj, "elts")) { + if (_PyObject_HasAttrId(obj, &PyId_elts)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "elts"); + tmp = _PyObject_GetAttrId(obj, &PyId_elts); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Tuple field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -5820,9 +5912,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"elts\" missing from Tuple"); return 1; } - if (PyObject_HasAttrString(obj, "ctx")) { + if (_PyObject_HasAttrId(obj, &PyId_ctx)) { int res; - tmp = PyObject_GetAttrString(obj, "ctx"); + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; @@ -5921,9 +6013,9 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) expr_ty upper; expr_ty step; - if (PyObject_HasAttrString(obj, "lower")) { + if (_PyObject_HasAttrId(obj, &PyId_lower)) { int res; - tmp = PyObject_GetAttrString(obj, "lower"); + tmp = _PyObject_GetAttrId(obj, &PyId_lower); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &lower, arena); if (res != 0) goto failed; @@ -5932,9 +6024,9 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) } else { lower = NULL; } - if (PyObject_HasAttrString(obj, "upper")) { + if (_PyObject_HasAttrId(obj, &PyId_upper)) { int res; - tmp = PyObject_GetAttrString(obj, "upper"); + tmp = _PyObject_GetAttrId(obj, &PyId_upper); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &upper, arena); if (res != 0) goto failed; @@ -5943,9 +6035,9 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) } else { upper = NULL; } - if (PyObject_HasAttrString(obj, "step")) { + if (_PyObject_HasAttrId(obj, &PyId_step)) { int res; - tmp = PyObject_GetAttrString(obj, "step"); + tmp = _PyObject_GetAttrId(obj, &PyId_step); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &step, arena); if (res != 0) goto failed; @@ -5965,11 +6057,11 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) if (isinstance) { asdl_seq* dims; - if (PyObject_HasAttrString(obj, "dims")) { + if (_PyObject_HasAttrId(obj, &PyId_dims)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "dims"); + tmp = _PyObject_GetAttrId(obj, &PyId_dims); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ExtSlice field \"dims\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -6001,9 +6093,9 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -6296,9 +6388,9 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena) expr_ty iter; asdl_seq* ifs; - if (PyObject_HasAttrString(obj, "target")) { + if (_PyObject_HasAttrId(obj, &PyId_target)) { int res; - tmp = PyObject_GetAttrString(obj, "target"); + tmp = _PyObject_GetAttrId(obj, &PyId_target); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &target, arena); if (res != 0) goto failed; @@ -6308,9 +6400,9 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from comprehension"); return 1; } - if (PyObject_HasAttrString(obj, "iter")) { + if (_PyObject_HasAttrId(obj, &PyId_iter)) { int res; - tmp = PyObject_GetAttrString(obj, "iter"); + tmp = _PyObject_GetAttrId(obj, &PyId_iter); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &iter, arena); if (res != 0) goto failed; @@ -6320,11 +6412,11 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from comprehension"); return 1; } - if (PyObject_HasAttrString(obj, "ifs")) { + if (_PyObject_HasAttrId(obj, &PyId_ifs)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "ifs"); + tmp = _PyObject_GetAttrId(obj, &PyId_ifs); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "comprehension field \"ifs\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -6365,9 +6457,9 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) *out = NULL; return 0; } - if (PyObject_HasAttrString(obj, "lineno")) { + if (_PyObject_HasAttrId(obj, &PyId_lineno)) { int res; - tmp = PyObject_GetAttrString(obj, "lineno"); + tmp = _PyObject_GetAttrId(obj, &PyId_lineno); if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &lineno, arena); if (res != 0) goto failed; @@ -6377,9 +6469,9 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from excepthandler"); return 1; } - if (PyObject_HasAttrString(obj, "col_offset")) { + if (_PyObject_HasAttrId(obj, &PyId_col_offset)) { int res; - tmp = PyObject_GetAttrString(obj, "col_offset"); + tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &col_offset, arena); if (res != 0) goto failed; @@ -6398,9 +6490,9 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) identifier name; asdl_seq* body; - if (PyObject_HasAttrString(obj, "type")) { + if (_PyObject_HasAttrId(obj, &PyId_type)) { int res; - tmp = PyObject_GetAttrString(obj, "type"); + tmp = _PyObject_GetAttrId(obj, &PyId_type); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &type, arena); if (res != 0) goto failed; @@ -6409,9 +6501,9 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) } else { type = NULL; } - if (PyObject_HasAttrString(obj, "name")) { + if (_PyObject_HasAttrId(obj, &PyId_name)) { int res; - tmp = PyObject_GetAttrString(obj, "name"); + tmp = _PyObject_GetAttrId(obj, &PyId_name); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; @@ -6420,11 +6512,11 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) } else { name = NULL; } - if (PyObject_HasAttrString(obj, "body")) { + if (_PyObject_HasAttrId(obj, &PyId_body)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); + tmp = _PyObject_GetAttrId(obj, &PyId_body); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ExceptHandler field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -6470,11 +6562,11 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) asdl_seq* defaults; asdl_seq* kw_defaults; - if (PyObject_HasAttrString(obj, "args")) { + if (_PyObject_HasAttrId(obj, &PyId_args)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "args"); + tmp = _PyObject_GetAttrId(obj, &PyId_args); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "arguments field \"args\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -6495,9 +6587,9 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from arguments"); return 1; } - if (PyObject_HasAttrString(obj, "vararg")) { + if (_PyObject_HasAttrId(obj, &PyId_vararg)) { int res; - tmp = PyObject_GetAttrString(obj, "vararg"); + tmp = _PyObject_GetAttrId(obj, &PyId_vararg); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &vararg, arena); if (res != 0) goto failed; @@ -6506,9 +6598,9 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } else { vararg = NULL; } - if (PyObject_HasAttrString(obj, "varargannotation")) { + if (_PyObject_HasAttrId(obj, &PyId_varargannotation)) { int res; - tmp = PyObject_GetAttrString(obj, "varargannotation"); + tmp = _PyObject_GetAttrId(obj, &PyId_varargannotation); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &varargannotation, arena); if (res != 0) goto failed; @@ -6517,11 +6609,11 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } else { varargannotation = NULL; } - if (PyObject_HasAttrString(obj, "kwonlyargs")) { + if (_PyObject_HasAttrId(obj, &PyId_kwonlyargs)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "kwonlyargs"); + tmp = _PyObject_GetAttrId(obj, &PyId_kwonlyargs); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "arguments field \"kwonlyargs\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -6542,9 +6634,9 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"kwonlyargs\" missing from arguments"); return 1; } - if (PyObject_HasAttrString(obj, "kwarg")) { + if (_PyObject_HasAttrId(obj, &PyId_kwarg)) { int res; - tmp = PyObject_GetAttrString(obj, "kwarg"); + tmp = _PyObject_GetAttrId(obj, &PyId_kwarg); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &kwarg, arena); if (res != 0) goto failed; @@ -6553,9 +6645,9 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } else { kwarg = NULL; } - if (PyObject_HasAttrString(obj, "kwargannotation")) { + if (_PyObject_HasAttrId(obj, &PyId_kwargannotation)) { int res; - tmp = PyObject_GetAttrString(obj, "kwargannotation"); + tmp = _PyObject_GetAttrId(obj, &PyId_kwargannotation); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &kwargannotation, arena); if (res != 0) goto failed; @@ -6564,11 +6656,11 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } else { kwargannotation = NULL; } - if (PyObject_HasAttrString(obj, "defaults")) { + if (_PyObject_HasAttrId(obj, &PyId_defaults)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "defaults"); + tmp = _PyObject_GetAttrId(obj, &PyId_defaults); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "arguments field \"defaults\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -6589,11 +6681,11 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"defaults\" missing from arguments"); return 1; } - if (PyObject_HasAttrString(obj, "kw_defaults")) { + if (_PyObject_HasAttrId(obj, &PyId_kw_defaults)) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "kw_defaults"); + tmp = _PyObject_GetAttrId(obj, &PyId_kw_defaults); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "arguments field \"kw_defaults\" must be a list, not a %.200s", tmp->ob_type->tp_name); @@ -6629,9 +6721,9 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) identifier arg; expr_ty annotation; - if (PyObject_HasAttrString(obj, "arg")) { + if (_PyObject_HasAttrId(obj, &PyId_arg)) { int res; - tmp = PyObject_GetAttrString(obj, "arg"); + tmp = _PyObject_GetAttrId(obj, &PyId_arg); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &arg, arena); if (res != 0) goto failed; @@ -6641,9 +6733,9 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"arg\" missing from arg"); return 1; } - if (PyObject_HasAttrString(obj, "annotation")) { + if (_PyObject_HasAttrId(obj, &PyId_annotation)) { int res; - tmp = PyObject_GetAttrString(obj, "annotation"); + tmp = _PyObject_GetAttrId(obj, &PyId_annotation); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &annotation, arena); if (res != 0) goto failed; @@ -6666,9 +6758,9 @@ obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena) identifier arg; expr_ty value; - if (PyObject_HasAttrString(obj, "arg")) { + if (_PyObject_HasAttrId(obj, &PyId_arg)) { int res; - tmp = PyObject_GetAttrString(obj, "arg"); + tmp = _PyObject_GetAttrId(obj, &PyId_arg); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &arg, arena); if (res != 0) goto failed; @@ -6678,9 +6770,9 @@ obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"arg\" missing from keyword"); return 1; } - if (PyObject_HasAttrString(obj, "value")) { + if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; @@ -6704,9 +6796,9 @@ obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena) identifier name; identifier asname; - if (PyObject_HasAttrString(obj, "name")) { + if (_PyObject_HasAttrId(obj, &PyId_name)) { int res; - tmp = PyObject_GetAttrString(obj, "name"); + tmp = _PyObject_GetAttrId(obj, &PyId_name); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; @@ -6716,9 +6808,9 @@ obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from alias"); return 1; } - if (PyObject_HasAttrString(obj, "asname")) { + if (_PyObject_HasAttrId(obj, &PyId_asname)) { int res; - tmp = PyObject_GetAttrString(obj, "asname"); + tmp = _PyObject_GetAttrId(obj, &PyId_asname); if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &asname, arena); if (res != 0) goto failed; @@ -6734,6 +6826,43 @@ failed: return 1; } +int +obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena) +{ + PyObject* tmp = NULL; + expr_ty context_expr; + expr_ty optional_vars; + + if (_PyObject_HasAttrId(obj, &PyId_context_expr)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_context_expr); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &context_expr, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from withitem"); + return 1; + } + if (_PyObject_HasAttrId(obj, &PyId_optional_vars)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_optional_vars); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &optional_vars, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + optional_vars = NULL; + } + *out = withitem(context_expr, optional_vars, arena); + return 0; +failed: + Py_XDECREF(tmp); + return 1; +} + static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, "_ast" @@ -6750,8 +6879,6 @@ PyInit__ast(void) NULL; if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) return NULL; - if (PyModule_AddStringConstant(m, "__version__", "82163") < 0) - return NULL; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return NULL; if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0) @@ -6785,10 +6912,8 @@ PyInit__ast(void) NULL; if (PyDict_SetItemString(d, "Raise", (PyObject*)Raise_type) < 0) return NULL; - if (PyDict_SetItemString(d, "TryExcept", (PyObject*)TryExcept_type) < - 0) return NULL; - if (PyDict_SetItemString(d, "TryFinally", (PyObject*)TryFinally_type) < - 0) return NULL; + if (PyDict_SetItemString(d, "Try", (PyObject*)Try_type) < 0) return + NULL; if (PyDict_SetItemString(d, "Assert", (PyObject*)Assert_type) < 0) return NULL; if (PyDict_SetItemString(d, "Import", (PyObject*)Import_type) < 0) @@ -6833,6 +6958,8 @@ PyInit__ast(void) (PyObject*)GeneratorExp_type) < 0) return NULL; if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return NULL; + if (PyDict_SetItemString(d, "YieldFrom", (PyObject*)YieldFrom_type) < + 0) return NULL; if (PyDict_SetItemString(d, "Compare", (PyObject*)Compare_type) < 0) return NULL; if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return @@ -6951,6 +7078,8 @@ PyInit__ast(void) return NULL; if (PyDict_SetItemString(d, "alias", (PyObject*)alias_type) < 0) return NULL; + if (PyDict_SetItemString(d, "withitem", (PyObject*)withitem_type) < 0) + return NULL; return m; } diff --git a/Python/_warnings.c b/Python/_warnings.c index c12db44..f33e477 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -18,11 +18,12 @@ static int check_matched(PyObject *obj, PyObject *arg) { PyObject *result; + _Py_IDENTIFIER(match); int rc; if (obj == Py_None) return 1; - result = PyObject_CallMethod(obj, "match", "O", arg); + result = _PyObject_CallMethodId(obj, &PyId_match, "O", arg); if (result == NULL) return -1; @@ -202,13 +203,13 @@ normalize_module(PyObject *filename) mod_str = _PyUnicode_AsString(filename); if (mod_str == NULL) - return NULL; - len = PyUnicode_GetSize(filename); + return NULL; + len = PyUnicode_GetLength(filename); if (len < 0) return NULL; if (len >= 3 && strncmp(mod_str + (len - 3), ".py", 3) == 0) { - module = PyUnicode_FromStringAndSize(mod_str, len-3); + module = PyUnicode_Substring(filename, 0, len-3); } else { module = filename; @@ -246,10 +247,11 @@ show_warning(PyObject *filename, int lineno, PyObject *text, PyObject PyObject *f_stderr; PyObject *name; char lineno_str[128]; + _Py_IDENTIFIER(__name__); PyOS_snprintf(lineno_str, sizeof(lineno_str), ":%d: ", lineno); - name = PyObject_GetAttrString(category, "__name__"); + name = _PyObject_GetAttrId(category, &PyId___name__); if (name == NULL) /* XXX Can an object lack a '__name__' attribute? */ return; @@ -409,10 +411,10 @@ warn_explicit(PyObject *category, PyObject *message, else { PyObject *res; - if (!PyMethod_Check(show_fxn) && !PyFunction_Check(show_fxn)) { + if (!PyCallable_Check(show_fxn)) { PyErr_SetString(PyExc_TypeError, "warnings.showwarning() must be set to a " - "function or method"); + "callable"); Py_DECREF(show_fxn); goto cleanup; } @@ -497,18 +499,28 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno, /* Setup filename. */ *filename = PyDict_GetItemString(globals, "__file__"); if (*filename != NULL && PyUnicode_Check(*filename)) { - Py_ssize_t len = PyUnicode_GetSize(*filename); - Py_UNICODE *unicode = PyUnicode_AS_UNICODE(*filename); + Py_ssize_t len; + int kind; + void *data; + + if (PyUnicode_READY(*filename)) + goto handle_error; + + len = PyUnicode_GetLength(*filename); + kind = PyUnicode_KIND(*filename); + data = PyUnicode_DATA(*filename); +#define ascii_lower(c) ((c <= 127) ? Py_TOLOWER(c) : 0) /* if filename.lower().endswith((".pyc", ".pyo")): */ if (len >= 4 && - unicode[len-4] == '.' && - Py_UNICODE_TOLOWER(unicode[len-3]) == 'p' && - Py_UNICODE_TOLOWER(unicode[len-2]) == 'y' && - (Py_UNICODE_TOLOWER(unicode[len-1]) == 'c' || - Py_UNICODE_TOLOWER(unicode[len-1]) == 'o')) + PyUnicode_READ(kind, data, len-4) == '.' && + ascii_lower(PyUnicode_READ(kind, data, len-3)) == 'p' && + ascii_lower(PyUnicode_READ(kind, data, len-2)) == 'y' && + (ascii_lower(PyUnicode_READ(kind, data, len-1)) == 'c' || + ascii_lower(PyUnicode_READ(kind, data, len-1)) == 'o')) { - *filename = PyUnicode_FromUnicode(unicode, len-1); + *filename = PyUnicode_Substring(*filename, 0, + PyUnicode_GET_LENGTH(*filename)-1); if (*filename == NULL) goto handle_error; } @@ -643,8 +655,9 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) return NULL; if (module_globals) { - static PyObject *get_source_name = NULL; - static PyObject *splitlines_name = NULL; + _Py_IDENTIFIER(get_source); + _Py_IDENTIFIER(splitlines); + PyObject *tmp; PyObject *loader; PyObject *module_name; PyObject *source; @@ -652,16 +665,10 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) PyObject *source_line; PyObject *returned; - if (get_source_name == NULL) { - get_source_name = PyUnicode_InternFromString("get_source"); - if (!get_source_name) - return NULL; - } - if (splitlines_name == NULL) { - splitlines_name = PyUnicode_InternFromString("splitlines"); - if (!splitlines_name) - return NULL; - } + if ((tmp = _PyUnicode_FromId(&PyId_get_source)) == NULL) + return NULL; + if ((tmp = _PyUnicode_FromId(&PyId_splitlines)) == NULL) + return NULL; /* Check/get the requisite pieces needed for the loader. */ loader = PyDict_GetItemString(module_globals, "__loader__"); @@ -671,11 +678,11 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) goto standard_call; /* Make sure the loader implements the optional get_source() method. */ - if (!PyObject_HasAttrString(loader, "get_source")) + if (!_PyObject_HasAttrId(loader, &PyId_get_source)) goto standard_call; /* Call get_source() to get the source code. */ - source = PyObject_CallMethodObjArgs(loader, get_source_name, - module_name, NULL); + source = PyObject_CallMethodObjArgs(loader, PyId_get_source.object, + module_name, NULL); if (!source) return NULL; else if (source == Py_None) { @@ -684,8 +691,9 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) } /* Split the source into lines. */ - source_list = PyObject_CallMethodObjArgs(source, splitlines_name, - NULL); + source_list = PyObject_CallMethodObjArgs(source, + PyId_splitlines.object, + NULL); Py_DECREF(source); if (!source_list) return NULL; @@ -950,23 +958,30 @@ _PyWarnings_Init(void) if (m == NULL) return NULL; - _filters = init_filters(); - if (_filters == NULL) - return NULL; + if (_filters == NULL) { + _filters = init_filters(); + if (_filters == NULL) + return NULL; + } Py_INCREF(_filters); if (PyModule_AddObject(m, "filters", _filters) < 0) return NULL; - _once_registry = PyDict_New(); - if (_once_registry == NULL) - return NULL; + if (_once_registry == NULL) { + _once_registry = PyDict_New(); + if (_once_registry == NULL) + return NULL; + } Py_INCREF(_once_registry); if (PyModule_AddObject(m, "_onceregistry", _once_registry) < 0) return NULL; - _default_action = PyUnicode_FromString("default"); - if (_default_action == NULL) - return NULL; + if (_default_action == NULL) { + _default_action = PyUnicode_FromString("default"); + if (_default_action == NULL) + return NULL; + } + Py_INCREF(_default_action); if (PyModule_AddObject(m, "_defaultaction", _default_action) < 0) return NULL; return m; diff --git a/Python/ast.c b/Python/ast.c index 6269c64..c565642 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1,24 +1,507 @@ /* * This file includes functions to transform a concrete syntax tree (CST) to - * an abstract syntax tree (AST). The main function is PyAST_FromNode(). + * an abstract syntax tree (AST). The main function is PyAST_FromNode(). * */ #include "Python.h" #include "Python-ast.h" -#include "grammar.h" #include "node.h" #include "ast.h" #include "token.h" -#include "parsetok.h" -#include "graminit.h" #include <assert.h> +static int validate_stmts(asdl_seq *); +static int validate_exprs(asdl_seq *, expr_context_ty, int); +static int validate_nonempty_seq(asdl_seq *, const char *, const char *); +static int validate_stmt(stmt_ty); +static int validate_expr(expr_ty, expr_context_ty); + +static int +validate_comprehension(asdl_seq *gens) +{ + int i; + if (!asdl_seq_LEN(gens)) { + PyErr_SetString(PyExc_ValueError, "comprehension with no generators"); + return 0; + } + for (i = 0; i < asdl_seq_LEN(gens); i++) { + comprehension_ty comp = asdl_seq_GET(gens, i); + if (!validate_expr(comp->target, Store) || + !validate_expr(comp->iter, Load) || + !validate_exprs(comp->ifs, Load, 0)) + return 0; + } + return 1; +} + +static int +validate_slice(slice_ty slice) +{ + switch (slice->kind) { + case Slice_kind: + return (!slice->v.Slice.lower || validate_expr(slice->v.Slice.lower, Load)) && + (!slice->v.Slice.upper || validate_expr(slice->v.Slice.upper, Load)) && + (!slice->v.Slice.step || validate_expr(slice->v.Slice.step, Load)); + case ExtSlice_kind: { + int i; + if (!validate_nonempty_seq(slice->v.ExtSlice.dims, "dims", "ExtSlice")) + return 0; + for (i = 0; i < asdl_seq_LEN(slice->v.ExtSlice.dims); i++) + if (!validate_slice(asdl_seq_GET(slice->v.ExtSlice.dims, i))) + return 0; + return 1; + } + case Index_kind: + return validate_expr(slice->v.Index.value, Load); + default: + PyErr_SetString(PyExc_SystemError, "unknown slice node"); + return 0; + } +} + +static int +validate_keywords(asdl_seq *keywords) +{ + int i; + for (i = 0; i < asdl_seq_LEN(keywords); i++) + if (!validate_expr(((keyword_ty)asdl_seq_GET(keywords, i))->value, Load)) + return 0; + return 1; +} + +static int +validate_args(asdl_seq *args) +{ + int i; + for (i = 0; i < asdl_seq_LEN(args); i++) { + arg_ty arg = asdl_seq_GET(args, i); + if (arg->annotation && !validate_expr(arg->annotation, Load)) + return 0; + } + return 1; +} + +static const char * +expr_context_name(expr_context_ty ctx) +{ + switch (ctx) { + case Load: + return "Load"; + case Store: + return "Store"; + case Del: + return "Del"; + case AugLoad: + return "AugLoad"; + case AugStore: + return "AugStore"; + case Param: + return "Param"; + default: + assert(0); + return "(unknown)"; + } +} + +static int +validate_arguments(arguments_ty args) +{ + if (!validate_args(args->args)) + return 0; + if (args->varargannotation) { + if (!args->vararg) { + PyErr_SetString(PyExc_ValueError, "varargannotation but no vararg on arguments"); + return 0; + } + if (!validate_expr(args->varargannotation, Load)) + return 0; + } + if (!validate_args(args->kwonlyargs)) + return 0; + if (args->kwargannotation) { + if (!args->kwarg) { + PyErr_SetString(PyExc_ValueError, "kwargannotation but no kwarg on arguments"); + return 0; + } + if (!validate_expr(args->kwargannotation, Load)) + return 0; + } + if (asdl_seq_LEN(args->defaults) > asdl_seq_LEN(args->args)) { + PyErr_SetString(PyExc_ValueError, "more positional defaults than args on arguments"); + return 0; + } + if (asdl_seq_LEN(args->kw_defaults) != asdl_seq_LEN(args->kwonlyargs)) { + PyErr_SetString(PyExc_ValueError, "length of kwonlyargs is not the same as " + "kw_defaults on arguments"); + return 0; + } + return validate_exprs(args->defaults, Load, 0) && validate_exprs(args->kw_defaults, Load, 1); +} + +static int +validate_expr(expr_ty exp, expr_context_ty ctx) +{ + int check_ctx = 1; + expr_context_ty actual_ctx; + + /* First check expression context. */ + switch (exp->kind) { + case Attribute_kind: + actual_ctx = exp->v.Attribute.ctx; + break; + case Subscript_kind: + actual_ctx = exp->v.Subscript.ctx; + break; + case Starred_kind: + actual_ctx = exp->v.Starred.ctx; + break; + case Name_kind: + actual_ctx = exp->v.Name.ctx; + break; + case List_kind: + actual_ctx = exp->v.List.ctx; + break; + case Tuple_kind: + actual_ctx = exp->v.Tuple.ctx; + break; + default: + if (ctx != Load) { + PyErr_Format(PyExc_ValueError, "expression which can't be " + "assigned to in %s context", expr_context_name(ctx)); + return 0; + } + check_ctx = 0; + } + if (check_ctx && actual_ctx != ctx) { + PyErr_Format(PyExc_ValueError, "expression must have %s context but has %s instead", + expr_context_name(ctx), expr_context_name(actual_ctx)); + return 0; + } + + /* Now validate expression. */ + switch (exp->kind) { + case BoolOp_kind: + if (asdl_seq_LEN(exp->v.BoolOp.values) < 2) { + PyErr_SetString(PyExc_ValueError, "BoolOp with less than 2 values"); + return 0; + } + return validate_exprs(exp->v.BoolOp.values, Load, 0); + case BinOp_kind: + return validate_expr(exp->v.BinOp.left, Load) && + validate_expr(exp->v.BinOp.right, Load); + case UnaryOp_kind: + return validate_expr(exp->v.UnaryOp.operand, Load); + case Lambda_kind: + return validate_arguments(exp->v.Lambda.args) && + validate_expr(exp->v.Lambda.body, Load); + case IfExp_kind: + return validate_expr(exp->v.IfExp.test, Load) && + validate_expr(exp->v.IfExp.body, Load) && + validate_expr(exp->v.IfExp.orelse, Load); + case Dict_kind: + if (asdl_seq_LEN(exp->v.Dict.keys) != asdl_seq_LEN(exp->v.Dict.values)) { + PyErr_SetString(PyExc_ValueError, + "Dict doesn't have the same number of keys as values"); + return 0; + } + return validate_exprs(exp->v.Dict.keys, Load, 0) && + validate_exprs(exp->v.Dict.values, Load, 0); + case Set_kind: + return validate_exprs(exp->v.Set.elts, Load, 0); +#define COMP(NAME) \ + case NAME ## _kind: \ + return validate_comprehension(exp->v.NAME.generators) && \ + validate_expr(exp->v.NAME.elt, Load); + COMP(ListComp) + COMP(SetComp) + COMP(GeneratorExp) +#undef COMP + case DictComp_kind: + return validate_comprehension(exp->v.DictComp.generators) && + validate_expr(exp->v.DictComp.key, Load) && + validate_expr(exp->v.DictComp.value, Load); + case Yield_kind: + return !exp->v.Yield.value || validate_expr(exp->v.Yield.value, Load); + case YieldFrom_kind: + return !exp->v.YieldFrom.value || + validate_expr(exp->v.YieldFrom.value, Load); + case Compare_kind: + if (!asdl_seq_LEN(exp->v.Compare.comparators)) { + PyErr_SetString(PyExc_ValueError, "Compare with no comparators"); + return 0; + } + if (asdl_seq_LEN(exp->v.Compare.comparators) != + asdl_seq_LEN(exp->v.Compare.ops)) { + PyErr_SetString(PyExc_ValueError, "Compare has a different number " + "of comparators and operands"); + return 0; + } + return validate_exprs(exp->v.Compare.comparators, Load, 0) && + validate_expr(exp->v.Compare.left, Load); + case Call_kind: + return validate_expr(exp->v.Call.func, Load) && + validate_exprs(exp->v.Call.args, Load, 0) && + validate_keywords(exp->v.Call.keywords) && + (!exp->v.Call.starargs || validate_expr(exp->v.Call.starargs, Load)) && + (!exp->v.Call.kwargs || validate_expr(exp->v.Call.kwargs, Load)); + case Num_kind: { + PyObject *n = exp->v.Num.n; + if (!PyLong_CheckExact(n) && !PyFloat_CheckExact(n) && + !PyComplex_CheckExact(n)) { + PyErr_SetString(PyExc_TypeError, "non-numeric type in Num"); + return 0; + } + return 1; + } + case Str_kind: { + PyObject *s = exp->v.Str.s; + if (!PyUnicode_CheckExact(s)) { + PyErr_SetString(PyExc_TypeError, "non-string type in Str"); + return 0; + } + return 1; + } + case Bytes_kind: { + PyObject *b = exp->v.Bytes.s; + if (!PyBytes_CheckExact(b)) { + PyErr_SetString(PyExc_TypeError, "non-bytes type in Bytes"); + return 0; + } + return 1; + } + case Attribute_kind: + return validate_expr(exp->v.Attribute.value, Load); + case Subscript_kind: + return validate_slice(exp->v.Subscript.slice) && + validate_expr(exp->v.Subscript.value, Load); + case Starred_kind: + return validate_expr(exp->v.Starred.value, ctx); + case List_kind: + return validate_exprs(exp->v.List.elts, ctx, 0); + case Tuple_kind: + return validate_exprs(exp->v.Tuple.elts, ctx, 0); + /* These last cases don't have any checking. */ + case Name_kind: + case Ellipsis_kind: + return 1; + default: + PyErr_SetString(PyExc_SystemError, "unexpected expression"); + return 0; + } +} + +static int +validate_nonempty_seq(asdl_seq *seq, const char *what, const char *owner) +{ + if (asdl_seq_LEN(seq)) + return 1; + PyErr_Format(PyExc_ValueError, "empty %s on %s", what, owner); + return 0; +} + +static int +validate_assignlist(asdl_seq *targets, expr_context_ty ctx) +{ + return validate_nonempty_seq(targets, "targets", ctx == Del ? "Delete" : "Assign") && + validate_exprs(targets, ctx, 0); +} + +static int +validate_body(asdl_seq *body, const char *owner) +{ + return validate_nonempty_seq(body, "body", owner) && validate_stmts(body); +} + +static int +validate_stmt(stmt_ty stmt) +{ + int i; + switch (stmt->kind) { + case FunctionDef_kind: + return validate_body(stmt->v.FunctionDef.body, "FunctionDef") && + validate_arguments(stmt->v.FunctionDef.args) && + validate_exprs(stmt->v.FunctionDef.decorator_list, Load, 0) && + (!stmt->v.FunctionDef.returns || + validate_expr(stmt->v.FunctionDef.returns, Load)); + case ClassDef_kind: + return validate_body(stmt->v.ClassDef.body, "ClassDef") && + validate_exprs(stmt->v.ClassDef.bases, Load, 0) && + validate_keywords(stmt->v.ClassDef.keywords) && + validate_exprs(stmt->v.ClassDef.decorator_list, Load, 0) && + (!stmt->v.ClassDef.starargs || validate_expr(stmt->v.ClassDef.starargs, Load)) && + (!stmt->v.ClassDef.kwargs || validate_expr(stmt->v.ClassDef.kwargs, Load)); + case Return_kind: + return !stmt->v.Return.value || validate_expr(stmt->v.Return.value, Load); + case Delete_kind: + return validate_assignlist(stmt->v.Delete.targets, Del); + case Assign_kind: + return validate_assignlist(stmt->v.Assign.targets, Store) && + validate_expr(stmt->v.Assign.value, Load); + case AugAssign_kind: + return validate_expr(stmt->v.AugAssign.target, Store) && + validate_expr(stmt->v.AugAssign.value, Load); + case For_kind: + return validate_expr(stmt->v.For.target, Store) && + validate_expr(stmt->v.For.iter, Load) && + validate_body(stmt->v.For.body, "For") && + validate_stmts(stmt->v.For.orelse); + case While_kind: + return validate_expr(stmt->v.While.test, Load) && + validate_body(stmt->v.While.body, "While") && + validate_stmts(stmt->v.While.orelse); + case If_kind: + return validate_expr(stmt->v.If.test, Load) && + validate_body(stmt->v.If.body, "If") && + validate_stmts(stmt->v.If.orelse); + case With_kind: + if (!validate_nonempty_seq(stmt->v.With.items, "items", "With")) + return 0; + for (i = 0; i < asdl_seq_LEN(stmt->v.With.items); i++) { + withitem_ty item = asdl_seq_GET(stmt->v.With.items, i); + if (!validate_expr(item->context_expr, Load) || + (item->optional_vars && !validate_expr(item->optional_vars, Store))) + return 0; + } + return validate_body(stmt->v.With.body, "With"); + case Raise_kind: + if (stmt->v.Raise.exc) { + return validate_expr(stmt->v.Raise.exc, Load) && + (!stmt->v.Raise.cause || validate_expr(stmt->v.Raise.cause, Load)); + } + if (stmt->v.Raise.cause) { + PyErr_SetString(PyExc_ValueError, "Raise with cause but no exception"); + return 0; + } + return 1; + case Try_kind: + if (!validate_body(stmt->v.Try.body, "Try")) + return 0; + if (!asdl_seq_LEN(stmt->v.Try.handlers) && + !asdl_seq_LEN(stmt->v.Try.finalbody)) { + PyErr_SetString(PyExc_ValueError, "Try has neither except handlers nor finalbody"); + return 0; + } + if (!asdl_seq_LEN(stmt->v.Try.handlers) && + asdl_seq_LEN(stmt->v.Try.orelse)) { + PyErr_SetString(PyExc_ValueError, "Try has orelse but no except handlers"); + return 0; + } + for (i = 0; i < asdl_seq_LEN(stmt->v.Try.handlers); i++) { + excepthandler_ty handler = asdl_seq_GET(stmt->v.Try.handlers, i); + if ((handler->v.ExceptHandler.type && + !validate_expr(handler->v.ExceptHandler.type, Load)) || + !validate_body(handler->v.ExceptHandler.body, "ExceptHandler")) + return 0; + } + return (!asdl_seq_LEN(stmt->v.Try.finalbody) || + validate_stmts(stmt->v.Try.finalbody)) && + (!asdl_seq_LEN(stmt->v.Try.orelse) || + validate_stmts(stmt->v.Try.orelse)); + case Assert_kind: + return validate_expr(stmt->v.Assert.test, Load) && + (!stmt->v.Assert.msg || validate_expr(stmt->v.Assert.msg, Load)); + case Import_kind: + return validate_nonempty_seq(stmt->v.Import.names, "names", "Import"); + case ImportFrom_kind: + if (stmt->v.ImportFrom.level < -1) { + PyErr_SetString(PyExc_ValueError, "ImportFrom level less than -1"); + return 0; + } + return validate_nonempty_seq(stmt->v.ImportFrom.names, "names", "ImportFrom"); + case Global_kind: + return validate_nonempty_seq(stmt->v.Global.names, "names", "Global"); + case Nonlocal_kind: + return validate_nonempty_seq(stmt->v.Nonlocal.names, "names", "Nonlocal"); + case Expr_kind: + return validate_expr(stmt->v.Expr.value, Load); + case Pass_kind: + case Break_kind: + case Continue_kind: + return 1; + default: + PyErr_SetString(PyExc_SystemError, "unexpected statement"); + return 0; + } +} + +static int +validate_stmts(asdl_seq *seq) +{ + int i; + for (i = 0; i < asdl_seq_LEN(seq); i++) { + stmt_ty stmt = asdl_seq_GET(seq, i); + if (stmt) { + if (!validate_stmt(stmt)) + return 0; + } + else { + PyErr_SetString(PyExc_ValueError, + "None disallowed in statement list"); + return 0; + } + } + return 1; +} + +static int +validate_exprs(asdl_seq *exprs, expr_context_ty ctx, int null_ok) +{ + int i; + for (i = 0; i < asdl_seq_LEN(exprs); i++) { + expr_ty expr = asdl_seq_GET(exprs, i); + if (expr) { + if (!validate_expr(expr, ctx)) + return 0; + } + else if (!null_ok) { + PyErr_SetString(PyExc_ValueError, + "None disallowed in expression list"); + return 0; + } + + } + return 1; +} + +int +PyAST_Validate(mod_ty mod) +{ + int res = 0; + + switch (mod->kind) { + case Module_kind: + res = validate_stmts(mod->v.Module.body); + break; + case Interactive_kind: + res = validate_stmts(mod->v.Interactive.body); + break; + case Expression_kind: + res = validate_expr(mod->v.Expression.body, Load); + break; + case Suite_kind: + PyErr_SetString(PyExc_ValueError, "Suite is not valid in the CPython compiler"); + break; + default: + PyErr_SetString(PyExc_SystemError, "impossible module node"); + res = 0; + break; + } + return res; +} + +/* This is done here, so defines like "test" don't interfere with AST use above. */ +#include "grammar.h" +#include "parsetok.h" +#include "graminit.h" + /* Data structure used internally */ struct compiling { char *c_encoding; /* source encoding */ PyArena *c_arena; /* arena for allocating memeory */ const char *c_filename; /* filename */ + PyObject *c_normalize; /* Normalization function from unicodedata. */ + PyObject *c_normalize_args; /* Normalization argument tuple. */ }; static asdl_seq *seq_for_testlist(struct compiling *, const node *); @@ -38,45 +521,58 @@ static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode); static PyObject *parsestrplus(struct compiling *, const node *n, int *bytesmode); -#ifndef LINENO -#define LINENO(n) ((n)->n_lineno) -#endif - #define COMP_GENEXP 0 #define COMP_LISTCOMP 1 #define COMP_SETCOMP 2 +static int +init_normalization(struct compiling *c) +{ + PyObject *m = PyImport_ImportModuleNoBlock("unicodedata"); + if (!m) + return 0; + c->c_normalize = PyObject_GetAttrString(m, "normalize"); + Py_DECREF(m); + if (!c->c_normalize) + return 0; + c->c_normalize_args = Py_BuildValue("(sN)", "NFKC", Py_None); + PyTuple_SET_ITEM(c->c_normalize_args, 1, NULL); + if (!c->c_normalize_args) { + Py_CLEAR(c->c_normalize); + return 0; + } + return 1; +} + static identifier -new_identifier(const char* n, PyArena *arena) +new_identifier(const char* n, struct compiling *c) { PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL); - Py_UNICODE *u; if (!id) return NULL; - u = PyUnicode_AS_UNICODE(id); + /* PyUnicode_DecodeUTF8 should always return a ready string. */ + assert(PyUnicode_IS_READY(id)); /* Check whether there are non-ASCII characters in the identifier; if so, normalize to NFKC. */ - for (; *u; u++) { - if (*u >= 128) { - PyObject *m = PyImport_ImportModuleNoBlock("unicodedata"); - PyObject *id2; - if (!m) - return NULL; - id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id); - Py_DECREF(m); - if (!id2) - return NULL; + if (!PyUnicode_IS_ASCII(id)) { + PyObject *id2; + if (!c->c_normalize && !init_normalization(c)) { Py_DECREF(id); - id = id2; - break; + return NULL; } + PyTuple_SET_ITEM(c->c_normalize_args, 1, id); + id2 = PyObject_Call(c->c_normalize, c->c_normalize_args, NULL); + Py_DECREF(id); + if (!id2) + return NULL; + id = id2; } PyUnicode_InternInPlace(&id); - PyArena_AddPyObject(arena, id); + PyArena_AddPyObject(c->c_arena, id); return id; } -#define NEW_IDENTIFIER(n) new_identifier(STR(n), c->c_arena) +#define NEW_IDENTIFIER(n) new_identifier(STR(n), c) /* This routine provides an invalid object for the syntax error. The outermost routine must unpack this error and create the @@ -226,13 +722,14 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename, stmt_ty s; node *ch; struct compiling c; + mod_ty res = NULL; if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) { c.c_encoding = "utf-8"; if (TYPE(n) == encoding_decl) { #if 0 ast_error(n, "encoding declaration in Unicode string"); - goto error; + goto out; #endif n = CHILD(n, 0); } @@ -245,13 +742,14 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename, } c.c_arena = arena; c.c_filename = filename; + c.c_normalize = c.c_normalize_args = NULL; k = 0; switch (TYPE(n)) { case file_input: stmts = asdl_seq_new(num_stmts(n), arena); if (!stmts) - return NULL; + goto out; for (i = 0; i < NCH(n) - 1; i++) { ch = CHILD(n, i); if (TYPE(ch) == NEWLINE) @@ -261,7 +759,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename, if (num == 1) { s = ast_for_stmt(&c, ch); if (!s) - goto error; + goto out; asdl_seq_SET(stmts, k++, s); } else { @@ -270,42 +768,44 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename, for (j = 0; j < num; j++) { s = ast_for_stmt(&c, CHILD(ch, j * 2)); if (!s) - goto error; + goto out; asdl_seq_SET(stmts, k++, s); } } } - return Module(stmts, arena); + res = Module(stmts, arena); + break; case eval_input: { expr_ty testlist_ast; /* XXX Why not comp_for here? */ testlist_ast = ast_for_testlist(&c, CHILD(n, 0)); if (!testlist_ast) - goto error; - return Expression(testlist_ast, arena); + goto out; + res = Expression(testlist_ast, arena); + break; } case single_input: if (TYPE(CHILD(n, 0)) == NEWLINE) { stmts = asdl_seq_new(1, arena); if (!stmts) - goto error; + goto out; asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset, arena)); if (!asdl_seq_GET(stmts, 0)) - goto error; - return Interactive(stmts, arena); + goto out; + res = Interactive(stmts, arena); } else { n = CHILD(n, 0); num = num_stmts(n); stmts = asdl_seq_new(num, arena); if (!stmts) - goto error; + goto out; if (num == 1) { s = ast_for_stmt(&c, n); if (!s) - goto error; + goto out; asdl_seq_SET(stmts, 0, s); } else { @@ -316,21 +816,28 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename, break; s = ast_for_stmt(&c, CHILD(n, i)); if (!s) - goto error; + goto out; asdl_seq_SET(stmts, i / 2, s); } } - return Interactive(stmts, arena); + res = Interactive(stmts, arena); } + break; default: PyErr_Format(PyExc_SystemError, "invalid node %d for PyAST_FromNode", TYPE(n)); - goto error; + goto out; } - error: - ast_error_finish(filename); - return NULL; + out: + if (c.c_normalize) { + Py_DECREF(c.c_normalize); + PyTuple_SET_ITEM(c.c_normalize_args, 1, NULL); + Py_DECREF(c.c_normalize_args); + } + if (!res) + ast_error_finish(filename); + return res; } /* Return the AST repr. of the operator represented as syntax (|, ^, etc.) @@ -468,6 +975,7 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) expr_name = "generator expression"; break; case Yield_kind: + case YieldFrom_kind: expr_name = "yield expression"; break; case ListComp_kind: @@ -1895,12 +2403,25 @@ ast_for_expr(struct compiling *c, const node *n) } return ast_for_binop(c, n); case yield_expr: { + node *an = NULL; + node *en = NULL; + int is_from = 0; expr_ty exp = NULL; - if (NCH(n) == 2) { - exp = ast_for_testlist(c, CHILD(n, 1)); + if (NCH(n) > 1) + an = CHILD(n, 1); /* yield_arg */ + if (an) { + en = CHILD(an, NCH(an) - 1); + if (NCH(an) == 2) { + is_from = 1; + exp = ast_for_expr(c, en); + } + else + exp = ast_for_testlist(c, en); if (!exp) return NULL; } + if (is_from) + return YieldFrom(exp, LINENO(n), n->n_col_offset, c->c_arena); return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena); } case factor: @@ -1925,7 +2446,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) /* arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) - argument: [test '='] test [comp_for] # Really [keyword '='] test + argument: [test '='] (test) [comp_for] # Really [keyword '='] test */ int i, nargs, nkeywords, ngens; @@ -2009,13 +2530,13 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) * then is very confusing. */ if (e->kind == Lambda_kind) { - ast_error(CHILD(ch, 0), "lambda cannot contain assignment"); - return NULL; + ast_error(CHILD(ch, 0), "lambda cannot contain assignment"); + return NULL; } else if (e->kind != Name_kind) { - ast_error(CHILD(ch, 0), "keyword can't be an expression"); - return NULL; + ast_error(CHILD(ch, 0), "keyword can't be an expression"); + return NULL; } else if (forbidden_name(e->v.Name.id, ch, 1)) { - return NULL; + return NULL; } key = e->v.Name.id; for (k = 0; k < nkeywords; k++) { @@ -2219,7 +2740,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n) continue_stmt: 'continue' return_stmt: 'return' [testlist] yield_stmt: yield_expr - yield_expr: 'yield' testlist + yield_expr: 'yield' testlist | 'yield' 'from' test raise_stmt: 'raise' [test [',' test [',' test]]] */ node *ch; @@ -2893,7 +3414,7 @@ ast_for_try_stmt(struct compiling *c, const node *n) { const int nch = NCH(n); int n_except = (nch - 3)/3; - asdl_seq *body, *orelse = NULL, *finally = NULL; + asdl_seq *body, *handlers = NULL, *orelse = NULL, *finally = NULL; REQ(n, try_stmt); @@ -2934,9 +3455,8 @@ ast_for_try_stmt(struct compiling *c, const node *n) if (n_except > 0) { int i; - stmt_ty except_st; /* process except statements to create a try ... except */ - asdl_seq *handlers = asdl_seq_new(n_except, c->c_arena); + handlers = asdl_seq_new(n_except, c->c_arena); if (handlers == NULL) return NULL; @@ -2947,28 +3467,15 @@ ast_for_try_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(handlers, i, e); } - - except_st = TryExcept(body, handlers, orelse, LINENO(n), - n->n_col_offset, c->c_arena); - if (!finally) - return except_st; - - /* if a 'finally' is present too, we nest the TryExcept within a - TryFinally to emulate try ... except ... finally */ - body = asdl_seq_new(1, c->c_arena); - if (body == NULL) - return NULL; - asdl_seq_SET(body, 0, except_st); } - /* must be a try ... finally (except clauses are in body, if any exist) */ - assert(finally != NULL); - return TryFinally(body, finally, LINENO(n), n->n_col_offset, c->c_arena); + assert(finally != NULL || asdl_seq_LEN(handlers)); + return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset, c->c_arena); } /* with_item: test ['as' expr] */ -static stmt_ty -ast_for_with_item(struct compiling *c, const node *n, asdl_seq *content) +static withitem_ty +ast_for_with_item(struct compiling *c, const node *n) { expr_ty context_expr, optional_vars = NULL; @@ -2987,43 +3494,32 @@ ast_for_with_item(struct compiling *c, const node *n, asdl_seq *content) } } - return With(context_expr, optional_vars, content, LINENO(n), - n->n_col_offset, c->c_arena); + return withitem(context_expr, optional_vars, c->c_arena); } /* with_stmt: 'with' with_item (',' with_item)* ':' suite */ static stmt_ty ast_for_with_stmt(struct compiling *c, const node *n) { - int i; - stmt_ty ret; - asdl_seq *inner; + int i, n_items; + asdl_seq *items, *body; REQ(n, with_stmt); - /* process the with items inside-out */ - i = NCH(n) - 1; - /* the suite of the innermost with item is the suite of the with stmt */ - inner = ast_for_suite(c, CHILD(n, i)); - if (!inner) - return NULL; - - for (;;) { - i -= 2; - ret = ast_for_with_item(c, CHILD(n, i), inner); - if (!ret) + n_items = (NCH(n) - 2) / 2; + items = asdl_seq_new(n_items, c->c_arena); + for (i = 1; i < NCH(n) - 2; i += 2) { + withitem_ty item = ast_for_with_item(c, CHILD(n, i)); + if (!item) return NULL; - /* was this the last item? */ - if (i == 1) - break; - /* if not, wrap the result so far in a new sequence */ - inner = asdl_seq_new(1, c->c_arena); - if (!inner) - return NULL; - asdl_seq_SET(inner, 0, ret); + asdl_seq_SET(items, (i - 1) / 2, item); } - return ret; + body = ast_for_suite(c, CHILD(n, NCH(n) - 1)); + if (!body) + return NULL; + + return With(items, body, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty @@ -3207,20 +3703,14 @@ parsenumber(struct compiling *c, const char *s) } static PyObject * -decode_utf8(struct compiling *c, const char **sPtr, const char *end, char* encoding) +decode_utf8(struct compiling *c, const char **sPtr, const char *end) { - PyObject *u, *v; char *s, *t; t = s = (char *)*sPtr; /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */ while (s < end && (*s & 0x80)) s++; *sPtr = s; - u = PyUnicode_DecodeUTF8(t, s - t, NULL); - if (u == NULL) - return NULL; - v = PyUnicode_AsEncodedString(u, encoding, NULL); - Py_DECREF(u); - return v; + return PyUnicode_DecodeUTF8(t, s - t, NULL); } static PyObject * @@ -3232,7 +3722,6 @@ decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, cons const char *end; if (encoding == NULL) { - buf = (char *)s; u = NULL; } else { /* check for integer overflow */ @@ -3255,22 +3744,20 @@ decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, cons } if (*s & 0x80) { /* XXX inefficient */ PyObject *w; - char *r; - Py_ssize_t rn, i; - w = decode_utf8(c, &s, end, "utf-32-be"); + int kind; + void *data; + Py_ssize_t len, i; + w = decode_utf8(c, &s, end); if (w == NULL) { Py_DECREF(u); return NULL; } - r = PyBytes_AS_STRING(w); - rn = Py_SIZE(w); - assert(rn % 4 == 0); - for (i = 0; i < rn; i += 4) { - sprintf(p, "\\U%02x%02x%02x%02x", - r[i + 0] & 0xFF, - r[i + 1] & 0xFF, - r[i + 2] & 0xFF, - r[i + 3] & 0xFF); + kind = PyUnicode_KIND(w); + data = PyUnicode_DATA(w); + len = PyUnicode_GET_LENGTH(w); + for (i = 0; i < len; i++) { + Py_UCS4 chr = PyUnicode_READ(kind, data, i); + sprintf(p, "\\U%08x", chr); p += 10; } /* Should be impossible to overflow */ @@ -3304,13 +3791,18 @@ parsestr(struct compiling *c, const node *n, int *bytesmode) int rawmode = 0; int need_encoding; if (isalpha(quote)) { - if (quote == 'b' || quote == 'B') { - quote = *++s; - *bytesmode = 1; - } - if (quote == 'r' || quote == 'R') { - quote = *++s; - rawmode = 1; + while (!*bytesmode || !rawmode) { + if (quote == 'b' || quote == 'B') { + quote = *++s; + *bytesmode = 1; + } + else if (quote == 'r' || quote == 'R') { + quote = *++s; + rawmode = 1; + } + else { + break; + } } } if (quote != '\'' && quote != '\"') { diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index e2ff0ba..81402fc 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -6,6 +6,9 @@ #include "node.h" #include "code.h" +#include "asdl.h" +#include "ast.h" + #include <ctype.h> #ifdef HAVE_LANGINFO_H @@ -18,20 +21,20 @@ Don't forget to modify PyUnicode_DecodeFSDefault() if you touch any of the values for Py_FileSystemDefaultEncoding! */ -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#ifdef HAVE_MBCS const char *Py_FileSystemDefaultEncoding = "mbcs"; int Py_HasFileSystemDefaultEncoding = 1; #elif defined(__APPLE__) const char *Py_FileSystemDefaultEncoding = "utf-8"; int Py_HasFileSystemDefaultEncoding = 1; -#elif defined(HAVE_LANGINFO_H) && defined(CODESET) +#else const char *Py_FileSystemDefaultEncoding = NULL; /* set by initfsencoding() */ int Py_HasFileSystemDefaultEncoding = 0; -#else -const char *Py_FileSystemDefaultEncoding = "utf-8"; -int Py_HasFileSystemDefaultEncoding = 1; #endif +_Py_IDENTIFIER(fileno); +_Py_IDENTIFIER(flush); + static PyObject * builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds) { @@ -39,6 +42,7 @@ builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds) PyObject *cls = NULL; Py_ssize_t nargs; int isclass; + _Py_IDENTIFIER(__prepare__); assert(args != NULL); if (!PyTuple_Check(args)) { @@ -99,6 +103,7 @@ builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds) Py_INCREF(meta); isclass = 1; /* meta is really a class */ } + if (isclass) { /* meta is really a class, so check for a more derived metaclass, or possible metaclass conflicts: */ @@ -118,7 +123,7 @@ builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds) } /* else: meta is not a class, so we cannot do the metaclass calculation, so we will use the explicitly given object as it is */ - prep = PyObject_GetAttrString(meta, "__prepare__"); + prep = _PyObject_GetAttrId(meta, &PyId___prepare__); if (prep == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -181,17 +186,14 @@ builtin___import__(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"name", "globals", "locals", "fromlist", "level", 0}; - char *name; - PyObject *globals = NULL; - PyObject *locals = NULL; - PyObject *fromlist = NULL; + PyObject *name, *globals = NULL, *locals = NULL, *fromlist = NULL; int level = -1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|OOOi:__import__", + if (!PyArg_ParseTupleAndKeywords(args, kwds, "U|OOOi:__import__", kwlist, &name, &globals, &locals, &fromlist, &level)) return NULL; - return PyImport_ImportModuleLevel(name, globals, locals, - fromlist, level); + return PyImport_ImportModuleLevelObject(name, globals, locals, + fromlist, level); } PyDoc_STRVAR(import_doc, @@ -516,17 +518,10 @@ builtin_chr(PyObject *self, PyObject *args) return PyUnicode_FromOrdinal(x); } -PyDoc_VAR(chr_doc) = PyDoc_STR( +PyDoc_STRVAR(chr_doc, "chr(i) -> Unicode character\n\ \n\ -Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff." -) -#ifndef Py_UNICODE_WIDE -PyDoc_STR( -"\nIf 0x10000 <= i, a surrogate pair is returned." -) -#endif -; +Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff."); static char * @@ -537,8 +532,8 @@ source_as_string(PyObject *cmd, char *funcname, char *what, PyCompilerFlags *cf) if (PyUnicode_Check(cmd)) { cf->cf_flags |= PyCF_IGNORE_COOKIE; - cmd = _PyUnicode_AsDefaultEncodedString(cmd, NULL); - if (cmd == NULL) + str = PyUnicode_AsUTF8AndSize(cmd, &size); + if (str == NULL) return NULL; } else if (!PyObject_CheckReadBuffer(cmd)) { @@ -547,9 +542,10 @@ source_as_string(PyObject *cmd, char *funcname, char *what, PyCompilerFlags *cf) funcname, what); return NULL; } - if (PyObject_AsReadBuffer(cmd, (const void **)&str, &size) < 0) { + else if (PyObject_AsReadBuffer(cmd, (const void **)&str, &size) < 0) { return NULL; } + if (strlen(str) != size) { PyErr_SetString(PyExc_TypeError, "source code string cannot contain null bytes"); @@ -636,6 +632,10 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds) PyArena_Free(arena); goto error; } + if (!PyAST_Validate(mod)) { + PyArena_Free(arena); + goto error; + } result = (PyObject*)PyAST_CompileEx(mod, filename, &cf, optimize, arena); PyArena_Free(arena); @@ -791,7 +791,6 @@ builtin_exec(PyObject *self, PyObject *args) { PyObject *v; PyObject *prog, *globals = Py_None, *locals = Py_None; - int plain = 0; if (!PyArg_UnpackTuple(args, "exec", 1, 3, &prog, &globals, &locals)) return NULL; @@ -800,7 +799,6 @@ builtin_exec(PyObject *self, PyObject *args) globals = PyEval_GetGlobals(); if (locals == Py_None) { locals = PyEval_GetLocals(); - plain = 1; } if (!globals || !locals) { PyErr_SetString(PyExc_SystemError, @@ -1116,7 +1114,7 @@ builtin_next(PyObject *self, PyObject *args) return NULL; if (!PyIter_Check(it)) { PyErr_Format(PyExc_TypeError, - "%.200s object is not an iterator", + "'%.200s' object is not an iterator", it->ob_type->tp_name); return NULL; } @@ -1422,24 +1420,13 @@ builtin_ord(PyObject *self, PyObject* obj) } } else if (PyUnicode_Check(obj)) { - size = PyUnicode_GET_SIZE(obj); + if (PyUnicode_READY(obj) == -1) + return NULL; + size = PyUnicode_GET_LENGTH(obj); if (size == 1) { - ord = (long)*PyUnicode_AS_UNICODE(obj); + ord = (long)PyUnicode_READ_CHAR(obj, 0); return PyLong_FromLong(ord); } -#ifndef Py_UNICODE_WIDE - if (size == 2) { - /* Decode a valid surrogate pair */ - int c0 = PyUnicode_AS_UNICODE(obj)[0]; - int c1 = PyUnicode_AS_UNICODE(obj)[1]; - if (0xD800 <= c0 && c0 <= 0xDBFF && - 0xDC00 <= c1 && c1 <= 0xDFFF) { - ord = ((((c0 & 0x03FF) << 10) | (c1 & 0x03FF)) + - 0x00010000); - return PyLong_FromLong(ord); - } - } -#endif } else if (PyByteArray_Check(obj)) { /* XXX Hopefully this is temporary */ @@ -1497,15 +1484,15 @@ equivalent to (x**y) % z, but may be more efficient (e.g. for longs)."); static PyObject * builtin_print(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"sep", "end", "file", 0}; + static char *kwlist[] = {"sep", "end", "file", "flush", 0}; static PyObject *dummy_args; - PyObject *sep = NULL, *end = NULL, *file = NULL; + PyObject *sep = NULL, *end = NULL, *file = NULL, *flush = NULL; int i, err; if (dummy_args == NULL && !(dummy_args = PyTuple_New(0))) - return NULL; - if (!PyArg_ParseTupleAndKeywords(dummy_args, kwds, "|OOO:print", - kwlist, &sep, &end, &file)) + return NULL; + if (!PyArg_ParseTupleAndKeywords(dummy_args, kwds, "|OOOO:print", + kwlist, &sep, &end, &file, &flush)) return NULL; if (file == NULL || file == Py_None) { file = PySys_GetObject("stdout"); @@ -1556,6 +1543,20 @@ builtin_print(PyObject *self, PyObject *args, PyObject *kwds) if (err) return NULL; + if (flush != NULL) { + PyObject *tmp; + int do_flush = PyObject_IsTrue(flush); + if (do_flush == -1) + return NULL; + else if (do_flush) { + tmp = PyObject_CallMethod(file, "flush", ""); + if (tmp == NULL) + return NULL; + else + Py_DECREF(tmp); + } + } + Py_RETURN_NONE; } @@ -1602,7 +1603,7 @@ builtin_input(PyObject *self, PyObject *args) } /* First of all, flush stderr */ - tmp = PyObject_CallMethod(ferr, "flush", ""); + tmp = _PyObject_CallMethodId(ferr, &PyId_flush, ""); if (tmp == NULL) PyErr_Clear(); else @@ -1611,7 +1612,7 @@ builtin_input(PyObject *self, PyObject *args) /* We should only use (GNU) readline if Python's sys.stdin and sys.stdout are the same as C's stdin and stdout, because we need to pass it those. */ - tmp = PyObject_CallMethod(fin, "fileno", ""); + tmp = _PyObject_CallMethodId(fin, &PyId_fileno, ""); if (tmp == NULL) { PyErr_Clear(); tty = 0; @@ -1624,7 +1625,7 @@ builtin_input(PyObject *self, PyObject *args) tty = fd == fileno(stdin) && isatty(fd); } if (tty) { - tmp = PyObject_CallMethod(fout, "fileno", ""); + tmp = _PyObject_CallMethodId(fout, &PyId_fileno, ""); if (tmp == NULL) PyErr_Clear(); else { @@ -1646,9 +1647,11 @@ builtin_input(PyObject *self, PyObject *args) char *stdin_encoding_str, *stdin_errors_str; PyObject *result; size_t len; + _Py_IDENTIFIER(encoding); + _Py_IDENTIFIER(errors); - stdin_encoding = PyObject_GetAttrString(fin, "encoding"); - stdin_errors = PyObject_GetAttrString(fin, "errors"); + stdin_encoding = _PyObject_GetAttrId(fin, &PyId_encoding); + stdin_errors = _PyObject_GetAttrId(fin, &PyId_errors); if (!stdin_encoding || !stdin_errors) /* stdin is a text stream, so it must have an encoding. */ @@ -1657,7 +1660,7 @@ builtin_input(PyObject *self, PyObject *args) stdin_errors_str = _PyUnicode_AsString(stdin_errors); if (!stdin_encoding_str || !stdin_errors_str) goto _readline_errors; - tmp = PyObject_CallMethod(fout, "flush", ""); + tmp = _PyObject_CallMethodId(fout, &PyId_flush, ""); if (tmp == NULL) PyErr_Clear(); else @@ -1666,8 +1669,8 @@ builtin_input(PyObject *self, PyObject *args) /* We have a prompt, encode it as stdout would */ char *stdout_encoding_str, *stdout_errors_str; PyObject *stringpo; - stdout_encoding = PyObject_GetAttrString(fout, "encoding"); - stdout_errors = PyObject_GetAttrString(fout, "errors"); + stdout_encoding = _PyObject_GetAttrId(fout, &PyId_encoding); + stdout_errors = _PyObject_GetAttrId(fout, &PyId_errors); if (!stdout_encoding || !stdout_errors) goto _readline_errors; stdout_encoding_str = _PyUnicode_AsString(stdout_encoding); @@ -1737,7 +1740,7 @@ builtin_input(PyObject *self, PyObject *args) if (PyFile_WriteObject(promptarg, fout, Py_PRINT_RAW) != 0) return NULL; } - tmp = PyObject_CallMethod(fout, "flush", ""); + tmp = _PyObject_CallMethodId(fout, &PyId_flush, ""); if (tmp == NULL) PyErr_Clear(); else @@ -1819,6 +1822,7 @@ builtin_sorted(PyObject *self, PyObject *args, PyObject *kwds) PyObject *callable; static char *kwlist[] = {"iterable", "key", "reverse", 0}; int reverse; + _Py_IDENTIFIER(sort); /* args 1-3 should match listsort in Objects/listobject.c */ if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Oi:sorted", @@ -1829,7 +1833,7 @@ builtin_sorted(PyObject *self, PyObject *args, PyObject *kwds) if (newlist == NULL) return NULL; - callable = PyObject_GetAttrString(newlist, "sort"); + callable = _PyObject_GetAttrId(newlist, &PyId_sort); if (callable == NULL) { Py_DECREF(newlist); return NULL; @@ -1875,7 +1879,8 @@ builtin_vars(PyObject *self, PyObject *args) Py_INCREF(d); } else { - d = PyObject_GetAttrString(v, "__dict__"); + _Py_IDENTIFIER(__dict__); + d = _PyObject_GetAttrId(v, &PyId___dict__); if (d == NULL) { PyErr_SetString(PyExc_TypeError, "vars() argument must have __dict__ attribute"); @@ -1919,12 +1924,18 @@ builtin_sum(PyObject *self, PyObject *args) Py_DECREF(iter); return NULL; } - if (PyByteArray_Check(result)) { + if (PyBytes_Check(result)) { PyErr_SetString(PyExc_TypeError, "sum() can't sum bytes [use b''.join(seq) instead]"); Py_DECREF(iter); return NULL; } + if (PyByteArray_Check(result)) { + PyErr_SetString(PyExc_TypeError, + "sum() can't sum bytearray [use b''.join(seq) instead]"); + Py_DECREF(iter); + return NULL; + } Py_INCREF(result); } diff --git a/Python/ceval.c b/Python/ceval.c index f0ea7c9..06bff4c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -138,7 +138,7 @@ static void format_exc_check_arg(PyObject *, const char *, PyObject *); static void format_exc_unbound(PyCodeObject *co, int oparg); static PyObject * unicode_concatenate(PyObject *, PyObject *, PyFrameObject *, unsigned char *); -static PyObject * special_lookup(PyObject *, char *, PyObject **); +static PyObject * special_lookup(PyObject *, _Py_Identifier *); #define NAME_ERROR_MSG \ "name '%.200s' is not defined" @@ -372,6 +372,7 @@ PyEval_ReleaseThread(PyThreadState *tstate) void PyEval_ReInitThreads(void) { + _Py_IDENTIFIER(_after_fork); PyObject *threading, *result; PyThreadState *tstate = PyThreadState_GET(); @@ -392,7 +393,7 @@ PyEval_ReInitThreads(void) PyErr_Clear(); return; } - result = PyObject_CallMethod(threading, "_after_fork", NULL); + result = _PyObject_CallMethodId(threading, &PyId__after_fork, NULL); if (result == NULL) PyErr_WriteUnraisable(threading); else @@ -491,7 +492,6 @@ static struct { } pendingcalls[NPENDINGCALLS]; static int pendingfirst = 0; static int pendinglast = 0; -static char pendingbusy = 0; int Py_AddPendingCall(int (*func)(void *), void *arg) @@ -538,6 +538,7 @@ Py_AddPendingCall(int (*func)(void *), void *arg) int Py_MakePendingCalls(void) { + static int busy = 0; int i; int r = 0; @@ -552,9 +553,9 @@ Py_MakePendingCalls(void) if (main_thread && PyThread_get_thread_ident() != main_thread) return 0; /* don't perform recursive pending calls */ - if (pendingbusy) + if (busy) return 0; - pendingbusy = 1; + busy = 1; /* perform a bounded number of calls, in case of recursion */ for (i=0; i<NPENDINGCALLS; i++) { int j; @@ -583,7 +584,7 @@ Py_MakePendingCalls(void) if (r) break; } - pendingbusy = 0; + busy = 0; return r; } @@ -749,6 +750,9 @@ enum why_code { WHY_SILENCED = 0x0080 /* Exception silenced by 'with' */ }; +static void save_exc_state(PyThreadState *, PyFrameObject *); +static void swap_exc_state(PyThreadState *, PyFrameObject *); +static void restore_and_clear_exc_state(PyThreadState *, PyFrameObject *); static enum why_code do_raise(PyObject *, PyObject *); static int unpack_iterable(PyObject *, int, int, PyObject **); @@ -817,10 +821,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) unsigned char *first_instr; PyObject *names; PyObject *consts; -#if defined(Py_DEBUG) || defined(LLTRACE) - /* Make it easier to find out where we are with a debugger */ - char *filename; -#endif /* Computed GOTOs, or the-optimization-commonly-but-improperly-known-as-"threaded code" @@ -1114,59 +1114,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_XDECREF(traceback); \ } -#define SAVE_EXC_STATE() \ - { \ - PyObject *type, *value, *traceback; \ - Py_XINCREF(tstate->exc_type); \ - Py_XINCREF(tstate->exc_value); \ - Py_XINCREF(tstate->exc_traceback); \ - type = f->f_exc_type; \ - value = f->f_exc_value; \ - traceback = f->f_exc_traceback; \ - f->f_exc_type = tstate->exc_type; \ - f->f_exc_value = tstate->exc_value; \ - f->f_exc_traceback = tstate->exc_traceback; \ - Py_XDECREF(type); \ - Py_XDECREF(value); \ - Py_XDECREF(traceback); \ - } - -#define SWAP_EXC_STATE() \ - { \ - PyObject *tmp; \ - tmp = tstate->exc_type; \ - tstate->exc_type = f->f_exc_type; \ - f->f_exc_type = tmp; \ - tmp = tstate->exc_value; \ - tstate->exc_value = f->f_exc_value; \ - f->f_exc_value = tmp; \ - tmp = tstate->exc_traceback; \ - tstate->exc_traceback = f->f_exc_traceback; \ - f->f_exc_traceback = tmp; \ - } - -#define RESTORE_AND_CLEAR_EXC_STATE() \ - { \ - PyObject *type, *value, *tb; \ - type = tstate->exc_type; \ - value = tstate->exc_value; \ - tb = tstate->exc_traceback; \ - tstate->exc_type = f->f_exc_type; \ - tstate->exc_value = f->f_exc_value; \ - tstate->exc_traceback = f->f_exc_traceback; \ - f->f_exc_type = NULL; \ - f->f_exc_value = NULL; \ - f->f_exc_traceback = NULL; \ - Py_XDECREF(type); \ - Py_XDECREF(value); \ - Py_XDECREF(tb); \ - } - /* Start of code */ - if (f == NULL) - return NULL; - /* push frame */ if (Py_EnterRecursiveCall("")) return NULL; @@ -1240,28 +1189,15 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* We were in an except handler when we left, restore the exception state which was put aside (see YIELD_VALUE). */ - SWAP_EXC_STATE(); - } - else { - SAVE_EXC_STATE(); + swap_exc_state(tstate, f); } + else + save_exc_state(tstate, f); } #ifdef LLTRACE lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL; #endif -#if defined(Py_DEBUG) || defined(LLTRACE) - { - PyObject *error_type, *error_value, *error_traceback; - PyErr_Fetch(&error_type, &error_value, &error_traceback); - filename = _PyUnicode_AsString(co->co_filename); - if (filename == NULL && tstate->overflowed) { - /* maximum recursion depth exceeded */ - goto exit_eval_frame; - } - PyErr_Restore(error_type, error_value, error_traceback); - } -#endif why = WHY_NOT; err = 0; @@ -1412,8 +1348,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) x to NULL, err to nonzero, or why to anything but WHY_NOT, and that no operation that succeeds does this! */ - /* case STOP_CODE: this is an error! */ - TARGET(NOP) FAST_DISPATCH(); @@ -1894,6 +1828,53 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) why = WHY_RETURN; goto fast_block_end; + TARGET(YIELD_FROM) + u = POP(); + x = PyObject_GetIter(u); + Py_DECREF(u); + if (x == NULL) + break; + /* x is now the iterator, make the first next() call */ + retval = (*Py_TYPE(x)->tp_iternext)(x); + if (!retval) { + PyObject *et, *ev, *tb; + /* iter may be exhausted */ + Py_CLEAR(x); + if (PyErr_Occurred() && + !PyErr_ExceptionMatches(PyExc_StopIteration)) { + /* some other exception */ + break; + } + /* try to get return value from exception */ + PyErr_Fetch(&et, &ev, &tb); + Py_XDECREF(et); + Py_XDECREF(tb); + /* u is return value */ + u = NULL; + if (ev) { + u = PyObject_GetAttrString(ev, "value"); + Py_DECREF(ev); + if (u == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + /* some other exception */ + break; + } + PyErr_Clear(); + } + } + if (u == NULL) { + u = Py_None; + Py_INCREF(u); + } + PUSH(u); + continue; + } + /* x is iterator, retval is value to be yielded */ + f->f_yieldfrom = x; + f->f_stacktop = stack_pointer; + why = WHY_YIELD; + goto fast_yield; + TARGET(YIELD_VALUE) retval = POP(); f->f_stacktop = stack_pointer; @@ -2121,7 +2102,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* Inline the PyDict_GetItem() calls. WARNING: this is an extreme speed hack. Do not try this at home. */ - Py_hash_t hash = ((PyUnicodeObject *)w)->hash; + Py_hash_t hash = ((PyASCIIObject *)w)->hash; if (hash != -1) { PyDictObject *d; PyDictEntry *e; @@ -2559,13 +2540,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) TARGET(SETUP_WITH) { - static PyObject *exit, *enter; + _Py_IDENTIFIER(__exit__); + _Py_IDENTIFIER(__enter__); w = TOP(); - x = special_lookup(w, "__exit__", &exit); + x = special_lookup(w, &PyId___exit__); if (!x) break; SET_TOP(x); - u = special_lookup(w, "__enter__", &enter); + u = special_lookup(w, &PyId___enter__); Py_DECREF(w); if (!u) { x = NULL; @@ -2725,7 +2707,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(*pfunc); *pfunc = self; na++; - n++; + /* n++; */ } else Py_INCREF(func); sp = stack_pointer; @@ -2753,9 +2735,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) int kwdefaults = (oparg>>8) & 0xff; int num_annotations = (oparg >> 16) & 0x7fff; + w = POP(); /* qualname */ v = POP(); /* code object */ - x = PyFunction_New(v, f->f_globals); + x = PyFunction_NewWithQualName(v, f->f_globals, w); Py_DECREF(v); + Py_DECREF(w); if (x != NULL && opcode == MAKE_CLOSURE) { v = POP(); @@ -3049,9 +3033,9 @@ fast_yield: break; if (i == f->f_iblock) /* We did not create this exception. */ - RESTORE_AND_CLEAR_EXC_STATE() + restore_and_clear_exc_state(tstate, f); else - SWAP_EXC_STATE() + swap_exc_state(tstate, f); } if (tstate->use_tracing) { @@ -3081,7 +3065,7 @@ fast_yield: PyTrace_RETURN, retval)) { Py_XDECREF(retval); retval = NULL; - why = WHY_EXCEPTION; + /* why = WHY_EXCEPTION; */ } } } @@ -3094,6 +3078,154 @@ exit_eval_frame: return retval; } +static void +format_missing(const char *kind, PyCodeObject *co, PyObject *names) +{ + int err; + Py_ssize_t len = PyList_GET_SIZE(names); + PyObject *name_str, *comma, *tail, *tmp; + + assert(PyList_CheckExact(names)); + assert(len >= 1); + /* Deal with the joys of natural language. */ + switch (len) { + case 1: + name_str = PyList_GET_ITEM(names, 0); + Py_INCREF(name_str); + break; + case 2: + name_str = PyUnicode_FromFormat("%U and %U", + PyList_GET_ITEM(names, len - 2), + PyList_GET_ITEM(names, len - 1)); + break; + default: + tail = PyUnicode_FromFormat(", %U, and %U", + PyList_GET_ITEM(names, len - 2), + PyList_GET_ITEM(names, len - 1)); + /* Chop off the last two objects in the list. This shouldn't actually + fail, but we can't be too careful. */ + err = PyList_SetSlice(names, len - 2, len, NULL); + if (err == -1) { + Py_DECREF(tail); + return; + } + /* Stitch everything up into a nice comma-separated list. */ + comma = PyUnicode_FromString(", "); + if (comma == NULL) { + Py_DECREF(tail); + return; + } + tmp = PyUnicode_Join(comma, names); + Py_DECREF(comma); + if (tmp == NULL) { + Py_DECREF(tail); + return; + } + name_str = PyUnicode_Concat(tmp, tail); + Py_DECREF(tmp); + Py_DECREF(tail); + break; + } + if (name_str == NULL) + return; + PyErr_Format(PyExc_TypeError, + "%U() missing %i required %s argument%s: %U", + co->co_name, + len, + kind, + len == 1 ? "" : "s", + name_str); + Py_DECREF(name_str); +} + +static void +missing_arguments(PyCodeObject *co, int missing, int defcount, + PyObject **fastlocals) +{ + int i, j = 0; + int start, end; + int positional = defcount != -1; + const char *kind = positional ? "positional" : "keyword-only"; + PyObject *missing_names; + + /* Compute the names of the arguments that are missing. */ + missing_names = PyList_New(missing); + if (missing_names == NULL) + return; + if (positional) { + start = 0; + end = co->co_argcount - defcount; + } + else { + start = co->co_argcount; + end = start + co->co_kwonlyargcount; + } + for (i = start; i < end; i++) { + if (GETLOCAL(i) == NULL) { + PyObject *raw = PyTuple_GET_ITEM(co->co_varnames, i); + PyObject *name = PyObject_Repr(raw); + if (name == NULL) { + Py_DECREF(missing_names); + return; + } + PyList_SET_ITEM(missing_names, j++, name); + } + } + assert(j == missing); + format_missing(kind, co, missing_names); + Py_DECREF(missing_names); +} + +static void +too_many_positional(PyCodeObject *co, int given, int defcount, PyObject **fastlocals) +{ + int plural; + int kwonly_given = 0; + int i; + PyObject *sig, *kwonly_sig; + + assert((co->co_flags & CO_VARARGS) == 0); + /* Count missing keyword-only args. */ + for (i = co->co_argcount; i < co->co_argcount + co->co_kwonlyargcount; i++) + if (GETLOCAL(i) != NULL) + kwonly_given++; + if (defcount) { + int atleast = co->co_argcount - defcount; + plural = 1; + sig = PyUnicode_FromFormat("from %d to %d", atleast, co->co_argcount); + } + else { + plural = co->co_argcount != 1; + sig = PyUnicode_FromFormat("%d", co->co_argcount); + } + if (sig == NULL) + return; + if (kwonly_given) { + const char *format = " positional argument%s (and %d keyword-only argument%s)"; + kwonly_sig = PyUnicode_FromFormat(format, given != 1 ? "s" : "", kwonly_given, + kwonly_given != 1 ? "s" : ""); + if (kwonly_sig == NULL) { + Py_DECREF(sig); + return; + } + } + else { + /* This will not fail. */ + kwonly_sig = PyUnicode_FromString(""); + assert(kwonly_sig != NULL); + } + PyErr_Format(PyExc_TypeError, + "%U() takes %U positional argument%s but %d%U %s given", + co->co_name, + sig, + plural ? "s" : "", + given, + kwonly_sig, + given == 1 && !kwonly_given ? "was" : "were"); + Py_DECREF(sig); + Py_DECREF(kwonly_sig); +} + /* This is gonna seem *real weird*, but if you put some other code between PyEval_EvalFrame() and PyEval_EvalCodeEx() you will need to adjust the test in the if statements in Misc/gdbinit (pystack and pystackv). */ @@ -3110,6 +3242,9 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, PyThreadState *tstate = PyThreadState_GET(); PyObject *x, *u; int total_args = co->co_argcount + co->co_kwonlyargcount; + int i; + int n = argcount; + PyObject *kwdict = NULL; if (globals == NULL) { PyErr_SetString(PyExc_SystemError, @@ -3126,212 +3261,153 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, fastlocals = f->f_localsplus; freevars = f->f_localsplus + co->co_nlocals; - if (total_args || co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) { - int i; - int n = argcount; - PyObject *kwdict = NULL; - if (co->co_flags & CO_VARKEYWORDS) { - kwdict = PyDict_New(); - if (kwdict == NULL) - goto fail; - i = total_args; - if (co->co_flags & CO_VARARGS) - i++; - SETLOCAL(i, kwdict); - } - if (argcount > co->co_argcount) { - if (!(co->co_flags & CO_VARARGS)) { - PyErr_Format(PyExc_TypeError, - "%U() takes %s %d " - "positional argument%s (%d given)", - co->co_name, - defcount ? "at most" : "exactly", - co->co_argcount, - co->co_argcount == 1 ? "" : "s", - argcount + kwcount); - goto fail; - } - n = co->co_argcount; - } - for (i = 0; i < n; i++) { + /* Parse arguments. */ + if (co->co_flags & CO_VARKEYWORDS) { + kwdict = PyDict_New(); + if (kwdict == NULL) + goto fail; + i = total_args; + if (co->co_flags & CO_VARARGS) + i++; + SETLOCAL(i, kwdict); + } + if (argcount > co->co_argcount) + n = co->co_argcount; + for (i = 0; i < n; i++) { + x = args[i]; + Py_INCREF(x); + SETLOCAL(i, x); + } + if (co->co_flags & CO_VARARGS) { + u = PyTuple_New(argcount - n); + if (u == NULL) + goto fail; + SETLOCAL(total_args, u); + for (i = n; i < argcount; i++) { x = args[i]; Py_INCREF(x); - SETLOCAL(i, x); + PyTuple_SET_ITEM(u, i-n, x); } - if (co->co_flags & CO_VARARGS) { - u = PyTuple_New(argcount - n); - if (u == NULL) - goto fail; - SETLOCAL(total_args, u); - for (i = n; i < argcount; i++) { - x = args[i]; - Py_INCREF(x); - PyTuple_SET_ITEM(u, i-n, x); - } + } + for (i = 0; i < kwcount; i++) { + PyObject **co_varnames; + PyObject *keyword = kws[2*i]; + PyObject *value = kws[2*i + 1]; + int j; + if (keyword == NULL || !PyUnicode_Check(keyword)) { + PyErr_Format(PyExc_TypeError, + "%U() keywords must be strings", + co->co_name); + goto fail; } - for (i = 0; i < kwcount; i++) { - PyObject **co_varnames; - PyObject *keyword = kws[2*i]; - PyObject *value = kws[2*i + 1]; - int j; - if (keyword == NULL || !PyUnicode_Check(keyword)) { - PyErr_Format(PyExc_TypeError, - "%U() keywords must be strings", - co->co_name); - goto fail; - } - /* Speed hack: do raw pointer compares. As names are - normally interned this should almost always hit. */ - co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; - for (j = 0; j < total_args; j++) { - PyObject *nm = co_varnames[j]; - if (nm == keyword) - goto kw_found; - } - /* Slow fallback, just in case */ - for (j = 0; j < total_args; j++) { - PyObject *nm = co_varnames[j]; - int cmp = PyObject_RichCompareBool( - keyword, nm, Py_EQ); - if (cmp > 0) - goto kw_found; - else if (cmp < 0) - goto fail; - } - if (j >= total_args && kwdict == NULL) { - PyErr_Format(PyExc_TypeError, - "%U() got an unexpected " - "keyword argument '%S'", - co->co_name, - keyword); + /* Speed hack: do raw pointer compares. As names are + normally interned this should almost always hit. */ + co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; + for (j = 0; j < total_args; j++) { + PyObject *nm = co_varnames[j]; + if (nm == keyword) + goto kw_found; + } + /* Slow fallback, just in case */ + for (j = 0; j < total_args; j++) { + PyObject *nm = co_varnames[j]; + int cmp = PyObject_RichCompareBool( + keyword, nm, Py_EQ); + if (cmp > 0) + goto kw_found; + else if (cmp < 0) goto fail; - } - PyDict_SetItem(kwdict, keyword, value); - continue; - kw_found: - if (GETLOCAL(j) != NULL) { - PyErr_Format(PyExc_TypeError, + } + if (j >= total_args && kwdict == NULL) { + PyErr_Format(PyExc_TypeError, + "%U() got an unexpected " + "keyword argument '%S'", + co->co_name, + keyword); + goto fail; + } + PyDict_SetItem(kwdict, keyword, value); + continue; + kw_found: + if (GETLOCAL(j) != NULL) { + PyErr_Format(PyExc_TypeError, "%U() got multiple " - "values for keyword " - "argument '%S'", + "values for argument '%S'", co->co_name, keyword); - goto fail; - } - Py_INCREF(value); - SETLOCAL(j, value); + goto fail; } - if (co->co_kwonlyargcount > 0) { - for (i = co->co_argcount; i < total_args; i++) { - PyObject *name; - if (GETLOCAL(i) != NULL) - continue; - name = PyTuple_GET_ITEM(co->co_varnames, i); - if (kwdefs != NULL) { - PyObject *def = PyDict_GetItem(kwdefs, name); - if (def) { - Py_INCREF(def); - SETLOCAL(i, def); - continue; - } - } - PyErr_Format(PyExc_TypeError, - "%U() needs keyword-only argument %S", - co->co_name, name); - goto fail; - } + Py_INCREF(value); + SETLOCAL(j, value); + } + if (argcount > co->co_argcount && !(co->co_flags & CO_VARARGS)) { + too_many_positional(co, argcount, defcount, fastlocals); + goto fail; + } + if (argcount < co->co_argcount) { + int m = co->co_argcount - defcount; + int missing = 0; + for (i = argcount; i < m; i++) + if (GETLOCAL(i) == NULL) + missing++; + if (missing) { + missing_arguments(co, missing, defcount, fastlocals); + goto fail; } - if (argcount < co->co_argcount) { - int m = co->co_argcount - defcount; - for (i = argcount; i < m; i++) { - if (GETLOCAL(i) == NULL) { - int j, given = 0; - for (j = 0; j < co->co_argcount; j++) - if (GETLOCAL(j)) - given++; - PyErr_Format(PyExc_TypeError, - "%U() takes %s %d " - "argument%s " - "(%d given)", - co->co_name, - ((co->co_flags & CO_VARARGS) || - defcount) ? "at least" - : "exactly", - m, m == 1 ? "" : "s", given); - goto fail; - } + if (n > m) + i = n - m; + else + i = 0; + for (; i < defcount; i++) { + if (GETLOCAL(m+i) == NULL) { + PyObject *def = defs[i]; + Py_INCREF(def); + SETLOCAL(m+i, def); } - if (n > m) - i = n - m; - else - i = 0; - for (; i < defcount; i++) { - if (GETLOCAL(m+i) == NULL) { - PyObject *def = defs[i]; + } + } + if (co->co_kwonlyargcount > 0) { + int missing = 0; + for (i = co->co_argcount; i < total_args; i++) { + PyObject *name; + if (GETLOCAL(i) != NULL) + continue; + name = PyTuple_GET_ITEM(co->co_varnames, i); + if (kwdefs != NULL) { + PyObject *def = PyDict_GetItem(kwdefs, name); + if (def) { Py_INCREF(def); - SETLOCAL(m+i, def); + SETLOCAL(i, def); + continue; } } + missing++; + } + if (missing) { + missing_arguments(co, missing, -1, fastlocals); + goto fail; } } - else if (argcount > 0 || kwcount > 0) { - PyErr_Format(PyExc_TypeError, - "%U() takes no arguments (%d given)", - co->co_name, - argcount + kwcount); - goto fail; - } + /* Allocate and initialize storage for cell vars, and copy free - vars into frame. This isn't too efficient right now. */ - if (PyTuple_GET_SIZE(co->co_cellvars)) { - int i, j, nargs, found; - Py_UNICODE *cellname, *argname; + vars into frame. */ + for (i = 0; i < PyTuple_GET_SIZE(co->co_cellvars); ++i) { PyObject *c; - - nargs = total_args; - if (co->co_flags & CO_VARARGS) - nargs++; - if (co->co_flags & CO_VARKEYWORDS) - nargs++; - - /* Initialize each cell var, taking into account - cell vars that are initialized from arguments. - - Should arrange for the compiler to put cellvars - that are arguments at the beginning of the cellvars - list so that we can march over it more efficiently? - */ - for (i = 0; i < PyTuple_GET_SIZE(co->co_cellvars); ++i) { - cellname = PyUnicode_AS_UNICODE( - PyTuple_GET_ITEM(co->co_cellvars, i)); - found = 0; - for (j = 0; j < nargs; j++) { - argname = PyUnicode_AS_UNICODE( - PyTuple_GET_ITEM(co->co_varnames, j)); - if (Py_UNICODE_strcmp(cellname, argname) == 0) { - c = PyCell_New(GETLOCAL(j)); - if (c == NULL) - goto fail; - GETLOCAL(co->co_nlocals + i) = c; - found = 1; - break; - } - } - if (found == 0) { - c = PyCell_New(NULL); - if (c == NULL) - goto fail; - SETLOCAL(co->co_nlocals + i, c); - } - } + int arg; + /* Possibly account for the cell variable being an argument. */ + if (co->co_cell2arg != NULL && + (arg = co->co_cell2arg[i]) != CO_CELL_NOT_AN_ARG) + c = PyCell_New(GETLOCAL(arg)); + else + c = PyCell_New(NULL); + if (c == NULL) + goto fail; + SETLOCAL(co->co_nlocals + i, c); } - if (PyTuple_GET_SIZE(co->co_freevars)) { - int i; - for (i = 0; i < PyTuple_GET_SIZE(co->co_freevars); ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - Py_INCREF(o); - freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o; - } + for (i = 0; i < PyTuple_GET_SIZE(co->co_freevars); ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + Py_INCREF(o); + freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o; } if (co->co_flags & CO_GENERATOR) { @@ -3365,18 +3441,72 @@ fail: /* Jump here from prelude on failure */ static PyObject * -special_lookup(PyObject *o, char *meth, PyObject **cache) +special_lookup(PyObject *o, _Py_Identifier *id) { PyObject *res; - res = _PyObject_LookupSpecial(o, meth, cache); + res = _PyObject_LookupSpecial(o, id); if (res == NULL && !PyErr_Occurred()) { - PyErr_SetObject(PyExc_AttributeError, *cache); + PyErr_SetObject(PyExc_AttributeError, id->object); return NULL; } return res; } +/* These 3 functions deal with the exception state of generators. */ + +static void +save_exc_state(PyThreadState *tstate, PyFrameObject *f) +{ + PyObject *type, *value, *traceback; + Py_XINCREF(tstate->exc_type); + Py_XINCREF(tstate->exc_value); + Py_XINCREF(tstate->exc_traceback); + type = f->f_exc_type; + value = f->f_exc_value; + traceback = f->f_exc_traceback; + f->f_exc_type = tstate->exc_type; + f->f_exc_value = tstate->exc_value; + f->f_exc_traceback = tstate->exc_traceback; + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); +} + +static void +swap_exc_state(PyThreadState *tstate, PyFrameObject *f) +{ + PyObject *tmp; + tmp = tstate->exc_type; + tstate->exc_type = f->f_exc_type; + f->f_exc_type = tmp; + tmp = tstate->exc_value; + tstate->exc_value = f->f_exc_value; + f->f_exc_value = tmp; + tmp = tstate->exc_traceback; + tstate->exc_traceback = f->f_exc_traceback; + f->f_exc_traceback = tmp; +} + +static void +restore_and_clear_exc_state(PyThreadState *tstate, PyFrameObject *f) +{ + PyObject *type, *value, *tb; + type = tstate->exc_type; + value = tstate->exc_value; + tb = tstate->exc_traceback; + tstate->exc_type = f->f_exc_type; + tstate->exc_value = f->f_exc_value; + tstate->exc_traceback = f->f_exc_traceback; + f->f_exc_type = NULL; + f->f_exc_value = NULL; + f->f_exc_traceback = NULL; + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(tb); +} + + /* Logic for the raise statement (too complicated for inlining). This *consumes* a reference count to each of its arguments. */ static enum why_code @@ -4192,7 +4322,7 @@ ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk) if (PyErr_ExceptionMatches(PyExc_TypeError)) { PyErr_Format(PyExc_TypeError, "%.200s%.200s argument after * " - "must be a sequence, not %200s", + "must be a sequence, not %.200s", PyEval_GetFuncName(func), PyEval_GetFuncDesc(func), stararg->ob_type->tp_name); @@ -4342,7 +4472,9 @@ import_from(PyObject *v, PyObject *name) static int import_all_from(PyObject *locals, PyObject *v) { - PyObject *all = PyObject_GetAttrString(v, "__all__"); + _Py_IDENTIFIER(__all__); + _Py_IDENTIFIER(__dict__); + PyObject *all = _PyObject_GetAttrId(v, &PyId___all__); PyObject *dict, *name, *value; int skip_leading_underscores = 0; int pos, err; @@ -4351,7 +4483,7 @@ import_all_from(PyObject *locals, PyObject *v) if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return -1; /* Unexpected error */ PyErr_Clear(); - dict = PyObject_GetAttrString(v, "__dict__"); + dict = _PyObject_GetAttrId(v, &PyId___dict__); if (dict == NULL) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return -1; @@ -4377,7 +4509,8 @@ import_all_from(PyObject *locals, PyObject *v) } if (skip_leading_underscores && PyUnicode_Check(name) && - PyUnicode_AS_UNICODE(name)[0] == '_') + PyUnicode_READY(name) != -1 && + PyUnicode_READ_CHAR(name, 0) == '_') { Py_DECREF(name); continue; @@ -4437,19 +4570,9 @@ format_exc_unbound(PyCodeObject *co, int oparg) static PyObject * unicode_concatenate(PyObject *v, PyObject *w, - PyFrameObject *f, unsigned char *next_instr) + PyFrameObject *f, unsigned char *next_instr) { - /* This function implements 'variable += expr' when both arguments - are (Unicode) strings. */ - Py_ssize_t v_len = PyUnicode_GET_SIZE(v); - Py_ssize_t w_len = PyUnicode_GET_SIZE(w); - Py_ssize_t new_len = v_len + w_len; - if (new_len < 0) { - PyErr_SetString(PyExc_OverflowError, - "strings are too large to concat"); - return NULL; - } - + PyObject *res; if (Py_REFCNT(v) == 2) { /* In the common case, there are 2 references to the value * stored in 'variable' when the += is performed: one on the @@ -4490,31 +4613,9 @@ unicode_concatenate(PyObject *v, PyObject *w, } } } - - if (Py_REFCNT(v) == 1 && !PyUnicode_CHECK_INTERNED(v)) { - /* Now we own the last reference to 'v', so we can resize it - * in-place. - */ - if (PyUnicode_Resize(&v, new_len) != 0) { - /* XXX if PyUnicode_Resize() fails, 'v' has been - * deallocated so it cannot be put back into - * 'variable'. The MemoryError is raised when there - * is no value in 'variable', which might (very - * remotely) be a cause of incompatibilities. - */ - return NULL; - } - /* copy 'w' into the newly allocated area of 'v' */ - memcpy(PyUnicode_AS_UNICODE(v) + v_len, - PyUnicode_AS_UNICODE(w), w_len*sizeof(Py_UNICODE)); - return v; - } - else { - /* When in-place resizing is not an option. */ - w = PyUnicode_Concat(v, w); - Py_DECREF(v); - return w; - } + res = v; + PyUnicode_Append(&res, w); + return res; } #ifdef DYNAMIC_EXECUTION_PROFILE diff --git a/Python/codecs.c b/Python/codecs.c index c7f4a9c..607feea 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -11,6 +11,8 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include <ctype.h> +const char *Py_hexdigits = "0123456789abcdef"; + /* --- Codec Registry ----------------------------------------------------- */ /* Import the standard encodings package which will register the first @@ -465,9 +467,11 @@ PyObject *PyCodec_LookupError(const char *name) static void wrong_exception_type(PyObject *exc) { - PyObject *type = PyObject_GetAttrString(exc, "__class__"); + _Py_IDENTIFIER(__class__); + _Py_IDENTIFIER(__name__); + PyObject *type = _PyObject_GetAttrId(exc, &PyId___class__); if (type != NULL) { - PyObject *name = PyObject_GetAttrString(type, "__name__"); + PyObject *name = _PyObject_GetAttrId(type, &PyId___name__); Py_DECREF(type); if (name != NULL) { PyErr_Format(PyExc_TypeError, @@ -506,57 +510,56 @@ PyObject *PyCodec_IgnoreErrors(PyObject *exc) wrong_exception_type(exc); return NULL; } - /* ouch: passing NULL, 0, pos gives None instead of u'' */ - return Py_BuildValue("(u#n)", &end, 0, end); + return Py_BuildValue("(Nn)", PyUnicode_New(0, 0), end); } PyObject *PyCodec_ReplaceErrors(PyObject *exc) { - PyObject *restuple; - Py_ssize_t start; - Py_ssize_t end; - Py_ssize_t i; + Py_ssize_t start, end, i, len; if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { PyObject *res; - Py_UNICODE *p; + int kind; + void *data; if (PyUnicodeEncodeError_GetStart(exc, &start)) return NULL; if (PyUnicodeEncodeError_GetEnd(exc, &end)) return NULL; - res = PyUnicode_FromUnicode(NULL, end-start); + len = end - start; + res = PyUnicode_New(len, '?'); if (res == NULL) return NULL; - for (p = PyUnicode_AS_UNICODE(res), i = start; - i<end; ++p, ++i) - *p = '?'; - restuple = Py_BuildValue("(On)", res, end); - Py_DECREF(res); - return restuple; + kind = PyUnicode_KIND(res); + data = PyUnicode_DATA(res); + for (i = 0; i < len; ++i) + PyUnicode_WRITE(kind, data, i, '?'); + return Py_BuildValue("(Nn)", res, end); } else if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) { - Py_UNICODE res = Py_UNICODE_REPLACEMENT_CHARACTER; if (PyUnicodeDecodeError_GetEnd(exc, &end)) return NULL; - return Py_BuildValue("(u#n)", &res, 1, end); + return Py_BuildValue("(Cn)", + (int)Py_UNICODE_REPLACEMENT_CHARACTER, + end); } else if (PyObject_IsInstance(exc, PyExc_UnicodeTranslateError)) { PyObject *res; - Py_UNICODE *p; + int kind; + void *data; if (PyUnicodeTranslateError_GetStart(exc, &start)) return NULL; if (PyUnicodeTranslateError_GetEnd(exc, &end)) return NULL; - res = PyUnicode_FromUnicode(NULL, end-start); + len = end - start; + res = PyUnicode_New(len, Py_UNICODE_REPLACEMENT_CHARACTER); if (res == NULL) return NULL; - for (p = PyUnicode_AS_UNICODE(res), i = start; - i<end; ++p, ++i) - *p = Py_UNICODE_REPLACEMENT_CHARACTER; - restuple = Py_BuildValue("(On)", res, end); - Py_DECREF(res); - return restuple; + kind = PyUnicode_KIND(res); + data = PyUnicode_DATA(res); + for (i=0; i < len; i++) + PyUnicode_WRITE(kind, data, i, Py_UNICODE_REPLACEMENT_CHARACTER); + return Py_BuildValue("(Nn)", res, end); } else { wrong_exception_type(exc); @@ -569,82 +572,72 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { PyObject *restuple; PyObject *object; + Py_ssize_t i; Py_ssize_t start; Py_ssize_t end; PyObject *res; - Py_UNICODE *p; - Py_UNICODE *startp; - Py_UNICODE *outp; + unsigned char *outp; int ressize; + Py_UCS4 ch; if (PyUnicodeEncodeError_GetStart(exc, &start)) return NULL; if (PyUnicodeEncodeError_GetEnd(exc, &end)) return NULL; if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; - startp = PyUnicode_AS_UNICODE(object); - for (p = startp+start, ressize = 0; p < startp+end; ++p) { - if (*p<10) + for (i = start, ressize = 0; i < end; ++i) { + /* object is guaranteed to be "ready" */ + ch = PyUnicode_READ_CHAR(object, i); + if (ch<10) ressize += 2+1+1; - else if (*p<100) + else if (ch<100) ressize += 2+2+1; - else if (*p<1000) + else if (ch<1000) ressize += 2+3+1; - else if (*p<10000) + else if (ch<10000) ressize += 2+4+1; -#ifndef Py_UNICODE_WIDE - else - ressize += 2+5+1; -#else - else if (*p<100000) + else if (ch<100000) ressize += 2+5+1; - else if (*p<1000000) + else if (ch<1000000) ressize += 2+6+1; else ressize += 2+7+1; -#endif } /* allocate replacement */ - res = PyUnicode_FromUnicode(NULL, ressize); + res = PyUnicode_New(ressize, 127); if (res == NULL) { Py_DECREF(object); return NULL; } + outp = PyUnicode_1BYTE_DATA(res); /* generate replacement */ - for (p = startp+start, outp = PyUnicode_AS_UNICODE(res); - p < startp+end; ++p) { - Py_UNICODE c = *p; + for (i = start; i < end; ++i) { int digits; int base; + ch = PyUnicode_READ_CHAR(object, i); *outp++ = '&'; *outp++ = '#'; - if (*p<10) { + if (ch<10) { digits = 1; base = 1; } - else if (*p<100) { + else if (ch<100) { digits = 2; base = 10; } - else if (*p<1000) { + else if (ch<1000) { digits = 3; base = 100; } - else if (*p<10000) { + else if (ch<10000) { digits = 4; base = 1000; } -#ifndef Py_UNICODE_WIDE - else { + else if (ch<100000) { digits = 5; base = 10000; } -#else - else if (*p<100000) { - digits = 5; - base = 10000; - } - else if (*p<1000000) { + else if (ch<1000000) { digits = 6; base = 100000; } @@ -652,10 +645,9 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) digits = 7; base = 1000000; } -#endif while (digits-->0) { - *outp++ = '0' + c/base; - c %= base; + *outp++ = '0' + ch/base; + ch %= base; base /= 10; } *outp++ = ';'; @@ -671,83 +663,61 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) } } -static Py_UNICODE hexdigits[] = { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' -}; - PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) { -#ifndef Py_UNICODE_WIDE -#define IS_SURROGATE_PAIR(p, end) \ - (*p >= 0xD800 && *p <= 0xDBFF && (p + 1) < end && \ - *(p + 1) >= 0xDC00 && *(p + 1) <= 0xDFFF) -#else -#define IS_SURROGATE_PAIR(p, end) 0 -#endif if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { PyObject *restuple; PyObject *object; + Py_ssize_t i; Py_ssize_t start; Py_ssize_t end; PyObject *res; - Py_UNICODE *p; - Py_UNICODE *startp; - Py_UNICODE *outp; + unsigned char *outp; int ressize; + Py_UCS4 c; if (PyUnicodeEncodeError_GetStart(exc, &start)) return NULL; if (PyUnicodeEncodeError_GetEnd(exc, &end)) return NULL; if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; - startp = PyUnicode_AS_UNICODE(object); - for (p = startp+start, ressize = 0; p < startp+end; ++p) { -#ifdef Py_UNICODE_WIDE - if (*p >= 0x00010000) + for (i = start, ressize = 0; i < end; ++i) { + /* object is guaranteed to be "ready" */ + c = PyUnicode_READ_CHAR(object, i); + if (c >= 0x10000) { ressize += 1+1+8; - else -#endif - if (*p >= 0x100) { - if (IS_SURROGATE_PAIR(p, startp+end)) { - ressize += 1+1+8; - ++p; - } - else - ressize += 1+1+4; + } + else if (c >= 0x100) { + ressize += 1+1+4; } else ressize += 1+1+2; } - res = PyUnicode_FromUnicode(NULL, ressize); + res = PyUnicode_New(ressize, 127); if (res==NULL) return NULL; - for (p = startp+start, outp = PyUnicode_AS_UNICODE(res); - p < startp+end; ++p) { - Py_UCS4 c = (Py_UCS4) *p; + for (i = start, outp = PyUnicode_1BYTE_DATA(res); + i < end; ++i) { + c = PyUnicode_READ_CHAR(object, i); *outp++ = '\\'; - if (IS_SURROGATE_PAIR(p, startp+end)) { - c = ((*p & 0x3FF) << 10) + (*(p + 1) & 0x3FF) + 0x10000; - ++p; - } if (c >= 0x00010000) { *outp++ = 'U'; - *outp++ = hexdigits[(c>>28)&0xf]; - *outp++ = hexdigits[(c>>24)&0xf]; - *outp++ = hexdigits[(c>>20)&0xf]; - *outp++ = hexdigits[(c>>16)&0xf]; - *outp++ = hexdigits[(c>>12)&0xf]; - *outp++ = hexdigits[(c>>8)&0xf]; + *outp++ = Py_hexdigits[(c>>28)&0xf]; + *outp++ = Py_hexdigits[(c>>24)&0xf]; + *outp++ = Py_hexdigits[(c>>20)&0xf]; + *outp++ = Py_hexdigits[(c>>16)&0xf]; + *outp++ = Py_hexdigits[(c>>12)&0xf]; + *outp++ = Py_hexdigits[(c>>8)&0xf]; } else if (c >= 0x100) { *outp++ = 'u'; - *outp++ = hexdigits[(c>>12)&0xf]; - *outp++ = hexdigits[(c>>8)&0xf]; + *outp++ = Py_hexdigits[(c>>12)&0xf]; + *outp++ = Py_hexdigits[(c>>8)&0xf]; } else *outp++ = 'x'; - *outp++ = hexdigits[(c>>4)&0xf]; - *outp++ = hexdigits[c&0xf]; + *outp++ = Py_hexdigits[(c>>4)&0xf]; + *outp++ = Py_hexdigits[c&0xf]; } restuple = Py_BuildValue("(On)", res, end); @@ -759,7 +729,6 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) wrong_exception_type(exc); return NULL; } -#undef IS_SURROGATE_PAIR } /* This handler is declared static until someone demonstrates @@ -769,12 +738,11 @@ PyCodec_SurrogatePassErrors(PyObject *exc) { PyObject *restuple; PyObject *object; + Py_ssize_t i; Py_ssize_t start; Py_ssize_t end; PyObject *res; if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { - Py_UNICODE *p; - Py_UNICODE *startp; char *outp; if (PyUnicodeEncodeError_GetStart(exc, &start)) return NULL; @@ -782,15 +750,15 @@ PyCodec_SurrogatePassErrors(PyObject *exc) return NULL; if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; - startp = PyUnicode_AS_UNICODE(object); res = PyBytes_FromStringAndSize(NULL, 3*(end-start)); if (!res) { Py_DECREF(object); return NULL; } outp = PyBytes_AsString(res); - for (p = startp+start; p < startp+end; p++) { - Py_UNICODE ch = *p; + for (i = start; i < end; i++) { + /* object is guaranteed to be "ready" */ + Py_UCS4 ch = PyUnicode_READ_CHAR(object, i); if (ch < 0xd800 || ch > 0xdfff) { /* Not a surrogate, fail with original exception */ PyErr_SetObject(PyExceptionInstance_Class(exc), exc); @@ -809,7 +777,7 @@ PyCodec_SurrogatePassErrors(PyObject *exc) } else if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) { unsigned char *p; - Py_UNICODE ch = 0; + Py_UCS4 ch = 0; if (PyUnicodeDecodeError_GetStart(exc, &start)) return NULL; if (!(object = PyUnicodeDecodeError_GetObject(exc))) @@ -835,7 +803,10 @@ PyCodec_SurrogatePassErrors(PyObject *exc) PyErr_SetObject(PyExceptionInstance_Class(exc), exc); return NULL; } - return Py_BuildValue("(u#n)", &ch, 1, start+3); + res = PyUnicode_FromOrdinal(ch); + if (res == NULL) + return NULL; + return Py_BuildValue("(Nn)", res, start+3); } else { wrong_exception_type(exc); @@ -848,12 +819,11 @@ PyCodec_SurrogateEscapeErrors(PyObject *exc) { PyObject *restuple; PyObject *object; + Py_ssize_t i; Py_ssize_t start; Py_ssize_t end; PyObject *res; if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { - Py_UNICODE *p; - Py_UNICODE *startp; char *outp; if (PyUnicodeEncodeError_GetStart(exc, &start)) return NULL; @@ -861,15 +831,15 @@ PyCodec_SurrogateEscapeErrors(PyObject *exc) return NULL; if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; - startp = PyUnicode_AS_UNICODE(object); res = PyBytes_FromStringAndSize(NULL, end-start); if (!res) { Py_DECREF(object); return NULL; } outp = PyBytes_AsString(res); - for (p = startp+start; p < startp+end; p++) { - Py_UNICODE ch = *p; + for (i = start; i < end; i++) { + /* object is guaranteed to be "ready" */ + Py_UCS4 ch = PyUnicode_READ_CHAR(object, i); if (ch < 0xdc80 || ch > 0xdcff) { /* Not a UTF-8b surrogate, fail with original exception */ PyErr_SetObject(PyExceptionInstance_Class(exc), exc); @@ -885,8 +855,9 @@ PyCodec_SurrogateEscapeErrors(PyObject *exc) return restuple; } else if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) { + PyObject *str; unsigned char *p; - Py_UNICODE ch[4]; /* decode up to 4 bad bytes. */ + Py_UCS2 ch[4]; /* decode up to 4 bad bytes. */ int consumed = 0; if (PyUnicodeDecodeError_GetStart(exc, &start)) return NULL; @@ -911,7 +882,10 @@ PyCodec_SurrogateEscapeErrors(PyObject *exc) PyErr_SetObject(PyExceptionInstance_Class(exc), exc); return NULL; } - return Py_BuildValue("(u#n)", ch, consumed, start+consumed); + str = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, ch, consumed); + if (str == NULL) + return NULL; + return Py_BuildValue("(Nn)", str, start+consumed); } else { wrong_exception_type(exc); @@ -1048,7 +1022,7 @@ static int _PyCodecRegistry_Init(void) interp->codec_error_registry = PyDict_New(); if (interp->codec_error_registry) { - for (i = 0; i < sizeof(methods)/sizeof(methods[0]); ++i) { + for (i = 0; i < Py_ARRAY_LENGTH(methods); ++i) { PyObject *func = PyCFunction_New(&methods[i].def, NULL); int res; if (!func) diff --git a/Python/compile.c b/Python/compile.c index ba593a0..b64c800 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -90,6 +90,13 @@ struct fblockinfo { basicblock *fb_block; }; +enum { + COMPILER_SCOPE_MODULE, + COMPILER_SCOPE_CLASS, + COMPILER_SCOPE_FUNCTION, + COMPILER_SCOPE_COMPREHENSION, +}; + /* The following items change on entry and exit of code blocks. They must be saved and restored when returning to a block. */ @@ -97,6 +104,9 @@ struct compiler_unit { PySTEntryObject *u_ste; PyObject *u_name; + PyObject *u_qualname; /* dot-separated qualified name (lazy) */ + int u_scope_type; + /* The following fields are dicts that map objects to the index of them in co_XXX. The index is used as the argument for opcodes that refer to those collections. @@ -135,6 +145,7 @@ managed by compiler_enter_scope() and compiler_exit_scope(). struct compiler { const char *c_filename; + PyObject *c_filename_obj; struct symtable *c_st; PyFutureFeatures *c_future; /* pointer to module's __future__ */ PyCompilerFlags *c_flags; @@ -148,7 +159,7 @@ struct compiler { PyArena *c_arena; /* pointer to memory allocation arena */ }; -static int compiler_enter_scope(struct compiler *, identifier, void *, int); +static int compiler_enter_scope(struct compiler *, identifier, int, void *, int); static void compiler_free(struct compiler *); static basicblock *compiler_new_block(struct compiler *); static int compiler_next_instr(struct compiler *, basicblock *); @@ -178,12 +189,13 @@ static int compiler_in_loop(struct compiler *); static int inplace_binop(struct compiler *, operator_ty); static int expr_constant(struct compiler *, expr_ty); -static int compiler_with(struct compiler *, stmt_ty); +static int compiler_with(struct compiler *, stmt_ty, int); static int compiler_call_helper(struct compiler *c, int n, asdl_seq *args, asdl_seq *keywords, expr_ty starargs, expr_ty kwargs); +static int compiler_try_except(struct compiler *, stmt_ty); static PyCodeObject *assemble(struct compiler *, int addNone); static PyObject *__doc__; @@ -195,16 +207,17 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) { /* Name mangling: __private becomes _classname__private. This is independent from how the name is used. */ - const Py_UNICODE *p, *name = PyUnicode_AS_UNICODE(ident); - Py_UNICODE *buffer; - size_t nlen, plen; + PyObject *result; + size_t nlen, plen, ipriv; + Py_UCS4 maxchar; if (privateobj == NULL || !PyUnicode_Check(privateobj) || - name == NULL || name[0] != '_' || name[1] != '_') { + PyUnicode_READ_CHAR(ident, 0) != '_' || + PyUnicode_READ_CHAR(ident, 1) != '_') { Py_INCREF(ident); return ident; } - p = PyUnicode_AS_UNICODE(privateobj); - nlen = Py_UNICODE_strlen(name); + nlen = PyUnicode_GET_LENGTH(ident); + plen = PyUnicode_GET_LENGTH(privateobj); /* Don't mangle __id__ or names with dots. The only time a name with a dot can occur is when @@ -214,32 +227,43 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) TODO(jhylton): Decide whether we want to support mangling of the module name, e.g. __M.X. */ - if ((name[nlen-1] == '_' && name[nlen-2] == '_') - || Py_UNICODE_strchr(name, '.')) { + if ((PyUnicode_READ_CHAR(ident, nlen-1) == '_' && + PyUnicode_READ_CHAR(ident, nlen-2) == '_') || + PyUnicode_FindChar(ident, '.', 0, nlen, 1) != -1) { Py_INCREF(ident); return ident; /* Don't mangle __whatever__ */ } /* Strip leading underscores from class name */ - while (*p == '_') - p++; - if (*p == 0) { + ipriv = 0; + while (PyUnicode_READ_CHAR(privateobj, ipriv) == '_') + ipriv++; + if (ipriv == plen) { Py_INCREF(ident); return ident; /* Don't mangle if class is just underscores */ } - plen = Py_UNICODE_strlen(p); + plen -= ipriv; assert(1 <= PY_SSIZE_T_MAX - nlen); assert(1 + nlen <= PY_SSIZE_T_MAX - plen); - ident = PyUnicode_FromStringAndSize(NULL, 1 + nlen + plen); - if (!ident) + maxchar = PyUnicode_MAX_CHAR_VALUE(ident); + if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar) + maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj); + + result = PyUnicode_New(1 + nlen + plen, maxchar); + if (!result) return 0; - /* ident = "_" + p[:plen] + name # i.e. 1+plen+nlen bytes */ - buffer = PyUnicode_AS_UNICODE(ident); - buffer[0] = '_'; - Py_UNICODE_strncpy(buffer+1, p, plen); - Py_UNICODE_strcpy(buffer+1+plen, name); - return ident; + /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */ + PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_'); + if (PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen) < 0) { + Py_DECREF(result); + return NULL; + } + if (PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen) < 0) { + Py_DECREF(result); + return NULL; + } + return result; } static int @@ -272,6 +296,9 @@ PyAST_CompileEx(mod_ty mod, const char *filename, PyCompilerFlags *flags, if (!compiler_init(&c)) return NULL; c.c_filename = filename; + c.c_filename_obj = PyUnicode_DecodeFSDefault(filename); + if (!c.c_filename_obj) + goto finally; c.c_arena = arena; c.c_future = PyFuture_FromAST(mod, filename); if (c.c_future == NULL) @@ -324,6 +351,8 @@ compiler_free(struct compiler *c) PySymtable_Free(c->c_st); if (c->c_future) PyObject_Free(c->c_future); + if (c->c_filename_obj) + Py_DECREF(c->c_filename_obj); Py_DECREF(c->c_stack); } @@ -438,6 +467,7 @@ compiler_unit_free(struct compiler_unit *u) } Py_CLEAR(u->u_ste); Py_CLEAR(u->u_name); + Py_CLEAR(u->u_qualname); Py_CLEAR(u->u_consts); Py_CLEAR(u->u_names); Py_CLEAR(u->u_varnames); @@ -448,8 +478,8 @@ compiler_unit_free(struct compiler_unit *u) } static int -compiler_enter_scope(struct compiler *c, identifier name, void *key, - int lineno) +compiler_enter_scope(struct compiler *c, identifier name, + int scope_type, void *key, int lineno) { struct compiler_unit *u; @@ -460,6 +490,7 @@ compiler_enter_scope(struct compiler *c, identifier name, void *key, return 0; } memset(u, 0, sizeof(struct compiler_unit)); + u->u_scope_type = scope_type; u->u_argcount = 0; u->u_kwonlyargcount = 0; u->u_ste = PySymtable_Lookup(c->c_st, key); @@ -547,6 +578,59 @@ compiler_exit_scope(struct compiler *c) } +static PyObject * +compiler_scope_qualname(struct compiler *c) +{ + Py_ssize_t stack_size, i; + _Py_static_string(dot, "."); + _Py_static_string(locals, "<locals>"); + struct compiler_unit *u; + PyObject *capsule, *name, *seq, *dot_str, *locals_str; + + u = c->u; + if (u->u_qualname != NULL) { + Py_INCREF(u->u_qualname); + return u->u_qualname; + } + + seq = PyList_New(0); + if (seq == NULL) + return NULL; + + stack_size = PyList_GET_SIZE(c->c_stack); + for (i = 0; i < stack_size; i++) { + capsule = PyList_GET_ITEM(c->c_stack, i); + u = (struct compiler_unit *)PyCapsule_GetPointer(capsule, COMPILER_CAPSULE_NAME_COMPILER_UNIT); + assert(u); + if (u->u_scope_type == COMPILER_SCOPE_MODULE) + continue; + if (PyList_Append(seq, u->u_name)) + goto _error; + if (u->u_scope_type == COMPILER_SCOPE_FUNCTION) { + locals_str = _PyUnicode_FromId(&locals); + if (locals_str == NULL) + goto _error; + if (PyList_Append(seq, locals_str)) + goto _error; + } + } + u = c->u; + if (PyList_Append(seq, u->u_name)) + goto _error; + dot_str = _PyUnicode_FromId(&dot); + if (dot_str == NULL) + goto _error; + name = PyUnicode_Join(dot_str, seq); + Py_DECREF(seq); + u->u_qualname = name; + Py_XINCREF(name); + return name; + +_error: + Py_XDECREF(seq); + return NULL; +} + /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -756,6 +840,7 @@ opcode_stack_effect(int opcode, int oparg) case IMPORT_STAR: return -1; case YIELD_VALUE: + case YIELD_FROM: return 0; case POP_BLOCK: @@ -843,9 +928,9 @@ opcode_stack_effect(int opcode, int oparg) case CALL_FUNCTION_VAR_KW: return -NARGS(oparg)-2; case MAKE_FUNCTION: - return -NARGS(oparg) - ((oparg >> 16) & 0xffff); + return -1 -NARGS(oparg) - ((oparg >> 16) & 0xffff); case MAKE_CLOSURE: - return -1 - NARGS(oparg) - ((oparg >> 16) & 0xffff); + return -2 - NARGS(oparg) - ((oparg >> 16) & 0xffff); #undef NARGS case BUILD_SLICE: if (oparg == 3) @@ -1175,7 +1260,7 @@ compiler_mod(struct compiler *c, mod_ty mod) return NULL; } /* Use 0 for firstlineno initially, will fixup in assemble(). */ - if (!compiler_enter_scope(c, module, mod, 0)) + if (!compiler_enter_scope(c, module, COMPILER_SCOPE_MODULE, mod, 0)) return NULL; switch (mod->kind) { case Module_kind: @@ -1251,11 +1336,15 @@ compiler_lookup_arg(PyObject *dict, PyObject *name) } static int -compiler_make_closure(struct compiler *c, PyCodeObject *co, int args) +compiler_make_closure(struct compiler *c, PyCodeObject *co, int args, PyObject *qualname) { int i, free = PyCode_GetNumFree(co); + if (qualname == NULL) + qualname = co->co_name; + if (free == 0) { ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); + ADDOP_O(c, LOAD_CONST, qualname, consts); ADDOP_I(c, MAKE_FUNCTION, args); return 1; } @@ -1292,6 +1381,7 @@ compiler_make_closure(struct compiler *c, PyCodeObject *co, int args) } ADDOP_I(c, BUILD_TUPLE, free); ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); + ADDOP_O(c, LOAD_CONST, qualname, consts); ADDOP_I(c, MAKE_CLOSURE, args); return 1; } @@ -1433,7 +1523,7 @@ static int compiler_function(struct compiler *c, stmt_ty s) { PyCodeObject *co; - PyObject *first_const = Py_None; + PyObject *qualname, *first_const = Py_None; arguments_ty args = s->v.FunctionDef.args; expr_ty returns = s->v.FunctionDef.returns; asdl_seq* decos = s->v.FunctionDef.decorator_list; @@ -1459,7 +1549,8 @@ compiler_function(struct compiler *c, stmt_ty s) return 0; assert((num_annotations & 0xFFFF) == num_annotations); - if (!compiler_enter_scope(c, s->v.FunctionDef.name, (void *)s, + if (!compiler_enter_scope(c, s->v.FunctionDef.name, + COMPILER_SCOPE_FUNCTION, (void *)s, s->lineno)) return 0; @@ -1481,14 +1572,19 @@ compiler_function(struct compiler *c, stmt_ty s) VISIT_IN_SCOPE(c, stmt, st); } co = assemble(c, 1); + qualname = compiler_scope_qualname(c); compiler_exit_scope(c); - if (co == NULL) + if (qualname == NULL || co == NULL) { + Py_XDECREF(qualname); + Py_XDECREF(co); return 0; + } arglength = asdl_seq_LEN(args->defaults); arglength |= kw_default_count << 8; arglength |= num_annotations << 16; - compiler_make_closure(c, co, arglength); + compiler_make_closure(c, co, arglength, qualname); + Py_DECREF(qualname); Py_DECREF(co); /* decorators */ @@ -1523,7 +1619,8 @@ compiler_class(struct compiler *c, stmt_ty s) */ /* 1. compile the class body into a code object */ - if (!compiler_enter_scope(c, s->v.ClassDef.name, (void *)s, s->lineno)) + if (!compiler_enter_scope(c, s->v.ClassDef.name, + COMPILER_SCOPE_CLASS, (void *)s, s->lineno)) return 0; /* this block represents what we do in the new scope */ { @@ -1553,13 +1650,28 @@ compiler_class(struct compiler *c, stmt_ty s) return 0; } Py_DECREF(str); + /* store the __qualname__ */ + str = compiler_scope_qualname(c); + if (!str) { + compiler_exit_scope(c); + return 0; + } + ADDOP_O(c, LOAD_CONST, str, consts); + Py_DECREF(str); + str = PyUnicode_InternFromString("__qualname__"); + if (!str || !compiler_nameop(c, str, Store)) { + Py_XDECREF(str); + compiler_exit_scope(c); + return 0; + } + Py_DECREF(str); /* compile the body proper */ if (!compiler_body(c, s->v.ClassDef.body)) { compiler_exit_scope(c); return 0; } /* return the (empty) __class__ cell */ - str = PyUnicode_InternFromString("__class__"); + str = PyUnicode_InternFromString("@__class__"); if (str == NULL) { compiler_exit_scope(c); return 0; @@ -1589,7 +1701,7 @@ compiler_class(struct compiler *c, stmt_ty s) ADDOP(c, LOAD_BUILD_CLASS); /* 3. load a function (or closure) made from the code object */ - compiler_make_closure(c, co, 0); + compiler_make_closure(c, co, 0, NULL); Py_DECREF(co); /* 4. load class name */ @@ -1640,6 +1752,7 @@ static int compiler_lambda(struct compiler *c, expr_ty e) { PyCodeObject *co; + PyObject *qualname; static identifier name; int kw_default_count = 0, arglength; arguments_ty args = e->v.Lambda.args; @@ -1659,7 +1772,8 @@ compiler_lambda(struct compiler *c, expr_ty e) } if (args->defaults) VISIT_SEQ(c, expr, args->defaults); - if (!compiler_enter_scope(c, name, (void *)e, e->lineno)) + if (!compiler_enter_scope(c, name, COMPILER_SCOPE_FUNCTION, + (void *)e, e->lineno)) return 0; /* Make None the first constant, so the lambda can't have a @@ -1677,13 +1791,15 @@ compiler_lambda(struct compiler *c, expr_ty e) ADDOP_IN_SCOPE(c, RETURN_VALUE); } co = assemble(c, 1); + qualname = compiler_scope_qualname(c); compiler_exit_scope(c); - if (co == NULL) + if (qualname == NULL || co == NULL) return 0; arglength = asdl_seq_LEN(args->defaults); arglength |= kw_default_count << 8; - compiler_make_closure(c, co, arglength); + compiler_make_closure(c, co, arglength, qualname); + Py_DECREF(qualname); Py_DECREF(co); return 1; @@ -1892,7 +2008,13 @@ compiler_try_finally(struct compiler *c, stmt_ty s) compiler_use_next_block(c, body); if (!compiler_push_fblock(c, FINALLY_TRY, body)) return 0; - VISIT_SEQ(c, stmt, s->v.TryFinally.body); + if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) { + if (!compiler_try_except(c, s)) + return 0; + } + else { + VISIT_SEQ(c, stmt, s->v.Try.body); + } ADDOP(c, POP_BLOCK); compiler_pop_fblock(c, FINALLY_TRY, body); @@ -1900,7 +2022,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) compiler_use_next_block(c, end); if (!compiler_push_fblock(c, FINALLY_END, end)) return 0; - VISIT_SEQ(c, stmt, s->v.TryFinally.finalbody); + VISIT_SEQ(c, stmt, s->v.Try.finalbody); ADDOP(c, END_FINALLY); compiler_pop_fblock(c, FINALLY_END, end); @@ -1954,15 +2076,15 @@ compiler_try_except(struct compiler *c, stmt_ty s) compiler_use_next_block(c, body); if (!compiler_push_fblock(c, EXCEPT, body)) return 0; - VISIT_SEQ(c, stmt, s->v.TryExcept.body); + VISIT_SEQ(c, stmt, s->v.Try.body); ADDOP(c, POP_BLOCK); compiler_pop_fblock(c, EXCEPT, body); ADDOP_JREL(c, JUMP_FORWARD, orelse); - n = asdl_seq_LEN(s->v.TryExcept.handlers); + n = asdl_seq_LEN(s->v.Try.handlers); compiler_use_next_block(c, except); for (i = 0; i < n; i++) { excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - s->v.TryExcept.handlers, i); + s->v.Try.handlers, i); if (!handler->v.ExceptHandler.type && i < n-1) return compiler_error(c, "default 'except:' must be last"); c->u->u_lineno_set = 0; @@ -2049,12 +2171,21 @@ compiler_try_except(struct compiler *c, stmt_ty s) } ADDOP(c, END_FINALLY); compiler_use_next_block(c, orelse); - VISIT_SEQ(c, stmt, s->v.TryExcept.orelse); + VISIT_SEQ(c, stmt, s->v.Try.orelse); compiler_use_next_block(c, end); return 1; } static int +compiler_try(struct compiler *c, stmt_ty s) { + if (s->v.Try.finalbody && asdl_seq_LEN(s->v.Try.finalbody)) + return compiler_try_finally(c, s); + else + return compiler_try_except(c, s); +} + + +static int compiler_import_as(struct compiler *c, identifier name, identifier asname) { /* The IMPORT_NAME opcode was already generated. This function @@ -2063,22 +2194,27 @@ compiler_import_as(struct compiler *c, identifier name, identifier asname) If there is a dot in name, we need to split it and emit a LOAD_ATTR for each name. */ - const Py_UNICODE *src = PyUnicode_AS_UNICODE(name); - const Py_UNICODE *dot = Py_UNICODE_strchr(src, '.'); - if (dot) { + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, + PyUnicode_GET_LENGTH(name), 1); + if (dot == -2) + return -1; + if (dot != -1) { /* Consume the base module name to get the first attribute */ - src = dot + 1; - while (dot) { - /* NB src is only defined when dot != NULL */ + Py_ssize_t pos = dot + 1; + while (dot != -1) { PyObject *attr; - dot = Py_UNICODE_strchr(src, '.'); - attr = PyUnicode_FromUnicode(src, - dot ? dot - src : Py_UNICODE_strlen(src)); + dot = PyUnicode_FindChar(name, '.', pos, + PyUnicode_GET_LENGTH(name), 1); + if (dot == -2) + return -1; + attr = PyUnicode_Substring(name, pos, + (dot != -1) ? dot : + PyUnicode_GET_LENGTH(name)); if (!attr) return -1; ADDOP_O(c, LOAD_ATTR, attr, names); Py_DECREF(attr); - src = dot + 1; + pos = dot + 1; } } return compiler_nameop(c, asname, Store); @@ -2117,13 +2253,12 @@ compiler_import(struct compiler *c, stmt_ty s) } else { identifier tmp = alias->name; - const Py_UNICODE *base = PyUnicode_AS_UNICODE(alias->name); - Py_UNICODE *dot = Py_UNICODE_strchr(base, '.'); - if (dot) - tmp = PyUnicode_FromUnicode(base, - dot - base); + Py_ssize_t dot = PyUnicode_FindChar( + alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1); + if (dot != -1) + tmp = PyUnicode_Substring(alias->name, 0, dot); r = compiler_nameop(c, tmp, Store); - if (dot) { + if (dot != -1) { Py_DECREF(tmp); } if (!r) @@ -2186,7 +2321,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); identifier store_name; - if (i == 0 && *PyUnicode_AS_UNICODE(alias->name) == '*') { + if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') { assert(n == 1); ADDOP(c, IMPORT_STAR); return 1; @@ -2301,10 +2436,8 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) } ADDOP_I(c, RAISE_VARARGS, n); break; - case TryExcept_kind: - return compiler_try_except(c, s); - case TryFinally_kind: - return compiler_try_finally(c, s); + case Try_kind: + return compiler_try(c, s); case Assert_kind: return compiler_assert(c, s); case Import_kind: @@ -2335,7 +2468,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) case Continue_kind: return compiler_continue(c); case With_kind: - return compiler_with(c, s); + return compiler_with(c, s, 0); } return 1; } @@ -2502,7 +2635,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) } /* XXX Leave assert here, but handle __doc__ and the like better */ - assert(scope || PyUnicode_AS_UNICODE(name)[0] == '_'); + assert(scope || PyUnicode_READ_CHAR(name, 0) == '_'); switch (optype) { case OP_DEREF: @@ -2880,11 +3013,13 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name, { PyCodeObject *co = NULL; expr_ty outermost_iter; + PyObject *qualname = NULL; outermost_iter = ((comprehension_ty) asdl_seq_GET(generators, 0))->iter; - if (!compiler_enter_scope(c, name, (void *)e, e->lineno)) + if (!compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, + (void *)e, e->lineno)) goto error; if (type != COMP_GENEXP) { @@ -2917,12 +3052,14 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name, } co = assemble(c, 1); + qualname = compiler_scope_qualname(c); compiler_exit_scope(c); - if (co == NULL) + if (qualname == NULL || co == NULL) goto error; - if (!compiler_make_closure(c, co, 0)) + if (!compiler_make_closure(c, co, 0, qualname)) goto error; + Py_DECREF(qualname); Py_DECREF(co); VISIT(c, expr, outermost_iter); @@ -2932,6 +3069,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name, error_in_scope: compiler_exit_scope(c); error: + Py_XDECREF(qualname); Py_XDECREF(co); return 0; } @@ -3025,8 +3163,7 @@ expr_constant(struct compiler *c, expr_ty e) return PyObject_IsTrue(e->v.Str.s); case Name_kind: /* optimize away names that can't be reassigned */ - id = PyBytes_AS_STRING( - _PyUnicode_AsDefaultEncodedString(e->v.Name.id, NULL)); + id = PyUnicode_AsUTF8(e->v.Name.id); if (strcmp(id, "True") == 0) return 1; if (strcmp(id, "False") == 0) return 0; if (strcmp(id, "None") == 0) return 0; @@ -3062,9 +3199,10 @@ expr_constant(struct compiler *c, expr_ty e) exit(*exc) */ static int -compiler_with(struct compiler *c, stmt_ty s) +compiler_with(struct compiler *c, stmt_ty s, int pos) { basicblock *block, *finally; + withitem_ty item = asdl_seq_GET(s->v.With.items, pos); assert(s->kind == With_kind); @@ -3074,7 +3212,7 @@ compiler_with(struct compiler *c, stmt_ty s) return 0; /* Evaluate EXPR */ - VISIT(c, expr, s->v.With.context_expr); + VISIT(c, expr, item->context_expr); ADDOP_JREL(c, SETUP_WITH, finally); /* SETUP_WITH pushes a finally block. */ @@ -3083,16 +3221,20 @@ compiler_with(struct compiler *c, stmt_ty s) return 0; } - if (s->v.With.optional_vars) { - VISIT(c, expr, s->v.With.optional_vars); + if (item->optional_vars) { + VISIT(c, expr, item->optional_vars); } else { /* Discard result from context.__enter__() */ ADDOP(c, POP_TOP); } - /* BLOCK code */ - VISIT_SEQ(c, stmt, s->v.With.body); + pos++; + if (pos == asdl_seq_LEN(s->v.With.items)) + /* BLOCK code */ + VISIT_SEQ(c, stmt, s->v.With.body) + else if (!compiler_with(c, s, pos)) + return 0; /* End of try block; start the finally block */ ADDOP(c, POP_BLOCK); @@ -3169,16 +3311,25 @@ compiler_visit_expr(struct compiler *c, expr_ty e) case DictComp_kind: return compiler_dictcomp(c, e); case Yield_kind: + case YieldFrom_kind: { + expr_ty value; if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'yield' outside function"); - if (e->v.Yield.value) { - VISIT(c, expr, e->v.Yield.value); + value = (e->kind == YieldFrom_kind) ? e->v.YieldFrom.value : e->v.Yield.value; + if (value) { + VISIT(c, expr, value); } else { ADDOP_O(c, LOAD_CONST, Py_None, consts); } - ADDOP(c, YIELD_VALUE); + if (e->kind == YieldFrom_kind) { + ADDOP(c, YIELD_FROM); + } + else { + ADDOP(c, YIELD_VALUE); + } break; + } case Compare_kind: return compiler_compare(c, e); case Call_kind: @@ -3361,7 +3512,7 @@ compiler_in_loop(struct compiler *c) { static int compiler_error(struct compiler *c, const char *errstr) { - PyObject *loc, *filename; + PyObject *loc; PyObject *u = NULL, *v = NULL; loc = PyErr_ProgramText(c->c_filename, c->u->u_lineno); @@ -3369,16 +3520,7 @@ compiler_error(struct compiler *c, const char *errstr) Py_INCREF(Py_None); loc = Py_None; } - if (c->c_filename != NULL) { - filename = PyUnicode_DecodeFSDefault(c->c_filename); - if (!filename) - goto exit; - } - else { - Py_INCREF(Py_None); - filename = Py_None; - } - u = Py_BuildValue("(NiiO)", filename, c->u->u_lineno, + u = Py_BuildValue("(OiiO)", c->c_filename_obj, c->u->u_lineno, c->u->u_col_offset, loc); if (!u) goto exit; @@ -3927,7 +4069,6 @@ makecode(struct compiler *c, struct assembler *a) PyObject *consts = NULL; PyObject *names = NULL; PyObject *varnames = NULL; - PyObject *filename = NULL; PyObject *name = NULL; PyObject *freevars = NULL; PyObject *cellvars = NULL; @@ -3951,10 +4092,6 @@ makecode(struct compiler *c, struct assembler *a) freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars)); if (!freevars) goto error; - filename = PyUnicode_DecodeFSDefault(c->c_filename); - if (!filename) - goto error; - nlocals = PyDict_Size(c->u->u_varnames); flags = compute_code_flags(c); if (flags < 0) @@ -3974,14 +4111,13 @@ makecode(struct compiler *c, struct assembler *a) nlocals, stackdepth(c), flags, bytecode, consts, names, varnames, freevars, cellvars, - filename, c->u->u_name, + c->c_filename_obj, c->u->u_name, c->u->u_firstlineno, a->a_lnotab); error: Py_XDECREF(consts); Py_XDECREF(names); Py_XDECREF(varnames); - Py_XDECREF(filename); Py_XDECREF(name); Py_XDECREF(freevars); Py_XDECREF(cellvars); diff --git a/Python/dtoa.c b/Python/dtoa.c index 44dc01f..82b6faa 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -2055,7 +2055,7 @@ _Py_dg_strtod(const char *s00, char **se) + Exp_msk1 ; word1(&rv) = 0; - dsign = 0; + /* dsign = 0; */ break; } } @@ -2092,7 +2092,7 @@ _Py_dg_strtod(const char *s00, char **se) goto undfl; } } - dsign = 1 - dsign; + /* dsign = 1 - dsign; */ break; } if ((aadj = ratio(delta, bs)) <= 2.) { diff --git a/Python/dynload_aix.c b/Python/dynload_aix.c index 149990d..8346f06 100644 --- a/Python/dynload_aix.c +++ b/Python/dynload_aix.c @@ -28,7 +28,6 @@ typedef struct Module { const struct filedescr _PyImport_DynLoadFiletab[] = { {".so", "rb", C_EXTENSION}, - {"module.so", "rb", C_EXTENSION}, {0, 0} }; @@ -129,7 +128,6 @@ aix_loaderror(const char *pathname) {L_ERROR_ERRNO, NULL} }; -#define LOAD_ERRTAB_LEN (sizeof(load_errtab)/sizeof(load_errtab[0])) #define ERRBUF_APPEND(s) strncat(errbuf, s, sizeof(errbuf)-strlen(errbuf)-1) PyOS_snprintf(errbuf, sizeof(errbuf), "from module %.200s ", pathname); @@ -140,7 +138,7 @@ aix_loaderror(const char *pathname) } for(i = 0; message[i] && *message[i]; i++) { int nerr = atoi(message[i]); - for (j=0; j<LOAD_ERRTAB_LEN ; j++) { + for (j=0; j < Py_ARRAY_LENGTH(load_errtab); j++) { if (nerr == load_errtab[j].errNo && load_errtab[j].errstr) ERRBUF_APPEND(load_errtab[j].errstr); } @@ -154,7 +152,7 @@ aix_loaderror(const char *pathname) } -dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, +dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname, const char *pathname, FILE *fp) { dl_funcptr p; diff --git a/Python/dynload_dl.c b/Python/dynload_dl.c index 2606e1e..a914a08 100644 --- a/Python/dynload_dl.c +++ b/Python/dynload_dl.c @@ -10,17 +10,16 @@ extern char *Py_GetProgramName(void); const struct filedescr _PyImport_DynLoadFiletab[] = { - {".o", "rb", C_EXTENSION}, - {"module.o", "rb", C_EXTENSION}, - {0, 0} + {".o", "rb", C_EXTENSION}, + {0, 0} }; -dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, - const char *pathname, FILE *fp) +dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname, + const char *pathname, FILE *fp) { - char funcname[258]; + char funcname[258]; - PyOS_snprintf(funcname, sizeof(funcname), "PyInit_%.200s", shortname); - return dl_loadmod(Py_GetProgramName(), pathname, funcname); + PyOS_snprintf(funcname, sizeof(funcname), "PyInit_%.200s", shortname); + return dl_loadmod(Py_GetProgramName(), pathname, funcname); } diff --git a/Python/dynload_hpux.c b/Python/dynload_hpux.c index 18a8137..1004010 100644 --- a/Python/dynload_hpux.c +++ b/Python/dynload_hpux.c @@ -15,11 +15,10 @@ const struct filedescr _PyImport_DynLoadFiletab[] = { {SHLIB_EXT, "rb", C_EXTENSION}, - {"module"SHLIB_EXT, "rb", C_EXTENSION}, {0, 0} }; -dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, +dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname, const char *pathname, FILE *fp) { dl_funcptr p; diff --git a/Python/dynload_next.c b/Python/dynload_next.c index cabf9b9..5caff8b 100644 --- a/Python/dynload_next.c +++ b/Python/dynload_next.c @@ -10,7 +10,6 @@ const struct filedescr _PyImport_DynLoadFiletab[] = { {".so", "rb", C_EXTENSION}, - {"module.so", "rb", C_EXTENSION}, {0, 0} }; @@ -31,8 +30,8 @@ const struct filedescr _PyImport_DynLoadFiletab[] = { #define LINKOPTIONS NSLINKMODULE_OPTION_BINDNOW| \ NSLINKMODULE_OPTION_RETURN_ON_ERROR|NSLINKMODULE_OPTION_PRIVATE #endif -dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, - const char *pathname, FILE *fp) +dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname, + const char *pathname, FILE *fp) { dl_funcptr p = NULL; char funcname[258]; diff --git a/Python/dynload_os2.c b/Python/dynload_os2.c index 101c024..f7264da 100644 --- a/Python/dynload_os2.c +++ b/Python/dynload_os2.c @@ -15,7 +15,7 @@ const struct filedescr _PyImport_DynLoadFiletab[] = { {0, 0} }; -dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, +dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname, const char *pathname, FILE *fp) { dl_funcptr p; diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 7ea510e..ab24238 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -39,7 +39,6 @@ const struct filedescr _PyImport_DynLoadFiletab[] = { #ifdef __CYGWIN__ {".dll", "rb", C_EXTENSION}, - {"module.dll", "rb", C_EXTENSION}, #else /* !__CYGWIN__ */ #if defined(PYOS_OS2) && defined(PYCC_GCC) {".pyd", "rb", C_EXTENSION}, @@ -48,15 +47,10 @@ const struct filedescr _PyImport_DynLoadFiletab[] = { #ifdef __VMS {".exe", "rb", C_EXTENSION}, {".EXE", "rb", C_EXTENSION}, - {"module.exe", "rb", C_EXTENSION}, - {"MODULE.EXE", "rb", C_EXTENSION}, #else /* !__VMS */ {"." SOABI ".so", "rb", C_EXTENSION}, - {"module." SOABI ".so", "rb", C_EXTENSION}, {".abi" PYTHON_ABI_STRING ".so", "rb", C_EXTENSION}, - {"module.abi" PYTHON_ABI_STRING ".so", "rb", C_EXTENSION}, {".so", "rb", C_EXTENSION}, - {"module.so", "rb", C_EXTENSION}, #endif /* __VMS */ #endif /* defined(PYOS_OS2) && defined(PYCC_GCC) */ #endif /* __CYGWIN__ */ @@ -75,7 +69,7 @@ static struct { static int nhandles = 0; -dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, +dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname, const char *pathname, FILE *fp) { dl_funcptr p; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 73a1dcf..2cbfe9f 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -171,43 +171,38 @@ static char *GetPythonImport (HINSTANCE hModule) return NULL; } -dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, - const char *pathname, FILE *fp) +dl_funcptr _PyImport_GetDynLoadWindows(const char *shortname, + PyObject *pathname, FILE *fp) { dl_funcptr p; char funcname[258], *import_python; + wchar_t *wpathname; #ifndef _DEBUG _Py_CheckPython3(); #endif + wpathname = PyUnicode_AsUnicode(pathname); + if (wpathname == NULL) + return NULL; + PyOS_snprintf(funcname, sizeof(funcname), "PyInit_%.200s", shortname); { HINSTANCE hDLL = NULL; - char pathbuf[260]; - LPTSTR dummy; unsigned int old_mode; ULONG_PTR cookie = 0; - /* We use LoadLibraryEx so Windows looks for dependent DLLs - in directory of pathname first. However, Windows95 - can sometimes not work correctly unless the absolute - path is used. If GetFullPathName() fails, the LoadLibrary - will certainly fail too, so use its error code */ /* Don't display a message box when Python can't load a DLL */ old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); - if (GetFullPathName(pathname, - sizeof(pathbuf), - pathbuf, - &dummy)) { - ULONG_PTR cookie = _Py_ActivateActCtx(); - /* XXX This call doesn't exist in Windows CE */ - hDLL = LoadLibraryEx(pathname, NULL, - LOAD_WITH_ALTERED_SEARCH_PATH); - _Py_DeactivateActCtx(cookie); - } + cookie = _Py_ActivateActCtx(); + /* We use LoadLibraryEx so Windows looks for dependent DLLs + in directory of pathname first. */ + /* XXX This call doesn't exist in Windows CE */ + hDLL = LoadLibraryExW(wpathname, NULL, + LOAD_WITH_ALTERED_SEARCH_PATH); + _Py_DeactivateActCtx(cookie); /* restore old error mode settings */ SetErrorMode(old_mode); @@ -254,31 +249,33 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, "DLL load failed: "); PyUnicode_AppendAndDel(&message, - PyUnicode_FromUnicode( + PyUnicode_FromWideChar( theInfo, theLength)); } - PyErr_SetObject(PyExc_ImportError, message); - Py_XDECREF(message); + if (message != NULL) { + PyErr_SetObject(PyExc_ImportError, message); + Py_DECREF(message); + } return NULL; } else { char buffer[256]; + PyOS_snprintf(buffer, sizeof(buffer), #ifdef _DEBUG - PyOS_snprintf(buffer, sizeof(buffer), "python%d%d_d.dll", + "python%d%d_d.dll", #else - PyOS_snprintf(buffer, sizeof(buffer), "python%d%d.dll", + "python%d%d.dll", #endif PY_MAJOR_VERSION,PY_MINOR_VERSION); import_python = GetPythonImport(hDLL); if (import_python && strcasecmp(buffer,import_python)) { - PyOS_snprintf(buffer, sizeof(buffer), - "Module use of %.150s conflicts " - "with this version of Python.", - import_python); - PyErr_SetString(PyExc_ImportError,buffer); + PyErr_Format(PyExc_ImportError, + "Module use of %.150s conflicts " + "with this version of Python.", + import_python); FreeLibrary(hDLL); return NULL; } diff --git a/Python/errors.c b/Python/errors.c index 5a9a624..31fa9e2 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -341,11 +341,9 @@ PyObject * PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject) { PyObject *message; - PyObject *v; + PyObject *v, *args; int i = errno; -#ifndef MS_WINDOWS - char *s; -#else +#ifdef MS_WINDOWS WCHAR *s_buf = NULL; #endif /* Unix/Windows */ @@ -355,11 +353,14 @@ PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject) #endif #ifndef MS_WINDOWS - if (i == 0) - s = "Error"; /* Sometimes errno didn't get set */ - else - s = strerror(i); - message = PyUnicode_DecodeUTF8(s, strlen(s), "ignore"); + if (i != 0) { + char *s = strerror(i); + message = PyUnicode_DecodeLocale(s, "surrogateescape"); + } + else { + /* Sometimes errno didn't get set */ + message = PyUnicode_FromString("Error"); + } #else if (i == 0) message = PyUnicode_FromString("Error"); /* Sometimes errno didn't get set */ @@ -395,7 +396,7 @@ PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject) /* remove trailing cr/lf and dots */ while (len > 0 && (s_buf[len-1] <= L' ' || s_buf[len-1] == L'.')) s_buf[--len] = L'\0'; - message = PyUnicode_FromUnicode(s_buf, len); + message = PyUnicode_FromWideChar(s_buf, len); } } } @@ -410,14 +411,18 @@ PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject) } if (filenameObject != NULL) - v = Py_BuildValue("(iOO)", i, message, filenameObject); + args = Py_BuildValue("(iOO)", i, message, filenameObject); else - v = Py_BuildValue("(iO)", i, message); + args = Py_BuildValue("(iO)", i, message); Py_DECREF(message); - if (v != NULL) { - PyErr_SetObject(exc, v); - Py_DECREF(v); + if (args != NULL) { + v = PyObject_Call(exc, args, NULL); + Py_DECREF(args); + if (v != NULL) { + PyErr_SetObject((PyObject *) Py_TYPE(v), v); + Py_DECREF(v); + } } #ifdef MS_WINDOWS LocalFree(s_buf); @@ -464,7 +469,7 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilenameObject( int len; WCHAR *s_buf = NULL; /* Free via LocalFree */ PyObject *message; - PyObject *v; + PyObject *args, *v; DWORD err = (DWORD)ierr; if (err==0) err = GetLastError(); len = FormatMessageW( @@ -487,7 +492,7 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilenameObject( /* remove trailing cr/lf and dots */ while (len > 0 && (s_buf[len-1] <= L' ' || s_buf[len-1] == L'.')) s_buf[--len] = L'\0'; - message = PyUnicode_FromUnicode(s_buf, len); + message = PyUnicode_FromWideChar(s_buf, len); } if (message == NULL) @@ -496,15 +501,20 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilenameObject( return NULL; } - if (filenameObject != NULL) - v = Py_BuildValue("(iOO)", err, message, filenameObject); - else - v = Py_BuildValue("(iO)", err, message); + if (filenameObject == NULL) + filenameObject = Py_None; + /* This is the constructor signature for passing a Windows error code. + The POSIX translation will be figured out by the constructor. */ + args = Py_BuildValue("(iOOi)", 0, message, filenameObject, err); Py_DECREF(message); - if (v != NULL) { - PyErr_SetObject(exc, v); - Py_DECREF(v); + if (args != NULL) { + v = PyObject_Call(exc, args, NULL); + Py_DECREF(args); + if (v != NULL) { + PyErr_SetObject((PyObject *) Py_TYPE(v), v); + Py_DECREF(v); + } } LocalFree(s_buf); return NULL; @@ -656,7 +666,7 @@ PyErr_NewException(const char *name, PyObject *base, PyObject *dict) if (bases == NULL) goto failure; } - /* Create a real new-style class. */ + /* Create a real class. */ result = PyObject_CallFunction((PyObject *)&PyType_Type, "sOO", dot+1, bases, dict); failure: @@ -707,6 +717,7 @@ PyErr_NewExceptionWithDoc(const char *name, const char *doc, void PyErr_WriteUnraisable(PyObject *obj) { + _Py_IDENTIFIER(__module__); PyObject *f, *t, *v, *tb; PyErr_Fetch(&t, &v, &tb); f = PySys_GetObject("stderr"); @@ -723,7 +734,7 @@ PyErr_WriteUnraisable(PyObject *obj) className = dot+1; } - moduleName = PyObject_GetAttrString(t, "__module__"); + moduleName = _PyObject_GetAttrId(t, &PyId___module__); if (moduleName == NULL) PyFile_WriteString("<unknown>", f); else { @@ -774,6 +785,12 @@ void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) { PyObject *exc, *v, *tb, *tmp; + _Py_IDENTIFIER(filename); + _Py_IDENTIFIER(lineno); + _Py_IDENTIFIER(msg); + _Py_IDENTIFIER(offset); + _Py_IDENTIFIER(print_file_and_line); + _Py_IDENTIFIER(text); /* add attributes for the line number and filename for the error */ PyErr_Fetch(&exc, &v, &tb); @@ -784,7 +801,7 @@ PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) if (tmp == NULL) PyErr_Clear(); else { - if (PyObject_SetAttrString(v, "lineno", tmp)) + if (_PyObject_SetAttrId(v, &PyId_lineno, tmp)) PyErr_Clear(); Py_DECREF(tmp); } @@ -793,7 +810,7 @@ PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) if (tmp == NULL) PyErr_Clear(); else { - if (PyObject_SetAttrString(v, "offset", tmp)) + if (_PyObject_SetAttrId(v, &PyId_offset, tmp)) PyErr_Clear(); Py_DECREF(tmp); } @@ -803,35 +820,35 @@ PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) if (tmp == NULL) PyErr_Clear(); else { - if (PyObject_SetAttrString(v, "filename", tmp)) + if (_PyObject_SetAttrId(v, &PyId_filename, tmp)) PyErr_Clear(); Py_DECREF(tmp); } tmp = PyErr_ProgramText(filename, lineno); if (tmp) { - if (PyObject_SetAttrString(v, "text", tmp)) + if (_PyObject_SetAttrId(v, &PyId_text, tmp)) PyErr_Clear(); Py_DECREF(tmp); } } - if (PyObject_SetAttrString(v, "offset", Py_None)) { + if (_PyObject_SetAttrId(v, &PyId_offset, Py_None)) { PyErr_Clear(); } if (exc != PyExc_SyntaxError) { - if (!PyObject_HasAttrString(v, "msg")) { + if (!_PyObject_HasAttrId(v, &PyId_msg)) { tmp = PyObject_Str(v); if (tmp) { - if (PyObject_SetAttrString(v, "msg", tmp)) + if (_PyObject_SetAttrId(v, &PyId_msg, tmp)) PyErr_Clear(); Py_DECREF(tmp); } else { PyErr_Clear(); } } - if (!PyObject_HasAttrString(v, "print_file_and_line")) { - if (PyObject_SetAttrString(v, "print_file_and_line", - Py_None)) + if (!_PyObject_HasAttrId(v, &PyId_print_file_and_line)) { + if (_PyObject_SetAttrId(v, &PyId_print_file_and_line, + Py_None)) PyErr_Clear(); } } diff --git a/Python/fileutils.c b/Python/fileutils.c index c563eaa..8993c8c 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -16,7 +16,9 @@ Return a pointer to a newly allocated wide character string (use PyMem_Free() to free the memory) and write the number of written wide characters excluding the null character into *size if size is not NULL, or - NULL on error (conversion or memory allocation error). + NULL on error (decoding or memory allocation error). If size is not NULL, + *size is set to (size_t)-1 on memory error and (size_t)-2 on decoding + error. Conversion errors should never happen, unless there is a bug in the C library. */ @@ -82,8 +84,9 @@ _Py_char2wchar(const char* arg, size_t *size) since we provide everything that we have - unless there is a bug in the C library, or I misunderstood how mbrtowc works. */ - fprintf(stderr, "unexpected mbrtowc result -2\n"); PyMem_Free(res); + if (size != NULL) + *size = (size_t)-2; return NULL; } if (converted == (size_t)-1) { @@ -112,7 +115,8 @@ _Py_char2wchar(const char* arg, size_t *size) is ASCII (i.e. escape all bytes > 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t)); - if (!res) goto oom; + if (!res) + goto oom; in = (unsigned char*)arg; out = res; while(*in) @@ -126,7 +130,8 @@ _Py_char2wchar(const char* arg, size_t *size) *size = out - res; return res; oom: - fprintf(stderr, "out of memory\n"); + if (size != NULL) + *size = (size_t)-1; return NULL; } @@ -137,10 +142,10 @@ oom: This function is the reverse of _Py_char2wchar(). Return a pointer to a newly allocated byte string (use PyMem_Free() to free - the memory), or NULL on conversion or memory allocation error. + the memory), or NULL on encoding or memory allocation error. If error_pos is not NULL: *error_pos is the index of the invalid character - on conversion error, or (size_t)-1 otherwise. */ + on encoding error, or (size_t)-1 otherwise. */ char* _Py_wchar2char(const wchar_t *text, size_t *error_pos) { @@ -235,8 +240,8 @@ _Py_wstat(const wchar_t* path, struct stat *buf) /* Call _wstat() on Windows, or encode the path to the filesystem encoding and call stat() otherwise. Only fill st_mode attribute on Windows. - Return 0 on success, -1 on _wstat() / stat() error or (if PyErr_Occurred()) - unicode error. */ + Return 0 on success, -1 on _wstat() / stat() error, -2 if an exception was + raised. */ int _Py_stat(PyObject *path, struct stat *statbuf) @@ -244,8 +249,12 @@ _Py_stat(PyObject *path, struct stat *statbuf) #ifdef MS_WINDOWS int err; struct _stat wstatbuf; + wchar_t *wpath; - err = _wstat(PyUnicode_AS_UNICODE(path), &wstatbuf); + wpath = PyUnicode_AsUnicode(path); + if (wpath == NULL) + return -2; + err = _wstat(wpath, &wstatbuf); if (!err) statbuf->st_mode = wstatbuf.st_mode; return err; @@ -253,7 +262,7 @@ _Py_stat(PyObject *path, struct stat *statbuf) int ret; PyObject *bytes = PyUnicode_EncodeFSDefault(path); if (bytes == NULL) - return -1; + return -2; ret = stat(PyBytes_AS_STRING(bytes), statbuf); Py_DECREF(bytes); return ret; @@ -297,18 +306,29 @@ FILE* _Py_fopen(PyObject *path, const char *mode) { #ifdef MS_WINDOWS + wchar_t *wpath; wchar_t wmode[10]; int usize; + if (!PyUnicode_Check(path)) { + PyErr_Format(PyExc_TypeError, + "str file path expected under Windows, got %R", + Py_TYPE(path)); + return NULL; + } + wpath = PyUnicode_AsUnicode(path); + if (wpath == NULL) + return NULL; + usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, wmode, sizeof(wmode)); if (usize == 0) return NULL; - return _wfopen(PyUnicode_AS_UNICODE(path), wmode); + return _wfopen(wpath, wmode); #else FILE *f; - PyObject *bytes = PyUnicode_EncodeFSDefault(path); - if (bytes == NULL) + PyObject *bytes; + if (!PyUnicode_FSConverter(path, &bytes)) return NULL; f = fopen(PyBytes_AS_STRING(bytes), mode); Py_DECREF(bytes); @@ -319,7 +339,7 @@ _Py_fopen(PyObject *path, const char *mode) #ifdef HAVE_READLINK /* Read value of symbolic link. Encode the path to the locale encoding, decode - the result from the locale encoding. */ + the result from the locale encoding. Return -1 on error. */ int _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz) @@ -363,7 +383,8 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz) #ifdef HAVE_REALPATH /* Return the canonicalized absolute pathname. Encode path to the locale - encoding, decode the result from the locale encoding. */ + encoding, decode the result from the locale encoding. + Return NULL on error. */ wchar_t* _Py_wrealpath(const wchar_t *path, @@ -401,7 +422,8 @@ _Py_wrealpath(const wchar_t *path, #endif /* Get the current directory. size is the buffer size in wide characters - including the null character. Decode the path from the locale encoding. */ + including the null character. Decode the path from the locale encoding. + Return NULL on error. */ wchar_t* _Py_wgetcwd(wchar_t *buf, size_t size) diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index c350907..ed716a5 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -3,12 +3,1477 @@ of int.__float__, etc., that take and return unicode objects */ #include "Python.h" -#include "../Objects/stringlib/unicodedefs.h" +#include <locale.h> +/* Raises an exception about an unknown presentation type for this + * type. */ -#define FORMAT_STRING _PyUnicode_FormatAdvanced -#define FORMAT_LONG _PyLong_FormatAdvanced -#define FORMAT_FLOAT _PyFloat_FormatAdvanced -#define FORMAT_COMPLEX _PyComplex_FormatAdvanced +static void +unknown_presentation_type(Py_UCS4 presentation_type, + const char* type_name) +{ + /* %c might be out-of-range, hence the two cases. */ + if (presentation_type > 32 && presentation_type < 128) + PyErr_Format(PyExc_ValueError, + "Unknown format code '%c' " + "for object of type '%.200s'", + (char)presentation_type, + type_name); + else + PyErr_Format(PyExc_ValueError, + "Unknown format code '\\x%x' " + "for object of type '%.200s'", + (unsigned int)presentation_type, + type_name); +} -#include "../Objects/stringlib/formatter.h" +static void +invalid_comma_type(Py_UCS4 presentation_type) +{ + if (presentation_type > 32 && presentation_type < 128) + PyErr_Format(PyExc_ValueError, + "Cannot specify ',' with '%c'.", + (char)presentation_type); + else + PyErr_Format(PyExc_ValueError, + "Cannot specify ',' with '\\x%x'.", + (unsigned int)presentation_type); +} + +/* + get_integer consumes 0 or more decimal digit characters from an + input string, updates *result with the corresponding positive + integer, and returns the number of digits consumed. + + returns -1 on error. +*/ +static int +get_integer(PyObject *str, Py_ssize_t *pos, Py_ssize_t end, + Py_ssize_t *result) +{ + Py_ssize_t accumulator, digitval; + int numdigits; + accumulator = numdigits = 0; + for (;;(*pos)++, numdigits++) { + if (*pos >= end) + break; + digitval = Py_UNICODE_TODECIMAL(PyUnicode_READ_CHAR(str, *pos)); + if (digitval < 0) + break; + /* + Detect possible overflow before it happens: + + accumulator * 10 + digitval > PY_SSIZE_T_MAX if and only if + accumulator > (PY_SSIZE_T_MAX - digitval) / 10. + */ + if (accumulator > (PY_SSIZE_T_MAX - digitval) / 10) { + PyErr_Format(PyExc_ValueError, + "Too many decimal digits in format string"); + return -1; + } + accumulator = accumulator * 10 + digitval; + } + *result = accumulator; + return numdigits; +} + +/************************************************************************/ +/*********** standard format specifier parsing **************************/ +/************************************************************************/ + +/* returns true if this character is a specifier alignment token */ +Py_LOCAL_INLINE(int) +is_alignment_token(Py_UCS4 c) +{ + switch (c) { + case '<': case '>': case '=': case '^': + return 1; + default: + return 0; + } +} + +/* returns true if this character is a sign element */ +Py_LOCAL_INLINE(int) +is_sign_element(Py_UCS4 c) +{ + switch (c) { + case ' ': case '+': case '-': + return 1; + default: + return 0; + } +} + + +typedef struct { + Py_UCS4 fill_char; + Py_UCS4 align; + int alternate; + Py_UCS4 sign; + Py_ssize_t width; + int thousands_separators; + Py_ssize_t precision; + Py_UCS4 type; +} InternalFormatSpec; + +#if 0 +/* Occassionally useful for debugging. Should normally be commented out. */ +static void +DEBUG_PRINT_FORMAT_SPEC(InternalFormatSpec *format) +{ + printf("internal format spec: fill_char %d\n", format->fill_char); + printf("internal format spec: align %d\n", format->align); + printf("internal format spec: alternate %d\n", format->alternate); + printf("internal format spec: sign %d\n", format->sign); + printf("internal format spec: width %zd\n", format->width); + printf("internal format spec: thousands_separators %d\n", + format->thousands_separators); + printf("internal format spec: precision %zd\n", format->precision); + printf("internal format spec: type %c\n", format->type); + printf("\n"); +} +#endif + + +/* + ptr points to the start of the format_spec, end points just past its end. + fills in format with the parsed information. + returns 1 on success, 0 on failure. + if failure, sets the exception +*/ +static int +parse_internal_render_format_spec(PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end, + InternalFormatSpec *format, + char default_type, + char default_align) +{ + Py_ssize_t pos = start; + /* end-pos is used throughout this code to specify the length of + the input string */ +#define READ_spec(index) PyUnicode_READ_CHAR(format_spec, index) + + Py_ssize_t consumed; + int align_specified = 0; + + format->fill_char = '\0'; + format->align = default_align; + format->alternate = 0; + format->sign = '\0'; + format->width = -1; + format->thousands_separators = 0; + format->precision = -1; + format->type = default_type; + + /* If the second char is an alignment token, + then parse the fill char */ + if (end-pos >= 2 && is_alignment_token(READ_spec(pos+1))) { + format->align = READ_spec(pos+1); + format->fill_char = READ_spec(pos); + align_specified = 1; + pos += 2; + } + else if (end-pos >= 1 && is_alignment_token(READ_spec(pos))) { + format->align = READ_spec(pos); + align_specified = 1; + ++pos; + } + + /* Parse the various sign options */ + if (end-pos >= 1 && is_sign_element(READ_spec(pos))) { + format->sign = READ_spec(pos); + ++pos; + } + + /* If the next character is #, we're in alternate mode. This only + applies to integers. */ + if (end-pos >= 1 && READ_spec(pos) == '#') { + format->alternate = 1; + ++pos; + } + + /* The special case for 0-padding (backwards compat) */ + if (format->fill_char == '\0' && end-pos >= 1 && READ_spec(pos) == '0') { + format->fill_char = '0'; + if (!align_specified) { + format->align = '='; + } + ++pos; + } + + consumed = get_integer(format_spec, &pos, end, &format->width); + if (consumed == -1) + /* Overflow error. Exception already set. */ + return 0; + + /* If consumed is 0, we didn't consume any characters for the + width. In that case, reset the width to -1, because + get_integer() will have set it to zero. -1 is how we record + that the width wasn't specified. */ + if (consumed == 0) + format->width = -1; + + /* Comma signifies add thousands separators */ + if (end-pos && READ_spec(pos) == ',') { + format->thousands_separators = 1; + ++pos; + } + + /* Parse field precision */ + if (end-pos && READ_spec(pos) == '.') { + ++pos; + + consumed = get_integer(format_spec, &pos, end, &format->precision); + if (consumed == -1) + /* Overflow error. Exception already set. */ + return 0; + + /* Not having a precision after a dot is an error. */ + if (consumed == 0) { + PyErr_Format(PyExc_ValueError, + "Format specifier missing precision"); + return 0; + } + + } + + /* Finally, parse the type field. */ + + if (end-pos > 1) { + /* More than one char remain, invalid format specifier. */ + PyErr_Format(PyExc_ValueError, "Invalid format specifier"); + return 0; + } + + if (end-pos == 1) { + format->type = READ_spec(pos); + ++pos; + } + + /* Do as much validating as we can, just by looking at the format + specifier. Do not take into account what type of formatting + we're doing (int, float, string). */ + + if (format->thousands_separators) { + switch (format->type) { + case 'd': + case 'e': + case 'f': + case 'g': + case 'E': + case 'G': + case '%': + case 'F': + case '\0': + /* These are allowed. See PEP 378.*/ + break; + default: + invalid_comma_type(format->type); + return 0; + } + } + + assert (format->align <= 127); + assert (format->sign <= 127); + return 1; +} + +/* Calculate the padding needed. */ +static void +calc_padding(Py_ssize_t nchars, Py_ssize_t width, Py_UCS4 align, + Py_ssize_t *n_lpadding, Py_ssize_t *n_rpadding, + Py_ssize_t *n_total) +{ + if (width >= 0) { + if (nchars > width) + *n_total = nchars; + else + *n_total = width; + } + else { + /* not specified, use all of the chars and no more */ + *n_total = nchars; + } + + /* Figure out how much leading space we need, based on the + aligning */ + if (align == '>') + *n_lpadding = *n_total - nchars; + else if (align == '^') + *n_lpadding = (*n_total - nchars) / 2; + else if (align == '<' || align == '=') + *n_lpadding = 0; + else { + /* We should never have an unspecified alignment. */ + *n_lpadding = 0; + assert(0); + } + + *n_rpadding = *n_total - nchars - *n_lpadding; +} + +/* Do the padding, and return a pointer to where the caller-supplied + content goes. */ +static Py_ssize_t +fill_padding(PyObject *s, Py_ssize_t start, Py_ssize_t nchars, + Py_UCS4 fill_char, Py_ssize_t n_lpadding, + Py_ssize_t n_rpadding) +{ + /* Pad on left. */ + if (n_lpadding) + PyUnicode_Fill(s, start, start + n_lpadding, fill_char); + + /* Pad on right. */ + if (n_rpadding) + PyUnicode_Fill(s, start + nchars + n_lpadding, + start + nchars + n_lpadding + n_rpadding, fill_char); + + /* Pointer to the user content. */ + return start + n_lpadding; +} + +/************************************************************************/ +/*********** common routines for numeric formatting *********************/ +/************************************************************************/ + +/* Locale type codes. */ +#define LT_CURRENT_LOCALE 0 +#define LT_DEFAULT_LOCALE 1 +#define LT_NO_LOCALE 2 + +/* Locale info needed for formatting integers and the part of floats + before and including the decimal. Note that locales only support + 8-bit chars, not unicode. */ +typedef struct { + char *decimal_point; + char *thousands_sep; + char *grouping; +} LocaleInfo; + +/* describes the layout for an integer, see the comment in + calc_number_widths() for details */ +typedef struct { + Py_ssize_t n_lpadding; + Py_ssize_t n_prefix; + Py_ssize_t n_spadding; + Py_ssize_t n_rpadding; + char sign; + Py_ssize_t n_sign; /* number of digits needed for sign (0/1) */ + Py_ssize_t n_grouped_digits; /* Space taken up by the digits, including + any grouping chars. */ + Py_ssize_t n_decimal; /* 0 if only an integer */ + Py_ssize_t n_remainder; /* Digits in decimal and/or exponent part, + excluding the decimal itself, if + present. */ + + /* These 2 are not the widths of fields, but are needed by + STRINGLIB_GROUPING. */ + Py_ssize_t n_digits; /* The number of digits before a decimal + or exponent. */ + Py_ssize_t n_min_width; /* The min_width we used when we computed + the n_grouped_digits width. */ +} NumberFieldWidths; + + +/* Given a number of the form: + digits[remainder] + where ptr points to the start and end points to the end, find where + the integer part ends. This could be a decimal, an exponent, both, + or neither. + If a decimal point is present, set *has_decimal and increment + remainder beyond it. + Results are undefined (but shouldn't crash) for improperly + formatted strings. +*/ +static void +parse_number(PyObject *s, Py_ssize_t pos, Py_ssize_t end, + Py_ssize_t *n_remainder, int *has_decimal) +{ + Py_ssize_t remainder; + + while (pos<end && isdigit(PyUnicode_READ_CHAR(s, pos))) + ++pos; + remainder = pos; + + /* Does remainder start with a decimal point? */ + *has_decimal = pos<end && PyUnicode_READ_CHAR(s, remainder) == '.'; + + /* Skip the decimal point. */ + if (*has_decimal) + remainder++; + + *n_remainder = end - remainder; +} + +/* not all fields of format are used. for example, precision is + unused. should this take discrete params in order to be more clear + about what it does? or is passing a single format parameter easier + and more efficient enough to justify a little obfuscation? */ +static Py_ssize_t +calc_number_widths(NumberFieldWidths *spec, Py_ssize_t n_prefix, + Py_UCS4 sign_char, PyObject *number, Py_ssize_t n_start, + Py_ssize_t n_end, Py_ssize_t n_remainder, + int has_decimal, const LocaleInfo *locale, + const InternalFormatSpec *format) +{ + Py_ssize_t n_non_digit_non_padding; + Py_ssize_t n_padding; + + spec->n_digits = n_end - n_start - n_remainder - (has_decimal?1:0); + spec->n_lpadding = 0; + spec->n_prefix = n_prefix; + spec->n_decimal = has_decimal ? strlen(locale->decimal_point) : 0; + spec->n_remainder = n_remainder; + spec->n_spadding = 0; + spec->n_rpadding = 0; + spec->sign = '\0'; + spec->n_sign = 0; + + /* the output will look like: + | | + | <lpadding> <sign> <prefix> <spadding> <grouped_digits> <decimal> <remainder> <rpadding> | + | | + + sign is computed from format->sign and the actual + sign of the number + + prefix is given (it's for the '0x' prefix) + + digits is already known + + the total width is either given, or computed from the + actual digits + + only one of lpadding, spadding, and rpadding can be non-zero, + and it's calculated from the width and other fields + */ + + /* compute the various parts we're going to write */ + switch (format->sign) { + case '+': + /* always put a + or - */ + spec->n_sign = 1; + spec->sign = (sign_char == '-' ? '-' : '+'); + break; + case ' ': + spec->n_sign = 1; + spec->sign = (sign_char == '-' ? '-' : ' '); + break; + default: + /* Not specified, or the default (-) */ + if (sign_char == '-') { + spec->n_sign = 1; + spec->sign = '-'; + } + } + + /* The number of chars used for non-digits and non-padding. */ + n_non_digit_non_padding = spec->n_sign + spec->n_prefix + spec->n_decimal + + spec->n_remainder; + + /* min_width can go negative, that's okay. format->width == -1 means + we don't care. */ + if (format->fill_char == '0' && format->align == '=') + spec->n_min_width = format->width - n_non_digit_non_padding; + else + spec->n_min_width = 0; + + if (spec->n_digits == 0) + /* This case only occurs when using 'c' formatting, we need + to special case it because the grouping code always wants + to have at least one character. */ + spec->n_grouped_digits = 0; + else + spec->n_grouped_digits = _PyUnicode_InsertThousandsGrouping( + NULL, PyUnicode_1BYTE_KIND, NULL, 0, NULL, + spec->n_digits, spec->n_min_width, + locale->grouping, locale->thousands_sep); + + /* Given the desired width and the total of digit and non-digit + space we consume, see if we need any padding. format->width can + be negative (meaning no padding), but this code still works in + that case. */ + n_padding = format->width - + (n_non_digit_non_padding + spec->n_grouped_digits); + if (n_padding > 0) { + /* Some padding is needed. Determine if it's left, space, or right. */ + switch (format->align) { + case '<': + spec->n_rpadding = n_padding; + break; + case '^': + spec->n_lpadding = n_padding / 2; + spec->n_rpadding = n_padding - spec->n_lpadding; + break; + case '=': + spec->n_spadding = n_padding; + break; + case '>': + spec->n_lpadding = n_padding; + break; + default: + /* Shouldn't get here, but treat it as '>' */ + spec->n_lpadding = n_padding; + assert(0); + break; + } + } + return spec->n_lpadding + spec->n_sign + spec->n_prefix + + spec->n_spadding + spec->n_grouped_digits + spec->n_decimal + + spec->n_remainder + spec->n_rpadding; +} + +/* Fill in the digit parts of a numbers's string representation, + as determined in calc_number_widths(). + Return -1 on error, or 0 on success. */ +static int +fill_number(PyObject *out, Py_ssize_t pos, const NumberFieldWidths *spec, + PyObject *digits, Py_ssize_t d_start, Py_ssize_t d_end, + PyObject *prefix, Py_ssize_t p_start, + Py_UCS4 fill_char, + LocaleInfo *locale, int toupper) +{ + /* Used to keep track of digits, decimal, and remainder. */ + Py_ssize_t d_pos = d_start; + unsigned int kind = PyUnicode_KIND(out); + void *data = PyUnicode_DATA(out); + +#ifndef NDEBUG + Py_ssize_t r; +#endif + + if (spec->n_lpadding) { + PyUnicode_Fill(out, pos, pos + spec->n_lpadding, fill_char); + pos += spec->n_lpadding; + } + if (spec->n_sign == 1) { + PyUnicode_WRITE(kind, data, pos++, spec->sign); + } + if (spec->n_prefix) { + if (PyUnicode_CopyCharacters(out, pos, + prefix, p_start, + spec->n_prefix) < 0) + return -1; + if (toupper) { + Py_ssize_t t; + for (t = 0; t < spec->n_prefix; t++) { + Py_UCS4 c = PyUnicode_READ(kind, data, pos + t); + c = Py_TOUPPER(c); + assert (c <= 127); + PyUnicode_WRITE(kind, data, pos + t, c); + } + } + pos += spec->n_prefix; + } + if (spec->n_spadding) { + PyUnicode_Fill(out, pos, pos + spec->n_spadding, fill_char); + pos += spec->n_spadding; + } + + /* Only for type 'c' special case, it has no digits. */ + if (spec->n_digits != 0) { + /* Fill the digits with InsertThousandsGrouping. */ + char *pdigits; + if (PyUnicode_READY(digits)) + return -1; + pdigits = PyUnicode_DATA(digits); + if (PyUnicode_KIND(digits) < kind) { + pdigits = _PyUnicode_AsKind(digits, kind); + if (pdigits == NULL) + return -1; + } +#ifndef NDEBUG + r = +#endif + _PyUnicode_InsertThousandsGrouping( + out, kind, + (char*)data + kind * pos, + spec->n_grouped_digits, + pdigits + kind * d_pos, + spec->n_digits, spec->n_min_width, + locale->grouping, locale->thousands_sep); +#ifndef NDEBUG + assert(r == spec->n_grouped_digits); +#endif + if (PyUnicode_KIND(digits) < kind) + PyMem_Free(pdigits); + d_pos += spec->n_digits; + } + if (toupper) { + Py_ssize_t t; + for (t = 0; t < spec->n_grouped_digits; t++) { + Py_UCS4 c = PyUnicode_READ(kind, data, pos + t); + c = Py_TOUPPER(c); + if (c > 127) { + PyErr_SetString(PyExc_SystemError, "non-ascii grouped digit"); + return -1; + } + PyUnicode_WRITE(kind, data, pos + t, c); + } + } + pos += spec->n_grouped_digits; + + if (spec->n_decimal) { + Py_ssize_t t; + for (t = 0; t < spec->n_decimal; ++t) + PyUnicode_WRITE(kind, data, pos + t, + locale->decimal_point[t]); + pos += spec->n_decimal; + d_pos += 1; + } + + if (spec->n_remainder) { + if (PyUnicode_CopyCharacters(out, pos, digits, d_pos, spec->n_remainder) < 0) + return -1; + pos += spec->n_remainder; + d_pos += spec->n_remainder; + } + + if (spec->n_rpadding) { + PyUnicode_Fill(out, pos, pos + spec->n_rpadding, fill_char); + pos += spec->n_rpadding; + } + return 0; +} + +static char no_grouping[1] = {CHAR_MAX}; + +/* Find the decimal point character(s?), thousands_separator(s?), and + grouping description, either for the current locale if type is + LT_CURRENT_LOCALE, a hard-coded locale if LT_DEFAULT_LOCALE, or + none if LT_NO_LOCALE. */ +static void +get_locale_info(int type, LocaleInfo *locale_info) +{ + switch (type) { + case LT_CURRENT_LOCALE: { + struct lconv *locale_data = localeconv(); + locale_info->decimal_point = locale_data->decimal_point; + locale_info->thousands_sep = locale_data->thousands_sep; + locale_info->grouping = locale_data->grouping; + break; + } + case LT_DEFAULT_LOCALE: + locale_info->decimal_point = "."; + locale_info->thousands_sep = ","; + locale_info->grouping = "\3"; /* Group every 3 characters. The + (implicit) trailing 0 means repeat + infinitely. */ + break; + case LT_NO_LOCALE: + locale_info->decimal_point = "."; + locale_info->thousands_sep = ""; + locale_info->grouping = no_grouping; + break; + default: + assert(0); + } +} + +/************************************************************************/ +/*********** string formatting ******************************************/ +/************************************************************************/ + +static PyObject * +format_string_internal(PyObject *value, const InternalFormatSpec *format) +{ + Py_ssize_t lpad; + Py_ssize_t rpad; + Py_ssize_t total; + Py_ssize_t pos; + Py_ssize_t len = PyUnicode_GET_LENGTH(value); + PyObject *result = NULL; + Py_UCS4 maxchar = 127; + + /* sign is not allowed on strings */ + if (format->sign != '\0') { + PyErr_SetString(PyExc_ValueError, + "Sign not allowed in string format specifier"); + goto done; + } + + /* alternate is not allowed on strings */ + if (format->alternate) { + PyErr_SetString(PyExc_ValueError, + "Alternate form (#) not allowed in string format " + "specifier"); + goto done; + } + + /* '=' alignment not allowed on strings */ + if (format->align == '=') { + PyErr_SetString(PyExc_ValueError, + "'=' alignment not allowed " + "in string format specifier"); + goto done; + } + + /* if precision is specified, output no more that format.precision + characters */ + if (format->precision >= 0 && len >= format->precision) { + len = format->precision; + } + + calc_padding(len, format->width, format->align, &lpad, &rpad, &total); + + if (lpad != 0 || rpad != 0) + maxchar = Py_MAX(maxchar, format->fill_char); + + /* allocate the resulting string */ + result = PyUnicode_New(total, maxchar); + if (result == NULL) + goto done; + + /* Write into that space. First the padding. */ + pos = fill_padding(result, 0, len, + format->fill_char=='\0'?' ':format->fill_char, + lpad, rpad); + + /* Then the source string. */ + if (PyUnicode_CopyCharacters(result, pos, value, 0, len) < 0) + Py_CLEAR(result); + +done: + assert(!result || _PyUnicode_CheckConsistency(result, 1)); + return result; +} + + +/************************************************************************/ +/*********** long formatting ********************************************/ +/************************************************************************/ + +typedef PyObject* +(*IntOrLongToString)(PyObject *value, int base); + +static PyObject * +format_int_or_long_internal(PyObject *value, const InternalFormatSpec *format, + IntOrLongToString tostring) +{ + PyObject *result = NULL; + Py_UCS4 maxchar = 127; + PyObject *tmp = NULL; + Py_ssize_t inumeric_chars; + Py_UCS4 sign_char = '\0'; + Py_ssize_t n_digits; /* count of digits need from the computed + string */ + Py_ssize_t n_remainder = 0; /* Used only for 'c' formatting, which + produces non-digits */ + Py_ssize_t n_prefix = 0; /* Count of prefix chars, (e.g., '0x') */ + Py_ssize_t n_total; + Py_ssize_t prefix = 0; + NumberFieldWidths spec; + long x; + int err; + + /* Locale settings, either from the actual locale or + from a hard-code pseudo-locale */ + LocaleInfo locale; + + /* no precision allowed on integers */ + if (format->precision != -1) { + PyErr_SetString(PyExc_ValueError, + "Precision not allowed in integer format specifier"); + goto done; + } + + /* special case for character formatting */ + if (format->type == 'c') { + /* error to specify a sign */ + if (format->sign != '\0') { + PyErr_SetString(PyExc_ValueError, + "Sign not allowed with integer" + " format specifier 'c'"); + goto done; + } + + /* taken from unicodeobject.c formatchar() */ + /* Integer input truncated to a character */ + x = PyLong_AsLong(value); + if (x == -1 && PyErr_Occurred()) + goto done; + if (x < 0 || x > 0x10ffff) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x110000)"); + goto done; + } + tmp = PyUnicode_FromOrdinal(x); + inumeric_chars = 0; + n_digits = 1; + maxchar = Py_MAX(maxchar, (Py_UCS4)x); + + /* As a sort-of hack, we tell calc_number_widths that we only + have "remainder" characters. calc_number_widths thinks + these are characters that don't get formatted, only copied + into the output string. We do this for 'c' formatting, + because the characters are likely to be non-digits. */ + n_remainder = 1; + } + else { + int base; + int leading_chars_to_skip = 0; /* Number of characters added by + PyNumber_ToBase that we want to + skip over. */ + + /* Compute the base and how many characters will be added by + PyNumber_ToBase */ + switch (format->type) { + case 'b': + base = 2; + leading_chars_to_skip = 2; /* 0b */ + break; + case 'o': + base = 8; + leading_chars_to_skip = 2; /* 0o */ + break; + case 'x': + case 'X': + base = 16; + leading_chars_to_skip = 2; /* 0x */ + break; + default: /* shouldn't be needed, but stops a compiler warning */ + case 'd': + case 'n': + base = 10; + break; + } + + /* The number of prefix chars is the same as the leading + chars to skip */ + if (format->alternate) + n_prefix = leading_chars_to_skip; + + /* Do the hard part, converting to a string in a given base */ + tmp = tostring(value, base); + if (tmp == NULL || PyUnicode_READY(tmp) == -1) + goto done; + + inumeric_chars = 0; + n_digits = PyUnicode_GET_LENGTH(tmp); + + prefix = inumeric_chars; + + /* Is a sign character present in the output? If so, remember it + and skip it */ + if (PyUnicode_READ_CHAR(tmp, inumeric_chars) == '-') { + sign_char = '-'; + ++prefix; + ++leading_chars_to_skip; + } + + /* Skip over the leading chars (0x, 0b, etc.) */ + n_digits -= leading_chars_to_skip; + inumeric_chars += leading_chars_to_skip; + } + + /* Determine the grouping, separator, and decimal point, if any. */ + get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : + (format->thousands_separators ? + LT_DEFAULT_LOCALE : + LT_NO_LOCALE), + &locale); + + /* Calculate how much memory we'll need. */ + n_total = calc_number_widths(&spec, n_prefix, sign_char, tmp, inumeric_chars, + inumeric_chars + n_digits, n_remainder, 0, &locale, format); + + if (spec.n_lpadding || spec.n_spadding || spec.n_rpadding) + maxchar = Py_MAX(maxchar, format->fill_char); + + /* Allocate the memory. */ + result = PyUnicode_New(n_total, maxchar); + if (!result) + goto done; + + /* Populate the memory. */ + err = fill_number(result, 0, &spec, + tmp, inumeric_chars, inumeric_chars + n_digits, + tmp, prefix, + format->fill_char == '\0' ? ' ' : format->fill_char, + &locale, format->type == 'X'); + if (err) + Py_CLEAR(result); + +done: + Py_XDECREF(tmp); + assert(!result || _PyUnicode_CheckConsistency(result, 1)); + return result; +} + +/************************************************************************/ +/*********** float formatting *******************************************/ +/************************************************************************/ + +static PyObject* +strtounicode(char *charbuffer, Py_ssize_t len) +{ + return PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, charbuffer, len); +} + +/* much of this is taken from unicodeobject.c */ +static PyObject * +format_float_internal(PyObject *value, + const InternalFormatSpec *format) +{ + char *buf = NULL; /* buffer returned from PyOS_double_to_string */ + Py_ssize_t n_digits; + Py_ssize_t n_remainder; + Py_ssize_t n_total; + int has_decimal; + double val; + Py_ssize_t precision = format->precision; + Py_ssize_t default_precision = 6; + Py_UCS4 type = format->type; + int add_pct = 0; + Py_ssize_t index; + NumberFieldWidths spec; + int flags = 0; + PyObject *result = NULL; + Py_UCS4 maxchar = 127; + Py_UCS4 sign_char = '\0'; + int float_type; /* Used to see if we have a nan, inf, or regular float. */ + PyObject *unicode_tmp = NULL; + int err; + + /* Locale settings, either from the actual locale or + from a hard-code pseudo-locale */ + LocaleInfo locale; + + if (format->alternate) + flags |= Py_DTSF_ALT; + + if (type == '\0') { + /* Omitted type specifier. Behaves in the same way as repr(x) + and str(x) if no precision is given, else like 'g', but with + at least one digit after the decimal point. */ + flags |= Py_DTSF_ADD_DOT_0; + type = 'r'; + default_precision = 0; + } + + if (type == 'n') + /* 'n' is the same as 'g', except for the locale used to + format the result. We take care of that later. */ + type = 'g'; + + val = PyFloat_AsDouble(value); + if (val == -1.0 && PyErr_Occurred()) + goto done; + + if (type == '%') { + type = 'f'; + val *= 100; + add_pct = 1; + } + + if (precision < 0) + precision = default_precision; + else if (type == 'r') + type = 'g'; + + /* Cast "type", because if we're in unicode we need to pass a + 8-bit char. This is safe, because we've restricted what "type" + can be. */ + buf = PyOS_double_to_string(val, (char)type, precision, flags, + &float_type); + if (buf == NULL) + goto done; + n_digits = strlen(buf); + + if (add_pct) { + /* We know that buf has a trailing zero (since we just called + strlen() on it), and we don't use that fact any more. So we + can just write over the trailing zero. */ + buf[n_digits] = '%'; + n_digits += 1; + } + + /* Since there is no unicode version of PyOS_double_to_string, + just use the 8 bit version and then convert to unicode. */ + unicode_tmp = strtounicode(buf, n_digits); + if (unicode_tmp == NULL) + goto done; + index = 0; + + /* Is a sign character present in the output? If so, remember it + and skip it */ + if (PyUnicode_READ_CHAR(unicode_tmp, index) == '-') { + sign_char = '-'; + ++index; + --n_digits; + } + + /* Determine if we have any "remainder" (after the digits, might include + decimal or exponent or both (or neither)) */ + parse_number(unicode_tmp, index, index + n_digits, &n_remainder, &has_decimal); + + /* Determine the grouping, separator, and decimal point, if any. */ + get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : + (format->thousands_separators ? + LT_DEFAULT_LOCALE : + LT_NO_LOCALE), + &locale); + + /* Calculate how much memory we'll need. */ + n_total = calc_number_widths(&spec, 0, sign_char, unicode_tmp, index, + index + n_digits, n_remainder, has_decimal, + &locale, format); + + if (spec.n_lpadding || spec.n_spadding || spec.n_rpadding) + maxchar = Py_MAX(maxchar, format->fill_char); + + /* Allocate the memory. */ + result = PyUnicode_New(n_total, maxchar); + if (result == NULL) + goto done; + + /* Populate the memory. */ + err = fill_number(result, 0, &spec, + unicode_tmp, index, index + n_digits, + NULL, 0, + format->fill_char == '\0' ? ' ' : format->fill_char, + &locale, 0); + if (err) + Py_CLEAR(result); + +done: + PyMem_Free(buf); + Py_DECREF(unicode_tmp); + assert(!result || _PyUnicode_CheckConsistency(result, 1)); + return result; +} + +/************************************************************************/ +/*********** complex formatting *****************************************/ +/************************************************************************/ + +static PyObject * +format_complex_internal(PyObject *value, + const InternalFormatSpec *format) +{ + double re; + double im; + char *re_buf = NULL; /* buffer returned from PyOS_double_to_string */ + char *im_buf = NULL; /* buffer returned from PyOS_double_to_string */ + + InternalFormatSpec tmp_format = *format; + Py_ssize_t n_re_digits; + Py_ssize_t n_im_digits; + Py_ssize_t n_re_remainder; + Py_ssize_t n_im_remainder; + Py_ssize_t n_re_total; + Py_ssize_t n_im_total; + int re_has_decimal; + int im_has_decimal; + Py_ssize_t precision = format->precision; + Py_ssize_t default_precision = 6; + Py_UCS4 type = format->type; + Py_ssize_t i_re; + Py_ssize_t i_im; + NumberFieldWidths re_spec; + NumberFieldWidths im_spec; + int flags = 0; + PyObject *result = NULL; + Py_UCS4 maxchar = 127; + int rkind; + void *rdata; + Py_ssize_t index; + Py_UCS4 re_sign_char = '\0'; + Py_UCS4 im_sign_char = '\0'; + int re_float_type; /* Used to see if we have a nan, inf, or regular float. */ + int im_float_type; + int add_parens = 0; + int skip_re = 0; + Py_ssize_t lpad; + Py_ssize_t rpad; + Py_ssize_t total; + PyObject *re_unicode_tmp = NULL; + PyObject *im_unicode_tmp = NULL; + int err; + + /* Locale settings, either from the actual locale or + from a hard-code pseudo-locale */ + LocaleInfo locale; + + /* Zero padding is not allowed. */ + if (format->fill_char == '0') { + PyErr_SetString(PyExc_ValueError, + "Zero padding is not allowed in complex format " + "specifier"); + goto done; + } + + /* Neither is '=' alignment . */ + if (format->align == '=') { + PyErr_SetString(PyExc_ValueError, + "'=' alignment flag is not allowed in complex format " + "specifier"); + goto done; + } + + re = PyComplex_RealAsDouble(value); + if (re == -1.0 && PyErr_Occurred()) + goto done; + im = PyComplex_ImagAsDouble(value); + if (im == -1.0 && PyErr_Occurred()) + goto done; + + if (format->alternate) + flags |= Py_DTSF_ALT; + + if (type == '\0') { + /* Omitted type specifier. Should be like str(self). */ + type = 'r'; + default_precision = 0; + if (re == 0.0 && copysign(1.0, re) == 1.0) + skip_re = 1; + else + add_parens = 1; + } + + if (type == 'n') + /* 'n' is the same as 'g', except for the locale used to + format the result. We take care of that later. */ + type = 'g'; + + if (precision < 0) + precision = default_precision; + else if (type == 'r') + type = 'g'; + + /* Cast "type", because if we're in unicode we need to pass a + 8-bit char. This is safe, because we've restricted what "type" + can be. */ + re_buf = PyOS_double_to_string(re, (char)type, precision, flags, + &re_float_type); + if (re_buf == NULL) + goto done; + im_buf = PyOS_double_to_string(im, (char)type, precision, flags, + &im_float_type); + if (im_buf == NULL) + goto done; + + n_re_digits = strlen(re_buf); + n_im_digits = strlen(im_buf); + + /* Since there is no unicode version of PyOS_double_to_string, + just use the 8 bit version and then convert to unicode. */ + re_unicode_tmp = strtounicode(re_buf, n_re_digits); + if (re_unicode_tmp == NULL) + goto done; + i_re = 0; + + im_unicode_tmp = strtounicode(im_buf, n_im_digits); + if (im_unicode_tmp == NULL) + goto done; + i_im = 0; + + /* Is a sign character present in the output? If so, remember it + and skip it */ + if (PyUnicode_READ_CHAR(re_unicode_tmp, i_re) == '-') { + re_sign_char = '-'; + ++i_re; + --n_re_digits; + } + if (PyUnicode_READ_CHAR(im_unicode_tmp, i_im) == '-') { + im_sign_char = '-'; + ++i_im; + --n_im_digits; + } + + /* Determine if we have any "remainder" (after the digits, might include + decimal or exponent or both (or neither)) */ + parse_number(re_unicode_tmp, i_re, i_re + n_re_digits, + &n_re_remainder, &re_has_decimal); + parse_number(im_unicode_tmp, i_im, i_im + n_im_digits, + &n_im_remainder, &im_has_decimal); + + /* Determine the grouping, separator, and decimal point, if any. */ + get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : + (format->thousands_separators ? + LT_DEFAULT_LOCALE : + LT_NO_LOCALE), + &locale); + + /* Turn off any padding. We'll do it later after we've composed + the numbers without padding. */ + tmp_format.fill_char = '\0'; + tmp_format.align = '<'; + tmp_format.width = -1; + + /* Calculate how much memory we'll need. */ + n_re_total = calc_number_widths(&re_spec, 0, re_sign_char, re_unicode_tmp, + i_re, i_re + n_re_digits, n_re_remainder, + re_has_decimal, &locale, &tmp_format); + + /* Same formatting, but always include a sign, unless the real part is + * going to be omitted, in which case we use whatever sign convention was + * requested by the original format. */ + if (!skip_re) + tmp_format.sign = '+'; + n_im_total = calc_number_widths(&im_spec, 0, im_sign_char, im_unicode_tmp, + i_im, i_im + n_im_digits, n_im_remainder, + im_has_decimal, &locale, &tmp_format); + + if (skip_re) + n_re_total = 0; + + /* Add 1 for the 'j', and optionally 2 for parens. */ + calc_padding(n_re_total + n_im_total + 1 + add_parens * 2, + format->width, format->align, &lpad, &rpad, &total); + + if (re_spec.n_lpadding || re_spec.n_spadding || re_spec.n_rpadding + || im_spec.n_lpadding || im_spec.n_spadding || im_spec.n_rpadding + || lpad || rpad) + maxchar = Py_MAX(maxchar, format->fill_char); + + result = PyUnicode_New(total, maxchar); + if (result == NULL) + goto done; + rkind = PyUnicode_KIND(result); + rdata = PyUnicode_DATA(result); + + /* Populate the memory. First, the padding. */ + index = fill_padding(result, 0, + n_re_total + n_im_total + 1 + add_parens * 2, + format->fill_char=='\0' ? ' ' : format->fill_char, + lpad, rpad); + + if (add_parens) + PyUnicode_WRITE(rkind, rdata, index++, '('); + + if (!skip_re) { + err = fill_number(result, index, &re_spec, + re_unicode_tmp, i_re, i_re + n_re_digits, + NULL, 0, + 0, + &locale, 0); + if (err) { + Py_CLEAR(result); + goto done; + } + index += n_re_total; + } + err = fill_number(result, index, &im_spec, + im_unicode_tmp, i_im, i_im + n_im_digits, + NULL, 0, + 0, + &locale, 0); + if (err) { + Py_CLEAR(result); + goto done; + } + index += n_im_total; + PyUnicode_WRITE(rkind, rdata, index++, 'j'); + + if (add_parens) + PyUnicode_WRITE(rkind, rdata, index++, ')'); + +done: + PyMem_Free(re_buf); + PyMem_Free(im_buf); + Py_XDECREF(re_unicode_tmp); + Py_XDECREF(im_unicode_tmp); + assert(!result || _PyUnicode_CheckConsistency(result, 1)); + return result; +} + +/************************************************************************/ +/*********** built in formatters ****************************************/ +/************************************************************************/ +PyObject * +_PyUnicode_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) +{ + InternalFormatSpec format; + PyObject *result; + + /* check for the special case of zero length format spec, make + it equivalent to str(obj) */ + if (start == end) + return PyObject_Str(obj); + + /* parse the format_spec */ + if (!parse_internal_render_format_spec(format_spec, start, end, + &format, 's', '<')) + return NULL; + + /* type conversion? */ + switch (format.type) { + case 's': + /* no type conversion needed, already a string. do the formatting */ + result = format_string_internal(obj, &format); + if (result != NULL) + assert(_PyUnicode_CheckConsistency(result, 1)); + break; + default: + /* unknown */ + unknown_presentation_type(format.type, obj->ob_type->tp_name); + result = NULL; + } + return result; +} + +static PyObject* +format_int_or_long(PyObject* obj, PyObject* format_spec, + Py_ssize_t start, Py_ssize_t end, + IntOrLongToString tostring) +{ + PyObject *result = NULL; + PyObject *tmp = NULL; + InternalFormatSpec format; + + /* check for the special case of zero length format spec, make + it equivalent to str(obj) */ + if (start == end) { + result = PyObject_Str(obj); + goto done; + } + + /* parse the format_spec */ + if (!parse_internal_render_format_spec(format_spec, start, end, + &format, 'd', '>')) + goto done; + + /* type conversion? */ + switch (format.type) { + case 'b': + case 'c': + case 'd': + case 'o': + case 'x': + case 'X': + case 'n': + /* no type conversion needed, already an int (or long). do + the formatting */ + result = format_int_or_long_internal(obj, &format, tostring); + break; + + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + case '%': + /* convert to float */ + tmp = PyNumber_Float(obj); + if (tmp == NULL) + goto done; + result = format_float_internal(tmp, &format); + break; + + default: + /* unknown */ + unknown_presentation_type(format.type, obj->ob_type->tp_name); + goto done; + } + +done: + Py_XDECREF(tmp); + return result; +} + +/* Need to define long_format as a function that will convert a long + to a string. In 3.0, _PyLong_Format has the correct signature. */ +#define long_format _PyLong_Format + +PyObject * +_PyLong_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) +{ + return format_int_or_long(obj, format_spec, start, end, + long_format); +} + +PyObject * +_PyFloat_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) +{ + PyObject *result = NULL; + InternalFormatSpec format; + + /* check for the special case of zero length format spec, make + it equivalent to str(obj) */ + if (start == end) { + result = PyObject_Str(obj); + goto done; + } + + /* parse the format_spec */ + if (!parse_internal_render_format_spec(format_spec, start, end, + &format, '\0', '>')) + goto done; + + /* type conversion? */ + switch (format.type) { + case '\0': /* No format code: like 'g', but with at least one decimal. */ + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + case 'n': + case '%': + /* no conversion, already a float. do the formatting */ + result = format_float_internal(obj, &format); + break; + + default: + /* unknown */ + unknown_presentation_type(format.type, obj->ob_type->tp_name); + goto done; + } + +done: + return result; +} + +PyObject * +_PyComplex_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) +{ + PyObject *result = NULL; + InternalFormatSpec format; + + /* check for the special case of zero length format spec, make + it equivalent to str(obj) */ + if (start == end) { + result = PyObject_Str(obj); + goto done; + } + + /* parse the format_spec */ + if (!parse_internal_render_format_spec(format_spec, start, end, + &format, '\0', '>')) + goto done; + + /* type conversion? */ + switch (format.type) { + case '\0': /* No format code: like 'g', but with at least one decimal. */ + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + case 'n': + /* no conversion, already a complex. do the formatting */ + result = format_complex_internal(obj, &format); + break; + + default: + /* unknown */ + unknown_presentation_type(format.type, obj->ob_type->tp_name); + goto done; + } + +done: + return result; +} diff --git a/Python/getargs.c b/Python/getargs.c index 600941d..77f27be 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -310,20 +310,18 @@ vgetargs1(PyObject *args, const char *format, va_list *p_va, int flags) if (max == 0) { if (args == NULL) return 1; - PyOS_snprintf(msgbuf, sizeof(msgbuf), - "%.200s%s takes no arguments", - fname==NULL ? "function" : fname, - fname==NULL ? "" : "()"); - PyErr_SetString(PyExc_TypeError, msgbuf); + PyErr_Format(PyExc_TypeError, + "%.200s%s takes no arguments", + fname==NULL ? "function" : fname, + fname==NULL ? "" : "()"); return 0; } else if (min == 1 && max == 1) { if (args == NULL) { - PyOS_snprintf(msgbuf, sizeof(msgbuf), - "%.200s%s takes at least one argument", - fname==NULL ? "function" : fname, - fname==NULL ? "" : "()"); - PyErr_SetString(PyExc_TypeError, msgbuf); + PyErr_Format(PyExc_TypeError, + "%.200s%s takes at least one argument", + fname==NULL ? "function" : fname, + fname==NULL ? "" : "()"); return 0; } msg = convertitem(args, &format, p_va, flags, levels, @@ -349,20 +347,18 @@ vgetargs1(PyObject *args, const char *format, va_list *p_va, int flags) len = PyTuple_GET_SIZE(args); if (len < min || max < len) { - if (message == NULL) { - PyOS_snprintf(msgbuf, sizeof(msgbuf), - "%.150s%s takes %s %d argument%s " - "(%ld given)", - fname==NULL ? "function" : fname, - fname==NULL ? "" : "()", - min==max ? "exactly" - : len < min ? "at least" : "at most", - len < min ? min : max, - (len < min ? min : max) == 1 ? "" : "s", - Py_SAFE_DOWNCAST(len, Py_ssize_t, long)); - message = msgbuf; - } - PyErr_SetString(PyExc_TypeError, message); + if (message == NULL) + PyErr_Format(PyExc_TypeError, + "%.150s%s takes %s %d argument%s (%ld given)", + fname==NULL ? "function" : fname, + fname==NULL ? "" : "()", + min==max ? "exactly" + : len < min ? "at least" : "at most", + len < min ? min : max, + (len < min ? min : max) == 1 ? "" : "s", + Py_SAFE_DOWNCAST(len, Py_ssize_t, long)); + else + PyErr_SetString(PyExc_TypeError, message); return 0; } @@ -550,9 +546,6 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, -#define UNICODE_DEFAULT_ENCODING(arg) \ - _PyUnicode_AsDefaultEncodedString(arg, NULL) - /* Format an error message generated by convertsimple(). */ static char * @@ -615,7 +608,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; char c = *format++; - PyObject *uarg; + char *sarg; switch (c) { @@ -832,6 +825,8 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *p = va_arg(*p_va, char *); if (PyBytes_Check(arg) && PyBytes_Size(arg) == 1) *p = PyBytes_AS_STRING(arg)[0]; + else if (PyByteArray_Check(arg) && PyByteArray_Size(arg) == 1) + *p = PyByteArray_AS_STRING(arg)[0]; else return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; @@ -839,11 +834,21 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'C': {/* unicode char */ int *p = va_arg(*p_va, int *); - if (PyUnicode_Check(arg) && - PyUnicode_GET_SIZE(arg) == 1) - *p = PyUnicode_AS_UNICODE(arg)[0]; - else + int kind; + void *data; + + if (!PyUnicode_Check(arg)) return converterr("a unicode character", arg, msgbuf, bufsize); + + if (PyUnicode_READY(arg)) + RETURN_ERR_OCCURRED; + + if (PyUnicode_GET_LENGTH(arg) != 1) + return converterr("a unicode character", arg, msgbuf, bufsize); + + kind = PyUnicode_KIND(arg); + data = PyUnicode_DATA(arg); + *p = PyUnicode_READ(kind, data, 0); break; } @@ -891,13 +896,12 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (c == 'z' && arg == Py_None) PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); else if (PyUnicode_Check(arg)) { - uarg = UNICODE_DEFAULT_ENCODING(arg); - if (uarg == NULL) + Py_ssize_t len; + sarg = PyUnicode_AsUTF8AndSize(arg, &len); + if (sarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); - PyBuffer_FillInfo(p, arg, - PyBytes_AS_STRING(uarg), PyBytes_GET_SIZE(uarg), - 1, 0); + PyBuffer_FillInfo(p, arg, sarg, len, 1, 0); } else { /* any buffer-like object */ char *buf; @@ -920,12 +924,13 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, STORE_SIZE(0); } else if (PyUnicode_Check(arg)) { - uarg = UNICODE_DEFAULT_ENCODING(arg); - if (uarg == NULL) + Py_ssize_t len; + sarg = PyUnicode_AsUTF8AndSize(arg, &len); + if (sarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); - *p = PyBytes_AS_STRING(uarg); - STORE_SIZE(PyBytes_GET_SIZE(uarg)); + *p = sarg; + STORE_SIZE(len); } else { /* any buffer-like object */ /* XXX Really? */ @@ -939,25 +944,25 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else { /* "s" or "z" */ char **p = va_arg(*p_va, char **); - uarg = NULL; + Py_ssize_t len; + sarg = NULL; if (c == 'z' && arg == Py_None) *p = NULL; else if (PyUnicode_Check(arg)) { - uarg = UNICODE_DEFAULT_ENCODING(arg); - if (uarg == NULL) + sarg = PyUnicode_AsUTF8AndSize(arg, &len); + if (sarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); - *p = PyBytes_AS_STRING(uarg); + *p = sarg; } else return converterr(c == 'z' ? "str or None" : "str", arg, msgbuf, bufsize); - if (*p != NULL && uarg != NULL && - (Py_ssize_t) strlen(*p) != PyBytes_GET_SIZE(uarg)) + if (*p != NULL && sarg != NULL && (Py_ssize_t) strlen(*p) != len) return converterr( - c == 'z' ? "str without null bytes or None" - : "str without null bytes", + c == 'z' ? "str without null characters or None" + : "str without null characters", arg, msgbuf, bufsize); } break; @@ -966,9 +971,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'u': /* raw unicode buffer (Py_UNICODE *) */ case 'Z': /* raw unicode buffer or None */ { + Py_UNICODE **p = va_arg(*p_va, Py_UNICODE **); + if (*format == '#') { /* any buffer-like object */ /* "s#" or "Z#" */ - Py_UNICODE **p = va_arg(*p_va, Py_UNICODE **); FETCH_SIZE; if (c == 'Z' && arg == Py_None) { @@ -976,23 +982,27 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, STORE_SIZE(0); } else if (PyUnicode_Check(arg)) { - *p = PyUnicode_AS_UNICODE(arg); - STORE_SIZE(PyUnicode_GET_SIZE(arg)); + Py_ssize_t len; + *p = PyUnicode_AsUnicodeAndSize(arg, &len); + if (*p == NULL) + RETURN_ERR_OCCURRED; + STORE_SIZE(len); } else return converterr("str or None", arg, msgbuf, bufsize); format++; } else { /* "s" or "Z" */ - Py_UNICODE **p = va_arg(*p_va, Py_UNICODE **); - if (c == 'Z' && arg == Py_None) *p = NULL; else if (PyUnicode_Check(arg)) { - *p = PyUnicode_AS_UNICODE(arg); - if (Py_UNICODE_strlen(*p) != PyUnicode_GET_SIZE(arg)) + Py_ssize_t len; + *p = PyUnicode_AsUnicodeAndSize(arg, &len); + if (*p == NULL) + RETURN_ERR_OCCURRED; + if (Py_UNICODE_strlen(*p) != len) return converterr( - "str without null character or None", + "str without null characters or None", arg, msgbuf, bufsize); } else return converterr(c == 'Z' ? "str or None" : "str", @@ -1459,8 +1469,8 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, nargs = PyTuple_GET_SIZE(args); nkeywords = (keywords == NULL) ? 0 : PyDict_Size(keywords); if (nargs + nkeywords > len) { - PyErr_Format(PyExc_TypeError, "%s%s takes at most %d " - "argument%s (%d given)", + PyErr_Format(PyExc_TypeError, + "%s%s takes at most %d argument%s (%d given)", (fname == NULL) ? "function" : fname, (fname == NULL) ? "" : "()", len, diff --git a/Python/graminit.c b/Python/graminit.c index cabc4d6..e04999b 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -1791,7 +1791,7 @@ static arc arcs_80_0[1] = { {167, 1}, }; static arc arcs_80_1[2] = { - {9, 2}, + {168, 2}, {0, 1}, }; static arc arcs_80_2[1] = { @@ -1802,171 +1802,188 @@ static state states_80[3] = { {2, arcs_80_1}, {1, arcs_80_2}, }; -static dfa dfas[81] = { +static arc arcs_81_0[2] = { + {73, 1}, + {9, 2}, +}; +static arc arcs_81_1[1] = { + {24, 2}, +}; +static arc arcs_81_2[1] = { + {0, 2}, +}; +static state states_81[3] = { + {2, arcs_81_0}, + {1, arcs_81_1}, + {1, arcs_81_2}, +}; +static dfa dfas[82] = { {256, "single_input", 0, 3, states_0, - "\004\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204"}, + "\004\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204\000"}, {257, "file_input", 0, 2, states_1, - "\204\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204"}, + "\204\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204\000"}, {258, "eval_input", 0, 3, states_2, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {259, "decorator", 0, 7, states_3, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {260, "decorators", 0, 2, states_4, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {261, "decorated", 0, 3, states_5, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {262, "funcdef", 0, 8, states_6, - "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {263, "parameters", 0, 4, states_7, - "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {264, "typedargslist", 0, 18, states_8, - "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {265, "tfpdef", 0, 4, states_9, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {266, "varargslist", 0, 18, states_10, - "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {267, "vfpdef", 0, 2, states_11, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {268, "stmt", 0, 2, states_12, - "\000\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204"}, + "\000\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204\000"}, {269, "simple_stmt", 0, 4, states_13, - "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200"}, + "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200\000"}, {270, "small_stmt", 0, 2, states_14, - "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200"}, + "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200\000"}, {271, "expr_stmt", 0, 6, states_15, - "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {272, "testlist_star_expr", 0, 3, states_16, - "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {273, "augassign", 0, 2, states_17, - "\000\000\000\000\000\000\376\037\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\376\037\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {274, "del_stmt", 0, 3, states_18, - "\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {275, "pass_stmt", 0, 2, states_19, - "\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {276, "flow_stmt", 0, 2, states_20, - "\000\000\000\000\000\000\000\000\340\001\000\000\000\000\000\000\000\000\000\000\200"}, + "\000\000\000\000\000\000\000\000\340\001\000\000\000\000\000\000\000\000\000\000\200\000"}, {277, "break_stmt", 0, 2, states_21, - "\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {278, "continue_stmt", 0, 2, states_22, - "\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {279, "return_stmt", 0, 3, states_23, - "\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {280, "yield_stmt", 0, 2, states_24, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000"}, {281, "raise_stmt", 0, 5, states_25, - "\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000"}, {282, "import_stmt", 0, 2, states_26, - "\000\000\000\000\000\000\000\000\000\022\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\022\000\000\000\000\000\000\000\000\000\000\000\000"}, {283, "import_name", 0, 3, states_27, - "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000"}, {284, "import_from", 0, 8, states_28, - "\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000"}, {285, "import_as_name", 0, 4, states_29, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {286, "dotted_as_name", 0, 4, states_30, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {287, "import_as_names", 0, 3, states_31, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {288, "dotted_as_names", 0, 2, states_32, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {289, "dotted_name", 0, 2, states_33, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {290, "global_stmt", 0, 3, states_34, - "\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000"}, {291, "nonlocal_stmt", 0, 3, states_35, - "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000"}, {292, "assert_stmt", 0, 5, states_36, - "\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000"}, {293, "compound_stmt", 0, 2, states_37, - "\000\010\020\000\000\000\000\000\000\000\000\220\045\000\000\000\000\000\000\000\004"}, + "\000\010\020\000\000\000\000\000\000\000\000\220\045\000\000\000\000\000\000\000\004\000"}, {294, "if_stmt", 0, 8, states_38, - "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, {295, "while_stmt", 0, 8, states_39, - "\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000"}, {296, "for_stmt", 0, 10, states_40, - "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, {297, "try_stmt", 0, 13, states_41, - "\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000"}, {298, "with_stmt", 0, 5, states_42, - "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"}, {299, "with_item", 0, 4, states_43, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {300, "except_clause", 0, 5, states_44, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000"}, {301, "suite", 0, 5, states_45, - "\004\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200"}, + "\004\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200\000"}, {302, "test", 0, 6, states_46, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {303, "test_nocond", 0, 2, states_47, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {304, "lambdef", 0, 5, states_48, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000"}, {305, "lambdef_nocond", 0, 5, states_49, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000"}, {306, "or_test", 0, 2, states_50, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000\000"}, {307, "and_test", 0, 2, states_51, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000\000"}, {308, "not_test", 0, 3, states_52, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000\000"}, {309, "comparison", 0, 2, states_53, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {310, "comp_op", 0, 4, states_54, - "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\220\177\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\220\177\000\000\000\000\000\000"}, {311, "star_expr", 0, 3, states_55, - "\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {312, "expr", 0, 2, states_56, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {313, "xor_expr", 0, 2, states_57, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {314, "and_expr", 0, 2, states_58, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {315, "shift_expr", 0, 2, states_59, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {316, "arith_expr", 0, 2, states_60, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {317, "term", 0, 2, states_61, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {318, "factor", 0, 3, states_62, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {319, "power", 0, 4, states_63, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000\000"}, {320, "atom", 0, 9, states_64, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000\000"}, {321, "testlist_comp", 0, 5, states_65, - "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {322, "trailer", 0, 7, states_66, - "\000\040\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\020\000\000"}, + "\000\040\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\020\000\000\000"}, {323, "subscriptlist", 0, 3, states_67, - "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {324, "subscript", 0, 5, states_68, - "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {325, "sliceop", 0, 3, states_69, - "\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {326, "exprlist", 0, 3, states_70, - "\000\040\040\200\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {327, "testlist", 0, 3, states_71, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {328, "dictorsetmaker", 0, 11, states_72, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {329, "classdef", 0, 8, states_73, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000"}, {330, "arglist", 0, 8, states_74, - "\000\040\040\200\001\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\200\001\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {331, "argument", 0, 4, states_75, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {332, "comp_iter", 0, 2, states_76, - "\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000\000"}, {333, "comp_for", 0, 6, states_77, - "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, {334, "comp_if", 0, 4, states_78, - "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, {335, "encoding_decl", 0, 2, states_79, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {336, "yield_expr", 0, 3, states_80, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000"}, + {337, "yield_arg", 0, 3, states_81, + "\000\040\040\000\000\000\000\000\000\202\000\000\000\200\020\000\000\206\120\076\000\000"}, }; -static label labels[168] = { +static label labels[169] = { {0, "EMPTY"}, {256, 0}, {4, 0}, @@ -2135,10 +2152,11 @@ static label labels[168] = { {334, 0}, {335, 0}, {1, "yield"}, + {337, 0}, }; grammar _PyParser_Grammar = { - 81, + 82, dfas, - {168, labels}, + {169, labels}, 256 }; diff --git a/Python/import.c b/Python/import.c index beda7f97..8bd7a61 100644 --- a/Python/import.c +++ b/Python/import.c @@ -102,6 +102,9 @@ typedef unsigned short mode_t; Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) tag: cpython-32 Python 3.2a2 3180 (add DELETE_DEREF) + Python 3.3a0 3190 __class__ super closure changed + Python 3.3a0 3200 (__qualname__ added) + 3210 (added size modulo 2**32 to the pyc header) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -110,22 +113,30 @@ typedef unsigned short mode_t; TAG must change for each major Python release. The magic number will take care of any bytecode changes that occur during development. */ -#define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) -#define TAG "cpython-32" +#define QUOTE(arg) #arg +#define STRIFY(name) QUOTE(name) +#define MAJOR STRIFY(PY_MAJOR_VERSION) +#define MINOR STRIFY(PY_MINOR_VERSION) +#define MAGIC (3210 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; +#undef QUOTE +#undef STRIFY +#undef MAJOR +#undef MINOR -/* See _PyImport_FixupExtensionUnicode() below */ +/* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; +/* Function from Parser/tokenizer.c */ +extern char * PyTokenizer_FindEncodingFilename(int, PyObject *); + /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; -/* Method from Parser/tokenizer.c */ -extern char * PyTokenizer_FindEncoding(int); - struct _inittab *PyImport_Inittab = _PyImport_Inittab; /* these tables define the module suffixes that Python recognizes */ @@ -140,6 +151,8 @@ static const struct filedescr _PyImport_StandardFiletab[] = { {0, 0} }; +static PyObject *initstr = NULL; +_Py_IDENTIFIER(__path__); /* Initialize things */ @@ -151,6 +164,10 @@ _PyImport_Init(void) int countD = 0; int countS = 0; + initstr = PyUnicode_InternFromString("__init__"); + if (initstr == NULL) + Py_FatalError("Can't initialize import variables"); + /* prepare _PyImport_Filetab: copy entries from _PyImport_DynLoadFiletab and _PyImport_StandardFiletab. */ @@ -229,8 +246,9 @@ _PyImportHooks_Init(void) PySys_WriteStderr("# can't import zipimport\n"); } else { - PyObject *zipimporter = PyObject_GetAttrString(zimpimport, - "zipimporter"); + _Py_IDENTIFIER(zipimporter); + PyObject *zipimporter = _PyObject_GetAttrId(zimpimport, + &PyId_zipimporter); Py_DECREF(zimpimport); if (zipimporter == NULL) { PyErr_Clear(); /* No zipimporter object -- okay */ @@ -319,7 +337,7 @@ _PyImport_ReInitLock(void) /* Forked as a side effect of import */ long me = PyThread_get_thread_ident(); PyThread_acquire_lock(import_lock, 0); - /* XXX: can the previous line fail? */ + /* XXX: can the previous line fail? */ import_lock_thread = me; import_lock_level--; } else { @@ -423,7 +441,6 @@ void PyImport_Cleanup(void) { Py_ssize_t pos, ndone; - char *name; PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; PyObject *modules = interp->modules; @@ -496,14 +513,13 @@ PyImport_Cleanup(void) if (value->ob_refcnt != 1) continue; if (PyUnicode_Check(key) && PyModule_Check(value)) { - name = _PyUnicode_AsString(key); - if (strcmp(name, "builtins") == 0) + if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) continue; - if (strcmp(name, "sys") == 0) + if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) continue; if (Py_VerboseFlag) - PySys_WriteStderr( - "# cleanup[1] %s\n", name); + PySys_FormatStderr( + "# cleanup[1] %U\n", key); _PyModule_Clear(value); PyDict_SetItem(modules, key, Py_None); ndone++; @@ -515,13 +531,12 @@ PyImport_Cleanup(void) pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyUnicode_Check(key) && PyModule_Check(value)) { - name = _PyUnicode_AsString(key); - if (strcmp(name, "builtins") == 0) + if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) continue; - if (strcmp(name, "sys") == 0) + if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) continue; if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup[2] %s\n", name); + PySys_FormatStderr("# cleanup[2] %U\n", key); _PyModule_Clear(value); PyDict_SetItem(modules, key, Py_None); } @@ -571,10 +586,10 @@ PyImport_GetMagicTag(void) once, we keep a static dictionary 'extensions' keyed by module name (for built-in modules) or by filename (for dynamically loaded modules), containing these modules. A copy of the module's - dictionary is stored by calling _PyImport_FixupExtensionUnicode() + dictionary is stored by calling _PyImport_FixupExtensionObject() immediately after the module initialization function succeeds. A copy can be retrieved from there by calling - _PyImport_FindExtensionUnicode(). + _PyImport_FindExtensionObject(). Modules which do support multiple initialization set their m_size field to a non-negative number (indicating the size of the @@ -583,7 +598,8 @@ PyImport_GetMagicTag(void) */ int -_PyImport_FixupExtensionUnicode(PyObject *mod, char *name, PyObject *filename) +_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, + PyObject *filename) { PyObject *modules, *dict; struct PyModuleDef *def; @@ -602,10 +618,10 @@ _PyImport_FixupExtensionUnicode(PyObject *mod, char *name, PyObject *filename) return -1; } modules = PyImport_GetModuleDict(); - if (PyDict_SetItemString(modules, name, mod) < 0) + if (PyDict_SetItem(modules, name, mod) < 0) return -1; if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItemString(modules, name); + PyDict_DelItem(modules, name); return -1; } if (def->m_size == -1) { @@ -631,17 +647,17 @@ int _PyImport_FixupBuiltin(PyObject *mod, char *name) { int res; - PyObject *filename; - filename = PyUnicode_FromString(name); - if (filename == NULL) + PyObject *nameobj; + nameobj = PyUnicode_InternFromString(name); + if (nameobj == NULL) return -1; - res = _PyImport_FixupExtensionUnicode(mod, name, filename); - Py_DECREF(filename); + res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); + Py_DECREF(nameobj); return res; } PyObject * -_PyImport_FindExtensionUnicode(char *name, PyObject *filename) +_PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { PyObject *mod, *mdict; PyModuleDef* def; @@ -654,7 +670,7 @@ _PyImport_FindExtensionUnicode(char *name, PyObject *filename) /* Module does not support repeated initialization */ if (def->m_base.m_copy == NULL) return NULL; - mod = PyImport_AddModule(name); + mod = PyImport_AddModuleObject(name); if (mod == NULL) return NULL; mdict = PyModule_GetDict(mod); @@ -669,30 +685,30 @@ _PyImport_FindExtensionUnicode(char *name, PyObject *filename) mod = def->m_base.m_init(); if (mod == NULL) return NULL; - PyDict_SetItemString(PyImport_GetModuleDict(), name, mod); + PyDict_SetItem(PyImport_GetModuleDict(), name, mod); Py_DECREF(mod); } if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItemString(PyImport_GetModuleDict(), name); + PyDict_DelItem(PyImport_GetModuleDict(), name); Py_DECREF(mod); return NULL; } if (Py_VerboseFlag) - PySys_FormatStderr("import %s # previously loaded (%U)\n", + PySys_FormatStderr("import %U # previously loaded (%R)\n", name, filename); return mod; } PyObject * -_PyImport_FindBuiltin(char *name) +_PyImport_FindBuiltin(const char *name) { - PyObject *res, *filename; - filename = PyUnicode_FromString(name); - if (filename == NULL) + PyObject *res, *nameobj; + nameobj = PyUnicode_InternFromString(name); + if (nameobj == NULL) return NULL; - res = _PyImport_FindExtensionUnicode(name, filename); - Py_DECREF(filename); + res = _PyImport_FindExtensionObject(nameobj, nameobj); + Py_DECREF(nameobj); return res; } @@ -703,18 +719,18 @@ _PyImport_FindBuiltin(char *name) 'NEW' REFERENCE! */ PyObject * -PyImport_AddModule(const char *name) +PyImport_AddModuleObject(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m; - if ((m = PyDict_GetItemString(modules, name)) != NULL && + if ((m = PyDict_GetItem(modules, name)) != NULL && PyModule_Check(m)) return m; - m = PyModule_New(name); + m = PyModule_NewObject(name); if (m == NULL) return NULL; - if (PyDict_SetItemString(modules, name, m) != 0) { + if (PyDict_SetItem(modules, name, m) != 0) { Py_DECREF(m); return NULL; } @@ -723,22 +739,34 @@ PyImport_AddModule(const char *name) return m; } +PyObject * +PyImport_AddModule(const char *name) +{ + PyObject *nameobj, *module; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + module = PyImport_AddModuleObject(nameobj); + Py_DECREF(nameobj); + return module; +} + + /* Remove name from sys.modules, if it's there. */ static void -remove_module(const char *name) +remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_GetItemString(modules, name) == NULL) + if (PyDict_GetItem(modules, name) == NULL) return; - if (PyDict_DelItemString(modules, name) < 0) + if (PyDict_DelItem(modules, name) < 0) Py_FatalError("import: deleting existing key in" "sys.modules failed"); } -static PyObject * get_sourcefile(char *file); -static char *make_source_pathname(char *pathname, char *buf); -static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen, - int debug); +static PyObject * get_sourcefile(PyObject *filename); +static PyObject *make_source_pathname(PyObject *pathname); +static PyObject* make_compiled_pathname(PyObject *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -768,10 +796,41 @@ PyObject * PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname) { + PyObject *m = NULL; + PyObject *nameobj, *pathobj = NULL, *cpathobj = NULL; + + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + + if (pathname != NULL) { + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) + goto error; + } else + pathobj = NULL; + if (cpathname != NULL) { + cpathobj = PyUnicode_DecodeFSDefault(cpathname); + if (cpathobj == NULL) + goto error; + } else + cpathobj = NULL; + m = PyImport_ExecCodeModuleObject(nameobj, co, pathobj, cpathobj); +error: + Py_DECREF(nameobj); + Py_XDECREF(pathobj); + Py_XDECREF(cpathobj); + return m; +} + +PyObject* +PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, + PyObject *cpathname) +{ PyObject *modules = PyImport_GetModuleDict(); PyObject *m, *d, *v; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) return NULL; /* If the module is being reloaded, we get the old module back @@ -783,12 +842,13 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, goto error; } /* Remember the filename as the __file__ attribute */ - v = NULL; if (pathname != NULL) { v = get_sourcefile(pathname); if (v == NULL) PyErr_Clear(); } + else + v = NULL; if (v == NULL) { v = ((PyCodeObject *)co)->co_filename; Py_INCREF(v); @@ -798,27 +858,21 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, Py_DECREF(v); /* Remember the pyc path name as the __cached__ attribute. */ - if (cpathname == NULL) { - v = Py_None; - Py_INCREF(v); - } - else if ((v = PyUnicode_FromString(cpathname)) == NULL) { - PyErr_Clear(); /* Not important enough to report */ + if (cpathname != NULL) + v = cpathname; + else v = Py_None; - Py_INCREF(v); - } if (PyDict_SetItemString(d, "__cached__", v) != 0) PyErr_Clear(); /* Not important enough to report */ - Py_DECREF(v); v = PyEval_EvalCode(co, d, d); if (v == NULL) goto error; Py_DECREF(v); - if ((m = PyDict_GetItemString(modules, name)) == NULL) { + if ((m = PyDict_GetItem(modules, name)) == NULL) { PyErr_Format(PyExc_ImportError, - "Loaded module %.200s not found in sys.modules", + "Loaded module %R not found in sys.modules", name); return NULL; } @@ -836,10 +890,10 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, /* Like strrchr(string, '/') but searches for the rightmost of either SEP or ALTSEP, if the latter is defined. */ -static char * -rightmost_sep(char *s) +static Py_UCS4* +rightmost_sep(Py_UCS4 *s) { - char *found, c; + Py_UCS4 *found, c; for (found = NULL; (c = *s); s++) { if (c == SEP #ifdef ALTSEP @@ -853,113 +907,89 @@ rightmost_sep(char *s) return found; } +/* Like rightmost_sep, but operate on unicode objects. */ +static Py_ssize_t +rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end) +{ + Py_ssize_t found, i; + Py_UCS4 c; + for (found = -1, i = start; i < end; i++) { + c = PyUnicode_READ_CHAR(o, i); + if (c == SEP +#ifdef ALTSEP + || c == ALTSEP +#endif + ) + { + found = i; + } + } + return found; +} /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname for the compiled file, or NULL if there's no space in the buffer. - Doesn't set an exception. */ + Doesn't set an exception. -static char * -make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) -{ - /* foo.py -> __pycache__/foo.<tag>.pyc */ - size_t len = strlen(pathname); - size_t i, save; - char *pos; - int sep = SEP; - - /* Sanity check that the buffer has roughly enough space to hold what - will eventually be the full path to the compiled file. The 5 extra - bytes include the slash afer __pycache__, the two extra dots, the - extra trailing character ('c' or 'o') and null. This isn't exact - because the contents of the buffer can affect how many actual - characters of the string get into the buffer. We'll do a final - sanity check before writing the extension to ensure we do not - overflow the buffer. - */ - if (len + strlen(CACHEDIR) + strlen(pyc_tag) + 5 > buflen) - return NULL; + foo.py -> __pycache__/foo.<tag>.pyc - /* Find the last path separator and copy everything from the start of - the source string up to and including the separator. - */ - if ((pos = rightmost_sep(pathname)) == NULL) { - i = 0; - } - else { - sep = *pos; - i = pos - pathname + 1; - strncpy(buf, pathname, i); - } - - save = i; - buf[i++] = '\0'; - /* Add __pycache__/ */ - strcat(buf, CACHEDIR); - i += strlen(CACHEDIR) - 1; - buf[i++] = sep; - buf[i++] = '\0'; - /* Add the base filename, but remove the .py or .pyw extension, since - the tag name must go before the extension. - */ - strcat(buf, pathname + save); - if ((pos = strrchr(buf, '.')) != NULL) - *++pos = '\0'; - strcat(buf, pyc_tag); - /* The length test above assumes that we're only adding one character - to the end of what would normally be the extension. What if there - is no extension, or the string ends in '.' or '.p', and otherwise - fills the buffer? By appending 4 more characters onto the string - here, we could overrun the buffer. - - As a simple example, let's say buflen=32 and the input string is - 'xxx.py'. strlen() would be 6 and the test above would yield: - - (6 + 11 + 10 + 5 == 32) > 32 - - which is false and so the name mangling would continue. This would - be fine because we'd end up with this string in buf: - - __pycache__/xxx.cpython-32.pyc\0 - - strlen(of that) == 30 + the nul fits inside a 32 character buffer. - We can even handle an input string of say 'xxxxx' above because - that's (5 + 11 + 10 + 5 == 31) > 32 which is also false. Name - mangling that yields: - - __pycache__/xxxxxcpython-32.pyc\0 - - which is 32 characters including the nul, and thus fits in the - buffer. However, an input string of 'xxxxxx' would yield a result - string of: - - __pycache__/xxxxxxcpython-32.pyc\0 - - which is 33 characters long (including the nul), thus overflowing - the buffer, even though the first test would fail, i.e.: the input - string is also 6 characters long, so 32 > 32 is false. - - The reason the first test fails but we still overflow the buffer is - that the test above only expects to add one extra character to be - added to the extension, and here we're adding three (pyc). We - don't add the first dot, so that reclaims one of expected - positions, leaving us overflowing by 1 byte (3 extra - 1 reclaimed - dot - 1 expected extra == 1 overflowed). - - The best we can do is ensure that we still have enough room in the - target buffer before we write the extension. Because it's always - only the extension that can cause the overflow, and never the other - path bytes we've written, it's sufficient to just do one more test - here. Still, the assertion that follows can't hurt. - */ -#if 0 - printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); -#endif - if (strlen(buf) + 5 > buflen) + pathstr is assumed to be "ready". +*/ + +static PyObject* +make_compiled_pathname(PyObject *pathstr, int debug) +{ + PyObject *result; + Py_ssize_t fname, ext, len, i, pos, taglen; + Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1; + int kind; + void *data; + Py_UCS4 lastsep; + + /* Compute the output string size. */ + len = PyUnicode_GET_LENGTH(pathstr); + /* If there is no separator, this returns -1, so + fname will be 0. */ + fname = rightmost_sep_obj(pathstr, 0, len) + 1; + /* Windows: re-use the last separator character (/ or \\) when + appending the __pycache__ path. */ + if (fname > 0) + lastsep = PyUnicode_READ_CHAR(pathstr, fname -1); + else + lastsep = SEP; + ext = fname - 1; + for(i = fname; i < len; i++) + if (PyUnicode_READ_CHAR(pathstr, i) == '.') + ext = i + 1; + if (ext < fname) + /* No dot in filename; use entire filename */ + ext = len; + + /* result = pathstr[:fname] + "__pycache__" + SEP + + pathstr[fname:ext] + tag + ".py[co]" */ + taglen = strlen(pyc_tag); + result = PyUnicode_New(ext + pycache_len + 1 + taglen + 4, + PyUnicode_MAX_CHAR_VALUE(pathstr)); + if (!result) return NULL; - strcat(buf, debug ? ".pyc" : ".pyo"); - assert(strlen(buf) < buflen); - return buf; + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); + pos = fname; + for (i = 0; i < pycache_len; i++) + PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); + PyUnicode_WRITE(kind, data, pos++, lastsep); + PyUnicode_CopyCharacters(result, pos, pathstr, + fname, ext - fname); + pos += ext - fname; + for (i = 0; pyc_tag[i]; i++) + PyUnicode_WRITE(kind, data, pos++, pyc_tag[i]); + PyUnicode_WRITE(kind, data, pos++, '.'); + PyUnicode_WRITE(kind, data, pos++, 'p'); + PyUnicode_WRITE(kind, data, pos++, 'y'); + PyUnicode_WRITE(kind, data, pos++, debug ? 'c' : 'o'); + return result; } @@ -967,42 +997,52 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) source file, if the path matches the PEP 3147 format. This does not check for any file existence, however, if the pyc file name does not match PEP 3147 style, NULL is returned. buf must be at least as big as pathname; - the resulting path will always be shorter. */ + the resulting path will always be shorter. + + (...)/__pycache__/foo.<tag>.pyc -> (...)/foo.py */ -static char * -make_source_pathname(char *pathname, char *buf) +static PyObject* +make_source_pathname(PyObject *path) { - /* __pycache__/foo.<tag>.pyc -> foo.py */ - size_t i, j; - char *left, *right, *dot0, *dot1, sep; + Py_ssize_t left, right, dot0, dot1, len; + Py_ssize_t i, j; + PyObject *result; + int kind; + void *data; + + len = PyUnicode_GET_LENGTH(path); + if (len > MAXPATHLEN) + return NULL; /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - if ((right = rightmost_sep(pathname)) == NULL) - return NULL; - sep = *right; - *right = '\0'; - left = rightmost_sep(pathname); - *right = sep; - if (left == NULL) - left = pathname; + right = rightmost_sep_obj(path, 0, len); + if (right == -1) + return NULL; + left = rightmost_sep_obj(path, 0, right); + if (left == -1) + left = 0; else left++; - if (right-left != strlen(CACHEDIR) || - strncmp(left, CACHEDIR, right-left) != 0) + if (right-left != sizeof(CACHEDIR)-1) return NULL; + for (i = 0; i < sizeof(CACHEDIR)-1; i++) + if (PyUnicode_READ_CHAR(path, left+i) != CACHEDIR[i]) + return NULL; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = strchr(right + 1, '.')) == NULL) + dot0 = PyUnicode_FindChar(path, '.', right+1, len, 1); + if (dot0 < 0) return NULL; - if ((dot1 = strchr(dot0 + 1, '.')) == NULL) + dot1 = PyUnicode_FindChar(path, '.', dot0+1, len, 1); + if (dot1 < 0) return NULL; /* Too many dots? */ - if (strchr(dot1 + 1, '.') != NULL) + if (PyUnicode_FindChar(path, '.', dot1+1, len, 1) != -1) return NULL; /* This is a PEP 3147 path. Start by copying everything from the @@ -1010,10 +1050,18 @@ make_source_pathname(char *pathname, char *buf) copy the file's basename, removing the magic tag and adding a .py suffix. */ - strncpy(buf, pathname, (i=left-pathname)); - strncpy(buf+i, right+1, (j=dot0-right)); - strcpy(buf+i+j, "py"); - return buf; + result = PyUnicode_New(left + (dot0-right) + 2, + PyUnicode_MAX_CHAR_VALUE(path)); + if (!result) + return NULL; + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + PyUnicode_CopyCharacters(result, 0, path, 0, (i = left)); + PyUnicode_CopyCharacters(result, left, path, right+1, + (j = dot0-right)); + PyUnicode_WRITE(kind, data, i+j, 'p'); + PyUnicode_WRITE(kind, data, i+j+1, 'y'); + return result; } /* Given a pathname for a Python source file, its time of last @@ -1024,31 +1072,39 @@ make_source_pathname(char *pathname, char *buf) Doesn't set an exception. */ static FILE * -check_compiled_module(char *pathname, time_t mtime, char *cpathname) +check_compiled_module(PyObject *pathname, struct stat *srcstat, PyObject *cpathname) { FILE *fp; long magic; long pyc_mtime; + long pyc_size; - fp = fopen(cpathname, "rb"); + fp = _Py_fopen(cpathname, "rb"); if (fp == NULL) return NULL; magic = PyMarshal_ReadLongFromFile(fp); if (magic != pyc_magic) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad magic\n", cpathname); + PySys_FormatStderr("# %R has bad magic\n", cpathname); fclose(fp); return NULL; } pyc_mtime = PyMarshal_ReadLongFromFile(fp); - if (pyc_mtime != mtime) { + if (pyc_mtime != srcstat->st_mtime) { + if (Py_VerboseFlag) + PySys_FormatStderr("# %R has bad mtime\n", cpathname); + fclose(fp); + return NULL; + } + pyc_size = PyMarshal_ReadLongFromFile(fp); + if (pyc_size != (srcstat->st_size & 0xFFFFFFFF)) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad mtime\n", cpathname); + PySys_FormatStderr("# %R has bad size\n", cpathname); fclose(fp); return NULL; } if (Py_VerboseFlag) - PySys_WriteStderr("# %s matches %s\n", cpathname, pathname); + PySys_FormatStderr("# %R matches %R\n", cpathname, pathname); return fp; } @@ -1056,7 +1112,7 @@ check_compiled_module(char *pathname, time_t mtime, char *cpathname) /* Read a code object from a file and check it for validity */ static PyCodeObject * -read_compiled_module(char *cpathname, FILE *fp) +read_compiled_module(PyObject *cpathname, FILE *fp) { PyObject *co; @@ -1065,7 +1121,7 @@ read_compiled_module(char *cpathname, FILE *fp) return NULL; if (!PyCode_Check(co)) { PyErr_Format(PyExc_ImportError, - "Non-code object in %.200s", cpathname); + "Non-code object in %R", cpathname); Py_DECREF(co); return NULL; } @@ -1077,7 +1133,7 @@ read_compiled_module(char *cpathname, FILE *fp) module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_compiled_module(char *name, char *cpathname, FILE *fp) +load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) { long magic; PyCodeObject *co; @@ -1086,18 +1142,20 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) magic = PyMarshal_ReadLongFromFile(fp); if (magic != pyc_magic) { PyErr_Format(PyExc_ImportError, - "Bad magic number in %.200s", cpathname); + "Bad magic number in %R", cpathname); return NULL; } + /* Skip mtime and size */ + (void) PyMarshal_ReadLongFromFile(fp); (void) PyMarshal_ReadLongFromFile(fp); co = read_compiled_module(cpathname, fp); if (co == NULL) return NULL; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # precompiled from %s\n", - name, cpathname); - m = PyImport_ExecCodeModuleWithPathnames( - name, (PyObject *)co, cpathname, cpathname); + PySys_FormatStderr("import %U # precompiled from %R\n", + name, cpathname); + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + cpathname, cpathname); Py_DECREF(co); return m; @@ -1106,134 +1164,233 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) /* Parse a source file and return the corresponding code object */ static PyCodeObject * -parse_source_module(const char *pathname, FILE *fp) +parse_source_module(PyObject *pathname, FILE *fp) { - PyCodeObject *co = NULL; + PyCodeObject *co; + PyObject *pathbytes; mod_ty mod; PyCompilerFlags flags; - PyArena *arena = PyArena_New(); - if (arena == NULL) + PyArena *arena; + + pathbytes = PyUnicode_EncodeFSDefault(pathname); + if (pathbytes == NULL) return NULL; + arena = PyArena_New(); + if (arena == NULL) { + Py_DECREF(pathbytes); + return NULL; + } + flags.cf_flags = 0; - mod = PyParser_ASTFromFile(fp, pathname, NULL, + mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL, Py_file_input, 0, 0, &flags, NULL, arena); - if (mod) { - co = PyAST_Compile(mod, pathname, NULL, arena); - } + if (mod != NULL) + co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena); + else + co = NULL; + Py_DECREF(pathbytes); PyArena_Free(arena); return co; } - -/* Helper to open a bytecode file for writing in exclusive mode */ - -static FILE * -open_exclusive(char *filename, mode_t mode) -{ -#if defined(O_EXCL)&&defined(O_CREAT)&&defined(O_WRONLY)&&defined(O_TRUNC) - /* Use O_EXCL to avoid a race condition when another process tries to - write the same file. When that happens, our open() call fails, - which is just fine (since it's only a cache). - XXX If the file exists and is writable but the directory is not - writable, the file will never be written. Oh well. - */ - int fd; - (void) unlink(filename); - fd = open(filename, O_EXCL|O_CREAT|O_WRONLY|O_TRUNC -#ifdef O_BINARY - |O_BINARY /* necessary for Windows */ -#endif -#ifdef __VMS - , mode, "ctxt=bin", "shr=nil" -#else - , mode -#endif - ); - if (fd < 0) - return NULL; - return fdopen(fd, "wb"); -#else - /* Best we can do -- on Windows this can't happen anyway */ - return fopen(filename, "wb"); -#endif -} - - /* Write a compiled module to a file, placing the time of last modification of its source into the header. Errors are ignored, if a write error occurs an attempt is made to remove the file. */ static void -write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) +write_compiled_module(PyCodeObject *co, PyObject *cpathname, + struct stat *srcstat) { + Py_UCS4 *cpathname_ucs4; FILE *fp; - char *dirpath; time_t mtime = srcstat->st_mtime; + long size = srcstat->st_size & 0xFFFFFFFF; + PyObject *cpathname_tmp; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; + wchar_t *wdirname, *wpathname, *wpathname_tmp; #else - mode_t mode = srcstat->st_mode & ~S_IXUSR & ~S_IXGRP & ~S_IXOTH; mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); + PyObject *dirbytes; + PyObject *cpathbytes, *cpathbytes_tmp; #endif - int saved; + int fd; + PyObject *dirname; + Py_UCS4 *dirsep; + int res, ok; /* Ensure that the __pycache__ directory exists. */ - dirpath = rightmost_sep(cpathname); - if (dirpath == NULL) { + cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname); + if (!cpathname_ucs4) + return; + dirsep = rightmost_sep(cpathname_ucs4); + if (dirsep == NULL) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# no %s path found %s\n", - CACHEDIR, cpathname); + PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); + return; + } + dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + cpathname_ucs4, + dirsep - cpathname_ucs4); + PyMem_Free(cpathname_ucs4); + if (dirname == NULL) { + PyErr_Clear(); return; } - saved = *dirpath; - *dirpath = '\0'; #ifdef MS_WINDOWS - if (_mkdir(cpathname) < 0 && errno != EEXIST) { + wdirname = PyUnicode_AsUnicode(dirname); + if (wdirname == NULL) { + PyErr_Clear(); + return; + } + res = CreateDirectoryW(wdirname, NULL); + ok = (res != 0); + if (!ok && GetLastError() == ERROR_ALREADY_EXISTS) + ok = 1; #else - if (mkdir(cpathname, dirmode) < 0 && errno != EEXIST) { + dirbytes = PyUnicode_EncodeFSDefault(dirname); + if (dirbytes == NULL) { + PyErr_Clear(); + return; + } + res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode); + Py_DECREF(dirbytes); + if (0 <= res) + ok = 1; + else + ok = (errno == EEXIST); #endif - *dirpath = saved; + if (!ok) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# cannot create cache dir %s\n", cpathname); + PySys_FormatStderr("# cannot create cache dir %R\n", dirname); + Py_DECREF(dirname); + return; + } + Py_DECREF(dirname); + + /* We first write to a tmp file and then take advantage + of atomic renaming (which *should* be true even under Windows). + As in importlib, we use id(something) to generate a pseudo-random + filename. mkstemp() can't be used since it doesn't allow specifying + the file access permissions. + */ + cpathname_tmp = PyUnicode_FromFormat("%U.%zd", + cpathname, (Py_ssize_t) co); + if (cpathname_tmp == NULL) { + PyErr_Clear(); + return; + } +#ifdef MS_WINDOWS + wpathname = PyUnicode_AsUnicode(cpathname); + if (wpathname == NULL) { + PyErr_Clear(); + return; + } + wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp); + if (wpathname_tmp == NULL) { + PyErr_Clear(); return; } - *dirpath = saved; - fp = open_exclusive(cpathname, mode); + (void)DeleteFileW(wpathname_tmp); + fd = _wopen(wpathname_tmp, + O_EXCL | O_CREAT | O_WRONLY | O_BINARY, + mode); + if (0 <= fd) + fp = fdopen(fd, "wb"); + else + fp = NULL; +#else + cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp); + Py_DECREF(cpathname_tmp); + if (cpathbytes_tmp == NULL) { + PyErr_Clear(); + return; + } + cpathbytes = PyUnicode_EncodeFSDefault(cpathname); + if (cpathbytes == NULL) { + PyErr_Clear(); + return; + } + fd = open(PyBytes_AS_STRING(cpathbytes_tmp), + O_CREAT | O_EXCL | O_WRONLY, 0666); + if (0 <= fd) + fp = fdopen(fd, "wb"); + else + fp = NULL; +#endif if (fp == NULL) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# can't create %s\n", cpathname); + PySys_FormatStderr( + "# can't create %R\n", cpathname); +#ifdef MS_WINDOWS + Py_DECREF(cpathname_tmp); +#else + Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); +#endif return; } PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); - /* First write a 0 for mtime */ + /* First write a 0 for mtime and size */ + PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); + fflush(fp); + /* Now write the true mtime and size (as 32-bit fields) */ + fseek(fp, 4L, 0); + assert(mtime <= 0xFFFFFFFF); + PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); + PyMarshal_WriteLongToFile(size, fp, Py_MARSHAL_VERSION); if (fflush(fp) != 0 || ferror(fp)) { if (Py_VerboseFlag) - PySys_WriteStderr("# can't write %s\n", cpathname); + PySys_FormatStderr("# can't write %R\n", cpathname); /* Don't keep partial file */ fclose(fp); - (void) unlink(cpathname); +#ifdef MS_WINDOWS + (void)DeleteFileW(wpathname_tmp); + Py_DECREF(cpathname_tmp); +#else + (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); + Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); +#endif return; } - /* Now write the true mtime (as a 32-bit field) */ - fseek(fp, 4L, 0); - assert(mtime <= 0xFFFFFFFF); - PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); - fflush(fp); fclose(fp); + /* Do a (hopefully) atomic rename */ +#ifdef MS_WINDOWS + if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) { + if (Py_VerboseFlag) + PySys_FormatStderr("# can't write %R\n", cpathname); + /* Don't keep tmp file */ + (void) DeleteFileW(wpathname_tmp); + Py_DECREF(cpathname_tmp); + return; + } + Py_DECREF(cpathname_tmp); +#else + if (rename(PyBytes_AS_STRING(cpathbytes_tmp), + PyBytes_AS_STRING(cpathbytes))) { + if (Py_VerboseFlag) + PySys_FormatStderr("# can't write %R\n", cpathname); + /* Don't keep tmp file */ + unlink(PyBytes_AS_STRING(cpathbytes_tmp)); + Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); + return; + } + Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); +#endif if (Py_VerboseFlag) - PySys_WriteStderr("# wrote %s\n", cpathname); + PySys_FormatStderr("# wrote %R\n", cpathname); } static void @@ -1260,26 +1417,44 @@ update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) } } -static int -update_compiled_module(PyCodeObject *co, char *pathname) +static void +update_compiled_module(PyCodeObject *co, PyObject *newname) { - PyObject *oldname, *newname; - - newname = PyUnicode_DecodeFSDefault(pathname); - if (newname == NULL) - return -1; + PyObject *oldname; - if (!PyUnicode_Compare(co->co_filename, newname)) { - Py_DECREF(newname); - return 0; - } + if (PyUnicode_Compare(co->co_filename, newname) == 0) + return; oldname = co->co_filename; Py_INCREF(oldname); update_code_filenames(co, oldname, newname); Py_DECREF(oldname); - Py_DECREF(newname); - return 1; +} + +static PyObject * +imp_fix_co_filename(PyObject *self, PyObject *args) +{ + PyObject *co; + PyObject *file_path; + + if (!PyArg_ParseTuple(args, "OO:_fix_co_filename", &co, &file_path)) + return NULL; + + if (!PyCode_Check(co)) { + PyErr_SetString(PyExc_TypeError, + "first argument must be a code object"); + return NULL; + } + + if (!PyUnicode_Check(file_path)) { + PyErr_SetString(PyExc_TypeError, + "second argument must be a string"); + return NULL; + } + + update_compiled_module((PyCodeObject*)co, file_path); + + Py_RETURN_NONE; } /* Load a source module from a given file and return its module @@ -1287,20 +1462,19 @@ update_compiled_module(PyCodeObject *co, char *pathname) byte-compiled file, use that instead. */ static PyObject * -load_source_module(char *name, char *pathname, FILE *fp) +load_source_module(PyObject *name, PyObject *pathname, FILE *fp) { struct stat st; FILE *fpc; - char buf[MAXPATHLEN+1]; - char *cpathname; + PyObject *cpathname = NULL, *cpathbytes = NULL; PyCodeObject *co; - PyObject *m; + PyObject *m = NULL; if (fstat(fileno(fp), &st) != 0) { PyErr_Format(PyExc_RuntimeError, - "unable to get file status from '%s'", + "unable to get file status from %R", pathname); - return NULL; + goto error; } if (sizeof st.st_mtime > 4) { /* Python's .pyc timestamp handling presumes that the timestamp fits @@ -1310,38 +1484,47 @@ load_source_module(char *name, char *pathname, FILE *fp) */ st.st_mtime &= 0xFFFFFFFF; } - cpathname = make_compiled_pathname( - pathname, buf, (size_t)MAXPATHLEN + 1, !Py_OptimizeFlag); - if (cpathname != NULL && - (fpc = check_compiled_module(pathname, st.st_mtime, cpathname))) { + if (PyUnicode_READY(pathname) < 0) + return NULL; + cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); + + if (cpathname != NULL) + fpc = check_compiled_module(pathname, &st, cpathname); + else + fpc = NULL; + + if (fpc) { co = read_compiled_module(cpathname, fpc); fclose(fpc); if (co == NULL) - return NULL; - if (update_compiled_module(co, pathname) < 0) - return NULL; + goto error; + update_compiled_module(co, pathname); if (Py_VerboseFlag) - PySys_WriteStderr("import %s # precompiled from %s\n", - name, cpathname); - pathname = cpathname; + PySys_FormatStderr("import %U # precompiled from %R\n", + name, cpathname); + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + cpathname, cpathname); } else { co = parse_source_module(pathname, fp); if (co == NULL) - return NULL; + goto error; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # from %s\n", - name, pathname); - if (cpathname) { + PySys_FormatStderr("import %U # from %R\n", + name, pathname); + if (cpathname != NULL) { PyObject *ro = PySys_GetObject("dont_write_bytecode"); if (ro == NULL || !PyObject_IsTrue(ro)) write_compiled_module(co, cpathname, &st); } + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + pathname, cpathname); } - m = PyImport_ExecCodeModuleWithPathnames( - name, (PyObject *)co, pathname, cpathname); Py_DECREF(co); +error: + Py_XDECREF(cpathbytes); + Py_XDECREF(cpathname); return m; } @@ -1349,101 +1532,115 @@ load_source_module(char *name, char *pathname, FILE *fp) * Returns the path to the py file if available, else the given path */ static PyObject * -get_sourcefile(char *file) +get_sourcefile(PyObject *filename) { - char py[MAXPATHLEN + 1]; Py_ssize_t len; - PyObject *u; + Py_UCS4 *fileuni; + PyObject *py; struct stat statbuf; + int err; - if (!file || !*file) { + len = PyUnicode_GET_LENGTH(filename); + if (len == 0) Py_RETURN_NONE; - } - len = strlen(file); - /* match '*.py?' */ - if (len > MAXPATHLEN || PyOS_strnicmp(&file[len-4], ".py", 3) != 0) { - return PyUnicode_DecodeFSDefault(file); - } + /* don't match *.pyc or *.pyo? */ + fileuni = PyUnicode_AsUCS4Copy(filename); + if (!fileuni) + return NULL; + if (len < 5 + || fileuni[len-4] != '.' + || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P') + || (fileuni[len-2] != 'y' && fileuni[len-2] != 'Y')) + goto unchanged; /* Start by trying to turn PEP 3147 path into source path. If that * fails, just chop off the trailing character, i.e. legacy pyc path * to py. */ - if (make_source_pathname(file, py) == NULL) { - strncpy(py, file, len-1); - py[len-1] = '\0'; + py = make_source_pathname(filename); + if (py == NULL) { + PyErr_Clear(); + py = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, fileuni, len - 1); } + if (py == NULL) + goto error; - if (stat(py, &statbuf) == 0 && - S_ISREG(statbuf.st_mode)) { - u = PyUnicode_DecodeFSDefault(py); - } - else { - u = PyUnicode_DecodeFSDefault(file); + err = _Py_stat(py, &statbuf); + if (err == -2) + goto error; + if (err == 0 && S_ISREG(statbuf.st_mode)) { + PyMem_Free(fileuni); + return py; } - return u; + Py_DECREF(py); + goto unchanged; + +error: + PyErr_Clear(); +unchanged: + PyMem_Free(fileuni); + Py_INCREF(filename); + return filename; } /* Forward */ -static PyObject *load_module(char *, FILE *, char *, int, PyObject *); -static struct filedescr *find_module(char *, char *, PyObject *, - char *, size_t, FILE **, PyObject **); -static struct _frozen * find_frozen(char *); +static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); +static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, + PyObject **, FILE **, PyObject **); +static struct _frozen * find_frozen(PyObject *); /* Load a package and return its module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_package(char *name, char *pathname) +load_package(PyObject *name, PyObject *pathname) { - PyObject *m, *d; - PyObject *file = NULL; - PyObject *path = NULL; + PyObject *m, *d, *bufobj; + PyObject *file = NULL, *path_list = NULL; int err; - char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) return NULL; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # directory %s\n", - name, pathname); - d = PyModule_GetDict(m); + PySys_FormatStderr("import %U # directory %R\n", + name, pathname); file = get_sourcefile(pathname); if (file == NULL) - goto error; - path = Py_BuildValue("[O]", file); - if (path == NULL) - goto error; + return NULL; + path_list = Py_BuildValue("[O]", file); + if (path_list == NULL) { + Py_DECREF(file); + return NULL; + } + d = PyModule_GetDict(m); err = PyDict_SetItemString(d, "__file__", file); + Py_DECREF(file); if (err == 0) - err = PyDict_SetItemString(d, "__path__", path); - if (err != 0) - goto error; - buf[0] = '\0'; - fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL); + err = PyDict_SetItemString(d, "__path__", path_list); + if (err != 0) { + Py_DECREF(path_list); + return NULL; + } + fdp = find_module(name, initstr, path_list, + &bufobj, &fp, NULL); + Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { PyErr_Clear(); Py_INCREF(m); + return m; } else - m = NULL; - goto cleanup; + return NULL; } - m = load_module(name, fp, buf, fdp->type, NULL); + m = load_module(name, fp, bufobj, fdp->type, NULL); + Py_XDECREF(bufobj); if (fp != NULL) fclose(fp); - goto cleanup; - - error: - m = NULL; - cleanup: - Py_XDECREF(path); - Py_XDECREF(file); return m; } @@ -1451,11 +1648,12 @@ load_package(char *name, char *pathname) /* Helper to test for built-in module */ static int -is_builtin(char *name) +is_builtin(PyObject *name) { - int i; + int i, cmp; for (i = 0; PyImport_Inittab[i].name != NULL; i++) { - if (strcmp(name, PyImport_Inittab[i].name) == 0) { + cmp = PyUnicode_CompareWithASCIIString(name, PyImport_Inittab[i].name); + if (cmp == 0) { if (PyImport_Inittab[i].initfunc == NULL) return -1; else @@ -1549,46 +1747,314 @@ PyImport_GetImporter(PyObject *path) { pathname and an open file. Return NULL if the module is not found. */ #ifdef MS_COREDLL -extern FILE *PyWin_FindRegisteredModule(const char *, struct filedescr **, - char *, Py_ssize_t); +extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, + PyObject **p_path); #endif -static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); -static int find_init_module(char *); /* Forward */ +/* Forward */ +static int case_ok(PyObject *, Py_ssize_t, PyObject *); +static int find_init_module(PyObject *); static struct filedescr importhookdescr = {"", "", IMP_HOOK}; -static struct filedescr * -find_module(char *fullname, char *subname, PyObject *path, char *buf, - size_t buflen, FILE **p_fp, PyObject **p_loader) +/* Get the path of a module: get its importer and call importer.find_module() + hook, or check if the module if a package (if path/__init__.py exists). + + -1: error: a Python error occurred + 0: ignore: an error occurred because of invalid data, but the error is not + important enough to be reported. + 1: get path: module not found, but *buf contains its path + 2: found: *p_fd is the file descriptor (IMP_HOOK or PKG_DIRECTORY) + and *buf is the path */ + +static int +find_module_path(PyObject *fullname, PyObject *name, PyObject *path, + PyObject *path_hooks, PyObject *path_importer_cache, + PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) +{ + PyObject *path_unicode, *filename = NULL; + Py_ssize_t len, pos; + struct stat statbuf; + static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + int err, result, addsep; + + if (PyUnicode_Check(path)) { + Py_INCREF(path); + path_unicode = path; + } + else if (PyBytes_Check(path)) { + path_unicode = PyUnicode_DecodeFSDefaultAndSize( + PyBytes_AS_STRING(path), PyBytes_GET_SIZE(path)); + if (path_unicode == NULL) + return -1; + } + else + return 0; + + if (PyUnicode_READY(path_unicode)) + return -1; + + len = PyUnicode_GET_LENGTH(path_unicode); + if (PyUnicode_FindChar(path_unicode, 0, 0, len, 1) != -1) { + result = 0; + goto out; /* path contains '\0' */ + } + + /* sys.path_hooks import hook */ + if (p_loader != NULL) { + _Py_IDENTIFIER(find_module); + PyObject *importer; + + importer = get_path_importer(path_importer_cache, + path_hooks, path); + if (importer == NULL) { + result = -1; + goto out; + } + /* Note: importer is a borrowed reference */ + if (importer != Py_None) { + PyObject *loader; + loader = _PyObject_CallMethodId(importer, + &PyId_find_module, "O", fullname); + if (loader == NULL) { + result = -1; /* error */ + goto out; + } + if (loader != Py_None) { + /* a loader was found */ + *p_loader = loader; + *p_fd = &importhookdescr; + result = 2; + goto out; + } + Py_DECREF(loader); + result = 0; + goto out; + } + } + /* no hook was found, use builtin import */ + + addsep = 0; + if (len > 0 && PyUnicode_READ_CHAR(path_unicode, len-1) != SEP +#ifdef ALTSEP + && PyUnicode_READ_CHAR(path_unicode, len-1) != ALTSEP +#endif + ) + addsep = 1; + filename = PyUnicode_New(len + PyUnicode_GET_LENGTH(name) + addsep, + Py_MAX(PyUnicode_MAX_CHAR_VALUE(path_unicode), + PyUnicode_MAX_CHAR_VALUE(name))); + if (filename == NULL) { + result = -1; + goto out; + } + PyUnicode_CopyCharacters(filename, 0, path_unicode, 0, len); + pos = len; + if (addsep) + PyUnicode_WRITE(PyUnicode_KIND(filename), + PyUnicode_DATA(filename), + pos++, SEP); + PyUnicode_CopyCharacters(filename, pos, name, 0, + PyUnicode_GET_LENGTH(name)); + + /* Check for package import (buf holds a directory name, + and there's an __init__ module in that directory */ +#ifdef HAVE_STAT + err = _Py_stat(filename, &statbuf); + if (err == -2) { + result = -1; + goto out; + } + if (err == 0 && /* it exists */ + S_ISDIR(statbuf.st_mode)) /* it's a directory */ + { + int match; + + match = case_ok(filename, 0, name); + if (match < 0) { + result = -1; + goto out; + } + if (match) { /* case matches */ + if (find_init_module(filename)) { /* and has __init__.py */ + *p_path = filename; + filename = NULL; + *p_fd = &fd_package; + result = 2; + goto out; + } + else { + int err; + err = PyErr_WarnFormat(PyExc_ImportWarning, 1, + "Not importing directory %R: missing __init__.py", + filename); + if (err) { + result = -1; + goto out; + } + } + } + } +#endif + *p_path = filename; + filename = NULL; + result = 1; + out: + Py_DECREF(path_unicode); + Py_XDECREF(filename); + return result; +} + +/* Find a module in search_path_list. For each path, try + find_module_path() or try each _PyImport_Filetab suffix. + + If the module is found, return a file descriptor, write the path in + *p_filename, write the pointer to the file object into *p_fp, and (if + p_loader is not NULL) the loader into *p_loader. + + Otherwise, raise an exception and return NULL. */ + +static struct filedescr* +find_module_path_list(PyObject *fullname, PyObject *name, + PyObject *search_path_list, PyObject *path_hooks, + PyObject *path_importer_cache, + PyObject **p_path, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; - size_t len, namelen; struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - PyObject *path_hooks, *path_importer_cache; - struct stat statbuf; + PyObject *prefix, *filename; + int match; + int err; + + npath = PyList_Size(search_path_list); + for (i = 0; i < npath; i++) { + PyObject *path; + int ok; + + path = PyList_GetItem(search_path_list, i); + if (path == NULL) + return NULL; + + prefix = NULL; + ok = find_module_path(fullname, name, path, + path_hooks, path_importer_cache, + &prefix, p_loader, &fdp); + if (ok < 0) + return NULL; + if (ok == 0) + continue; + if (ok == 2) { + *p_path = prefix; + return fdp; + } + + for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { + struct stat statbuf; + + filemode = fdp->mode; + if (filemode[0] == 'U') + filemode = "r" PY_STDIOTEXTMODE; + + filename = PyUnicode_FromFormat("%U%s", prefix, fdp->suffix); + if (filename == NULL) { + Py_DECREF(prefix); + return NULL; + } + + if (Py_VerboseFlag > 1) + PySys_FormatStderr("# trying %R\n", filename); + + err = _Py_stat(filename, &statbuf); + if (err == -2) { + Py_DECREF(prefix); + Py_DECREF(filename); + return NULL; + } + if (err != 0 || S_ISDIR(statbuf.st_mode)) { + /* it doesn't exist, or it's a directory */ + Py_DECREF(filename); + continue; + } + + fp = _Py_fopen(filename, filemode); + if (fp == NULL) { + Py_DECREF(filename); + if (PyErr_Occurred()) { + Py_DECREF(prefix); + return NULL; + } + continue; + } + match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); + if (match < 0) { + Py_DECREF(prefix); + Py_DECREF(filename); + return NULL; + } + if (match) { + Py_DECREF(prefix); + *p_path = filename; + *p_fp = fp; + return fdp; + } + Py_DECREF(filename); + + fclose(fp); + fp = NULL; + } + Py_DECREF(prefix); + } + PyErr_Format(PyExc_ImportError, + "No module named %R", name); + return NULL; +} + +/* Find a module: + + - try find_module() of each sys.meta_path hook + - try find_frozen() + - try is_builtin() + - try _PyWin_FindRegisteredModule() (Windows only) + - otherwise, call find_module_path_list() with search_path_list (if not + NULL) or sys.path + + fullname can be NULL, but only if p_loader is NULL. + + Return: + + - &fd_builtin (C_BUILTIN) if it is a builtin + - &fd_frozen (PY_FROZEN) if it is frozen + - &fd_package (PKG_DIRECTORY) and write the filename into *p_path + if it is a package + - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a + importer loader was found + - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or + PY_CODERESOURCE: see _PyImport_Filetab), write the filename into + *p_path and the pointer to the open file into *p_fp + - NULL on error + + By default, *p_path, *p_fp and *p_loader (if set) are set to NULL. + Eg. *p_path is set to NULL for a builtin package. +*/ + +static struct filedescr * +find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, + PyObject **p_path, FILE **p_fp, PyObject **p_loader) +{ + Py_ssize_t i, npath; static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; - static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - char name[MAXPATHLEN+1]; -#if defined(PYOS_OS2) - size_t saved_len; - size_t saved_namelen; - char *saved_buf = NULL; -#endif + PyObject *path_hooks, *path_importer_cache; + + *p_path = NULL; + *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; - if (strlen(subname) > MAXPATHLEN) { - PyErr_SetString(PyExc_OverflowError, - "module name is too long"); - return NULL; - } - strcpy(name, subname); - /* sys.meta_path import hook */ if (p_loader != NULL) { + _Py_IDENTIFIER(find_module); PyObject *meta_path; meta_path = PySys_GetObject("meta_path"); @@ -1603,10 +2069,10 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, for (i = 0; i < npath; i++) { PyObject *loader; PyObject *hook = PyList_GetItem(meta_path, i); - loader = PyObject_CallMethod(hook, "find_module", - "sO", fullname, - path != NULL ? - path : Py_None); + loader = _PyObject_CallMethodId(hook, &PyId_find_module, + "OO", fullname, + search_path_list != NULL ? + search_path_list : Py_None); if (loader == NULL) { Py_DECREF(meta_path); return NULL; /* true error */ @@ -1622,27 +2088,29 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, Py_DECREF(meta_path); } - if (find_frozen(fullname) != NULL) { - strcpy(buf, fullname); + if (find_frozen(fullname) != NULL) return &fd_frozen; - } - if (path == NULL) { - if (is_builtin(name)) { - strcpy(buf, name); + if (search_path_list == NULL) { +#ifdef MS_COREDLL + FILE *fp; + struct filedescr *fdp; +#endif + if (is_builtin(name)) return &fd_builtin; - } #ifdef MS_COREDLL - fp = PyWin_FindRegisteredModule(name, &fdp, buf, buflen); + fp = _PyWin_FindRegisteredModule(name, &fdp, p_path); if (fp != NULL) { *p_fp = fp; return fdp; } + else if (PyErr_Occurred()) + return NULL; #endif - path = PySys_GetObject("path"); + search_path_list = PySys_GetObject("path"); } - if (path == NULL || !PyList_Check(path)) { + if (search_path_list == NULL || !PyList_Check(search_path_list)) { PyErr_SetString(PyExc_RuntimeError, "sys.path must be a list of directory names"); return NULL; @@ -1663,183 +2131,13 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, return NULL; } - npath = PyList_Size(path); - namelen = strlen(name); - for (i = 0; i < npath; i++) { - PyObject *v = PyList_GetItem(path, i); - PyObject *origv = v; - const char *base; - Py_ssize_t size; - if (!v) - return NULL; - if (PyUnicode_Check(v)) { - v = PyUnicode_EncodeFSDefault(v); - if (v == NULL) - return NULL; - } - else if (!PyBytes_Check(v)) - continue; - else - Py_INCREF(v); - - base = PyBytes_AS_STRING(v); - size = PyBytes_GET_SIZE(v); - len = size; - if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { - Py_DECREF(v); - continue; /* Too long */ - } - strcpy(buf, base); - Py_DECREF(v); - - if (strlen(buf) != len) { - continue; /* v contains '\0' */ - } - - /* sys.path_hooks import hook */ - if (p_loader != NULL) { - PyObject *importer; - - importer = get_path_importer(path_importer_cache, - path_hooks, origv); - if (importer == NULL) { - return NULL; - } - /* Note: importer is a borrowed reference */ - if (importer != Py_None) { - PyObject *loader; - loader = PyObject_CallMethod(importer, - "find_module", - "s", fullname); - if (loader == NULL) - return NULL; /* error */ - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - return &importhookdescr; - } - Py_DECREF(loader); - continue; - } - } - /* no hook was found, use builtin import */ - - if (len > 0 && buf[len-1] != SEP -#ifdef ALTSEP - && buf[len-1] != ALTSEP -#endif - ) - buf[len++] = SEP; - strcpy(buf+len, name); - len += namelen; - - /* Check for package import (buf holds a directory name, - and there's an __init__ module in that directory */ -#ifdef HAVE_STAT - if (stat(buf, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, name)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ - return &fd_package; - } - else { - int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return NULL; - err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) - return NULL; - } - } -#endif -#if defined(PYOS_OS2) - /* take a snapshot of the module spec for restoration - * after the 8 character DLL hackery - */ - saved_buf = strdup(buf); - saved_len = len; - saved_namelen = namelen; -#endif /* PYOS_OS2 */ - for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { - struct stat statbuf; -#if defined(PYOS_OS2) && defined(HAVE_DYNAMIC_LOADING) - /* OS/2 limits DLLs to 8 character names (w/o - extension) - * so if the name is longer than that and its a - * dynamically loaded module we're going to try, - * truncate the name before trying - */ - if (strlen(subname) > 8) { - /* is this an attempt to load a C extension? */ - const struct filedescr *scan; - scan = _PyImport_DynLoadFiletab; - while (scan->suffix != NULL) { - if (!strcmp(scan->suffix, fdp->suffix)) - break; - else - scan++; - } - if (scan->suffix != NULL) { - /* yes, so truncate the name */ - namelen = 8; - len -= strlen(subname) - namelen; - buf[len] = '\0'; - } - } -#endif /* PYOS_OS2 */ - strcpy(buf+len, fdp->suffix); - if (Py_VerboseFlag > 1) - PySys_WriteStderr("# trying %s\n", buf); - - filemode = fdp->mode; - if (filemode[0] == 'U') - filemode = "r" PY_STDIOTEXTMODE; - - if (stat(buf, &statbuf) == 0 && S_ISDIR(statbuf.st_mode)) - /* it's a directory */ - fp = NULL; - else - fp = fopen(buf, filemode); - if (fp != NULL) { - if (case_ok(buf, len, namelen, name)) - break; - else { /* continue search */ - fclose(fp); - fp = NULL; - } - } -#if defined(PYOS_OS2) - /* restore the saved snapshot */ - strcpy(buf, saved_buf); - len = saved_len; - namelen = saved_namelen; -#endif - } -#if defined(PYOS_OS2) - /* don't need/want the module name snapshot anymore */ - if (saved_buf) - { - free(saved_buf); - saved_buf = NULL; - } -#endif - if (fp != NULL) - break; - } - if (fp == NULL) { - PyErr_Format(PyExc_ImportError, - "No module named %.200s", name); - return NULL; - } - *p_fp = fp; - return fdp; + return find_module_path_list(fullname, name, + search_path_list, path_hooks, + path_importer_cache, + p_path, p_fp, p_loader); } -/* case_ok(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) +/* case_bytes(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) * The arguments here are tricky, best shown by example: * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 * ^ ^ ^ ^ @@ -1853,18 +2151,18 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, * We've already done a successful stat() or fopen() on buf, so know that * there's some match, possibly case-insensitive. * - * case_ok() is to return 1 if there's a case-sensitive match for - * name, else 0. case_ok() is also to return 1 if envar PYTHONCASEOK + * case_bytes() is to return 1 if there's a case-sensitive match for + * name, else 0. case_bytes() is also to return 1 if envar PYTHONCASEOK * exists. * - * case_ok() is used to implement case-sensitive import semantics even + * case_bytes() is used to implement case-sensitive import semantics even * on platforms with case-insensitive filesystems. It's trivial to implement * for case-sensitive filesystems. It's pretty much a cross-platform * nightmare for systems with case-insensitive filesystems. */ /* First we may need a pile of platform-specific header files; the sequence - * of #if's here should match the sequence in the body of case_ok(). + * of #if's here should match the sequence in the body of case_bytes(). */ #if defined(MS_WINDOWS) #include <windows.h> @@ -1883,33 +2181,24 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, #include <os2.h> #endif +#if defined(DJGPP) \ + || ((defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) \ + && defined(HAVE_DIRENT_H)) \ + || defined(PYOS_OS2) +# define USE_CASE_OK_BYTES +#endif + + +#ifdef USE_CASE_OK_BYTES static int -case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, char *name) +case_bytes(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) { /* Pick a platform-specific implementation; the sequence of #if's here should * match the sequence just above. */ -/* MS_WINDOWS */ -#if defined(MS_WINDOWS) - WIN32_FIND_DATA data; - HANDLE h; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - h = FindFirstFile(buf, &data); - if (h == INVALID_HANDLE_VALUE) { - PyErr_Format(PyExc_NameError, - "Can't find file for module %.100s\n(filename %.300s)", - name, buf); - return 0; - } - FindClose(h); - return strncmp(data.cFileName, name, namelen) == 0; - /* DJGPP */ -#elif defined(DJGPP) +#if defined(DJGPP) struct ffblk ffblk; int done; @@ -1921,7 +2210,7 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, char *name) PyErr_Format(PyExc_NameError, "Can't find file for module %.100s\n(filename %.300s)", name, buf); - return 0; + return -1; } return strncmp(ffblk.ff_name, name, namelen) == 0; @@ -1988,6 +2277,79 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, char *name) /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ #else +# error "USE_CASE_OK_BYTES is not correctly defined" +#endif +} +#endif + +/* + * Check if a filename case matchs the name case. We've already done a + * successful stat() or fopen() on buf, so know that there's some match, + * possibly case-insensitive. + * + * case_ok() is to return 1 if there's a case-sensitive match for name, 0 if it + * the filename doesn't match, or -1 on error. case_ok() is also to return 1 + * if envar PYTHONCASEOK exists. + * + * case_ok() is used to implement case-sensitive import semantics even + * on platforms with case-insensitive filesystems. It's trivial to implement + * for case-sensitive filesystems. It's pretty much a cross-platform + * nightmare for systems with case-insensitive filesystems. + */ + +static int +case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) +{ +#ifdef MS_WINDOWS + WIN32_FIND_DATAW data; + HANDLE h; + int cmp; + wchar_t *wfilename, *wname; + Py_ssize_t wname_len; + + if (Py_GETENV("PYTHONCASEOK") != NULL) + return 1; + + wfilename = PyUnicode_AsUnicode(filename); + if (wfilename == NULL) + return -1; + + h = FindFirstFileW(wfilename, &data); + if (h == INVALID_HANDLE_VALUE) { + PyErr_Format(PyExc_NameError, + "Can't find file for module %R\n(filename %R)", + name, filename); + return -1; + } + FindClose(h); + + wname = PyUnicode_AsUnicodeAndSize(name, &wname_len); + if (wname == NULL) + return -1; + + cmp = wcsncmp(data.cFileName, wname, wname_len); + return cmp == 0; +#elif defined(USE_CASE_OK_BYTES) + int match; + PyObject *filebytes, *namebytes; + filebytes = PyUnicode_EncodeFSDefault(filename); + if (filebytes == NULL) + return -1; + namebytes = PyUnicode_EncodeFSDefault(name); + if (namebytes == NULL) { + Py_DECREF(filebytes); + return -1; + } + match = case_bytes( + PyBytes_AS_STRING(filebytes), + PyBytes_GET_SIZE(filebytes) + prefix_delta, + PyBytes_GET_SIZE(namebytes), + PyBytes_AS_STRING(namebytes)); + Py_DECREF(filebytes); + Py_DECREF(namebytes); + return match; +#else + /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ return 1; #endif @@ -1995,89 +2357,93 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, char *name) #ifdef HAVE_STAT -/* Helper to look for __init__.py or __init__.py[co] in potential package */ +/* Helper to look for __init__.py or __init__.py[co] in potential package. + Return 1 if __init__ was found, 0 if not, or -1 on error. */ static int -find_init_module(char *buf) +find_init_module(PyObject *directory) { - const size_t save_len = strlen(buf); - size_t i = save_len; - char *pname; /* pointer to start of __init__ */ struct stat statbuf; + PyObject *filename; + int match; + int err; -/* For calling case_ok(buf, len, namelen, name): - * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 - * ^ ^ ^ ^ - * |--------------------- buf ---------------------| - * |------------------- len ------------------| - * |------ name -------| - * |----- namelen -----| - */ - if (save_len + 13 >= MAXPATHLEN) - return 0; - buf[i++] = SEP; - pname = buf + i; - strcpy(pname, "__init__.py"); - if (stat(buf, &statbuf) == 0) { - if (case_ok(buf, - save_len + 9, /* len("/__init__") */ - 8, /* len("__init__") */ - pname)) { - buf[save_len] = '\0'; + filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); + if (filename == NULL) + return -1; + err = _Py_stat(filename, &statbuf); + if (err == -2) + return -1; + if (err == 0) { + /* 3=len(".py") */ + match = case_ok(filename, -3, initstr); + if (match < 0) { + Py_DECREF(filename); + return -1; + } + if (match) { + Py_DECREF(filename); return 1; } } - i += strlen(pname); - strcpy(buf+i, Py_OptimizeFlag ? "o" : "c"); - if (stat(buf, &statbuf) == 0) { - if (case_ok(buf, - save_len + 9, /* len("/__init__") */ - 8, /* len("__init__") */ - pname)) { - buf[save_len] = '\0'; + Py_DECREF(filename); + + filename = PyUnicode_FromFormat("%U%c__init__.py%c", + directory, SEP, Py_OptimizeFlag ? 'o' : 'c'); + if (filename == NULL) + return -1; + err = _Py_stat(filename, &statbuf); + if (err == -2) { + Py_DECREF(filename); + return -1; + } + if (err == 0) { + /* 4=len(".pyc") */ + match = case_ok(filename, -4, initstr); + if (match < 0) { + Py_DECREF(filename); + return -1; + } + if (match) { + Py_DECREF(filename); return 1; } } - buf[save_len] = '\0'; + Py_DECREF(filename); return 0; } #endif /* HAVE_STAT */ -static int init_builtin(char *); /* Forward */ +static int init_builtin(PyObject *); /* Forward */ static PyObject* -load_builtin(char *name, char *pathname, int type) +load_builtin(PyObject *name, int type) { PyObject *m, *modules; int err; - if (pathname != NULL && pathname[0] != '\0') - name = pathname; - if (type == C_BUILTIN) err = init_builtin(name); else - err = PyImport_ImportFrozenModule(name); + err = PyImport_ImportFrozenModuleObject(name); if (err < 0) return NULL; if (err == 0) { PyErr_Format(PyExc_ImportError, - "Purported %s module %.200s not found", - type == C_BUILTIN ? - "builtin" : "frozen", + "Purported %s module %R not found", + type == C_BUILTIN ? "builtin" : "frozen", name); return NULL; } modules = PyImport_GetModuleDict(); - m = PyDict_GetItemString(modules, name); + m = PyDict_GetItem(modules, name); if (m == NULL) { PyErr_Format( PyExc_ImportError, - "%s module %.200s not properly initialized", - type == C_BUILTIN ? - "builtin" : "frozen", + "%s module %R not properly initialized", + type == C_BUILTIN ? "builtin" : "frozen", name); return NULL; } @@ -2089,7 +2455,7 @@ load_builtin(char *name, char *pathname, int type) its module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) +load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *loader) { PyObject *m; @@ -2099,7 +2465,7 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) case PY_COMPILED: if (fp == NULL) { PyErr_Format(PyExc_ValueError, - "file object required for import (type code %d)", + "file object required for import (type code %d)", type); return NULL; } @@ -2127,22 +2493,23 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) case C_BUILTIN: case PY_FROZEN: - m = load_builtin(name, pathname, type); + m = load_builtin(name, type); break; case IMP_HOOK: { + _Py_IDENTIFIER(load_module); if (loader == NULL) { PyErr_SetString(PyExc_ImportError, "import hook without loader"); return NULL; } - m = PyObject_CallMethod(loader, "load_module", "s", name); + m = _PyObject_CallMethodId(loader, &PyId_load_module, "O", name); break; } default: PyErr_Format(PyExc_ImportError, - "Don't know how to import %.200s (type code %d)", + "Don't know how to import %R (type code %d)", name, type); m = NULL; @@ -2157,32 +2524,32 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) an exception set if the initialization failed. */ static int -init_builtin(char *name) +init_builtin(PyObject *name) { struct _inittab *p; - if (_PyImport_FindBuiltin(name) != NULL) + if (_PyImport_FindExtensionObject(name, name) != NULL) return 1; for (p = PyImport_Inittab; p->name != NULL; p++) { PyObject *mod; PyModuleDef *def; - if (strcmp(name, p->name) == 0) { + if (PyUnicode_CompareWithASCIIString(name, p->name) == 0) { if (p->initfunc == NULL) { PyErr_Format(PyExc_ImportError, - "Cannot re-init internal module %.200s", + "Cannot re-init internal module %R", name); return -1; } if (Py_VerboseFlag) - PySys_WriteStderr("import %s # builtin\n", name); + PySys_FormatStderr("import %U # builtin\n", name); mod = (*p->initfunc)(); if (mod == 0) return -1; /* Remember pointer to module init function. */ def = PyModule_GetDef(mod); def->m_base.m_init = p->initfunc; - if (_PyImport_FixupBuiltin(mod, name) < 0) + if (_PyImport_FixupExtensionObject(mod, name, name) < 0) return -1; /* FixupExtension has put the module into sys.modules, so we can release our own reference. */ @@ -2197,37 +2564,37 @@ init_builtin(char *name) /* Frozen modules */ static struct _frozen * -find_frozen(char *name) +find_frozen(PyObject *name) { struct _frozen *p; - if (!name) + if (name == NULL) return NULL; for (p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) return NULL; - if (strcmp(p->name, name) == 0) + if (PyUnicode_CompareWithASCIIString(name, p->name) == 0) break; } return p; } static PyObject * -get_frozen_object(char *name) +get_frozen_object(PyObject *name) { struct _frozen *p = find_frozen(name); int size; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "No such frozen object named %.200s", + "No such frozen object named %R", name); return NULL; } if (p->code == NULL) { PyErr_Format(PyExc_ImportError, - "Excluded frozen object named %.200s", + "Excluded frozen object named %R", name); return NULL; } @@ -2238,14 +2605,14 @@ get_frozen_object(char *name) } static PyObject * -is_frozen_package(char *name) +is_frozen_package(PyObject *name) { struct _frozen *p = find_frozen(name); int size; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "No such frozen object named %.200s", + "No such frozen object named %R", name); return NULL; } @@ -2265,19 +2632,20 @@ is_frozen_package(char *name) This function is also used from frozenmain.c */ int -PyImport_ImportFrozenModule(char *name) +PyImport_ImportFrozenModuleObject(PyObject *name) { - struct _frozen *p = find_frozen(name); - PyObject *co; - PyObject *m; + struct _frozen *p; + PyObject *co, *m, *path; int ispackage; int size; + p = find_frozen(name); + if (p == NULL) return 0; if (p->code == NULL) { PyErr_Format(PyExc_ImportError, - "Excluded frozen object named %.200s", + "Excluded frozen object named %R", name); return -1; } @@ -2286,40 +2654,41 @@ PyImport_ImportFrozenModule(char *name) if (ispackage) size = -size; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # frozen%s\n", + PySys_FormatStderr("import %U # frozen%s\n", name, ispackage ? " package" : ""); co = PyMarshal_ReadObjectFromString((char *)p->code, size); if (co == NULL) return -1; if (!PyCode_Check(co)) { PyErr_Format(PyExc_TypeError, - "frozen object %.200s is not a code object", + "frozen object %R is not a code object", name); goto err_return; } if (ispackage) { /* Set __path__ to the package name */ - PyObject *d, *s, *l; + PyObject *d, *l; int err; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) goto err_return; d = PyModule_GetDict(m); - s = PyUnicode_InternFromString(name); - if (s == NULL) - goto err_return; l = PyList_New(1); if (l == NULL) { - Py_DECREF(s); goto err_return; } - PyList_SET_ITEM(l, 0, s); + Py_INCREF(name); + PyList_SET_ITEM(l, 0, name); err = PyDict_SetItemString(d, "__path__", l); Py_DECREF(l); if (err != 0) goto err_return; } - m = PyImport_ExecCodeModuleEx(name, co, "<frozen>"); + path = PyUnicode_FromString("<frozen>"); + if (path == NULL) + goto err_return; + m = PyImport_ExecCodeModuleObject(name, co, path, NULL); + Py_DECREF(path); if (m == NULL) goto err_return; Py_DECREF(co); @@ -2330,6 +2699,19 @@ err_return: return -1; } +int +PyImport_ImportFrozenModule(char *name) +{ + PyObject *nameobj; + int ret; + nameobj = PyUnicode_InternFromString(name); + if (nameobj == NULL) + return -1; + ret = PyImport_ImportFrozenModuleObject(nameobj); + Py_DECREF(nameobj); + return ret; +} + /* Import a module, either built-in, frozen, or external, and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -2360,8 +2742,7 @@ PyImport_ImportModule(const char *name) PyObject * PyImport_ImportModuleNoBlock(const char *name) { - PyObject *result; - PyObject *modules; + PyObject *nameobj, *modules, *result; #ifdef WITH_THREAD long me; #endif @@ -2371,14 +2752,16 @@ PyImport_ImportModuleNoBlock(const char *name) if (modules == NULL) return NULL; - result = PyDict_GetItemString(modules, name); + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + result = PyDict_GetItem(modules, nameobj); if (result != NULL) { + Py_DECREF(nameobj); Py_INCREF(result); return result; } - else { - PyErr_Clear(); - } + PyErr_Clear(); #ifdef WITH_THREAD /* check the import lock * me might be -1 but I ignore the error here, the lock function @@ -2386,79 +2769,106 @@ PyImport_ImportModuleNoBlock(const char *name) me = PyThread_get_thread_ident(); if (import_lock_thread == -1 || import_lock_thread == me) { /* no thread or me is holding the lock */ - return PyImport_ImportModule(name); + result = PyImport_Import(nameobj); } else { PyErr_Format(PyExc_ImportError, - "Failed to import %.200s because the import lock" + "Failed to import %R because the import lock" "is held by another thread.", - name); - return NULL; + nameobj); + result = NULL; } #else - return PyImport_ImportModule(name); + result = PyImport_Import(nameobj); #endif + Py_DECREF(nameobj); + return result; } /* Forward declarations for helper routines */ -static PyObject *get_parent(PyObject *globals, char *buf, - Py_ssize_t *p_buflen, int level); +static PyObject *get_parent(PyObject *globals, + PyObject **p_name, + int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, - char **p_name, char *buf, Py_ssize_t *p_buflen); -static int mark_miss(char *name); + PyObject *inputname, PyObject **p_outputname, + PyObject **p_prefix); +static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, - char *buf, Py_ssize_t buflen, int recursive); -static PyObject * import_submodule(PyObject *mod, char *name, char *fullname); + PyObject *buf, int recursive); +static PyObject * import_submodule(PyObject *mod, PyObject *name, + PyObject *fullname); /* The Magnum Opus of dotted-name import :-) */ static PyObject * -import_module_level(char *name, PyObject *globals, PyObject *locals, +import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - char buf[MAXPATHLEN+1]; - Py_ssize_t buflen = 0; - PyObject *parent, *head, *next, *tail; + PyObject *parent, *next, *inputname, *outputname; + PyObject *head = NULL; + PyObject *tail = NULL; + PyObject *prefix = NULL; + PyObject *result = NULL; + Py_ssize_t sep, altsep; + + if (PyUnicode_READY(name)) + return NULL; - if (strchr(name, '/') != NULL -#ifdef MS_WINDOWS - || strchr(name, '\\') != NULL + sep = PyUnicode_FindChar(name, SEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (sep == -2) + return NULL; +#ifdef ALTSEP + altsep = PyUnicode_FindChar(name, ALTSEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (altsep == -2) + return NULL; +#else + altsep = -1; #endif - ) { + if (sep != -1 || altsep != -1) + { PyErr_SetString(PyExc_ImportError, "Import by filename is not supported."); return NULL; } - parent = get_parent(globals, buf, &buflen, level); - if (parent == NULL) + parent = get_parent(globals, &prefix, level); + if (parent == NULL) { + return NULL; + } + + if (PyUnicode_READY(prefix)) return NULL; - head = load_next(parent, level < 0 ? Py_None : parent, &name, buf, - &buflen); + head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, + &prefix); if (head == NULL) - return NULL; + goto out; tail = head; Py_INCREF(tail); - while (name) { - next = load_next(tail, tail, &name, buf, &buflen); - Py_DECREF(tail); - if (next == NULL) { - Py_DECREF(head); - return NULL; + + if (outputname != NULL) { + while (1) { + inputname = outputname; + next = load_next(tail, tail, inputname, &outputname, + &prefix); + Py_CLEAR(tail); + Py_CLEAR(inputname); + if (next == NULL) + goto out; + tail = next; + + if (outputname == NULL) { + break; + } } - tail = next; } if (tail == Py_None) { /* If tail is Py_None, both get_parent and load_next found an empty module name: someone called __import__("") or doctored faulty bytecode */ - Py_DECREF(tail); - Py_DECREF(head); - PyErr_SetString(PyExc_ValueError, - "Empty module name"); - return NULL; + PyErr_SetString(PyExc_ValueError, "Empty module name"); + goto out; } if (fromlist != NULL) { @@ -2467,49 +2877,70 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, } if (fromlist == NULL) { - Py_DECREF(tail); - return head; + result = head; + Py_INCREF(result); + goto out; } - Py_DECREF(head); - if (!ensure_fromlist(tail, fromlist, buf, buflen, 0)) { - Py_DECREF(tail); - return NULL; - } + if (!ensure_fromlist(tail, fromlist, prefix, 0)) + goto out; - return tail; + result = tail; + Py_INCREF(result); + out: + Py_XDECREF(head); + Py_XDECREF(tail); + Py_XDECREF(prefix); + return result; } PyObject * -PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) +PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, + PyObject *locals, PyObject *fromlist, + int level) { - PyObject *result; + PyObject *mod; _PyImport_AcquireLock(); - result = import_module_level(name, globals, locals, fromlist, level); + mod = import_module_level(name, globals, locals, fromlist, level); if (_PyImport_ReleaseLock() < 0) { - Py_XDECREF(result); + Py_XDECREF(mod); PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; } - return result; + return mod; } +PyObject * +PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, + PyObject *fromlist, int level) +{ + PyObject *nameobj, *mod; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + mod = PyImport_ImportModuleLevelObject(nameobj, globals, locals, + fromlist, level); + Py_DECREF(nameobj); + return mod; +} + + /* Return the package that an import is being performed in. If globals comes from the module foo.bar.bat (not itself a package), this returns the sys.modules entry for foo.bar. If globals is from a package's __init__.py, the package's entry in sys.modules is returned, as a borrowed reference. - The *name* of the returned package is returned in buf, with the length of - the name in *p_buflen. + The name of the returned package is returned in *p_name. If globals doesn't come from a package or a module in a package, or a corresponding entry is not found in sys.modules, Py_None is returned. */ static PyObject * -get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) +get_parent(PyObject *globals, PyObject **p_name, int level) { + PyObject *nameobj; + static PyObject *namestr = NULL; static PyObject *pathstr = NULL; static PyObject *pkgstr = NULL; @@ -2517,7 +2948,7 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) int orig_level = level; if (globals == NULL || !PyDict_Check(globals) || !level) - return Py_None; + goto return_none; if (namestr == NULL) { namestr = PyUnicode_InternFromString("__name__"); @@ -2535,260 +2966,274 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) return NULL; } - *buf = '\0'; - *p_buflen = 0; pkgname = PyDict_GetItem(globals, pkgstr); if ((pkgname != NULL) && (pkgname != Py_None)) { /* __package__ is set, so use it */ - char *pkgname_str; - Py_ssize_t len; - if (!PyUnicode_Check(pkgname)) { PyErr_SetString(PyExc_ValueError, "__package__ set to non-string"); return NULL; } - pkgname_str = _PyUnicode_AsStringAndSize(pkgname, &len); - if (len == 0) { + if (PyUnicode_GET_LENGTH(pkgname) == 0) { if (level > 0) { PyErr_SetString(PyExc_ValueError, "Attempted relative import in non-package"); return NULL; } - return Py_None; + goto return_none; } - if (len > MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Package name too long"); - return NULL; - } - strcpy(buf, pkgname_str); + Py_INCREF(pkgname); + nameobj = pkgname; } else { /* __package__ not set, so figure it out and set it */ modname = PyDict_GetItem(globals, namestr); if (modname == NULL || !PyUnicode_Check(modname)) - return Py_None; + goto return_none; modpath = PyDict_GetItem(globals, pathstr); if (modpath != NULL) { /* __path__ is set, so modname is already the package name */ - char *modname_str; - Py_ssize_t len; int error; - modname_str = _PyUnicode_AsStringAndSize(modname, &len); - if (len > MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; - } - strcpy(buf, modname_str); error = PyDict_SetItem(globals, pkgstr, modname); if (error) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } + Py_INCREF(modname); + nameobj = modname; } else { /* Normal module, so work out the package name if any */ - char *start = _PyUnicode_AsString(modname); - char *lastdot = strrchr(start, '.'); - size_t len; - int error; - if (lastdot == NULL && level > 0) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import in non-package"); + Py_ssize_t len; + len = PyUnicode_FindChar(modname, '.', + 0, PyUnicode_GET_LENGTH(modname), -1); + if (len == -2) return NULL; - } - if (lastdot == NULL) { - error = PyDict_SetItem(globals, pkgstr, Py_None); - if (error) { + if (len < 0) { + if (level > 0) { + PyErr_SetString(PyExc_ValueError, + "Attempted relative import in non-package"); + return NULL; + } + if (PyDict_SetItem(globals, pkgstr, Py_None)) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } - return Py_None; + goto return_none; } - len = lastdot - start; - if (len >= MAXPATHLEN) { + pkgname = PyUnicode_Substring(modname, 0, len); + if (pkgname == NULL) + return NULL; + if (PyDict_SetItem(globals, pkgstr, pkgname)) { + Py_DECREF(pkgname); PyErr_SetString(PyExc_ValueError, - "Module name too long"); + "Could not set __package__"); return NULL; } - strncpy(buf, start, len); - buf[len] = '\0'; - pkgname = PyUnicode_FromString(buf); - if (pkgname == NULL) { + nameobj = pkgname; + } + } + if (level > 1) { + Py_ssize_t dot, end = PyUnicode_GET_LENGTH(nameobj); + PyObject *newname; + while (--level > 0) { + dot = PyUnicode_FindChar(nameobj, '.', 0, end, -1); + if (dot == -2) { + Py_DECREF(nameobj); return NULL; } - error = PyDict_SetItem(globals, pkgstr, pkgname); - Py_DECREF(pkgname); - if (error) { + if (dot < 0) { + Py_DECREF(nameobj); PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); + "Attempted relative import beyond " + "toplevel package"); return NULL; } + end = dot; } - } - while (--level > 0) { - char *dot = strrchr(buf, '.'); - if (dot == NULL) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import beyond " - "toplevel package"); + newname = PyUnicode_Substring(nameobj, 0, end); + Py_DECREF(nameobj); + if (newname == NULL) return NULL; - } - *dot = '\0'; + nameobj = newname; } - *p_buflen = strlen(buf); modules = PyImport_GetModuleDict(); - parent = PyDict_GetItemString(modules, buf); + parent = PyDict_GetItem(modules, nameobj); if (parent == NULL) { - if (orig_level < 1) { - PyObject *err_msg = PyBytes_FromFormat( - "Parent module '%.200s' not found " - "while handling absolute import", buf); - if (err_msg == NULL) { - return NULL; - } - if (!PyErr_WarnEx(PyExc_RuntimeWarning, - PyBytes_AsString(err_msg), 1)) { - *buf = '\0'; - *p_buflen = 0; - parent = Py_None; - } - Py_DECREF(err_msg); - } else { + int err; + + if (orig_level >= 1) { PyErr_Format(PyExc_SystemError, - "Parent module '%.200s' not loaded, " - "cannot perform relative import", buf); + "Parent module %R not loaded, " + "cannot perform relative import", nameobj); + Py_DECREF(nameobj); + return NULL; } + + err = PyErr_WarnFormat( + PyExc_RuntimeWarning, 1, + "Parent module %R not found while handling absolute import", + nameobj); + Py_DECREF(nameobj); + if (err) + return NULL; + + goto return_none; } + *p_name = nameobj; return parent; /* We expect, but can't guarantee, if parent != None, that: - - parent.__name__ == buf + - parent.__name__ == name - parent.__dict__ is globals If this is violated... Who cares? */ + +return_none: + nameobj = PyUnicode_New(0, 0); + if (nameobj == NULL) + return NULL; + *p_name = nameobj; + return Py_None; } /* altmod is either None or same as mod */ static PyObject * -load_next(PyObject *mod, PyObject *altmod, char **p_name, char *buf, - Py_ssize_t *p_buflen) +load_next(PyObject *mod, PyObject *altmod, + PyObject *inputname, PyObject **p_outputname, + PyObject **p_prefix) { - char *name = *p_name; - char *dot = strchr(name, '.'); - size_t len; - char *p; - PyObject *result; + Py_ssize_t dot; + Py_ssize_t len; + PyObject *fullname, *name = NULL, *result; + + *p_outputname = NULL; - if (strlen(name) == 0) { + len = PyUnicode_GET_LENGTH(inputname); + if (len == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); - *p_name = NULL; return mod; } - if (dot == NULL) { - *p_name = NULL; - len = strlen(name); - } - else { - *p_name = dot+1; - len = dot-name; - } - if (len == 0) { - PyErr_SetString(PyExc_ValueError, - "Empty module name"); - return NULL; + + dot = PyUnicode_FindChar(inputname, '.', 0, len, 1); + if (dot >= 0) { + len = dot; + if (len == 0) { + PyErr_SetString(PyExc_ValueError, + "Empty module name"); + goto error; + } } - p = buf + *p_buflen; - if (p != buf) - *p++ = '.'; - if (p+len-buf >= MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; + /* name = inputname[:len] */ + name = PyUnicode_Substring(inputname, 0, len); + if (name == NULL) + goto error; + + if (PyUnicode_GET_LENGTH(*p_prefix)) { + /* fullname = prefix + "." + name */ + fullname = PyUnicode_FromFormat("%U.%U", *p_prefix, name); + if (fullname == NULL) + goto error; + } + else { + fullname = name; + Py_INCREF(fullname); } - strncpy(p, name, len); - p[len] = '\0'; - *p_buflen = p+len-buf; - result = import_submodule(mod, p, buf); + result = import_submodule(mod, name, fullname); + Py_DECREF(*p_prefix); + /* Transfer reference. */ + *p_prefix = fullname; if (result == Py_None && altmod != mod) { Py_DECREF(result); /* Here, altmod must be None and mod must not be None */ - result = import_submodule(altmod, p, p); + result = import_submodule(altmod, name, name); if (result != NULL && result != Py_None) { - if (mark_miss(buf) != 0) { + if (mark_miss(*p_prefix) != 0) { Py_DECREF(result); - return NULL; + goto error; } - strncpy(buf, name, len); - buf[len] = '\0'; - *p_buflen = len; + Py_DECREF(*p_prefix); + *p_prefix = name; + Py_INCREF(*p_prefix); } } if (result == NULL) - return NULL; + goto error; if (result == Py_None) { Py_DECREF(result); PyErr_Format(PyExc_ImportError, - "No module named %.200s", name); - return NULL; + "No module named %R", inputname); + goto error; + } + + if (dot >= 0) { + *p_outputname = PyUnicode_Substring(inputname, dot+1, + PyUnicode_GET_LENGTH(inputname)); + if (*p_outputname == NULL) { + Py_DECREF(result); + goto error; + } } + Py_DECREF(name); return result; + +error: + Py_XDECREF(name); + return NULL; } static int -mark_miss(char *name) +mark_miss(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - return PyDict_SetItemString(modules, name, Py_None); + return PyDict_SetItem(modules, name, Py_None); } static int -ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, +ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, int recursive) { int i; + PyObject *fullname; + Py_ssize_t fromlist_len; - if (!PyObject_HasAttrString(mod, "__path__")) + if (!_PyObject_HasAttrId(mod, &PyId___path__)) return 1; - for (i = 0; ; i++) { + fromlist_len = PySequence_Size(fromlist); + + for (i = 0; i < fromlist_len; i++) { PyObject *item = PySequence_GetItem(fromlist, i); int hasit; - if (item == NULL) { - if (PyErr_ExceptionMatches(PyExc_IndexError)) { - PyErr_Clear(); - return 1; - } + if (item == NULL) return 0; - } if (!PyUnicode_Check(item)) { PyErr_SetString(PyExc_TypeError, "Item in ``from list'' not a string"); Py_DECREF(item); return 0; } - if (PyUnicode_AS_UNICODE(item)[0] == '*') { + if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; + _Py_IDENTIFIER(__all__); Py_DECREF(item); /* See if the package defines __all__ */ if (recursive) continue; /* Avoid endless recursion */ - all = PyObject_GetAttrString(mod, "__all__"); + all = _PyObject_GetAttrId(mod, &PyId___all__); if (all == NULL) PyErr_Clear(); else { - int ret = ensure_fromlist(mod, all, buf, buflen, 1); + int ret = ensure_fromlist(mod, all, name, 1); Py_DECREF(all); if (!ret) return 0; @@ -2797,27 +3242,14 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, } hasit = PyObject_HasAttr(mod, item); if (!hasit) { - PyObject *item8; - char *subname; PyObject *submod; - char *p; - item8 = PyUnicode_EncodeFSDefault(item); - if (!item8) { - PyErr_SetString(PyExc_ValueError, "Cannot encode path item"); - return 0; - } - subname = PyBytes_AS_STRING(item8); - if (buflen + strlen(subname) >= MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - Py_DECREF(item); - return 0; + fullname = PyUnicode_FromFormat("%U.%U", name, item); + if (fullname != NULL) { + submod = import_submodule(mod, item, fullname); + Py_DECREF(fullname); } - p = buf + buflen; - *p++ = '.'; - strcpy(p, subname); - submod = import_submodule(mod, subname, buf); - Py_DECREF(item8); + else + submod = NULL; Py_XDECREF(submod); if (submod == NULL) { Py_DECREF(item); @@ -2827,12 +3259,12 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, Py_DECREF(item); } - /* NOTREACHED */ + return 1; } static int -add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, - PyObject *modules) +add_submodule(PyObject *mod, PyObject *submod, PyObject *fullname, + PyObject *subname, PyObject *modules) { if (mod == Py_None) return 1; @@ -2843,7 +3275,7 @@ add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, load failed with a SyntaxError -- then there's no trace in sys.modules. In that case, of course, do nothing extra.) */ if (submod == NULL) { - submod = PyDict_GetItemString(modules, fullname); + submod = PyDict_GetItem(modules, fullname); if (submod == NULL) return 1; } @@ -2854,68 +3286,66 @@ add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, PyObject *dict = PyModule_GetDict(mod); if (!dict) return 0; - if (PyDict_SetItemString(dict, subname, submod) < 0) + if (PyDict_SetItem(dict, subname, submod) < 0) return 0; } else { - if (PyObject_SetAttrString(mod, subname, submod) < 0) + if (PyObject_SetAttr(mod, subname, submod) < 0) return 0; } return 1; } static PyObject * -import_submodule(PyObject *mod, char *subname, char *fullname) +import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL; + PyObject *m = NULL, *bufobj, *path_list, *loader; + struct filedescr *fdp; + FILE *fp; /* Require: if mod == None: subname == fullname else: mod.__name__ + "." + subname == fullname */ - if ((m = PyDict_GetItemString(modules, fullname)) != NULL) { + if ((m = PyDict_GetItem(modules, fullname)) != NULL) { Py_INCREF(m); + return m; } - else { - PyObject *path, *loader = NULL; - char buf[MAXPATHLEN+1]; - struct filedescr *fdp; - FILE *fp = NULL; - if (mod == Py_None) - path = NULL; - else { - path = PyObject_GetAttrString(mod, "__path__"); - if (path == NULL) { - PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - } - - buf[0] = '\0'; - fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1, - &fp, &loader); - Py_XDECREF(path); - if (fdp == NULL) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - return NULL; + if (mod == Py_None) + path_list = NULL; + else { + path_list = _PyObject_GetAttrId(mod, &PyId___path__); + if (path_list == NULL) { PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } - m = load_module(fullname, fp, buf, fdp->type, loader); - Py_XDECREF(loader); - if (fp) - fclose(fp); - if (!add_submodule(mod, m, fullname, subname, modules)) { - Py_XDECREF(m); - m = NULL; - } } + fdp = find_module(fullname, subname, path_list, + &bufobj, &fp, &loader); + Py_XDECREF(path_list); + if (fdp == NULL) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + return NULL; + PyErr_Clear(); + Py_INCREF(Py_None); + return Py_None; + } + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_XDECREF(bufobj); + Py_XDECREF(loader); + if (fp) + fclose(fp); + if (m == NULL) + return NULL; + if (!add_submodule(mod, m, fullname, subname, modules)) { + Py_XDECREF(m); + return NULL; + } return m; } @@ -2929,12 +3359,12 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - PyObject *path = NULL, *loader = NULL, *existing_m = NULL; - char *name, *subname; - char buf[MAXPATHLEN+1]; + PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; + PyObject *name, *bufobj, *subname; + Py_ssize_t subname_start; struct filedescr *fdp; FILE *fp = NULL; - PyObject *newm; + PyObject *newm = NULL; if (modules_reloading == NULL) { Py_FatalError("PyImport_ReloadModule: " @@ -2947,61 +3377,72 @@ PyImport_ReloadModule(PyObject *m) "reload() argument must be module"); return NULL; } - name = (char*)PyModule_GetName(m); - if (name == NULL) + name = PyModule_GetNameObject(m); + if (name == NULL || PyUnicode_READY(name) == -1) return NULL; - if (m != PyDict_GetItemString(modules, name)) { + if (m != PyDict_GetItem(modules, name)) { PyErr_Format(PyExc_ImportError, - "reload(): module %.200s not in sys.modules", + "reload(): module %R not in sys.modules", name); + Py_DECREF(name); return NULL; } - existing_m = PyDict_GetItemString(modules_reloading, name); + existing_m = PyDict_GetItem(modules_reloading, name); if (existing_m != NULL) { /* Due to a recursive reload, this module is already being reloaded. */ + Py_DECREF(name); Py_INCREF(existing_m); return existing_m; } - if (PyDict_SetItemString(modules_reloading, name, m) < 0) + if (PyDict_SetItem(modules_reloading, name, m) < 0) { + Py_DECREF(name); return NULL; + } - subname = strrchr(name, '.'); - if (subname == NULL) + subname_start = PyUnicode_FindChar(name, '.', 0, + PyUnicode_GET_LENGTH(name), -1); + if (subname_start == -1) { + Py_INCREF(name); subname = name; + } else { PyObject *parentname, *parent; - parentname = PyUnicode_FromStringAndSize(name, (subname-name)); + Py_ssize_t len; + parentname = PyUnicode_Substring(name, 0, subname_start); if (parentname == NULL) { - imp_modules_reloading_clear(); - return NULL; + goto error; } parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %U not in sys.modules", + "reload(): parent %R not in sys.modules", parentname); Py_DECREF(parentname); - imp_modules_reloading_clear(); - return NULL; + goto error; } Py_DECREF(parentname); - subname++; - path = PyObject_GetAttrString(parent, "__path__"); - if (path == NULL) + path_list = _PyObject_GetAttrId(parent, &PyId___path__); + if (path_list == NULL) PyErr_Clear(); + subname_start++; + len = PyUnicode_GET_LENGTH(name) - (subname_start + 1); + subname = PyUnicode_Substring(name, subname_start, len); } - buf[0] = '\0'; - fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader); - Py_XDECREF(path); + if (subname == NULL) + goto error; + fdp = find_module(name, subname, path_list, + &bufobj, &fp, &loader); + Py_DECREF(subname); + Py_XDECREF(path_list); if (fdp == NULL) { Py_XDECREF(loader); - imp_modules_reloading_clear(); - return NULL; + goto error; } - newm = load_module(name, fp, buf, fdp->type, loader); + newm = load_module(name, fp, bufobj, fdp->type, loader); + Py_XDECREF(bufobj); Py_XDECREF(loader); if (fp) @@ -3012,9 +3453,12 @@ PyImport_ReloadModule(PyObject *m) * going to return NULL in this case regardless of whether * replacing name succeeds, so the return value is ignored. */ - PyDict_SetItemString(modules, name, m); + PyDict_SetItem(modules, name, m); } + +error: imp_modules_reloading_clear(); + Py_DECREF(name); return newm; } @@ -3162,22 +3606,21 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) } static PyObject * -call_find_module(char *name, PyObject *path) +call_find_module(PyObject *name, PyObject *path_list) { extern int fclose(FILE *); PyObject *fob, *ret; PyObject *pathobj; struct filedescr *fdp; - char pathname[MAXPATHLEN+1]; - FILE *fp = NULL; + FILE *fp; int fd = -1; char *found_encoding = NULL; char *encoding = NULL; - pathname[0] = '\0'; - if (path == Py_None) - path = NULL; - fdp = find_module(NULL, name, path, pathname, MAXPATHLEN+1, &fp, NULL); + if (path_list == Py_None) + path_list = NULL; + fdp = find_module(NULL, name, path_list, + &pathobj, &fp, NULL); if (fdp == NULL) return NULL; if (fp != NULL) { @@ -3185,22 +3628,29 @@ call_find_module(char *name, PyObject *path) if (fd != -1) fd = dup(fd); fclose(fp); + if (fd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } fp = NULL; } if (fd != -1) { if (strchr(fdp->mode, 'b') == NULL) { - /* PyTokenizer_FindEncoding() returns PyMem_MALLOC'ed + /* PyTokenizer_FindEncodingFilename() returns PyMem_MALLOC'ed memory. */ - found_encoding = PyTokenizer_FindEncoding(fd); + found_encoding = PyTokenizer_FindEncodingFilename(fd, pathobj); lseek(fd, 0, 0); /* Reset position */ - if (found_encoding == NULL && PyErr_Occurred()) + if (found_encoding == NULL && PyErr_Occurred()) { + Py_XDECREF(pathobj); return NULL; + } encoding = (found_encoding != NULL) ? found_encoding : (char*)PyUnicode_GetDefaultEncoding(); } - fob = PyFile_FromFd(fd, pathname, fdp->mode, -1, + fob = PyFile_FromFd(fd, NULL, fdp->mode, -1, (char*)encoding, NULL, NULL, 1); if (fob == NULL) { + Py_XDECREF(pathobj); close(fd); PyMem_FREE(found_encoding); return NULL; @@ -3210,7 +3660,10 @@ call_find_module(char *name, PyObject *path) fob = Py_None; Py_INCREF(fob); } - pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_INCREF(Py_None); + pathobj = Py_None; + } ret = Py_BuildValue("NN(ssi)", fob, pathobj, fdp->suffix, fdp->mode, fdp->type); PyMem_FREE(found_encoding); @@ -3221,24 +3674,20 @@ call_find_module(char *name, PyObject *path) static PyObject * imp_find_module(PyObject *self, PyObject *args) { - PyObject *name; - PyObject *ret, *path = NULL; - if (!PyArg_ParseTuple(args, "O&|O:find_module", - PyUnicode_FSConverter, &name, - &path)) + PyObject *name, *path_list = NULL; + if (!PyArg_ParseTuple(args, "U|O:find_module", + &name, &path_list)) return NULL; - ret = call_find_module(PyBytes_AS_STRING(name), path); - Py_DECREF(name); - return ret; + return call_find_module(name, path_list); } static PyObject * imp_init_builtin(PyObject *self, PyObject *args) { - char *name; + PyObject *name; int ret; PyObject *m; - if (!PyArg_ParseTuple(args, "s:init_builtin", &name)) + if (!PyArg_ParseTuple(args, "U:init_builtin", &name)) return NULL; ret = init_builtin(name); if (ret < 0) @@ -3247,7 +3696,7 @@ imp_init_builtin(PyObject *self, PyObject *args) Py_INCREF(Py_None); return Py_None; } - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); Py_XINCREF(m); return m; } @@ -3255,19 +3704,19 @@ imp_init_builtin(PyObject *self, PyObject *args) static PyObject * imp_init_frozen(PyObject *self, PyObject *args) { - char *name; + PyObject *name; int ret; PyObject *m; - if (!PyArg_ParseTuple(args, "s:init_frozen", &name)) + if (!PyArg_ParseTuple(args, "U:init_frozen", &name)) return NULL; - ret = PyImport_ImportFrozenModule(name); + ret = PyImport_ImportFrozenModuleObject(name); if (ret < 0) return NULL; if (ret == 0) { Py_INCREF(Py_None); return Py_None; } - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); Py_XINCREF(m); return m; } @@ -3275,9 +3724,9 @@ imp_init_frozen(PyObject *self, PyObject *args) static PyObject * imp_get_frozen_object(PyObject *self, PyObject *args) { - char *name; + PyObject *name; - if (!PyArg_ParseTuple(args, "s:get_frozen_object", &name)) + if (!PyArg_ParseTuple(args, "U:get_frozen_object", &name)) return NULL; return get_frozen_object(name); } @@ -3285,9 +3734,9 @@ imp_get_frozen_object(PyObject *self, PyObject *args) static PyObject * imp_is_frozen_package(PyObject *self, PyObject *args) { - char *name; + PyObject *name; - if (!PyArg_ParseTuple(args, "s:is_frozen_package", &name)) + if (!PyArg_ParseTuple(args, "U:is_frozen_package", &name)) return NULL; return is_frozen_package(name); } @@ -3295,8 +3744,8 @@ imp_is_frozen_package(PyObject *self, PyObject *args) static PyObject * imp_is_builtin(PyObject *self, PyObject *args) { - char *name; - if (!PyArg_ParseTuple(args, "s:is_builtin", &name)) + PyObject *name; + if (!PyArg_ParseTuple(args, "U:is_builtin", &name)) return NULL; return PyLong_FromLong(is_builtin(name)); } @@ -3304,63 +3753,74 @@ imp_is_builtin(PyObject *self, PyObject *args) static PyObject * imp_is_frozen(PyObject *self, PyObject *args) { - char *name; + PyObject *name; struct _frozen *p; - if (!PyArg_ParseTuple(args, "s:is_frozen", &name)) + if (!PyArg_ParseTuple(args, "U:is_frozen", &name)) return NULL; p = find_frozen(name); return PyBool_FromLong((long) (p == NULL ? 0 : p->size)); } static FILE * -get_file(char *pathname, PyObject *fob, char *mode) +get_file(PyObject *pathname, PyObject *fob, char *mode) { FILE *fp; if (mode[0] == 'U') mode = "r" PY_STDIOTEXTMODE; if (fob == NULL) { - fp = fopen(pathname, mode); + fp = _Py_fopen(pathname, mode); + if (!fp) { + if (!PyErr_Occurred()) + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } + return fp; } else { int fd = PyObject_AsFileDescriptor(fob); if (fd == -1) return NULL; - if (!_PyVerify_fd(fd)) - goto error; + if (!_PyVerify_fd(fd)) { + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } + /* the FILE struct gets a new fd, so that it can be closed * independently of the file descriptor given */ fd = dup(fd); - if (fd == -1) - goto error; + if (fd == -1) { + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } + fp = fdopen(fd, mode); - } - if (fp) + if (!fp) { + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } return fp; -error: - PyErr_SetFromErrno(PyExc_IOError); - return NULL; + } } static PyObject * imp_load_compiled(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "sO&|O:load_compiled", + if (!PyArg_ParseTuple(args, "UO&|O:load_compiled", &name, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - fp = get_file(PyBytes_AS_STRING(pathname), fob, "rb"); + fp = get_file(pathname, fob, "rb"); if (fp == NULL) { Py_DECREF(pathname); return NULL; } - m = load_compiled_module(name, PyBytes_AS_STRING(pathname), fp); + m = load_compiled_module(name, pathname, fp); fclose(fp); Py_DECREF(pathname); return m; @@ -3371,28 +3831,26 @@ imp_load_compiled(PyObject *self, PyObject *args) static PyObject * imp_load_dynamic(PyObject *self, PyObject *args) { - char *name; - PyObject *pathbytes; - char *pathname; - PyObject *fob = NULL; - PyObject *m; - FILE *fp = NULL; - if (!PyArg_ParseTuple(args, "sO&|O:load_dynamic", - &name, PyUnicode_FSConverter, &pathbytes, &fob)) + PyObject *name, *pathname, *fob = NULL, *mod; + FILE *fp; + + if (!PyArg_ParseTuple(args, "UO&|O:load_dynamic", + &name, PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - pathname = PyBytes_AS_STRING(pathbytes); - if (fob) { - fp = get_file(pathname, fob, "r"); + if (fob != NULL) { + fp = get_file(NULL, fob, "r"); if (fp == NULL) { - Py_DECREF(pathbytes); + Py_DECREF(pathname); return NULL; } } - m = _PyImport_LoadDynamicModule(name, pathname, fp); - Py_DECREF(pathbytes); + else + fp = NULL; + mod = _PyImport_LoadDynamicModule(name, pathname, fp); + Py_DECREF(pathname); if (fp) fclose(fp); - return m; + return mod; } #endif /* HAVE_DYNAMIC_LOADING */ @@ -3400,22 +3858,21 @@ imp_load_dynamic(PyObject *self, PyObject *args) static PyObject * imp_load_source(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "sO&|O:load_source", + if (!PyArg_ParseTuple(args, "UO&|O:load_source", &name, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - fp = get_file(PyBytes_AS_STRING(pathname), fob, "r"); + fp = get_file(pathname, fob, "r"); if (fp == NULL) { Py_DECREF(pathname); return NULL; } - m = load_source_module(name, PyBytes_AS_STRING(pathname), fp); + m = load_source_module(name, pathname, fp); Py_DECREF(pathname); fclose(fp); return m; @@ -3424,20 +3881,22 @@ imp_load_source(PyObject *self, PyObject *args) static PyObject * imp_load_module(PyObject *self, PyObject *args) { - char *name; - PyObject *fob; - PyObject *pathname; - PyObject * ret; + PyObject *name, *fob, *pathname, *pathname_obj, *ret; char *suffix; /* Unused */ char *mode; int type; FILE *fp; - if (!PyArg_ParseTuple(args, "sOO&(ssi):load_module", - &name, &fob, - PyUnicode_FSConverter, &pathname, - &suffix, &mode, &type)) + if (!PyArg_ParseTuple(args, "UOO(ssi):load_module", + &name, &fob, &pathname_obj, &suffix, &mode, &type)) return NULL; + if (pathname_obj != Py_None) { + if (!PyUnicode_FSDecoder(pathname_obj, &pathname)) + return NULL; + } + else + pathname = NULL; + if (*mode) { /* Mode must start with 'r' or 'U' and must not contain '+'. Implicit in this test is the assumption that the mode @@ -3446,7 +3905,7 @@ imp_load_module(PyObject *self, PyObject *args) if (!(*mode == 'r' || *mode == 'U') || strchr(mode, '+')) { PyErr_Format(PyExc_ValueError, "invalid file open mode %.200s", mode); - Py_DECREF(pathname); + Py_XDECREF(pathname); return NULL; } } @@ -3455,12 +3914,12 @@ imp_load_module(PyObject *self, PyObject *args) else { fp = get_file(NULL, fob, mode); if (fp == NULL) { - Py_DECREF(pathname); + Py_XDECREF(pathname); return NULL; } } - ret = load_module(name, fp, PyBytes_AS_STRING(pathname), type, NULL); - Py_DECREF(pathname); + ret = load_module(name, fp, pathname, type, NULL); + Py_XDECREF(pathname); if (fp) fclose(fp); return ret; @@ -3469,13 +3928,12 @@ imp_load_module(PyObject *self, PyObject *args) static PyObject * imp_load_package(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject * ret; - if (!PyArg_ParseTuple(args, "sO&:load_package", - &name, PyUnicode_FSConverter, &pathname)) + if (!PyArg_ParseTuple(args, "UO&:load_package", + &name, PyUnicode_FSDecoder, &pathname)) return NULL; - ret = load_package(name, PyBytes_AS_STRING(pathname)); + ret = load_package(name, pathname); Py_DECREF(pathname); return ret; } @@ -3483,10 +3941,10 @@ imp_load_package(PyObject *self, PyObject *args) static PyObject * imp_new_module(PyObject *self, PyObject *args) { - char *name; - if (!PyArg_ParseTuple(args, "s:new_module", &name)) + PyObject *name; + if (!PyArg_ParseTuple(args, "U:new_module", &name)) return NULL; - return PyModule_New(name); + return PyModule_NewObject(name); } static PyObject * @@ -3505,33 +3963,32 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) { static char *kwlist[] = {"path", "debug_override", NULL}; - char buf[MAXPATHLEN+1]; - PyObject *pathbytes; - char *cpathname; + PyObject *pathname, *cpathname; PyObject *debug_override = NULL; int debug = !Py_OptimizeFlag; if (!PyArg_ParseTupleAndKeywords( args, kws, "O&|O", kwlist, - PyUnicode_FSConverter, &pathbytes, &debug_override)) + PyUnicode_FSDecoder, &pathname, &debug_override)) return NULL; if (debug_override != NULL && (debug = PyObject_IsTrue(debug_override)) < 0) { - Py_DECREF(pathbytes); + Py_DECREF(pathname); return NULL; } - cpathname = make_compiled_pathname( - PyBytes_AS_STRING(pathbytes), - buf, MAXPATHLEN+1, debug); - Py_DECREF(pathbytes); + if (PyUnicode_READY(pathname) < 0) + return NULL; + + cpathname = make_compiled_pathname(pathname, debug); + Py_DECREF(pathname); if (cpathname == NULL) { PyErr_Format(PyExc_SystemError, "path buffer too short"); return NULL; } - return PyUnicode_DecodeFSDefault(buf); + return cpathname; } PyDoc_STRVAR(doc_cache_from_source, @@ -3549,25 +4006,22 @@ static PyObject * imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) { static char *kwlist[] = {"path", NULL}; - - PyObject *pathname_obj; - char *pathname; - char buf[MAXPATHLEN+1]; + PyObject *pathname, *source; if (!PyArg_ParseTupleAndKeywords( args, kws, "O&", kwlist, - PyUnicode_FSConverter, &pathname_obj)) + PyUnicode_FSDecoder, &pathname)) return NULL; - pathname = PyBytes_AS_STRING(pathname_obj); - if (make_source_pathname(pathname, buf) == NULL) { - PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %s", + source = make_source_pathname(pathname); + if (source == NULL) { + PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %R", pathname); - Py_DECREF(pathname_obj); + Py_DECREF(pathname); return NULL; } - Py_DECREF(pathname_obj); - return PyUnicode_FromString(buf); + Py_DECREF(pathname); + return source; } PyDoc_STRVAR(doc_source_from_cache, @@ -3659,6 +4113,7 @@ static PyMethodDef imp_methods[] = { #endif {"load_package", imp_load_package, METH_VARARGS}, {"load_source", imp_load_source, METH_VARARGS}, + {"_fix_co_filename", imp_fix_co_filename, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; @@ -3721,7 +4176,7 @@ NullImporter_init(NullImporter *self, PyObject *args, PyObject *kwds) &pathobj)) return -1; - if (PyUnicode_GET_SIZE(pathobj) == 0) { + if (PyUnicode_GET_LENGTH(pathobj) == 0) { PyErr_SetString(PyExc_ImportError, "empty pathname"); return -1; } diff --git a/Python/importdl.c b/Python/importdl.c index 9caed45..9127d61 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -12,49 +12,71 @@ #include "importdl.h" -extern dl_funcptr _PyImport_GetDynLoadFunc(const char *name, - const char *shortname, +#ifdef MS_WINDOWS +extern dl_funcptr _PyImport_GetDynLoadWindows(const char *shortname, + PyObject *pathname, FILE *fp); +#else +extern dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname, const char *pathname, FILE *fp); - - +#endif PyObject * -_PyImport_LoadDynamicModule(char *name, char *pathname, FILE *fp) +_PyImport_LoadDynamicModule(PyObject *name, PyObject *path, FILE *fp) { - PyObject *m; - PyObject *path; - char *lastdot, *shortname, *packagecontext, *oldcontext; + PyObject *m = NULL; +#ifndef MS_WINDOWS + PyObject *pathbytes; +#endif + PyObject *nameascii; + char *namestr, *lastdot, *shortname, *packagecontext, *oldcontext; dl_funcptr p0; PyObject* (*p)(void); struct PyModuleDef *def; - PyObject *result; - path = PyUnicode_DecodeFSDefault(pathname); - if (path == NULL) - return NULL; - - if ((m = _PyImport_FindExtensionUnicode(name, path)) != NULL) { + m = _PyImport_FindExtensionObject(name, path); + if (m != NULL) { Py_INCREF(m); - result = m; - goto finally; + return m; } - lastdot = strrchr(name, '.'); + + /* name must be encodable to ASCII because dynamic module must have a + function called "PyInit_NAME", they are written in C, and the C language + doesn't accept non-ASCII identifiers. */ + nameascii = PyUnicode_AsEncodedString(name, "ascii", NULL); + if (nameascii == NULL) + return NULL; + + namestr = PyBytes_AS_STRING(nameascii); + if (namestr == NULL) + goto error; + + lastdot = strrchr(namestr, '.'); if (lastdot == NULL) { packagecontext = NULL; - shortname = name; + shortname = namestr; } else { - packagecontext = name; + packagecontext = namestr; shortname = lastdot+1; } - p0 = _PyImport_GetDynLoadFunc(name, shortname, pathname, fp); +#ifdef MS_WINDOWS + p0 = _PyImport_GetDynLoadWindows(shortname, path, fp); +#else + pathbytes = PyUnicode_EncodeFSDefault(path); + if (pathbytes == NULL) + goto error; + p0 = _PyImport_GetDynLoadFunc(shortname, + PyBytes_AS_STRING(pathbytes), fp); + Py_DECREF(pathbytes); +#endif p = (PyObject*(*)(void))p0; if (PyErr_Occurred()) goto error; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "dynamic module does not define init function (PyInit_%.200s)", + "dynamic module does not define init function" + " (PyInit_%s)", shortname); goto error; } @@ -66,7 +88,6 @@ _PyImport_LoadDynamicModule(char *name, char *pathname, FILE *fp) goto error; if (PyErr_Occurred()) { - Py_DECREF(m); PyErr_Format(PyExc_SystemError, "initialization of %s raised unreported exception", shortname); @@ -83,20 +104,19 @@ _PyImport_LoadDynamicModule(char *name, char *pathname, FILE *fp) else Py_INCREF(path); - if (_PyImport_FixupExtensionUnicode(m, name, path) < 0) + if (_PyImport_FixupExtensionObject(m, name, path) < 0) goto error; if (Py_VerboseFlag) - PySys_WriteStderr( - "import %s # dynamically loaded from %s\n", - name, pathname); - result = m; - goto finally; + PySys_FormatStderr( + "import %U # dynamically loaded from %R\n", + name, path); + Py_DECREF(nameascii); + return m; error: - result = NULL; -finally: - Py_DECREF(path); - return result; + Py_DECREF(nameascii); + Py_XDECREF(m); + return NULL; } #endif /* HAVE_DYNAMIC_LOADING */ diff --git a/Python/importdl.h b/Python/importdl.h index b4d21be..e51e551 100644 --- a/Python/importdl.h +++ b/Python/importdl.h @@ -28,7 +28,7 @@ struct filedescr { extern struct filedescr * _PyImport_Filetab; extern const struct filedescr _PyImport_DynLoadFiletab[]; -extern PyObject *_PyImport_LoadDynamicModule(char *name, char *pathname, +extern PyObject *_PyImport_LoadDynamicModule(PyObject *name, PyObject *pathname, FILE *); /* Max length of module suffix searched for -- accommodates "module.slb" */ diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 5d8e5a9..d9a0855 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -23,9 +23,6 @@ def write_contents(f): opcode = find_module("opcode") targets = ['_unknown_opcode'] * 256 for opname, op in opcode.opmap.items(): - if opname == "STOP_CODE": - # XXX opcode not implemented - continue targets[op] = "TARGET_%s" % opname f.write("static void *opcode_targets[256] = {\n") f.write(",\n".join([" &&%s" % s for s in targets])) diff --git a/Python/marshal.c b/Python/marshal.c index 094f732..98bbbaf 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -59,9 +59,9 @@ typedef struct { /* If fp == NULL, the following are valid: */ PyObject *readable; /* Stream-like object being read from */ PyObject *str; + PyObject *current_filename; char *ptr; char *end; - PyObject *strings; /* dict on marshal, list on unmarshal */ int version; } WFILE; @@ -311,9 +311,7 @@ w_object(PyObject *v, WFILE *p) } else if (PyUnicode_CheckExact(v)) { PyObject *utf8; - utf8 = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(v), - PyUnicode_GET_SIZE(v), - "surrogatepass"); + utf8 = PyUnicode_AsEncodedString(v, "utf8", "surrogatepass"); if (utf8 == NULL) { p->depth--; p->error = WFERR_UNMARSHALLABLE; @@ -445,7 +443,6 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version) wf.fp = fp; wf.error = WFERR_OK; wf.depth = 0; - wf.strings = NULL; wf.version = version; w_long(x, &wf); } @@ -457,10 +454,8 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) wf.fp = fp; wf.error = WFERR_OK; wf.depth = 0; - wf.strings = (version > 0) ? PyDict_New() : NULL; wf.version = version; w_object(x, &wf); - Py_XDECREF(wf.strings); } typedef WFILE RFILE; /* Same struct with different invariants */ @@ -485,7 +480,9 @@ r_string(char *s, int n, RFILE *p) } } else { - PyObject *data = PyObject_CallMethod(p->readable, "read", "i", n); + _Py_IDENTIFIER(read); + + PyObject *data = _PyObject_CallMethodId(p->readable, &PyId_read, "i", n); read = 0; if (data != NULL) { if (!PyBytes_Check(data)) { @@ -1069,6 +1066,18 @@ r_object(RFILE *p) filename = r_object(p); if (filename == NULL) goto code_error; + if (PyUnicode_CheckExact(filename)) { + if (p->current_filename != NULL) { + if (!PyUnicode_Compare(filename, p->current_filename)) { + Py_DECREF(filename); + Py_INCREF(p->current_filename); + filename = p->current_filename; + } + } + else { + p->current_filename = filename; + } + } name = r_object(p); if (name == NULL) goto code_error; @@ -1131,7 +1140,7 @@ PyMarshal_ReadShortFromFile(FILE *fp) assert(fp); rf.readable = NULL; rf.fp = fp; - rf.strings = NULL; + rf.current_filename = NULL; rf.end = rf.ptr = NULL; return r_short(&rf); } @@ -1142,7 +1151,7 @@ PyMarshal_ReadLongFromFile(FILE *fp) RFILE rf; rf.fp = fp; rf.readable = NULL; - rf.strings = NULL; + rf.current_filename = NULL; rf.ptr = rf.end = NULL; return r_long(&rf); } @@ -1204,11 +1213,10 @@ PyMarshal_ReadObjectFromFile(FILE *fp) PyObject *result; rf.fp = fp; rf.readable = NULL; - rf.strings = PyList_New(0); + rf.current_filename = NULL; rf.depth = 0; rf.ptr = rf.end = NULL; result = r_object(&rf); - Py_DECREF(rf.strings); return result; } @@ -1219,12 +1227,11 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) PyObject *result; rf.fp = NULL; rf.readable = NULL; + rf.current_filename = NULL; rf.ptr = str; rf.end = str + len; - rf.strings = PyList_New(0); rf.depth = 0; result = r_object(&rf); - Py_DECREF(rf.strings); return result; } @@ -1244,9 +1251,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) wf.error = WFERR_OK; wf.depth = 0; wf.version = version; - wf.strings = (version > 0) ? PyDict_New() : NULL; w_object(x, &wf); - Py_XDECREF(wf.strings); if (wf.str != NULL) { char *base = PyBytes_AS_STRING((PyBytesObject *)wf.str); if (wf.ptr - base > PY_SSIZE_T_MAX) { @@ -1287,12 +1292,14 @@ marshal_dump(PyObject *self, PyObject *args) int version = Py_MARSHAL_VERSION; PyObject *s; PyObject *res; + _Py_IDENTIFIER(write); + if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version)) return NULL; s = PyMarshal_WriteObjectToString(x, version); if (s == NULL) return NULL; - res = PyObject_CallMethod(f, "write", "O", s); + res = _PyObject_CallMethodId(f, &PyId_write, "O", s); Py_DECREF(s); return res; } @@ -1314,6 +1321,7 @@ static PyObject * marshal_load(PyObject *self, PyObject *f) { PyObject *data, *result; + _Py_IDENTIFIER(read); RFILE rf; /* @@ -1321,7 +1329,7 @@ marshal_load(PyObject *self, PyObject *f) * This is to ensure that the object passed in at least * has a read method which returns bytes. */ - data = PyObject_CallMethod(f, "read", "i", 0); + data = _PyObject_CallMethodId(f, &PyId_read, "i", 0); if (data == NULL) return NULL; if (!PyBytes_Check(data)) { @@ -1331,12 +1339,11 @@ marshal_load(PyObject *self, PyObject *f) result = NULL; } else { - rf.strings = PyList_New(0); rf.depth = 0; rf.fp = NULL; rf.readable = f; + rf.current_filename = NULL; result = read_object(&rf); - Py_DECREF(rf.strings); } Py_DECREF(data); return result; @@ -1389,12 +1396,11 @@ marshal_loads(PyObject *self, PyObject *args) n = p.len; rf.fp = NULL; rf.readable = NULL; + rf.current_filename = NULL; rf.ptr = s; rf.end = s + n; - rf.strings = PyList_New(0); rf.depth = 0; result = read_object(&rf); - Py_DECREF(rf.strings); PyBuffer_Release(&p); return result; } diff --git a/Python/modsupport.c b/Python/modsupport.c index 85b0d66..428914f 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -148,15 +148,6 @@ do_mklist(const char **p_format, va_list *p_va, int endchar, int n, int flags) return v; } -static int -_ustrlen(Py_UNICODE *u) -{ - int i = 0; - Py_UNICODE *v = u; - while (*v != 0) { i++; v++; } - return i; -} - static PyObject * do_mktuple(const char **p_format, va_list *p_va, int endchar, int n, int flags) { @@ -269,7 +260,7 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) } else { if (n < 0) - n = _ustrlen(u); + n = Py_UNICODE_strlen(u); v = PyUnicode_FromUnicode(u, n); } return v; @@ -292,11 +283,6 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) case 'C': { int i = va_arg(*p_va, int); - if (i < 0 || i > PyUnicode_GetMax()) { - PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x110000)"); - return NULL; - } return PyUnicode_FromOrdinal(i); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index a91da79..2d6bcda 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -71,7 +71,7 @@ static void *opcode_targets[256] = { &&TARGET_STORE_LOCALS, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&_unknown_opcode, + &&TARGET_YIELD_FROM, &&_unknown_opcode, &&_unknown_opcode, &&TARGET_INPLACE_LSHIFT, diff --git a/Python/peephole.c b/Python/peephole.c index 359eda8..5d53677 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -23,6 +23,64 @@ #define ISBASICBLOCK(blocks, start, bytes) \ (blocks[start]==blocks[start+bytes-1]) + +#define CONST_STACK_CREATE() { \ + const_stack_size = 256; \ + const_stack = PyMem_New(PyObject *, const_stack_size); \ + load_const_stack = PyMem_New(Py_ssize_t, const_stack_size); \ + if (!const_stack || !load_const_stack) { \ + PyErr_NoMemory(); \ + goto exitError; \ + } \ + } + +#define CONST_STACK_DELETE() do { \ + if (const_stack) \ + PyMem_Free(const_stack); \ + if (load_const_stack) \ + PyMem_Free(load_const_stack); \ + } while(0) + +#define CONST_STACK_LEN() (const_stack_top + 1) + +#define CONST_STACK_PUSH_OP(i) do { \ + PyObject *_x; \ + assert(codestr[i] == LOAD_CONST); \ + assert(PyList_GET_SIZE(consts) > GETARG(codestr, i)); \ + _x = PyList_GET_ITEM(consts, GETARG(codestr, i)); \ + if (++const_stack_top >= const_stack_size) { \ + const_stack_size *= 2; \ + PyMem_Resize(const_stack, PyObject *, const_stack_size); \ + PyMem_Resize(load_const_stack, Py_ssize_t, const_stack_size); \ + if (!const_stack || !load_const_stack) { \ + PyErr_NoMemory(); \ + goto exitError; \ + } \ + } \ + load_const_stack[const_stack_top] = i; \ + const_stack[const_stack_top] = _x; \ + in_consts = 1; \ + } while(0) + +#define CONST_STACK_RESET() do { \ + const_stack_top = -1; \ + } while(0) + +#define CONST_STACK_TOP() \ + const_stack[const_stack_top] + +#define CONST_STACK_LASTN(i) \ + &const_stack[const_stack_top - i + 1] + +#define CONST_STACK_POP(i) do { \ + assert(const_stack_top + 1 >= i); \ + const_stack_top -= i; \ + } while(0) + +#define CONST_STACK_OP_LASTN(i) \ + ((const_stack_top >= i - 1) ? load_const_stack[const_stack_top - i + 1] : -1) + + /* Replace LOAD_CONST c1. LOAD_CONST c2 ... LOAD_CONST cn BUILD_TUPLE n with LOAD_CONST (c1, c2, ... cn). The consts table must still be in list form so that the @@ -33,17 +91,14 @@ test; for BUILD_SET it assembles a frozenset rather than a tuple. */ static int -tuple_of_constants(unsigned char *codestr, Py_ssize_t n, PyObject *consts) +tuple_of_constants(unsigned char *codestr, Py_ssize_t n, + PyObject *consts, PyObject **objs) { PyObject *newconst, *constant; - Py_ssize_t i, arg, len_consts; + Py_ssize_t i, len_consts; /* Pre-conditions */ assert(PyList_CheckExact(consts)); - assert(codestr[n*3] == BUILD_TUPLE || codestr[n*3] == BUILD_LIST || codestr[n*3] == BUILD_SET); - assert(GETARG(codestr, (n*3)) == n); - for (i=0 ; i<n ; i++) - assert(codestr[i*3] == LOAD_CONST); /* Buildup new tuple of constants */ newconst = PyTuple_New(n); @@ -51,16 +106,14 @@ tuple_of_constants(unsigned char *codestr, Py_ssize_t n, PyObject *consts) return 0; len_consts = PyList_GET_SIZE(consts); for (i=0 ; i<n ; i++) { - arg = GETARG(codestr, (i*3)); - assert(arg < len_consts); - constant = PyList_GET_ITEM(consts, arg); + constant = objs[i]; Py_INCREF(constant); PyTuple_SET_ITEM(newconst, i, constant); } /* If it's a BUILD_SET, use the PyTuple we just built to create a PyFrozenSet, and use that as the constant instead: */ - if (codestr[n*3] == BUILD_SET) { + if (codestr[0] == BUILD_SET) { PyObject *tuple = newconst; newconst = PyFrozenSet_New(tuple); Py_DECREF(tuple); @@ -77,9 +130,8 @@ tuple_of_constants(unsigned char *codestr, Py_ssize_t n, PyObject *consts) /* Write NOPs over old LOAD_CONSTS and add a new LOAD_CONST newconst on top of the BUILD_TUPLE n */ - memset(codestr, NOP, n*3); - codestr[n*3] = LOAD_CONST; - SETARG(codestr, (n*3), len_consts); + codestr[0] = LOAD_CONST; + SETARG(codestr, 0, len_consts); return 1; } @@ -87,14 +139,14 @@ tuple_of_constants(unsigned char *codestr, Py_ssize_t n, PyObject *consts) with LOAD_CONST binop(c1,c2) The consts table must still be in list form so that the new constant can be appended. - Called with codestr pointing to the first LOAD_CONST. + Called with codestr pointing to the BINOP. Abandons the transformation if the folding fails (i.e. 1+'a'). If the new constant is a sequence, only folds when the size is below a threshold value. That keeps pyc files from becoming large in the presence of code like: (None,)*1000. */ static int -fold_binops_on_constants(unsigned char *codestr, PyObject *consts) +fold_binops_on_constants(unsigned char *codestr, PyObject *consts, PyObject **objs) { PyObject *newconst, *v, *w; Py_ssize_t len_consts, size; @@ -102,13 +154,11 @@ fold_binops_on_constants(unsigned char *codestr, PyObject *consts) /* Pre-conditions */ assert(PyList_CheckExact(consts)); - assert(codestr[0] == LOAD_CONST); - assert(codestr[3] == LOAD_CONST); /* Create new constant */ - v = PyList_GET_ITEM(consts, GETARG(codestr, 0)); - w = PyList_GET_ITEM(consts, GETARG(codestr, 3)); - opcode = codestr[6]; + v = objs[0]; + w = objs[1]; + opcode = codestr[0]; switch (opcode) { case BINARY_POWER: newconst = PyNumber_Power(v, w, Py_None); @@ -133,24 +183,6 @@ fold_binops_on_constants(unsigned char *codestr, PyObject *consts) break; case BINARY_SUBSCR: newconst = PyObject_GetItem(v, w); - /* #5057: if v is unicode, there might be differences between - wide and narrow builds in cases like '\U00012345'[0]. - Wide builds will return a non-BMP char, whereas narrow builds - will return a surrogate. In both the cases skip the - optimization in order to produce compatible pycs. - */ - if (newconst != NULL && - PyUnicode_Check(v) && PyUnicode_Check(newconst)) { - Py_UNICODE ch = PyUnicode_AS_UNICODE(newconst)[0]; -#ifdef Py_UNICODE_WIDE - if (ch > 0xFFFF) { -#else - if (ch >= 0xD800 && ch <= 0xDFFF) { -#endif - Py_DECREF(newconst); - return 0; - } - } break; case BINARY_LSHIFT: newconst = PyNumber_Lshift(v, w); @@ -198,16 +230,15 @@ fold_binops_on_constants(unsigned char *codestr, PyObject *consts) Py_DECREF(newconst); /* Write NOP NOP NOP NOP LOAD_CONST newconst */ - memset(codestr, NOP, 4); - codestr[4] = LOAD_CONST; - SETARG(codestr, 4, len_consts); + codestr[-2] = LOAD_CONST; + SETARG(codestr, -2, len_consts); return 1; } static int -fold_unaryops_on_constants(unsigned char *codestr, PyObject *consts) +fold_unaryops_on_constants(unsigned char *codestr, PyObject *consts, PyObject *v) { - PyObject *newconst=NULL, *v; + PyObject *newconst; Py_ssize_t len_consts; int opcode; @@ -216,13 +247,10 @@ fold_unaryops_on_constants(unsigned char *codestr, PyObject *consts) assert(codestr[0] == LOAD_CONST); /* Create new constant */ - v = PyList_GET_ITEM(consts, GETARG(codestr, 0)); opcode = codestr[3]; switch (opcode) { case UNARY_NEGATIVE: - /* Preserve the sign of -0.0 */ - if (PyObject_IsTrue(v) == 1) - newconst = PyNumber_Negative(v); + newconst = PyNumber_Negative(v); break; case UNARY_INVERT: newconst = PyNumber_Invert(v); @@ -358,7 +386,11 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, unsigned char *lineno; int *addrmap = NULL; int new_line, cum_orig_line, last_line, tabsiz; - int cumlc=0, lastlc=0; /* Count runs of consecutive LOAD_CONSTs */ + PyObject **const_stack = NULL; + Py_ssize_t *load_const_stack = NULL; + Py_ssize_t const_stack_top = -1; + Py_ssize_t const_stack_size = 0; + int in_consts = 0; /* whether we are in a LOAD_CONST sequence */ unsigned int *blocks = NULL; char *name; @@ -404,12 +436,16 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, goto exitError; assert(PyList_Check(consts)); + CONST_STACK_CREATE(); + for (i=0 ; i<codelen ; i += CODESIZE(codestr[i])) { reoptimize_current: opcode = codestr[i]; - lastlc = cumlc; - cumlc = 0; + if (!in_consts) { + CONST_STACK_RESET(); + } + in_consts = 0; switch (opcode) { /* Replace UNARY_NOT POP_JUMP_IF_FALSE @@ -450,21 +486,21 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, goto exitError; else if (h == 0) continue; - cumlc = lastlc + 1; + CONST_STACK_PUSH_OP(i); break; /* Skip over LOAD_CONST trueconst POP_JUMP_IF_FALSE xx. This improves "while 1" performance. */ case LOAD_CONST: - cumlc = lastlc + 1; + CONST_STACK_PUSH_OP(i); j = GETARG(codestr, i); if (codestr[i+3] != POP_JUMP_IF_FALSE || !ISBASICBLOCK(blocks,i,6) || !PyObject_IsTrue(PyList_GET_ITEM(consts, j))) continue; memset(codestr+i, NOP, 6); - cumlc = 0; + CONST_STACK_RESET(); break; /* Try to fold tuples of constants (includes a case for lists and sets @@ -476,19 +512,23 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, case BUILD_LIST: case BUILD_SET: j = GETARG(codestr, i); - h = i - 3 * j; - if (h >= 0 && - j <= lastlc && + if (j == 0) + break; + h = CONST_STACK_OP_LASTN(j); + assert((h >= 0 || CONST_STACK_LEN() < j)); + if (h >= 0 && j > 0 && j <= CONST_STACK_LEN() && ((opcode == BUILD_TUPLE && - ISBASICBLOCK(blocks, h, 3*(j+1))) || + ISBASICBLOCK(blocks, h, i-h+3)) || ((opcode == BUILD_LIST || opcode == BUILD_SET) && codestr[i+3]==COMPARE_OP && - ISBASICBLOCK(blocks, h, 3*(j+2)) && + ISBASICBLOCK(blocks, h, i-h+6) && (GETARG(codestr,i+3)==6 || GETARG(codestr,i+3)==7))) && - tuple_of_constants(&codestr[h], j, consts)) { + tuple_of_constants(&codestr[i], j, consts, CONST_STACK_LASTN(j))) { assert(codestr[i] == LOAD_CONST); - cumlc = 1; + memset(&codestr[h], NOP, i - h); + CONST_STACK_POP(j); + CONST_STACK_PUSH_OP(i); break; } if (codestr[i+3] != UNPACK_SEQUENCE || @@ -501,10 +541,12 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, } else if (j == 2) { codestr[i] = ROT_TWO; memset(codestr+i+1, NOP, 5); + CONST_STACK_RESET(); } else if (j == 3) { codestr[i] = ROT_THREE; codestr[i+1] = ROT_TWO; memset(codestr+i+2, NOP, 4); + CONST_STACK_RESET(); } break; @@ -523,12 +565,18 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, case BINARY_AND: case BINARY_XOR: case BINARY_OR: - if (lastlc >= 2 && - ISBASICBLOCK(blocks, i-6, 7) && - fold_binops_on_constants(&codestr[i-6], consts)) { + /* NOTE: LOAD_CONST is saved at `i-2` since it has an arg + while BINOP hasn't */ + h = CONST_STACK_OP_LASTN(2); + assert((h >= 0 || CONST_STACK_LEN() < 2)); + if (h >= 0 && + ISBASICBLOCK(blocks, h, i-h+1) && + fold_binops_on_constants(&codestr[i], consts, CONST_STACK_LASTN(2))) { i -= 2; + memset(&codestr[h], NOP, i - h); assert(codestr[i] == LOAD_CONST); - cumlc = 1; + CONST_STACK_POP(2); + CONST_STACK_PUSH_OP(i); } break; @@ -537,12 +585,15 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, case UNARY_NEGATIVE: case UNARY_INVERT: case UNARY_POSITIVE: - if (lastlc >= 1 && - ISBASICBLOCK(blocks, i-3, 4) && - fold_unaryops_on_constants(&codestr[i-3], consts)) { + h = CONST_STACK_OP_LASTN(1); + assert((h >= 0 || CONST_STACK_LEN() < 1)); + if (h >= 0 && + ISBASICBLOCK(blocks, h, i-h+1) && + fold_unaryops_on_constants(&codestr[i-3], consts, CONST_STACK_TOP())) { i -= 2; assert(codestr[i] == LOAD_CONST); - cumlc = 1; + CONST_STACK_POP(1); + CONST_STACK_PUSH_OP(i); } break; @@ -699,6 +750,7 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, assert(h + nops == codelen); code = PyBytes_FromStringAndSize((char *)codestr, h); + CONST_STACK_DELETE(); PyMem_Free(addrmap); PyMem_Free(codestr); PyMem_Free(blocks); @@ -708,6 +760,7 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, code = NULL; exitUnchanged: + CONST_STACK_DELETE(); if (blocks != NULL) PyMem_Free(blocks); if (addrmap != NULL) diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 75e3032..7bf21c0 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -954,7 +954,7 @@ format_float_short(double d, char format_code, /* shouldn't get here: Gay's code should always return something starting with a digit, an 'I', or 'N' */ strncpy(p, "ERR", 3); - p += 3; + /* p += 3; */ assert(0); } goto exit; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 584a19b..a642c0b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -62,14 +62,17 @@ static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *, static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *, PyCompilerFlags *); static void err_input(perrdetail *); +static void err_free(perrdetail *); static void initsigs(void); static void call_py_exitfuncs(void); static void wait_for_thread_shutdown(void); static void call_ll_exitfuncs(void); -extern void _PyUnicode_Init(void); +extern int _PyUnicode_Init(void); extern void _PyUnicode_Fini(void); extern int _PyLong_Init(void); extern void PyLong_Fini(void); +extern int _PyFaulthandler_Init(void); +extern void _PyFaulthandler_Fini(void); #ifdef WITH_THREAD extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); @@ -139,12 +142,13 @@ get_codec_name(const char *encoding) { char *name_utf8, *name_str; PyObject *codec, *name = NULL; + _Py_IDENTIFIER(name); codec = _PyCodec_Lookup(encoding); if (!codec) goto error; - name = PyObject_GetAttrString(codec, "name"); + name = _PyObject_GetAttrId(codec, &PyId_name); Py_CLEAR(codec); if (!name) goto error; @@ -166,18 +170,25 @@ error: return NULL; } -#if defined(HAVE_LANGINFO_H) && defined(CODESET) static char* -get_codeset(void) +get_locale_encoding(void) { +#ifdef MS_WINDOWS + char codepage[100]; + PyOS_snprintf(codepage, sizeof(codepage), "cp%d", GetACP()); + return get_codec_name(codepage); +#elif defined(HAVE_LANGINFO_H) && defined(CODESET) char* codeset = nl_langinfo(CODESET); if (!codeset || codeset[0] == '\0') { PyErr_SetString(PyExc_ValueError, "CODESET is not set or empty"); return NULL; } return get_codec_name(codeset); -} +#else + PyErr_SetNone(PyExc_NotImplementedError); + return NULL; #endif +} void Py_InitializeEx(int install_sigs) @@ -257,7 +268,8 @@ Py_InitializeEx(int install_sigs) Py_FatalError("Py_Initialize: can't make modules_reloading dictionary"); /* Init Unicode implementation; relies on the codec registry */ - _PyUnicode_Init(); + if (_PyUnicode_Init() < 0) + Py_FatalError("Py_Initialize: can't initialize unicode"); bimod = _PyBuiltin_Init(); if (bimod == NULL) @@ -296,6 +308,10 @@ Py_InitializeEx(int install_sigs) _PyImportHooks_Init(); + /* initialize the faulthandler module */ + if (_PyFaulthandler_Init()) + Py_FatalError("Py_Initialize: can't initialize faulthandler"); + /* Initialize _warnings. */ _PyWarnings_Init(); @@ -361,9 +377,10 @@ flush_std_files(void) PyObject *fout = PySys_GetObject("stdout"); PyObject *ferr = PySys_GetObject("stderr"); PyObject *tmp; + _Py_IDENTIFIER(flush); if (fout != NULL && fout != Py_None && !file_is_closed(fout)) { - tmp = PyObject_CallMethod(fout, "flush", ""); + tmp = _PyObject_CallMethodId(fout, &PyId_flush, ""); if (tmp == NULL) PyErr_WriteUnraisable(fout); else @@ -371,7 +388,7 @@ flush_std_files(void) } if (ferr != NULL && ferr != Py_None && !file_is_closed(ferr)) { - tmp = PyObject_CallMethod(ferr, "flush", ""); + tmp = _PyObject_CallMethodId(ferr, &PyId_flush, ""); if (tmp == NULL) PyErr_Clear(); else @@ -464,7 +481,7 @@ Py_Finalize(void) flush_std_files(); /* Collect final garbage. This disposes of cycles created by - * new-style class definitions, for example. + * class definitions, for example. * XXX This is disabled because it caused too many problems. If * XXX a __del__ or weakref callback triggers here, Python code has * XXX a hard time running, because even the sys module has been @@ -485,6 +502,9 @@ Py_Finalize(void) /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ _PyImport_Fini(); + /* unload faulthandler module */ + _PyFaulthandler_Fini(); + /* Debugging stuff */ #ifdef COUNT_ALLOCS dump_counts(stdout); @@ -534,6 +554,7 @@ Py_Finalize(void) PyLong_Fini(); PyFloat_Fini(); PyDict_Fini(); + PySlice_Fini(); /* Cleanup Unicode implementation */ _PyUnicode_Fini(); @@ -759,24 +780,17 @@ static int initfsencoding(PyInterpreterState *interp) { PyObject *codec; -#if defined(HAVE_LANGINFO_H) && defined(CODESET) - char *codeset = NULL; - - if (Py_FileSystemDefaultEncoding == NULL) { - /* On Unix, set the file system encoding according to the - user's preference, if the CODESET names a well-known - Python codec, and Py_FileSystemDefaultEncoding isn't - initialized by other means. */ - codeset = get_codeset(); - if (codeset == NULL) + + if (Py_FileSystemDefaultEncoding == NULL) + { + Py_FileSystemDefaultEncoding = get_locale_encoding(); + if (Py_FileSystemDefaultEncoding == NULL) Py_FatalError("Py_Initialize: Unable to get the locale encoding"); - Py_FileSystemDefaultEncoding = codeset; Py_HasFileSystemDefaultEncoding = 0; interp->fscodec_initialized = 1; return 0; } -#endif /* the encoding is mbcs, utf-8 or ascii */ codec = _PyCodec_Lookup(Py_FileSystemDefaultEncoding); @@ -818,6 +832,11 @@ create_stdio(PyObject* io, const char* newline; PyObject *line_buffering; int buffering, isatty; + _Py_IDENTIFIER(open); + _Py_IDENTIFIER(isatty); + _Py_IDENTIFIER(TextIOWrapper); + _Py_IDENTIFIER(name); + _Py_IDENTIFIER(mode); /* stdin is always opened in buffered mode, first because it shouldn't make a difference in common use cases, second because TextIOWrapper @@ -832,14 +851,15 @@ create_stdio(PyObject* io, mode = "wb"; else mode = "rb"; - buf = PyObject_CallMethod(io, "open", "isiOOOi", - fd, mode, buffering, - Py_None, Py_None, Py_None, 0); + buf = _PyObject_CallMethodId(io, &PyId_open, "isiOOOi", + fd, mode, buffering, + Py_None, Py_None, Py_None, 0); if (buf == NULL) goto error; if (buffering) { - raw = PyObject_GetAttrString(buf, "raw"); + _Py_IDENTIFIER(raw); + raw = _PyObject_GetAttrId(buf, &PyId_raw); if (raw == NULL) goto error; } @@ -849,9 +869,9 @@ create_stdio(PyObject* io, } text = PyUnicode_FromString(name); - if (text == NULL || PyObject_SetAttrString(raw, "name", text) < 0) + if (text == NULL || _PyObject_SetAttrId(raw, &PyId_name, text) < 0) goto error; - res = PyObject_CallMethod(raw, "isatty", ""); + res = _PyObject_CallMethodId(raw, &PyId_isatty, ""); if (res == NULL) goto error; isatty = PyObject_IsTrue(res); @@ -874,9 +894,9 @@ create_stdio(PyObject* io, } #endif - stream = PyObject_CallMethod(io, "TextIOWrapper", "OsssO", - buf, encoding, errors, - newline, line_buffering); + stream = _PyObject_CallMethodId(io, &PyId_TextIOWrapper, "OsssO", + buf, encoding, errors, + newline, line_buffering); Py_CLEAR(buf); if (stream == NULL) goto error; @@ -886,7 +906,7 @@ create_stdio(PyObject* io, else mode = "r"; text = PyUnicode_FromString(mode); - if (!text || PyObject_SetAttrString(stream, "mode", text) < 0) + if (!text || _PyObject_SetAttrId(stream, &PyId_mode, text) < 0) goto error; Py_CLEAR(text); return stream; @@ -1126,13 +1146,14 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags PyArena *arena; char *ps1 = "", *ps2 = "", *enc = NULL; int errcode = 0; + _Py_IDENTIFIER(encoding); if (fp == stdin) { /* Fetch encoding from sys.stdin */ v = PySys_GetObject("stdin"); if (v == NULL || v == Py_None) return -1; - oenc = PyObject_GetAttrString(v, "encoding"); + oenc = _PyObject_GetAttrId(v, &PyId_encoding); if (!oenc) return -1; enc = _PyUnicode_AsString(oenc); @@ -1329,19 +1350,19 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, { long hold; PyObject *v; - - /* old style errors */ - if (PyTuple_Check(err)) - return PyArg_ParseTuple(err, "O(ziiz)", message, filename, - lineno, offset, text); + _Py_IDENTIFIER(msg); + _Py_IDENTIFIER(filename); + _Py_IDENTIFIER(lineno); + _Py_IDENTIFIER(offset); + _Py_IDENTIFIER(text); /* new style errors. `err' is an instance */ - if (! (v = PyObject_GetAttrString(err, "msg"))) + if (! (v = _PyObject_GetAttrId(err, &PyId_msg))) goto finally; *message = v; - if (!(v = PyObject_GetAttrString(err, "filename"))) + if (!(v = _PyObject_GetAttrId(err, &PyId_filename))) goto finally; if (v == Py_None) *filename = NULL; @@ -1349,7 +1370,7 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, goto finally; Py_DECREF(v); - if (!(v = PyObject_GetAttrString(err, "lineno"))) + if (!(v = _PyObject_GetAttrId(err, &PyId_lineno))) goto finally; hold = PyLong_AsLong(v); Py_DECREF(v); @@ -1358,7 +1379,7 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, goto finally; *lineno = (int)hold; - if (!(v = PyObject_GetAttrString(err, "offset"))) + if (!(v = _PyObject_GetAttrId(err, &PyId_offset))) goto finally; if (v == Py_None) { *offset = -1; @@ -1373,7 +1394,7 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, *offset = (int)hold; } - if (!(v = PyObject_GetAttrString(err, "text"))) + if (!(v = _PyObject_GetAttrId(err, &PyId_text))) goto finally; if (v == Py_None) *text = NULL; @@ -1442,7 +1463,8 @@ handle_system_exit(void) goto done; if (PyExceptionInstance_Check(value)) { /* The error code should be in the `code' attribute. */ - PyObject *code = PyObject_GetAttrString(value, "code"); + _Py_IDENTIFIER(code); + PyObject *code = _PyObject_GetAttrId(value, &PyId_code); if (code) { Py_DECREF(value); value = code; @@ -1549,6 +1571,7 @@ print_exception(PyObject *f, PyObject *value) { int err = 0; PyObject *type, *tb; + _Py_IDENTIFIER(print_file_and_line); if (!PyExceptionInstance_Check(value)) { PyFile_WriteString("TypeError: print_exception(): Exception expected for value, ", f); @@ -1564,7 +1587,7 @@ print_exception(PyObject *f, PyObject *value) if (tb && tb != Py_None) err = PyTraceBack_Print(tb, f); if (err == 0 && - PyObject_HasAttrString(value, "print_file_and_line")) + _PyObject_HasAttrId(value, &PyId_print_file_and_line)) { PyObject *message; const char *filename, *text; @@ -1599,6 +1622,7 @@ print_exception(PyObject *f, PyObject *value) else { PyObject* moduleName; char* className; + _Py_IDENTIFIER(__module__); assert(PyExceptionClass_Check(type)); className = PyExceptionClass_Name(type); if (className != NULL) { @@ -1607,10 +1631,10 @@ print_exception(PyObject *f, PyObject *value) className = dot+1; } - moduleName = PyObject_GetAttrString(type, "__module__"); + moduleName = _PyObject_GetAttrId(type, &PyId___module__); if (moduleName == NULL || !PyUnicode_Check(moduleName)) { - Py_DECREF(moduleName); + Py_XDECREF(moduleName); err = PyFile_WriteString("<unknown>", f); } else { @@ -1637,7 +1661,7 @@ print_exception(PyObject *f, PyObject *value) if (s == NULL) err = -1; else if (!PyUnicode_Check(s) || - PyUnicode_GetSize(s) != 0) + PyUnicode_GetLength(s) != 0) err = PyFile_WriteString(": ", f); if (err == 0) err = PyFile_WriteObject(s, f, Py_PRINT_RAW); @@ -1774,13 +1798,14 @@ flush_io(void) { PyObject *f, *r; PyObject *type, *value, *traceback; + _Py_IDENTIFIER(flush); /* Save the current exception */ PyErr_Fetch(&type, &value, &traceback); f = PySys_GetObject("stderr"); if (f != NULL) { - r = PyObject_CallMethod(f, "flush", ""); + r = _PyObject_CallMethodId(f, &PyId_flush, ""); if (r) Py_DECREF(r); else @@ -1788,7 +1813,7 @@ flush_io(void) } f = PySys_GetObject("stdout"); if (f != NULL) { - r = PyObject_CallMethod(f, "flush", ""); + r = _PyObject_CallMethodId(f, &PyId_flush, ""); if (r) Py_DECREF(r); else @@ -1827,6 +1852,8 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, "Bad magic number in .pyc file"); return NULL; } + /* Skip mtime and size */ + (void) PyMarshal_ReadLongFromFile(fp); (void) PyMarshal_ReadLongFromFile(fp); v = PyMarshal_ReadLastObjectFromFile(fp); fclose(fp); @@ -1919,12 +1946,13 @@ PyParser_ASTFromString(const char *s, const char *filename, int start, flags->cf_flags |= iflags & PyCF_MASK; mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); - return mod; } else { err_input(&err); - return NULL; + mod = NULL; } + err_free(&err); + return mod; } mod_ty @@ -1949,14 +1977,15 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc, flags->cf_flags |= iflags & PyCF_MASK; mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); - return mod; } else { err_input(&err); if (errcode) *errcode = err.error; - return NULL; + mod = NULL; } + err_free(&err); + return mod; } /* Simplified interface to parsefile -- return node or set exception */ @@ -1970,6 +1999,7 @@ PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int fla start, NULL, NULL, &err, flags); if (n == NULL) err_input(&err); + err_free(&err); return n; } @@ -1984,6 +2014,7 @@ PyParser_SimpleParseStringFlags(const char *str, int start, int flags) start, &err, flags); if (n == NULL) err_input(&err); + err_free(&err); return n; } @@ -1996,6 +2027,7 @@ PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename, &_PyParser_Grammar, start, &err, flags); if (n == NULL) err_input(&err); + err_free(&err); return n; } @@ -2009,11 +2041,23 @@ PyParser_SimpleParseStringFilename(const char *str, const char *filename, int st even parser modules. */ void +PyParser_ClearError(perrdetail *err) +{ + err_free(err); +} + +void PyParser_SetError(perrdetail *err) { err_input(err); } +static void +err_free(perrdetail *err) +{ + Py_CLEAR(err->filename); +} + /* Set the error appropriate to the given input error code (see errcode.h) */ static void @@ -2021,7 +2065,6 @@ err_input(perrdetail *err) { PyObject *v, *w, *errtype, *errtext; PyObject *msg_obj = NULL; - PyObject *filename; char *msg = NULL; errtype = PyExc_SyntaxError; @@ -2093,6 +2136,9 @@ err_input(perrdetail *err) case E_IDENTIFIER: msg = "invalid character in identifier"; break; + case E_BADSINGLE: + msg = "multiple statements found while compiling a single statement"; + break; default: fprintf(stderr, "error=%d\n", err->error); msg = "unknown parsing error"; @@ -2107,17 +2153,8 @@ err_input(perrdetail *err) errtext = PyUnicode_DecodeUTF8(err->text, strlen(err->text), "replace"); } - if (err->filename != NULL) - filename = PyUnicode_DecodeFSDefault(err->filename); - else { - Py_INCREF(Py_None); - filename = Py_None; - } - if (filename != NULL) - v = Py_BuildValue("(NiiN)", filename, - err->lineno, err->offset, errtext); - else - v = NULL; + v = Py_BuildValue("(OiiN)", err->filename, + err->lineno, err->offset, errtext); if (v != NULL) { if (msg_obj) w = Py_BuildValue("(OO)", msg_obj, v); @@ -2141,11 +2178,24 @@ cleanup: void Py_FatalError(const char *msg) { + const int fd = fileno(stderr); + PyThreadState *tstate; + fprintf(stderr, "Fatal Python error: %s\n", msg); fflush(stderr); /* it helps in Windows debug build */ if (PyErr_Occurred()) { PyErr_PrintEx(0); } + else { + tstate = _Py_atomic_load_relaxed(&_PyThreadState_Current); + if (tstate != NULL) { + fputc('\n', stderr); + fflush(stderr); + _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + } + _PyFaulthandler_Fini(); + } + #ifdef MS_WINDOWS { size_t len = strlen(msg); @@ -2200,6 +2250,7 @@ static void wait_for_thread_shutdown(void) { #ifdef WITH_THREAD + _Py_IDENTIFIER(_shutdown); PyObject *result; PyThreadState *tstate = PyThreadState_GET(); PyObject *threading = PyMapping_GetItemString(tstate->interp->modules, @@ -2209,7 +2260,7 @@ wait_for_thread_shutdown(void) PyErr_Clear(); return; } - result = PyObject_CallMethod(threading, "_shutdown", ""); + result = _PyObject_CallMethodId(threading, &PyId__shutdown, ""); if (result == NULL) { PyErr_WriteUnraisable(threading); } diff --git a/Python/pytime.c b/Python/pytime.c index 6fb7695..bec1c71 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,7 +1,9 @@ #include "Python.h" +#ifdef MS_WINDOWS +#include <windows.h> +#endif -#ifdef __APPLE__ -#if defined(HAVE_GETTIMEOFDAY) && defined(HAVE_FTIME) +#if defined(__APPLE__) && defined(HAVE_GETTIMEOFDAY) && defined(HAVE_FTIME) /* * _PyTime_gettimeofday falls back to ftime when getttimeofday fails because the latter * might fail on some platforms. This fallback is unwanted on MacOSX because @@ -10,18 +12,30 @@ */ # undef HAVE_FTIME #endif -#endif -#ifdef HAVE_FTIME +#if defined(HAVE_FTIME) && !defined(MS_WINDOWS) #include <sys/timeb.h> -#if !defined(MS_WINDOWS) && !defined(PYOS_OS2) extern int ftime(struct timeb *); -#endif /* MS_WINDOWS */ -#endif /* HAVE_FTIME */ +#endif void _PyTime_gettimeofday(_PyTime_timeval *tp) { +#ifdef MS_WINDOWS + FILETIME system_time; + ULARGE_INTEGER large; + ULONGLONG microseconds; + + GetSystemTimeAsFileTime(&system_time); + large.u.LowPart = system_time.dwLowDateTime; + large.u.HighPart = system_time.dwHighDateTime; + /* 11,644,473,600,000,000: number of microseconds between + the 1st january 1601 and the 1st january 1970 (369 years + 89 leap + days). */ + microseconds = large.QuadPart / 10 - 11644473600000000; + tp->tv_sec = microseconds / 1000000; + tp->tv_usec = microseconds % 1000000; +#else /* There are three ways to get the time: (1) gettimeofday() -- resolution in microseconds (2) ftime() -- resolution in milliseconds @@ -30,6 +44,7 @@ _PyTime_gettimeofday(_PyTime_timeval *tp) Since on some systems (e.g. SCO ODT 3.0) gettimeofday() may fail, so we fall back on ftime() or time(). Note: clock resolution does not imply clock accuracy! */ + #ifdef HAVE_GETTIMEOFDAY #ifdef GETTIMEOFDAY_NO_TZ if (gettimeofday(tp) == 0) @@ -39,6 +54,7 @@ _PyTime_gettimeofday(_PyTime_timeval *tp) return; #endif /* !GETTIMEOFDAY_NO_TZ */ #endif /* !HAVE_GETTIMEOFDAY */ + #if defined(HAVE_FTIME) { struct timeb t; @@ -50,7 +66,8 @@ _PyTime_gettimeofday(_PyTime_timeval *tp) tp->tv_sec = time(NULL); tp->tv_usec = 0; #endif /* !HAVE_FTIME */ - return; + +#endif /* MS_WINDOWS */ } void diff --git a/Python/symtable.c b/Python/symtable.c index 1ec51f7..b622835 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -19,10 +19,6 @@ #define IMPORT_STAR_WARNING "import * only allowed at module level" -#define RETURN_VAL_IN_GENERATOR \ - "'return' with argument inside generator" - - static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, int lineno, int col_offset) @@ -185,6 +181,7 @@ static int symtable_visit_params(struct symtable *st, asdl_seq *args); static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); static int symtable_visit_annotations(struct symtable *st, stmt_ty s); +static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static identifier top = NULL, lambda = NULL, genexpr = NULL, @@ -224,10 +221,17 @@ symtable_new(void) struct symtable * PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) { - struct symtable *st = symtable_new(); + struct symtable *st; asdl_seq *seq; int i; + if (__class__ == NULL) { + __class__ = PyUnicode_InternFromString("@__class__"); + if (__class__ == NULL) + return NULL; + } + + st = symtable_new(); if (st == NULL) return st; st->st_filename = filename; @@ -743,14 +747,12 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } else { /* Special-case __class__ */ - if (!GET_IDENTIFIER(__class__)) - goto error; assert(PySet_Contains(local, __class__) == 1); if (PySet_Add(newbound, __class__) < 0) goto error; } - /* Recursively call analyze_block() on each child block. + /* Recursively call analyze_child_block() on each child block. newbound, newglobal now contain the names visible in nested blocks. The free variables in the children will @@ -782,7 +784,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, NULL)) goto error; else if (ste->ste_type == ClassBlock && !analyze_cells(scopes, newfree, - "__class__")) + "@__class__")) goto error; /* Records the results of the analysis in the symbol table entry */ if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, @@ -893,18 +895,15 @@ symtable_warn(struct symtable *st, char *msg, int lineno) static int symtable_exit_block(struct symtable *st, void *ast) { - Py_ssize_t end; - - Py_CLEAR(st->st_cur); - end = PyList_GET_SIZE(st->st_stack) - 1; - if (end >= 0) { - st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, - end); - if (st->st_cur == NULL) - return 0; - Py_INCREF(st->st_cur); - if (PySequence_DelItem(st->st_stack, end) < 0) + Py_ssize_t size; + + st->st_cur = NULL; + size = PyList_GET_SIZE(st->st_stack); + if (size) { + if (PyList_SetSlice(st->st_stack, size - 1, size, NULL) < 0) return 0; + if (--size) + st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, size - 1); } return 1; } @@ -913,23 +912,23 @@ static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, void *ast, int lineno, int col_offset) { - PySTEntryObject *prev = NULL; + PySTEntryObject *prev = NULL, *ste; - if (st->st_cur) { - prev = st->st_cur; - if (PyList_Append(st->st_stack, (PyObject *)st->st_cur) < 0) { - return 0; - } - Py_DECREF(st->st_cur); - } - st->st_cur = ste_new(st, name, block, ast, lineno, col_offset); - if (st->st_cur == NULL) + ste = ste_new(st, name, block, ast, lineno, col_offset); + if (ste == NULL) return 0; + if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) { + Py_DECREF(ste); + return 0; + } + prev = st->st_cur; + /* The entry is owned by the stack. Borrow it for st_cur. */ + Py_DECREF(ste); + st->st_cur = ste; if (block == ModuleBlock) st->st_global = st->st_cur->ste_symbols; if (prev) { - if (PyList_Append(prev->ste_children, - (PyObject *)st->st_cur) < 0) { + if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) { return 0; } } @@ -1023,12 +1022,6 @@ error: if (!symtable_visit_ ## TYPE((ST), (V))) \ return 0; -#define VISIT_IN_BLOCK(ST, TYPE, V, S) \ - if (!symtable_visit_ ## TYPE((ST), (V))) { \ - symtable_exit_block((ST), (S)); \ - return 0; \ - } - #define VISIT_SEQ(ST, TYPE, SEQ) { \ int i; \ asdl_seq *seq = (SEQ); /* avoid variable capture */ \ @@ -1039,18 +1032,6 @@ error: } \ } -#define VISIT_SEQ_IN_BLOCK(ST, TYPE, SEQ, S) { \ - int i; \ - asdl_seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) { \ - symtable_exit_block((ST), (S)); \ - return 0; \ - } \ - } \ -} - #define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \ int i; \ asdl_seq *seq = (SEQ); /* avoid variable capture */ \ @@ -1061,18 +1042,6 @@ error: } \ } -#define VISIT_SEQ_TAIL_IN_BLOCK(ST, TYPE, SEQ, START, S) { \ - int i; \ - asdl_seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = (START); i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) { \ - symtable_exit_block((ST), (S)); \ - return 0; \ - } \ - } \ -} - #define VISIT_KWONLYDEFAULTS(ST, KW_DEFAULTS) { \ int i = 0; \ asdl_seq *seq = (KW_DEFAULTS); /* avoid variable capture */ \ @@ -1122,8 +1091,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) FunctionBlock, (void *)s, s->lineno, s->col_offset)) return 0; - VISIT_IN_BLOCK(st, arguments, s->v.FunctionDef.args, s); - VISIT_SEQ_IN_BLOCK(st, stmt, s->v.FunctionDef.body, s); + VISIT(st, arguments, s->v.FunctionDef.args); + VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st, s)) return 0; break; @@ -1142,8 +1111,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, (void *)s, s->lineno, s->col_offset)) return 0; - if (!GET_IDENTIFIER(__class__) || - !symtable_add_def(st, __class__, DEF_LOCAL) || + if (!symtable_add_def(st, __class__, DEF_LOCAL) || !GET_IDENTIFIER(__locals__) || !symtable_add_def(st, __locals__, DEF_PARAM)) { symtable_exit_block(st, s); @@ -1151,7 +1119,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } tmp = st->st_private; st->st_private = s->v.ClassDef.name; - VISIT_SEQ_IN_BLOCK(st, stmt, s->v.ClassDef.body, s); + VISIT_SEQ(st, stmt, s->v.ClassDef.body); st->st_private = tmp; if (!symtable_exit_block(st, s)) return 0; @@ -1161,14 +1129,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) if (s->v.Return.value) { VISIT(st, expr, s->v.Return.value); st->st_cur->ste_returns_value = 1; - if (st->st_cur->ste_generator) { - PyErr_SetString(PyExc_SyntaxError, - RETURN_VAL_IN_GENERATOR); - PyErr_SyntaxLocationEx(st->st_filename, - s->lineno, - s->col_offset); - return 0; - } } break; case Delete_kind: @@ -1205,19 +1165,16 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) case Raise_kind: if (s->v.Raise.exc) { VISIT(st, expr, s->v.Raise.exc); - if (s->v.Raise.cause) { - VISIT(st, expr, s->v.Raise.cause); - } + if (s->v.Raise.cause) { + VISIT(st, expr, s->v.Raise.cause); + } } break; - case TryExcept_kind: - VISIT_SEQ(st, stmt, s->v.TryExcept.body); - VISIT_SEQ(st, stmt, s->v.TryExcept.orelse); - VISIT_SEQ(st, excepthandler, s->v.TryExcept.handlers); - break; - case TryFinally_kind: - VISIT_SEQ(st, stmt, s->v.TryFinally.body); - VISIT_SEQ(st, stmt, s->v.TryFinally.finalbody); + case Try_kind: + VISIT_SEQ(st, stmt, s->v.Try.body); + VISIT_SEQ(st, stmt, s->v.Try.orelse); + VISIT_SEQ(st, excepthandler, s->v.Try.handlers); + VISIT_SEQ(st, stmt, s->v.Try.finalbody); break; case Assert_kind: VISIT(st, expr, s->v.Assert.test); @@ -1305,10 +1262,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) /* nothing to do here */ break; case With_kind: - VISIT(st, expr, s->v.With.context_expr); - if (s->v.With.optional_vars) { - VISIT(st, expr, s->v.With.optional_vars); - } + VISIT_SEQ(st, withitem, s->v.With.items); VISIT_SEQ(st, stmt, s->v.With.body); break; } @@ -1341,8 +1295,8 @@ symtable_visit_expr(struct symtable *st, expr_ty e) FunctionBlock, (void *)e, e->lineno, e->col_offset)) return 0; - VISIT_IN_BLOCK(st, arguments, e->v.Lambda.args, (void*)e); - VISIT_IN_BLOCK(st, expr, e->v.Lambda.body, (void*)e); + VISIT(st, arguments, e->v.Lambda.args); + VISIT(st, expr, e->v.Lambda.body); if (!symtable_exit_block(st, (void *)e)) return 0; break; @@ -1376,17 +1330,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e) return 0; break; case Yield_kind: - if (e->v.Yield.value) - VISIT(st, expr, e->v.Yield.value); + case YieldFrom_kind: { + expr_ty value; + value = (e->kind == YieldFrom_kind) ? e->v.YieldFrom.value : e->v.Yield.value; + if (value) + VISIT(st, expr, value); st->st_cur->ste_generator = 1; - if (st->st_cur->ste_returns_value) { - PyErr_SetString(PyExc_SyntaxError, - RETURN_VAL_IN_GENERATOR); - PyErr_SyntaxLocationEx(st->st_filename, - e->lineno, e->col_offset); - return 0; - } break; + } case Compare_kind: VISIT(st, expr, e->v.Compare.left); VISIT_SEQ(st, expr, e->v.Compare.comparators); @@ -1425,8 +1376,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) if (e->v.Name.ctx == Load && st->st_cur->ste_type == FunctionBlock && !PyUnicode_CompareWithASCIIString(e->v.Name.id, "super")) { - if (!GET_IDENTIFIER(__class__) || - !symtable_add_def(st, __class__, USE)) + if (!symtable_add_def(st, __class__, USE)) return 0; } break; @@ -1543,6 +1493,16 @@ symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh) return 1; } +static int +symtable_visit_withitem(struct symtable *st, withitem_ty item) +{ + VISIT(st, expr, item->context_expr); + if (item->optional_vars) { + VISIT(st, expr, item->optional_vars); + } + return 1; +} + static int symtable_visit_alias(struct symtable *st, alias_ty a) @@ -1553,10 +1513,10 @@ symtable_visit_alias(struct symtable *st, alias_ty a) */ PyObject *store_name; PyObject *name = (a->asname == NULL) ? a->name : a->asname; - const Py_UNICODE *base = PyUnicode_AS_UNICODE(name); - Py_UNICODE *dot = Py_UNICODE_strchr(base, '.'); - if (dot) { - store_name = PyUnicode_FromUnicode(base, dot - base); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, + PyUnicode_GET_LENGTH(name), 1); + if (dot != -1) { + store_name = PyUnicode_Substring(name, 0, dot); if (!store_name) return 0; } @@ -1653,13 +1613,12 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, symtable_exit_block(st, (void *)e); return 0; } - VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e); - VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e); - VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension, - generators, 1, (void*)e); + VISIT(st, expr, outermost->target); + VISIT_SEQ(st, expr, outermost->ifs); + VISIT_SEQ_TAIL(st, comprehension, generators, 1); if (value) - VISIT_IN_BLOCK(st, expr, value, (void*)e); - VISIT_IN_BLOCK(st, expr, elt, (void*)e); + VISIT(st, expr, value); + VISIT(st, expr, elt); return symtable_exit_block(st, (void *)e); } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index c4f27d0..c434b5a 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -17,6 +17,7 @@ Data members: #include "Python.h" #include "code.h" #include "frameobject.h" +#include "pythread.h" #include "osdefs.h" @@ -78,8 +79,10 @@ sys_displayhook_unencodable(PyObject *outf, PyObject *o) PyObject *encoded, *escaped_str, *repr_str, *buffer, *result; char *stdout_encoding_str; int ret; + _Py_IDENTIFIER(encoding); + _Py_IDENTIFIER(buffer); - stdout_encoding = PyObject_GetAttrString(outf, "encoding"); + stdout_encoding = _PyObject_GetAttrId(outf, &PyId_encoding); if (stdout_encoding == NULL) goto error; stdout_encoding_str = _PyUnicode_AsString(stdout_encoding); @@ -96,9 +99,10 @@ sys_displayhook_unencodable(PyObject *outf, PyObject *o) if (encoded == NULL) goto error; - buffer = PyObject_GetAttrString(outf, "buffer"); + buffer = _PyObject_GetAttrId(outf, &PyId_buffer); if (buffer) { - result = PyObject_CallMethod(buffer, "write", "(O)", encoded); + _Py_IDENTIFIER(write); + result = _PyObject_CallMethodId(buffer, &PyId_write, "(O)", encoded); Py_DECREF(buffer); Py_DECREF(encoded); if (result == NULL) @@ -135,6 +139,7 @@ sys_displayhook(PyObject *self, PyObject *o) PyObject *modules = interp->modules; PyObject *builtins = PyDict_GetItemString(modules, "builtins"); int err; + _Py_IDENTIFIER(_); if (builtins == NULL) { PyErr_SetString(PyExc_RuntimeError, "lost builtins module"); @@ -148,7 +153,7 @@ sys_displayhook(PyObject *self, PyObject *o) Py_INCREF(Py_None); return Py_None; } - if (PyObject_SetAttrString(builtins, "_", Py_None) != 0) + if (_PyObject_SetAttrId(builtins, &PyId__, Py_None) != 0) return NULL; outf = PySys_GetObject("stdout"); if (outf == NULL || outf == Py_None) { @@ -170,7 +175,7 @@ sys_displayhook(PyObject *self, PyObject *o) } if (PyFile_WriteString("\n", outf) != 0) return NULL; - if (PyObject_SetAttrString(builtins, "_", o) != 0) + if (_PyObject_SetAttrId(builtins, &PyId__, o) != 0) return NULL; Py_INCREF(Py_None); return Py_None; @@ -770,9 +775,7 @@ interpreter loads extension modules. Among other things, this will enable\n\ a lazy resolving of symbols when importing a module, if called as\n\ sys.setdlopenflags(0). To share symbols across extension modules, call as\n\ sys.setdlopenflags(ctypes.RTLD_GLOBAL). Symbolic names for the flag modules\n\ -can be either found in the ctypes module, or in the DLFCN module. If DLFCN\n\ -is not available, it can be generated from /usr/include/dlfcn.h using the\n\ -h2py script."); +can be found in the os module (RTLD_xxx constants, e.g. os.RTLD_LAZY)."); static PyObject * sys_getdlopenflags(PyObject *self, PyObject *args) @@ -811,10 +814,11 @@ static PyObject * sys_getsizeof(PyObject *self, PyObject *args, PyObject *kwds) { PyObject *res = NULL; - static PyObject *str__sizeof__ = NULL, *gc_head_size = NULL; + static PyObject *gc_head_size = NULL; static char *kwlist[] = {"object", "default", 0}; PyObject *o, *dflt = NULL; PyObject *method; + _Py_IDENTIFIER(__sizeof__); if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:getsizeof", kwlist, &o, &dflt)) @@ -831,8 +835,7 @@ sys_getsizeof(PyObject *self, PyObject *args, PyObject *kwds) if (PyType_Ready(Py_TYPE(o)) < 0) return NULL; - method = _PyObject_LookupSpecial(o, "__sizeof__", - &str__sizeof__); + method = _PyObject_LookupSpecial(o, &PyId___sizeof__); if (method == NULL) { if (!PyErr_Occurred()) PyErr_Format(PyExc_TypeError, @@ -1177,7 +1180,6 @@ PySys_AddXOption(const wchar_t *s) PyObject *opts; PyObject *name = NULL, *value = NULL; const wchar_t *name_end; - int r; opts = get_xoptions(); if (opts == NULL) @@ -1195,7 +1197,7 @@ PySys_AddXOption(const wchar_t *s) } if (name == NULL || value == NULL) goto error; - r = PyDict_SetItem(opts, name, value); + PyDict_SetItem(opts, name, value); Py_DECREF(name); Py_DECREF(value); return; @@ -1252,21 +1254,21 @@ PyDoc_STR( "\n\ Static objects:\n\ \n\ -float_info -- a dict with information about the float implementation.\n\ -int_info -- a struct sequence with information about the int implementation.\n\ -maxsize -- the largest supported length of containers.\n\ -maxunicode -- the largest supported character\n\ builtin_module_names -- tuple of module names built into this interpreter\n\ -subversion -- subversion information of the build as tuple\n\ -version -- the version of this interpreter as a string\n\ -version_info -- version information as a named tuple\n\ -hexversion -- version information encoded as a single integer\n\ copyright -- copyright notice pertaining to this interpreter\n\ -platform -- platform identifier\n\ -executable -- absolute path of the executable binary of the Python interpreter\n\ -prefix -- prefix used to find the Python library\n\ exec_prefix -- prefix used to find the machine-specific Python library\n\ +executable -- absolute path of the executable binary of the Python interpreter\n\ +float_info -- a struct sequence with information about the float implementation.\n\ float_repr_style -- string indicating the style of repr() output for floats\n\ +hexversion -- version information encoded as a single integer\n\ +int_info -- a struct sequence with information about the int implementation.\n\ +maxsize -- the largest supported length of containers.\n\ +maxunicode -- the value of the largest Unicode codepoint\n\ +platform -- platform identifier\n\ +prefix -- prefix used to find the Python library\n\ +thread_info -- a struct sequence with information about the thread implementation.\n\ +version -- the version of this interpreter as a string\n\ +version_info -- version information as a named tuple\n\ " ) #ifdef MS_WINDOWS @@ -1305,43 +1307,6 @@ settrace() -- set the global debug tracing function\n\ ) /* end of sys_doc */ ; -/* Subversion branch and revision management */ -static int svn_initialized; -static char patchlevel_revision[50]; /* Just the number */ -static char branch[50]; -static char shortbranch[50]; -static const char *svn_revision; - -static void -svnversion_init(void) -{ - if (svn_initialized) - return; - - svn_initialized = 1; - *patchlevel_revision = '\0'; - strcpy(branch, ""); - strcpy(shortbranch, "unknown"); - svn_revision = ""; -} - -/* Return svnversion output if available. - Else return Revision of patchlevel.h if on branch. - Else return empty string */ -const char* -Py_SubversionRevision() -{ - svnversion_init(); - return svn_revision; -} - -const char* -Py_SubversionShortBranch() -{ - svnversion_init(); - return shortbranch; -} - PyDoc_STRVAR(flags__doc__, "sys.flags\n\ @@ -1352,7 +1317,6 @@ static PyTypeObject FlagsType; static PyStructSequence_Field flags_fields[] = { {"debug", "-d"}, - {"division_warning", "-Q"}, {"inspect", "-i"}, {"interactive", "-i"}, {"optimize", "-O or -OO"}, @@ -1377,9 +1341,9 @@ static PyStructSequence_Desc flags_desc = { flags__doc__, /* doc */ flags_fields, /* fields */ #ifdef RISCOS - 14 -#else 13 +#else + 12 #endif }; @@ -1397,7 +1361,6 @@ make_flags(void) PyStructSequence_SET_ITEM(seq, pos++, PyLong_FromLong(flag)) SetFlag(Py_DebugFlag); - SetFlag(Py_DivisionWarningFlag); SetFlag(Py_InspectFlag); SetFlag(Py_InteractiveFlag); SetFlag(Py_OptimizeFlag); @@ -1547,10 +1510,6 @@ _PySys_Init(void) PyUnicode_FromString(Py_GetVersion())); SET_SYS_FROM_STRING("hexversion", PyLong_FromLong(PY_VERSION_HEX)); - svnversion_init(); - SET_SYS_FROM_STRING("subversion", - Py_BuildValue("(sss)", "CPython", branch, - svn_revision)); SET_SYS_FROM_STRING("_mercurial", Py_BuildValue("(szz)", "CPython", _Py_hgidentifier(), _Py_hgversion())); @@ -1581,7 +1540,7 @@ _PySys_Init(void) SET_SYS_FROM_STRING("hash_info", get_hash_info()); SET_SYS_FROM_STRING("maxunicode", - PyLong_FromLong(PyUnicode_GetMax())); + PyLong_FromLong(0x10FFFF)); SET_SYS_FROM_STRING("builtin_module_names", list_builtin_module_names()); { @@ -1658,6 +1617,10 @@ _PySys_Init(void) PyUnicode_FromString("legacy")); #endif +#ifdef WITH_THREAD + SET_SYS_FROM_STRING("thread_info", PyThread_GetInfo()); +#endif + #undef SET_SYS_FROM_STRING if (PyErr_Occurred()) return NULL; @@ -1812,7 +1775,7 @@ sys_update_path(int argc, wchar_t **argv) the argument must be the full path anyway. */ wchar_t *ptemp; if (GetFullPathNameW(argv0, - sizeof(fullpath)/sizeof(fullpath[0]), + Py_ARRAY_LENGTH(fullpath), fullpath, &ptemp)) { argv0 = fullpath; @@ -1881,11 +1844,12 @@ sys_pyfile_write_unicode(PyObject *unicode, PyObject *file) { PyObject *writer = NULL, *args = NULL, *result = NULL; int err; + _Py_IDENTIFIER(write); if (file == NULL) return -1; - writer = PyObject_GetAttrString(file, "write"); + writer = _PyObject_GetAttrId(file, &PyId_write); if (writer == NULL) goto error; diff --git a/Python/thread.c b/Python/thread.c index d224046..e55d342 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -7,7 +7,6 @@ #include "Python.h" - #ifndef _POSIX_THREADS /* This means pthreads are not implemented in libc headers, hence the macro not present in unistd.h. But they still can be implemented as an external @@ -27,18 +26,6 @@ #ifndef _POSIX_THREADS -#ifdef __sgi -#define SGI_THREADS -#endif - -#ifdef HAVE_THREAD_H -#define SOLARIS_THREADS -#endif - -#if defined(sun) && !defined(SOLARIS_THREADS) -#define SUN_LWP -#endif - /* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then enough of the Posix threads package is implemented to support python threads. @@ -94,47 +81,21 @@ PyThread_init_thread(void) or the size specified by the THREAD_STACK_SIZE macro. */ static size_t _pythread_stacksize = 0; -#ifdef SGI_THREADS -#error SGI Irix threads are now unsupported, and code will be removed in 3.3. -#include "thread_sgi.h" -#endif - -#ifdef SOLARIS_THREADS -#include "thread_solaris.h" -#endif - -#ifdef SUN_LWP -#error SunOS lightweight processes are now unsupported, and code will be removed in 3.3. -#include "thread_lwp.h" -#endif - -#ifdef HAVE_PTH -#error GNU pth threads are now unsupported, and code will be removed in 3.3. -#include "thread_pth.h" -#undef _POSIX_THREADS -#endif - #ifdef _POSIX_THREADS +#define PYTHREAD_NAME "pthread" #include "thread_pthread.h" #endif -#ifdef C_THREADS -#error Mach C Threads are now unsupported, and code will be removed in 3.3. -#include "thread_cthread.h" -#endif - #ifdef NT_THREADS +#define PYTHREAD_NAME "nt" #include "thread_nt.h" #endif #ifdef OS2_THREADS +#define PYTHREAD_NAME "os2" #include "thread_os2.h" #endif -#ifdef PLAN9_THREADS -#include "thread_plan9.h" -#endif - /* #ifdef FOOBAR_THREADS #include "thread_foobar.h" @@ -409,3 +370,84 @@ PyThread_ReInitTLS(void) } #endif /* Py_HAVE_NATIVE_TLS */ + +PyDoc_STRVAR(threadinfo__doc__, +"sys.thread_info\n\ +\n\ +A struct sequence holding information about the thread implementation."); + +static PyStructSequence_Field threadinfo_fields[] = { + {"name", "name of the thread implementation"}, + {"lock", "name of the lock implementation"}, + {"version", "name and version of the thread library"}, + {0} +}; + +static PyStructSequence_Desc threadinfo_desc = { + "sys.thread_info", /* name */ + threadinfo__doc__, /* doc */ + threadinfo_fields, /* fields */ + 3 +}; + +static PyTypeObject ThreadInfoType; + +PyObject* +PyThread_GetInfo(void) +{ + PyObject *threadinfo, *value; + int pos = 0; +#if (defined(_POSIX_THREADS) && defined(HAVE_CONFSTR) \ + && defined(_CS_GNU_LIBPTHREAD_VERSION)) + char buffer[255]; + int len; +#endif + + if (ThreadInfoType.tp_name == 0) + PyStructSequence_InitType(&ThreadInfoType, &threadinfo_desc); + + threadinfo = PyStructSequence_New(&ThreadInfoType); + if (threadinfo == NULL) + return NULL; + + value = PyUnicode_FromString(PYTHREAD_NAME); + if (value == NULL) { + Py_DECREF(threadinfo); + return NULL; + } + PyStructSequence_SET_ITEM(threadinfo, pos++, value); + +#ifdef _POSIX_THREADS +#ifdef USE_SEMAPHORES + value = PyUnicode_FromString("semaphore"); +#else + value = PyUnicode_FromString("mutex+cond"); +#endif + if (value == NULL) { + Py_DECREF(threadinfo); + return NULL; + } +#else + Py_INCREF(Py_None); + value = Py_None; +#endif + PyStructSequence_SET_ITEM(threadinfo, pos++, value); + +#if (defined(_POSIX_THREADS) && defined(HAVE_CONFSTR) \ + && defined(_CS_GNU_LIBPTHREAD_VERSION)) + value = NULL; + len = confstr(_CS_GNU_LIBPTHREAD_VERSION, buffer, sizeof(buffer)); + if (1 < len && len < sizeof(buffer)) { + value = PyUnicode_DecodeFSDefaultAndSize(buffer, len-1); + if (value == NULL) + PyErr_Clear(); + } + if (value == NULL) +#endif + { + Py_INCREF(Py_None); + value = Py_None; + } + PyStructSequence_SET_ITEM(threadinfo, pos++, value); + return threadinfo; +} diff --git a/Python/thread_cthread.h b/Python/thread_cthread.h deleted file mode 100644 index 1b3e390..0000000 --- a/Python/thread_cthread.h +++ /dev/null @@ -1,112 +0,0 @@ - -#ifdef MACH_C_THREADS -#include <mach/cthreads.h> -#endif - -#ifdef HURD_C_THREADS -#include <cthreads.h> -#endif - -/* - * Initialization. - */ -static void -PyThread__init_thread(void) -{ -#ifndef HURD_C_THREADS - /* Roland McGrath said this should not be used since this is - done while linking to threads */ - cthread_init(); -#else -/* do nothing */ - ; -#endif -} - -/* - * Thread support. - */ -long -PyThread_start_new_thread(void (*func)(void *), void *arg) -{ - int success = 0; /* init not needed when SOLARIS_THREADS and */ - /* C_THREADS implemented properly */ - - dprintf(("PyThread_start_new_thread called\n")); - if (!initialized) - PyThread_init_thread(); - /* looks like solaris detaches the thread to never rejoin - * so well do it here - */ - cthread_detach(cthread_fork((cthread_fn_t) func, arg)); - return success < 0 ? -1 : 0; -} - -long -PyThread_get_thread_ident(void) -{ - if (!initialized) - PyThread_init_thread(); - return (long) cthread_self(); -} - -void -PyThread_exit_thread(void) -{ - dprintf(("PyThread_exit_thread called\n")); - if (!initialized) - exit(0); - cthread_exit(0); -} - -/* - * Lock support. - */ -PyThread_type_lock -PyThread_allocate_lock(void) -{ - mutex_t lock; - - dprintf(("PyThread_allocate_lock called\n")); - if (!initialized) - PyThread_init_thread(); - - lock = mutex_alloc(); - if (mutex_init(lock)) { - perror("mutex_init"); - free((void *) lock); - lock = 0; - } - dprintf(("PyThread_allocate_lock() -> %p\n", lock)); - return (PyThread_type_lock) lock; -} - -void -PyThread_free_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_free_lock(%p) called\n", lock)); - mutex_free(lock); -} - -int -PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) -{ - int success = FALSE; - - dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, waitflag)); - if (waitflag) { /* blocking */ - mutex_lock((mutex_t)lock); - success = TRUE; - } else { /* non blocking */ - success = mutex_try_lock((mutex_t)lock); - } - dprintf(("PyThread_acquire_lock(%p, %d) -> %d\n", lock, waitflag, success)); - return success; -} - -void -PyThread_release_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_release_lock(%p) called\n", lock)); - mutex_unlock((mutex_t )lock); -} diff --git a/Python/thread_lwp.h b/Python/thread_lwp.h deleted file mode 100644 index ba7b37a..0000000 --- a/Python/thread_lwp.h +++ /dev/null @@ -1,113 +0,0 @@ - -#include <stdlib.h> -#include <lwp/lwp.h> -#include <lwp/stackdep.h> - -#define STACKSIZE 1000 /* stacksize for a thread */ -#define NSTACKS 2 /* # stacks to be put in cache initially */ - -struct lock { - int lock_locked; - cv_t lock_condvar; - mon_t lock_monitor; -}; - - -/* - * Initialization. - */ -static void PyThread__init_thread(void) -{ - lwp_setstkcache(STACKSIZE, NSTACKS); -} - -/* - * Thread support. - */ - - -long PyThread_start_new_thread(void (*func)(void *), void *arg) -{ - thread_t tid; - int success; - dprintf(("PyThread_start_new_thread called\n")); - if (!initialized) - PyThread_init_thread(); - success = lwp_create(&tid, func, MINPRIO, 0, lwp_newstk(), 1, arg); - return success < 0 ? -1 : 0; -} - -long PyThread_get_thread_ident(void) -{ - thread_t tid; - if (!initialized) - PyThread_init_thread(); - if (lwp_self(&tid) < 0) - return -1; - return tid.thread_id; -} - -void PyThread_exit_thread(void) -{ - dprintf(("PyThread_exit_thread called\n")); - if (!initialized) - exit(0); - lwp_destroy(SELF); -} - -/* - * Lock support. - */ -PyThread_type_lock PyThread_allocate_lock(void) -{ - struct lock *lock; - extern char *malloc(size_t); - - dprintf(("PyThread_allocate_lock called\n")); - if (!initialized) - PyThread_init_thread(); - - lock = (struct lock *) malloc(sizeof(struct lock)); - lock->lock_locked = 0; - (void) mon_create(&lock->lock_monitor); - (void) cv_create(&lock->lock_condvar, lock->lock_monitor); - dprintf(("PyThread_allocate_lock() -> %p\n", lock)); - return (PyThread_type_lock) lock; -} - -void PyThread_free_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_free_lock(%p) called\n", lock)); - mon_destroy(((struct lock *) lock)->lock_monitor); - free((char *) lock); -} - -int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) -{ - int success; - - dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, waitflag)); - success = 0; - - (void) mon_enter(((struct lock *) lock)->lock_monitor); - if (waitflag) - while (((struct lock *) lock)->lock_locked) - cv_wait(((struct lock *) lock)->lock_condvar); - if (!((struct lock *) lock)->lock_locked) { - success = 1; - ((struct lock *) lock)->lock_locked = 1; - } - cv_broadcast(((struct lock *) lock)->lock_condvar); - mon_exit(((struct lock *) lock)->lock_monitor); - dprintf(("PyThread_acquire_lock(%p, %d) -> %d\n", lock, waitflag, success)); - return success; -} - -void PyThread_release_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_release_lock(%p) called\n", lock)); - (void) mon_enter(((struct lock *) lock)->lock_monitor); - ((struct lock *) lock)->lock_locked = 0; - cv_broadcast(((struct lock *) lock)->lock_condvar); - mon_exit(((struct lock *) lock)->lock_monitor); -} diff --git a/Python/thread_pth.h b/Python/thread_pth.h index 82a00e7..5704266 100644 --- a/Python/thread_pth.h +++ b/Python/thread_pth.h @@ -69,9 +69,8 @@ long PyThread_get_thread_ident(void) volatile pth_t threadid; if (!initialized) PyThread_init_thread(); - /* Jump through some hoops for Alpha OSF/1 */ threadid = pth_self(); - return (long) *(long *) &threadid; + return (long) threadid; } void PyThread_exit_thread(void) diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 3efccf6..4b61e18 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -144,7 +144,7 @@ typedef struct { * Initialization. */ -#ifdef _HAVE_BSDI +#if defined(_HAVE_BSDI) static void _noop(void) { @@ -240,8 +240,7 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) hosed" because: - It does not guarantee the promise that a non-zero integer is returned. - The cast to long is inherently unsafe. - - It is not clear that the 'volatile' (for AIX?) and ugly casting in the - latter return statement (for Alpha OSF/1) are any longer necessary. + - It is not clear that the 'volatile' (for AIX?) are any longer necessary. */ long PyThread_get_thread_ident(void) @@ -249,13 +248,8 @@ PyThread_get_thread_ident(void) volatile pthread_t threadid; if (!initialized) PyThread_init_thread(); - /* Jump through some hoops for Alpha OSF/1 */ threadid = pthread_self(); -#if SIZEOF_PTHREAD_T <= SIZEOF_LONG return (long) threadid; -#else - return (long) *(long *) &threadid; -#endif } void diff --git a/Python/thread_sgi.h b/Python/thread_sgi.h deleted file mode 100644 index 771ab2c..0000000 --- a/Python/thread_sgi.h +++ /dev/null @@ -1,259 +0,0 @@ - -#include <stdlib.h> -#include <stdio.h> -#include <signal.h> -#include <sys/types.h> -#include <sys/wait.h> -#include <sys/prctl.h> -#include <ulocks.h> -#include <errno.h> - -#define HDR_SIZE 2680 /* sizeof(ushdr_t) */ -#define MAXPROC 100 /* max # of threads that can be started */ - -static usptr_t *shared_arena; -static ulock_t count_lock; /* protection for some variables */ -static ulock_t wait_lock; /* lock used to wait for other threads */ -static int waiting_for_threads; /* protected by count_lock */ -static int nthreads; /* protected by count_lock */ -static int exit_status; -static int exiting; /* we're already exiting (for maybe_exit) */ -static pid_t my_pid; /* PID of main thread */ -static struct pidlist { - pid_t parent; - pid_t child; -} pidlist[MAXPROC]; /* PIDs of other threads; protected by count_lock */ -static int maxpidindex; /* # of PIDs in pidlist */ -/* - * Initialization. - */ -static void PyThread__init_thread(void) -{ -#ifdef USE_DL - long addr, size; -#endif /* USE_DL */ - - -#ifdef USE_DL - if ((size = usconfig(CONF_INITSIZE, 64*1024)) < 0) - perror("usconfig - CONF_INITSIZE (check)"); - if (usconfig(CONF_INITSIZE, size) < 0) - perror("usconfig - CONF_INITSIZE (reset)"); - addr = (long) dl_getrange(size + HDR_SIZE); - dprintf(("trying to use addr %p-%p for shared arena\n", addr, addr+size)); - errno = 0; - if ((addr = usconfig(CONF_ATTACHADDR, addr)) < 0 && errno != 0) - perror("usconfig - CONF_ATTACHADDR (set)"); -#endif /* USE_DL */ - if (usconfig(CONF_INITUSERS, 16) < 0) - perror("usconfig - CONF_INITUSERS"); - my_pid = getpid(); /* so that we know which is the main thread */ - if (usconfig(CONF_ARENATYPE, US_SHAREDONLY) < 0) - perror("usconfig - CONF_ARENATYPE"); - usconfig(CONF_LOCKTYPE, US_DEBUG); /* XXX */ -#ifdef Py_DEBUG - if (thread_debug & 4) - usconfig(CONF_LOCKTYPE, US_DEBUGPLUS); - else if (thread_debug & 2) - usconfig(CONF_LOCKTYPE, US_DEBUG); -#endif /* Py_DEBUG */ - if ((shared_arena = usinit(tmpnam(0))) == 0) - perror("usinit"); -#ifdef USE_DL - if (usconfig(CONF_ATTACHADDR, addr) < 0) /* reset address */ - perror("usconfig - CONF_ATTACHADDR (reset)"); -#endif /* USE_DL */ - if ((count_lock = usnewlock(shared_arena)) == NULL) - perror("usnewlock (count_lock)"); - (void) usinitlock(count_lock); - if ((wait_lock = usnewlock(shared_arena)) == NULL) - perror("usnewlock (wait_lock)"); - dprintf(("arena start: %p, arena size: %ld\n", shared_arena, (long) usconfig(CONF_GETSIZE, shared_arena))); -} - -/* - * Thread support. - */ - -static void clean_threads(void) -{ - int i, j; - pid_t mypid, pid; - - /* clean up any exited threads */ - mypid = getpid(); - i = 0; - while (i < maxpidindex) { - if (pidlist[i].parent == mypid && (pid = pidlist[i].child) > 0) { - pid = waitpid(pid, 0, WNOHANG); - if (pid > 0) { - /* a thread has exited */ - pidlist[i] = pidlist[--maxpidindex]; - /* remove references to children of dead proc */ - for (j = 0; j < maxpidindex; j++) - if (pidlist[j].parent == pid) - pidlist[j].child = -1; - continue; /* don't increment i */ - } - } - i++; - } - /* clean up the list */ - i = 0; - while (i < maxpidindex) { - if (pidlist[i].child == -1) { - pidlist[i] = pidlist[--maxpidindex]; - continue; /* don't increment i */ - } - i++; - } -} - -long PyThread_start_new_thread(void (*func)(void *), void *arg) -{ -#ifdef USE_DL - long addr, size; - static int local_initialized = 0; -#endif /* USE_DL */ - int success = 0; /* init not needed when SOLARIS_THREADS and */ - /* C_THREADS implemented properly */ - - dprintf(("PyThread_start_new_thread called\n")); - if (!initialized) - PyThread_init_thread(); - switch (ussetlock(count_lock)) { - case 0: return 0; - case -1: perror("ussetlock (count_lock)"); - } - if (maxpidindex >= MAXPROC) - success = -1; - else { -#ifdef USE_DL - if (!local_initialized) { - if ((size = usconfig(CONF_INITSIZE, 64*1024)) < 0) - perror("usconfig - CONF_INITSIZE (check)"); - if (usconfig(CONF_INITSIZE, size) < 0) - perror("usconfig - CONF_INITSIZE (reset)"); - addr = (long) dl_getrange(size + HDR_SIZE); - dprintf(("trying to use addr %p-%p for sproc\n", - addr, addr+size)); - errno = 0; - if ((addr = usconfig(CONF_ATTACHADDR, addr)) < 0 && - errno != 0) - perror("usconfig - CONF_ATTACHADDR (set)"); - } -#endif /* USE_DL */ - clean_threads(); - if ((success = sproc(func, PR_SALL, arg)) < 0) - perror("sproc"); -#ifdef USE_DL - if (!local_initialized) { - if (usconfig(CONF_ATTACHADDR, addr) < 0) - /* reset address */ - perror("usconfig - CONF_ATTACHADDR (reset)"); - local_initialized = 1; - } -#endif /* USE_DL */ - if (success >= 0) { - nthreads++; - pidlist[maxpidindex].parent = getpid(); - pidlist[maxpidindex++].child = success; - dprintf(("pidlist[%d] = %d\n", - maxpidindex-1, success)); - } - } - if (usunsetlock(count_lock) < 0) - perror("usunsetlock (count_lock)"); - return success; -} - -long PyThread_get_thread_ident(void) -{ - return getpid(); -} - -void PyThread_exit_thread(void) -{ - dprintf(("PyThread_exit_thread called\n")); - if (!initialized) - exit(0); - if (ussetlock(count_lock) < 0) - perror("ussetlock (count_lock)"); - nthreads--; - if (getpid() == my_pid) { - /* main thread; wait for other threads to exit */ - exiting = 1; - waiting_for_threads = 1; - if (ussetlock(wait_lock) < 0) - perror("ussetlock (wait_lock)"); - for (;;) { - if (nthreads < 0) { - dprintf(("really exit (%d)\n", exit_status)); - exit(exit_status); - } - if (usunsetlock(count_lock) < 0) - perror("usunsetlock (count_lock)"); - dprintf(("waiting for other threads (%d)\n", nthreads)); - if (ussetlock(wait_lock) < 0) - perror("ussetlock (wait_lock)"); - if (ussetlock(count_lock) < 0) - perror("ussetlock (count_lock)"); - } - } - /* not the main thread */ - if (waiting_for_threads) { - dprintf(("main thread is waiting\n")); - if (usunsetlock(wait_lock) < 0) - perror("usunsetlock (wait_lock)"); - } - if (usunsetlock(count_lock) < 0) - perror("usunsetlock (count_lock)"); - _exit(0); -} - -/* - * Lock support. - */ -PyThread_type_lock PyThread_allocate_lock(void) -{ - ulock_t lock; - - dprintf(("PyThread_allocate_lock called\n")); - if (!initialized) - PyThread_init_thread(); - - if ((lock = usnewlock(shared_arena)) == NULL) - perror("usnewlock"); - (void) usinitlock(lock); - dprintf(("PyThread_allocate_lock() -> %p\n", lock)); - return (PyThread_type_lock) lock; -} - -void PyThread_free_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_free_lock(%p) called\n", lock)); - usfreelock((ulock_t) lock, shared_arena); -} - -int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) -{ - int success; - - dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, waitflag)); - errno = 0; /* clear it just in case */ - if (waitflag) - success = ussetlock((ulock_t) lock); - else - success = uscsetlock((ulock_t) lock, 1); /* Try it once */ - if (success < 0) - perror(waitflag ? "ussetlock" : "uscsetlock"); - dprintf(("PyThread_acquire_lock(%p, %d) -> %d\n", lock, waitflag, success)); - return success; -} - -void PyThread_release_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_release_lock(%p) called\n", lock)); - if (usunsetlock((ulock_t) lock) < 0) - perror("usunsetlock"); -} diff --git a/Python/thread_solaris.h b/Python/thread_solaris.h deleted file mode 100644 index 1ce1cfc..0000000 --- a/Python/thread_solaris.h +++ /dev/null @@ -1,130 +0,0 @@ - -#include <stdlib.h> -#include <stdio.h> -#include <errno.h> -#include </usr/include/thread.h> -#undef _POSIX_THREADS - - -/* - * Initialization. - */ -static void PyThread__init_thread(void) -{ -} - -/* - * Thread support. - */ -struct func_arg { - void (*func)(void *); - void *arg; -}; - -static void * -new_func(void *funcarg) -{ - void (*func)(void *); - void *arg; - - func = ((struct func_arg *) funcarg)->func; - arg = ((struct func_arg *) funcarg)->arg; - free(funcarg); - (*func)(arg); - return 0; -} - - -long -PyThread_start_new_thread(void (*func)(void *), void *arg) -{ - thread_t tid; - struct func_arg *funcarg; - - dprintf(("PyThread_start_new_thread called\n")); - if (!initialized) - PyThread_init_thread(); - funcarg = (struct func_arg *) malloc(sizeof(struct func_arg)); - funcarg->func = func; - funcarg->arg = arg; - if (thr_create(0, 0, new_func, funcarg, - THR_DETACHED | THR_NEW_LWP, &tid)) { - perror("thr_create"); - free((void *) funcarg); - return -1; - } - return tid; -} - -long -PyThread_get_thread_ident(void) -{ - if (!initialized) - PyThread_init_thread(); - return thr_self(); -} - -void -PyThread_exit_thread(void) -{ - dprintf(("PyThread_exit_thread called\n")); - if (!initialized) - exit(0); - thr_exit(0); -} - -/* - * Lock support. - */ -PyThread_type_lock -PyThread_allocate_lock(void) -{ - mutex_t *lock; - - dprintf(("PyThread_allocate_lock called\n")); - if (!initialized) - PyThread_init_thread(); - - lock = (mutex_t *) malloc(sizeof(mutex_t)); - if (mutex_init(lock, USYNC_THREAD, 0)) { - perror("mutex_init"); - free((void *) lock); - lock = 0; - } - dprintf(("PyThread_allocate_lock() -> %p\n", lock)); - return (PyThread_type_lock) lock; -} - -void -PyThread_free_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_free_lock(%p) called\n", lock)); - mutex_destroy((mutex_t *) lock); - free((void *) lock); -} - -int -PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) -{ - int success; - - dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, waitflag)); - if (waitflag) - success = mutex_lock((mutex_t *) lock); - else - success = mutex_trylock((mutex_t *) lock); - if (success < 0) - perror(waitflag ? "mutex_lock" : "mutex_trylock"); - else - success = !success; /* solaris does it the other way round */ - dprintf(("PyThread_acquire_lock(%p, %d) -> %d\n", lock, waitflag, success)); - return success; -} - -void -PyThread_release_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_release_lock(%p) called\n", lock)); - if (mutex_unlock((mutex_t *) lock)) - perror("mutex_unlock"); -} diff --git a/Python/thread_wince.h b/Python/thread_wince.h deleted file mode 100644 index 51ddc02..0000000 --- a/Python/thread_wince.h +++ /dev/null @@ -1,136 +0,0 @@ - -/* This code implemented by Mark Hammond (MHammond@skippinet.com.au) */ - -#include <windows.h> -#include <limits.h> -#include <pydebug.h> - -long PyThread_get_thread_ident(void); - -/* - * Change all headers to pure ANSI as no one will use K&R style on an - * NT - */ - -/* - * Initialization of the C package, should not be needed. - */ -static void PyThread__init_thread(void) -{ -} - -/* - * Thread support. - */ -long PyThread_start_new_thread(void (*func)(void *), void *arg) -{ - long rv; - int success = -1; - - dprintf(("%ld: PyThread_start_new_thread called\n", PyThread_get_thread_ident())); - if (!initialized) - PyThread_init_thread(); - - rv = _beginthread(func, 0, arg); /* use default stack size */ - - if (rv != -1) { - success = 0; - dprintf(("%ld: PyThread_start_new_thread succeeded:\n", PyThread_get_thread_ident())); - } - - return success; -} - -/* - * Return the thread Id instead of an handle. The Id is said to uniquely identify the - * thread in the system - */ -long PyThread_get_thread_ident(void) -{ - if (!initialized) - PyThread_init_thread(); - - return GetCurrentThreadId(); -} - -void PyThread_exit_thread(void) -{ - dprintf(("%ld: PyThread_exit_thread called\n", PyThread_get_thread_ident())); - if (!initialized) - exit(0); - _endthread(); -} - -/* - * Lock support. It has to be implemented using Mutexes, as - * CE doesnt support semaphores. Therefore we use some hacks to - * simulate the non reentrant requirements of Python locks - */ -PyThread_type_lock PyThread_allocate_lock(void) -{ - HANDLE aLock; - - dprintf(("PyThread_allocate_lock called\n")); - if (!initialized) - PyThread_init_thread(); - - aLock = CreateEvent(NULL, /* Security attributes */ - 0, /* Manual-Reset */ - 1, /* Is initially signalled */ - NULL); /* Name of event */ - - dprintf(("%ld: PyThread_allocate_lock() -> %p\n", PyThread_get_thread_ident(), aLock)); - - return (PyThread_type_lock) aLock; -} - -void PyThread_free_lock(PyThread_type_lock aLock) -{ - dprintf(("%ld: PyThread_free_lock(%p) called\n", PyThread_get_thread_ident(),aLock)); - - CloseHandle(aLock); -} - -/* - * Return 1 on success if the lock was acquired - * - * and 0 if the lock was not acquired. This means a 0 is returned - * if the lock has already been acquired by this thread! - */ -int PyThread_acquire_lock(PyThread_type_lock aLock, int waitflag) -{ - int success = 1; - DWORD waitResult; - - dprintf(("%ld: PyThread_acquire_lock(%p, %d) called\n", PyThread_get_thread_ident(),aLock, waitflag)); - -#ifndef DEBUG - waitResult = WaitForSingleObject(aLock, (waitflag ? INFINITE : 0)); -#else - /* To aid in debugging, we regularly wake up. This allows us to - break into the debugger */ - while (TRUE) { - waitResult = WaitForSingleObject(aLock, waitflag ? 3000 : 0); - if (waitflag==0 || (waitflag && waitResult == WAIT_OBJECT_0)) - break; - } -#endif - - if (waitResult != WAIT_OBJECT_0) { - success = 0; /* We failed */ - } - - dprintf(("%ld: PyThread_acquire_lock(%p, %d) -> %d\n", PyThread_get_thread_ident(),aLock, waitflag, success)); - - return success; -} - -void PyThread_release_lock(PyThread_type_lock aLock) -{ - dprintf(("%ld: PyThread_release_lock(%p) called\n", PyThread_get_thread_ident(),aLock)); - - if (!SetEvent(aLock)) - dprintf(("%ld: Could not PyThread_release_lock(%p) error: %l\n", PyThread_get_thread_ident(), aLock, GetLastError())); -} - - diff --git a/Python/traceback.c b/Python/traceback.c index 59bb3f0..c8b3ee1 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -13,8 +13,13 @@ #define OFF(x) offsetof(PyTracebackObject, x) -/* Method from Parser/tokenizer.c */ -extern char * PyTokenizer_FindEncoding(int); +#define PUTS(fd, str) write(fd, str, strlen(str)) +#define MAX_STRING_LENGTH 100 +#define MAX_FRAME_DEPTH 100 +#define MAX_NTHREADS 100 + +/* Function from Parser/tokenizer.c */ +extern char * PyTokenizer_FindEncodingFilename(int, PyObject *); static PyObject * tb_dir(PyTracebackObject *self) @@ -147,6 +152,7 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * const char* filepath; Py_ssize_t len; PyObject* result; + _Py_IDENTIFIER(open); filebytes = PyUnicode_EncodeFSDefault(filename); if (filebytes == NULL) { @@ -194,7 +200,7 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * namebuf[len++] = SEP; strcpy(namebuf+len, tail); - binary = PyObject_CallMethod(io, "open", "ss", namebuf, "rb"); + binary = _PyObject_CallMethodId(io, &PyId_open, "ss", namebuf, "rb"); if (binary != NULL) { result = binary; goto finally; @@ -224,8 +230,11 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent) PyObject *lineobj = NULL; PyObject *res; char buf[MAXPATHLEN+1]; - Py_UNICODE *u, *p; - Py_ssize_t len; + int kind; + void *data; + _Py_IDENTIFIER(close); + _Py_IDENTIFIER(open); + _Py_IDENTIFIER(TextIOWrapper); /* open the file */ if (filename == NULL) @@ -234,7 +243,7 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent) io = PyImport_ImportModuleNoBlock("io"); if (io == NULL) return -1; - binary = PyObject_CallMethod(io, "open", "Os", filename, "rb"); + binary = _PyObject_CallMethodId(io, &PyId_open, "Os", filename, "rb"); if (binary == NULL) { binary = _Py_FindSourceFile(filename, buf, sizeof(buf), io); @@ -246,10 +255,10 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent) /* use the right encoding to decode the file as unicode */ fd = PyObject_AsFileDescriptor(binary); - found_encoding = PyTokenizer_FindEncoding(fd); + found_encoding = PyTokenizer_FindEncodingFilename(fd, filename); encoding = (found_encoding != NULL) ? found_encoding : "utf-8"; lseek(fd, 0, 0); /* Reset position */ - fob = PyObject_CallMethod(io, "TextIOWrapper", "Os", binary, encoding); + fob = _PyObject_CallMethodId(io, &PyId_TextIOWrapper, "Os", binary, encoding); Py_DECREF(io); Py_DECREF(binary); PyMem_FREE(found_encoding); @@ -268,7 +277,7 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent) break; } } - res = PyObject_CallMethod(fob, "close", ""); + res = _PyObject_CallMethodId(fob, &PyId_close, ""); if (res) Py_DECREF(res); else @@ -280,13 +289,16 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent) } /* remove the indentation of the line */ - u = PyUnicode_AS_UNICODE(lineobj); - len = PyUnicode_GET_SIZE(lineobj); - for (p=u; *p == ' ' || *p == '\t' || *p == '\014'; p++) - len--; - if (u != p) { + kind = PyUnicode_KIND(lineobj); + data = PyUnicode_DATA(lineobj); + for (i=0; i < PyUnicode_GET_LENGTH(lineobj); i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch != ' ' && ch != '\t' && ch != '\014') + break; + } + if (i) { PyObject *truncated; - truncated = PyUnicode_FromUnicode(p, len); + truncated = PyUnicode_Substring(lineobj, i, PyUnicode_GET_LENGTH(lineobj)); if (truncated) { Py_DECREF(lineobj); lineobj = truncated; @@ -402,3 +414,250 @@ PyTraceBack_Print(PyObject *v, PyObject *f) err = tb_printinternal((PyTracebackObject *)v, f, limit); return err; } + +/* Reverse a string. For example, "abcd" becomes "dcba". + + This function is signal safe. */ + +static void +reverse_string(char *text, const size_t len) +{ + char tmp; + size_t i, j; + if (len == 0) + return; + for (i=0, j=len-1; i < j; i++, j--) { + tmp = text[i]; + text[i] = text[j]; + text[j] = tmp; + } +} + +/* Format an integer in range [0; 999999] to decimal, + and write it into the file fd. + + This function is signal safe. */ + +static void +dump_decimal(int fd, int value) +{ + char buffer[7]; + int len; + if (value < 0 || 999999 < value) + return; + len = 0; + do { + buffer[len] = '0' + (value % 10); + value /= 10; + len++; + } while (value); + reverse_string(buffer, len); + write(fd, buffer, len); +} + +/* Format an integer in range [0; 0xffffffff] to hexdecimal of 'width' digits, + and write it into the file fd. + + This function is signal safe. */ + +static void +dump_hexadecimal(int width, unsigned long value, int fd) +{ + int len; + char buffer[sizeof(unsigned long) * 2 + 1]; + len = 0; + do { + buffer[len] = Py_hexdigits[value & 15]; + value >>= 4; + len++; + } while (len < width || value); + reverse_string(buffer, len); + write(fd, buffer, len); +} + +/* Write an unicode object into the file fd using ascii+backslashreplace. + + This function is signal safe. */ + +static void +dump_ascii(int fd, PyObject *text) +{ + PyASCIIObject *ascii = (PyASCIIObject *)text; + Py_ssize_t i, size; + int truncated; + int kind; + void *data = NULL; + wchar_t *wstr = NULL; + Py_UCS4 ch; + + size = ascii->length; + kind = ascii->state.kind; + if (ascii->state.compact) { + if (ascii->state.ascii) + data = ((PyASCIIObject*)text) + 1; + else + data = ((PyCompactUnicodeObject*)text) + 1; + } + else if (kind != PyUnicode_WCHAR_KIND) { + data = ((PyUnicodeObject *)text)->data.any; + if (data == NULL) + return; + } + else { + wstr = ((PyASCIIObject *)text)->wstr; + if (wstr == NULL) + return; + size = ((PyCompactUnicodeObject *)text)->wstr_length; + } + + if (MAX_STRING_LENGTH < size) { + size = MAX_STRING_LENGTH; + truncated = 1; + } + else + truncated = 0; + + for (i=0; i < size; i++) { + if (kind != PyUnicode_WCHAR_KIND) + ch = PyUnicode_READ(kind, data, i); + else + ch = wstr[i]; + if (ch < 128) { + char c = (char)ch; + write(fd, &c, 1); + } + else if (ch < 0xff) { + PUTS(fd, "\\x"); + dump_hexadecimal(2, ch, fd); + } + else if (ch < 0xffff) { + PUTS(fd, "\\u"); + dump_hexadecimal(4, ch, fd); + } + else { + PUTS(fd, "\\U"); + dump_hexadecimal(8, ch, fd); + } + } + if (truncated) + PUTS(fd, "..."); +} + +/* Write a frame into the file fd: "File "xxx", line xxx in xxx". + + This function is signal safe. */ + +static void +dump_frame(int fd, PyFrameObject *frame) +{ + PyCodeObject *code; + int lineno; + + code = frame->f_code; + PUTS(fd, " File "); + if (code != NULL && code->co_filename != NULL + && PyUnicode_Check(code->co_filename)) + { + write(fd, "\"", 1); + dump_ascii(fd, code->co_filename); + write(fd, "\"", 1); + } else { + PUTS(fd, "???"); + } + + /* PyFrame_GetLineNumber() was introduced in Python 2.7.0 and 3.2.0 */ + lineno = PyCode_Addr2Line(code, frame->f_lasti); + PUTS(fd, ", line "); + dump_decimal(fd, lineno); + PUTS(fd, " in "); + + if (code != NULL && code->co_name != NULL + && PyUnicode_Check(code->co_name)) + dump_ascii(fd, code->co_name); + else + PUTS(fd, "???"); + + write(fd, "\n", 1); +} + +static void +dump_traceback(int fd, PyThreadState *tstate, int write_header) +{ + PyFrameObject *frame; + unsigned int depth; + + if (write_header) + PUTS(fd, "Traceback (most recent call first):\n"); + + frame = _PyThreadState_GetFrame(tstate); + if (frame == NULL) + return; + + depth = 0; + while (frame != NULL) { + if (MAX_FRAME_DEPTH <= depth) { + PUTS(fd, " ...\n"); + break; + } + if (!PyFrame_Check(frame)) + break; + dump_frame(fd, frame); + frame = frame->f_back; + depth++; + } +} + +void +_Py_DumpTraceback(int fd, PyThreadState *tstate) +{ + dump_traceback(fd, tstate, 1); +} + +/* Write the thread identifier into the file 'fd': "Current thread 0xHHHH:\" if + is_current is true, "Thread 0xHHHH:\n" otherwise. + + This function is signal safe. */ + +static void +write_thread_id(int fd, PyThreadState *tstate, int is_current) +{ + if (is_current) + PUTS(fd, "Current thread 0x"); + else + PUTS(fd, "Thread 0x"); + dump_hexadecimal(sizeof(long)*2, (unsigned long)tstate->thread_id, fd); + PUTS(fd, ":\n"); +} + +const char* +_Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, + PyThreadState *current_thread) +{ + PyThreadState *tstate; + unsigned int nthreads; + + /* Get the current interpreter from the current thread */ + tstate = PyInterpreterState_ThreadHead(interp); + if (tstate == NULL) + return "unable to get the thread head state"; + + /* Dump the traceback of each thread */ + tstate = PyInterpreterState_ThreadHead(interp); + nthreads = 0; + do + { + if (nthreads != 0) + write(fd, "\n", 1); + if (nthreads >= MAX_NTHREADS) { + PUTS(fd, "...\n"); + break; + } + write_thread_id(fd, tstate, tstate == current_thread); + dump_traceback(fd, tstate, 0); + tstate = PyThreadState_Next(tstate); + nthreads++; + } while (tstate != NULL); + + return NULL; +} + |