diff options
Diffstat (limited to 'Python/ceval.c')
| -rw-r--r-- | Python/ceval.c | 795 |
1 files changed, 487 insertions, 308 deletions
diff --git a/Python/ceval.c b/Python/ceval.c index 8e396fd..bd0cbe7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -113,7 +113,7 @@ static PyObject * call_function(PyObject ***, int, uint64*, uint64*); #else static PyObject * call_function(PyObject ***, int); #endif -static PyObject * fast_function(PyObject *, PyObject ***, int, int, int); +static PyObject * fast_function(PyObject *, PyObject **, int, int, int); static PyObject * do_call(PyObject *, PyObject ***, int, int); static PyObject * ext_do_call(PyObject *, PyObject ***, int, int, int); static PyObject * update_keyword_args(PyObject *, int, PyObject ***, @@ -125,7 +125,7 @@ static PyObject * load_args(PyObject ***, int); #ifdef LLTRACE static int lltrace; -static int prtrace(PyObject *, char *); +static int prtrace(PyObject *, const char *); #endif static int call_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, @@ -139,12 +139,13 @@ static int maybe_call_line_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, int *, int *, int *); static PyObject * cmp_outcome(int, PyObject *, PyObject *); +static PyObject * import_name(PyFrameObject *, PyObject *, PyObject *, PyObject *); static PyObject * import_from(PyObject *, PyObject *); static int import_all_from(PyObject *, PyObject *); static void format_exc_check_arg(PyObject *, const char *, PyObject *); static void format_exc_unbound(PyCodeObject *co, int oparg); static PyObject * unicode_concatenate(PyObject *, PyObject *, - PyFrameObject *, unsigned char *); + PyFrameObject *, const unsigned short *); static PyObject * special_lookup(PyObject *, _Py_Identifier *); #define NAME_ERROR_MSG \ @@ -800,7 +801,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) int lastopcode = 0; #endif PyObject **stack_pointer; /* Next free slot in value stack */ - unsigned char *next_instr; + const unsigned short *next_instr; int opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ enum why_code why; /* Reason for block stack unwind */ @@ -818,7 +819,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) time it is tested. */ int instr_ub = -1, instr_lb = 0, instr_prev = -1; - unsigned char *first_instr; + const unsigned short *first_instr; PyObject *names; PyObject *consts; @@ -886,24 +887,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* Import the static jump table */ #include "opcode_targets.h" -/* This macro is used when several opcodes defer to the same implementation - (e.g. SETUP_LOOP, SETUP_FINALLY) */ -#define TARGET_WITH_IMPL(op, impl) \ - TARGET_##op: \ - opcode = op; \ - if (HAS_ARG(op)) \ - oparg = NEXTARG(); \ - case op: \ - goto impl; \ - #define TARGET(op) \ TARGET_##op: \ - opcode = op; \ - if (HAS_ARG(op)) \ - oparg = NEXTARG(); \ case op: - #define DISPATCH() \ { \ if (!_Py_atomic_load_relaxed(&eval_breaker)) { \ @@ -917,7 +904,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { \ if (!lltrace && !_Py_TracingPossible) { \ f->f_lasti = INSTR_OFFSET(); \ - goto *opcode_targets[*next_instr++]; \ + NEXTOPARG(); \ + goto *opcode_targets[opcode]; \ } \ goto fast_next_opcode; \ } @@ -926,7 +914,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { \ if (!_Py_TracingPossible) { \ f->f_lasti = INSTR_OFFSET(); \ - goto *opcode_targets[*next_instr++]; \ + NEXTOPARG(); \ + goto *opcode_targets[opcode]; \ } \ goto fast_next_opcode; \ } @@ -935,10 +924,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #else #define TARGET(op) \ case op: -#define TARGET_WITH_IMPL(op, impl) \ - /* silence compiler warnings about `impl` unused */ \ - if (0) goto impl; \ - case op: + #define DISPATCH() continue #define FAST_DISPATCH() goto fast_next_opcode #endif @@ -994,28 +980,38 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* Code access macros */ -#define INSTR_OFFSET() ((int)(next_instr - first_instr)) -#define NEXTOP() (*next_instr++) -#define NEXTARG() (next_instr += 2, (next_instr[-1]<<8) + next_instr[-2]) -#define PEEKARG() ((next_instr[2]<<8) + next_instr[1]) -#define JUMPTO(x) (next_instr = first_instr + (x)) -#define JUMPBY(x) (next_instr += (x)) +#ifdef WORDS_BIGENDIAN + #define OPCODE(word) ((word) >> 8) + #define OPARG(word) ((word) & 255) +#else + #define OPCODE(word) ((word) & 255) + #define OPARG(word) ((word) >> 8) +#endif +/* The integer overflow is checked by an assertion below. */ +#define INSTR_OFFSET() (2*(int)(next_instr - first_instr)) +#define NEXTOPARG() do { \ + unsigned short word = *next_instr; \ + opcode = OPCODE(word); \ + oparg = OPARG(word); \ + next_instr++; \ + } while (0) +#define JUMPTO(x) (next_instr = first_instr + (x)/2) +#define JUMPBY(x) (next_instr += (x)/2) /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to predict the second code when the first is run. For example, - COMPARE_OP is often followed by JUMP_IF_FALSE or JUMP_IF_TRUE. And, - those opcodes are often followed by a POP_TOP. + COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE. Verifying the prediction costs a single high-speed test of a register variable against a constant. If the pairing was good, then the processor's own internal branch predication has a high likelihood of success, resulting in a nearly zero-overhead transition to the next opcode. A successful prediction saves a trip through the eval-loop - including its two unpredictable branches, the HAS_ARG test and the - switch-case. Combined with the processor's internal branch prediction, - a successful PREDICT has the effect of making the two opcodes run as if - they were a single new opcode with the bodies combined. + including its unpredictable switch-case branch. Combined with the + processor's internal branch prediction, a successful PREDICT has the + effect of making the two opcodes run as if they were a single new opcode + with the bodies combined. If collecting opcode statistics, your choices are to either keep the predictions turned-on and interpret the results as if some opcodes @@ -1030,13 +1026,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #if defined(DYNAMIC_EXECUTION_PROFILE) || USE_COMPUTED_GOTOS #define PREDICT(op) if (0) goto PRED_##op -#define PREDICTED(op) PRED_##op: -#define PREDICTED_WITH_ARG(op) PRED_##op: #else -#define PREDICT(op) if (*next_instr == op) goto PRED_##op -#define PREDICTED(op) PRED_##op: next_instr++ -#define PREDICTED_WITH_ARG(op) PRED_##op: oparg = PEEKARG(); next_instr += 3 +#define PREDICT(op) \ + do{ \ + unsigned short word = *next_instr; \ + opcode = OPCODE(word); \ + if (opcode == op){ \ + oparg = OPARG(word); \ + next_instr++; \ + goto PRED_##op; \ + } \ + } while(0) #endif +#define PREDICTED(op) PRED_##op: /* Stack manipulation macros */ @@ -1100,7 +1102,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } #define UNWIND_EXCEPT_HANDLER(b) \ - { \ + do { \ PyObject *type, *value, *traceback; \ assert(STACK_LEVEL() >= (b)->b_level + 3); \ while (STACK_LEVEL() > (b)->b_level + 3) { \ @@ -1116,7 +1118,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_XDECREF(type); \ Py_XDECREF(value); \ Py_XDECREF(traceback); \ - } + } while(0) /* Start of code */ @@ -1165,16 +1167,16 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) consts = co->co_consts; fastlocals = f->f_localsplus; freevars = f->f_localsplus + co->co_nlocals; - first_instr = (unsigned char*) PyBytes_AS_STRING(co->co_code); - /* An explanation is in order for the next line. - - f->f_lasti now refers to the index of the last instruction - executed. You might think this was obvious from the name, but - this wasn't always true before 2.3! PyFrame_New now sets - f->f_lasti to -1 (i.e. the index *before* the first instruction) - and YIELD_VALUE doesn't fiddle with f_lasti any more. So this - does work. Promise. - YIELD_FROM sets f_lasti to itself, in order to repeated yield + assert(PyBytes_Check(co->co_code)); + assert(PyBytes_GET_SIZE(co->co_code) <= INT_MAX); + assert(PyBytes_GET_SIZE(co->co_code) % 2 == 0); + assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(co->co_code), 2)); + first_instr = (unsigned short*) PyBytes_AS_STRING(co->co_code); + /* + f->f_lasti refers to the index of the last instruction, + unless it's -1 in which case next_instr should be first_instr. + + YIELD_FROM sets f_lasti to itself, in order to repeatedly yield multiple values. When the PREDICT() macros are enabled, some opcode pairs follow in @@ -1183,9 +1185,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) were a single new opcode; accordingly,f->f_lasti will point to the first code in the pair (for instance, GET_ITER followed by FOR_ITER is effectively a single opcode and f->f_lasti will point - at to the beginning of the combined pair.) + to the beginning of the combined pair.) */ - next_instr = first_instr + f->f_lasti + 1; + next_instr = first_instr; + if (f->f_lasti >= 0) { + assert(f->f_lasti % 2 == 0); + next_instr += f->f_lasti/2 + 1; + } stack_pointer = f->f_stacktop; assert(stack_pointer != NULL); f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */ @@ -1249,7 +1255,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_MakePendingCalls() above. */ if (_Py_atomic_load_relaxed(&eval_breaker)) { - if (*next_instr == SETUP_FINALLY) { + if (OPCODE(*next_instr) == SETUP_FINALLY) { /* Make the last opcode before a try: finally: block uninterruptible. */ goto fast_next_opcode; @@ -1322,11 +1328,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* Extract opcode and argument */ - opcode = NEXTOP(); - oparg = 0; /* allows oparg to be stored in a register because - it doesn't have to be remembered across a full loop */ - if (HAS_ARG(opcode)) - oparg = NEXTARG(); + NEXTOPARG(); dispatch_opcode: #ifdef DYNAMIC_EXECUTION_PROFILE #ifdef DXPAIRS @@ -1377,6 +1379,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) FAST_DISPATCH(); } + PREDICTED(LOAD_CONST); TARGET(LOAD_CONST) { PyObject *value = GETITEM(consts, oparg); Py_INCREF(value); @@ -1384,7 +1387,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) FAST_DISPATCH(); } - PREDICTED_WITH_ARG(STORE_FAST); + PREDICTED(STORE_FAST); TARGET(STORE_FAST) { PyObject *value = POP(); SETLOCAL(oparg, value); @@ -2006,6 +2009,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } SET_TOP(awaitable); + PREDICT(LOAD_CONST); DISPATCH(); } @@ -2048,9 +2052,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(next_iter); PUSH(awaitable); + PREDICT(LOAD_CONST); DISPATCH(); } + PREDICTED(GET_AWAITABLE); TARGET(GET_AWAITABLE) { PyObject *iterable = TOP(); PyObject *iter = _PyCoro_GetAwaitableIter(iterable); @@ -2078,6 +2084,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) goto error; } + PREDICT(LOAD_CONST); DISPATCH(); } @@ -2111,7 +2118,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) f->f_stacktop = stack_pointer; why = WHY_YIELD; /* and repeat... */ - f->f_lasti--; + f->f_lasti -= 2; goto fast_yield; } @@ -2133,6 +2140,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } + PREDICTED(POP_BLOCK); TARGET(POP_BLOCK) { PyTryBlock *b = PyFrame_BlockPop(f); UNWIND_BLOCK(b); @@ -2249,7 +2257,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - PREDICTED_WITH_ARG(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); TARGET(UNPACK_SEQUENCE) { PyObject *seq = POP(), *item, **items; if (PyTuple_CheckExact(seq) && @@ -2358,7 +2366,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } else { v = PyObject_GetItem(locals, name); - if (v == NULL && _PyErr_OCCURRED()) { + if (v == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); @@ -2398,26 +2406,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyObject *name = GETITEM(names, oparg); PyObject *v; if (PyDict_CheckExact(f->f_globals) - && PyDict_CheckExact(f->f_builtins)) { + && PyDict_CheckExact(f->f_builtins)) + { v = _PyDict_LoadGlobal((PyDictObject *)f->f_globals, (PyDictObject *)f->f_builtins, name); if (v == NULL) { - if (!_PyErr_OCCURRED()) + if (!_PyErr_OCCURRED()) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ format_exc_check_arg(PyExc_NameError, NAME_ERROR_MSG, name); + } goto error; } Py_INCREF(v); } else { /* Slow-path if globals or builtins is not a dict */ + + /* namespace 1: globals */ v = PyObject_GetItem(f->f_globals, name); if (v == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); + /* namespace 2: builtins */ v = PyObject_GetItem(f->f_builtins, name); if (v == NULL) { if (PyErr_ExceptionMatches(PyExc_KeyError)) @@ -2477,7 +2492,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } else { value = PyObject_GetItem(locals, name); - if (value == NULL && PyErr_Occurred()) { + if (value == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); @@ -2540,9 +2555,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - TARGET_WITH_IMPL(BUILD_TUPLE_UNPACK, _build_list_unpack) - TARGET(BUILD_LIST_UNPACK) - _build_list_unpack: { + TARGET(BUILD_TUPLE_UNPACK) + TARGET(BUILD_LIST_UNPACK) { int convert_to_tuple = opcode == BUILD_TUPLE_UNPACK; int i; PyObject *sum = PyList_New(0); @@ -2639,9 +2653,41 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - TARGET_WITH_IMPL(BUILD_MAP_UNPACK_WITH_CALL, _build_map_unpack) - TARGET(BUILD_MAP_UNPACK) - _build_map_unpack: { + TARGET(BUILD_CONST_KEY_MAP) { + int i; + PyObject *map; + PyObject *keys = TOP(); + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + PyErr_SetString(PyExc_SystemError, + "bad BUILD_CONST_KEY_MAP keys argument"); + goto error; + } + map = _PyDict_NewPresized((Py_ssize_t)oparg); + if (map == NULL) { + goto error; + } + for (i = oparg; i > 0; i--) { + int err; + PyObject *key = PyTuple_GET_ITEM(keys, oparg - i); + PyObject *value = PEEK(i + 1); + err = PyDict_SetItem(map, key, value); + if (err != 0) { + Py_DECREF(map); + goto error; + } + } + + Py_DECREF(POP()); + while (oparg--) { + Py_DECREF(POP()); + } + PUSH(map); + DISPATCH(); + } + + TARGET(BUILD_MAP_UNPACK_WITH_CALL) + TARGET(BUILD_MAP_UNPACK) { int with_call = opcode == BUILD_MAP_UNPACK_WITH_CALL; int num_maps; int function_location; @@ -2763,45 +2809,15 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } TARGET(IMPORT_NAME) { - _Py_IDENTIFIER(__import__); PyObject *name = GETITEM(names, oparg); - PyObject *func = _PyDict_GetItemId(f->f_builtins, &PyId___import__); - PyObject *from, *level, *args, *res; - if (func == NULL) { - PyErr_SetString(PyExc_ImportError, - "__import__ not found"); - goto error; - } - Py_INCREF(func); - from = POP(); - level = TOP(); - if (PyLong_AsLong(level) != -1 || PyErr_Occurred()) - args = PyTuple_Pack(5, - name, - f->f_globals, - f->f_locals == NULL ? - Py_None : f->f_locals, - from, - level); - else - args = PyTuple_Pack(4, - name, - f->f_globals, - f->f_locals == NULL ? - Py_None : f->f_locals, - from); - Py_DECREF(level); - Py_DECREF(from); - if (args == NULL) { - Py_DECREF(func); - STACKADJ(-1); - goto error; - } + PyObject *fromlist = POP(); + PyObject *level = TOP(); + PyObject *res; READ_TIMESTAMP(intr0); - res = PyEval_CallObject(func, args); + res = import_name(f, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); READ_TIMESTAMP(intr1); - Py_DECREF(args); - Py_DECREF(func); SET_TOP(res); if (res == NULL) goto error; @@ -2848,7 +2864,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) FAST_DISPATCH(); } - PREDICTED_WITH_ARG(POP_JUMP_IF_FALSE); + PREDICTED(POP_JUMP_IF_FALSE); TARGET(POP_JUMP_IF_FALSE) { PyObject *cond = POP(); int err; @@ -2872,7 +2888,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - PREDICTED_WITH_ARG(POP_JUMP_IF_TRUE); + PREDICTED(POP_JUMP_IF_TRUE); TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = POP(); int err; @@ -2949,7 +2965,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - PREDICTED_WITH_ARG(JUMP_ABSOLUTE); + PREDICTED(JUMP_ABSOLUTE); TARGET(JUMP_ABSOLUTE) { JUMPTO(oparg); #if FAST_LOOPS @@ -2975,6 +2991,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (iter == NULL) goto error; PREDICT(FOR_ITER); + PREDICT(CALL_FUNCTION); DISPATCH(); } @@ -3003,10 +3020,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (iter == NULL) goto error; } + PREDICT(LOAD_CONST); DISPATCH(); } - PREDICTED_WITH_ARG(FOR_ITER); + PREDICTED(FOR_ITER); TARGET(FOR_ITER) { /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); @@ -3028,6 +3046,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) STACKADJ(-1); Py_DECREF(iter); JUMPBY(oparg); + PREDICT(POP_BLOCK); DISPATCH(); } @@ -3044,10 +3063,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) goto fast_block_end; } - TARGET_WITH_IMPL(SETUP_LOOP, _setup_finally) - TARGET_WITH_IMPL(SETUP_EXCEPT, _setup_finally) - TARGET(SETUP_FINALLY) - _setup_finally: { + TARGET(SETUP_LOOP) + TARGET(SETUP_EXCEPT) + TARGET(SETUP_FINALLY) { /* NOTE: If you add any new block-setup opcodes that are not try/except/finally handlers, you may need to update the PyGen_NeedsFinalizing() function. @@ -3078,6 +3096,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (res == NULL) goto error; PUSH(res); + PREDICT(GET_AWAITABLE); DISPATCH(); } @@ -3226,6 +3245,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } + PREDICTED(CALL_FUNCTION); TARGET(CALL_FUNCTION) { PyObject **sp, *res; PCALL(PCALL_ALL); @@ -3242,10 +3262,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw) - TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw) - TARGET(CALL_FUNCTION_VAR_KW) - _call_function_var_kw: { + TARGET(CALL_FUNCTION_VAR) + TARGET(CALL_FUNCTION_KW) + TARGET(CALL_FUNCTION_VAR_KW) { int na = oparg & 0xff; int nk = (oparg>>8) & 0xff; int flags = (opcode - CALL_FUNCTION) & 3; @@ -3287,117 +3306,36 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - TARGET_WITH_IMPL(MAKE_CLOSURE, _make_function) - TARGET(MAKE_FUNCTION) - _make_function: { - int posdefaults = oparg & 0xff; - int kwdefaults = (oparg>>8) & 0xff; - int num_annotations = (oparg >> 16) & 0x7fff; + TARGET(MAKE_FUNCTION) { + PyObject *qualname = POP(); + PyObject *codeobj = POP(); + PyFunctionObject *func = (PyFunctionObject *) + PyFunction_NewWithQualName(codeobj, f->f_globals, qualname); - PyObject *qualname = POP(); /* qualname */ - PyObject *code = POP(); /* code object */ - PyObject *func = PyFunction_NewWithQualName(code, f->f_globals, qualname); - Py_DECREF(code); + Py_DECREF(codeobj); Py_DECREF(qualname); - - if (func == NULL) + if (func == NULL) { goto error; - - if (opcode == MAKE_CLOSURE) { - PyObject *closure = POP(); - if (PyFunction_SetClosure(func, closure) != 0) { - /* Can't happen unless bytecode is corrupt. */ - Py_DECREF(func); - Py_DECREF(closure); - goto error; - } - Py_DECREF(closure); } - if (num_annotations > 0) { - Py_ssize_t name_ix; - PyObject *names = POP(); /* names of args with annotations */ - PyObject *anns = PyDict_New(); - if (anns == NULL) { - Py_DECREF(func); - Py_DECREF(names); - goto error; - } - name_ix = PyTuple_Size(names); - assert(num_annotations == name_ix+1); - while (name_ix > 0) { - PyObject *name, *value; - int err; - --name_ix; - name = PyTuple_GET_ITEM(names, name_ix); - value = POP(); - err = PyDict_SetItem(anns, name, value); - Py_DECREF(value); - if (err != 0) { - Py_DECREF(anns); - Py_DECREF(func); - Py_DECREF(names); - goto error; - } - } - Py_DECREF(names); - - if (PyFunction_SetAnnotations(func, anns) != 0) { - /* Can't happen unless - PyFunction_SetAnnotations changes. */ - Py_DECREF(anns); - Py_DECREF(func); - goto error; - } - Py_DECREF(anns); + if (oparg & 0x08) { + assert(PyTuple_CheckExact(TOP())); + func ->func_closure = POP(); } - - /* XXX Maybe this should be a separate opcode? */ - if (kwdefaults > 0) { - PyObject *defs = PyDict_New(); - if (defs == NULL) { - Py_DECREF(func); - goto error; - } - while (--kwdefaults >= 0) { - PyObject *v = POP(); /* default value */ - PyObject *key = POP(); /* kw only arg name */ - int err = PyDict_SetItem(defs, key, v); - Py_DECREF(v); - Py_DECREF(key); - if (err != 0) { - Py_DECREF(defs); - Py_DECREF(func); - goto error; - } - } - if (PyFunction_SetKwDefaults(func, defs) != 0) { - /* Can't happen unless - PyFunction_SetKwDefaults changes. */ - Py_DECREF(func); - Py_DECREF(defs); - goto error; - } - Py_DECREF(defs); + if (oparg & 0x04) { + assert(PyDict_CheckExact(TOP())); + func->func_annotations = POP(); } - if (posdefaults > 0) { - PyObject *defs = PyTuple_New(posdefaults); - if (defs == NULL) { - Py_DECREF(func); - goto error; - } - while (--posdefaults >= 0) - PyTuple_SET_ITEM(defs, posdefaults, POP()); - if (PyFunction_SetDefaults(func, defs) != 0) { - /* Can't happen unless - PyFunction_SetDefaults changes. */ - Py_DECREF(defs); - Py_DECREF(func); - goto error; - } - Py_DECREF(defs); + if (oparg & 0x02) { + assert(PyDict_CheckExact(TOP())); + func->func_kwdefaults = POP(); } - PUSH(func); + if (oparg & 0x01) { + assert(PyTuple_CheckExact(TOP())); + func->func_defaults = POP(); + } + + PUSH((PyObject *)func); DISPATCH(); } @@ -3419,9 +3357,68 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } + TARGET(FORMAT_VALUE) { + /* Handles f-string value formatting. */ + PyObject *result; + PyObject *fmt_spec; + PyObject *value; + PyObject *(*conv_fn)(PyObject *); + int which_conversion = oparg & FVC_MASK; + int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC; + + fmt_spec = have_fmt_spec ? POP() : NULL; + value = POP(); + + /* See if any conversion is specified. */ + switch (which_conversion) { + case FVC_STR: conv_fn = PyObject_Str; break; + case FVC_REPR: conv_fn = PyObject_Repr; break; + case FVC_ASCII: conv_fn = PyObject_ASCII; break; + + /* Must be 0 (meaning no conversion), since only four + values are allowed by (oparg & FVC_MASK). */ + default: conv_fn = NULL; break; + } + + /* If there's a conversion function, call it and replace + value with that result. Otherwise, just use value, + without conversion. */ + if (conv_fn != NULL) { + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) { + Py_XDECREF(fmt_spec); + goto error; + } + value = result; + } + + /* If value is a unicode object, and there's no fmt_spec, + then we know the result of format(value) is value + itself. In that case, skip calling format(). I plan to + move this optimization in to PyObject_Format() + itself. */ + if (PyUnicode_CheckExact(value) && fmt_spec == NULL) { + /* Do nothing, just transfer ownership to result. */ + result = value; + } else { + /* Actually call format(). */ + result = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_XDECREF(fmt_spec); + if (result == NULL) { + goto error; + } + } + + PUSH(result); + DISPATCH(); + } + TARGET(EXTENDED_ARG) { - opcode = NEXTOP(); - oparg = oparg<<16 | NEXTARG(); + int oldoparg = oparg; + NEXTOPARG(); + oparg |= oldoparg << 8; goto dispatch_opcode; } @@ -3782,6 +3779,7 @@ too_many_positional(PyCodeObject *co, int given, int defcount, PyObject **fastlo Py_DECREF(kwonly_sig); } + /* This is gonna seem *real weird*, but if you put some other code between PyEval_EvalFrame() and PyEval_EvalCodeEx() you will need to adjust the test in the if statements in Misc/gdbinit (pystack and pystackv). */ @@ -3796,12 +3794,13 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, PyFrameObject *f; PyObject *retval = NULL; PyObject **fastlocals, **freevars; - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate; PyObject *x, *u; - int total_args = co->co_argcount + co->co_kwonlyargcount; - int i; - int n = argcount; - PyObject *kwdict = NULL; + const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; + Py_ssize_t i, n; + PyObject *kwdict; + + assert((kwcount == 0) || (kws != NULL)); if (globals == NULL) { PyErr_SetString(PyExc_SystemError, @@ -3809,36 +3808,50 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, return NULL; } + /* Create the frame */ + tstate = PyThreadState_GET(); assert(tstate != NULL); - assert(globals != NULL); f = PyFrame_New(tstate, co, globals, locals); - if (f == NULL) + if (f == NULL) { return NULL; - + } fastlocals = f->f_localsplus; freevars = f->f_localsplus + co->co_nlocals; - /* Parse arguments. */ + /* Create a dictionary for keyword parameters (**kwags) */ if (co->co_flags & CO_VARKEYWORDS) { kwdict = PyDict_New(); if (kwdict == NULL) goto fail; i = total_args; - if (co->co_flags & CO_VARARGS) + if (co->co_flags & CO_VARARGS) { i++; + } SETLOCAL(i, kwdict); } - if (argcount > co->co_argcount) + else { + kwdict = NULL; + } + + /* Copy positional arguments into local variables */ + if (argcount > co->co_argcount) { n = co->co_argcount; + } + else { + n = argcount; + } for (i = 0; i < n; i++) { x = args[i]; Py_INCREF(x); SETLOCAL(i, x); } + + /* Pack other positional arguments into the *args argument */ if (co->co_flags & CO_VARARGS) { u = PyTuple_New(argcount - n); - if (u == NULL) + if (u == NULL) { goto fail; + } SETLOCAL(total_args, u); for (i = n; i < argcount; i++) { x = args[i]; @@ -3846,17 +3859,21 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, PyTuple_SET_ITEM(u, i-n, x); } } + + /* Handle keyword arguments (passed as an array of (key, value)) */ for (i = 0; i < kwcount; i++) { PyObject **co_varnames; PyObject *keyword = kws[2*i]; PyObject *value = kws[2*i + 1]; - int j; + Py_ssize_t j; + if (keyword == NULL || !PyUnicode_Check(keyword)) { PyErr_Format(PyExc_TypeError, "%U() keywords must be strings", co->co_name); goto fail; } + /* Speed hack: do raw pointer compares. As names are normally interned this should almost always hit. */ co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; @@ -3865,6 +3882,7 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, if (nm == keyword) goto kw_found; } + /* Slow fallback, just in case */ for (j = 0; j < total_args; j++) { PyObject *nm = co_varnames[j]; @@ -3875,6 +3893,7 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, else if (cmp < 0) goto fail; } + if (j >= total_args && kwdict == NULL) { PyErr_Format(PyExc_TypeError, "%U() got an unexpected " @@ -3883,10 +3902,12 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, keyword); goto fail; } + if (PyDict_SetItem(kwdict, keyword, value) == -1) { goto fail; } continue; + kw_found: if (GETLOCAL(j) != NULL) { PyErr_Format(PyExc_TypeError, @@ -3899,16 +3920,22 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, Py_INCREF(value); SETLOCAL(j, value); } + + /* Check the number of positional arguments */ if (argcount > co->co_argcount && !(co->co_flags & CO_VARARGS)) { too_many_positional(co, argcount, defcount, fastlocals); goto fail; } + + /* Add missing positional arguments (copy default values from defs) */ if (argcount < co->co_argcount) { - int m = co->co_argcount - defcount; - int missing = 0; - for (i = argcount; i < m; i++) - if (GETLOCAL(i) == NULL) + Py_ssize_t m = co->co_argcount - defcount; + Py_ssize_t missing = 0; + for (i = argcount; i < m; i++) { + if (GETLOCAL(i) == NULL) { missing++; + } + } if (missing) { missing_arguments(co, missing, defcount, fastlocals); goto fail; @@ -3925,8 +3952,10 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, } } } + + /* Add missing keyword arguments (copy default values from kwdefs) */ if (co->co_kwonlyargcount > 0) { - int missing = 0; + Py_ssize_t missing = 0; for (i = co->co_argcount; i < total_args; i++) { PyObject *name; if (GETLOCAL(i) != NULL) @@ -3967,12 +3996,15 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, goto fail; SETLOCAL(co->co_nlocals + i, c); } + + /* Copy closure variables to free variables */ for (i = 0; i < PyTuple_GET_SIZE(co->co_freevars); ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); Py_INCREF(o); freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o; } + /* Handle generator/coroutine */ if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) { PyObject *gen; PyObject *coro_wrapper = tstate->coroutine_wrapper; @@ -4037,8 +4069,10 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure) { return _PyEval_EvalCodeWithName(_co, globals, locals, - args, argcount, kws, kwcount, - defs, defcount, kwdefs, closure, + args, argcount, + kws, kwcount, + defs, defcount, + kwdefs, closure, NULL, NULL); } @@ -4297,7 +4331,7 @@ Error: #ifdef LLTRACE static int -prtrace(PyObject *v, char *str) +prtrace(PyObject *v, const char *str) { printf("%s ", str); if (PyObject_Print(v, stdout, 0) != 0) @@ -4546,7 +4580,7 @@ PyEval_MergeCompilerFlags(PyCompilerFlags *cf) The arg must be a tuple or NULL. The kw must be a dict or NULL. */ PyObject * -PyEval_CallObjectWithKeywords(PyObject *func, PyObject *arg, PyObject *kw) +PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) { PyObject *result; @@ -4557,28 +4591,33 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *arg, PyObject *kw) assert(!PyErr_Occurred()); #endif - if (arg == NULL) { - arg = PyTuple_New(0); - if (arg == NULL) + if (args == NULL) { + if (kwargs == NULL) { + return _PyObject_FastCall(func, NULL, 0, 0); + } + + args = PyTuple_New(0); + if (args == NULL) return NULL; } - else if (!PyTuple_Check(arg)) { + else if (!PyTuple_Check(args)) { PyErr_SetString(PyExc_TypeError, "argument list must be a tuple"); return NULL; } - else - Py_INCREF(arg); + else { + Py_INCREF(args); + } - if (kw != NULL && !PyDict_Check(kw)) { + if (kwargs != NULL && !PyDict_Check(kwargs)) { PyErr_SetString(PyExc_TypeError, "keyword list must be a dictionary"); - Py_DECREF(arg); + Py_DECREF(args); return NULL; } - result = PyObject_Call(func, arg, kw); - Py_DECREF(arg); + result = PyObject_Call(func, args, kwargs); + Py_DECREF(args); return result; } @@ -4726,10 +4765,12 @@ call_function(PyObject ***pp_stack, int oparg } else Py_INCREF(func); READ_TIMESTAMP(*pintr0); - if (PyFunction_Check(func)) - x = fast_function(func, pp_stack, n, na, nk); - else + if (PyFunction_Check(func)) { + x = fast_function(func, (*pp_stack) - n, n, na, nk); + } + else { x = do_call(func, pp_stack, na, nk); + } READ_TIMESTAMP(*pintr1); Py_DECREF(func); @@ -4759,62 +4800,144 @@ call_function(PyObject ***pp_stack, int oparg done before evaluating the frame. */ +static PyObject* +_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) +{ + PyFrameObject *f; + PyThreadState *tstate = PyThreadState_GET(); + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + + PCALL(PCALL_FASTER_FUNCTION); + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + + fastlocals = f->f_localsplus; + + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + + return result; +} + +/* Similar to _PyFunction_FastCall() but keywords are passed a (key, value) + pairs in stack */ static PyObject * -fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk) +fast_function(PyObject *func, PyObject **stack, int n, int nargs, int nk) { PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); PyObject *globals = PyFunction_GET_GLOBALS(func); PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *kwdefs = PyFunction_GET_KW_DEFAULTS(func); - PyObject *name = ((PyFunctionObject *)func) -> func_name; - PyObject *qualname = ((PyFunctionObject *)func) -> func_qualname; - PyObject **d = NULL; - int nd = 0; + PyObject *kwdefs, *closure, *name, *qualname; + PyObject **d; + int nd; PCALL(PCALL_FUNCTION); PCALL(PCALL_FAST_FUNCTION); - if (argdefs == NULL && co->co_argcount == n && - co->co_kwonlyargcount == 0 && nk==0 && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - PyFrameObject *f; - PyObject *retval = NULL; - PyThreadState *tstate = PyThreadState_GET(); - PyObject **fastlocals, **stack; - int i; - PCALL(PCALL_FASTER_FUNCTION); - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) - return NULL; + if (co->co_kwonlyargcount == 0 && nk == 0 && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) + { + if (argdefs == NULL && co->co_argcount == nargs) { + return _PyFunction_FastCallNoKw(co, stack, nargs, globals); + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + stack = &PyTuple_GET_ITEM(argdefs, 0); + return _PyFunction_FastCallNoKw(co, stack, Py_SIZE(argdefs), + globals); + } + } + + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + closure = PyFunction_GET_CLOSURE(func); + name = ((PyFunctionObject *)func) -> func_name; + qualname = ((PyFunctionObject *)func) -> func_qualname; + + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + return _PyEval_EvalCodeWithName((PyObject*)co, globals, (PyObject *)NULL, + stack, nargs, + stack + nargs, nk, + d, nd, kwdefs, + closure, name, qualname); +} + +PyObject * +_PyFunction_FastCall(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) +{ + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *kwdefs, *closure, *name, *qualname; + PyObject **d; + int nd; + + PCALL(PCALL_FUNCTION); + PCALL(PCALL_FAST_FUNCTION); - fastlocals = f->f_localsplus; - stack = (*pp_stack) - n; + /* issue #27128: support for keywords will come later */ + assert(kwargs == NULL); - for (i = 0; i < n; i++) { - Py_INCREF(*stack); - fastlocals[i] = *stack++; + if (co->co_kwonlyargcount == 0 && kwargs == NULL && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) + { + if (argdefs == NULL && co->co_argcount == nargs) { + return _PyFunction_FastCallNoKw(co, args, nargs, globals); + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + return _PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), + globals); } - retval = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return retval; } + + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + closure = PyFunction_GET_CLOSURE(func); + name = ((PyFunctionObject *)func) -> func_name; + qualname = ((PyFunctionObject *)func) -> func_qualname; + if (argdefs != NULL) { d = &PyTuple_GET_ITEM(argdefs, 0); nd = Py_SIZE(argdefs); } - return _PyEval_EvalCodeWithName((PyObject*)co, globals, - (PyObject *)NULL, (*pp_stack)-n, na, - (*pp_stack)-2*nk, nk, d, nd, kwdefs, - PyFunction_GET_CLOSURE(func), - name, qualname); + else { + d = NULL; + nd = 0; + } + return _PyEval_EvalCodeWithName((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + NULL, 0, + d, nd, kwdefs, + closure, name, qualname); } static PyObject * @@ -4863,6 +4986,21 @@ update_star_args(int nstack, int nstar, PyObject *stararg, { PyObject *callargs, *w; + if (!nstack) { + if (!stararg) { + /* There are no positional arguments on the stack and there is no + sequence to be unpacked. */ + return PyTuple_New(0); + } + if (PyTuple_CheckExact(stararg)) { + /* No arguments are passed on the stack and the sequence is not a + tuple subclass so we can just pass the stararg tuple directly + to the function. */ + Py_INCREF(stararg); + return stararg; + } + } + callargs = PyTuple_New(nstack + nstar); if (callargs == NULL) { return NULL; @@ -4951,7 +5089,7 @@ ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk) if (flags & CALL_FLAG_KW) { kwdict = EXT_POP(*pp_stack); - if (!PyDict_Check(kwdict)) { + if (!PyDict_CheckExact(kwdict)) { PyObject *d; d = PyDict_New(); if (d == NULL) @@ -5126,6 +5264,46 @@ cmp_outcome(int op, PyObject *v, PyObject *w) } static PyObject * +import_name(PyFrameObject *f, PyObject *name, PyObject *fromlist, PyObject *level) +{ + _Py_IDENTIFIER(__import__); + PyObject *import_func, *res; + PyObject* stack[5]; + + import_func = _PyDict_GetItemId(f->f_builtins, &PyId___import__); + if (import_func == NULL) { + PyErr_SetString(PyExc_ImportError, "__import__ not found"); + return NULL; + } + + /* Fast path for not overloaded __import__. */ + if (import_func == PyThreadState_GET()->interp->import_func) { + int ilevel = _PyLong_AsInt(level); + if (ilevel == -1 && PyErr_Occurred()) { + return NULL; + } + res = PyImport_ImportModuleLevelObject( + name, + f->f_globals, + f->f_locals == NULL ? Py_None : f->f_locals, + fromlist, + ilevel); + return res; + } + + Py_INCREF(import_func); + + stack[0] = name; + stack[1] = f->f_globals; + stack[2] = f->f_locals == NULL ? Py_None : f->f_locals; + stack[3] = fromlist; + stack[4] = level; + res = _PyObject_FastCall(import_func, stack, 5, NULL); + Py_DECREF(import_func); + return res; +} + +static PyObject * import_from(PyObject *v, PyObject *name) { PyObject *x; @@ -5261,7 +5439,7 @@ format_exc_unbound(PyCodeObject *co, int oparg) static PyObject * unicode_concatenate(PyObject *v, PyObject *w, - PyFrameObject *f, unsigned char *next_instr) + PyFrameObject *f, const unsigned short *next_instr) { PyObject *res; if (Py_REFCNT(v) == 2) { @@ -5271,10 +5449,11 @@ unicode_concatenate(PyObject *v, PyObject *w, * 'variable'. We try to delete the variable now to reduce * the refcnt to 1. */ - switch (*next_instr) { + int opcode, oparg; + NEXTOPARG(); + switch (opcode) { case STORE_FAST: { - int oparg = PEEKARG(); PyObject **fastlocals = f->f_localsplus; if (GETLOCAL(oparg) == v) SETLOCAL(oparg, NULL); @@ -5284,7 +5463,7 @@ unicode_concatenate(PyObject *v, PyObject *w, { PyObject **freevars = (f->f_localsplus + f->f_code->co_nlocals); - PyObject *c = freevars[PEEKARG()]; + PyObject *c = freevars[oparg]; if (PyCell_GET(c) == v) PyCell_Set(c, NULL); break; @@ -5292,7 +5471,7 @@ unicode_concatenate(PyObject *v, PyObject *w, case STORE_NAME: { PyObject *names = f->f_code->co_names; - PyObject *name = GETITEM(names, PEEKARG()); + PyObject *name = GETITEM(names, oparg); PyObject *locals = f->f_locals; if (PyDict_CheckExact(locals) && PyDict_GetItem(locals, name) == v) { |