diff options
Diffstat (limited to 'Python/ceval.c')
| -rw-r--r-- | Python/ceval.c | 497 |
1 files changed, 309 insertions, 188 deletions
diff --git a/Python/ceval.c b/Python/ceval.c index fcf8efb..e0ce92e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -705,6 +705,114 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) char *filename; #endif +/* Computed GOTOs, or + the-optimization-commonly-but-improperly-known-as-"threaded code" + using gcc's labels-as-values extension + (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html). + + The traditional bytecode evaluation loop uses a "switch" statement, which + decent compilers will optimize as a single indirect branch instruction + combined with a lookup table of jump addresses. However, since the + indirect jump instruction is shared by all opcodes, the CPU will have a + hard time making the right prediction for where to jump next (actually, + it will be always wrong except in the uncommon case of a sequence of + several identical opcodes). + + "Threaded code" in contrast, uses an explicit jump table and an explicit + indirect jump instruction at the end of each opcode. Since the jump + instruction is at a different address for each opcode, the CPU will make a + separate prediction for each of these instructions, which is equivalent to + predicting the second opcode of each opcode pair. These predictions have + a much better chance to turn out valid, especially in small bytecode loops. + + A mispredicted branch on a modern CPU flushes the whole pipeline and + can cost several CPU cycles (depending on the pipeline depth), + and potentially many more instructions (depending on the pipeline width). + A correctly predicted branch, however, is nearly free. + + At the time of this writing, the "threaded code" version is up to 15-20% + faster than the normal "switch" version, depending on the compiler and the + CPU architecture. + + We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined, + because it would render the measurements invalid. + + + NOTE: care must be taken that the compiler doesn't try to "optimize" the + indirect jumps by sharing them between all opcodes. Such optimizations + can be disabled on gcc by using the -fno-gcse flag (or possibly + -fno-crossjumping). +*/ + +#if defined(USE_COMPUTED_GOTOS) && defined(DYNAMIC_EXECUTION_PROFILE) +#undef USE_COMPUTED_GOTOS +#endif + +#ifdef USE_COMPUTED_GOTOS +/* Import the static jump table */ +#include "opcode_targets.h" + +/* This macro is used when several opcodes defer to the same implementation + (e.g. SETUP_LOOP, SETUP_FINALLY) */ +#define TARGET_WITH_IMPL(op, impl) \ + TARGET_##op: \ + opcode = op; \ + if (HAS_ARG(op)) \ + oparg = NEXTARG(); \ + case op: \ + goto impl; \ + +#define TARGET(op) \ + TARGET_##op: \ + opcode = op; \ + if (HAS_ARG(op)) \ + oparg = NEXTARG(); \ + case op: + + +#define DISPATCH() \ + { \ + /* Avoid multiple loads from _Py_Ticker despite `volatile` */ \ + int _tick = _Py_Ticker - 1; \ + _Py_Ticker = _tick; \ + if (_tick >= 0) { \ + FAST_DISPATCH(); \ + } \ + continue; \ + } + +#ifdef LLTRACE +#define FAST_DISPATCH() \ + { \ + if (!lltrace && !_Py_TracingPossible) { \ + f->f_lasti = INSTR_OFFSET(); \ + goto *opcode_targets[*next_instr++]; \ + } \ + goto fast_next_opcode; \ + } +#else +#define FAST_DISPATCH() \ + { \ + if (!_Py_TracingPossible) { \ + f->f_lasti = INSTR_OFFSET(); \ + goto *opcode_targets[*next_instr++]; \ + } \ + goto fast_next_opcode; \ + } +#endif + +#else +#define TARGET(op) \ + case op: +#define TARGET_WITH_IMPL(op, impl) \ + /* silence compiler warnings about `impl` unused */ \ + if (0) goto impl; \ + case op: +#define DISPATCH() continue +#define FAST_DISPATCH() goto fast_next_opcode +#endif + + /* Tuple access macros */ #ifndef Py_DEBUG @@ -782,16 +890,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) predictions turned-on and interpret the results as if some opcodes had been combined or turn-off predictions so that the opcode frequency counter updates for both opcodes. + + Opcode prediction is disabled with threaded code, since the latter allows + the CPU to record separate branch prediction information for each + opcode. + */ -#ifdef DYNAMIC_EXECUTION_PROFILE +#if defined(DYNAMIC_EXECUTION_PROFILE) || defined(USE_COMPUTED_GOTOS) #define PREDICT(op) if (0) goto PRED_##op +#define PREDICTED(op) PRED_##op: +#define PREDICTED_WITH_ARG(op) PRED_##op: #else #define PREDICT(op) if (*next_instr == op) goto PRED_##op -#endif - #define PREDICTED(op) PRED_##op: next_instr++ #define PREDICTED_WITH_ARG(op) PRED_##op: oparg = PEEKARG(); next_instr += 3 +#endif + /* Stack manipulation macros */ @@ -1153,56 +1268,56 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* case STOP_CODE: this is an error! */ - case NOP: - goto fast_next_opcode; + TARGET(NOP) + FAST_DISPATCH(); - case LOAD_FAST: + TARGET(LOAD_FAST) x = GETLOCAL(oparg); if (x != NULL) { Py_INCREF(x); PUSH(x); - goto fast_next_opcode; + FAST_DISPATCH(); } format_exc_check_arg(PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(co->co_varnames, oparg)); break; - case LOAD_CONST: + TARGET(LOAD_CONST) x = GETITEM(consts, oparg); Py_INCREF(x); PUSH(x); - goto fast_next_opcode; + FAST_DISPATCH(); PREDICTED_WITH_ARG(STORE_FAST); - case STORE_FAST: + TARGET(STORE_FAST) v = POP(); SETLOCAL(oparg, v); - goto fast_next_opcode; + FAST_DISPATCH(); PREDICTED(POP_TOP); - case POP_TOP: + TARGET(POP_TOP) v = POP(); Py_DECREF(v); - goto fast_next_opcode; + FAST_DISPATCH(); - case ROT_TWO: + TARGET(ROT_TWO) v = TOP(); w = SECOND(); SET_TOP(w); SET_SECOND(v); - goto fast_next_opcode; + FAST_DISPATCH(); - case ROT_THREE: + TARGET(ROT_THREE) v = TOP(); w = SECOND(); x = THIRD(); SET_TOP(w); SET_SECOND(x); SET_THIRD(v); - goto fast_next_opcode; + FAST_DISPATCH(); - case ROT_FOUR: + TARGET(ROT_FOUR) u = TOP(); v = SECOND(); w = THIRD(); @@ -1211,15 +1326,15 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) SET_SECOND(w); SET_THIRD(x); SET_FOURTH(u); - goto fast_next_opcode; + FAST_DISPATCH(); - case DUP_TOP: + TARGET(DUP_TOP) v = TOP(); Py_INCREF(v); PUSH(v); - goto fast_next_opcode; + FAST_DISPATCH(); - case DUP_TOPX: + TARGET(DUP_TOPX) if (oparg == 2) { x = TOP(); Py_INCREF(x); @@ -1228,7 +1343,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) STACKADJ(2); SET_TOP(x); SET_SECOND(w); - goto fast_next_opcode; + FAST_DISPATCH(); } else if (oparg == 3) { x = TOP(); Py_INCREF(x); @@ -1240,106 +1355,106 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) SET_TOP(x); SET_SECOND(w); SET_THIRD(v); - goto fast_next_opcode; + FAST_DISPATCH(); } Py_FatalError("invalid argument to DUP_TOPX" " (bytecode corruption?)"); /* Never returns, so don't bother to set why. */ break; - case UNARY_POSITIVE: + TARGET(UNARY_POSITIVE) v = TOP(); x = PyNumber_Positive(v); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case UNARY_NEGATIVE: + TARGET(UNARY_NEGATIVE) v = TOP(); x = PyNumber_Negative(v); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case UNARY_NOT: + TARGET(UNARY_NOT) v = TOP(); err = PyObject_IsTrue(v); Py_DECREF(v); if (err == 0) { Py_INCREF(Py_True); SET_TOP(Py_True); - continue; + DISPATCH(); } else if (err > 0) { Py_INCREF(Py_False); SET_TOP(Py_False); err = 0; - continue; + DISPATCH(); } STACKADJ(-1); break; - case UNARY_INVERT: + TARGET(UNARY_INVERT) v = TOP(); x = PyNumber_Invert(v); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_POWER: + TARGET(BINARY_POWER) w = POP(); v = TOP(); x = PyNumber_Power(v, w, Py_None); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_MULTIPLY: + TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_TRUE_DIVIDE: + TARGET(BINARY_TRUE_DIVIDE) w = POP(); v = TOP(); x = PyNumber_TrueDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_FLOOR_DIVIDE: + TARGET(BINARY_FLOOR_DIVIDE) w = POP(); v = TOP(); x = PyNumber_FloorDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_MODULO: + TARGET(BINARY_MODULO) w = POP(); v = TOP(); x = PyNumber_Remainder(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_ADD: + TARGET(BINARY_ADD) w = POP(); v = TOP(); if (PyUnicode_CheckExact(v) && @@ -1355,152 +1470,152 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) skip_decref_vx: Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_SUBTRACT: + TARGET(BINARY_SUBTRACT) w = POP(); v = TOP(); x = PyNumber_Subtract(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_SUBSCR: + TARGET(BINARY_SUBSCR) w = POP(); v = TOP(); x = PyObject_GetItem(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_LSHIFT: + TARGET(BINARY_LSHIFT) w = POP(); v = TOP(); x = PyNumber_Lshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_RSHIFT: + TARGET(BINARY_RSHIFT) w = POP(); v = TOP(); x = PyNumber_Rshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_AND: + TARGET(BINARY_AND) w = POP(); v = TOP(); x = PyNumber_And(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_XOR: + TARGET(BINARY_XOR) w = POP(); v = TOP(); x = PyNumber_Xor(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_OR: + TARGET(BINARY_OR) w = POP(); v = TOP(); x = PyNumber_Or(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case LIST_APPEND: + TARGET(LIST_APPEND) w = POP(); v = stack_pointer[-oparg]; err = PyList_Append(v, w); Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); - continue; + DISPATCH(); } break; - case SET_ADD: + TARGET(SET_ADD) w = POP(); v = stack_pointer[-oparg]; err = PySet_Add(v, w); Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); - continue; + DISPATCH(); } break; - case INPLACE_POWER: + TARGET(INPLACE_POWER) w = POP(); v = TOP(); x = PyNumber_InPlacePower(v, w, Py_None); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_MULTIPLY: + TARGET(INPLACE_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_InPlaceMultiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_TRUE_DIVIDE: + TARGET(INPLACE_TRUE_DIVIDE) w = POP(); v = TOP(); x = PyNumber_InPlaceTrueDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_FLOOR_DIVIDE: + TARGET(INPLACE_FLOOR_DIVIDE) w = POP(); v = TOP(); x = PyNumber_InPlaceFloorDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_MODULO: + TARGET(INPLACE_MODULO) w = POP(); v = TOP(); x = PyNumber_InPlaceRemainder(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_ADD: + TARGET(INPLACE_ADD) w = POP(); v = TOP(); if (PyUnicode_CheckExact(v) && @@ -1516,70 +1631,70 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) skip_decref_v: Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_SUBTRACT: + TARGET(INPLACE_SUBTRACT) w = POP(); v = TOP(); x = PyNumber_InPlaceSubtract(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_LSHIFT: + TARGET(INPLACE_LSHIFT) w = POP(); v = TOP(); x = PyNumber_InPlaceLshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_RSHIFT: + TARGET(INPLACE_RSHIFT) w = POP(); v = TOP(); x = PyNumber_InPlaceRshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_AND: + TARGET(INPLACE_AND) w = POP(); v = TOP(); x = PyNumber_InPlaceAnd(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_XOR: + TARGET(INPLACE_XOR) w = POP(); v = TOP(); x = PyNumber_InPlaceXor(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_OR: + TARGET(INPLACE_OR) w = POP(); v = TOP(); x = PyNumber_InPlaceOr(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case STORE_SUBSCR: + TARGET(STORE_SUBSCR) w = TOP(); v = SECOND(); u = THIRD(); @@ -1589,10 +1704,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(u); Py_DECREF(v); Py_DECREF(w); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case DELETE_SUBSCR: + TARGET(DELETE_SUBSCR) w = TOP(); v = SECOND(); STACKADJ(-2); @@ -1600,10 +1715,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) err = PyObject_DelItem(v, w); Py_DECREF(v); Py_DECREF(w); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case PRINT_EXPR: + TARGET(PRINT_EXPR) v = POP(); w = PySys_GetObject("displayhook"); if (w == NULL) { @@ -1630,7 +1745,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #ifdef CASE_TOO_BIG default: switch (opcode) { #endif - case RAISE_VARARGS: + TARGET(RAISE_VARARGS) v = w = NULL; switch (oparg) { case 2: @@ -1648,19 +1763,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } break; - case STORE_LOCALS: + TARGET(STORE_LOCALS) x = POP(); v = f->f_locals; Py_XDECREF(v); f->f_locals = x; - continue; + DISPATCH(); - case RETURN_VALUE: + TARGET(RETURN_VALUE) retval = POP(); why = WHY_RETURN; goto fast_block_end; - case YIELD_VALUE: + TARGET(YIELD_VALUE) retval = POP(); f->f_stacktop = stack_pointer; why = WHY_YIELD; @@ -1670,7 +1785,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) SWAP_EXC_STATE(); goto fast_yield; - case POP_EXCEPT: + TARGET(POP_EXCEPT) { PyTryBlock *b = PyFrame_BlockPop(f); if (b->b_type != EXCEPT_HANDLER) { @@ -1681,17 +1796,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } UNWIND_EXCEPT_HANDLER(b); } - continue; + DISPATCH(); - case POP_BLOCK: + TARGET(POP_BLOCK) { PyTryBlock *b = PyFrame_BlockPop(f); UNWIND_BLOCK(b); } - continue; + DISPATCH(); PREDICTED(END_FINALLY); - case END_FINALLY: + TARGET(END_FINALLY) v = POP(); if (PyLong_Check(v)) { why = (enum why_code) PyLong_AS_LONG(v); @@ -1731,7 +1846,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(v); break; - case LOAD_BUILD_CLASS: + TARGET(LOAD_BUILD_CLASS) x = PyDict_GetItemString(f->f_builtins, "__build_class__"); if (x == NULL) { @@ -1743,7 +1858,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PUSH(x); break; - case STORE_NAME: + TARGET(STORE_NAME) w = GETITEM(names, oparg); v = POP(); if ((x = f->f_locals) != NULL) { @@ -1752,14 +1867,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) else err = PyObject_SetItem(x, w, v); Py_DECREF(v); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; } PyErr_Format(PyExc_SystemError, "no locals found when storing %R", w); break; - case DELETE_NAME: + TARGET(DELETE_NAME) w = GETITEM(names, oparg); if ((x = f->f_locals) != NULL) { if ((err = PyObject_DelItem(x, w)) != 0) @@ -1773,7 +1888,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) break; PREDICTED_WITH_ARG(UNPACK_SEQUENCE); - case UNPACK_SEQUENCE: + TARGET(UNPACK_SEQUENCE) v = POP(); if (PyTuple_CheckExact(v) && PyTuple_GET_SIZE(v) == oparg) { @@ -1785,7 +1900,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PUSH(w); } Py_DECREF(v); - continue; + DISPATCH(); } else if (PyList_CheckExact(v) && PyList_GET_SIZE(v) == oparg) { PyObject **items = \ @@ -1805,7 +1920,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(v); break; - case UNPACK_EX: + TARGET(UNPACK_EX) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); v = POP(); @@ -1820,7 +1935,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) break; } - case STORE_ATTR: + TARGET(STORE_ATTR) w = GETITEM(names, oparg); v = TOP(); u = SECOND(); @@ -1828,10 +1943,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) err = PyObject_SetAttr(v, w, u); /* v.w = u */ Py_DECREF(v); Py_DECREF(u); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case DELETE_ATTR: + TARGET(DELETE_ATTR) w = GETITEM(names, oparg); v = POP(); err = PyObject_SetAttr(v, w, (PyObject *)NULL); @@ -1839,22 +1954,22 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(v); break; - case STORE_GLOBAL: + TARGET(STORE_GLOBAL) w = GETITEM(names, oparg); v = POP(); err = PyDict_SetItem(f->f_globals, w, v); Py_DECREF(v); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case DELETE_GLOBAL: + TARGET(DELETE_GLOBAL) w = GETITEM(names, oparg); if ((err = PyDict_DelItem(f->f_globals, w)) != 0) format_exc_check_arg( PyExc_NameError, GLOBAL_NAME_ERROR_MSG, w); break; - case LOAD_NAME: + TARGET(LOAD_NAME) w = GETITEM(names, oparg); if ((v = f->f_locals) == NULL) { PyErr_Format(PyExc_SystemError, @@ -1889,9 +2004,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_INCREF(x); } PUSH(x); - continue; + DISPATCH(); - case LOAD_GLOBAL: + TARGET(LOAD_GLOBAL) w = GETITEM(names, oparg); if (PyUnicode_CheckExact(w)) { /* Inline the PyDict_GetItem() calls. @@ -1911,7 +2026,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (x != NULL) { Py_INCREF(x); PUSH(x); - continue; + DISPATCH(); } d = (PyDictObject *)(f->f_builtins); e = d->ma_lookup(d, w, hash); @@ -1923,7 +2038,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (x != NULL) { Py_INCREF(x); PUSH(x); - continue; + DISPATCH(); } goto load_global_error; } @@ -1942,13 +2057,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } Py_INCREF(x); PUSH(x); - continue; + DISPATCH(); - case DELETE_FAST: + TARGET(DELETE_FAST) x = GETLOCAL(oparg); if (x != NULL) { SETLOCAL(oparg, NULL); - continue; + DISPATCH(); } format_exc_check_arg( PyExc_UnboundLocalError, @@ -1957,19 +2072,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) ); break; - case LOAD_CLOSURE: + TARGET(LOAD_CLOSURE) x = freevars[oparg]; Py_INCREF(x); PUSH(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case LOAD_DEREF: + TARGET(LOAD_DEREF) x = freevars[oparg]; w = PyCell_Get(x); if (w != NULL) { PUSH(w); - continue; + DISPATCH(); } err = -1; /* Don't stomp existing exception */ @@ -1990,14 +2105,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } break; - case STORE_DEREF: + TARGET(STORE_DEREF) w = POP(); x = freevars[oparg]; PyCell_Set(x, w); Py_DECREF(w); - continue; + DISPATCH(); - case BUILD_TUPLE: + TARGET(BUILD_TUPLE) x = PyTuple_New(oparg); if (x != NULL) { for (; --oparg >= 0;) { @@ -2005,11 +2120,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyTuple_SET_ITEM(x, oparg, w); } PUSH(x); - continue; + DISPATCH(); } break; - case BUILD_LIST: + TARGET(BUILD_LIST) x = PyList_New(oparg); if (x != NULL) { for (; --oparg >= 0;) { @@ -2017,11 +2132,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyList_SET_ITEM(x, oparg, w); } PUSH(x); - continue; + DISPATCH(); } break; - case BUILD_SET: + TARGET(BUILD_SET) x = PySet_New(NULL); if (x != NULL) { for (; --oparg >= 0;) { @@ -2035,17 +2150,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) break; } PUSH(x); - continue; + DISPATCH(); } break; - case BUILD_MAP: + TARGET(BUILD_MAP) x = _PyDict_NewPresized((Py_ssize_t)oparg); PUSH(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case STORE_MAP: + TARGET(STORE_MAP) w = TOP(); /* key */ u = SECOND(); /* value */ v = THIRD(); /* dict */ @@ -2054,10 +2169,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) err = PyDict_SetItem(v, w, u); /* v[w] = u */ Py_DECREF(u); Py_DECREF(w); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case MAP_ADD: + TARGET(MAP_ADD) w = TOP(); /* key */ u = SECOND(); /* value */ STACKADJ(-2); @@ -2068,20 +2183,20 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); - continue; + DISPATCH(); } break; - case LOAD_ATTR: + TARGET(LOAD_ATTR) w = GETITEM(names, oparg); v = TOP(); x = PyObject_GetAttr(v, w); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case COMPARE_OP: + TARGET(COMPARE_OP) w = POP(); v = TOP(); x = cmp_outcome(oparg, v, w); @@ -2091,9 +2206,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (x == NULL) break; PREDICT(JUMP_IF_FALSE); PREDICT(JUMP_IF_TRUE); - continue; + DISPATCH(); - case IMPORT_NAME: + TARGET(IMPORT_NAME) w = GETITEM(names, oparg); x = PyDict_GetItemString(f->f_builtins, "__import__"); if (x == NULL) { @@ -2134,10 +2249,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) READ_TIMESTAMP(intr1); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case IMPORT_STAR: + TARGET(IMPORT_STAR) v = POP(); PyFrame_FastToLocals(f); if ((x = f->f_locals) == NULL) { @@ -2150,33 +2265,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) READ_TIMESTAMP(intr1); PyFrame_LocalsToFast(f, 0); Py_DECREF(v); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case IMPORT_FROM: + TARGET(IMPORT_FROM) w = GETITEM(names, oparg); v = TOP(); READ_TIMESTAMP(intr0); x = import_from(v, w); READ_TIMESTAMP(intr1); PUSH(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case JUMP_FORWARD: + TARGET(JUMP_FORWARD) JUMPBY(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); PREDICTED_WITH_ARG(JUMP_IF_FALSE); - case JUMP_IF_FALSE: + TARGET(JUMP_IF_FALSE) w = TOP(); if (w == Py_True) { PREDICT(POP_TOP); - goto fast_next_opcode; + FAST_DISPATCH(); } if (w == Py_False) { JUMPBY(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); } err = PyObject_IsTrue(w); if (err > 0) @@ -2185,18 +2300,18 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) JUMPBY(oparg); else break; - continue; + DISPATCH(); PREDICTED_WITH_ARG(JUMP_IF_TRUE); - case JUMP_IF_TRUE: + TARGET(JUMP_IF_TRUE) w = TOP(); if (w == Py_False) { PREDICT(POP_TOP); - goto fast_next_opcode; + FAST_DISPATCH(); } if (w == Py_True) { JUMPBY(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); } err = PyObject_IsTrue(w); if (err > 0) { @@ -2207,10 +2322,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) ; else break; - continue; + DISPATCH(); PREDICTED_WITH_ARG(JUMP_ABSOLUTE); - case JUMP_ABSOLUTE: + TARGET(JUMP_ABSOLUTE) JUMPTO(oparg); #if FAST_LOOPS /* Enabling this path speeds-up all while and for-loops by bypassing @@ -2218,14 +2333,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) because it prevents detection of a control-break in tight loops like "while 1: pass". Compile with this option turned-on when you need the speed-up and do not need break checking inside tight loops (ones - that contain only instructions ending with goto fast_next_opcode). + that contain only instructions ending with FAST_DISPATCH). */ - goto fast_next_opcode; + FAST_DISPATCH(); #else - continue; + DISPATCH(); #endif - case GET_ITER: + TARGET(GET_ITER) /* before: [obj]; after [getiter(obj)] */ v = TOP(); x = PyObject_GetIter(v); @@ -2233,13 +2348,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (x != NULL) { SET_TOP(x); PREDICT(FOR_ITER); - continue; + DISPATCH(); } STACKADJ(-1); break; PREDICTED_WITH_ARG(FOR_ITER); - case FOR_ITER: + TARGET(FOR_ITER) /* before: [iter]; after: [iter, iter()] *or* [] */ v = TOP(); x = (*v->ob_type->tp_iternext)(v); @@ -2247,7 +2362,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PUSH(x); PREDICT(STORE_FAST); PREDICT(UNPACK_SEQUENCE); - continue; + DISPATCH(); } if (PyErr_Occurred()) { if (!PyErr_ExceptionMatches( @@ -2259,13 +2374,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) x = v = POP(); Py_DECREF(v); JUMPBY(oparg); - continue; + DISPATCH(); - case BREAK_LOOP: + TARGET(BREAK_LOOP) why = WHY_BREAK; goto fast_block_end; - case CONTINUE_LOOP: + TARGET(CONTINUE_LOOP) retval = PyLong_FromLong(oparg); if (!retval) { x = NULL; @@ -2274,9 +2389,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) why = WHY_CONTINUE; goto fast_block_end; - case SETUP_LOOP: - case SETUP_EXCEPT: - case SETUP_FINALLY: + TARGET_WITH_IMPL(SETUP_LOOP, _setup_finally) + TARGET_WITH_IMPL(SETUP_EXCEPT, _setup_finally) + TARGET(SETUP_FINALLY) + _setup_finally: /* NOTE: If you add any new block-setup opcodes that are not try/except/finally handlers, you may need to update the PyGen_NeedsFinalizing() function. @@ -2284,9 +2400,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyFrame_BlockSetup(f, opcode, INSTR_OFFSET() + oparg, STACK_LEVEL()); - continue; + DISPATCH(); - case WITH_CLEANUP: + TARGET(WITH_CLEANUP) { /* At the top of the stack are 1-3 values indicating how/why we entered the finally clause: @@ -2350,7 +2466,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) break; } - case CALL_FUNCTION: + TARGET(CALL_FUNCTION) { PyObject **sp; PCALL(PCALL_ALL); @@ -2363,13 +2479,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) stack_pointer = sp; PUSH(x); if (x != NULL) - continue; + DISPATCH(); break; } - case CALL_FUNCTION_VAR: - case CALL_FUNCTION_KW: - case CALL_FUNCTION_VAR_KW: + TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw) + TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw) + TARGET(CALL_FUNCTION_VAR_KW) + _call_function_var_kw: { int na = oparg & 0xff; int nk = (oparg>>8) & 0xff; @@ -2409,12 +2526,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } PUSH(x); if (x != NULL) - continue; + DISPATCH(); break; } - case MAKE_CLOSURE: - case MAKE_FUNCTION: + TARGET_WITH_IMPL(MAKE_CLOSURE, _make_function) + TARGET(MAKE_FUNCTION) + _make_function: { int posdefaults = oparg & 0xff; int kwdefaults = (oparg>>8) & 0xff; @@ -2507,7 +2625,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) break; } - case BUILD_SLICE: + TARGET(BUILD_SLICE) if (oparg == 3) w = POP(); else @@ -2519,14 +2637,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(v); Py_XDECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case EXTENDED_ARG: + TARGET(EXTENDED_ARG) opcode = NEXTOP(); oparg = oparg<<16 | NEXTARG(); goto dispatch_opcode; +#ifdef USE_COMPUTED_GOTOS + _unknown_opcode: +#endif default: fprintf(stderr, "XXX lineno: %d, opcode: %d\n", |