Issue #4753: By enabling a configure option named '--with-computed-gotos'

on compilers that support it (notably: gcc, SunPro, icc), the bytecode
evaluation loop is compiled with a new dispatch mechanism which gives
speedups of up to 20%, depending on the system, on various benchmarks.

1 files changed, 309 insertions, 188 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index fcf8efb..e0ce92e 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -705,6 +705,114 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 	char *filename;
 #endif
 
+/* Computed GOTOs, or
+       the-optimization-commonly-but-improperly-known-as-"threaded code"
+   using gcc's labels-as-values extension
+   (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).
+
+   The traditional bytecode evaluation loop uses a "switch" statement, which
+   decent compilers will optimize as a single indirect branch instruction 
+   combined with a lookup table of jump addresses. However, since the
+   indirect jump instruction is shared by all opcodes, the CPU will have a
+   hard time making the right prediction for where to jump next (actually,
+   it will be always wrong except in the uncommon case of a sequence of
+   several identical opcodes).
+
+   "Threaded code" in contrast, uses an explicit jump table and an explicit
+   indirect jump instruction at the end of each opcode. Since the jump
+   instruction is at a different address for each opcode, the CPU will make a
+   separate prediction for each of these instructions, which is equivalent to
+   predicting the second opcode of each opcode pair. These predictions have
+   a much better chance to turn out valid, especially in small bytecode loops.
+
+   A mispredicted branch on a modern CPU flushes the whole pipeline and
+   can cost several CPU cycles (depending on the pipeline depth), 
+   and potentially many more instructions (depending on the pipeline width).
+   A correctly predicted branch, however, is nearly free.
+
+   At the time of this writing, the "threaded code" version is up to 15-20%
+   faster than the normal "switch" version, depending on the compiler and the
+   CPU architecture.
+
+   We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined,
+   because it would render the measurements invalid.
+
+
+   NOTE: care must be taken that the compiler doesn't try to "optimize" the
+   indirect jumps by sharing them between all opcodes. Such optimizations
+   can be disabled on gcc by using the -fno-gcse flag (or possibly
+   -fno-crossjumping).
+*/
+
+#if defined(USE_COMPUTED_GOTOS) && defined(DYNAMIC_EXECUTION_PROFILE)
+#undef USE_COMPUTED_GOTOS
+#endif
+
+#ifdef USE_COMPUTED_GOTOS
+/* Import the static jump table */
+#include "opcode_targets.h"
+
+/* This macro is used when several opcodes defer to the same implementation
+   (e.g. SETUP_LOOP, SETUP_FINALLY) */
+#define TARGET_WITH_IMPL(op, impl) \
+	TARGET_##op: \
+		opcode = op; \
+		if (HAS_ARG(op)) \
+			oparg = NEXTARG(); \
+	case op: \
+		goto impl; \
+
+#define TARGET(op) \
+	TARGET_##op: \
+		opcode = op; \
+		if (HAS_ARG(op)) \
+			oparg = NEXTARG(); \
+	case op:
+
+
+#define DISPATCH() \
+	{ \
+		/* Avoid multiple loads from _Py_Ticker despite `volatile` */ \
+		int _tick = _Py_Ticker - 1; \
+		_Py_Ticker = _tick; \
+		if (_tick >= 0) { \
+			FAST_DISPATCH(); \
+		} \
+		continue; \
+	}
+
+#ifdef LLTRACE
+#define FAST_DISPATCH() \
+	{ \
+		if (!lltrace && !_Py_TracingPossible) { \
+			f->f_lasti = INSTR_OFFSET(); \
+			goto *opcode_targets[*next_instr++]; \
+		} \
+		goto fast_next_opcode; \
+	}
+#else
+#define FAST_DISPATCH() \
+	{ \
+		if (!_Py_TracingPossible) { \
+			f->f_lasti = INSTR_OFFSET(); \
+			goto *opcode_targets[*next_instr++]; \
+		} \
+		goto fast_next_opcode; \
+	}
+#endif
+
+#else
+#define TARGET(op) \
+	case op:
+#define TARGET_WITH_IMPL(op, impl) \
+	/* silence compiler warnings about `impl` unused */ \
+	if (0) goto impl; \
+	case op:
+#define DISPATCH() continue
+#define FAST_DISPATCH() goto fast_next_opcode
+#endif
+
+
 /* Tuple access macros */
 
 #ifndef Py_DEBUG
@@ -782,16 +890,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 	predictions turned-on and interpret the results as if some opcodes
 	had been combined or turn-off predictions so that the opcode frequency
 	counter updates for both opcodes.
+
+    Opcode prediction is disabled with threaded code, since the latter allows
+	the CPU to record separate branch prediction information for each
+	opcode.
+
 */
 
-#ifdef DYNAMIC_EXECUTION_PROFILE
+#if defined(DYNAMIC_EXECUTION_PROFILE) || defined(USE_COMPUTED_GOTOS)
 #define PREDICT(op)		if (0) goto PRED_##op
+#define PREDICTED(op)		PRED_##op:
+#define PREDICTED_WITH_ARG(op)	PRED_##op:
 #else
 #define PREDICT(op)		if (*next_instr == op) goto PRED_##op
-#endif
-
 #define PREDICTED(op)		PRED_##op: next_instr++
 #define PREDICTED_WITH_ARG(op)	PRED_##op: oparg = PEEKARG(); next_instr += 3
+#endif
+
 
 /* Stack manipulation macros */
 
@@ -1153,56 +1268,56 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
 		/* case STOP_CODE: this is an error! */
 
-		case NOP:
-			goto fast_next_opcode;
+		TARGET(NOP)
+			FAST_DISPATCH();
 
-		case LOAD_FAST:
+		TARGET(LOAD_FAST)
 			x = GETLOCAL(oparg);
 			if (x != NULL) {
 				Py_INCREF(x);
 				PUSH(x);
-				goto fast_next_opcode;
+				FAST_DISPATCH();
 			}
 			format_exc_check_arg(PyExc_UnboundLocalError,
 				UNBOUNDLOCAL_ERROR_MSG,
 				PyTuple_GetItem(co->co_varnames, oparg));
 			break;
 
-		case LOAD_CONST:
+		TARGET(LOAD_CONST)
 			x = GETITEM(consts, oparg);
 			Py_INCREF(x);
 			PUSH(x);
-			goto fast_next_opcode;
+			FAST_DISPATCH();
 
 		PREDICTED_WITH_ARG(STORE_FAST);
-		case STORE_FAST:
+		TARGET(STORE_FAST)
 			v = POP();
 			SETLOCAL(oparg, v);
-			goto fast_next_opcode;
+			FAST_DISPATCH();
 
 		PREDICTED(POP_TOP);
-		case POP_TOP:
+		TARGET(POP_TOP)
 			v = POP();
 			Py_DECREF(v);
-			goto fast_next_opcode;
+			FAST_DISPATCH();
 
-		case ROT_TWO:
+		TARGET(ROT_TWO)
 			v = TOP();
 			w = SECOND();
 			SET_TOP(w);
 			SET_SECOND(v);
-			goto fast_next_opcode;
+			FAST_DISPATCH();
 
-		case ROT_THREE:
+		TARGET(ROT_THREE)
 			v = TOP();
 			w = SECOND();
 			x = THIRD();
 			SET_TOP(w);
 			SET_SECOND(x);
 			SET_THIRD(v);
-			goto fast_next_opcode;
+			FAST_DISPATCH();
 
-		case ROT_FOUR:
+		TARGET(ROT_FOUR)
 			u = TOP();
 			v = SECOND();
 			w = THIRD();
@@ -1211,15 +1326,15 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			SET_SECOND(w);
 			SET_THIRD(x);
 			SET_FOURTH(u);
-			goto fast_next_opcode;
+			FAST_DISPATCH();
 
-		case DUP_TOP:
+		TARGET(DUP_TOP)
 			v = TOP();
 			Py_INCREF(v);
 			PUSH(v);
-			goto fast_next_opcode;
+			FAST_DISPATCH();
 
-		case DUP_TOPX:
+		TARGET(DUP_TOPX)
 			if (oparg == 2) {
 				x = TOP();
 				Py_INCREF(x);
@@ -1228,7 +1343,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 				STACKADJ(2);
 				SET_TOP(x);
 				SET_SECOND(w);
-				goto fast_next_opcode;
+				FAST_DISPATCH();
 			} else if (oparg == 3) {
 				x = TOP();
 				Py_INCREF(x);
@@ -1240,106 +1355,106 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 				SET_TOP(x);
 				SET_SECOND(w);
 				SET_THIRD(v);
-				goto fast_next_opcode;
+				FAST_DISPATCH();
 			}
 			Py_FatalError("invalid argument to DUP_TOPX"
 				      " (bytecode corruption?)");
 			/* Never returns, so don't bother to set why. */
 			break;
 
-		case UNARY_POSITIVE:
+		TARGET(UNARY_POSITIVE)
 			v = TOP();
 			x = PyNumber_Positive(v);
 			Py_DECREF(v);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case UNARY_NEGATIVE:
+		TARGET(UNARY_NEGATIVE)
 			v = TOP();
 			x = PyNumber_Negative(v);
 			Py_DECREF(v);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case UNARY_NOT:
+		TARGET(UNARY_NOT)
 			v = TOP();
 			err = PyObject_IsTrue(v);
 			Py_DECREF(v);
 			if (err == 0) {
 				Py_INCREF(Py_True);
 				SET_TOP(Py_True);
-				continue;
+				DISPATCH();
 			}
 			else if (err > 0) {
 				Py_INCREF(Py_False);
 				SET_TOP(Py_False);
 				err = 0;
-				continue;
+				DISPATCH();
 			}
 			STACKADJ(-1);
 			break;
 
-		case UNARY_INVERT:
+		TARGET(UNARY_INVERT)
 			v = TOP();
 			x = PyNumber_Invert(v);
 			Py_DECREF(v);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_POWER:
+		TARGET(BINARY_POWER)
 			w = POP();
 			v = TOP();
 			x = PyNumber_Power(v, w, Py_None);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_MULTIPLY:
+		TARGET(BINARY_MULTIPLY)
 			w = POP();
 			v = TOP();
 			x = PyNumber_Multiply(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_TRUE_DIVIDE:
+		TARGET(BINARY_TRUE_DIVIDE)
 			w = POP();
 			v = TOP();
 			x = PyNumber_TrueDivide(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_FLOOR_DIVIDE:
+		TARGET(BINARY_FLOOR_DIVIDE)
 			w = POP();
 			v = TOP();
 			x = PyNumber_FloorDivide(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_MODULO:
+		TARGET(BINARY_MODULO)
 			w = POP();
 			v = TOP();
 			x = PyNumber_Remainder(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_ADD:
+		TARGET(BINARY_ADD)
 			w = POP();
 			v = TOP();
 			if (PyUnicode_CheckExact(v) &&
@@ -1355,152 +1470,152 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 		  skip_decref_vx:
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_SUBTRACT:
+		TARGET(BINARY_SUBTRACT)
 			w = POP();
 			v = TOP();
 			x = PyNumber_Subtract(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_SUBSCR:
+		TARGET(BINARY_SUBSCR)
 			w = POP();
 			v = TOP();
 			x = PyObject_GetItem(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_LSHIFT:
+		TARGET(BINARY_LSHIFT)
 			w = POP();
 			v = TOP();
 			x = PyNumber_Lshift(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_RSHIFT:
+		TARGET(BINARY_RSHIFT)
 			w = POP();
 			v = TOP();
 			x = PyNumber_Rshift(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_AND:
+		TARGET(BINARY_AND)
 			w = POP();
 			v = TOP();
 			x = PyNumber_And(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_XOR:
+		TARGET(BINARY_XOR)
 			w = POP();
 			v = TOP();
 			x = PyNumber_Xor(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case BINARY_OR:
+		TARGET(BINARY_OR)
 			w = POP();
 			v = TOP();
 			x = PyNumber_Or(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case LIST_APPEND:
+		TARGET(LIST_APPEND)
 			w = POP();
 			v = stack_pointer[-oparg];
 			err = PyList_Append(v, w);
 			Py_DECREF(w);
 			if (err == 0) {
 				PREDICT(JUMP_ABSOLUTE);
-				continue;
+				DISPATCH();
 			}
 			break;
 
-		case SET_ADD:
+		TARGET(SET_ADD)
 			w = POP();
 			v = stack_pointer[-oparg];
 			err = PySet_Add(v, w);
 			Py_DECREF(w);
 			if (err == 0) {
 				PREDICT(JUMP_ABSOLUTE);
-				continue;
+				DISPATCH();
 			}
 			break;
 
-		case INPLACE_POWER:
+		TARGET(INPLACE_POWER)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlacePower(v, w, Py_None);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_MULTIPLY:
+		TARGET(INPLACE_MULTIPLY)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceMultiply(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_TRUE_DIVIDE:
+		TARGET(INPLACE_TRUE_DIVIDE)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceTrueDivide(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_FLOOR_DIVIDE:
+		TARGET(INPLACE_FLOOR_DIVIDE)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceFloorDivide(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_MODULO:
+		TARGET(INPLACE_MODULO)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceRemainder(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_ADD:
+		TARGET(INPLACE_ADD)
 			w = POP();
 			v = TOP();
 			if (PyUnicode_CheckExact(v) &&
@@ -1516,70 +1631,70 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 		  skip_decref_v:
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_SUBTRACT:
+		TARGET(INPLACE_SUBTRACT)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceSubtract(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_LSHIFT:
+		TARGET(INPLACE_LSHIFT)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceLshift(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_RSHIFT:
+		TARGET(INPLACE_RSHIFT)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceRshift(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_AND:
+		TARGET(INPLACE_AND)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceAnd(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_XOR:
+		TARGET(INPLACE_XOR)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceXor(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case INPLACE_OR:
+		TARGET(INPLACE_OR)
 			w = POP();
 			v = TOP();
 			x = PyNumber_InPlaceOr(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case STORE_SUBSCR:
+		TARGET(STORE_SUBSCR)
 			w = TOP();
 			v = SECOND();
 			u = THIRD();
@@ -1589,10 +1704,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			Py_DECREF(u);
 			Py_DECREF(v);
 			Py_DECREF(w);
-			if (err == 0) continue;
+			if (err == 0) DISPATCH();
 			break;
 
-		case DELETE_SUBSCR:
+		TARGET(DELETE_SUBSCR)
 			w = TOP();
 			v = SECOND();
 			STACKADJ(-2);
@@ -1600,10 +1715,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			err = PyObject_DelItem(v, w);
 			Py_DECREF(v);
 			Py_DECREF(w);
-			if (err == 0) continue;
+			if (err == 0) DISPATCH();
 			break;
 
-		case PRINT_EXPR:
+		TARGET(PRINT_EXPR)
 			v = POP();
 			w = PySys_GetObject("displayhook");
 			if (w == NULL) {
@@ -1630,7 +1745,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 #ifdef CASE_TOO_BIG
 		default: switch (opcode) {
 #endif
-		case RAISE_VARARGS:
+		TARGET(RAISE_VARARGS)
 			v = w = NULL;
 			switch (oparg) {
 			case 2:
@@ -1648,19 +1763,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			}
 			break;
 
-		case STORE_LOCALS:
+		TARGET(STORE_LOCALS)
 			x = POP();
 			v = f->f_locals;
 			Py_XDECREF(v);
 			f->f_locals = x;
-			continue;
+			DISPATCH();
 
-		case RETURN_VALUE:
+		TARGET(RETURN_VALUE)
 			retval = POP();
 			why = WHY_RETURN;
 			goto fast_block_end;
 
-		case YIELD_VALUE:
+		TARGET(YIELD_VALUE)
 			retval = POP();
 			f->f_stacktop = stack_pointer;
 			why = WHY_YIELD;
@@ -1670,7 +1785,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			SWAP_EXC_STATE();
 			goto fast_yield;
 
-		case POP_EXCEPT:
+		TARGET(POP_EXCEPT)
 			{
 				PyTryBlock *b = PyFrame_BlockPop(f);
 				if (b->b_type != EXCEPT_HANDLER) {
@@ -1681,17 +1796,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 				}
 				UNWIND_EXCEPT_HANDLER(b);
 			}
-			continue;
+			DISPATCH();
 
-		case POP_BLOCK:
+		TARGET(POP_BLOCK)
 			{
 				PyTryBlock *b = PyFrame_BlockPop(f);
 				UNWIND_BLOCK(b);
 			}
-			continue;
+			DISPATCH();
 
 		PREDICTED(END_FINALLY);
-		case END_FINALLY:
+		TARGET(END_FINALLY)
 			v = POP();
 			if (PyLong_Check(v)) {
 				why = (enum why_code) PyLong_AS_LONG(v);
@@ -1731,7 +1846,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			Py_DECREF(v);
 			break;
 
-		case LOAD_BUILD_CLASS:
+		TARGET(LOAD_BUILD_CLASS)
 			x = PyDict_GetItemString(f->f_builtins,
 						 "__build_class__");
 			if (x == NULL) {
@@ -1743,7 +1858,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			PUSH(x);
 			break;
 
-		case STORE_NAME:
+		TARGET(STORE_NAME)
 			w = GETITEM(names, oparg);
 			v = POP();
 			if ((x = f->f_locals) != NULL) {
@@ -1752,14 +1867,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 				else
 					err = PyObject_SetItem(x, w, v);
 				Py_DECREF(v);
-				if (err == 0) continue;
+				if (err == 0) DISPATCH();
 				break;
 			}
 			PyErr_Format(PyExc_SystemError,
 				     "no locals found when storing %R", w);
 			break;
 
-		case DELETE_NAME:
+		TARGET(DELETE_NAME)
 			w = GETITEM(names, oparg);
 			if ((x = f->f_locals) != NULL) {
 				if ((err = PyObject_DelItem(x, w)) != 0)
@@ -1773,7 +1888,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			break;
 
 		PREDICTED_WITH_ARG(UNPACK_SEQUENCE);
-		case UNPACK_SEQUENCE:
+		TARGET(UNPACK_SEQUENCE)
 			v = POP();
 			if (PyTuple_CheckExact(v) &&
 			    PyTuple_GET_SIZE(v) == oparg) {
@@ -1785,7 +1900,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 					PUSH(w);
 				}
 				Py_DECREF(v);
-				continue;
+				DISPATCH();
 			} else if (PyList_CheckExact(v) &&
 				   PyList_GET_SIZE(v) == oparg) {
 				PyObject **items = \
@@ -1805,7 +1920,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			Py_DECREF(v);
 			break;
 
-		case UNPACK_EX:
+		TARGET(UNPACK_EX)
 		{
 			int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
 			v = POP();
@@ -1820,7 +1935,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			break;
 		}
 
-		case STORE_ATTR:
+		TARGET(STORE_ATTR)
 			w = GETITEM(names, oparg);
 			v = TOP();
 			u = SECOND();
@@ -1828,10 +1943,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			err = PyObject_SetAttr(v, w, u); /* v.w = u */
 			Py_DECREF(v);
 			Py_DECREF(u);
-			if (err == 0) continue;
+			if (err == 0) DISPATCH();
 			break;
 
-		case DELETE_ATTR:
+		TARGET(DELETE_ATTR)
 			w = GETITEM(names, oparg);
 			v = POP();
 			err = PyObject_SetAttr(v, w, (PyObject *)NULL);
@@ -1839,22 +1954,22 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			Py_DECREF(v);
 			break;
 
-		case STORE_GLOBAL:
+		TARGET(STORE_GLOBAL)
 			w = GETITEM(names, oparg);
 			v = POP();
 			err = PyDict_SetItem(f->f_globals, w, v);
 			Py_DECREF(v);
-			if (err == 0) continue;
+			if (err == 0) DISPATCH();
 			break;
 
-		case DELETE_GLOBAL:
+		TARGET(DELETE_GLOBAL)
 			w = GETITEM(names, oparg);
 			if ((err = PyDict_DelItem(f->f_globals, w)) != 0)
 				format_exc_check_arg(
 				    PyExc_NameError, GLOBAL_NAME_ERROR_MSG, w);
 			break;
 
-		case LOAD_NAME:
+		TARGET(LOAD_NAME)
 			w = GETITEM(names, oparg);
 			if ((v = f->f_locals) == NULL) {
 				PyErr_Format(PyExc_SystemError,
@@ -1889,9 +2004,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 				Py_INCREF(x);
 			}
 			PUSH(x);
-			continue;
+			DISPATCH();
 
-		case LOAD_GLOBAL:
+		TARGET(LOAD_GLOBAL)
 			w = GETITEM(names, oparg);
 			if (PyUnicode_CheckExact(w)) {
 				/* Inline the PyDict_GetItem() calls.
@@ -1911,7 +2026,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 					if (x != NULL) {
 						Py_INCREF(x);
 						PUSH(x);
-						continue;
+						DISPATCH();
 					}
 					d = (PyDictObject *)(f->f_builtins);
 					e = d->ma_lookup(d, w, hash);
@@ -1923,7 +2038,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 					if (x != NULL) {
 						Py_INCREF(x);
 						PUSH(x);
-						continue;
+						DISPATCH();
 					}
 					goto load_global_error;
 				}
@@ -1942,13 +2057,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			}
 			Py_INCREF(x);
 			PUSH(x);
-			continue;
+			DISPATCH();
 
-		case DELETE_FAST:
+		TARGET(DELETE_FAST)
 			x = GETLOCAL(oparg);
 			if (x != NULL) {
 				SETLOCAL(oparg, NULL);
-				continue;
+				DISPATCH();
 			}
 			format_exc_check_arg(
 				PyExc_UnboundLocalError,
@@ -1957,19 +2072,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 				);
 			break;
 
-		case LOAD_CLOSURE:
+		TARGET(LOAD_CLOSURE)
 			x = freevars[oparg];
 			Py_INCREF(x);
 			PUSH(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case LOAD_DEREF:
+		TARGET(LOAD_DEREF)
 			x = freevars[oparg];
 			w = PyCell_Get(x);
 			if (w != NULL) {
 				PUSH(w);
-				continue;
+				DISPATCH();
 			}
 			err = -1;
 			/* Don't stomp existing exception */
@@ -1990,14 +2105,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			}
 			break;
 
-		case STORE_DEREF:
+		TARGET(STORE_DEREF)
 			w = POP();
 			x = freevars[oparg];
 			PyCell_Set(x, w);
 			Py_DECREF(w);
-			continue;
+			DISPATCH();
 
-		case BUILD_TUPLE:
+		TARGET(BUILD_TUPLE)
 			x = PyTuple_New(oparg);
 			if (x != NULL) {
 				for (; --oparg >= 0;) {
@@ -2005,11 +2120,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 					PyTuple_SET_ITEM(x, oparg, w);
 				}
 				PUSH(x);
-				continue;
+				DISPATCH();
 			}
 			break;
 
-		case BUILD_LIST:
+		TARGET(BUILD_LIST)
 			x =  PyList_New(oparg);
 			if (x != NULL) {
 				for (; --oparg >= 0;) {
@@ -2017,11 +2132,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 					PyList_SET_ITEM(x, oparg, w);
 				}
 				PUSH(x);
-				continue;
+				DISPATCH();
 			}
 			break;
 
-		case BUILD_SET:
+		TARGET(BUILD_SET)
 			x = PySet_New(NULL);
 			if (x != NULL) {
 				for (; --oparg >= 0;) {
@@ -2035,17 +2150,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 					break;
 				}
 				PUSH(x);
-				continue;
+				DISPATCH();
 			}
 			break;
 
-		case BUILD_MAP:
+		TARGET(BUILD_MAP)
 			x = _PyDict_NewPresized((Py_ssize_t)oparg);
 			PUSH(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case STORE_MAP:
+		TARGET(STORE_MAP)
 			w = TOP();     /* key */
 			u = SECOND();  /* value */
 			v = THIRD();   /* dict */
@@ -2054,10 +2169,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			err = PyDict_SetItem(v, w, u);  /* v[w] = u */
 			Py_DECREF(u);
 			Py_DECREF(w);
-			if (err == 0) continue;
+			if (err == 0) DISPATCH();
 			break;
 
-		case MAP_ADD:
+		TARGET(MAP_ADD)
 			w = TOP();     /* key */
 			u = SECOND();  /* value */
 			STACKADJ(-2);
@@ -2068,20 +2183,20 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			Py_DECREF(w);
 			if (err == 0) {
 				PREDICT(JUMP_ABSOLUTE);
-				continue;
+				DISPATCH();
 			}
 			break;
 
-		case LOAD_ATTR:
+		TARGET(LOAD_ATTR)
 			w = GETITEM(names, oparg);
 			v = TOP();
 			x = PyObject_GetAttr(v, w);
 			Py_DECREF(v);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case COMPARE_OP:
+		TARGET(COMPARE_OP)
 			w = POP();
 			v = TOP();
 			x = cmp_outcome(oparg, v, w);
@@ -2091,9 +2206,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			if (x == NULL) break;
 			PREDICT(JUMP_IF_FALSE);
 			PREDICT(JUMP_IF_TRUE);
-			continue;
+			DISPATCH();
 
-		case IMPORT_NAME:
+		TARGET(IMPORT_NAME)
 			w = GETITEM(names, oparg);
 			x = PyDict_GetItemString(f->f_builtins, "__import__");
 			if (x == NULL) {
@@ -2134,10 +2249,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			READ_TIMESTAMP(intr1);
 			Py_DECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case IMPORT_STAR:
+		TARGET(IMPORT_STAR)
 			v = POP();
 			PyFrame_FastToLocals(f);
 			if ((x = f->f_locals) == NULL) {
@@ -2150,33 +2265,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			READ_TIMESTAMP(intr1);
 			PyFrame_LocalsToFast(f, 0);
 			Py_DECREF(v);
-			if (err == 0) continue;
+			if (err == 0) DISPATCH();
 			break;
 
-		case IMPORT_FROM:
+		TARGET(IMPORT_FROM)
 			w = GETITEM(names, oparg);
 			v = TOP();
 			READ_TIMESTAMP(intr0);
 			x = import_from(v, w);
 			READ_TIMESTAMP(intr1);
 			PUSH(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case JUMP_FORWARD:
+		TARGET(JUMP_FORWARD)
 			JUMPBY(oparg);
-			goto fast_next_opcode;
+			FAST_DISPATCH();
 
 		PREDICTED_WITH_ARG(JUMP_IF_FALSE);
-		case JUMP_IF_FALSE:
+		TARGET(JUMP_IF_FALSE)
 			w = TOP();
 			if (w == Py_True) {
 				PREDICT(POP_TOP);
-				goto fast_next_opcode;
+				FAST_DISPATCH();
 			}
 			if (w == Py_False) {
 				JUMPBY(oparg);
-				goto fast_next_opcode;
+				FAST_DISPATCH();
 			}
 			err = PyObject_IsTrue(w);
 			if (err > 0)
@@ -2185,18 +2300,18 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 				JUMPBY(oparg);
 			else
 				break;
-			continue;
+			DISPATCH();
 
 		PREDICTED_WITH_ARG(JUMP_IF_TRUE);
-		case JUMP_IF_TRUE:
+		TARGET(JUMP_IF_TRUE)
 			w = TOP();
 			if (w == Py_False) {
 				PREDICT(POP_TOP);
-				goto fast_next_opcode;
+				FAST_DISPATCH();
 			}
 			if (w == Py_True) {
 				JUMPBY(oparg);
-				goto fast_next_opcode;
+				FAST_DISPATCH();
 			}
 			err = PyObject_IsTrue(w);
 			if (err > 0) {
@@ -2207,10 +2322,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 				;
 			else
 				break;
-			continue;
+			DISPATCH();
 
 		PREDICTED_WITH_ARG(JUMP_ABSOLUTE);
-		case JUMP_ABSOLUTE:
+		TARGET(JUMP_ABSOLUTE)
 			JUMPTO(oparg);
 #if FAST_LOOPS
 			/* Enabling this path speeds-up all while and for-loops by bypassing
@@ -2218,14 +2333,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                            because it prevents detection of a control-break in tight loops like
                            "while 1: pass".  Compile with this option turned-on when you need
                            the speed-up and do not need break checking inside tight loops (ones
-                           that contain only instructions ending with goto fast_next_opcode).
+                           that contain only instructions ending with FAST_DISPATCH).
                         */
-			goto fast_next_opcode;
+			FAST_DISPATCH();
 #else
-			continue;
+			DISPATCH();
 #endif
 
-		case GET_ITER:
+		TARGET(GET_ITER)
 			/* before: [obj]; after [getiter(obj)] */
 			v = TOP();
 			x = PyObject_GetIter(v);
@@ -2233,13 +2348,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			if (x != NULL) {
 				SET_TOP(x);
 				PREDICT(FOR_ITER);
-				continue;
+				DISPATCH();
 			}
 			STACKADJ(-1);
 			break;
 
 		PREDICTED_WITH_ARG(FOR_ITER);
-		case FOR_ITER:
+		TARGET(FOR_ITER)
 			/* before: [iter]; after: [iter, iter()] *or* [] */
 			v = TOP();
 			x = (*v->ob_type->tp_iternext)(v);
@@ -2247,7 +2362,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 				PUSH(x);
 				PREDICT(STORE_FAST);
 				PREDICT(UNPACK_SEQUENCE);
-				continue;
+				DISPATCH();
 			}
 			if (PyErr_Occurred()) {
 				if (!PyErr_ExceptionMatches(
@@ -2259,13 +2374,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
  			x = v = POP();
 			Py_DECREF(v);
 			JUMPBY(oparg);
-			continue;
+			DISPATCH();
 
-		case BREAK_LOOP:
+		TARGET(BREAK_LOOP)
 			why = WHY_BREAK;
 			goto fast_block_end;
 
-		case CONTINUE_LOOP:
+		TARGET(CONTINUE_LOOP)
 			retval = PyLong_FromLong(oparg);
 			if (!retval) {
 				x = NULL;
@@ -2274,9 +2389,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			why = WHY_CONTINUE;
 			goto fast_block_end;
 
-		case SETUP_LOOP:
-		case SETUP_EXCEPT:
-		case SETUP_FINALLY:
+		TARGET_WITH_IMPL(SETUP_LOOP, _setup_finally)
+		TARGET_WITH_IMPL(SETUP_EXCEPT, _setup_finally)
+		TARGET(SETUP_FINALLY)
+		_setup_finally:
 			/* NOTE: If you add any new block-setup opcodes that
 		           are not try/except/finally handlers, you may need
 		           to update the PyGen_NeedsFinalizing() function.
@@ -2284,9 +2400,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
 			PyFrame_BlockSetup(f, opcode, INSTR_OFFSET() + oparg,
 					   STACK_LEVEL());
-			continue;
+			DISPATCH();
 
-		case WITH_CLEANUP:
+		TARGET(WITH_CLEANUP)
 		{
 			/* At the top of the stack are 1-3 values indicating
 			   how/why we entered the finally clause:
@@ -2350,7 +2466,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			break;
 		}
 
-		case CALL_FUNCTION:
+		TARGET(CALL_FUNCTION)
 		{
 			PyObject **sp;
 			PCALL(PCALL_ALL);
@@ -2363,13 +2479,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			stack_pointer = sp;
 			PUSH(x);
 			if (x != NULL)
-				continue;
+				DISPATCH();
 			break;
 		}
 
-		case CALL_FUNCTION_VAR:
-		case CALL_FUNCTION_KW:
-		case CALL_FUNCTION_VAR_KW:
+		TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw)
+		TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw)
+		TARGET(CALL_FUNCTION_VAR_KW)
+		_call_function_var_kw:
 		{
 		    int na = oparg & 0xff;
 		    int nk = (oparg>>8) & 0xff;
@@ -2409,12 +2526,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 		    }
 		    PUSH(x);
 		    if (x != NULL)
-			    continue;
+			    DISPATCH();
 		    break;
 		}
 
-		case MAKE_CLOSURE:
-		case MAKE_FUNCTION:
+		TARGET_WITH_IMPL(MAKE_CLOSURE, _make_function)
+		TARGET(MAKE_FUNCTION)
+		_make_function:
 		{
 		    int posdefaults = oparg & 0xff;
 		    int kwdefaults = (oparg>>8) & 0xff;
@@ -2507,7 +2625,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			break;
 		}
 
-		case BUILD_SLICE:
+		TARGET(BUILD_SLICE)
 			if (oparg == 3)
 				w = POP();
 			else
@@ -2519,14 +2637,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			Py_DECREF(v);
 			Py_XDECREF(w);
 			SET_TOP(x);
-			if (x != NULL) continue;
+			if (x != NULL) DISPATCH();
 			break;
 
-		case EXTENDED_ARG:
+		TARGET(EXTENDED_ARG)
 			opcode = NEXTOP();
 			oparg = oparg<<16 | NEXTARG();
 			goto dispatch_opcode;
 
+#ifdef USE_COMPUTED_GOTOS
+		_unknown_opcode:
+#endif
 		default:
 			fprintf(stderr,
 				"XXX lineno: %d, opcode: %d\n",