bpo-43760: Check for tracing using 'bitwise or' instead of branch in dispatch. (GH-28723)

7 files changed, 249 insertions, 217 deletions

diff --git a/Include/opcode.h b/Include/opcode.h
index 2789525..8817a4d 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -167,6 +167,7 @@ extern "C" {
 #define LOAD_FAST__LOAD_CONST   134
 #define LOAD_CONST__LOAD_FAST   140
 #define STORE_FAST__STORE_FAST  143
+#define DO_TRACING              255
 #ifdef NEED_OPCODE_JUMP_TABLES
 static uint32_t _PyOpcode_RelativeJump[8] = {
     0U,
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-04-16-11-50.bpo-43760.R9QoUv.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-04-16-11-50.bpo-43760.R9QoUv.rst
new file mode 100644
index 0000000..1809b42
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-04-16-11-50.bpo-43760.R9QoUv.rst
@@ -0,0 +1,3 @@
+The number of hardware branches per instruction dispatch is reduced from two
+to one by adding a special instruction for tracing. Patch by Mark Shannon.
+
diff --git a/Python/ceval.c b/Python/ceval.c
index 8f65bb3..2dbc291 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1218,7 +1218,7 @@ eval_frame_handle_pending(PyThreadState *tstate)
 #endif
 
 #ifdef WITH_DTRACE
-#define OR_DTRACE_LINE || PyDTrace_LINE_ENABLED()
+#define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0)
 #else
 #define OR_DTRACE_LINE
 #endif
@@ -1240,11 +1240,13 @@ eval_frame_handle_pending(PyThreadState *tstate)
     #define USE_COMPUTED_GOTOS 0
 #endif
 
+#define INSTRUCTION_START() frame->f_lasti = INSTR_OFFSET(); next_instr++
+
 #if USE_COMPUTED_GOTOS
-#define TARGET(op) TARGET_##op
+#define TARGET(op) TARGET_##op: INSTRUCTION_START();
 #define DISPATCH_GOTO() goto *opcode_targets[opcode]
 #else
-#define TARGET(op) case op
+#define TARGET(op) case op: INSTRUCTION_START();
 #define DISPATCH_GOTO() goto dispatch_opcode
 #endif
 
@@ -1274,12 +1276,11 @@ eval_frame_handle_pending(PyThreadState *tstate)
 #endif
 #endif
 #ifndef PRE_DISPATCH_GOTO
-#define PRE_DISPATCH_GOTO() do { LLTRACE_INSTR();  RECORD_DXPROFILE(); } while (0)
+#define PRE_DISPATCH_GOTO() do { LLTRACE_INSTR(); RECORD_DXPROFILE(); } while (0)
 #endif
 
 #define NOTRACE_DISPATCH() \
     { \
-        frame->f_lasti = INSTR_OFFSET(); \
         NEXTOPARG(); \
         PRE_DISPATCH_GOTO(); \
         DISPATCH_GOTO(); \
@@ -1288,10 +1289,11 @@ eval_frame_handle_pending(PyThreadState *tstate)
 /* Do interpreter dispatch accounting for tracing and instrumentation */
 #define DISPATCH() \
     { \
-        if (cframe.use_tracing OR_DTRACE_LINE) { \
-            goto tracing_dispatch; \
-        } \
-        NOTRACE_DISPATCH(); \
+        NEXTOPARG(); \
+        PRE_DISPATCH_GOTO(); \
+        assert(cframe.use_tracing == 0 || cframe.use_tracing == 255); \
+        opcode |= cframe.use_tracing OR_DTRACE_LINE; \
+        DISPATCH_GOTO(); \
     }
 
 #define CHECK_EVAL_BREAKER() \
@@ -1316,7 +1318,6 @@ eval_frame_handle_pending(PyThreadState *tstate)
         _Py_CODEUNIT word = *next_instr; \
         opcode = _Py_OPCODE(word); \
         oparg = _Py_OPARG(word); \
-        next_instr++; \
     } while (0)
 #define JUMPTO(x)       (next_instr = first_instr + (x))
 #define JUMPBY(x)       (next_instr += (x))
@@ -1326,7 +1327,6 @@ eval_frame_handle_pending(PyThreadState *tstate)
         _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code))[INSTR_OFFSET()]; \
         opcode = _Py_OPCODE(word); \
         oparg = _Py_OPARG(word); \
-        next_instr++; \
     } while (0)
 
 /* OpCode prediction macros
@@ -1363,10 +1363,10 @@ eval_frame_handle_pending(PyThreadState *tstate)
 #define PREDICT(op) \
     do { \
         _Py_CODEUNIT word = *next_instr; \
-        opcode = _Py_OPCODE(word); \
+        opcode = _Py_OPCODE(word) | cframe.use_tracing OR_DTRACE_LINE; \
         if (opcode == op) { \
             oparg = _Py_OPARG(word); \
-            next_instr++; \
+            INSTRUCTION_START(); \
             goto PREDICT_ID(op); \
         } \
     } while(0)
@@ -1516,6 +1516,15 @@ trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame)
     return 0;
 }
 
+static int
+skip_backwards_over_extended_args(PyCodeObject *code, int offset) {
+    _Py_CODEUNIT *instrs = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code);
+    while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) {
+        offset--;
+    }
+    return offset;
+}
+
 PyObject* _Py_HOT_FUNCTION
 _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int throwflag)
 {
@@ -1668,42 +1677,6 @@ check_eval_breaker:
 
     DISPATCH();
 
-    tracing_dispatch:
-    {
-        int instr_prev = frame->f_lasti;
-        frame->f_lasti = INSTR_OFFSET();
-        TRACING_NEXTOPARG();
-
-        if (PyDTrace_LINE_ENABLED())
-            maybe_dtrace_line(frame, &tstate->trace_info, instr_prev);
-
-        /* line-by-line tracing support */
-
-        if (cframe.use_tracing &&
-            tstate->c_tracefunc != NULL && !tstate->tracing) {
-            int err;
-            /* see maybe_call_line_trace()
-               for expository comments */
-            _PyFrame_SetStackPointer(frame, stack_pointer);
-
-            err = maybe_call_line_trace(tstate->c_tracefunc,
-                                        tstate->c_traceobj,
-                                        tstate, frame, instr_prev);
-            if (err) {
-                /* trace function raised an exception */
-                goto error;
-            }
-            /* Reload possibly changed frame fields */
-            JUMPTO(frame->f_lasti);
-
-            stack_pointer = _PyFrame_GetStackPointer(frame);
-            frame->stacktop = -1;
-            TRACING_NEXTOPARG();
-        }
-        PRE_DISPATCH_GOTO();
-        DISPATCH_GOTO();
-    }
-
     /* Start instructions */
 #if USE_COMPUTED_GOTOS
     {
@@ -1716,13 +1689,12 @@ check_eval_breaker:
            It is essential that any operation that fails must goto error
            and that all operation that succeed call DISPATCH() ! */
 
-        TARGET(NOP): {
+        TARGET(NOP) {
             DISPATCH();
         }
 
         /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */
-        TARGET(LOAD_CLOSURE):
-        TARGET(LOAD_FAST): {
+        TARGET(LOAD_CLOSURE) {
             PyObject *value = GETLOCAL(oparg);
             if (value == NULL) {
                 goto unbound_local_error;
@@ -1732,7 +1704,17 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_CONST): {
+        TARGET(LOAD_FAST) {
+            PyObject *value = GETLOCAL(oparg);
+            if (value == NULL) {
+                goto unbound_local_error;
+            }
+            Py_INCREF(value);
+            PUSH(value);
+            DISPATCH();
+        }
+
+        TARGET(LOAD_CONST) {
             PREDICTED(LOAD_CONST);
             PyObject *value = GETITEM(consts, oparg);
             Py_INCREF(value);
@@ -1740,19 +1722,20 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(STORE_FAST): {
+        TARGET(STORE_FAST) {
             PREDICTED(STORE_FAST);
             PyObject *value = POP();
             SETLOCAL(oparg, value);
             DISPATCH();
         }
 
-        TARGET(LOAD_FAST__LOAD_FAST): {
+        TARGET(LOAD_FAST__LOAD_FAST) {
             PyObject *value = GETLOCAL(oparg);
             if (value == NULL) {
                 goto unbound_local_error;
             }
             NEXTOPARG();
+            next_instr++;
             Py_INCREF(value);
             PUSH(value);
             value = GETLOCAL(oparg);
@@ -1764,12 +1747,13 @@ check_eval_breaker:
             NOTRACE_DISPATCH();
         }
 
-        TARGET(LOAD_FAST__LOAD_CONST): {
+        TARGET(LOAD_FAST__LOAD_CONST) {
             PyObject *value = GETLOCAL(oparg);
             if (value == NULL) {
                 goto unbound_local_error;
             }
             NEXTOPARG();
+            next_instr++;
             Py_INCREF(value);
             PUSH(value);
             value = GETITEM(consts, oparg);
@@ -1778,10 +1762,11 @@ check_eval_breaker:
             NOTRACE_DISPATCH();
         }
 
-        TARGET(STORE_FAST__LOAD_FAST): {
+        TARGET(STORE_FAST__LOAD_FAST) {
             PyObject *value = POP();
             SETLOCAL(oparg, value);
             NEXTOPARG();
+            next_instr++;
             value = GETLOCAL(oparg);
             if (value == NULL) {
                 goto unbound_local_error;
@@ -1791,18 +1776,20 @@ check_eval_breaker:
             NOTRACE_DISPATCH();
         }
 
-        TARGET(STORE_FAST__STORE_FAST): {
+        TARGET(STORE_FAST__STORE_FAST) {
             PyObject *value = POP();
             SETLOCAL(oparg, value);
             NEXTOPARG();
+            next_instr++;
             value = POP();
             SETLOCAL(oparg, value);
             NOTRACE_DISPATCH();
         }
 
-        TARGET(LOAD_CONST__LOAD_FAST): {
+        TARGET(LOAD_CONST__LOAD_FAST) {
             PyObject *value = GETITEM(consts, oparg);
             NEXTOPARG();
+            next_instr++;
             Py_INCREF(value);
             PUSH(value);
             value = GETLOCAL(oparg);
@@ -1814,13 +1801,13 @@ check_eval_breaker:
             NOTRACE_DISPATCH();
         }
 
-        TARGET(POP_TOP): {
+        TARGET(POP_TOP) {
             PyObject *value = POP();
             Py_DECREF(value);
             DISPATCH();
         }
 
-        TARGET(ROT_TWO): {
+        TARGET(ROT_TWO) {
             PyObject *top = TOP();
             PyObject *second = SECOND();
             SET_TOP(second);
@@ -1828,7 +1815,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(ROT_THREE): {
+        TARGET(ROT_THREE) {
             PyObject *top = TOP();
             PyObject *second = SECOND();
             PyObject *third = THIRD();
@@ -1838,7 +1825,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(ROT_FOUR): {
+        TARGET(ROT_FOUR) {
             PyObject *top = TOP();
             PyObject *second = SECOND();
             PyObject *third = THIRD();
@@ -1850,14 +1837,14 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(DUP_TOP): {
+        TARGET(DUP_TOP) {
             PyObject *top = TOP();
             Py_INCREF(top);
             PUSH(top);
             DISPATCH();
         }
 
-        TARGET(DUP_TOP_TWO): {
+        TARGET(DUP_TOP_TWO) {
             PyObject *top = TOP();
             PyObject *second = SECOND();
             Py_INCREF(top);
@@ -1868,7 +1855,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(UNARY_POSITIVE): {
+        TARGET(UNARY_POSITIVE) {
             PyObject *value = TOP();
             PyObject *res = PyNumber_Positive(value);
             Py_DECREF(value);
@@ -1878,7 +1865,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(UNARY_NEGATIVE): {
+        TARGET(UNARY_NEGATIVE) {
             PyObject *value = TOP();
             PyObject *res = PyNumber_Negative(value);
             Py_DECREF(value);
@@ -1888,7 +1875,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(UNARY_NOT): {
+        TARGET(UNARY_NOT) {
             PyObject *value = TOP();
             int err = PyObject_IsTrue(value);
             Py_DECREF(value);
@@ -1906,7 +1893,7 @@ check_eval_breaker:
             goto error;
         }
 
-        TARGET(UNARY_INVERT): {
+        TARGET(UNARY_INVERT) {
             PyObject *value = TOP();
             PyObject *res = PyNumber_Invert(value);
             Py_DECREF(value);
@@ -1916,7 +1903,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_POWER): {
+        TARGET(BINARY_POWER) {
             PyObject *exp = POP();
             PyObject *base = TOP();
             PyObject *res = PyNumber_Power(base, exp, Py_None);
@@ -1928,7 +1915,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_MULTIPLY): {
+        TARGET(BINARY_MULTIPLY) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_Multiply(left, right);
@@ -1940,7 +1927,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_MATRIX_MULTIPLY): {
+        TARGET(BINARY_MATRIX_MULTIPLY) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_MatrixMultiply(left, right);
@@ -1952,7 +1939,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_TRUE_DIVIDE): {
+        TARGET(BINARY_TRUE_DIVIDE) {
             PyObject *divisor = POP();
             PyObject *dividend = TOP();
             PyObject *quotient = PyNumber_TrueDivide(dividend, divisor);
@@ -1964,7 +1951,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_FLOOR_DIVIDE): {
+        TARGET(BINARY_FLOOR_DIVIDE) {
             PyObject *divisor = POP();
             PyObject *dividend = TOP();
             PyObject *quotient = PyNumber_FloorDivide(dividend, divisor);
@@ -1976,7 +1963,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_MODULO): {
+        TARGET(BINARY_MODULO) {
             PyObject *divisor = POP();
             PyObject *dividend = TOP();
             PyObject *res;
@@ -1996,7 +1983,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_ADD): {
+        TARGET(BINARY_ADD) {
             PREDICTED(BINARY_ADD);
             STAT_INC(BINARY_ADD, unquickened);
             PyObject *right = POP();
@@ -2011,7 +1998,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_ADD_ADAPTIVE): {
+        TARGET(BINARY_ADD_ADAPTIVE) {
             if (oparg == 0) {
                 PyObject *left = SECOND();
                 PyObject *right = TOP();
@@ -2029,7 +2016,7 @@ check_eval_breaker:
             }
         }
 
-        TARGET(BINARY_ADD_UNICODE): {
+        TARGET(BINARY_ADD_UNICODE) {
             PyObject *left = SECOND();
             PyObject *right = TOP();
             DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_ADD);
@@ -2047,7 +2034,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_ADD_UNICODE_INPLACE_FAST): {
+        TARGET(BINARY_ADD_UNICODE_INPLACE_FAST) {
             PyObject *left = SECOND();
             PyObject *right = TOP();
             DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_ADD);
@@ -2076,7 +2063,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_ADD_FLOAT): {
+        TARGET(BINARY_ADD_FLOAT) {
             PyObject *left = SECOND();
             PyObject *right = TOP();
             DEOPT_IF(!PyFloat_CheckExact(left), BINARY_ADD);
@@ -2096,7 +2083,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_ADD_INT): {
+        TARGET(BINARY_ADD_INT) {
             PyObject *left = SECOND();
             PyObject *right = TOP();
             DEOPT_IF(!PyLong_CheckExact(left), BINARY_ADD);
@@ -2114,7 +2101,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_SUBTRACT): {
+        TARGET(BINARY_SUBTRACT) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *diff = PyNumber_Subtract(left, right);
@@ -2126,7 +2113,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_SUBSCR): {
+        TARGET(BINARY_SUBSCR) {
             PREDICTED(BINARY_SUBSCR);
             STAT_INC(BINARY_SUBSCR, unquickened);
             PyObject *sub = POP();
@@ -2140,7 +2127,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_SUBSCR_ADAPTIVE): {
+        TARGET(BINARY_SUBSCR_ADAPTIVE) {
             if (oparg == 0) {
                 PyObject *sub = TOP();
                 PyObject *container = SECOND();
@@ -2161,7 +2148,7 @@ check_eval_breaker:
             }
         }
 
-        TARGET(BINARY_SUBSCR_LIST_INT): {
+        TARGET(BINARY_SUBSCR_LIST_INT) {
             PyObject *sub = TOP();
             PyObject *list = SECOND();
             DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
@@ -2185,7 +2172,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_SUBSCR_TUPLE_INT): {
+        TARGET(BINARY_SUBSCR_TUPLE_INT) {
             PyObject *sub = TOP();
             PyObject *tuple = SECOND();
             DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
@@ -2209,7 +2196,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_SUBSCR_DICT): {
+        TARGET(BINARY_SUBSCR_DICT) {
             PyObject *dict = SECOND();
             DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR);
             STAT_INC(BINARY_SUBSCR, hit);
@@ -2226,7 +2213,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_LSHIFT): {
+        TARGET(BINARY_LSHIFT) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_Lshift(left, right);
@@ -2238,7 +2225,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_RSHIFT): {
+        TARGET(BINARY_RSHIFT) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_Rshift(left, right);
@@ -2250,7 +2237,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_AND): {
+        TARGET(BINARY_AND) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_And(left, right);
@@ -2262,7 +2249,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_XOR): {
+        TARGET(BINARY_XOR) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_Xor(left, right);
@@ -2274,7 +2261,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BINARY_OR): {
+        TARGET(BINARY_OR) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_Or(left, right);
@@ -2286,7 +2273,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LIST_APPEND): {
+        TARGET(LIST_APPEND) {
             PyObject *v = POP();
             PyObject *list = PEEK(oparg);
             int err;
@@ -2298,7 +2285,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(SET_ADD): {
+        TARGET(SET_ADD) {
             PyObject *v = POP();
             PyObject *set = PEEK(oparg);
             int err;
@@ -2310,7 +2297,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_POWER): {
+        TARGET(INPLACE_POWER) {
             PyObject *exp = POP();
             PyObject *base = TOP();
             PyObject *res = PyNumber_InPlacePower(base, exp, Py_None);
@@ -2322,7 +2309,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_MULTIPLY): {
+        TARGET(INPLACE_MULTIPLY) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_InPlaceMultiply(left, right);
@@ -2334,7 +2321,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_MATRIX_MULTIPLY): {
+        TARGET(INPLACE_MATRIX_MULTIPLY) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_InPlaceMatrixMultiply(left, right);
@@ -2346,7 +2333,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_TRUE_DIVIDE): {
+        TARGET(INPLACE_TRUE_DIVIDE) {
             PyObject *divisor = POP();
             PyObject *dividend = TOP();
             PyObject *quotient = PyNumber_InPlaceTrueDivide(dividend, divisor);
@@ -2358,7 +2345,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_FLOOR_DIVIDE): {
+        TARGET(INPLACE_FLOOR_DIVIDE) {
             PyObject *divisor = POP();
             PyObject *dividend = TOP();
             PyObject *quotient = PyNumber_InPlaceFloorDivide(dividend, divisor);
@@ -2370,7 +2357,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_MODULO): {
+        TARGET(INPLACE_MODULO) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *mod = PyNumber_InPlaceRemainder(left, right);
@@ -2382,7 +2369,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_ADD): {
+        TARGET(INPLACE_ADD) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *sum;
@@ -2401,7 +2388,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_SUBTRACT): {
+        TARGET(INPLACE_SUBTRACT) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *diff = PyNumber_InPlaceSubtract(left, right);
@@ -2413,7 +2400,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_LSHIFT): {
+        TARGET(INPLACE_LSHIFT) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_InPlaceLshift(left, right);
@@ -2425,7 +2412,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_RSHIFT): {
+        TARGET(INPLACE_RSHIFT) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_InPlaceRshift(left, right);
@@ -2437,7 +2424,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_AND): {
+        TARGET(INPLACE_AND) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_InPlaceAnd(left, right);
@@ -2449,7 +2436,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_XOR): {
+        TARGET(INPLACE_XOR) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_InPlaceXor(left, right);
@@ -2461,7 +2448,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(INPLACE_OR): {
+        TARGET(INPLACE_OR) {
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_InPlaceOr(left, right);
@@ -2473,7 +2460,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(STORE_SUBSCR): {
+        TARGET(STORE_SUBSCR) {
             PyObject *sub = TOP();
             PyObject *container = SECOND();
             PyObject *v = THIRD();
@@ -2489,7 +2476,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(DELETE_SUBSCR): {
+        TARGET(DELETE_SUBSCR) {
             PyObject *sub = TOP();
             PyObject *container = SECOND();
             int err;
@@ -2503,7 +2490,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(PRINT_EXPR): {
+        TARGET(PRINT_EXPR) {
             _Py_IDENTIFIER(displayhook);
             PyObject *value = POP();
             PyObject *hook = _PySys_GetObjectId(&PyId_displayhook);
@@ -2522,7 +2509,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(RAISE_VARARGS): {
+        TARGET(RAISE_VARARGS) {
             PyObject *cause = NULL, *exc = NULL;
             switch (oparg) {
             case 2:
@@ -2544,7 +2531,7 @@ check_eval_breaker:
             goto error;
         }
 
-        TARGET(RETURN_VALUE): {
+        TARGET(RETURN_VALUE) {
             retval = POP();
             assert(EMPTY());
             frame->f_state = FRAME_RETURNED;
@@ -2552,7 +2539,7 @@ check_eval_breaker:
             goto exiting;
         }
 
-        TARGET(GET_AITER): {
+        TARGET(GET_AITER) {
             unaryfunc getter = NULL;
             PyObject *iter = NULL;
             PyObject *obj = TOP();
@@ -2596,7 +2583,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(GET_ANEXT): {
+        TARGET(GET_ANEXT) {
             unaryfunc getter = NULL;
             PyObject *next_iter = NULL;
             PyObject *awaitable = NULL;
@@ -2647,7 +2634,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(GET_AWAITABLE): {
+        TARGET(GET_AWAITABLE) {
             PREDICTED(GET_AWAITABLE);
             PyObject *iterable = TOP();
             PyObject *iter = _PyCoro_GetAwaitableIter(iterable);
@@ -2688,7 +2675,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(YIELD_FROM): {
+        TARGET(YIELD_FROM) {
             PyObject *v = POP();
             PyObject *receiver = TOP();
             PySendResult gen_status;
@@ -2740,7 +2727,7 @@ check_eval_breaker:
             goto exiting;
         }
 
-        TARGET(YIELD_VALUE): {
+        TARGET(YIELD_VALUE) {
             retval = POP();
 
             if (co->co_flags & CO_ASYNC_GENERATOR) {
@@ -2757,7 +2744,7 @@ check_eval_breaker:
             goto exiting;
         }
 
-        TARGET(GEN_START): {
+        TARGET(GEN_START) {
             PyObject *none = POP();
             Py_DECREF(none);
             if (!Py_IsNone(none)) {
@@ -2781,7 +2768,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(POP_EXCEPT): {
+        TARGET(POP_EXCEPT) {
             PyObject *type, *value, *traceback;
             _PyErr_StackItem *exc_info;
             exc_info = tstate->exc_info;
@@ -2797,7 +2784,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(POP_EXCEPT_AND_RERAISE): {
+        TARGET(POP_EXCEPT_AND_RERAISE) {
             PyObject *lasti = PEEK(4);
             if (PyLong_Check(lasti)) {
                 frame->f_lasti = PyLong_AsLong(lasti);
@@ -2827,7 +2814,7 @@ check_eval_breaker:
             goto exception_unwind;
         }
 
-        TARGET(RERAISE): {
+        TARGET(RERAISE) {
             if (oparg) {
                 PyObject *lasti = PEEK(oparg+3);
                 if (PyLong_Check(lasti)) {
@@ -2848,7 +2835,7 @@ check_eval_breaker:
             goto exception_unwind;
         }
 
-        TARGET(END_ASYNC_FOR): {
+        TARGET(END_ASYNC_FOR) {
             PyObject *exc = POP();
             PyObject *val = POP();
             PyObject *tb = POP();
@@ -2866,14 +2853,14 @@ check_eval_breaker:
             }
         }
 
-        TARGET(LOAD_ASSERTION_ERROR): {
+        TARGET(LOAD_ASSERTION_ERROR) {
             PyObject *value = PyExc_AssertionError;
             Py_INCREF(value);
             PUSH(value);
             DISPATCH();
         }
 
-        TARGET(LOAD_BUILD_CLASS): {
+        TARGET(LOAD_BUILD_CLASS) {
             _Py_IDENTIFIER(__build_class__);
 
             PyObject *bc;
@@ -2904,7 +2891,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(STORE_NAME): {
+        TARGET(STORE_NAME) {
             PyObject *name = GETITEM(names, oparg);
             PyObject *v = POP();
             PyObject *ns = LOCALS();
@@ -2925,7 +2912,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(DELETE_NAME): {
+        TARGET(DELETE_NAME) {
             PyObject *name = GETITEM(names, oparg);
             PyObject *ns = LOCALS();
             int err;
@@ -2944,7 +2931,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(UNPACK_SEQUENCE): {
+        TARGET(UNPACK_SEQUENCE) {
             PREDICTED(UNPACK_SEQUENCE);
             PyObject *seq = POP(), *item, **items;
             if (PyTuple_CheckExact(seq) &&
@@ -2975,7 +2962,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(UNPACK_EX): {
+        TARGET(UNPACK_EX) {
             int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
             PyObject *seq = POP();
 
@@ -2990,7 +2977,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(STORE_ATTR): {
+        TARGET(STORE_ATTR) {
             PREDICTED(STORE_ATTR);
             STAT_INC(STORE_ATTR, unquickened);
             PyObject *name = GETITEM(names, oparg);
@@ -3006,7 +2993,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(DELETE_ATTR): {
+        TARGET(DELETE_ATTR) {
             PyObject *name = GETITEM(names, oparg);
             PyObject *owner = POP();
             int err;
@@ -3017,7 +3004,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(STORE_GLOBAL): {
+        TARGET(STORE_GLOBAL) {
             PyObject *name = GETITEM(names, oparg);
             PyObject *v = POP();
             int err;
@@ -3028,7 +3015,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(DELETE_GLOBAL): {
+        TARGET(DELETE_GLOBAL) {
             PyObject *name = GETITEM(names, oparg);
             int err;
             err = PyDict_DelItem(GLOBALS(), name);
@@ -3042,7 +3029,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_NAME): {
+        TARGET(LOAD_NAME) {
             PyObject *name = GETITEM(names, oparg);
             PyObject *locals = LOCALS();
             PyObject *v;
@@ -3106,7 +3093,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_GLOBAL): {
+        TARGET(LOAD_GLOBAL) {
             PREDICTED(LOAD_GLOBAL);
             STAT_INC(LOAD_GLOBAL, unquickened);
             PyObject *name = GETITEM(names, oparg);
@@ -3156,7 +3143,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_GLOBAL_ADAPTIVE): {
+        TARGET(LOAD_GLOBAL_ADAPTIVE) {
             assert(cframe.use_tracing == 0);
             SpecializedCacheEntry *cache = GET_CACHE();
             if (cache->adaptive.counter == 0) {
@@ -3176,7 +3163,7 @@ check_eval_breaker:
             }
         }
 
-        TARGET(LOAD_GLOBAL_MODULE): {
+        TARGET(LOAD_GLOBAL_MODULE) {
             assert(cframe.use_tracing == 0);
             DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
             PyDictObject *dict = (PyDictObject *)GLOBALS();
@@ -3194,7 +3181,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_GLOBAL_BUILTIN): {
+        TARGET(LOAD_GLOBAL_BUILTIN) {
             assert(cframe.use_tracing == 0);
             DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
             DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL);
@@ -3215,7 +3202,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(DELETE_FAST): {
+        TARGET(DELETE_FAST) {
             PyObject *v = GETLOCAL(oparg);
             if (v != NULL) {
                 SETLOCAL(oparg, NULL);
@@ -3229,7 +3216,7 @@ check_eval_breaker:
             goto error;
         }
 
-        TARGET(MAKE_CELL): {
+        TARGET(MAKE_CELL) {
             // "initial" is probably NULL but not if it's an arg (or set
             // via PyFrame_LocalsToFast() before MAKE_CELL has run).
             PyObject *initial = GETLOCAL(oparg);
@@ -3241,7 +3228,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(DELETE_DEREF): {
+        TARGET(DELETE_DEREF) {
             PyObject *cell = GETLOCAL(oparg);
             PyObject *oldobj = PyCell_GET(cell);
             if (oldobj != NULL) {
@@ -3253,7 +3240,7 @@ check_eval_breaker:
             goto error;
         }
 
-        TARGET(LOAD_CLASSDEREF): {
+        TARGET(LOAD_CLASSDEREF) {
             PyObject *name, *value, *locals = LOCALS();
             assert(locals);
             assert(oparg >= 0 && oparg < co->co_nlocalsplus);
@@ -3289,7 +3276,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_DEREF): {
+        TARGET(LOAD_DEREF) {
             PyObject *cell = GETLOCAL(oparg);
             PyObject *value = PyCell_GET(cell);
             if (value == NULL) {
@@ -3301,7 +3288,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(STORE_DEREF): {
+        TARGET(STORE_DEREF) {
             PyObject *v = POP();
             PyObject *cell = GETLOCAL(oparg);
             PyObject *oldobj = PyCell_GET(cell);
@@ -3310,7 +3297,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BUILD_STRING): {
+        TARGET(BUILD_STRING) {
             PyObject *str;
             PyObject *empty = PyUnicode_New(0, 0);
             if (empty == NULL) {
@@ -3328,7 +3315,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BUILD_TUPLE): {
+        TARGET(BUILD_TUPLE) {
             PyObject *tup = PyTuple_New(oparg);
             if (tup == NULL)
                 goto error;
@@ -3340,7 +3327,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BUILD_LIST): {
+        TARGET(BUILD_LIST) {
             PyObject *list =  PyList_New(oparg);
             if (list == NULL)
                 goto error;
@@ -3352,7 +3339,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LIST_TO_TUPLE): {
+        TARGET(LIST_TO_TUPLE) {
             PyObject *list = POP();
             PyObject *tuple = PyList_AsTuple(list);
             Py_DECREF(list);
@@ -3363,7 +3350,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LIST_EXTEND): {
+        TARGET(LIST_EXTEND) {
             PyObject *iterable = POP();
             PyObject *list = PEEK(oparg);
             PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
@@ -3384,7 +3371,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(SET_UPDATE): {
+        TARGET(SET_UPDATE) {
             PyObject *iterable = POP();
             PyObject *set = PEEK(oparg);
             int err = _PySet_Update(set, iterable);
@@ -3395,7 +3382,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BUILD_SET): {
+        TARGET(BUILD_SET) {
             PyObject *set = PySet_New(NULL);
             int err = 0;
             int i;
@@ -3416,7 +3403,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BUILD_MAP): {
+        TARGET(BUILD_MAP) {
             Py_ssize_t i;
             PyObject *map = _PyDict_NewPresized((Py_ssize_t)oparg);
             if (map == NULL)
@@ -3440,7 +3427,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(SETUP_ANNOTATIONS): {
+        TARGET(SETUP_ANNOTATIONS) {
             _Py_IDENTIFIER(__annotations__);
             int err;
             PyObject *ann_dict;
@@ -3499,7 +3486,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BUILD_CONST_KEY_MAP): {
+        TARGET(BUILD_CONST_KEY_MAP) {
             Py_ssize_t i;
             PyObject *map;
             PyObject *keys = TOP();
@@ -3532,7 +3519,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(DICT_UPDATE): {
+        TARGET(DICT_UPDATE) {
             PyObject *update = POP();
             PyObject *dict = PEEK(oparg);
             if (PyDict_Update(dict, update) < 0) {
@@ -3548,7 +3535,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(DICT_MERGE): {
+        TARGET(DICT_MERGE) {
             PyObject *update = POP();
             PyObject *dict = PEEK(oparg);
 
@@ -3562,7 +3549,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(MAP_ADD): {
+        TARGET(MAP_ADD) {
             PyObject *value = TOP();
             PyObject *key = SECOND();
             PyObject *map;
@@ -3579,7 +3566,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_ATTR): {
+        TARGET(LOAD_ATTR) {
             PREDICTED(LOAD_ATTR);
             STAT_INC(LOAD_ATTR, unquickened);
             PyObject *name = GETITEM(names, oparg);
@@ -3593,7 +3580,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_ATTR_ADAPTIVE): {
+        TARGET(LOAD_ATTR_ADAPTIVE) {
             assert(cframe.use_tracing == 0);
             SpecializedCacheEntry *cache = GET_CACHE();
             if (cache->adaptive.counter == 0) {
@@ -3614,7 +3601,7 @@ check_eval_breaker:
             }
         }
 
-        TARGET(LOAD_ATTR_SPLIT_KEYS): {
+        TARGET(LOAD_ATTR_SPLIT_KEYS) {
             assert(cframe.use_tracing == 0);
             PyObject *owner = TOP();
             PyObject *res;
@@ -3639,7 +3626,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_ATTR_MODULE): {
+        TARGET(LOAD_ATTR_MODULE) {
             assert(cframe.use_tracing == 0);
             // shared with LOAD_METHOD_MODULE
             PyObject *owner = TOP();
@@ -3650,7 +3637,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_ATTR_WITH_HINT): {
+        TARGET(LOAD_ATTR_WITH_HINT) {
             assert(cframe.use_tracing == 0);
             PyObject *owner = TOP();
             PyObject *res;
@@ -3679,7 +3666,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_ATTR_SLOT): {
+        TARGET(LOAD_ATTR_SLOT) {
             assert(cframe.use_tracing == 0);
             PyObject *owner = TOP();
             PyObject *res;
@@ -3700,7 +3687,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(STORE_ATTR_ADAPTIVE): {
+        TARGET(STORE_ATTR_ADAPTIVE) {
             assert(cframe.use_tracing == 0);
             SpecializedCacheEntry *cache = GET_CACHE();
             if (cache->adaptive.counter == 0) {
@@ -3721,7 +3708,7 @@ check_eval_breaker:
             }
         }
 
-        TARGET(STORE_ATTR_SPLIT_KEYS): {
+        TARGET(STORE_ATTR_SPLIT_KEYS) {
             assert(cframe.use_tracing == 0);
             PyObject *owner = TOP();
             PyTypeObject *tp = Py_TYPE(owner);
@@ -3759,7 +3746,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(STORE_ATTR_WITH_HINT): {
+        TARGET(STORE_ATTR_WITH_HINT) {
             assert(cframe.use_tracing == 0);
             PyObject *owner = TOP();
             PyTypeObject *tp = Py_TYPE(owner);
@@ -3795,7 +3782,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(STORE_ATTR_SLOT): {
+        TARGET(STORE_ATTR_SLOT) {
             assert(cframe.use_tracing == 0);
             PyObject *owner = TOP();
             PyTypeObject *tp = Py_TYPE(owner);
@@ -3816,7 +3803,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(COMPARE_OP): {
+        TARGET(COMPARE_OP) {
             assert(oparg <= Py_GE);
             PyObject *right = POP();
             PyObject *left = TOP();
@@ -3831,7 +3818,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(IS_OP): {
+        TARGET(IS_OP) {
             PyObject *right = POP();
             PyObject *left = TOP();
             int res = Py_Is(left, right) ^ oparg;
@@ -3845,7 +3832,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(CONTAINS_OP): {
+        TARGET(CONTAINS_OP) {
             PyObject *right = POP();
             PyObject *left = POP();
             int res = PySequence_Contains(right, left);
@@ -3865,7 +3852,7 @@ check_eval_breaker:
 #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\
                          "BaseException is not allowed"
 
-        TARGET(JUMP_IF_NOT_EXC_MATCH): {
+        TARGET(JUMP_IF_NOT_EXC_MATCH) {
             PyObject *right = POP();
             PyObject *left = POP();
             if (PyTuple_Check(right)) {
@@ -3906,7 +3893,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(IMPORT_NAME): {
+        TARGET(IMPORT_NAME) {
             PyObject *name = GETITEM(names, oparg);
             PyObject *fromlist = POP();
             PyObject *level = TOP();
@@ -3920,7 +3907,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(IMPORT_STAR): {
+        TARGET(IMPORT_STAR) {
             PyObject *from = POP(), *locals;
             int err;
             if (_PyFrame_FastToLocalsWithError(frame) < 0) {
@@ -3943,7 +3930,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(IMPORT_FROM): {
+        TARGET(IMPORT_FROM) {
             PyObject *name = GETITEM(names, oparg);
             PyObject *from = TOP();
             PyObject *res;
@@ -3954,12 +3941,12 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(JUMP_FORWARD): {
+        TARGET(JUMP_FORWARD) {
             JUMPBY(oparg);
             DISPATCH();
         }
 
-        TARGET(POP_JUMP_IF_FALSE): {
+        TARGET(POP_JUMP_IF_FALSE) {
             PREDICTED(POP_JUMP_IF_FALSE);
             PyObject *cond = POP();
             int err;
@@ -3986,7 +3973,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(POP_JUMP_IF_TRUE): {
+        TARGET(POP_JUMP_IF_TRUE) {
             PREDICTED(POP_JUMP_IF_TRUE);
             PyObject *cond = POP();
             int err;
@@ -4013,7 +4000,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(JUMP_IF_FALSE_OR_POP): {
+        TARGET(JUMP_IF_FALSE_OR_POP) {
             PyObject *cond = TOP();
             int err;
             if (Py_IsTrue(cond)) {
@@ -4037,7 +4024,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(JUMP_IF_TRUE_OR_POP): {
+        TARGET(JUMP_IF_TRUE_OR_POP) {
             PyObject *cond = TOP();
             int err;
             if (Py_IsFalse(cond)) {
@@ -4062,7 +4049,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(JUMP_ABSOLUTE): {
+        TARGET(JUMP_ABSOLUTE) {
             PREDICTED(JUMP_ABSOLUTE);
             if (oparg < INSTR_OFFSET()) {
                 /* Increment the warmup counter and quicken if warm enough
@@ -4084,13 +4071,13 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(JUMP_ABSOLUTE_QUICK): {
+        TARGET(JUMP_ABSOLUTE_QUICK) {
             JUMPTO(oparg);
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
-        TARGET(GET_LEN): {
+        TARGET(GET_LEN) {
             // PUSH(len(TOS))
             Py_ssize_t len_i = PyObject_Length(TOP());
             if (len_i < 0) {
@@ -4104,7 +4091,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(MATCH_CLASS): {
+        TARGET(MATCH_CLASS) {
             // Pop TOS. On success, set TOS to True and TOS1 to a tuple of
             // attributes. On failure, set TOS to False.
             PyObject *names = POP();
@@ -4127,7 +4114,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(MATCH_MAPPING): {
+        TARGET(MATCH_MAPPING) {
             PyObject *subject = TOP();
             int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
             PyObject *res = match ? Py_True : Py_False;
@@ -4136,7 +4123,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(MATCH_SEQUENCE): {
+        TARGET(MATCH_SEQUENCE) {
             PyObject *subject = TOP();
             int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
             PyObject *res = match ? Py_True : Py_False;
@@ -4145,7 +4132,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(MATCH_KEYS): {
+        TARGET(MATCH_KEYS) {
             // On successful match for all keys, PUSH(values) and PUSH(True).
             // Otherwise, PUSH(None) and PUSH(False).
             PyObject *keys = TOP();
@@ -4166,7 +4153,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(COPY_DICT_WITHOUT_KEYS): {
+        TARGET(COPY_DICT_WITHOUT_KEYS) {
             // rest = dict(TOS1)
             // for key in TOS:
             //     del rest[key]
@@ -4192,7 +4179,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(GET_ITER): {
+        TARGET(GET_ITER) {
             /* before: [obj]; after [getiter(obj)] */
             PyObject *iterable = TOP();
             PyObject *iter = PyObject_GetIter(iterable);
@@ -4205,7 +4192,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(GET_YIELD_FROM_ITER): {
+        TARGET(GET_YIELD_FROM_ITER) {
             /* before: [obj]; after [getiter(obj)] */
             PyObject *iterable = TOP();
             PyObject *iter;
@@ -4234,7 +4221,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(FOR_ITER): {
+        TARGET(FOR_ITER) {
             PREDICTED(FOR_ITER);
             /* before: [iter]; after: [iter, iter()] *or* [] */
             PyObject *iter = TOP();
@@ -4261,7 +4248,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BEFORE_ASYNC_WITH): {
+        TARGET(BEFORE_ASYNC_WITH) {
             _Py_IDENTIFIER(__aenter__);
             _Py_IDENTIFIER(__aexit__);
             PyObject *mgr = TOP();
@@ -4299,7 +4286,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BEFORE_WITH): {
+        TARGET(BEFORE_WITH) {
             _Py_IDENTIFIER(__enter__);
             _Py_IDENTIFIER(__exit__);
             PyObject *mgr = TOP();
@@ -4337,7 +4324,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(WITH_EXCEPT_START): {
+        TARGET(WITH_EXCEPT_START) {
             /* At the top of the stack are 8 values:
                - (TOP, SECOND, THIRD) = exc_info()
                - (FOURTH, FIFTH, SIXTH) = previous exception
@@ -4367,7 +4354,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(PUSH_EXC_INFO): {
+        TARGET(PUSH_EXC_INFO) {
             PyObject *type = TOP();
             PyObject *value = SECOND();
             PyObject *tb = THIRD();
@@ -4398,7 +4385,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_METHOD): {
+        TARGET(LOAD_METHOD) {
             PREDICTED(LOAD_METHOD);
             STAT_INC(LOAD_METHOD, unquickened);
             /* Designed to work in tandem with CALL_METHOD. */
@@ -4437,7 +4424,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_METHOD_ADAPTIVE): {
+        TARGET(LOAD_METHOD_ADAPTIVE) {
             assert(cframe.use_tracing == 0);
             SpecializedCacheEntry *cache = GET_CACHE();
             if (cache->adaptive.counter == 0) {
@@ -4458,7 +4445,7 @@ check_eval_breaker:
             }
         }
 
-        TARGET(LOAD_METHOD_CACHED): {
+        TARGET(LOAD_METHOD_CACHED) {
             /* LOAD_METHOD, with cached method object */
             assert(cframe.use_tracing == 0);
             PyObject *self = TOP();
@@ -4495,7 +4482,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_METHOD_MODULE): {
+        TARGET(LOAD_METHOD_MODULE) {
             /* LOAD_METHOD, for module methods */
             assert(cframe.use_tracing == 0);
             PyObject *owner = TOP();
@@ -4507,7 +4494,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(LOAD_METHOD_CLASS): {
+        TARGET(LOAD_METHOD_CLASS) {
             /* LOAD_METHOD, for class methods */
             assert(cframe.use_tracing == 0);
             SpecializedCacheEntry *caches = GET_CACHE();
@@ -4533,7 +4520,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(CALL_METHOD): {
+        TARGET(CALL_METHOD) {
             /* Designed to work in tamdem with LOAD_METHOD. */
             PyObject **sp, *res;
             int meth_found;
@@ -4577,7 +4564,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(CALL_METHOD_KW): {
+        TARGET(CALL_METHOD_KW) {
             /* Designed to work in tandem with LOAD_METHOD. Same as CALL_METHOD
             but pops TOS to get a tuple of keyword names. */
             PyObject **sp, *res;
@@ -4601,7 +4588,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(CALL_FUNCTION): {
+        TARGET(CALL_FUNCTION) {
             PREDICTED(CALL_FUNCTION);
             PyObject **sp, *res;
             sp = stack_pointer;
@@ -4615,7 +4602,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(CALL_FUNCTION_KW): {
+        TARGET(CALL_FUNCTION_KW) {
             PyObject **sp, *res, *names;
 
             names = POP();
@@ -4635,7 +4622,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(CALL_FUNCTION_EX): {
+        TARGET(CALL_FUNCTION_EX) {
             PREDICTED(CALL_FUNCTION_EX);
             PyObject *func, *callargs, *kwargs = NULL, *result;
             if (oparg & 0x01) {
@@ -4682,7 +4669,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(MAKE_FUNCTION): {
+        TARGET(MAKE_FUNCTION) {
             PyObject *codeobj = POP();
             PyFunctionObject *func = (PyFunctionObject *)
                 PyFunction_New(codeobj, GLOBALS());
@@ -4713,7 +4700,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(BUILD_SLICE): {
+        TARGET(BUILD_SLICE) {
             PyObject *start, *stop, *step, *slice;
             if (oparg == 3)
                 step = POP();
@@ -4731,7 +4718,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(FORMAT_VALUE): {
+        TARGET(FORMAT_VALUE) {
             /* Handles f-string value formatting. */
             PyObject *result;
             PyObject *fmt_spec;
@@ -4791,7 +4778,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(ROT_N): {
+        TARGET(ROT_N) {
             PyObject *top = TOP();
             memmove(&PEEK(oparg - 1), &PEEK(oparg),
                     sizeof(PyObject*) * (oparg - 1));
@@ -4799,7 +4786,7 @@ check_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(EXTENDED_ARG): {
+        TARGET(EXTENDED_ARG) {
             int oldoparg = oparg;
             NEXTOPARG();
             oparg |= oldoparg << 8;
@@ -4807,6 +4794,45 @@ check_eval_breaker:
             DISPATCH_GOTO();
         }
 
+#if USE_COMPUTED_GOTOS
+        TARGET_DO_TRACING: {
+#else
+        case DO_TRACING: {
+#endif
+            int instr_prev = skip_backwards_over_extended_args(co, frame->f_lasti);
+            frame->f_lasti = INSTR_OFFSET();
+            TRACING_NEXTOPARG();
+            if (PyDTrace_LINE_ENABLED()) {
+                maybe_dtrace_line(frame, &tstate->trace_info, instr_prev);
+            }
+            /* line-by-line tracing support */
+
+            if (cframe.use_tracing &&
+                tstate->c_tracefunc != NULL && !tstate->tracing) {
+                int err;
+                /* see maybe_call_line_trace()
+                for expository comments */
+                _PyFrame_SetStackPointer(frame, stack_pointer);
+
+                err = maybe_call_line_trace(tstate->c_tracefunc,
+                                            tstate->c_traceobj,
+                                            tstate, frame, instr_prev);
+                if (err) {
+                    /* trace function raised an exception */
+                    next_instr++;
+                    goto error;
+                }
+                /* Reload possibly changed frame fields */
+                JUMPTO(frame->f_lasti);
+
+                stack_pointer = _PyFrame_GetStackPointer(frame);
+                frame->stacktop = -1;
+                TRACING_NEXTOPARG();
+            }
+            PRE_DISPATCH_GOTO();
+            DISPATCH_GOTO();
+        }
+
 
 #if USE_COMPUTED_GOTOS
         _unknown_opcode:
@@ -6001,7 +6027,7 @@ call_trace(Py_tracefunc func, PyObject *obj,
     result = func(obj, f, what, arg);
     f->f_lineno = 0;
     tstate->cframe->use_tracing = ((tstate->c_tracefunc != NULL)
-                           || (tstate->c_profilefunc != NULL));
+                           || (tstate->c_profilefunc != NULL)) ? 255 : 0;
     tstate->tracing--;
     return result;
 }
@@ -6016,7 +6042,7 @@ _PyEval_CallTracing(PyObject *func, PyObject *args)
 
     tstate->tracing = 0;
     tstate->cframe->use_tracing = ((tstate->c_tracefunc != NULL)
-                           || (tstate->c_profilefunc != NULL));
+                           || (tstate->c_profilefunc != NULL)) ? 255 : 0;
     result = PyObject_Call(func, args, NULL);
     tstate->tracing = save_tracing;
     tstate->cframe->use_tracing = save_use_tracing;
@@ -6081,7 +6107,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
     tstate->c_profilefunc = func;
 
     /* Flag that tracing or profiling is turned on */
-    tstate->cframe->use_tracing = (func != NULL) || (tstate->c_tracefunc != NULL);
+    tstate->cframe->use_tracing = (func != NULL) || (tstate->c_tracefunc != NULL) ? 255 : 0;
     return 0;
 }
 
@@ -6114,7 +6140,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
     tstate->c_tracefunc = NULL;
     tstate->c_traceobj = NULL;
     /* Must make sure that profiling is not ignored if 'traceobj' is freed */
-    tstate->cframe->use_tracing = (tstate->c_profilefunc != NULL);
+    tstate->cframe->use_tracing = (tstate->c_profilefunc != NULL) ? 255 : 0;
     Py_XDECREF(traceobj);
 
     Py_XINCREF(arg);
@@ -6123,7 +6149,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
 
     /* Flag that tracing or profiling is turned on */
     tstate->cframe->use_tracing = ((func != NULL)
-                           || (tstate->c_profilefunc != NULL));
+                           || (tstate->c_profilefunc != NULL)) ? 255 : 0;
 
     return 0;
 }
@@ -6871,6 +6897,7 @@ unicode_concatenate(PyThreadState *tstate, PyObject *v, PyObject *w,
          */
         int opcode, oparg;
         NEXTOPARG();
+        next_instr++;
         switch (opcode) {
         case STORE_FAST:
         {
diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py
index 189d72a..3bf2e35 100755
--- a/Python/makeopcodetargets.py
+++ b/Python/makeopcodetargets.py
@@ -32,6 +32,7 @@ def write_contents(f):
     """
     opcode = find_module('opcode')
     targets = ['_unknown_opcode'] * 256
+    targets[255] = "TARGET_DO_TRACING"
     for opname, op in opcode.opmap.items():
         targets[op] = "TARGET_%s" % opname
     next_op = 1
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index f3bfae5..30df683 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -254,5 +254,5 @@ static void *opcode_targets[256] = {
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,
-    &&_unknown_opcode
+    &&TARGET_DO_TRACING
 };
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 6e7e45b..f2cd3a9 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -268,7 +268,7 @@ sys_audit_tstate(PyThreadState *ts, const char *event,
                 break;
             }
             if (canTrace) {
-                ts->cframe->use_tracing = (ts->c_tracefunc || ts->c_profilefunc);
+                ts->cframe->use_tracing = (ts->c_tracefunc || ts->c_profilefunc) ? 255 : 0;
                 ts->tracing--;
             }
             PyObject* args[2] = {eventName, eventArgs};
@@ -283,7 +283,7 @@ sys_audit_tstate(PyThreadState *ts, const char *event,
             Py_DECREF(o);
             Py_CLEAR(hook);
         }
-        ts->cframe->use_tracing = (ts->c_tracefunc || ts->c_profilefunc);
+        ts->cframe->use_tracing = (ts->c_tracefunc || ts->c_profilefunc) ? 255 : 0;
         ts->tracing--;
         if (_PyErr_Occurred(ts)) {
             goto exit;
diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py
index 48875d2..dca9bbc 100644
--- a/Tools/scripts/generate_opcode_h.py
+++ b/Tools/scripts/generate_opcode_h.py
@@ -72,7 +72,7 @@ def main(opcode_py, outfile='Include/opcode.h'):
                 next_op += 1
             fobj.write("#define %-23s %3s\n" % (name, next_op))
             used[next_op] = True
-
+        fobj.write("#define DO_TRACING              255\n")
         fobj.write("#ifdef NEED_OPCODE_JUMP_TABLES\n")
         write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj)
         write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj)