bpo-46841: Quicken code in-place (GH-31888)

* Moves the bytecode to the end of the corresponding PyCodeObject, and quickens it in-place.

* Removes the almost-always-unused co_varnames, co_freevars, and co_cellvars member caches

* _PyOpcode_Deopt is a new mapping from all opcodes to their un-quickened forms.

* _PyOpcode_InlineCacheEntries is renamed to _PyOpcode_Caches

* _Py_IncrementCountAndMaybeQuicken is renamed to _PyCode_Warmup

* _Py_Quicken is renamed to _PyCode_Quicken

* _co_quickened is renamed to _co_code_adaptive (and is now a read-only memoryview).

* Do not emit unused nonzero opargs anymore in the compiler.

3 files changed, 464 insertions, 300 deletions

diff --git a/Include/cpython/code.h b/Include/cpython/code.h
index ab827c5..1576783 100644
--- a/Include/cpython/code.h
+++ b/Include/cpython/code.h
@@ -26,91 +26,80 @@ typedef uint16_t _Py_CODEUNIT;
 // Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing:
 #define _Py_SET_OPCODE(word, opcode) (((unsigned char *)&(word))[0] = (opcode))
 
+// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
+// defined in this macro:
+#define _PyCode_DEF(SIZE) {                                                    \
+    PyObject_VAR_HEAD                                                          \
+                                                                               \
+    /* Note only the following fields are used in hash and/or comparisons      \
+     *                                                                         \
+     * - co_name                                                               \
+     * - co_argcount                                                           \
+     * - co_posonlyargcount                                                    \
+     * - co_kwonlyargcount                                                     \
+     * - co_nlocals                                                            \
+     * - co_stacksize                                                          \
+     * - co_flags                                                              \
+     * - co_firstlineno                                                        \
+     * - co_consts                                                             \
+     * - co_names                                                              \
+     * - co_localsplusnames                                                    \
+     * This is done to preserve the name and line number for tracebacks        \
+     * and debuggers; otherwise, constant de-duplication would collapse        \
+     * identical functions/lambdas defined on different lines.                 \
+     */                                                                        \
+                                                                               \
+    /* These fields are set with provided values on new code objects. */       \
+                                                                               \
+    /* The hottest fields (in the eval loop) are grouped here at the top. */   \
+    PyObject *co_consts;           /* list (constants used) */                 \
+    PyObject *co_names;            /* list of strings (names used) */          \
+    PyObject *co_exceptiontable;   /* Byte string encoding exception handling  \
+                                      table */                                 \
+    int co_flags;                  /* CO_..., see below */                     \
+    int co_warmup;                 /* Warmup counter for quickening */         \
+                                                                               \
+    /* The rest are not so impactful on performance. */                        \
+    int co_argcount;              /* #arguments, except *args */               \
+    int co_posonlyargcount;       /* #positional only arguments */             \
+    int co_kwonlyargcount;        /* #keyword only arguments */                \
+    int co_stacksize;             /* #entries needed for evaluation stack */   \
+    int co_firstlineno;           /* first source line number */               \
+                                                                               \
+    /* redundant values (derived from co_localsplusnames and                   \
+       co_localspluskinds) */                                                  \
+    int co_nlocalsplus;           /* number of local + cell + free variables   \
+                                  */                                           \
+    int co_nlocals;               /* number of local variables */              \
+    int co_nplaincellvars;        /* number of non-arg cell variables */       \
+    int co_ncellvars;             /* total number of cell variables */         \
+    int co_nfreevars;             /* number of free variables */               \
+                                                                               \
+    PyObject *co_localsplusnames; /* tuple mapping offsets to names */         \
+    PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte    \
+                                     per variable) */                          \
+    PyObject *co_filename;        /* unicode (where it was loaded from) */     \
+    PyObject *co_name;            /* unicode (name, for reference) */          \
+    PyObject *co_qualname;        /* unicode (qualname, for reference) */      \
+    PyObject *co_linetable;       /* bytes (encoding addr<->lineno mapping)    \
+                                     See Objects/lnotab_notes.txt for details. \
+                                  */                                           \
+    PyObject *co_endlinetable;    /* bytes object that holds end lineno for    \
+                                     instructions separated across different   \
+                                     lines */                                  \
+    PyObject *co_columntable;     /* bytes object that holds start/end column  \
+                                     offset each instruction */                \
+                                                                               \
+    PyObject *co_weakreflist;     /* to support weakrefs to code objects */    \
+    /* Scratch space for extra data relating to the code object.               \
+       Type is a void* to keep the format private in codeobject.c to force     \
+       people to go through the proper APIs. */                                \
+    void *co_extra;                                                            \
+    char co_code_adaptive[(SIZE)];                                             \
+}
 
 /* Bytecode object */
-struct PyCodeObject {
-    PyObject_HEAD
-
-    /* Note only the following fields are used in hash and/or comparisons
-     *
-     * - co_name
-     * - co_argcount
-     * - co_posonlyargcount
-     * - co_kwonlyargcount
-     * - co_nlocals
-     * - co_stacksize
-     * - co_flags
-     * - co_firstlineno
-     * - co_code
-     * - co_consts
-     * - co_names
-     * - co_varnames
-     * - co_freevars
-     * - co_cellvars
-     *
-     * This is done to preserve the name and line number for tracebacks
-     * and debuggers; otherwise, constant de-duplication would collapse
-     * identical functions/lambdas defined on different lines.
-     */
-
-    /* These fields are set with provided values on new code objects. */
-
-    // The hottest fields (in the eval loop) are grouped here at the top.
-    PyObject *co_consts;        /* list (constants used) */
-    PyObject *co_names;         /* list of strings (names used) */
-    _Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening.
-                                    Unlike the other "hot" fields, this one is
-                                    actually derived from co_code. */
-    PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
-    int co_flags;               /* CO_..., see below */
-    int co_warmup;              /* Warmup counter for quickening */
-
-    // The rest are not so impactful on performance.
-    int co_argcount;            /* #arguments, except *args */
-    int co_posonlyargcount;     /* #positional only arguments */
-    int co_kwonlyargcount;      /* #keyword only arguments */
-    int co_stacksize;           /* #entries needed for evaluation stack */
-    int co_firstlineno;         /* first source line number */
-    PyObject *co_code;          /* instruction opcodes */
-    PyObject *co_localsplusnames;  /* tuple mapping offsets to names */
-    PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */
-    PyObject *co_filename;      /* unicode (where it was loaded from) */
-    PyObject *co_name;          /* unicode (name, for reference) */
-    PyObject *co_qualname;      /* unicode (qualname, for reference) */
-    PyObject *co_linetable;     /* bytes (encoding addr<->lineno mapping) See
-                                   Objects/lnotab_notes.txt for details. */
-    PyObject *co_endlinetable;  /* bytes object that holds end lineno for
-                                   instructions separated across different
-                                   lines */
-    PyObject *co_columntable;   /* bytes object that holds start/end column
-                                   offset each instruction */
-
-    /* These fields are set with computed values on new code objects. */
-
-    // redundant values (derived from co_localsplusnames and co_localspluskinds)
-    int co_nlocalsplus;         /* number of local + cell + free variables */
-    int co_nlocals;             /* number of local variables */
-    int co_nplaincellvars;      /* number of non-arg cell variables */
-    int co_ncellvars;           /* total number of cell variables */
-    int co_nfreevars;           /* number of free variables */
-    // lazily-computed values
-    PyObject *co_varnames;      /* tuple of strings (local variable names) */
-    PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
-    PyObject *co_freevars;      /* tuple of strings (free variable names) */
-
-    /* The remaining fields are zeroed out on new code objects. */
-
-    PyObject *co_weakreflist;   /* to support weakrefs to code objects */
-    /* Scratch space for extra data relating to the code object.
-       Type is a void* to keep the format private in codeobject.c to force
-       people to go through the proper APIs. */
-    void *co_extra;
-    /* Quickened instructions and cache, or NULL
-     This should be treated as opaque by all code except the specializer and
-     interpreter. */
-    _Py_CODEUNIT *co_quickened;
-
-};
+struct PyCodeObject _PyCode_DEF(1);
 
 /* Masks for co_flags above */
 #define CO_OPTIMIZED    0x0001
@@ -151,6 +140,8 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;
 
 #define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type)
 #define PyCode_GetNumFree(op) ((op)->co_nfreevars)
+#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive)
+#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))
 
 /* Public interface */
 PyAPI_FUNC(PyCodeObject *) PyCode_New(
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 0d324e9..82dc9e4 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -92,30 +92,22 @@ typedef struct {
 
 #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
 
-/* Maximum size of code to quicken, in code units. */
-#define MAX_SIZE_TO_QUICKEN 10000
-
 #define QUICKENING_WARMUP_DELAY 8
 
 /* We want to compare to zero for efficiency, so we offset values accordingly */
 #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
-#define QUICKENING_WARMUP_COLDEST 1
 
-int _Py_Quicken(PyCodeObject *code);
+void _PyCode_Quicken(PyCodeObject *code);
 
-/* Returns 1 if quickening occurs.
- * -1 if an error occurs
- * 0 otherwise */
-static inline int
-_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code)
+static inline void
+_PyCode_Warmup(PyCodeObject *code)
 {
     if (code->co_warmup != 0) {
         code->co_warmup++;
         if (code->co_warmup == 0) {
-            return _Py_Quicken(code) ? -1 : 1;
+            _PyCode_Quicken(code);
         }
     }
-    return 0;
 }
 
 extern Py_ssize_t _Py_QuickenedCount;
@@ -225,6 +217,7 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
 extern PyObject* _PyCode_GetVarnames(PyCodeObject *);
 extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
 extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
+extern PyObject* _PyCode_GetCode(PyCodeObject *);
 
 /* Return the ending source code line number from a bytecode index. */
 extern int _PyCode_Addr2EndLine(PyCodeObject *, int);
diff --git a/Include/opcode.h b/Include/opcode.h
index 7bf0ba7..dfc7b72 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -7,185 +7,187 @@ extern "C" {
 
 
 /* Instruction opcodes for compiled code */
-#define CACHE                             0
-#define POP_TOP                           1
-#define PUSH_NULL                         2
-#define NOP                               9
-#define UNARY_POSITIVE                   10
-#define UNARY_NEGATIVE                   11
-#define UNARY_NOT                        12
-#define UNARY_INVERT                     15
-#define BINARY_SUBSCR                    25
-#define GET_LEN                          30
-#define MATCH_MAPPING                    31
-#define MATCH_SEQUENCE                   32
-#define MATCH_KEYS                       33
-#define PUSH_EXC_INFO                    35
-#define WITH_EXCEPT_START                49
-#define GET_AITER                        50
-#define GET_ANEXT                        51
-#define BEFORE_ASYNC_WITH                52
-#define BEFORE_WITH                      53
-#define END_ASYNC_FOR                    54
-#define STORE_SUBSCR                     60
-#define DELETE_SUBSCR                    61
-#define GET_ITER                         68
-#define GET_YIELD_FROM_ITER              69
-#define PRINT_EXPR                       70
-#define LOAD_BUILD_CLASS                 71
-#define LOAD_ASSERTION_ERROR             74
-#define RETURN_GENERATOR                 75
-#define LIST_TO_TUPLE                    82
-#define RETURN_VALUE                     83
-#define IMPORT_STAR                      84
-#define SETUP_ANNOTATIONS                85
-#define YIELD_VALUE                      86
-#define ASYNC_GEN_WRAP                   87
-#define PREP_RERAISE_STAR                88
-#define POP_EXCEPT                       89
-#define HAVE_ARGUMENT                    90
-#define STORE_NAME                       90
-#define DELETE_NAME                      91
-#define UNPACK_SEQUENCE                  92
-#define FOR_ITER                         93
-#define UNPACK_EX                        94
-#define STORE_ATTR                       95
-#define DELETE_ATTR                      96
-#define STORE_GLOBAL                     97
-#define DELETE_GLOBAL                    98
-#define SWAP                             99
-#define LOAD_CONST                      100
-#define LOAD_NAME                       101
-#define BUILD_TUPLE                     102
-#define BUILD_LIST                      103
-#define BUILD_SET                       104
-#define BUILD_MAP                       105
-#define LOAD_ATTR                       106
-#define COMPARE_OP                      107
-#define IMPORT_NAME                     108
-#define IMPORT_FROM                     109
-#define JUMP_FORWARD                    110
-#define JUMP_IF_FALSE_OR_POP            111
-#define JUMP_IF_TRUE_OR_POP             112
-#define JUMP_ABSOLUTE                   113
-#define POP_JUMP_IF_FALSE               114
-#define POP_JUMP_IF_TRUE                115
-#define LOAD_GLOBAL                     116
-#define IS_OP                           117
-#define CONTAINS_OP                     118
-#define RERAISE                         119
-#define COPY                            120
-#define JUMP_IF_NOT_EXC_MATCH           121
-#define BINARY_OP                       122
-#define SEND                            123
-#define LOAD_FAST                       124
-#define STORE_FAST                      125
-#define DELETE_FAST                     126
-#define JUMP_IF_NOT_EG_MATCH            127
-#define POP_JUMP_IF_NOT_NONE            128
-#define POP_JUMP_IF_NONE                129
-#define RAISE_VARARGS                   130
-#define GET_AWAITABLE                   131
-#define MAKE_FUNCTION                   132
-#define BUILD_SLICE                     133
-#define JUMP_NO_INTERRUPT               134
-#define MAKE_CELL                       135
-#define LOAD_CLOSURE                    136
-#define LOAD_DEREF                      137
-#define STORE_DEREF                     138
-#define DELETE_DEREF                    139
-#define CALL_FUNCTION_EX                142
-#define EXTENDED_ARG                    144
-#define LIST_APPEND                     145
-#define SET_ADD                         146
-#define MAP_ADD                         147
-#define LOAD_CLASSDEREF                 148
-#define COPY_FREE_VARS                  149
-#define RESUME                          151
-#define MATCH_CLASS                     152
-#define FORMAT_VALUE                    155
-#define BUILD_CONST_KEY_MAP             156
-#define BUILD_STRING                    157
-#define LOAD_METHOD                     160
-#define LIST_EXTEND                     162
-#define SET_UPDATE                      163
-#define DICT_MERGE                      164
-#define DICT_UPDATE                     165
-#define PRECALL                         166
-#define CALL                            171
-#define KW_NAMES                        172
-#define BINARY_OP_ADAPTIVE                3
-#define BINARY_OP_ADD_INT                 4
-#define BINARY_OP_ADD_FLOAT               5
-#define BINARY_OP_ADD_UNICODE             6
-#define BINARY_OP_INPLACE_ADD_UNICODE     7
-#define BINARY_OP_MULTIPLY_INT            8
-#define BINARY_OP_MULTIPLY_FLOAT         13
-#define BINARY_OP_SUBTRACT_INT           14
-#define BINARY_OP_SUBTRACT_FLOAT         16
-#define COMPARE_OP_ADAPTIVE              17
-#define COMPARE_OP_FLOAT_JUMP            18
-#define COMPARE_OP_INT_JUMP              19
-#define COMPARE_OP_STR_JUMP              20
-#define BINARY_SUBSCR_ADAPTIVE           21
-#define BINARY_SUBSCR_GETITEM            22
-#define BINARY_SUBSCR_LIST_INT           23
-#define BINARY_SUBSCR_TUPLE_INT          24
-#define BINARY_SUBSCR_DICT               26
-#define STORE_SUBSCR_ADAPTIVE            27
-#define STORE_SUBSCR_LIST_INT            28
-#define STORE_SUBSCR_DICT                29
-#define CALL_ADAPTIVE                    34
-#define CALL_PY_EXACT_ARGS               36
-#define CALL_PY_WITH_DEFAULTS            37
-#define JUMP_ABSOLUTE_QUICK              38
-#define LOAD_ATTR_ADAPTIVE               39
-#define LOAD_ATTR_INSTANCE_VALUE         40
-#define LOAD_ATTR_WITH_HINT              41
-#define LOAD_ATTR_SLOT                   42
-#define LOAD_ATTR_MODULE                 43
-#define LOAD_GLOBAL_ADAPTIVE             44
-#define LOAD_GLOBAL_MODULE               45
-#define LOAD_GLOBAL_BUILTIN              46
-#define LOAD_METHOD_ADAPTIVE             47
-#define LOAD_METHOD_CLASS                48
-#define LOAD_METHOD_MODULE               55
-#define LOAD_METHOD_NO_DICT              56
-#define LOAD_METHOD_WITH_DICT            57
-#define LOAD_METHOD_WITH_VALUES          58
-#define PRECALL_ADAPTIVE                 59
-#define PRECALL_BUILTIN_CLASS            62
-#define PRECALL_NO_KW_BUILTIN_O          63
-#define PRECALL_NO_KW_BUILTIN_FAST       64
-#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS  65
-#define PRECALL_NO_KW_LEN                66
-#define PRECALL_NO_KW_ISINSTANCE         67
-#define PRECALL_NO_KW_LIST_APPEND        72
-#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O  73
-#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS  76
-#define PRECALL_NO_KW_STR_1              77
-#define PRECALL_NO_KW_TUPLE_1            78
-#define PRECALL_NO_KW_TYPE_1             79
-#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST  80
-#define PRECALL_BOUND_METHOD             81
-#define PRECALL_PYFUNC                  140
-#define RESUME_QUICK                    141
-#define STORE_ATTR_ADAPTIVE             143
-#define STORE_ATTR_INSTANCE_VALUE       150
-#define STORE_ATTR_SLOT                 153
-#define STORE_ATTR_WITH_HINT            154
-#define UNPACK_SEQUENCE_ADAPTIVE        158
-#define UNPACK_SEQUENCE_LIST            159
-#define UNPACK_SEQUENCE_TUPLE           161
-#define UNPACK_SEQUENCE_TWO_TUPLE       167
-#define LOAD_FAST__LOAD_FAST            168
-#define STORE_FAST__LOAD_FAST           169
-#define LOAD_FAST__LOAD_CONST           170
-#define LOAD_CONST__LOAD_FAST           173
-#define STORE_FAST__STORE_FAST          174
-#define DO_TRACING                      255
+#define CACHE                                    0
+#define POP_TOP                                  1
+#define PUSH_NULL                                2
+#define NOP                                      9
+#define UNARY_POSITIVE                          10
+#define UNARY_NEGATIVE                          11
+#define UNARY_NOT                               12
+#define UNARY_INVERT                            15
+#define BINARY_SUBSCR                           25
+#define GET_LEN                                 30
+#define MATCH_MAPPING                           31
+#define MATCH_SEQUENCE                          32
+#define MATCH_KEYS                              33
+#define PUSH_EXC_INFO                           35
+#define WITH_EXCEPT_START                       49
+#define GET_AITER                               50
+#define GET_ANEXT                               51
+#define BEFORE_ASYNC_WITH                       52
+#define BEFORE_WITH                             53
+#define END_ASYNC_FOR                           54
+#define STORE_SUBSCR                            60
+#define DELETE_SUBSCR                           61
+#define GET_ITER                                68
+#define GET_YIELD_FROM_ITER                     69
+#define PRINT_EXPR                              70
+#define LOAD_BUILD_CLASS                        71
+#define LOAD_ASSERTION_ERROR                    74
+#define RETURN_GENERATOR                        75
+#define LIST_TO_TUPLE                           82
+#define RETURN_VALUE                            83
+#define IMPORT_STAR                             84
+#define SETUP_ANNOTATIONS                       85
+#define YIELD_VALUE                             86
+#define ASYNC_GEN_WRAP                          87
+#define PREP_RERAISE_STAR                       88
+#define POP_EXCEPT                              89
+#define HAVE_ARGUMENT                           90
+#define STORE_NAME                              90
+#define DELETE_NAME                             91
+#define UNPACK_SEQUENCE                         92
+#define FOR_ITER                                93
+#define UNPACK_EX                               94
+#define STORE_ATTR                              95
+#define DELETE_ATTR                             96
+#define STORE_GLOBAL                            97
+#define DELETE_GLOBAL                           98
+#define SWAP                                    99
+#define LOAD_CONST                             100
+#define LOAD_NAME                              101
+#define BUILD_TUPLE                            102
+#define BUILD_LIST                             103
+#define BUILD_SET                              104
+#define BUILD_MAP                              105
+#define LOAD_ATTR                              106
+#define COMPARE_OP                             107
+#define IMPORT_NAME                            108
+#define IMPORT_FROM                            109
+#define JUMP_FORWARD                           110
+#define JUMP_IF_FALSE_OR_POP                   111
+#define JUMP_IF_TRUE_OR_POP                    112
+#define JUMP_ABSOLUTE                          113
+#define POP_JUMP_IF_FALSE                      114
+#define POP_JUMP_IF_TRUE                       115
+#define LOAD_GLOBAL                            116
+#define IS_OP                                  117
+#define CONTAINS_OP                            118
+#define RERAISE                                119
+#define COPY                                   120
+#define JUMP_IF_NOT_EXC_MATCH                  121
+#define BINARY_OP                              122
+#define SEND                                   123
+#define LOAD_FAST                              124
+#define STORE_FAST                             125
+#define DELETE_FAST                            126
+#define JUMP_IF_NOT_EG_MATCH                   127
+#define POP_JUMP_IF_NOT_NONE                   128
+#define POP_JUMP_IF_NONE                       129
+#define RAISE_VARARGS                          130
+#define GET_AWAITABLE                          131
+#define MAKE_FUNCTION                          132
+#define BUILD_SLICE                            133
+#define JUMP_NO_INTERRUPT                      134
+#define MAKE_CELL                              135
+#define LOAD_CLOSURE                           136
+#define LOAD_DEREF                             137
+#define STORE_DEREF                            138
+#define DELETE_DEREF                           139
+#define CALL_FUNCTION_EX                       142
+#define EXTENDED_ARG                           144
+#define LIST_APPEND                            145
+#define SET_ADD                                146
+#define MAP_ADD                                147
+#define LOAD_CLASSDEREF                        148
+#define COPY_FREE_VARS                         149
+#define RESUME                                 151
+#define MATCH_CLASS                            152
+#define FORMAT_VALUE                           155
+#define BUILD_CONST_KEY_MAP                    156
+#define BUILD_STRING                           157
+#define LOAD_METHOD                            160
+#define LIST_EXTEND                            162
+#define SET_UPDATE                             163
+#define DICT_MERGE                             164
+#define DICT_UPDATE                            165
+#define PRECALL                                166
+#define CALL                                   171
+#define KW_NAMES                               172
+#define BINARY_OP_ADAPTIVE                       3
+#define BINARY_OP_ADD_FLOAT                      4
+#define BINARY_OP_ADD_INT                        5
+#define BINARY_OP_ADD_UNICODE                    6
+#define BINARY_OP_INPLACE_ADD_UNICODE            7
+#define BINARY_OP_MULTIPLY_FLOAT                 8
+#define BINARY_OP_MULTIPLY_INT                  13
+#define BINARY_OP_SUBTRACT_FLOAT                14
+#define BINARY_OP_SUBTRACT_INT                  16
+#define BINARY_SUBSCR_ADAPTIVE                  17
+#define BINARY_SUBSCR_DICT                      18
+#define BINARY_SUBSCR_GETITEM                   19
+#define BINARY_SUBSCR_LIST_INT                  20
+#define BINARY_SUBSCR_TUPLE_INT                 21
+#define CALL_ADAPTIVE                           22
+#define CALL_PY_EXACT_ARGS                      23
+#define CALL_PY_WITH_DEFAULTS                   24
+#define COMPARE_OP_ADAPTIVE                     26
+#define COMPARE_OP_FLOAT_JUMP                   27
+#define COMPARE_OP_INT_JUMP                     28
+#define COMPARE_OP_STR_JUMP                     29
+#define JUMP_ABSOLUTE_QUICK                     34
+#define LOAD_ATTR_ADAPTIVE                      36
+#define LOAD_ATTR_INSTANCE_VALUE                37
+#define LOAD_ATTR_MODULE                        38
+#define LOAD_ATTR_SLOT                          39
+#define LOAD_ATTR_WITH_HINT                     40
+#define LOAD_CONST__LOAD_FAST                   41
+#define LOAD_FAST__LOAD_CONST                   42
+#define LOAD_FAST__LOAD_FAST                    43
+#define LOAD_GLOBAL_ADAPTIVE                    44
+#define LOAD_GLOBAL_BUILTIN                     45
+#define LOAD_GLOBAL_MODULE                      46
+#define LOAD_METHOD_ADAPTIVE                    47
+#define LOAD_METHOD_CLASS                       48
+#define LOAD_METHOD_MODULE                      55
+#define LOAD_METHOD_NO_DICT                     56
+#define LOAD_METHOD_WITH_DICT                   57
+#define LOAD_METHOD_WITH_VALUES                 58
+#define PRECALL_ADAPTIVE                        59
+#define PRECALL_BOUND_METHOD                    62
+#define PRECALL_BUILTIN_CLASS                   63
+#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS      64
+#define PRECALL_NO_KW_BUILTIN_FAST              65
+#define PRECALL_NO_KW_BUILTIN_O                 66
+#define PRECALL_NO_KW_ISINSTANCE                67
+#define PRECALL_NO_KW_LEN                       72
+#define PRECALL_NO_KW_LIST_APPEND               73
+#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST    76
+#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS  77
+#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O       78
+#define PRECALL_NO_KW_STR_1                     79
+#define PRECALL_NO_KW_TUPLE_1                   80
+#define PRECALL_NO_KW_TYPE_1                    81
+#define PRECALL_PYFUNC                         140
+#define RESUME_QUICK                           141
+#define STORE_ATTR_ADAPTIVE                    143
+#define STORE_ATTR_INSTANCE_VALUE              150
+#define STORE_ATTR_SLOT                        153
+#define STORE_ATTR_WITH_HINT                   154
+#define STORE_FAST__LOAD_FAST                  158
+#define STORE_FAST__STORE_FAST                 159
+#define STORE_SUBSCR_ADAPTIVE                  161
+#define STORE_SUBSCR_DICT                      167
+#define STORE_SUBSCR_LIST_INT                  168
+#define UNPACK_SEQUENCE_ADAPTIVE               169
+#define UNPACK_SEQUENCE_LIST                   170
+#define UNPACK_SEQUENCE_TUPLE                  173
+#define UNPACK_SEQUENCE_TWO_TUPLE              174
+#define DO_TRACING                             255
 
-extern const uint8_t _PyOpcode_InlineCacheEntries[256];
+extern const uint8_t _PyOpcode_Caches[256];
+
+extern const uint8_t _PyOpcode_Deopt[256];
 
 #ifdef NEED_OPCODE_TABLES
 static const uint32_t _PyOpcode_RelativeJump[8] = {
@@ -209,7 +211,7 @@ static const uint32_t _PyOpcode_Jump[8] = {
     0U,
 };
 
-const uint8_t _PyOpcode_InlineCacheEntries[256] = {
+const uint8_t _PyOpcode_Caches[256] = {
     [BINARY_SUBSCR] = 4,
     [STORE_SUBSCR] = 1,
     [UNPACK_SEQUENCE] = 1,
@@ -222,6 +224,184 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = {
     [PRECALL] = 1,
     [CALL] = 4,
 };
+
+const uint8_t _PyOpcode_Deopt[256] = {
+    [ASYNC_GEN_WRAP] = ASYNC_GEN_WRAP,
+    [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH,
+    [BEFORE_WITH] = BEFORE_WITH,
+    [BINARY_OP] = BINARY_OP,
+    [BINARY_OP_ADAPTIVE] = BINARY_OP,
+    [BINARY_OP_ADD_FLOAT] = BINARY_OP,
+    [BINARY_OP_ADD_INT] = BINARY_OP,
+    [BINARY_OP_ADD_UNICODE] = BINARY_OP,
+    [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP,
+    [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP,
+    [BINARY_OP_MULTIPLY_INT] = BINARY_OP,
+    [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP,
+    [BINARY_OP_SUBTRACT_INT] = BINARY_OP,
+    [BINARY_SUBSCR] = BINARY_SUBSCR,
+    [BINARY_SUBSCR_ADAPTIVE] = BINARY_SUBSCR,
+    [BINARY_SUBSCR_DICT] = BINARY_SUBSCR,
+    [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR,
+    [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR,
+    [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR,
+    [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP,
+    [BUILD_LIST] = BUILD_LIST,
+    [BUILD_MAP] = BUILD_MAP,
+    [BUILD_SET] = BUILD_SET,
+    [BUILD_SLICE] = BUILD_SLICE,
+    [BUILD_STRING] = BUILD_STRING,
+    [BUILD_TUPLE] = BUILD_TUPLE,
+    [CACHE] = CACHE,
+    [CALL] = CALL,
+    [CALL_ADAPTIVE] = CALL,
+    [CALL_FUNCTION_EX] = CALL_FUNCTION_EX,
+    [CALL_PY_EXACT_ARGS] = CALL,
+    [CALL_PY_WITH_DEFAULTS] = CALL,
+    [COMPARE_OP] = COMPARE_OP,
+    [COMPARE_OP_ADAPTIVE] = COMPARE_OP,
+    [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP,
+    [COMPARE_OP_INT_JUMP] = COMPARE_OP,
+    [COMPARE_OP_STR_JUMP] = COMPARE_OP,
+    [CONTAINS_OP] = CONTAINS_OP,
+    [COPY] = COPY,
+    [COPY_FREE_VARS] = COPY_FREE_VARS,
+    [DELETE_ATTR] = DELETE_ATTR,
+    [DELETE_DEREF] = DELETE_DEREF,
+    [DELETE_FAST] = DELETE_FAST,
+    [DELETE_GLOBAL] = DELETE_GLOBAL,
+    [DELETE_NAME] = DELETE_NAME,
+    [DELETE_SUBSCR] = DELETE_SUBSCR,
+    [DICT_MERGE] = DICT_MERGE,
+    [DICT_UPDATE] = DICT_UPDATE,
+    [END_ASYNC_FOR] = END_ASYNC_FOR,
+    [EXTENDED_ARG] = EXTENDED_ARG,
+    [FORMAT_VALUE] = FORMAT_VALUE,
+    [FOR_ITER] = FOR_ITER,
+    [GET_AITER] = GET_AITER,
+    [GET_ANEXT] = GET_ANEXT,
+    [GET_AWAITABLE] = GET_AWAITABLE,
+    [GET_ITER] = GET_ITER,
+    [GET_LEN] = GET_LEN,
+    [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER,
+    [IMPORT_FROM] = IMPORT_FROM,
+    [IMPORT_NAME] = IMPORT_NAME,
+    [IMPORT_STAR] = IMPORT_STAR,
+    [IS_OP] = IS_OP,
+    [JUMP_ABSOLUTE] = JUMP_ABSOLUTE,
+    [JUMP_ABSOLUTE_QUICK] = JUMP_ABSOLUTE,
+    [JUMP_FORWARD] = JUMP_FORWARD,
+    [JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP,
+    [JUMP_IF_NOT_EG_MATCH] = JUMP_IF_NOT_EG_MATCH,
+    [JUMP_IF_NOT_EXC_MATCH] = JUMP_IF_NOT_EXC_MATCH,
+    [JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP,
+    [JUMP_NO_INTERRUPT] = JUMP_NO_INTERRUPT,
+    [KW_NAMES] = KW_NAMES,
+    [LIST_APPEND] = LIST_APPEND,
+    [LIST_EXTEND] = LIST_EXTEND,
+    [LIST_TO_TUPLE] = LIST_TO_TUPLE,
+    [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR,
+    [LOAD_ATTR] = LOAD_ATTR,
+    [LOAD_ATTR_ADAPTIVE] = LOAD_ATTR,
+    [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR,
+    [LOAD_ATTR_MODULE] = LOAD_ATTR,
+    [LOAD_ATTR_SLOT] = LOAD_ATTR,
+    [LOAD_ATTR_WITH_HINT] = LOAD_ATTR,
+    [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS,
+    [LOAD_CLASSDEREF] = LOAD_CLASSDEREF,
+    [LOAD_CLOSURE] = LOAD_CLOSURE,
+    [LOAD_CONST] = LOAD_CONST,
+    [LOAD_CONST__LOAD_FAST] = LOAD_CONST,
+    [LOAD_DEREF] = LOAD_DEREF,
+    [LOAD_FAST] = LOAD_FAST,
+    [LOAD_FAST__LOAD_CONST] = LOAD_FAST,
+    [LOAD_FAST__LOAD_FAST] = LOAD_FAST,
+    [LOAD_GLOBAL] = LOAD_GLOBAL,
+    [LOAD_GLOBAL_ADAPTIVE] = LOAD_GLOBAL,
+    [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL,
+    [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL,
+    [LOAD_METHOD] = LOAD_METHOD,
+    [LOAD_METHOD_ADAPTIVE] = LOAD_METHOD,
+    [LOAD_METHOD_CLASS] = LOAD_METHOD,
+    [LOAD_METHOD_MODULE] = LOAD_METHOD,
+    [LOAD_METHOD_NO_DICT] = LOAD_METHOD,
+    [LOAD_METHOD_WITH_DICT] = LOAD_METHOD,
+    [LOAD_METHOD_WITH_VALUES] = LOAD_METHOD,
+    [LOAD_NAME] = LOAD_NAME,
+    [MAKE_CELL] = MAKE_CELL,
+    [MAKE_FUNCTION] = MAKE_FUNCTION,
+    [MAP_ADD] = MAP_ADD,
+    [MATCH_CLASS] = MATCH_CLASS,
+    [MATCH_KEYS] = MATCH_KEYS,
+    [MATCH_MAPPING] = MATCH_MAPPING,
+    [MATCH_SEQUENCE] = MATCH_SEQUENCE,
+    [NOP] = NOP,
+    [POP_EXCEPT] = POP_EXCEPT,
+    [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,
+    [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,
+    [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,
+    [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,
+    [POP_TOP] = POP_TOP,
+    [PRECALL] = PRECALL,
+    [PRECALL_ADAPTIVE] = PRECALL,
+    [PRECALL_BOUND_METHOD] = PRECALL,
+    [PRECALL_BUILTIN_CLASS] = PRECALL,
+    [PRECALL_BUILTIN_FAST_WITH_KEYWORDS] = PRECALL,
+    [PRECALL_NO_KW_BUILTIN_FAST] = PRECALL,
+    [PRECALL_NO_KW_BUILTIN_O] = PRECALL,
+    [PRECALL_NO_KW_ISINSTANCE] = PRECALL,
+    [PRECALL_NO_KW_LEN] = PRECALL,
+    [PRECALL_NO_KW_LIST_APPEND] = PRECALL,
+    [PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST] = PRECALL,
+    [PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = PRECALL,
+    [PRECALL_NO_KW_METHOD_DESCRIPTOR_O] = PRECALL,
+    [PRECALL_NO_KW_STR_1] = PRECALL,
+    [PRECALL_NO_KW_TUPLE_1] = PRECALL,
+    [PRECALL_NO_KW_TYPE_1] = PRECALL,
+    [PRECALL_PYFUNC] = PRECALL,
+    [PREP_RERAISE_STAR] = PREP_RERAISE_STAR,
+    [PRINT_EXPR] = PRINT_EXPR,
+    [PUSH_EXC_INFO] = PUSH_EXC_INFO,
+    [PUSH_NULL] = PUSH_NULL,
+    [RAISE_VARARGS] = RAISE_VARARGS,
+    [RERAISE] = RERAISE,
+    [RESUME] = RESUME,
+    [RESUME_QUICK] = RESUME,
+    [RETURN_GENERATOR] = RETURN_GENERATOR,
+    [RETURN_VALUE] = RETURN_VALUE,
+    [SEND] = SEND,
+    [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS,
+    [SET_ADD] = SET_ADD,
+    [SET_UPDATE] = SET_UPDATE,
+    [STORE_ATTR] = STORE_ATTR,
+    [STORE_ATTR_ADAPTIVE] = STORE_ATTR,
+    [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR,
+    [STORE_ATTR_SLOT] = STORE_ATTR,
+    [STORE_ATTR_WITH_HINT] = STORE_ATTR,
+    [STORE_DEREF] = STORE_DEREF,
+    [STORE_FAST] = STORE_FAST,
+    [STORE_FAST__LOAD_FAST] = STORE_FAST,
+    [STORE_FAST__STORE_FAST] = STORE_FAST,
+    [STORE_GLOBAL] = STORE_GLOBAL,
+    [STORE_NAME] = STORE_NAME,
+    [STORE_SUBSCR] = STORE_SUBSCR,
+    [STORE_SUBSCR_ADAPTIVE] = STORE_SUBSCR,
+    [STORE_SUBSCR_DICT] = STORE_SUBSCR,
+    [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR,
+    [SWAP] = SWAP,
+    [UNARY_INVERT] = UNARY_INVERT,
+    [UNARY_NEGATIVE] = UNARY_NEGATIVE,
+    [UNARY_NOT] = UNARY_NOT,
+    [UNARY_POSITIVE] = UNARY_POSITIVE,
+    [UNPACK_EX] = UNPACK_EX,
+    [UNPACK_SEQUENCE] = UNPACK_SEQUENCE,
+    [UNPACK_SEQUENCE_ADAPTIVE] = UNPACK_SEQUENCE,
+    [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE,
+    [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE,
+    [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE,
+    [WITH_EXCEPT_START] = WITH_EXCEPT_START,
+    [YIELD_VALUE] = YIELD_VALUE,
+};
 #endif /* OPCODE_TABLES */
 
 #define HAS_CONST(op) (false\
@@ -229,32 +409,32 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = {
     || ((op) == 172) \
     )
 
-#define NB_ADD                            0
-#define NB_AND                            1
-#define NB_FLOOR_DIVIDE                   2
-#define NB_LSHIFT                         3
-#define NB_MATRIX_MULTIPLY                4
-#define NB_MULTIPLY                       5
-#define NB_REMAINDER                      6
-#define NB_OR                             7
-#define NB_POWER                          8
-#define NB_RSHIFT                         9
-#define NB_SUBTRACT                      10
-#define NB_TRUE_DIVIDE                   11
-#define NB_XOR                           12
-#define NB_INPLACE_ADD                   13
-#define NB_INPLACE_AND                   14
-#define NB_INPLACE_FLOOR_DIVIDE          15
-#define NB_INPLACE_LSHIFT                16
-#define NB_INPLACE_MATRIX_MULTIPLY       17
-#define NB_INPLACE_MULTIPLY              18
-#define NB_INPLACE_REMAINDER             19
-#define NB_INPLACE_OR                    20
-#define NB_INPLACE_POWER                 21
-#define NB_INPLACE_RSHIFT                22
-#define NB_INPLACE_SUBTRACT              23
-#define NB_INPLACE_TRUE_DIVIDE           24
-#define NB_INPLACE_XOR                   25
+#define NB_ADD                                   0
+#define NB_AND                                   1
+#define NB_FLOOR_DIVIDE                          2
+#define NB_LSHIFT                                3
+#define NB_MATRIX_MULTIPLY                       4
+#define NB_MULTIPLY                              5
+#define NB_REMAINDER                             6
+#define NB_OR                                    7
+#define NB_POWER                                 8
+#define NB_RSHIFT                                9
+#define NB_SUBTRACT                             10
+#define NB_TRUE_DIVIDE                          11
+#define NB_XOR                                  12
+#define NB_INPLACE_ADD                          13
+#define NB_INPLACE_AND                          14
+#define NB_INPLACE_FLOOR_DIVIDE                 15
+#define NB_INPLACE_LSHIFT                       16
+#define NB_INPLACE_MATRIX_MULTIPLY              17
+#define NB_INPLACE_MULTIPLY                     18
+#define NB_INPLACE_REMAINDER                    19
+#define NB_INPLACE_OR                           20
+#define NB_INPLACE_POWER                        21
+#define NB_INPLACE_RSHIFT                       22
+#define NB_INPLACE_SUBTRACT                     23
+#define NB_INPLACE_TRUE_DIVIDE                  24
+#define NB_INPLACE_XOR                          25
 
 #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT)