gh-115999: Implement thread-local bytecode and enable specialization for `BINARY_OP` (#123926)

Each thread specializes a thread-local copy of the bytecode, created on the first RESUME, in free-threaded builds. All copies of the bytecode for a code object are stored in the co_tlbc array on the code object. Threads reserve a globally unique index identifying its copy of the bytecode in all co_tlbc arrays at thread creation and release the index at thread destruction. The first entry in every co_tlbc array always points to the "main" copy of the bytecode that is stored at the end of the code object. This ensures that no bytecode is copied for programs that do not use threads.

Thread-local bytecode can be disabled at runtime by providing either -X tlbc=0 or PYTHON_TLBC=0. Disabling thread-local bytecode also disables specialization.

Concurrent modifications to the bytecode made by the specializing interpreter and instrumentation use atomics, with specialization taking care not to overwrite an instruction that was instrumented concurrently.

11 files changed, 253 insertions, 67 deletions

diff --git a/Include/cpython/code.h b/Include/cpython/code.h
index 2561b2b..370f1d2 100644
--- a/Include/cpython/code.h
+++ b/Include/cpython/code.h
@@ -72,6 +72,24 @@ typedef struct {
     uint8_t *per_instruction_tools;
 } _PyCoMonitoringData;
 
+#ifdef Py_GIL_DISABLED
+
+/* Each thread specializes a thread-local copy of the bytecode in free-threaded
+ * builds. These copies are stored on the code object in a `_PyCodeArray`. The
+ * first entry in the array always points to the "main" copy of the bytecode
+ * that is stored at the end of the code object.
+ */
+typedef struct {
+    Py_ssize_t size;
+    char *entries[1];
+} _PyCodeArray;
+
+#define _PyCode_DEF_THREAD_LOCAL_BYTECODE() \
+    _PyCodeArray *co_tlbc;
+#else
+#define _PyCode_DEF_THREAD_LOCAL_BYTECODE()
+#endif
+
 // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
 // defined in this macro:
 #define _PyCode_DEF(SIZE) {                                                    \
@@ -138,6 +156,7 @@ typedef struct {
        Type is a void* to keep the format private in codeobject.c to force     \
        people to go through the proper APIs. */                                \
     void *co_extra;                                                            \
+    _PyCode_DEF_THREAD_LOCAL_BYTECODE()                                        \
     char co_code_adaptive[(SIZE)];                                             \
 }
 
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index c2cb4e3..f69c586 100644
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -183,6 +183,7 @@ typedef struct PyConfig {
     int cpu_count;
 #ifdef Py_GIL_DISABLED
     int enable_gil;
+    int tlbc_enabled;
 #endif
 
     /* --- Path configuration inputs ------------ */
diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index 411bbff..80bd19a 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -174,6 +174,18 @@ _PyEval_IsGILEnabled(PyThreadState *tstate)
 extern int _PyEval_EnableGILTransient(PyThreadState *tstate);
 extern int _PyEval_EnableGILPermanent(PyThreadState *tstate);
 extern int _PyEval_DisableGIL(PyThreadState *state);
+
+
+static inline _Py_CODEUNIT *
+_PyEval_GetExecutableCode(PyThreadState *tstate, PyCodeObject *co)
+{
+    _Py_CODEUNIT *bc = _PyCode_GetTLBCFast(tstate, co);
+    if (bc != NULL) {
+        return bc;
+    }
+    return _PyCode_GetTLBC(co);
+}
+
 #endif
 
 extern void _PyEval_DeactivateOpCache(void);
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 57e0a14..a0acf76 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -11,6 +11,7 @@ extern "C" {
 #include "pycore_stackref.h"    // _PyStackRef
 #include "pycore_lock.h"        // PyMutex
 #include "pycore_backoff.h"     // _Py_BackoffCounter
+#include "pycore_tstate.h"      // _PyThreadStateImpl
 
 
 /* Each instruction in a code object is a fixed-width value,
@@ -313,11 +314,17 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range);
 /** API for executors */
 extern void _PyCode_Clear_Executors(PyCodeObject *code);
 
+
 #ifdef Py_GIL_DISABLED
 // gh-115999 tracks progress on addressing this.
 #define ENABLE_SPECIALIZATION 0
+// Use this to enable specialization families once they are thread-safe. All
+// uses will be replaced with ENABLE_SPECIALIZATION once all families are
+// thread-safe.
+#define ENABLE_SPECIALIZATION_FT 1
 #else
 #define ENABLE_SPECIALIZATION 1
+#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION
 #endif
 
 /* Specialization functions */
@@ -600,6 +607,40 @@ struct _PyCode8 _PyCode_DEF(8);
 
 PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup;
 
+#ifdef Py_GIL_DISABLED
+
+// Return a pointer to the thread-local bytecode for the current thread, if it
+// exists.
+static inline _Py_CODEUNIT *
+_PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co)
+{
+    _PyCodeArray *code = _Py_atomic_load_ptr_acquire(&co->co_tlbc);
+    int32_t idx = ((_PyThreadStateImpl*) tstate)->tlbc_index;
+    if (idx < code->size && code->entries[idx] != NULL) {
+        return (_Py_CODEUNIT *) code->entries[idx];
+    }
+    return NULL;
+}
+
+// Return a pointer to the thread-local bytecode for the current thread,
+// creating it if necessary.
+extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co);
+
+// Reserve an index for the current thread into thread-local bytecode
+// arrays
+//
+// Returns the reserved index or -1 on error.
+extern int32_t _Py_ReserveTLBCIndex(PyInterpreterState *interp);
+
+// Release the current thread's index into thread-local bytecode arrays
+extern void _Py_ClearTLBCIndex(_PyThreadStateImpl *tstate);
+
+// Free all TLBC copies not associated with live threads.
+//
+// Returns 0 on success or -1 on error.
+extern int _Py_ClearUnusedTLBC(PyInterpreterState *interp);
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
index c9ac381..8c01003 100644
--- a/Include/internal/pycore_frame.h
+++ b/Include/internal/pycore_frame.h
@@ -68,6 +68,10 @@ typedef struct _PyInterpreterFrame {
     PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */
     PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */
     _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */
+#ifdef Py_GIL_DISABLED
+    /* Index of thread-local bytecode containing instr_ptr. */
+    int32_t tlbc_index;
+#endif
     _PyStackRef *stackpointer;
     uint16_t return_offset;  /* Only relevant during a function call */
     char owner;
@@ -76,7 +80,7 @@ typedef struct _PyInterpreterFrame {
 } _PyInterpreterFrame;
 
 #define _PyInterpreterFrame_LASTI(IF) \
-    ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF))))
+    ((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF))))
 
 static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
     PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable);
@@ -84,6 +88,19 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
     return (PyCodeObject *)executable;
 }
 
+static inline _Py_CODEUNIT *
+_PyFrame_GetBytecode(_PyInterpreterFrame *f)
+{
+#ifdef Py_GIL_DISABLED
+    PyCodeObject *co = _PyFrame_GetCode(f);
+    _PyCodeArray *tlbc = _Py_atomic_load_ptr_acquire(&co->co_tlbc);
+    assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size);
+    return (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index];
+#else
+    return _PyCode_CODE(_PyFrame_GetCode(f));
+#endif
+}
+
 static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) {
     PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj);
     assert(PyFunction_Check(func));
@@ -144,13 +161,33 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *
 #endif
 }
 
+#ifdef Py_GIL_DISABLED
+static inline void
+_PyFrame_InitializeTLBC(PyThreadState *tstate, _PyInterpreterFrame *frame,
+                        PyCodeObject *code)
+{
+    _Py_CODEUNIT *tlbc = _PyCode_GetTLBCFast(tstate, code);
+    if (tlbc == NULL) {
+        // No thread-local bytecode exists for this thread yet; use the main
+        // thread's copy, deferring thread-local bytecode creation to the
+        // execution of RESUME.
+        frame->instr_ptr = _PyCode_CODE(code);
+        frame->tlbc_index = 0;
+    }
+    else {
+        frame->instr_ptr = tlbc;
+        frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index;
+    }
+}
+#endif
+
 /* Consumes reference to func and locals.
    Does not initialize frame->previous, which happens
    when frame is linked into the frame stack.
  */
 static inline void
 _PyFrame_Initialize(
-    _PyInterpreterFrame *frame, _PyStackRef func,
+    PyThreadState *tstate, _PyInterpreterFrame *frame, _PyStackRef func,
     PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous)
 {
     frame->previous = previous;
@@ -162,7 +199,12 @@ _PyFrame_Initialize(
     frame->f_locals = locals;
     frame->stackpointer = frame->localsplus + code->co_nlocalsplus;
     frame->frame_obj = NULL;
+#ifdef Py_GIL_DISABLED
+    _PyFrame_InitializeTLBC(tstate, frame, code);
+#else
+    (void)tstate;
     frame->instr_ptr = _PyCode_CODE(code);
+#endif
     frame->return_offset = 0;
     frame->owner = FRAME_OWNED_BY_THREAD;
 
@@ -224,7 +266,8 @@ _PyFrame_IsIncomplete(_PyInterpreterFrame *frame)
         return true;
     }
     return frame->owner != FRAME_OWNED_BY_GENERATOR &&
-        frame->instr_ptr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable;
+           frame->instr_ptr < _PyFrame_GetBytecode(frame) +
+                                  _PyFrame_GetCode(frame)->_co_firsttraceable;
 }
 
 static inline _PyInterpreterFrame *
@@ -315,7 +358,8 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, _PyStackRef func, int null_locals_
     _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top;
     tstate->datastack_top += code->co_framesize;
     assert(tstate->datastack_top < tstate->datastack_limit);
-    _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from, previous);
+    _PyFrame_Initialize(tstate, new_frame, func, NULL, code, null_locals_from,
+                        previous);
     return new_frame;
 }
 
@@ -339,7 +383,11 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     assert(stackdepth <= code->co_stacksize);
     frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth;
     frame->frame_obj = NULL;
+#ifdef Py_GIL_DISABLED
+    _PyFrame_InitializeTLBC(tstate, frame, code);
+#else
     frame->instr_ptr = _PyCode_CODE(code);
+#endif
     frame->owner = FRAME_OWNED_BY_THREAD;
     frame->return_offset = 0;
 
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
index b85957d..38a1c56 100644
--- a/Include/internal/pycore_gc.h
+++ b/Include/internal/pycore_gc.h
@@ -389,6 +389,10 @@ extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *ar
         }                                                               \
     } while (0)
 
+#ifdef Py_GIL_DISABLED
+extern void _PyGC_VisitObjectsWorldStopped(PyInterpreterState *interp,
+                                           gcvisitobjects_t callback, void *arg);
+#endif
 
 #ifdef __cplusplus
 }
diff --git a/Include/internal/pycore_index_pool.h b/Include/internal/pycore_index_pool.h
new file mode 100644
index 0000000..e81bfd4
--- /dev/null
+++ b/Include/internal/pycore_index_pool.h
@@ -0,0 +1,56 @@
+#ifndef Py_INTERNAL_INDEX_POOL_H
+#define Py_INTERNAL_INDEX_POOL_H
+
+#include "Python.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifdef Py_GIL_DISABLED
+
+// This contains code for allocating unique indices in an array. It is used by
+// the free-threaded build to assign each thread a globally unique index into
+// each code object's thread-local bytecode array.
+
+// A min-heap of indices
+typedef struct _PyIndexHeap {
+    int32_t *values;
+
+    // Number of items stored in values
+    Py_ssize_t size;
+
+    // Maximum number of items that can be stored in values
+    Py_ssize_t capacity;
+} _PyIndexHeap;
+
+// An unbounded pool of indices. Indices are allocated starting from 0. They
+// may be released back to the pool once they are no longer in use.
+typedef struct _PyIndexPool {
+    PyMutex mutex;
+
+    // Min heap of indices available for allocation
+    _PyIndexHeap free_indices;
+
+    // Next index to allocate if no free indices are available
+    int32_t next_index;
+} _PyIndexPool;
+
+// Allocate the smallest available index. Returns -1 on error.
+extern int32_t _PyIndexPool_AllocIndex(_PyIndexPool *indices);
+
+// Release `index` back to the pool
+extern void _PyIndexPool_FreeIndex(_PyIndexPool *indices, int32_t index);
+
+extern void _PyIndexPool_Fini(_PyIndexPool *indices);
+
+#endif // Py_GIL_DISABLED
+
+#ifdef __cplusplus
+}
+#endif
+#endif // !Py_INTERNAL_INDEX_POOL_H
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index 36cd71e..9e3b429 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -26,6 +26,7 @@ extern "C" {
 #include "pycore_genobject.h"     // _PyGen_FetchStopIterationValue
 #include "pycore_global_objects.h"// struct _Py_interp_cached_objects
 #include "pycore_import.h"        // struct _import_state
+#include "pycore_index_pool.h"     // _PyIndexPool
 #include "pycore_instruments.h"   // _PY_MONITORING_EVENTS
 #include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_mimalloc.h"      // struct _mimalloc_interp_state
@@ -222,6 +223,7 @@ struct _is {
     struct _brc_state brc;  // biased reference counting state
     struct _Py_unique_id_pool unique_ids;  // object ids for per-thread refcounts
     PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS];
+    _PyIndexPool tlbc_indices;
 #endif
 
     // Per-interpreter state for the obmalloc allocator.  For the main
diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h
index e0e7d5e..b8bea72 100644
--- a/Include/internal/pycore_tstate.h
+++ b/Include/internal/pycore_tstate.h
@@ -42,6 +42,9 @@ typedef struct _PyThreadStateImpl {
         int is_finalized;
     } refcounts;
 
+    // Index to use to retrieve thread-local bytecode for this thread
+    int32_t tlbc_index;
+
     // When >1, code objects do not immortalize their non-string constants.
     int suppress_co_const_immortalization;
 #endif
@@ -52,7 +55,6 @@ typedef struct _PyThreadStateImpl {
 
 } _PyThreadStateImpl;
 
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h
index de628d2..55416d2 100644
--- a/Include/internal/pycore_uop_ids.h
+++ b/Include/internal/pycore_uop_ids.h
@@ -193,106 +193,107 @@ extern "C" {
 #define _LOAD_ATTR_SLOT_1 423
 #define _LOAD_ATTR_WITH_HINT 424
 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
+#define _LOAD_BYTECODE 425
 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
 #define _LOAD_CONST LOAD_CONST
 #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL
-#define _LOAD_CONST_INLINE 425
-#define _LOAD_CONST_INLINE_BORROW 426
-#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 427
-#define _LOAD_CONST_INLINE_WITH_NULL 428
+#define _LOAD_CONST_INLINE 426
+#define _LOAD_CONST_INLINE_BORROW 427
+#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 428
+#define _LOAD_CONST_INLINE_WITH_NULL 429
 #define _LOAD_DEREF LOAD_DEREF
-#define _LOAD_FAST 429
-#define _LOAD_FAST_0 430
-#define _LOAD_FAST_1 431
-#define _LOAD_FAST_2 432
-#define _LOAD_FAST_3 433
-#define _LOAD_FAST_4 434
-#define _LOAD_FAST_5 435
-#define _LOAD_FAST_6 436
-#define _LOAD_FAST_7 437
+#define _LOAD_FAST 430
+#define _LOAD_FAST_0 431
+#define _LOAD_FAST_1 432
+#define _LOAD_FAST_2 433
+#define _LOAD_FAST_3 434
+#define _LOAD_FAST_4 435
+#define _LOAD_FAST_5 436
+#define _LOAD_FAST_6 437
+#define _LOAD_FAST_7 438
 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK
 #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
 #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
 #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
-#define _LOAD_GLOBAL 438
-#define _LOAD_GLOBAL_BUILTINS 439
-#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 440
-#define _LOAD_GLOBAL_MODULE 441
-#define _LOAD_GLOBAL_MODULE_FROM_KEYS 442
+#define _LOAD_GLOBAL 439
+#define _LOAD_GLOBAL_BUILTINS 440
+#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 441
+#define _LOAD_GLOBAL_MODULE 442
+#define _LOAD_GLOBAL_MODULE_FROM_KEYS 443
 #define _LOAD_LOCALS LOAD_LOCALS
 #define _LOAD_NAME LOAD_NAME
-#define _LOAD_SMALL_INT 443
-#define _LOAD_SMALL_INT_0 444
-#define _LOAD_SMALL_INT_1 445
-#define _LOAD_SMALL_INT_2 446
-#define _LOAD_SMALL_INT_3 447
+#define _LOAD_SMALL_INT 444
+#define _LOAD_SMALL_INT_0 445
+#define _LOAD_SMALL_INT_1 446
+#define _LOAD_SMALL_INT_2 447
+#define _LOAD_SMALL_INT_3 448
 #define _LOAD_SPECIAL LOAD_SPECIAL
 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR
 #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD
-#define _MAKE_CALLARGS_A_TUPLE 448
+#define _MAKE_CALLARGS_A_TUPLE 449
 #define _MAKE_CELL MAKE_CELL
 #define _MAKE_FUNCTION MAKE_FUNCTION
-#define _MAKE_WARM 449
+#define _MAKE_WARM 450
 #define _MAP_ADD MAP_ADD
 #define _MATCH_CLASS MATCH_CLASS
 #define _MATCH_KEYS MATCH_KEYS
 #define _MATCH_MAPPING MATCH_MAPPING
 #define _MATCH_SEQUENCE MATCH_SEQUENCE
-#define _MAYBE_EXPAND_METHOD 450
-#define _MAYBE_EXPAND_METHOD_KW 451
-#define _MONITOR_CALL 452
-#define _MONITOR_JUMP_BACKWARD 453
-#define _MONITOR_RESUME 454
+#define _MAYBE_EXPAND_METHOD 451
+#define _MAYBE_EXPAND_METHOD_KW 452
+#define _MONITOR_CALL 453
+#define _MONITOR_JUMP_BACKWARD 454
+#define _MONITOR_RESUME 455
 #define _NOP NOP
 #define _POP_EXCEPT POP_EXCEPT
-#define _POP_JUMP_IF_FALSE 455
-#define _POP_JUMP_IF_TRUE 456
+#define _POP_JUMP_IF_FALSE 456
+#define _POP_JUMP_IF_TRUE 457
 #define _POP_TOP POP_TOP
-#define _POP_TOP_LOAD_CONST_INLINE_BORROW 457
+#define _POP_TOP_LOAD_CONST_INLINE_BORROW 458
 #define _PUSH_EXC_INFO PUSH_EXC_INFO
-#define _PUSH_FRAME 458
+#define _PUSH_FRAME 459
 #define _PUSH_NULL PUSH_NULL
-#define _PY_FRAME_GENERAL 459
-#define _PY_FRAME_KW 460
-#define _QUICKEN_RESUME 461
-#define _REPLACE_WITH_TRUE 462
+#define _PY_FRAME_GENERAL 460
+#define _PY_FRAME_KW 461
+#define _QUICKEN_RESUME 462
+#define _REPLACE_WITH_TRUE 463
 #define _RESUME_CHECK RESUME_CHECK
 #define _RETURN_GENERATOR RETURN_GENERATOR
 #define _RETURN_VALUE RETURN_VALUE
-#define _SAVE_RETURN_OFFSET 463
-#define _SEND 464
-#define _SEND_GEN_FRAME 465
+#define _SAVE_RETURN_OFFSET 464
+#define _SEND 465
+#define _SEND_GEN_FRAME 466
 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
 #define _SET_ADD SET_ADD
 #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
 #define _SET_UPDATE SET_UPDATE
-#define _START_EXECUTOR 466
-#define _STORE_ATTR 467
-#define _STORE_ATTR_INSTANCE_VALUE 468
-#define _STORE_ATTR_SLOT 469
-#define _STORE_ATTR_WITH_HINT 470
+#define _START_EXECUTOR 467
+#define _STORE_ATTR 468
+#define _STORE_ATTR_INSTANCE_VALUE 469
+#define _STORE_ATTR_SLOT 470
+#define _STORE_ATTR_WITH_HINT 471
 #define _STORE_DEREF STORE_DEREF
-#define _STORE_FAST 471
-#define _STORE_FAST_0 472
-#define _STORE_FAST_1 473
-#define _STORE_FAST_2 474
-#define _STORE_FAST_3 475
-#define _STORE_FAST_4 476
-#define _STORE_FAST_5 477
-#define _STORE_FAST_6 478
-#define _STORE_FAST_7 479
+#define _STORE_FAST 472
+#define _STORE_FAST_0 473
+#define _STORE_FAST_1 474
+#define _STORE_FAST_2 475
+#define _STORE_FAST_3 476
+#define _STORE_FAST_4 477
+#define _STORE_FAST_5 478
+#define _STORE_FAST_6 479
+#define _STORE_FAST_7 480
 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
 #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
 #define _STORE_GLOBAL STORE_GLOBAL
 #define _STORE_NAME STORE_NAME
-#define _STORE_SLICE 480
-#define _STORE_SUBSCR 481
+#define _STORE_SLICE 481
+#define _STORE_SUBSCR 482
 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
 #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
 #define _SWAP SWAP
-#define _TIER2_RESUME_CHECK 482
-#define _TO_BOOL 483
+#define _TIER2_RESUME_CHECK 483
+#define _TO_BOOL 484
 #define _TO_BOOL_BOOL TO_BOOL_BOOL
 #define _TO_BOOL_INT TO_BOOL_INT
 #define _TO_BOOL_LIST TO_BOOL_LIST
@@ -302,13 +303,13 @@ extern "C" {
 #define _UNARY_NEGATIVE UNARY_NEGATIVE
 #define _UNARY_NOT UNARY_NOT
 #define _UNPACK_EX UNPACK_EX
-#define _UNPACK_SEQUENCE 484
+#define _UNPACK_SEQUENCE 485
 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
 #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
 #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
 #define _WITH_EXCEPT_START WITH_EXCEPT_START
 #define _YIELD_VALUE YIELD_VALUE
-#define MAX_UOP_ID 484
+#define MAX_UOP_ID 485
 
 #ifdef __cplusplus
 }
diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h
index 4cfdece..ade2972 100644
--- a/Include/internal/pycore_uop_metadata.h
+++ b/Include/internal/pycore_uop_metadata.h
@@ -289,7 +289,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_FATAL_ERROR] = 0,
     [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG,
     [_DEOPT] = 0,
-    [_ERROR_POP_N] = HAS_ARG_FLAG,
+    [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
     [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG,
 };