bpo-46841: Move the cache for `LOAD_GLOBAL` inline. (GH-31575)

2 files changed, 60 insertions, 17 deletions

diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 0e401d6..dfa15b8 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -27,11 +27,6 @@ typedef struct {
 } _PyAttrCache;
 
 typedef struct {
-    uint32_t module_keys_version;
-    uint32_t builtin_keys_version;
-} _PyLoadGlobalCache;
-
-typedef struct {
     /* Borrowed ref in LOAD_METHOD */
     PyObject *obj;
 } _PyObjectCache;
@@ -57,23 +52,35 @@ typedef union {
     _PyEntryZero zero;
     _PyAdaptiveEntry adaptive;
     _PyAttrCache attr;
-    _PyLoadGlobalCache load_global;
     _PyObjectCache obj;
     _PyCallCache call;
 } SpecializedCacheEntry;
 
 #define INSTRUCTIONS_PER_ENTRY (sizeof(SpecializedCacheEntry)/sizeof(_Py_CODEUNIT))
 
+/* Inline caches */
+
+#define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT))
+
+typedef struct {
+    _Py_CODEUNIT counter;
+    _Py_CODEUNIT index;
+    _Py_CODEUNIT module_keys_version;
+    _Py_CODEUNIT _m1;
+    _Py_CODEUNIT builtin_keys_version;
+} _PyLoadGlobalCache;
+
+#define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache)
+
 typedef struct {
     _Py_CODEUNIT counter;
 } _PyBinaryOpCache;
 
+#define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache)
 typedef struct {
     _Py_CODEUNIT counter;
 } _PyUnpackSequenceCache;
 
-#define INLINE_CACHE_ENTRIES_BINARY_OP \
-    (sizeof(_PyBinaryOpCache) / sizeof(_Py_CODEUNIT))
 
 #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \
     (sizeof(_PyUnpackSequenceCache) / sizeof(_Py_CODEUNIT))
@@ -307,7 +314,7 @@ cache_backoff(_PyAdaptiveEntry *entry) {
 
 extern int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
 extern int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
-extern int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
+extern int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name);
 extern int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
 extern int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache);
 extern int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr);
@@ -388,6 +395,38 @@ extern PyObject* _Py_GetSpecializationStats(void);
 #define OBJECT_STAT_INC(name) ((void)0)
 #endif
 
+// Cache values are only valid in memory, so use native endianness.
+#ifdef WORDS_BIGENDIAN
+
+static inline void
+write32(uint16_t *p, uint32_t val)
+{
+    p[0] = val >> 16;
+    p[1] = (uint16_t)val;
+}
+
+static inline uint32_t
+read32(uint16_t *p)
+{
+    return (p[0] << 16) | p[1];
+}
+
+#else
+
+static inline void
+write32(uint16_t *p, uint32_t val)
+{
+    p[0] = (uint16_t)val;
+    p[1] = val >> 16;
+}
+
+static inline uint32_t
+read32(uint16_t *p)
+{
+    return p[0] | (p[1] << 16);
+}
+
+#endif
 
 #ifdef __cplusplus
 }
diff --git a/Include/opcode.h b/Include/opcode.h
index ae21d92..99480de 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -185,8 +185,11 @@ extern "C" {
 #define STORE_FAST__STORE_FAST          175
 #define LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE 176
 #define DO_TRACING                      255
-#ifdef NEED_OPCODE_JUMP_TABLES
-static uint32_t _PyOpcode_RelativeJump[8] = {
+
+extern const uint8_t _PyOpcode_InlineCacheEntries[256];
+
+#ifdef NEED_OPCODE_TABLES
+static const uint32_t _PyOpcode_RelativeJump[8] = {
     0U,
     0U,
     536870912U,
@@ -196,7 +199,7 @@ static uint32_t _PyOpcode_RelativeJump[8] = {
     0U,
     0U,
 };
-static uint32_t _PyOpcode_Jump[8] = {
+static const uint32_t _PyOpcode_Jump[8] = {
     0U,
     0U,
     536870912U,
@@ -206,6 +209,12 @@ static uint32_t _PyOpcode_Jump[8] = {
     0U,
     0U,
 };
+
+const uint8_t _PyOpcode_InlineCacheEntries[256] = {
+    [UNPACK_SEQUENCE] = 1,
+    [LOAD_GLOBAL] = 5,
+    [BINARY_OP] = 1,
+};
 #endif /* OPCODE_TABLES */
 
 #define HAS_CONST(op) (false\
@@ -240,11 +249,6 @@ static uint32_t _PyOpcode_Jump[8] = {
 #define NB_INPLACE_TRUE_DIVIDE           24
 #define NB_INPLACE_XOR                   25
 
-static const uint8_t _PyOpcode_InlineCacheEntries[256] = {
-    [UNPACK_SEQUENCE] = 1,
-    [BINARY_OP] = 1,
-};
-
 #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT)
 
 /* Reserve some bytecodes for internal use in the compiler.