bpo-46823: Implement LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE superinstruction (GH-31484)

3 files changed, 88 insertions, 4 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index 8077570..f3bdaf1 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -3444,6 +3444,34 @@ handle_eval_breaker:
             }
         }
 
+        TARGET(LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE) {
+            assert(cframe.use_tracing == 0);
+            PyObject *owner = GETLOCAL(oparg); // borrowed
+            if (owner == NULL) {
+                goto unbound_local_error;
+            }
+            // GET_CACHE(), but for the following opcode
+            assert(_Py_OPCODE(*next_instr) == LOAD_ATTR_INSTANCE_VALUE);
+            SpecializedCacheEntry *caches = _GetSpecializedCacheEntryForInstruction(
+                first_instr, INSTR_OFFSET() + 1, _Py_OPARG(*next_instr));
+            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
+            assert(cache0->version != 0);
+            PyTypeObject *tp = Py_TYPE(owner);
+            // These DEOPT_IF miss branches do PUSH(Py_NewRef(owner)).
+            DEOPT_IF(tp->tp_version_tag != cache0->version,
+                     LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE);
+            assert(tp->tp_dictoffset < 0);
+            assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            PyDictValues *values = *_PyObject_ValuesPointer(owner);
+            DEOPT_IF(values == NULL, LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE);
+            PyObject *res = values->values[cache0->index];
+            DEOPT_IF(res == NULL, LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE);
+            STAT_INC(LOAD_ATTR, hit);
+            PUSH(Py_NewRef(res));
+            next_instr++;
+            NOTRACE_DISPATCH();
+        }
+
         TARGET(LOAD_ATTR_INSTANCE_VALUE) {
             assert(cframe.use_tracing == 0);
             PyObject *owner = TOP();
@@ -3452,13 +3480,13 @@ handle_eval_breaker:
             SpecializedCacheEntry *caches = GET_CACHE();
             _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
             assert(cache0->version != 0);
-            DEOPT_IF(tp->tp_version_tag != cache0->version, LOAD_ATTR);
+            DEOPT_IF(tp->tp_version_tag != cache0->version, LOAD_ATTR_INSTANCE_VALUE);
             assert(tp->tp_dictoffset < 0);
             assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
             PyDictValues *values = *_PyObject_ValuesPointer(owner);
-            DEOPT_IF(values == NULL, LOAD_ATTR);
+            DEOPT_IF(values == NULL, LOAD_ATTR_INSTANCE_VALUE);
             res = values->values[cache0->index];
-            DEOPT_IF(res == NULL, LOAD_ATTR);
+            DEOPT_IF(res == NULL, LOAD_ATTR_INSTANCE_VALUE);
             STAT_INC(LOAD_ATTR, hit);
             Py_INCREF(res);
             SET_TOP(res);
@@ -5515,6 +5543,52 @@ MISS_WITH_CACHE(BINARY_SUBSCR)
 MISS_WITH_CACHE(UNPACK_SEQUENCE)
 MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
 
+LOAD_ATTR_INSTANCE_VALUE_miss:
+        {
+            // Special-cased so that if LOAD_ATTR_INSTANCE_VALUE
+            // gets replaced, then any preceeding
+            // LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE gets replaced as well
+            STAT_INC(LOAD_ATTR_INSTANCE_VALUE, miss);
+            STAT_INC(LOAD_ATTR, miss);
+            _PyAdaptiveEntry *cache = &GET_CACHE()->adaptive;
+            cache->counter--;
+            if (cache->counter == 0) {
+                next_instr[-1] = _Py_MAKECODEUNIT(LOAD_ATTR_ADAPTIVE, _Py_OPARG(next_instr[-1]));
+                if (_Py_OPCODE(next_instr[-2]) == LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE) {
+                    next_instr[-2] = _Py_MAKECODEUNIT(LOAD_FAST, _Py_OPARG(next_instr[-2]));
+                    if (_Py_OPCODE(next_instr[-3]) == LOAD_FAST) {
+                        next_instr[-3] =  _Py_MAKECODEUNIT(LOAD_FAST__LOAD_FAST, _Py_OPARG(next_instr[-3]));
+                    }
+                }
+                STAT_INC(LOAD_ATTR, deopt);
+                cache_backoff(cache);
+            }
+            oparg = cache->original_oparg;
+            JUMP_TO_INSTRUCTION(LOAD_ATTR);
+        }
+
+LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE_miss:
+        {
+            // This is special-cased because we have a superinstruction
+            // that includes a specialized instruction.
+            // If the specialized portion misses, carry out
+            // the first instruction, then perform a miss
+            // for the second instruction as usual.
+
+            // Do LOAD_FAST
+            {
+                PyObject *value = GETLOCAL(oparg);
+                assert(value != NULL); // Already checked if unbound
+                Py_INCREF(value);
+                PUSH(value);
+                NEXTOPARG();
+                next_instr++;
+            }
+
+            // Now we are in the correct state for LOAD_ATTR
+            goto LOAD_ATTR_INSTANCE_VALUE_miss;
+        }
+
 binary_subscr_dict_error:
         {
             PyObject *sub = POP();
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index cf5bdc7..fac671e 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -173,7 +173,7 @@ static void *opcode_targets[256] = {
     &&TARGET_CALL,
     &&TARGET_KW_NAMES,
     &&TARGET_STORE_FAST__STORE_FAST,
-    &&_unknown_opcode,
+    &&TARGET_LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE,
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,
diff --git a/Python/specialize.c b/Python/specialize.c
index 816cca1..1641766 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -889,6 +889,16 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp
         return -1;
     }
     if (err) {
+        if (_Py_OPCODE(instr[0]) == LOAD_ATTR_INSTANCE_VALUE) {
+            // Note: instr[-1] exists because there's something on the stack,
+            // and instr[-2] exists because there's at least a RESUME as well.
+            if (_Py_OPCODE(instr[-1]) == LOAD_FAST) {
+                instr[-1] = _Py_MAKECODEUNIT(LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE, _Py_OPARG(instr[-1]));
+                if (_Py_OPCODE(instr[-2]) == LOAD_FAST__LOAD_FAST) {
+                    instr[-2] = _Py_MAKECODEUNIT(LOAD_FAST, _Py_OPARG(instr[-2]));
+                }
+            }
+        }
         goto success;
     }
 fail: