GH-115457: Support splitting and replication of micro ops. (GH-115558)

1 files changed, 383 insertions, 22 deletions

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index a18284d..b46885e 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -37,6 +37,102 @@
             break;
         }
 
+        case _LOAD_FAST_0: {
+            PyObject *value;
+            oparg = 0;
+            assert(oparg == CURRENT_OPARG());
+            value = GETLOCAL(oparg);
+            assert(value != NULL);
+            Py_INCREF(value);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            break;
+        }
+
+        case _LOAD_FAST_1: {
+            PyObject *value;
+            oparg = 1;
+            assert(oparg == CURRENT_OPARG());
+            value = GETLOCAL(oparg);
+            assert(value != NULL);
+            Py_INCREF(value);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            break;
+        }
+
+        case _LOAD_FAST_2: {
+            PyObject *value;
+            oparg = 2;
+            assert(oparg == CURRENT_OPARG());
+            value = GETLOCAL(oparg);
+            assert(value != NULL);
+            Py_INCREF(value);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            break;
+        }
+
+        case _LOAD_FAST_3: {
+            PyObject *value;
+            oparg = 3;
+            assert(oparg == CURRENT_OPARG());
+            value = GETLOCAL(oparg);
+            assert(value != NULL);
+            Py_INCREF(value);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            break;
+        }
+
+        case _LOAD_FAST_4: {
+            PyObject *value;
+            oparg = 4;
+            assert(oparg == CURRENT_OPARG());
+            value = GETLOCAL(oparg);
+            assert(value != NULL);
+            Py_INCREF(value);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            break;
+        }
+
+        case _LOAD_FAST_5: {
+            PyObject *value;
+            oparg = 5;
+            assert(oparg == CURRENT_OPARG());
+            value = GETLOCAL(oparg);
+            assert(value != NULL);
+            Py_INCREF(value);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            break;
+        }
+
+        case _LOAD_FAST_6: {
+            PyObject *value;
+            oparg = 6;
+            assert(oparg == CURRENT_OPARG());
+            value = GETLOCAL(oparg);
+            assert(value != NULL);
+            Py_INCREF(value);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            break;
+        }
+
+        case _LOAD_FAST_7: {
+            PyObject *value;
+            oparg = 7;
+            assert(oparg == CURRENT_OPARG());
+            value = GETLOCAL(oparg);
+            assert(value != NULL);
+            Py_INCREF(value);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            break;
+        }
+
         case _LOAD_FAST: {
             PyObject *value;
             oparg = CURRENT_OPARG();
@@ -69,6 +165,86 @@
             break;
         }
 
+        case _STORE_FAST_0: {
+            PyObject *value;
+            oparg = 0;
+            assert(oparg == CURRENT_OPARG());
+            value = stack_pointer[-1];
+            SETLOCAL(oparg, value);
+            stack_pointer += -1;
+            break;
+        }
+
+        case _STORE_FAST_1: {
+            PyObject *value;
+            oparg = 1;
+            assert(oparg == CURRENT_OPARG());
+            value = stack_pointer[-1];
+            SETLOCAL(oparg, value);
+            stack_pointer += -1;
+            break;
+        }
+
+        case _STORE_FAST_2: {
+            PyObject *value;
+            oparg = 2;
+            assert(oparg == CURRENT_OPARG());
+            value = stack_pointer[-1];
+            SETLOCAL(oparg, value);
+            stack_pointer += -1;
+            break;
+        }
+
+        case _STORE_FAST_3: {
+            PyObject *value;
+            oparg = 3;
+            assert(oparg == CURRENT_OPARG());
+            value = stack_pointer[-1];
+            SETLOCAL(oparg, value);
+            stack_pointer += -1;
+            break;
+        }
+
+        case _STORE_FAST_4: {
+            PyObject *value;
+            oparg = 4;
+            assert(oparg == CURRENT_OPARG());
+            value = stack_pointer[-1];
+            SETLOCAL(oparg, value);
+            stack_pointer += -1;
+            break;
+        }
+
+        case _STORE_FAST_5: {
+            PyObject *value;
+            oparg = 5;
+            assert(oparg == CURRENT_OPARG());
+            value = stack_pointer[-1];
+            SETLOCAL(oparg, value);
+            stack_pointer += -1;
+            break;
+        }
+
+        case _STORE_FAST_6: {
+            PyObject *value;
+            oparg = 6;
+            assert(oparg == CURRENT_OPARG());
+            value = stack_pointer[-1];
+            SETLOCAL(oparg, value);
+            stack_pointer += -1;
+            break;
+        }
+
+        case _STORE_FAST_7: {
+            PyObject *value;
+            oparg = 7;
+            assert(oparg == CURRENT_OPARG());
+            value = stack_pointer[-1];
+            SETLOCAL(oparg, value);
+            stack_pointer += -1;
+            break;
+        }
+
         case _STORE_FAST: {
             PyObject *value;
             oparg = CURRENT_OPARG();
@@ -1534,7 +1710,7 @@
             Py_DECREF(self);
             if (attr == NULL) goto pop_3_error_tier_two;
             stack_pointer[-3] = attr;
-            stack_pointer += -2 + ((0) ? 1 : 0);
+            stack_pointer += -2;
             break;
         }
 
@@ -1637,11 +1813,11 @@
             break;
         }
 
-        case _LOAD_ATTR_INSTANCE_VALUE: {
+        case _LOAD_ATTR_INSTANCE_VALUE_0: {
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
-            oparg = CURRENT_OPARG();
+            (void)null;
             owner = stack_pointer[-1];
             uint16_t index = (uint16_t)CURRENT_OPERAND();
             PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
@@ -1652,11 +1828,31 @@
             null = NULL;
             Py_DECREF(owner);
             stack_pointer[-1] = attr;
-            if (oparg & 1) stack_pointer[0] = null;
-            stack_pointer += (oparg & 1);
             break;
         }
 
+        case _LOAD_ATTR_INSTANCE_VALUE_1: {
+            PyObject *owner;
+            PyObject *attr;
+            PyObject *null = NULL;
+            (void)null;
+            owner = stack_pointer[-1];
+            uint16_t index = (uint16_t)CURRENT_OPERAND();
+            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            attr = _PyDictOrValues_GetValues(dorv)->values[index];
+            if (attr == NULL) goto deoptimize;
+            STAT_INC(LOAD_ATTR, hit);
+            Py_INCREF(attr);
+            null = NULL;
+            Py_DECREF(owner);
+            stack_pointer[-1] = attr;
+            stack_pointer[0] = null;
+            stack_pointer += 1;
+            break;
+        }
+
+        /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */
+
         case _CHECK_ATTR_MODULE: {
             PyObject *owner;
             owner = stack_pointer[-1];
@@ -1735,11 +1931,11 @@
             break;
         }
 
-        case _LOAD_ATTR_SLOT: {
+        case _LOAD_ATTR_SLOT_0: {
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
-            oparg = CURRENT_OPARG();
+            (void)null;
             owner = stack_pointer[-1];
             uint16_t index = (uint16_t)CURRENT_OPERAND();
             char *addr = (char *)owner + index;
@@ -1750,11 +1946,31 @@
             null = NULL;
             Py_DECREF(owner);
             stack_pointer[-1] = attr;
-            if (oparg & 1) stack_pointer[0] = null;
-            stack_pointer += (oparg & 1);
             break;
         }
 
+        case _LOAD_ATTR_SLOT_1: {
+            PyObject *owner;
+            PyObject *attr;
+            PyObject *null = NULL;
+            (void)null;
+            owner = stack_pointer[-1];
+            uint16_t index = (uint16_t)CURRENT_OPERAND();
+            char *addr = (char *)owner + index;
+            attr = *(PyObject **)addr;
+            if (attr == NULL) goto deoptimize;
+            STAT_INC(LOAD_ATTR, hit);
+            Py_INCREF(attr);
+            null = NULL;
+            Py_DECREF(owner);
+            stack_pointer[-1] = attr;
+            stack_pointer[0] = null;
+            stack_pointer += 1;
+            break;
+        }
+
+        /* _LOAD_ATTR_SLOT is split on (oparg & 1) */
+
         case _CHECK_ATTR_CLASS: {
             PyObject *owner;
             owner = stack_pointer[-1];
@@ -1765,11 +1981,11 @@
             break;
         }
 
-        case _LOAD_ATTR_CLASS: {
+        case _LOAD_ATTR_CLASS_0: {
             PyObject *owner;
             PyObject *attr;
             PyObject *null = NULL;
-            oparg = CURRENT_OPARG();
+            (void)null;
             owner = stack_pointer[-1];
             PyObject *descr = (PyObject *)CURRENT_OPERAND();
             STAT_INC(LOAD_ATTR, hit);
@@ -1778,11 +1994,29 @@
             null = NULL;
             Py_DECREF(owner);
             stack_pointer[-1] = attr;
-            if (oparg & 1) stack_pointer[0] = null;
-            stack_pointer += (oparg & 1);
             break;
         }
 
+        case _LOAD_ATTR_CLASS_1: {
+            PyObject *owner;
+            PyObject *attr;
+            PyObject *null = NULL;
+            (void)null;
+            owner = stack_pointer[-1];
+            PyObject *descr = (PyObject *)CURRENT_OPERAND();
+            STAT_INC(LOAD_ATTR, hit);
+            assert(descr != NULL);
+            attr = Py_NewRef(descr);
+            null = NULL;
+            Py_DECREF(owner);
+            stack_pointer[-1] = attr;
+            stack_pointer[0] = null;
+            stack_pointer += 1;
+            break;
+        }
+
+        /* _LOAD_ATTR_CLASS is split on (oparg & 1) */
+
         /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */
 
         /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */
@@ -2464,8 +2698,8 @@
             assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR));
             self = owner;
             stack_pointer[-1] = attr;
-            if (1) stack_pointer[0] = self;
-            stack_pointer += ((1) ? 1 : 0);
+            stack_pointer[0] = self;
+            stack_pointer += 1;
             break;
         }
 
@@ -2484,8 +2718,8 @@
             attr = Py_NewRef(descr);
             self = owner;
             stack_pointer[-1] = attr;
-            if (1) stack_pointer[0] = self;
-            stack_pointer += ((1) ? 1 : 0);
+            stack_pointer[0] = self;
+            stack_pointer += 1;
             break;
         }
 
@@ -2501,7 +2735,6 @@
             Py_DECREF(owner);
             attr = Py_NewRef(descr);
             stack_pointer[-1] = attr;
-            stack_pointer += ((0) ? 1 : 0);
             break;
         }
 
@@ -2518,7 +2751,6 @@
             Py_DECREF(owner);
             attr = Py_NewRef(descr);
             stack_pointer[-1] = attr;
-            stack_pointer += ((0) ? 1 : 0);
             break;
         }
 
@@ -2547,8 +2779,8 @@
             attr = Py_NewRef(descr);
             self = owner;
             stack_pointer[-1] = attr;
-            if (1) stack_pointer[0] = self;
-            stack_pointer += ((1) ? 1 : 0);
+            stack_pointer[0] = self;
+            stack_pointer += 1;
             break;
         }
 
@@ -2615,6 +2847,136 @@
             break;
         }
 
+        case _INIT_CALL_PY_EXACT_ARGS_0: {
+            PyObject **args;
+            PyObject *self_or_null;
+            PyObject *callable;
+            _PyInterpreterFrame *new_frame;
+            oparg = 0;
+            assert(oparg == CURRENT_OPARG());
+            args = &stack_pointer[-oparg];
+            self_or_null = stack_pointer[-1 - oparg];
+            callable = stack_pointer[-2 - oparg];
+            int argcount = oparg;
+            if (self_or_null != NULL) {
+                args--;
+                argcount++;
+            }
+            STAT_INC(CALL, hit);
+            PyFunctionObject *func = (PyFunctionObject *)callable;
+            new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
+            for (int i = 0; i < argcount; i++) {
+                new_frame->localsplus[i] = args[i];
+            }
+            stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+            stack_pointer += -1 - oparg;
+            break;
+        }
+
+        case _INIT_CALL_PY_EXACT_ARGS_1: {
+            PyObject **args;
+            PyObject *self_or_null;
+            PyObject *callable;
+            _PyInterpreterFrame *new_frame;
+            oparg = 1;
+            assert(oparg == CURRENT_OPARG());
+            args = &stack_pointer[-oparg];
+            self_or_null = stack_pointer[-1 - oparg];
+            callable = stack_pointer[-2 - oparg];
+            int argcount = oparg;
+            if (self_or_null != NULL) {
+                args--;
+                argcount++;
+            }
+            STAT_INC(CALL, hit);
+            PyFunctionObject *func = (PyFunctionObject *)callable;
+            new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
+            for (int i = 0; i < argcount; i++) {
+                new_frame->localsplus[i] = args[i];
+            }
+            stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+            stack_pointer += -1 - oparg;
+            break;
+        }
+
+        case _INIT_CALL_PY_EXACT_ARGS_2: {
+            PyObject **args;
+            PyObject *self_or_null;
+            PyObject *callable;
+            _PyInterpreterFrame *new_frame;
+            oparg = 2;
+            assert(oparg == CURRENT_OPARG());
+            args = &stack_pointer[-oparg];
+            self_or_null = stack_pointer[-1 - oparg];
+            callable = stack_pointer[-2 - oparg];
+            int argcount = oparg;
+            if (self_or_null != NULL) {
+                args--;
+                argcount++;
+            }
+            STAT_INC(CALL, hit);
+            PyFunctionObject *func = (PyFunctionObject *)callable;
+            new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
+            for (int i = 0; i < argcount; i++) {
+                new_frame->localsplus[i] = args[i];
+            }
+            stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+            stack_pointer += -1 - oparg;
+            break;
+        }
+
+        case _INIT_CALL_PY_EXACT_ARGS_3: {
+            PyObject **args;
+            PyObject *self_or_null;
+            PyObject *callable;
+            _PyInterpreterFrame *new_frame;
+            oparg = 3;
+            assert(oparg == CURRENT_OPARG());
+            args = &stack_pointer[-oparg];
+            self_or_null = stack_pointer[-1 - oparg];
+            callable = stack_pointer[-2 - oparg];
+            int argcount = oparg;
+            if (self_or_null != NULL) {
+                args--;
+                argcount++;
+            }
+            STAT_INC(CALL, hit);
+            PyFunctionObject *func = (PyFunctionObject *)callable;
+            new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
+            for (int i = 0; i < argcount; i++) {
+                new_frame->localsplus[i] = args[i];
+            }
+            stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+            stack_pointer += -1 - oparg;
+            break;
+        }
+
+        case _INIT_CALL_PY_EXACT_ARGS_4: {
+            PyObject **args;
+            PyObject *self_or_null;
+            PyObject *callable;
+            _PyInterpreterFrame *new_frame;
+            oparg = 4;
+            assert(oparg == CURRENT_OPARG());
+            args = &stack_pointer[-oparg];
+            self_or_null = stack_pointer[-1 - oparg];
+            callable = stack_pointer[-2 - oparg];
+            int argcount = oparg;
+            if (self_or_null != NULL) {
+                args--;
+                argcount++;
+            }
+            STAT_INC(CALL, hit);
+            PyFunctionObject *func = (PyFunctionObject *)callable;
+            new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
+            for (int i = 0; i < argcount; i++) {
+                new_frame->localsplus[i] = args[i];
+            }
+            stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+            stack_pointer += -1 - oparg;
+            break;
+        }
+
         case _INIT_CALL_PY_EXACT_ARGS: {
             PyObject **args;
             PyObject *self_or_null;
@@ -2660,7 +3022,6 @@
                 goto exit_unwind;
             }
             #endif
-            stack_pointer += ((0) ? 1 : 0);
             break;
         }