GH-91432: Specialize FOR_ITER (GH-91713)

* Adds FOR_ITER_LIST and FOR_ITER_RANGE specializations.

* Adds _PyLong_AssignValue() internal function to avoid temporary boxing of ints.

6 files changed, 104 insertions, 60 deletions

diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 7e21c5a..71dd3b3 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -85,6 +85,12 @@ typedef struct {
 
 #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
 
+typedef struct {
+    _Py_CODEUNIT counter;
+} _PyForIterCache;
+
+#define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
+
 #define QUICKENING_WARMUP_DELAY 8
 
 /* We want to compare to zero for efficiency, so we offset values accordingly */
@@ -243,6 +249,7 @@ extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
                                      _Py_CODEUNIT *instr, int oparg);
 extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr,
                                           int oparg);
+extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr);
 
 /* Deallocator function for static codeobjects used in deepfreeze.py */
 extern void _PyStaticCode_Dealloc(PyCodeObject *co);
diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h
index 860dce1..7721f6c 100644
--- a/Include/internal/pycore_list.h
+++ b/Include/internal/pycore_list.h
@@ -56,6 +56,12 @@ _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem)
     return _PyList_AppendTakeRefListResize(self, newitem);
 }
 
+typedef struct {
+    PyObject_HEAD
+    Py_ssize_t it_index;
+    PyListObject *it_seq; /* Set to NULL when iterator is exhausted */
+} _PyListIterObject;
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h
index a337624..67dd5c3 100644
--- a/Include/internal/pycore_long.h
+++ b/Include/internal/pycore_long.h
@@ -47,6 +47,8 @@ PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right);
 PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right);
 PyObject *_PyLong_Subtract(PyLongObject *left, PyLongObject *right);
 
+int _PyLong_AssignValue(PyObject **target, Py_ssize_t value);
+
 /* Used by Python/mystrtoul.c, _PyBytes_FromHex(),
    _PyBytes_DecodeEscape(), etc. */
 PyAPI_DATA(unsigned char) _PyLong_DigitValue[256];
diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h
index f0edbc7..b9195f5 100644
--- a/Include/internal/pycore_opcode.h
+++ b/Include/internal/pycore_opcode.h
@@ -44,6 +44,7 @@ const uint8_t _PyOpcode_Caches[256] = {
     [BINARY_SUBSCR] = 4,
     [STORE_SUBSCR] = 1,
     [UNPACK_SEQUENCE] = 1,
+    [FOR_ITER] = 1,
     [STORE_ATTR] = 4,
     [LOAD_ATTR] = 9,
     [COMPARE_OP] = 2,
@@ -123,6 +124,9 @@ const uint8_t _PyOpcode_Deopt[256] = {
     [EXTENDED_ARG_QUICK] = EXTENDED_ARG,
     [FORMAT_VALUE] = FORMAT_VALUE,
     [FOR_ITER] = FOR_ITER,
+    [FOR_ITER_ADAPTIVE] = FOR_ITER,
+    [FOR_ITER_LIST] = FOR_ITER,
+    [FOR_ITER_RANGE] = FOR_ITER,
     [GET_AITER] = GET_AITER,
     [GET_ANEXT] = GET_ANEXT,
     [GET_AWAITABLE] = GET_AWAITABLE,
@@ -304,6 +308,9 @@ const uint8_t _PyOpcode_Original[256] = {
     [EXTENDED_ARG_QUICK] = EXTENDED_ARG_QUICK,
     [FORMAT_VALUE] = FORMAT_VALUE,
     [FOR_ITER] = FOR_ITER,
+    [FOR_ITER_ADAPTIVE] = FOR_ITER,
+    [FOR_ITER_LIST] = FOR_ITER,
+    [FOR_ITER_RANGE] = FOR_ITER,
     [GET_AITER] = GET_AITER,
     [GET_ANEXT] = GET_ANEXT,
     [GET_AWAITABLE] = GET_AWAITABLE,
@@ -476,34 +483,34 @@ static const char *const _PyOpcode_OpName[256] = {
     [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP",
     [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP",
     [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK",
-    [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK",
+    [FOR_ITER_ADAPTIVE] = "FOR_ITER_ADAPTIVE",
     [STORE_SUBSCR] = "STORE_SUBSCR",
     [DELETE_SUBSCR] = "DELETE_SUBSCR",
+    [FOR_ITER_LIST] = "FOR_ITER_LIST",
+    [FOR_ITER_RANGE] = "FOR_ITER_RANGE",
+    [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK",
     [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE",
     [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS",
     [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE",
-    [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE",
-    [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY",
-    [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT",
     [GET_ITER] = "GET_ITER",
     [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER",
     [PRINT_EXPR] = "PRINT_EXPR",
     [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS",
-    [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT",
-    [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT",
+    [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE",
+    [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY",
     [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR",
     [RETURN_GENERATOR] = "RETURN_GENERATOR",
+    [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT",
+    [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT",
+    [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT",
     [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT",
     [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT",
     [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES",
-    [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
-    [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST",
-    [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
     [LIST_TO_TUPLE] = "LIST_TO_TUPLE",
     [RETURN_VALUE] = "RETURN_VALUE",
     [IMPORT_STAR] = "IMPORT_STAR",
     [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS",
-    [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE",
+    [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
     [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP",
     [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR",
     [POP_EXCEPT] = "POP_EXCEPT",
@@ -530,7 +537,7 @@ static const char *const _PyOpcode_OpName[256] = {
     [JUMP_FORWARD] = "JUMP_FORWARD",
     [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP",
     [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP",
-    [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN",
+    [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST",
     [POP_JUMP_FORWARD_IF_FALSE] = "POP_JUMP_FORWARD_IF_FALSE",
     [POP_JUMP_FORWARD_IF_TRUE] = "POP_JUMP_FORWARD_IF_TRUE",
     [LOAD_GLOBAL] = "LOAD_GLOBAL",
@@ -538,7 +545,7 @@ static const char *const _PyOpcode_OpName[256] = {
     [CONTAINS_OP] = "CONTAINS_OP",
     [RERAISE] = "RERAISE",
     [COPY] = "COPY",
-    [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE",
+    [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
     [BINARY_OP] = "BINARY_OP",
     [SEND] = "SEND",
     [LOAD_FAST] = "LOAD_FAST",
@@ -558,9 +565,9 @@ static const char *const _PyOpcode_OpName[256] = {
     [STORE_DEREF] = "STORE_DEREF",
     [DELETE_DEREF] = "DELETE_DEREF",
     [JUMP_BACKWARD] = "JUMP_BACKWARD",
-    [RESUME_QUICK] = "RESUME_QUICK",
+    [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE",
     [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
-    [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE",
+    [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN",
     [EXTENDED_ARG] = "EXTENDED_ARG",
     [LIST_APPEND] = "LIST_APPEND",
     [SET_ADD] = "SET_ADD",
@@ -570,34 +577,34 @@ static const char *const _PyOpcode_OpName[256] = {
     [YIELD_VALUE] = "YIELD_VALUE",
     [RESUME] = "RESUME",
     [MATCH_CLASS] = "MATCH_CLASS",
-    [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE",
-    [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT",
+    [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE",
+    [RESUME_QUICK] = "RESUME_QUICK",
     [FORMAT_VALUE] = "FORMAT_VALUE",
     [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP",
     [BUILD_STRING] = "BUILD_STRING",
+    [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE",
+    [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE",
+    [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT",
     [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT",
-    [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST",
-    [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST",
-    [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE",
     [LIST_EXTEND] = "LIST_EXTEND",
     [SET_UPDATE] = "SET_UPDATE",
     [DICT_MERGE] = "DICT_MERGE",
     [DICT_UPDATE] = "DICT_UPDATE",
+    [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST",
+    [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST",
+    [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE",
     [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT",
     [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT",
-    [UNPACK_SEQUENCE_ADAPTIVE] = "UNPACK_SEQUENCE_ADAPTIVE",
-    [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST",
-    [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE",
     [CALL] = "CALL",
     [KW_NAMES] = "KW_NAMES",
     [POP_JUMP_BACKWARD_IF_NOT_NONE] = "POP_JUMP_BACKWARD_IF_NOT_NONE",
     [POP_JUMP_BACKWARD_IF_NONE] = "POP_JUMP_BACKWARD_IF_NONE",
     [POP_JUMP_BACKWARD_IF_FALSE] = "POP_JUMP_BACKWARD_IF_FALSE",
     [POP_JUMP_BACKWARD_IF_TRUE] = "POP_JUMP_BACKWARD_IF_TRUE",
+    [UNPACK_SEQUENCE_ADAPTIVE] = "UNPACK_SEQUENCE_ADAPTIVE",
+    [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST",
+    [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE",
     [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE",
-    [178] = "<178>",
-    [179] = "<179>",
-    [180] = "<180>",
     [181] = "<181>",
     [182] = "<182>",
     [183] = "<183>",
@@ -677,9 +684,6 @@ static const char *const _PyOpcode_OpName[256] = {
 #endif
 
 #define EXTRA_CASES \
-    case 178: \
-    case 179: \
-    case 180: \
     case 181: \
     case 182: \
     case 183: \
diff --git a/Include/internal/pycore_range.h b/Include/internal/pycore_range.h
new file mode 100644
index 0000000..809e89a
--- /dev/null
+++ b/Include/internal/pycore_range.h
@@ -0,0 +1,22 @@
+#ifndef Py_INTERNAL_RANGE_H
+#define Py_INTERNAL_RANGE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    long index;
+    long start;
+    long step;
+    long len;
+} _PyRangeIterObject;
+
+#ifdef __cplusplus
+}
+#endif
+#endif   /* !Py_INTERNAL_RANGE_H */
diff --git a/Include/opcode.h b/Include/opcode.h
index b58b498..7a22d52 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -155,38 +155,41 @@ extern "C" {
 #define COMPARE_OP_INT_JUMP                     56
 #define COMPARE_OP_STR_JUMP                     57
 #define EXTENDED_ARG_QUICK                      58
-#define JUMP_BACKWARD_QUICK                     59
-#define LOAD_ATTR_ADAPTIVE                      62
-#define LOAD_ATTR_CLASS                         63
-#define LOAD_ATTR_INSTANCE_VALUE                64
-#define LOAD_ATTR_MODULE                        65
-#define LOAD_ATTR_PROPERTY                      66
-#define LOAD_ATTR_SLOT                          67
-#define LOAD_ATTR_WITH_HINT                     72
-#define LOAD_ATTR_METHOD_LAZY_DICT              73
-#define LOAD_ATTR_METHOD_NO_DICT                76
-#define LOAD_ATTR_METHOD_WITH_DICT              77
-#define LOAD_ATTR_METHOD_WITH_VALUES            78
-#define LOAD_CONST__LOAD_FAST                   79
-#define LOAD_FAST__LOAD_CONST                   80
-#define LOAD_FAST__LOAD_FAST                    81
-#define LOAD_GLOBAL_ADAPTIVE                    86
-#define LOAD_GLOBAL_BUILTIN                    113
-#define LOAD_GLOBAL_MODULE                     121
-#define RESUME_QUICK                           141
-#define STORE_ATTR_ADAPTIVE                    143
-#define STORE_ATTR_INSTANCE_VALUE              153
-#define STORE_ATTR_SLOT                        154
-#define STORE_ATTR_WITH_HINT                   158
-#define STORE_FAST__LOAD_FAST                  159
-#define STORE_FAST__STORE_FAST                 160
-#define STORE_SUBSCR_ADAPTIVE                  161
-#define STORE_SUBSCR_DICT                      166
-#define STORE_SUBSCR_LIST_INT                  167
-#define UNPACK_SEQUENCE_ADAPTIVE               168
-#define UNPACK_SEQUENCE_LIST                   169
-#define UNPACK_SEQUENCE_TUPLE                  170
-#define UNPACK_SEQUENCE_TWO_TUPLE              177
+#define FOR_ITER_ADAPTIVE                       59
+#define FOR_ITER_LIST                           62
+#define FOR_ITER_RANGE                          63
+#define JUMP_BACKWARD_QUICK                     64
+#define LOAD_ATTR_ADAPTIVE                      65
+#define LOAD_ATTR_CLASS                         66
+#define LOAD_ATTR_INSTANCE_VALUE                67
+#define LOAD_ATTR_MODULE                        72
+#define LOAD_ATTR_PROPERTY                      73
+#define LOAD_ATTR_SLOT                          76
+#define LOAD_ATTR_WITH_HINT                     77
+#define LOAD_ATTR_METHOD_LAZY_DICT              78
+#define LOAD_ATTR_METHOD_NO_DICT                79
+#define LOAD_ATTR_METHOD_WITH_DICT              80
+#define LOAD_ATTR_METHOD_WITH_VALUES            81
+#define LOAD_CONST__LOAD_FAST                   86
+#define LOAD_FAST__LOAD_CONST                  113
+#define LOAD_FAST__LOAD_FAST                   121
+#define LOAD_GLOBAL_ADAPTIVE                   141
+#define LOAD_GLOBAL_BUILTIN                    143
+#define LOAD_GLOBAL_MODULE                     153
+#define RESUME_QUICK                           154
+#define STORE_ATTR_ADAPTIVE                    158
+#define STORE_ATTR_INSTANCE_VALUE              159
+#define STORE_ATTR_SLOT                        160
+#define STORE_ATTR_WITH_HINT                   161
+#define STORE_FAST__LOAD_FAST                  166
+#define STORE_FAST__STORE_FAST                 167
+#define STORE_SUBSCR_ADAPTIVE                  168
+#define STORE_SUBSCR_DICT                      169
+#define STORE_SUBSCR_LIST_INT                  170
+#define UNPACK_SEQUENCE_ADAPTIVE               177
+#define UNPACK_SEQUENCE_LIST                   178
+#define UNPACK_SEQUENCE_TUPLE                  179
+#define UNPACK_SEQUENCE_TWO_TUPLE              180
 #define DO_TRACING                             255
 
 #define HAS_CONST(op) (false\