diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bytecodes.c | 32 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 227 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 72 | ||||
| -rw-r--r-- | Python/optimizer_bytecodes.c | 23 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 306 |
5 files changed, 204 insertions, 456 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cf5bf0b..8420d09 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -614,7 +614,7 @@ dummy_func( EXIT_IF(!_PyLong_IsCompact((PyLongObject *)value_o)); } - pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); @@ -624,12 +624,12 @@ dummy_func( STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); EXIT_IF(PyStackRef_IsNull(res)); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; INPUTS_DEAD(); } - pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { + pure op(_BINARY_OP_ADD_INT, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); @@ -639,12 +639,12 @@ dummy_func( STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); EXIT_IF(PyStackRef_IsNull(res)); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; INPUTS_DEAD(); } - pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); @@ -654,19 +654,19 @@ dummy_func( STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); EXIT_IF(PyStackRef_IsNull(res)); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; INPUTS_DEAD(); } macro(BINARY_OP_MULTIPLY_INT) = - _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT; + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT + _POP_TOP_INT + _POP_TOP_INT; macro(BINARY_OP_ADD_INT) = - _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT; + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT + _POP_TOP_INT + _POP_TOP_INT; macro(BINARY_OP_SUBTRACT_INT) = - _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT; + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT; op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -2701,7 +2701,7 @@ dummy_func( _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/1 + _COMPARE_OP_FLOAT; macro(COMPARE_OP_INT) = - _GUARD_TOS_INT + _GUARD_NOS_INT + unused/1 + _COMPARE_OP_INT; + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/1 + _COMPARE_OP_INT + _POP_TOP_INT + _POP_TOP_INT; macro(COMPARE_OP_STR) = _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/1 + _COMPARE_OP_STR; @@ -2724,7 +2724,7 @@ dummy_func( } // Similar to COMPARE_OP_FLOAT - op(_COMPARE_OP_INT, (left, right -- res)) { + op(_COMPARE_OP_INT, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -2737,9 +2737,9 @@ dummy_func( Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; DEAD(left); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); DEAD(right); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d8f33bc..b4d4684 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3353,12 +3353,14 @@ break; } - case _BINARY_OP_MULTIPLY_INT_r01: { + case _BINARY_OP_MULTIPLY_INT_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -3373,22 +3375,26 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _BINARY_OP_MULTIPLY_INT_r11: { + case _BINARY_OP_MULTIPLY_INT_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; right = _stack_item_0; left = stack_pointer[-1]; @@ -3405,22 +3411,26 @@ SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _BINARY_OP_MULTIPLY_INT_r21: { + case _BINARY_OP_MULTIPLY_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; @@ -3439,55 +3449,24 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - right = _stack_item_2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _BINARY_OP_ADD_INT_r01: { + case _BINARY_OP_ADD_INT_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -3502,22 +3481,26 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _BINARY_OP_ADD_INT_r11: { + case _BINARY_OP_ADD_INT_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; right = _stack_item_0; left = stack_pointer[-1]; @@ -3534,22 +3517,26 @@ SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _BINARY_OP_ADD_INT_r21: { + case _BINARY_OP_ADD_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; @@ -3568,55 +3555,24 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - right = _stack_item_2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _BINARY_OP_SUBTRACT_INT_r01: { + case _BINARY_OP_SUBTRACT_INT_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -3631,22 +3587,26 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _BINARY_OP_SUBTRACT_INT_r11: { + case _BINARY_OP_SUBTRACT_INT_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; right = _stack_item_0; left = stack_pointer[-1]; @@ -3663,22 +3623,26 @@ SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _BINARY_OP_SUBTRACT_INT_r21: { + case _BINARY_OP_SUBTRACT_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; @@ -3697,45 +3661,12 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - right = _stack_item_2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -9054,12 +8985,14 @@ break; } - case _COMPARE_OP_INT_r21: { + case _COMPARE_OP_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); @@ -9075,13 +9008,13 @@ Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + l = left; + r = right; res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1e7778c..3b8ad45 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -170,6 +170,8 @@ _PyStackRef left; _PyStackRef right; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; // _GUARD_TOS_INT { value = stack_pointer[-1]; @@ -206,8 +208,20 @@ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; @@ -511,6 +525,8 @@ _PyStackRef left; _PyStackRef right; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; // _GUARD_TOS_INT { value = stack_pointer[-1]; @@ -547,8 +563,20 @@ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1109,6 +1137,8 @@ _PyStackRef left; _PyStackRef right; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; // _GUARD_TOS_INT { value = stack_pointer[-1]; @@ -1145,8 +1175,20 @@ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; @@ -4544,6 +4586,8 @@ _PyStackRef left; _PyStackRef right; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; // _GUARD_TOS_INT { value = stack_pointer[-1]; @@ -4578,10 +4622,22 @@ Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + l = left; + r = right; res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 9e9af61..b97b4c3 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -251,37 +251,37 @@ dummy_func(void) { } } - op(_BINARY_OP_ADD_INT, (left, right -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + op(_BINARY_OP_ADD_INT, (left, right -- res, l, r)) { res = sym_new_compact_int(ctx); + l = left; + r = right; } - op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) { res = sym_new_compact_int(ctx); + l = left; + r = right; } - op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + op(_BINARY_OP_MULTIPLY_INT, (left, right -- res, l, r)) { res = sym_new_compact_int(ctx); + l = left; + r = right; } op(_BINARY_OP_ADD_FLOAT, (left, right -- res, l, r)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); res = sym_new_type(ctx, &PyFloat_Type); l = left; r = right; } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res, l, r)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); res = sym_new_type(ctx, &PyFloat_Type); l = left; r = right; } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res, l, r)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); res = sym_new_type(ctx, &PyFloat_Type); l = left; r = right; @@ -446,9 +446,10 @@ dummy_func(void) { } } - op(_COMPARE_OP_INT, (left, right -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + op(_COMPARE_OP_INT, (left, right -- res, l, r)) { res = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; } op(_COMPARE_OP_FLOAT, (left, right -- res)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b79e912..0d56464 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -497,50 +497,18 @@ JitOptRef right; JitOptRef left; JitOptRef res; + JitOptRef l; + JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res_stackref = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res_stackref )) { - ctx->done = true; - break; - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } - CHECK_STACK_BOUNDS(-1); - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } res = sym_new_compact_int(ctx); - CHECK_STACK_BOUNDS(-1); + l = left; + r = right; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -549,50 +517,18 @@ JitOptRef right; JitOptRef left; JitOptRef res; + JitOptRef l; + JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res_stackref = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res_stackref )) { - ctx->done = true; - break; - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } - CHECK_STACK_BOUNDS(-1); - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } res = sym_new_compact_int(ctx); - CHECK_STACK_BOUNDS(-1); + l = left; + r = right; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -601,50 +537,18 @@ JitOptRef right; JitOptRef left; JitOptRef res; + JitOptRef l; + JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res_stackref = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res_stackref )) { - ctx->done = true; - break; - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } - CHECK_STACK_BOUNDS(-1); - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } res = sym_new_compact_int(ctx); - CHECK_STACK_BOUNDS(-1); + l = left; + r = right; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -677,44 +581,6 @@ JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - _PyStackRef l_stackref; - _PyStackRef r_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - res_stackref = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { - JUMP_TO_LABEL(error); - } - l_stackref = left; - r_stackref = right; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - l = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(l_stackref)); - r = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(r_stackref)); - CHECK_STACK_BOUNDS(1); - stack_pointer[-2] = res; - stack_pointer[-1] = l; - stack_pointer[0] = r; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } res = sym_new_type(ctx, &PyFloat_Type); l = left; r = right; @@ -735,44 +601,6 @@ JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - _PyStackRef l_stackref; - _PyStackRef r_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - res_stackref = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { - JUMP_TO_LABEL(error); - } - l_stackref = left; - r_stackref = right; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - l = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(l_stackref)); - r = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(r_stackref)); - CHECK_STACK_BOUNDS(1); - stack_pointer[-2] = res; - stack_pointer[-1] = l; - stack_pointer[0] = r; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } res = sym_new_type(ctx, &PyFloat_Type); l = left; r = right; @@ -793,44 +621,6 @@ JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - _PyStackRef l_stackref; - _PyStackRef r_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - res_stackref = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { - JUMP_TO_LABEL(error); - } - l_stackref = left; - r_stackref = right; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - l = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(l_stackref)); - r = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(r_stackref)); - CHECK_STACK_BOUNDS(1); - stack_pointer[-2] = res; - stack_pointer[-1] = l; - stack_pointer[0] = r; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } res = sym_new_type(ctx, &PyFloat_Type); l = left; r = right; @@ -2028,50 +1818,18 @@ JitOptRef right; JitOptRef left; JitOptRef res; + JitOptRef l; + JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res_stackref = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } - CHECK_STACK_BOUNDS(-1); - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } res = sym_new_type(ctx, &PyBool_Type); - CHECK_STACK_BOUNDS(-1); + l = left; + r = right; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } |