diff options
| -rw-r--r-- | Include/internal/pycore_opcode_metadata.h | 14 | ||||
| -rw-r--r-- | Include/internal/pycore_uop_ids.h | 252 | ||||
| -rw-r--r-- | Include/internal/pycore_uop_metadata.h | 10 | ||||
| -rw-r--r-- | Include/opcode_ids.h | 37 | ||||
| -rw-r--r-- | Lib/_opcode_metadata.py | 37 | ||||
| -rw-r--r-- | Lib/test/test_generated_cases.py | 4 | ||||
| -rw-r--r-- | Python/bytecodes.c | 133 | ||||
| -rw-r--r-- | Python/ceval_macros.h | 10 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 38 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 738 | ||||
| -rw-r--r-- | Python/opcode_targets.h | 6 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 22 | ||||
| -rw-r--r-- | Tools/cases_generator/analyzer.py | 17 | ||||
| -rw-r--r-- | Tools/cases_generator/generators_common.py | 15 | ||||
| -rw-r--r-- | Tools/cases_generator/tier1_generator.py | 2 | ||||
| -rw-r--r-- | Tools/cases_generator/tier2_generator.py | 2 |
16 files changed, 820 insertions, 517 deletions
diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index c056ff1..c3cdbc3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -469,6 +469,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 5; case YIELD_VALUE: return 1; + case _DO_CALL_FUNCTION_EX: + return 3 + (oparg & 1); default: return -1; } @@ -916,6 +918,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 6; case YIELD_VALUE: return 1; + case _DO_CALL_FUNCTION_EX: + return 1; default: return -1; } @@ -1085,7 +1089,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, @@ -1155,7 +1159,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1198,6 +1202,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_DO_CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, @@ -1613,13 +1618,13 @@ const char *_PyOpcode_OpName[264] = { [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [YIELD_VALUE] = "YIELD_VALUE", + [_DO_CALL_FUNCTION_EX] = "_DO_CALL_FUNCTION_EX", }; #endif extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { - [JUMP_BACKWARD] = 1, [TO_BOOL] = 3, [BINARY_SUBSCR] = 1, [STORE_SUBSCR] = 1, @@ -1631,6 +1636,7 @@ const uint8_t _PyOpcode_Caches[256] = { [LOAD_ATTR] = 9, [COMPARE_OP] = 1, [CONTAINS_OP] = 1, + [JUMP_BACKWARD] = 1, [POP_JUMP_IF_TRUE] = 1, [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, @@ -1854,12 +1860,12 @@ const uint8_t _PyOpcode_Deopt[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, [WITH_EXCEPT_START] = WITH_EXCEPT_START, [YIELD_VALUE] = YIELD_VALUE, + [_DO_CALL_FUNCTION_EX] = _DO_CALL_FUNCTION_EX, }; #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 116: \ case 117: \ case 118: \ case 119: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 27d7f96..74c7cc9 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -39,11 +39,9 @@ extern "C" { #define _CALL_BUILTIN_FAST 315 #define _CALL_BUILTIN_FAST_WITH_KEYWORDS 316 #define _CALL_BUILTIN_O 317 -#define _CALL_FUNCTION_EX CALL_FUNCTION_EX #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_KW CALL_KW #define _CALL_LEN CALL_LEN #define _CALL_LIST_APPEND CALL_LIST_APPEND #define _CALL_METHOD_DESCRIPTOR_FAST 318 @@ -69,15 +67,16 @@ extern "C" { #define _CHECK_METHOD_VERSION 335 #define _CHECK_PEP_523 336 #define _CHECK_PERIODIC 337 -#define _CHECK_STACK_SPACE 338 -#define _CHECK_STACK_SPACE_OPERAND 339 -#define _CHECK_VALIDITY 340 -#define _CHECK_VALIDITY_AND_SET_IP 341 -#define _COMPARE_OP 342 -#define _COMPARE_OP_FLOAT 343 -#define _COMPARE_OP_INT 344 -#define _COMPARE_OP_STR 345 -#define _CONTAINS_OP 346 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 338 +#define _CHECK_STACK_SPACE 339 +#define _CHECK_STACK_SPACE_OPERAND 340 +#define _CHECK_VALIDITY 341 +#define _CHECK_VALIDITY_AND_SET_IP 342 +#define _COMPARE_OP 343 +#define _COMPARE_OP_FLOAT 344 +#define _COMPARE_OP_INT 345 +#define _COMPARE_OP_STR 346 +#define _CONTAINS_OP 347 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -89,61 +88,61 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 347 +#define _DEOPT 348 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 348 -#define _DYNAMIC_EXIT 349 +#define _DO_CALL 349 +#define _DO_CALL_KW 350 +#define _DYNAMIC_EXIT 351 #define _END_SEND END_SEND -#define _ERROR_POP_N 350 +#define _ERROR_POP_N 352 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 351 -#define _FATAL_ERROR 352 +#define _EXPAND_METHOD 353 +#define _FATAL_ERROR 354 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 353 -#define _FOR_ITER_GEN_FRAME 354 -#define _FOR_ITER_TIER_TWO 355 +#define _FOR_ITER 355 +#define _FOR_ITER_GEN_FRAME 356 +#define _FOR_ITER_TIER_TWO 357 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 356 -#define _GUARD_BOTH_INT 357 -#define _GUARD_BOTH_UNICODE 358 -#define _GUARD_BUILTINS_VERSION 359 -#define _GUARD_DORV_NO_DICT 360 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 361 -#define _GUARD_GLOBALS_VERSION 362 -#define _GUARD_IS_FALSE_POP 363 -#define _GUARD_IS_NONE_POP 364 -#define _GUARD_IS_NOT_NONE_POP 365 -#define _GUARD_IS_TRUE_POP 366 -#define _GUARD_KEYS_VERSION 367 -#define _GUARD_NOS_FLOAT 368 -#define _GUARD_NOS_INT 369 -#define _GUARD_NOT_EXHAUSTED_LIST 370 -#define _GUARD_NOT_EXHAUSTED_RANGE 371 -#define _GUARD_NOT_EXHAUSTED_TUPLE 372 -#define _GUARD_TOS_FLOAT 373 -#define _GUARD_TOS_INT 374 -#define _GUARD_TYPE_VERSION 375 +#define _GUARD_BOTH_FLOAT 358 +#define _GUARD_BOTH_INT 359 +#define _GUARD_BOTH_UNICODE 360 +#define _GUARD_BUILTINS_VERSION 361 +#define _GUARD_DORV_NO_DICT 362 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 363 +#define _GUARD_GLOBALS_VERSION 364 +#define _GUARD_IS_FALSE_POP 365 +#define _GUARD_IS_NONE_POP 366 +#define _GUARD_IS_NOT_NONE_POP 367 +#define _GUARD_IS_TRUE_POP 368 +#define _GUARD_KEYS_VERSION 369 +#define _GUARD_NOS_FLOAT 370 +#define _GUARD_NOS_INT 371 +#define _GUARD_NOT_EXHAUSTED_LIST 372 +#define _GUARD_NOT_EXHAUSTED_RANGE 373 +#define _GUARD_NOT_EXHAUSTED_TUPLE 374 +#define _GUARD_TOS_FLOAT 375 +#define _GUARD_TOS_INT 376 +#define _GUARD_TYPE_VERSION 377 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 376 -#define _INIT_CALL_PY_EXACT_ARGS 377 -#define _INIT_CALL_PY_EXACT_ARGS_0 378 -#define _INIT_CALL_PY_EXACT_ARGS_1 379 -#define _INIT_CALL_PY_EXACT_ARGS_2 380 -#define _INIT_CALL_PY_EXACT_ARGS_3 381 -#define _INIT_CALL_PY_EXACT_ARGS_4 382 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 378 +#define _INIT_CALL_PY_EXACT_ARGS 379 +#define _INIT_CALL_PY_EXACT_ARGS_0 380 +#define _INIT_CALL_PY_EXACT_ARGS_1 381 +#define _INIT_CALL_PY_EXACT_ARGS_2 382 +#define _INIT_CALL_PY_EXACT_ARGS_3 383 +#define _INIT_CALL_PY_EXACT_ARGS_4 384 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION -#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD #define _INSTRUMENTED_LINE INSTRUMENTED_LINE #define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR @@ -151,66 +150,65 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME -#define _INTERNAL_INCREMENT_OPT_COUNTER 383 -#define _IS_NONE 384 +#define _INTERNAL_INCREMENT_OPT_COUNTER 385 +#define _IS_NONE 386 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 385 -#define _ITER_CHECK_RANGE 386 -#define _ITER_CHECK_TUPLE 387 -#define _ITER_JUMP_LIST 388 -#define _ITER_JUMP_RANGE 389 -#define _ITER_JUMP_TUPLE 390 -#define _ITER_NEXT_LIST 391 -#define _ITER_NEXT_RANGE 392 -#define _ITER_NEXT_TUPLE 393 -#define _JUMP_TO_TOP 394 +#define _ITER_CHECK_LIST 387 +#define _ITER_CHECK_RANGE 388 +#define _ITER_CHECK_TUPLE 389 +#define _ITER_JUMP_LIST 390 +#define _ITER_JUMP_RANGE 391 +#define _ITER_JUMP_TUPLE 392 +#define _ITER_NEXT_LIST 393 +#define _ITER_NEXT_RANGE 394 +#define _ITER_NEXT_TUPLE 395 +#define _JUMP_TO_TOP 396 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 395 -#define _LOAD_ATTR_CLASS 396 -#define _LOAD_ATTR_CLASS_0 397 -#define _LOAD_ATTR_CLASS_1 398 +#define _LOAD_ATTR 397 +#define _LOAD_ATTR_CLASS 398 +#define _LOAD_ATTR_CLASS_0 399 +#define _LOAD_ATTR_CLASS_1 400 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 399 -#define _LOAD_ATTR_INSTANCE_VALUE_0 400 -#define _LOAD_ATTR_INSTANCE_VALUE_1 401 -#define _LOAD_ATTR_METHOD_LAZY_DICT 402 -#define _LOAD_ATTR_METHOD_NO_DICT 403 -#define _LOAD_ATTR_METHOD_WITH_VALUES 404 -#define _LOAD_ATTR_MODULE 405 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 406 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 407 -#define _LOAD_ATTR_PROPERTY_FRAME 408 -#define _LOAD_ATTR_SLOT 409 -#define _LOAD_ATTR_SLOT_0 410 -#define _LOAD_ATTR_SLOT_1 411 -#define _LOAD_ATTR_WITH_HINT 412 +#define _LOAD_ATTR_INSTANCE_VALUE 401 +#define _LOAD_ATTR_INSTANCE_VALUE_0 402 +#define _LOAD_ATTR_INSTANCE_VALUE_1 403 +#define _LOAD_ATTR_METHOD_LAZY_DICT 404 +#define _LOAD_ATTR_METHOD_NO_DICT 405 +#define _LOAD_ATTR_METHOD_WITH_VALUES 406 +#define _LOAD_ATTR_MODULE 407 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 408 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 409 +#define _LOAD_ATTR_PROPERTY_FRAME 410 +#define _LOAD_ATTR_SLOT 411 +#define _LOAD_ATTR_SLOT_0 412 +#define _LOAD_ATTR_SLOT_1 413 +#define _LOAD_ATTR_WITH_HINT 414 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 413 -#define _LOAD_CONST_INLINE_BORROW 414 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 415 -#define _LOAD_CONST_INLINE_WITH_NULL 416 +#define _LOAD_CONST_INLINE 415 +#define _LOAD_CONST_INLINE_BORROW 416 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 417 +#define _LOAD_CONST_INLINE_WITH_NULL 418 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 417 -#define _LOAD_FAST_0 418 -#define _LOAD_FAST_1 419 -#define _LOAD_FAST_2 420 -#define _LOAD_FAST_3 421 -#define _LOAD_FAST_4 422 -#define _LOAD_FAST_5 423 -#define _LOAD_FAST_6 424 -#define _LOAD_FAST_7 425 +#define _LOAD_FAST 419 +#define _LOAD_FAST_0 420 +#define _LOAD_FAST_1 421 +#define _LOAD_FAST_2 422 +#define _LOAD_FAST_3 423 +#define _LOAD_FAST_4 424 +#define _LOAD_FAST_5 425 +#define _LOAD_FAST_6 426 +#define _LOAD_FAST_7 427 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 426 -#define _LOAD_GLOBAL_BUILTINS 427 -#define _LOAD_GLOBAL_MODULE 428 +#define _LOAD_GLOBAL 428 +#define _LOAD_GLOBAL_BUILTINS 429 +#define _LOAD_GLOBAL_MODULE 430 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SPECIAL LOAD_SPECIAL @@ -223,55 +221,58 @@ extern "C" { #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 429 -#define _MONITOR_CALL 430 +#define _MAYBE_EXPAND_METHOD 431 +#define _MONITOR_CALL 432 +#define _MONITOR_JUMP_BACKWARD 433 +#define _MONITOR_RESUME 434 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 431 -#define _POP_JUMP_IF_TRUE 432 +#define _POP_JUMP_IF_FALSE 435 +#define _POP_JUMP_IF_TRUE 436 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 433 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 437 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 434 +#define _PUSH_FRAME 438 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 435 -#define _REPLACE_WITH_TRUE 436 +#define _PY_FRAME_GENERAL 439 +#define _QUICKEN_RESUME 440 +#define _REPLACE_WITH_TRUE 441 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 437 -#define _SEND 438 -#define _SEND_GEN_FRAME 439 +#define _SAVE_RETURN_OFFSET 442 +#define _SEND 443 +#define _SEND_GEN_FRAME 444 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 440 -#define _STORE_ATTR 441 -#define _STORE_ATTR_INSTANCE_VALUE 442 -#define _STORE_ATTR_SLOT 443 -#define _STORE_ATTR_WITH_HINT 444 +#define _START_EXECUTOR 445 +#define _STORE_ATTR 446 +#define _STORE_ATTR_INSTANCE_VALUE 447 +#define _STORE_ATTR_SLOT 448 +#define _STORE_ATTR_WITH_HINT 449 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 445 -#define _STORE_FAST_0 446 -#define _STORE_FAST_1 447 -#define _STORE_FAST_2 448 -#define _STORE_FAST_3 449 -#define _STORE_FAST_4 450 -#define _STORE_FAST_5 451 -#define _STORE_FAST_6 452 -#define _STORE_FAST_7 453 +#define _STORE_FAST 450 +#define _STORE_FAST_0 451 +#define _STORE_FAST_1 452 +#define _STORE_FAST_2 453 +#define _STORE_FAST_3 454 +#define _STORE_FAST_4 455 +#define _STORE_FAST_5 456 +#define _STORE_FAST_6 457 +#define _STORE_FAST_7 458 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 454 +#define _STORE_SUBSCR 459 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 455 -#define _TO_BOOL 456 +#define _TIER2_RESUME_CHECK 460 +#define _TO_BOOL 461 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -281,13 +282,14 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 457 +#define _UNPACK_SEQUENCE 462 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 457 +#define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX +#define MAX_UOP_ID 462 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f5c6664..ef9e7de 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -20,6 +20,8 @@ extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, + [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, @@ -202,7 +204,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG, - [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -334,6 +335,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", @@ -536,6 +538,10 @@ int _PyUop_num_popped(int opcode, int oparg) switch(opcode) { case _NOP: return 0; + case _CHECK_PERIODIC: + return 0; + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: + return 0; case _RESUME_CHECK: return 0; case _LOAD_FAST_CHECK: @@ -900,8 +906,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _MAYBE_EXPAND_METHOD: return 2 + oparg; - case _CHECK_PERIODIC: - return 0; case _PY_FRAME_GENERAL: return 2 + oparg; case _CHECK_FUNCTION_VERSION: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 3353e80..95984a9 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -126,6 +126,7 @@ extern "C" { #define UNPACK_EX 113 #define UNPACK_SEQUENCE 114 #define YIELD_VALUE 115 +#define _DO_CALL_FUNCTION_EX 116 #define RESUME 149 #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 @@ -200,24 +201,24 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 220 #define UNPACK_SEQUENCE_TUPLE 221 #define UNPACK_SEQUENCE_TWO_TUPLE 222 -#define INSTRUMENTED_RESUME 236 -#define INSTRUMENTED_END_FOR 237 -#define INSTRUMENTED_END_SEND 238 -#define INSTRUMENTED_LOAD_SUPER_ATTR 239 -#define INSTRUMENTED_FOR_ITER 240 -#define INSTRUMENTED_CALL_KW 241 -#define INSTRUMENTED_CALL_FUNCTION_EX 242 -#define INSTRUMENTED_INSTRUCTION 243 -#define INSTRUMENTED_JUMP_FORWARD 244 -#define INSTRUMENTED_JUMP_BACKWARD 245 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 246 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 247 -#define INSTRUMENTED_POP_JUMP_IF_NONE 248 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 249 -#define INSTRUMENTED_RETURN_VALUE 250 -#define INSTRUMENTED_RETURN_CONST 251 -#define INSTRUMENTED_YIELD_VALUE 252 -#define INSTRUMENTED_CALL 253 +#define INSTRUMENTED_END_FOR 236 +#define INSTRUMENTED_END_SEND 237 +#define INSTRUMENTED_LOAD_SUPER_ATTR 238 +#define INSTRUMENTED_FOR_ITER 239 +#define INSTRUMENTED_CALL_KW 240 +#define INSTRUMENTED_CALL_FUNCTION_EX 241 +#define INSTRUMENTED_INSTRUCTION 242 +#define INSTRUMENTED_JUMP_FORWARD 243 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 244 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 245 +#define INSTRUMENTED_POP_JUMP_IF_NONE 246 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 247 +#define INSTRUMENTED_RESUME 248 +#define INSTRUMENTED_RETURN_VALUE 249 +#define INSTRUMENTED_RETURN_CONST 250 +#define INSTRUMENTED_YIELD_VALUE 251 +#define INSTRUMENTED_CALL 252 +#define INSTRUMENTED_JUMP_BACKWARD 253 #define INSTRUMENTED_LINE 254 #define ENTER_EXECUTOR 255 #define JUMP 256 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 7c559f6..94c8a0a 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -305,24 +305,25 @@ opmap = { 'UNPACK_EX': 113, 'UNPACK_SEQUENCE': 114, 'YIELD_VALUE': 115, - 'INSTRUMENTED_RESUME': 236, - 'INSTRUMENTED_END_FOR': 237, - 'INSTRUMENTED_END_SEND': 238, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 239, - 'INSTRUMENTED_FOR_ITER': 240, - 'INSTRUMENTED_CALL_KW': 241, - 'INSTRUMENTED_CALL_FUNCTION_EX': 242, - 'INSTRUMENTED_INSTRUCTION': 243, - 'INSTRUMENTED_JUMP_FORWARD': 244, - 'INSTRUMENTED_JUMP_BACKWARD': 245, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 246, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 247, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 248, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 249, - 'INSTRUMENTED_RETURN_VALUE': 250, - 'INSTRUMENTED_RETURN_CONST': 251, - 'INSTRUMENTED_YIELD_VALUE': 252, - 'INSTRUMENTED_CALL': 253, + '_DO_CALL_FUNCTION_EX': 116, + 'INSTRUMENTED_END_FOR': 236, + 'INSTRUMENTED_END_SEND': 237, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 238, + 'INSTRUMENTED_FOR_ITER': 239, + 'INSTRUMENTED_CALL_KW': 240, + 'INSTRUMENTED_CALL_FUNCTION_EX': 241, + 'INSTRUMENTED_INSTRUCTION': 242, + 'INSTRUMENTED_JUMP_FORWARD': 243, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 244, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 245, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 246, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 247, + 'INSTRUMENTED_RESUME': 248, + 'INSTRUMENTED_RETURN_VALUE': 249, + 'INSTRUMENTED_RETURN_CONST': 250, + 'INSTRUMENTED_YIELD_VALUE': 251, + 'INSTRUMENTED_CALL': 252, + 'INSTRUMENTED_JUMP_BACKWARD': 253, 'JUMP': 256, 'JUMP_NO_INTERRUPT': 257, 'LOAD_CLOSURE': 258, diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index beafa54..7f82181 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -247,14 +247,13 @@ class TestGeneratedCases(unittest.TestCase): """ self.run_cases_test(input, output) - def test_predictions_and_eval_breaker(self): + def test_predictions(self): input = """ inst(OP1, (arg -- rest)) { } inst(OP3, (arg -- res)) { DEOPT_IF(xxx); res = Py_None; - CHECK_EVAL_BREAKER(); } family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; """ @@ -277,7 +276,6 @@ class TestGeneratedCases(unittest.TestCase): DEOPT_IF(xxx, OP1); res = Py_None; stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); DISPATCH(); } """ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ffa53bb..757c3f9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -10,6 +10,7 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_backoff.h" #include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_ceval.h" #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" @@ -146,36 +147,54 @@ dummy_func( RESUME_CHECK, }; - tier1 inst(RESUME, (--)) { - assert(frame == tstate->current_frame); + op(_CHECK_PERIODIC, (--)) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + ERROR_IF(err != 0, error); + } + } + + op(_CHECK_PERIODIC_IF_NOT_YIELD_FROM, (--)) { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + ERROR_IF(err != 0, error); + } + } + } + + op(_QUICKEN_RESUME, (--)) { + #if ENABLE_SPECIALIZATION + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION */ + } + + tier1 op(_MAYBE_INSTRUMENT, (--)) { if (tstate->tracing == 0) { - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - PyCodeObject* code = _PyFrame_GetCode(frame); - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); - assert((code_version & 255) == 0); + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - ERROR_IF(err, error); + if (err) { + ERROR_NO_POP(); + } next_instr = this_instr; DISPATCH(); } - assert(this_instr->op.code == RESUME || - this_instr->op.code == RESUME_CHECK || - this_instr->op.code == INSTRUMENTED_RESUME || - this_instr->op.code == ENTER_EXECUTOR); - if (this_instr->op.code == RESUME) { - #if ENABLE_SPECIALIZATION - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - #endif /* ENABLE_SPECIALIZATION */ - } - } - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); } } + macro(RESUME) = + _MAYBE_INSTRUMENT + + _QUICKEN_RESUME + + _CHECK_PERIODIC_IF_NOT_YIELD_FROM; + inst(RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0); @@ -187,33 +206,23 @@ dummy_func( DEOPT_IF(eval_breaker != version); } - inst(INSTRUMENTED_RESUME, (--)) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version && tstate->tracing == 0) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) { - ERROR_NO_POP(); - } - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - ERROR_IF(err, error); - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; - DISPATCH(); - } + op(_MONITOR_RESUME, (--)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + ERROR_IF(err, error); + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = frame->instr_ptr; } } + macro(INSTRUMENTED_RESUME) = + _MAYBE_INSTRUMENT + + _CHECK_PERIODIC_IF_NOT_YIELD_FROM + + _MONITOR_RESUME; + pseudo(LOAD_CLOSURE, (-- unused)) = { LOAD_FAST, }; @@ -2486,8 +2495,7 @@ dummy_func( JUMPBY(oparg); } - tier1 inst(JUMP_BACKWARD, (unused/1 --)) { - CHECK_EVAL_BREAKER(); + tier1 op(_JUMP_BACKWARD, (the_counter/1 --)) { assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); #ifdef _Py_TIER2 @@ -2519,6 +2527,10 @@ dummy_func( #endif /* _Py_TIER2 */ } + macro(JUMP_BACKWARD) = + _CHECK_PERIODIC + + _JUMP_BACKWARD; + pseudo(JUMP, (--)) = { JUMP_FORWARD, JUMP_BACKWARD, @@ -3265,10 +3277,6 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_CHECK_PERIODIC, (--)) { - CHECK_EVAL_BREAKER(); - } - op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { int is_meth = !PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); @@ -4012,7 +4020,7 @@ dummy_func( GO_TO_INSTRUCTION(CALL_KW); } - inst(CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { + op(_DO_CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -4094,14 +4102,17 @@ dummy_func( } ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); - CHECK_EVAL_BREAKER(); } + macro(CALL_KW) = + _DO_CALL_KW + + _CHECK_PERIODIC; + inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - inst(CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { + inst(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); @@ -4175,9 +4186,13 @@ dummy_func( DECREF_INPUTS(); assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); ERROR_IF(PyStackRef_IsNull(result), error); - CHECK_EVAL_BREAKER(); } + macro(CALL_FUNCTION_EX) = + _DO_CALL_FUNCTION_EX + + _CHECK_PERIODIC; + + inst(MAKE_FUNCTION, (codeobj_st -- func)) { PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); @@ -4381,11 +4396,15 @@ dummy_func( INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); } - inst(INSTRUMENTED_JUMP_BACKWARD, (unused/1 -- )) { - CHECK_EVAL_BREAKER(); + op(_MONITOR_JUMP_BACKWARD, (-- )) { INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); } + macro(INSTRUMENTED_JUMP_BACKWARD) = + unused/1 + + _CHECK_PERIODIC + + _MONITOR_JUMP_BACKWARD; + inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 8b25a5f..ed146a1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -133,16 +133,6 @@ do { \ // Use this instead of 'goto error' so Tier 2 can go to a different label #define GOTO_ERROR(LABEL) goto LABEL -#define CHECK_EVAL_BREAKER() \ - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ - if (_Py_HandlePending(tstate) != 0) { \ - GOTO_ERROR(error); \ - } \ - } - - /* Tuple access macros */ #ifndef Py_DEBUG diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0bccaf9..b03eb99 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -12,6 +12,31 @@ break; } + case _CHECK_PERIODIC: { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) JUMP_TO_ERROR(); + } + break; + } + + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + oparg = CURRENT_OPARG(); + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) JUMP_TO_ERROR(); + } + } + break; + } + + /* _QUICKEN_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _RESUME_CHECK: { #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { @@ -30,7 +55,7 @@ break; } - /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 because it is instrumented */ + /* _MONITOR_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _LOAD_FAST_CHECK: { _PyStackRef value; @@ -3555,11 +3580,6 @@ /* _DO_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _CHECK_PERIODIC: { - CHECK_EVAL_BREAKER(); - break; - } - /* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _PY_FRAME_GENERAL: { @@ -4657,11 +4677,11 @@ /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAKE_FUNCTION: { _PyStackRef codeobj_st; @@ -4882,7 +4902,7 @@ /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 because it is instrumented */ + /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ef00f6f..4a5554a6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -949,11 +949,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1275,11 +1285,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1344,11 +1364,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1412,11 +1442,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1466,11 +1506,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1485,89 +1535,105 @@ _PyStackRef callargs_st; _PyStackRef kwargs_st = PyStackRef_NULL; _PyStackRef result; + // __DO_CALL_FUNCTION_EX if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; } callargs_st = stack_pointer[-1 - (oparg & 1)]; func_st = stack_pointer[-3 - (oparg & 1)]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - int err = check_args_iterable(tstate, func, callargs); - if (err < 0) { - goto error; - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - goto error; - } - PyStackRef_CLOSE(callargs_st); - callargs_st = PyStackRef_FromPyObjectSteal(tuple); - callargs = tuple; - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - if (err) goto error; - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (PyStackRef_IsNull(result)) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); + { + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (!PyTuple_CheckExact(callargs)) { + int err = check_args_iterable(tstate, func, callargs); + if (err < 0) { + goto error; } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - PyStackRef_CLEAR(result); + PyObject *tuple = PySequence_Tuple(callargs); + if (tuple == NULL) { + goto error; + } + PyStackRef_CLOSE(callargs_st); + callargs_st = PyStackRef_FromPyObjectSteal(tuple); + callargs = tuple; + } + assert(PyTuple_CheckExact(callargs)); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + if (err) goto error; + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (PyStackRef_IsNull(result)) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + PyStackRef_CLEAR(result); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); } + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + } + PyStackRef_CLOSE(func_st); + PyStackRef_CLOSE(callargs_st); + PyStackRef_XCLOSE(kwargs_st); + assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); + if (PyStackRef_IsNull(result)) { + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + goto error; } } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); goto error; } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); } - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); - } - PyStackRef_CLOSE(func_st); - PyStackRef_CLOSE(callargs_st); - PyStackRef_XCLOSE(kwargs_st); - assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); - if (PyStackRef_IsNull(result)) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; } stack_pointer[-3 - (oparg & 1)] = result; stack_pointer += -2 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1664,107 +1730,123 @@ _PyStackRef *args; _PyStackRef kwnames; _PyStackRef res; + // _DO_CALL_KW kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null_o != NULL) { - args--; - total_args++; - } - if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o - ); - PyStackRef_CLOSE(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o + ); + PyStackRef_CLOSE(kwnames); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + if (err < 0) { + Py_CLEAR(res_o); + } + } } PyStackRef_CLOSE(kwnames); - if (true) { + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - if (err < 0) { - Py_CLEAR(res_o); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } } } - PyStackRef_CLOSE(kwnames); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1912,11 +1994,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1983,11 +2075,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2041,11 +2143,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2102,11 +2214,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2175,11 +2297,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2369,11 +2501,16 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) goto pop_2_error; + } } stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2405,11 +2542,16 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) goto pop_2_error; + } } stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3736,11 +3878,21 @@ } // _CHECK_PERIODIC { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3872,8 +4024,19 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ - CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) goto error; + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } DISPATCH(); } @@ -4018,19 +4181,34 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version && tstate->tracing == 0) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) { - goto error; + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) { + goto error; + } + next_instr = this_instr; + DISPATCH(); + } } - next_instr = this_instr; } - else { + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) goto error; + } } + } + // _MONITOR_RESUME + { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, this_instr); @@ -4039,7 +4217,6 @@ if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; - DISPATCH(); } } DISPATCH(); @@ -4244,37 +4421,49 @@ (void)this_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); - /* Skip 1 cache entry */ - CHECK_EVAL_BREAKER(); - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #ifdef _Py_TIER2 - #if ENABLE_SPECIALIZATION - _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - while (oparg > 255) { - oparg >>= 8; - start--; - } - _PyExecutorObject *executor; - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); - if (optimized < 0) goto error; - if (optimized) { - assert(tstate->previous_executor == NULL); - tstate->previous_executor = Py_None; - GOTO_TIER_TWO(executor); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) goto error; + } + } + // _JUMP_BACKWARD + { + uint16_t the_counter = read_u16(&this_instr[1].cache); + (void)the_counter; + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #ifdef _Py_TIER2 + #if ENABLE_SPECIALIZATION + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; + } + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + if (optimized < 0) goto error; + if (optimized) { + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); + } + else { + this_instr[1].counter = restart_backoff_counter(counter); + } } else { - this_instr[1].counter = restart_backoff_counter(counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } + #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ } - else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - #endif /* ENABLE_SPECIALIZATION */ - #endif /* _Py_TIER2 */ DISPATCH(); } @@ -5889,32 +6078,39 @@ PREDICTED(RESUME); _Py_CODEUNIT *this_instr = next_instr - 1; (void)this_instr; - assert(frame == tstate->current_frame); - if (tstate->tracing == 0) { - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - PyCodeObject* code = _PyFrame_GetCode(frame); - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; - DISPATCH(); + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) { + goto error; + } + next_instr = this_instr; + DISPATCH(); + } } - assert(this_instr->op.code == RESUME || - this_instr->op.code == RESUME_CHECK || - this_instr->op.code == INSTRUMENTED_RESUME || - this_instr->op.code == ENTER_EXECUTOR); - if (this_instr->op.code == RESUME) { - #if ENABLE_SPECIALIZATION + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - #endif /* ENABLE_SPECIALIZATION */ } + #endif /* ENABLE_SPECIALIZATION */ } - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + int err = _Py_HandlePending(tstate); + if (err != 0) goto error; + } + } } DISPATCH(); } @@ -7108,4 +7304,98 @@ assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } + + TARGET(_DO_CALL_FUNCTION_EX) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(_DO_CALL_FUNCTION_EX); + _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef result; + if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; } + callargs_st = stack_pointer[-1 - (oparg & 1)]; + func_st = stack_pointer[-3 - (oparg & 1)]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (!PyTuple_CheckExact(callargs)) { + int err = check_args_iterable(tstate, func, callargs); + if (err < 0) { + goto error; + } + PyObject *tuple = PySequence_Tuple(callargs); + if (tuple == NULL) { + goto error; + } + PyStackRef_CLOSE(callargs_st); + callargs_st = PyStackRef_FromPyObjectSteal(tuple); + callargs = tuple; + } + assert(PyTuple_CheckExact(callargs)); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + if (err) goto error; + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (PyStackRef_IsNull(result)) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + PyStackRef_CLEAR(result); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + } + PyStackRef_CLOSE(func_st); + PyStackRef_CLOSE(callargs_st); + PyStackRef_XCLOSE(kwargs_st); + assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); + if (PyStackRef_IsNull(result)) { + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } #undef TIER_ONE diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index db92b02..9ea01e2 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -115,7 +115,7 @@ static void *opcode_targets[256] = { &&TARGET_UNPACK_EX, &&TARGET_UNPACK_SEQUENCE, &&TARGET_YIELD_VALUE, - &&_unknown_opcode, + &&TARGET__DO_CALL_FUNCTION_EX, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -235,7 +235,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, @@ -244,15 +243,16 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_JUMP_FORWARD, - &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_RETURN_VALUE, &&TARGET_INSTRUMENTED_RETURN_CONST, &&TARGET_INSTRUMENTED_YIELD_VALUE, &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, }; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 3f4080d..866d7d9 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -7,11 +7,21 @@ break; } + case _CHECK_PERIODIC: { + break; + } + + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + break; + } + + /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */ + case _RESUME_CHECK: { break; } - /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { _Py_UopsSymbol *value; @@ -1644,10 +1654,6 @@ /* _DO_CALL is not a viable micro-op for tier 2 */ - case _CHECK_PERIODIC: { - break; - } - /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { @@ -1966,11 +1972,11 @@ /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ - /* _CALL_KW is not a viable micro-op for tier 2 */ + /* _DO_CALL_KW is not a viable micro-op for tier 2 */ /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ - /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { _Py_UopsSymbol *func; @@ -2100,7 +2106,7 @@ /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ - /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 */ /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 8c75165..3cc36b6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -14,7 +14,6 @@ class Properties: oparg: bool jumps: bool eval_breaker: bool - ends_with_eval_breaker: bool needs_this: bool always_exits: bool stores_sp: bool @@ -44,7 +43,6 @@ class Properties: oparg=any(p.oparg for p in properties), jumps=any(p.jumps for p in properties), eval_breaker=any(p.eval_breaker for p in properties), - ends_with_eval_breaker=any(p.ends_with_eval_breaker for p in properties), needs_this=any(p.needs_this for p in properties), always_exits=any(p.always_exits for p in properties), stores_sp=any(p.stores_sp for p in properties), @@ -70,7 +68,6 @@ SKIP_PROPERTIES = Properties( oparg=False, jumps=False, eval_breaker=False, - ends_with_eval_breaker=False, needs_this=False, always_exits=False, stores_sp=False, @@ -194,13 +191,6 @@ class Uop: return "has unused cache entries" if self.properties.error_with_pop and self.properties.error_without_pop: return "has both popping and not-popping errors" - if self.properties.eval_breaker: - if self.properties.error_with_pop or self.properties.error_without_pop: - return "has error handling and eval-breaker check" - if self.properties.side_exit: - return "exits and eval-breaker check" - if self.properties.deopts: - return "deopts and eval-breaker check" return None def is_viable(self) -> bool: @@ -587,10 +577,6 @@ EXITS = { } -def eval_breaker_at_end(op: parser.InstDef) -> bool: - return op.tokens[-5].text == "CHECK_EVAL_BREAKER" - - def always_exits(op: parser.InstDef) -> bool: depth = 0 tkn_iter = iter(op.tokens) @@ -678,8 +664,7 @@ def compute_properties(op: parser.InstDef) -> Properties: side_exit=exits_if, oparg=oparg_used(op), jumps=variable_used(op, "JUMPBY"), - eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"), - ends_with_eval_breaker=eval_breaker_at_end(op), + eval_breaker="CHECK_PERIODIC" in op.name, needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), stores_sp=variable_used(op, "SYNC_SP"), diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 6ed9d83..dd4057c 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -75,7 +75,6 @@ class Emitter: "ERROR_IF": self.error_if, "ERROR_NO_POP": self.error_no_pop, "DECREF_INPUTS": self.decref_inputs, - "CHECK_EVAL_BREAKER": self.check_eval_breaker, "SYNC_SP": self.sync_sp, "PyStackRef_FromPyObjectNew": self.py_stack_ref_from_py_object_new, } @@ -190,20 +189,6 @@ class Emitter: next(tkn_iter) stack.flush(self.out) - def check_eval_breaker( - self, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, - ) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - if not uop.properties.ends_with_eval_breaker: - self.out.emit_at("CHECK_EVAL_BREAKER();", tkn) - def py_stack_ref_from_py_object_new( self, tkn: Token, diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index c3456cd..c749896 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -201,8 +201,6 @@ def generate_tier1( out.start_line() if not inst.parts[-1].properties.always_exits: stack.flush(out) - if inst.parts[-1].properties.ends_with_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();\n") out.emit("DISPATCH();\n") out.start_line() out.emit("}") diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 7ed9376..b7c70fd 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -230,8 +230,6 @@ def generate_tier2( out.start_line() if not uop.properties.always_exits: stack.flush(out) - if uop.properties.ends_with_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();\n") out.emit("break;\n") out.start_line() out.emit("}") |